From 9d1a4624d9cf0cdffb1e909b64dd0a920e2644e9 Mon Sep 17 00:00:00 2001 From: forhappy Date: Sat, 17 Oct 2026 23:35:58 -0700 Subject: [PATCH 01/38] feat(query): retrieve broad implementation evidence Index bounded source-owned lexical evidence and rank implementation candidates without requiring exact symbols. Add opt-in CLI and MCP shortlists with source citations, relevance reasons, and whole-response budgets. Preserve deterministic JSON/store contracts and exact identity precedence. Document cache/profile migrations, trust boundaries, and incomplete comparative qualification. --- CHANGELOG.md | 164 ++ COMPATIBILITY.md | 464 ++++ MIGRATION.md | 141 + PERFORMANCE.md | 58 + SECURITY.md | 60 + crates/compass-cli/src/freshness.rs | 14 + crates/compass-cli/src/help.rs | 7 + crates/compass-cli/src/lib.rs | 139 +- crates/compass-cli/tests/code_query_cli.rs | 835 ++++++ .../tests/store_term_capabilities.rs | 169 ++ crates/compass-core/src/build_state.rs | 1 + crates/compass-core/src/lib.rs | 1 + crates/compass-core/src/pipeline.rs | 585 ++++- crates/compass-core/src/source_query.rs | 630 +++++ .../compass-core/tests/source_query_bounds.rs | 93 + .../tests/source_query_integrity.rs | 260 ++ .../tests/source_query_pipeline.rs | 606 +++++ .../tests/source_query_snapshot_contract.rs | 139 + crates/compass-files/src/cache.rs | 2 +- crates/compass-graph/src/snapshot.rs | 763 +++++- crates/compass-graph/src/v1.rs | 24 +- crates/compass-languages/src/engine.rs | 7 + .../compass-languages/src/evidence/build.rs | 102 +- .../src/evidence/documentation.rs | 184 ++ crates/compass-languages/src/evidence/mod.rs | 7 +- .../compass-languages/src/evidence/model.rs | 31 + .../src/evidence/typescript.rs | 35 +- .../src/evidence/validate.rs | 116 +- .../src/evidence_pipeline.rs | 16 +- crates/compass-languages/src/lib.rs | 9 +- crates/compass-languages/src/source_query.rs | 1086 ++++++++ .../tests/callable_initializer_definitions.rs | 77 + .../tests/leading_documentation.rs | 392 +++ .../tests/source_query_evidence.rs | 121 + .../tests/typescript_member_scopes.rs | 116 + .../tests/typescript_universal_evidence.rs | 4 +- .../tests/universal_evidence.rs | 148 +- crates/compass-mcp/src/code_query.rs | 62 +- crates/compass-mcp/src/lib.rs | 47 + crates/compass-mcp/tests/code_query_tools.rs | 224 ++ crates/compass-model/src/lib.rs | 1 + crates/compass-model/src/provenance.rs | 103 + crates/compass-model/src/search.rs | 238 +- crates/compass-model/src/source_query.rs | 419 +++ crates/compass-model/src/validation.rs | 9 + .../tests/leading_documentation.rs | 250 ++ crates/compass-model/tests/source_query.rs | 257 ++ crates/compass-output/src/lib.rs | 6 + crates/compass-output/src/ranked_query.rs | 787 ++++++ crates/compass-output/src/text_budget.rs | 2 +- crates/compass-output/tests/agent_query.rs | 4 +- crates/compass-output/tests/ranked_query.rs | 516 ++++ crates/compass-query/Cargo.toml | 2 +- .../examples/broad_retrieval_trace.rs | 39 + crates/compass-query/src/broad.rs | 2305 +++++++++++++++++ crates/compass-query/src/broad/compounds.rs | 208 ++ crates/compass-query/src/broad/excerpts.rs | 407 +++ crates/compass-query/src/broad/ownership.rs | 165 ++ .../compass-query/src/broad/relationships.rs | 212 ++ crates/compass-query/src/code_query.rs | 40 +- crates/compass-query/src/concepts.rs | 14 + crates/compass-query/src/discovery.rs | 179 +- crates/compass-query/src/discovery_text.rs | 93 +- crates/compass-query/src/graph_engine.rs | 32 +- crates/compass-query/src/index.rs | 427 ++- crates/compass-query/src/index_validation.rs | 649 +++++ crates/compass-query/src/intent.rs | 4 +- crates/compass-query/src/lib.rs | 17 +- crates/compass-query/src/ranking.rs | 17 + crates/compass-query/src/response_cache.rs | 11 +- crates/compass-query/src/semantic_search.rs | 182 +- crates/compass-query/src/source_query.rs | 186 ++ crates/compass-query/tests/broad_retrieval.rs | 2156 +++++++++++++++ .../compass-query/tests/broad_test_context.rs | 119 + .../compass-query/tests/fielded_projection.rs | 309 +++ crates/compass-query/tests/natural_intent.rs | 4 +- crates/compass-query/tests/store_engine.rs | 96 +- .../src/evidence/index/builder.rs | 58 + crates/compass-resolve/src/evidence/mod.rs | 113 +- .../src/evidence/projection/documentation.rs | 188 ++ .../src/evidence/projection/mod.rs | 13 +- .../src/evidence/projection/nodes.rs | 12 +- crates/compass-resolve/src/lib.rs | 5 +- .../tests/callable_initializer_definitions.rs | 112 + .../tests/evidence_resolution_contract.rs | 1 + .../tests/inference_admission.rs | 1 + .../tests/leading_documentation.rs | 194 ++ .../tests/source_query_evidence.rs | 59 + .../tests/typescript_member_scopes.rs | 120 + docs/design/language-architecture.md | 19 +- docs/reference/commands.md | 94 + docs/reference/outputs.md | 171 ++ docs/reference/universal-semantic-evidence.md | 67 +- .../callable_initializer_definitions.json | 42 + fixtures/typescript_member_scopes.json | 64 + 95 files changed, 19384 insertions(+), 353 deletions(-) create mode 100644 crates/compass-cli/tests/store_term_capabilities.rs create mode 100644 crates/compass-core/src/source_query.rs create mode 100644 crates/compass-core/tests/source_query_bounds.rs create mode 100644 crates/compass-core/tests/source_query_integrity.rs create mode 100644 crates/compass-core/tests/source_query_pipeline.rs create mode 100644 crates/compass-core/tests/source_query_snapshot_contract.rs create mode 100644 crates/compass-languages/src/evidence/documentation.rs create mode 100644 crates/compass-languages/src/source_query.rs create mode 100644 crates/compass-languages/tests/callable_initializer_definitions.rs create mode 100644 crates/compass-languages/tests/leading_documentation.rs create mode 100644 crates/compass-languages/tests/source_query_evidence.rs create mode 100644 crates/compass-languages/tests/typescript_member_scopes.rs create mode 100644 crates/compass-model/src/source_query.rs create mode 100644 crates/compass-model/tests/leading_documentation.rs create mode 100644 crates/compass-model/tests/source_query.rs create mode 100644 crates/compass-output/src/ranked_query.rs create mode 100644 crates/compass-output/tests/ranked_query.rs create mode 100644 crates/compass-query/examples/broad_retrieval_trace.rs create mode 100644 crates/compass-query/src/broad.rs create mode 100644 crates/compass-query/src/broad/compounds.rs create mode 100644 crates/compass-query/src/broad/excerpts.rs create mode 100644 crates/compass-query/src/broad/ownership.rs create mode 100644 crates/compass-query/src/broad/relationships.rs create mode 100644 crates/compass-query/src/index_validation.rs create mode 100644 crates/compass-query/src/source_query.rs create mode 100644 crates/compass-query/tests/broad_retrieval.rs create mode 100644 crates/compass-query/tests/broad_test_context.rs create mode 100644 crates/compass-query/tests/fielded_projection.rs create mode 100644 crates/compass-resolve/src/evidence/projection/documentation.rs create mode 100644 crates/compass-resolve/tests/callable_initializer_definitions.rs create mode 100644 crates/compass-resolve/tests/leading_documentation.rs create mode 100644 crates/compass-resolve/tests/source_query_evidence.rs create mode 100644 crates/compass-resolve/tests/typescript_member_scopes.rs create mode 100644 fixtures/callable_initializer_definitions.json create mode 100644 fixtures/typescript_member_scopes.json diff --git a/CHANGELOG.md b/CHANGELOG.md index ab47dda8a..a80bbef59 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,170 @@ ## Unreleased +- Prefer cohesive compound identifiers over scattered word mentions for broad + questions such as `load-only` field setup. Require separate owner-topic + evidence, retain Low inspection relevance and bounded real source citations, + and preserve exact identities. Code-based inspection preferences now reject + missing, empty and native-ellipsis-clipped signatures. Ranker14 refreshes + disposable response caches; graph facts and recall postings are unchanged. + +- Require frozen, independently reviewed exposure lineage and verified history + artifacts for heldout developer comparison captures. Reject known source or + linked-mechanism reuse across renamed tasks, question rewrites and commits; + preserve legacy diagnostic inputs and the existing numeric quality gates. + +- Recall indexed `item`/`element` and `copy`/`clone` alternatives when ordinary + broad-query words `entry` or `snapshot` have no indexed literal match. Require + separate source-owned literal topic evidence for lexical-only candidates; + keep dependent matches Low and disclose each translation. Preserve exact + identities and existing literal candidate admission. Refresh disposable + response caches; graph facts are unchanged. + +- Prefer callable-local operation identifiers with separate topic evidence for + broad single-operation inspection. Exclude signature-overlapping identifiers, + compound/negative/temporal conditions and mismatched document/test intent from + this preference. Preserve uncertainty and exact identity; version disposable + response caches as ranking semantics change. Graph facts are unchanged. + +- Include retained source citations on compact discovery seed lines before + relationship context, keeping selected candidates inspectable on the first + page without changing retrieval order or cursor semantics. + +- Preserve documentation coverage warnings in local SQLite queries, matching + JSON queries. Recover older snapshots through bounded diagnostic reads and + invalidate cached answers that could omit these warnings. Retain graph + coverage warnings when discovery finds no matching seed. + +- Score ranked code quotations without borrowing terms from retained overlapping + comments. Preserve the precise comment when it supplies the matching evidence. + Under tight text budgets, shorten or omit lower-ranked quotes before clipping + the first candidate's inspection evidence. Candidate ranking is unchanged. + +- Exclude Python lambda keyword tokens from auxiliary callable scopes. This + corrects scope counts and can recover source evidence at the scope quota; + semantic graph records are unchanged. + +- Add separately labelled nested anonymous-callable inspection context to + already retrieved implementations. Native AST parent facts preserve ownership + barriers and actual syntax anchors; no graph calls or execution are inferred. + Direct windows keep priority within shared limits. Source profile4, ranked + view4 and AST cache24 version the change; readers retain older profiles. + + +- Bound disposable JSON query-index validation. Retain the full integrity check; + slow eligible file checks can use a size-bounded, byte-identical inspection + image, while WAL and mismatched images keep original-file validation. Report + startup-budget exhaustion as a typed timeout without rebuilding the index. + Cancel and join operation-local SQLite interrupt timers before reusing the + connection, avoiding coarse progress-callback overruns in integrity walks. + +- Retain one exact callable-local identifier occurrence per spelling for body + retrieval, with explicit omitted-occurrence counts. This reduces repeated-site + pressure on bounded publication while preserving independently collected code + windows. AST cache24 rebuilds disposable source-query evidence. + +- Development ranked queries retain bounded AST-owned code windows for + comment-free implementations. Ranked JSON uses `compass.query.ranked-view/3`; + immutable legacy views and source-query profiles remain readable. Code windows + have independent quotas and do not change ranking or graph relationships. + + +- Keep a bare existing node ID as the sole ranked inspection target, excluding + unrelated matches for identifier prefixes such as `sha256`. Raw discovery + and optional structural context retain their contracts. + +- Preserve real source postings when their terms equal store capability markers. + Store publication uses separate capability records and retains legacy reads; + indexing Compass's own capability constant signatures no longer conflicts with + empty markers. Graph records and relationship evidence are unchanged. + +- Add bounded source-owned comment/documentation quotations to opt-in ranked + answers. Ranked JSON uses version 2 and retains version 1 reader compatibility; + text shortens/omits quotes before candidate rows to honor the output budget. + Candidate ranking, graph relationships and structural confidence are unchanged. + Omit clipped quotation windows that no longer contain matching query terms. + + +- Retain direct variable-initializer function definitions in source ranges and + TypeScript/TSX body-query evidence using exact AST owner/parent facts. Preserve + original binding provenance, stable identities and relationship occurrences; + reject ambiguous ownership. Source-query profile2 reads alongside immutable + profile1 artifacts. AST cache20 rebuilds disposable entries. JavaScript3 and + TypeScript4 remain `Qualifying` pending independent source audit and promotion. + +- Preserve non-file AST declaration/occurrence ranges during fact-neutral edits, + including declarations ending at EOF. Refresh existing file-node digest/size + metadata from file inventory. Refresh file-scoped container envelopes only + with current AST ownership and original declaration provenance; restore Kotlin + package updates after metadata-tail truncation while preserving explicit + namespace bounds and cold-build/edit-restore equivalence. + Reject non-callable body owners and report + unknown source-query schema/profile versions through typed error codes. + +- Apply broad implementation-search test preferences to colocated test/spec + files and typed test/fixture roles. Preserve explicit test intent and both + candidate sets; isolate cached answers with ranker/9. + +- Select ECMAScript member scopes by their structural parent instead of hashed + child IDs. Preserve accessor ambiguity and declared field/method targets across + filename changes, and recognize JavaScript class-field name anchors rather + than replacing them with constructor writes. JavaScript3 and TypeScript4 remain `Qualifying`; AST cache20 + rebuilds disposable facts without rewriting historical graphs. + +- Add a native, graph-bound callable-body query index for Python, Rust, Go and + TypeScript/TSX. Retain bounded AST identifiers/comments with source citations, + shared JSON/store recall and statistics, and Low inspection relevance when + body evidence supplies missing context. Publish coherent immutable artifacts, + refresh both incremental paths, and invalidate caches by content/profile. + Comparative quality and release qualification remain under verification. + +- Require topic evidence beyond generic kind/language matches for broad + questions with additional subjects. Preserve simple listings, exact identity + and tentative candidates supported by real topic evidence. + +- Recall callable owners of matching local identifiers through bounded exact + AST containment. Preserve unique ownership, source citations and shared work + ceilings; label inferred query relevance as Low inspection evidence. + +- Retain more ranked candidates within the text budget by abbreviating lengthy + explanations before dropping an owner. Preserve uncertainty, citations and + the byte limit; the JSON projection and query ranking are unchanged. + +- Recall broad implementation candidates through bounded, verified outgoing + call/reference evidence and retain source citations. Mark matches that depend + on referenced identifiers as Low inspection leads. Fix plural acronyms such + as TTLs becoming missing symbol names; preserve explicit identity guards. + +- Add bounded AST leading documentation for development Rust3, Go2 and + TypeScript4 producers, retained as source-proven rationale resources. Require + the exact excerpt and full owner anchors before broad retrieval uses leading + prose. Preserve citation ranges, report omitted excerpts in graph/query + warnings, and invalidate extraction/publication/ranking caches. These producer + versions remain `Qualifying` pending source audit and release promotion. + +- Improve broad native retrieval with fielded names, signatures, aliases, + paths and source-owned prose, shared JSON/store term statistics and bounded + declaration recall. Mark partial candidates with missing owner-specific + context and retain exact lookup behavior. Unknown ordinary wording can return + Low candidates; absent explicitly named subjects remain guarded. Negative + qualifications return inspection candidates with an explicit unverified + condition warning rather than claiming that the condition holds. + Preserve temporal conditions as explicitly unverified while retrieving on + substantive terms; keep quoted and qualified names literal. + Disposable JSON indexes rebuild to format 4; older immutable stores give an + explicit update instruction rather than claiming full field coverage. +- Recover query-index builds after a process is terminated by using persistent + operating-system locks. Recheck index validity and replace invalid indexes + only while holding the build lock; a live competing builder yields an + actionable bounded-wait error. Keep the new cache namespace separate from + older presence-lock builders. +- Add opt-in `query --view ranked` and MCP `query_graph` with `view: "ranked"`. + Show up to five source-located candidates with relevance reasons, + separate identity/evidence labels, honest recall bounds and scope refinement. + Optional `--include-context` / `include_context` appends witnessed one-hop context. + Publish the additive `compass.query.ranked-view/2` projection; existing raw + formats and discovery cursor contracts retain their shapes. + ## 0.4.2 - 2026-10-02 - Add source-scoped architecture and community views, compact hotspot rankings, diff --git a/COMPATIBILITY.md b/COMPATIBILITY.md index 6043437be..22c315170 100644 --- a/COMPATIBILITY.md +++ b/COMPATIBILITY.md @@ -1,5 +1,416 @@ # Compass compatibility +## Graph coverage diagnostic projections + +New current-store snapshots add point projections for all graph diagnostic +codes, retaining the existing first-occurrence lookup contract. This uses the +existing `diagnostic-code` metadata namespace and graph-index/1 layout; canonical +graph records, relationships and historical realizations are unchanged. +Readers recover missing projections from older immutable diagnostic supplements +within the standard 1,000-record/1 MiB scan limits. A truncated scan without a +match returns a limit error rather than asserting that no diagnostic exists. +Local query readers preserve this as a resource-limit error. Existing response +caches receive a new diagnostic-projection identity automatically, preventing +reuse of answers that omitted documentation coverage warnings. No manual cache +deletion or historical rewrite is required. +Discovery responses with no matching seed retain graph coverage warnings too; +an empty answer does not imply complete indexing. + +## Developer comparison-panel binding + +New heldout developer captures require the additive +`compass.broad-query-exposure-lineage/1` source-registration field described in +`benchmarks/agent_query/BROAD_QUERY_REVIEW.md`. Every task declares its split and +stable mechanism IDs. Hash-verified historical evidence and independently +confirmed history completeness are required before capture; known repository +or linked-mechanism exposure rejects confirmation, even with new question/task +IDs or commits. Original, lookup and union panels bind the same lineage. +Legacy registrations remain shape-valid and diagnostically readable, but cannot +qualify without lineage. Explicit development captures retain their existing +source-review requirement. This changes developer analysis/capture protocol, +not native product behavior. Freeze current normalizer/assessor hashes for a new +experiment; historical reports and their verifiers remain immutable. + +The developer benchmark adds `compass.broad-query-panels/1`, +`compass.broad-query-panel-review-input/1` and +`compass.broad-query-panel-review/1`. Unknown versions fail explicitly. Both the +original behavior panel and its declaration-lookup correction must pass the +existing canonical review gates; shared captures and costs are bound once. +The native product, graph formats and canonical review-report schema are +unchanged. Freeze the current assessor digests before collection. Historical +reports retain their original verifier provenance; changed analysis code +requires new reports and the corresponding pre-output protocol freeze. + +## Nested callable inspection context + +Current source evidence uses `source-query/4-nested-callable-context` and ranked +JSON uses `compass.query.ranked-view/4`. Readers retain source profiles 1–3 and +ranked views 1–3 with their original meanings; older readers reject the new +versions. AST cache24 refreshes disposable native scope metadata automatically. + +An already retrieved callable can expose `nested_callable_code` quotations from +an anonymous callable with explicit native AST ancestry. Each quotation carries +its exact `nestedScope` anchor, separate from its retained source slice. Named +nested functions, separately published callable bindings, ambiguous owners and +class/type boundaries cannot lend code to an enclosing owner. Lexical nesting +does not establish a call, execution or the requested behavior. + +Direct-owner windows receive capacity first. Direct and nested code share the +existing 16-window per-owner, 32,768-window and 8 MiB publication bounds; at most +eight nested scopes are retained per owner. Inspection omissions are explicit. +Incomplete nested evidence is regenerated on a fact-neutral update rather than +reusing a truncated prior projection. Ranked results still retain at most two +512-byte quotations per candidate within the first-page budget. Quote selection +uses the whole question; it does not change lexical terms, document frequencies, +candidate admission or node scores through quotations, or graph identities and +relationships. The separate scope-census correction below can recover evidence +in quota-limited files. No source +file is read at query time. + +Code-quotation priority excludes terms inside retained overlapping comment +anchors; those terms still support their separately labelled comment quote. +Under tight text budgets the first candidate's quote receives priority over +lower-ranked quotes, with omissions disclosed. JSON/source bytes, candidate +scores and graph relationships retain their existing contracts. + +## Disposable query-index validation + +JSON query-index format 4 and its cache keys are unchanged. Validation first +checks the original file with the existing full `PRAGMA integrity_check`. +After a two-second file probe, eligible rollback-mode indexes between 16 MiB and +768 MiB can be inspected through a serialized image. Its complete bytes must +equal the physical original, including EOF. The image is never published or +used as the query connection. WAL headers, size mismatches and unavailable +images retain original-file validation; they do not imply corruption. + +Metadata and integrity work share a 30-second validation budget. Expiry returns +`query_index_validation_timeout` with kind `Timeout` and leaves the cache in +place. A joined, operation-local SQLite interrupt timer complements progress +callbacks, so a long integrity operation cannot defer expiry until the next +coarse callback interval. The timer is cancelled before any subsequent SQL; +thread-start failure uses finer progress checks. Blocking filesystem operations +must still return before SQLite can observe interruption. A timeout must not be +treated as a corrupt index or an empty answer. The budget +does not cover loading the graph or constructing a missing index. Historical +graphs, source-query profiles, graph relationships and query response schemas +are unchanged. No manual cache deletion or data migration is required. + +## Callable initializer definition ranges + +JavaScript3 and TypeScript4 also publish the full definition range of a direct +variable-bound function. The binding's original AST anchor continues to identify +the node and its provenance. The resolver requires a uniquely owned function +scope parented by the binding scope; inherited callback scopes cannot substitute. +Multiple qualifying scopes remain ambiguous. No relationship is synthesized. + +Source-query profile2 uses the exact direct variable-declarator envelope for +TypeScript/TSX lexical body ownership. Immutable profile1 artifacts remain +readable with their original evidence; unknown profiles fail explicitly. +AST cache24 rebuilds disposable facts. These producers remain `Qualifying`; +the existing release-promotion decision is unchanged. + +## ECMAScript primary member scopes + +JavaScript3 and TypeScript4 select a unique owned child whose parent is the +declaration's binding scope for member lookup. Nested blocks and callable-type +scopes that inherit the same owner cannot win by hashed ID. Ambiguous accessor +fixtures remain unresolved; declared fields, methods and lexical shadowing keep +source anchors and occurrence evidence. JavaScript class fields read their +grammar's `property` name anchor rather than requiring TypeScript's `name` field. +These development producers are +`Qualifying` pending a new source audit and promotion decision. + +AST cache24 invalidates disposable extraction entries and current build state. +Run `compass update ` to obtain corrected current references. The +extraction/5 identity seed, evidence/2 and graph/1 formats are unchanged; published +historical realizations are not rewritten. + +## Broad source-evidence retrieval + +Current store snapshots record search capabilities in a separate term-index +namespace. Source terms equal to older capability-marker strings remain ordinary +searchable postings, including their complete node IDs and chunks. Readers still +accept historical empty-posting markers; new writers retain those legacy markers +where no real source posting occupies their key. Older readers may report the +affected capability unavailable on a marker collision and should be upgraded. +Unknown or malformed explicit capability versions fail rather than falling back +to a legacy marker. The additive records keep graph-index/1 and the graph's +canonical records, identities and relationships unchanged. Historical snapshots +are not rewritten. + +Broad native implementation questions now use a shared bounded field projection +of names, paths, existing signatures/types, aliases, kinds, roles, language and +source-owned document text. Each field is bounded to 8,192 UTF-8 bytes and 512 terms. Alias +enrichment reads at most nine relationships; more than eight aliases omits the +alias field and reports bounded recall instead of choosing an arbitrary subset. +Fielded aliases require nonempty, exact, nonheuristic provenance; ineligible raw +relationships still count toward the bound. Legacy strict alias lookup retains +its existing meaning. + +JSON query-index format 4 uses a separate cache key from older indexes and +rebuilds disposable format-2/3 indexes automatically. +New immutable snapshots publish additive field-frequency and declaration-channel +postings; older snapshots remain readable with an explicit incomplete-recall +warning. Run `compass update` on a current checkout to publish those postings. +Historical realizations are never rewritten. Graph identities, relationship +kinds/direction/multiplicity and exact-symbol command meanings are unchanged. +The active query ranker profile is `query-ranker/14-compound-identifier`; existing response +cache identities include that profile and do not reuse old ranked results. + +Broad retrieval can give an inspection preference to hyphenated query compounds +whose two to four normalized words occur contiguously in a callable's own name +or a single retained owner-local identifier. It considers at most eight unique +compounds; larger compound requests retain ordinary ranking without selecting +an arbitrary subset. It never joins separate identifier sites, uses prose or +namespace/path mentions as compound proof, or changes recall postings/counts. +Separate owner-field topic evidence outside the compound is required. Explicit +identities, document requests, mismatched test contexts, recognized negative or +temporal conditions and vocabulary-dependent candidates receive no compound +preference. Clipped field projections keep ordinary ranking. Dependent matches +remain Low, below exact identity tiers, with explicit unverified-behavior reasons. +Actual matching identifier anchors receive priority within the existing eight +body-source citation cap; omissions retain boundedness. This changes neither +source/graph schemas nor quotations' ownership. The cache epoch is +`compound-identifier/2`. + +Ordinary broad-query words `entry` and `snapshot` have directional vocabulary +fallbacks to `item`/`element` and `copy`/`clone`, respectively. A group expands +only when known corpus statistics show no indexed literal variant and an indexed +alternative exists. These statistics describe the index, not a source-wide or +query-scope absence proof. Queries with named contexts or explicit identities, +document requests and unavailable legacy statistics do not use this fallback. +At most four alternative-frequency probes share the existing work limits. +Lexical-only candidates depending on an alternative require separate literal +non-action topic evidence in an owner field; facets, paths, namespace prefixes +and generic document links cannot supply it. Existing explicit semantic-channel +admission remains separate. Dependent matches are Low, name the translations, +use the existing `synonym` concept-match method and receive no operation +preference. Known literal groups and exact identities retain their meaning; +expansion cannot veto a literal-only candidate that passed its original +supported-concept screen. Graph/source schemas and immutable artifacts are +unchanged. The additional response-cache epoch is `unindexed-vocabulary/1`. + +For broad implementation questions with one recognized operation concept, +callable-local operation identifiers and separate topic evidence receive an +ordinal inspection preference. An operation mentioned only in prose or a +retained signature does not supply it. Identifier postings include parameters, so terms +also present in the retained signature are excluded from this code signal; +missing/empty signatures, native display signatures ending in `…`, or clipped +search-field projections retain ordinary ranking for code-based signals. +This is not an AST body-role classification: an identifier absent from the +retained signature is still only a lexical inspection signal. A callable's own name +remains a direct name signal. Multiple-operation, negative and temporal questions, +document requests and mismatched test contexts retain ordinary ranking. This +preference changes neither relevance bands nor structural confidence and never +establishes execution. Exact IDs/full names retain their higher tiers. Graph +facts, source schemas and read limits are unchanged; the response-cache context +epoch also invalidates intermediate experimental answers. + +Broad method recall also probes at most eight relationship-term postings, +hydrates at most 64 candidate owners, and examines at most 32 outgoing calls or +references per owner. This channel charges its decoded records and probes to the +existing shared hard ceilings, with a separate 6,400-read/96-probe sub-budget. +It requires nonempty exact AST node/edge evidence and a relationship site inside +the callable's source range. Only the target's bounded name field contributes; +its qualified name, path and prose do not. Additional `code_reference` matched +fields retain source citations and actual relationship kinds/direction. A +candidate that needs this field to match a concept is Low unless its ID or full +name matches exactly: using an identifier does not prove the requested behavior. +Property leaves receive the same implementation-search penalty as fields and +variables. Older immutable snapshots without broad relationship postings retain +direct-call recall with a specific update warning. Reached quotas explicitly +report bounded recall. Ordinary plural acronyms such +as APIs and TTLs normalize as prose; quoted and qualified identities keep their +original identity guards. No graph schema, identity or published record changes. + +Broad implementation ranking recognizes colocated `.test.`/`.spec.` files, +conventional Python/Go/Rust test filenames and typed test/fixture roles. The +existing soft context preference remains reversible by an explicit test request; +it does not exclude candidates or change exact/directed lookup semantics. +Response caches use the new ranker profile automatically. + +Broad questions require subject evidence beyond `kind` and `language` when +other subjects are requested. Generic facets cannot stand in for missing topic +evidence. Queries whose non-action subjects all match those facets retain their +listing behavior; exact identity and explicit semantic fallback remain intact. +Unfamiliar wording can still yield tentative candidates that have other topic +evidence. The 40% supported-concept threshold and work ceilings are unchanged. + +Broad local-declaration recall examines up to 32 matching parameter, variable, +field, property or constant names and four incoming `contains` edges per +declaration. It requires one unique callable parent with exact AST node/edge +evidence and enclosed source ranges. Truncated parent lists cannot establish +uniqueness. Only bounded terminal-name terms contribute `owned_identifier`; +namespace/path matches and nested callable declarations do not. Answer scopes +apply to the parent after uniqueness is checked. This channel reserves 288 +decoded records and 32 probes within the existing shared hard ceilings; reached +limits remain explicit. Source citations and actual containment witnesses are +retained. A candidate needing this field is Low unless its ID or full name +matches exactly. Containment establishes ownership, not requested behavior. + +A witnessed documentation link contributes an owner's prose only when the +source range establishes that it is enclosed in that declaration, or when the +versioned leading-documentation AST proof retains both the exact excerpt and +full declaration ranges in the same provenance item. General links +can suggest a candidate only when the source-backed passage contains its actual +name and the question's subjects. They never establish ownership or structural +confidence. Missing subject +evidence yields no supporting indexed evidence, with scope/limit caveats. +Partial candidates name missing owner-specific subjects or named contexts and +carry Low relevance; corpus-wide word occurrence does not establish that a +candidate implements the requested behavior. Source-enclosed prose remains +available as supporting evidence rather than duplicate implementation rows. +Standalone `not`, `never` and `without` retain an unverified condition: lexical +retrieval returns Low inspection candidates with an explicit warning. Inspect +source or use a structural query to verify the condition. Optional native LSA +remains explicit and independently bounded; +it adds approximate candidates without establishing structural confidence. + +Query-index builders hold a persistent operating-system lock through validation, +replacement and publication. Termination releases ownership; the coordination +file remains and its existence alone does not indicate a live builder. A +five-second retry budget bounds lock contention, with a typed actionable +`query_index_lock_timeout`; it does not bound graph loading or index construction. +Invalid indexes are removed only under the lock, after rechecking whether another +builder already published a valid result. Format 4 isolates this protocol from +older binaries that used file presence as ownership. Graph/history formats and +query ranking are unaffected by this recovery change. + +## Source-body query evidence (development implementation) + +The native parser now retains bounded callable-local identifiers and inline +comments for Python, Rust, Go and TypeScript/TSX in a reserved per-file extraction +extension, `compass.source-query.extraction/1`, with +`source-query/3-owned-code-windows`. It preserves original UTF-8 source slices, +byte positions, a source digest and explicit omissions. Nested callables have +separate owners; class/type boundaries block inheritance. Arbitrary string +literal contents are excluded. Repeated occurrence omissions are counted +separately from missing term coverage. Work is bounded to a 4 MiB source file, +200,000 AST visits, depth 512, 512 scopes, 256 names per scope, one exact site +per spelling, eight comments per scope, 4 KiB per comment and 1 MiB retained text +per file. Omitted repeated occurrences increment the occurrence counter without +claiming missing term coverage. Code windows are collected independently at +identifier visits, including repeated spellings. Readers retain support for +historical projections containing up to four sites per spelling. +Unsupported or out-of-profile evidence is absent or explicitly omitted. + +AST cache24 rebuilds disposable extraction entries to retain this metadata. +The semantic extraction/5 seed, Fact IDs and dependency graph contract do not +change for this query-only metadata. A typed `compass.source-query/1` auxiliary +artifact retains bounded source excerpts and corpus-statistic validation. Native +updates now stage it with the graph, bind it to the exact graph/file digests and +one unique exact-AST callable range, and include it in atomic publication and +required build-state seals. Per-file inventory supports both fact-neutral +incremental paths; invalid or aggregate-truncated prior artifacts require full +reconstruction. Query-only metadata does not enter structural fact digests; +existing declaration source hashes still track body-comment edits. + +Current JSON and local-store readers share the bounded body projection for +postings, document-frequency increments and scoring. Body recall probes at most +eight terms and hydrates at most 64 owners within existing shared work ceilings. +The in-memory index admits at most 262,144 postings and 65,536 terms; exceeding +these limits is an explicit error. Auxiliary input is capped at 64 MiB, 32,768 +owners/files, 65,536 source sites and 16 MiB retained text. Reached extraction or +publication limits remain explicit. Body-only relevance remains Low inspection +evidence; source occurrence does not establish the requested behavior. Response +and long-lived engine cache identities include auxiliary content identity. +Present corrupt, mismatched or unknown-major artifacts fail explicitly. Absent +older artifacts remain readable with a missing-body-recall warning. Detached +in-memory graphs and historical selectors do not borrow current source metadata. +The reader reports unsupported auxiliary versions with `UnsupportedSchema` +and `unsupported_source_query_schema` / `unsupported_source_query_profile`, +before interpreting current payload fields. A retrieved owner must remain an +exact-AST callable. Neutral edits preserve declaration/provenance ranges even +when a declaration coincides with the old EOF. File nodes and source-proven +file-scoped language containers can refresh their envelopes. A container requires +one directly owned full-file scope, its file-root parent, and matching original +AST declaration provenance; ambiguous scopes use full publication. This restores +Kotlin package-envelope refresh without widening explicit namespaces or callable +bodies to include outside comments. Historical realizations remain immutable. +Integration checks are underway; +comparative quality and release qualification remain unproven. + +## Leading documentation evidence (development) + +Rust3, Go2 and TypeScript4 producers are `Qualifying` pending a new source audit +and release promotion decision. Their optional `DeclarationFact.documentation` +field under evidence/2 carries one exact leading AST comment group. Rust source +consumers constructing `DeclarationFact` literals must initialize `documentation` +(normally `None`); serde readers still accept an absent field. Older strict +evidence readers must admit this field; absent metadata remains readable. +Excerpts retain raw UTF-8 source bytes (including the Rust line-comment newline) +up to 4096 bytes, with 1 MiB per file and 128 MiB per resolver index. + +Publication/3 emits `Resource(rationale)` nodes and `documents` edges under +`universal-leading-documentation-v1`. The existing provenance-anchor vector +retains the excerpt and its unique full declaration range; graph/1 has no new +wire fields. A claimed rule with a missing or inconsistent ownership proof is +invalid. Document content must retain exact nonheuristic evidence with its AST +excerpt anchor; the owner must retain an exact AST anchor inside its full source +range. A precise edge cannot promote contradictory node evidence. Generic +documentation links retain their existing meaning. The query +ranker consumes the shared proof and preserves source citations; it does not +infer ownership from names. Excerpt/attachment limits and ambiguous definition +ranges publish a `leading_documentation_omission_summary` warning consumed by +both JSON and store query paths. Structural declarations remain available. + +Extraction/5 and AST cache24 invalidate prior disposable extraction facts. +Rebuild current graphs to obtain this evidence; immutable historical graphs are +not rewritten. Python docstrings and the JavaScript producer are unchanged. +Native regression checks do not establish comparative query quality or producer +qualification. + +## Ranked implementation view + +`compass query QUESTION --view ranked` adds an opt-in implementation shortlist. +Text shows up to five ranked owners and keeps the whole answer within +`--text-budget`, including coverage, omission guidance and a narrowing action. +The renderer first tries shorter combined explanations before dropping a ranked +owner. It labels abbreviated reasons, retains relevance/identity/evidence labels +and source citations, and keeps Low matches explicitly unverified. The bounded +JSON projection retains its original reasons. If metadata and one whole candidate cannot fit, it returns an explicit error. +`--view ranked --format json` emits `compass.query.ranked-view/4`; this is a +separate projection, not a change to raw discovery JSON or Agent View. +Ranked view rejects discovery cursors, result envelopes, evidence mode, +legacy traversal controls and `agent-json`. Existing modes remain available. +When the entire question is an existing node ID admitted as a discovery seed, +the ranked shortlist contains that seed alone. Raw discovery and optional +structural context retain their existing contracts; words inside an ID do not +make unrelated lexical matches additional ranked targets. + +MCP `query_graph` accepts `view: "ranked"` and an optional `text_budget` (1–65536, +default 800). Its existing `result` field retains `compass.query.discovery/1`; +the optional `rankedView` field carries the new projection. Strict envelope +decoders must admit that field when requesting this view. Default requests +retain their existing Agent View. Unknown ranked-view major versions fail +explicitly in the shared decoder. + +Ranked context is opt-in through `--include-context` / `include_context: true` +(default false). It appends up to eight witnessed one-hop relationships with +their endpoint records, direction, occurrence identity and source evidence. +Context has its own omission fields and never changes candidate order; text +drops whole context rows before candidates to satisfy the same answer budget. + +Relevance bands are ordinal retrieval judgments, not calibrated probabilities. +Unknown ordinary query wording can yield Low candidates with missing-context +reasons instead of suppressing every result. Explicit names (including quoted +and underscore identifiers) remain guarded when absent from the index. +Negative wording is retained as an unverified condition: candidates are Low +with an `incomplete_coverage` diagnostic and an explicit qualification warning. +It never establishes a negative behavior from lexical matches. Unquoted +`before`, `after`, `during`, and `until` likewise remain unverified temporal +conditions, with Low candidates and an explicit diagnostic. These connectives +do not count as lexical subject evidence; quoted and qualified identifiers +keep their literal terms. These retrieval +semantics use `query-ranker/14-compound-identifier`; prior profile constants remain +available and disposable response cache identities change automatically. +Name identity, structural evidence, graph coverage and candidate recall bounds +remain separate. `responseTruncated` retains discovery's aggregate bound, +including candidate/source-field/context limits; it does not assert that a +traversal itself was cut short. A zero-result answer reports no supporting +indexed evidence; it does not claim that the behavior is absent. Historical +realizations remain immutable. + Concept recall adds optional `conceptMatches` to `compass.query/1` with closed methods `synonym`, `document_link`, and `semantic_lsa`. Strict consumers must update to the matching enum/field manifest and fingerprint. Absence means no @@ -557,6 +968,10 @@ Discovery text pagination now uses node inventories. Older discovery cursors fail with an explicit version error; restart the query to obtain a new cursor. Natural discovery text now defaults to an 800-token page; `--text-budget N` and `--cursor` retain caller control. +Compact `SEED` lines now include their retained source anchor, when available, +before relationship context can consume the page. Missing anchors remain +unlocated. This additive text annotation preserves the version 3 entry order, +semantic positions and cursor contract; JSON and evidence-mode text are unchanged. The typed commands accept `--format agent-json`; default text is an answer-first presentation. MCP keeps `compass.mcp.tool-result/1` and adds the @@ -1740,3 +2155,52 @@ Graph, Program IR and immutable historical schemas are unchanged. Disposable impact/traversal cache headers `TRAILA04` and `TRAILT08` retain the pinned build commit; older caches are ignored and rebuilt. Missing Program IR exits with code 3 and recovery commands; unknown Program subcommands exit with code 2 before artifact access. + + +## Ranked source quotation projection + +The opt-in ranked answer projection now emits `compass.query.ranked-view/4`. +Native readers retain version 1 compatibility with absent quotations; version 1 +payloads containing the new field and unknown major versions fail explicitly. +Existing raw discovery JSON and cursor contracts are unchanged. + +At most two 512-byte immutable quotations are retained for each of the first five +candidates. They come only from source-query comments or proven raw leading +documentation. Normalized docstrings and general links retain their existing +retrieval pointers. Text may shorten/omit quotations to preserve candidate rows +inside the whole-response budget. This changes presentation, not ranking or +structural confidence. The response cache has a new projection epoch while +query-ranker/10, source-query schemas/profiles and immutable graphs are unchanged. + + +## Bounded code inspection windows (development) + +Source-query profile `source-query/3-owned-code-windows` adds optional +`codeWindows` and `omittedCodeWindows` per callable under `compass.source-query/1`. +New readers retain immutable profiles1/2; legacy profiles cannot claim these new +fields. Older readers reject profile3 and must update. AST cache24 rebuilds +disposable current extraction. Graph identities, relationships, extraction/5, +ranking profile10 and retrieval terms/statistics are unchanged. + +Each window retains the exact UTF-8 bytes of a small AST ancestor, excluding +nested callable/type subtrees and parser recovery. Selection walks at most eight +ancestors and 128 subtree nodes per attempt, with200,000 additional AST visits per +file, at most16windows of512bytes per owner and256KiB per file. Code text has +independent publication caps of8MiB and32,768windows; it cannot consume the +16MiB lexical budget. Omitted windows are counted separately. These are inspection +windows, not complete function bodies or proof of the requested behavior. + +Ranked output now emits `compass.query.ranked-view/4` to add the typed +`callable_code` excerpt kind. The reader accepts strict versions1/2; version1 +cannot contain excerpts, and version2 cannot contain code excerpts. Update +consumers before reading version3. Query projection epoch `ranked-source-excerpts/3` +invalidates disposable responses. Output retains at most two512byte excerpts +for each of the first five candidates, subject to the existing whole-answer +budget. Current source files are never opened during quotation selection. + +Python auxiliary scope extraction now requires a named AST node: the unnamed +`lambda` keyword token is not another callable. This corrects scope counts and +can recover later source owners when the per-file scope quota was previously +spent on keyword tokens. The change can affect source-field coverage and +retrieval scores in quota-limited files; it does not change semantic graph +records. AST cache24 rebuilds the affected disposable evidence. diff --git a/MIGRATION.md b/MIGRATION.md index d0de600fd..8e012bbc4 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -1,10 +1,125 @@ # Migrate from Graphify to Compass +Current updates rebuild AST cache24 and retain callable-body evidence for direct +TypeScript/TSX variable initializers. Their source ranges now cover the complete +definition; original binding identity and AST provenance stay distinct. Older +`source-query/1-ast-callable-lexemes` artifacts remain readable and historical +realizations are unchanged. New artifacts use profile4; older binaries must be +upgraded to read that projection. JavaScript3 and TypeScript4 remain `Qualifying` +pending source audit and release promotion. + Compass uses its own executable, output directory, environment variable, and sidecars. Its output root now preserves the familiar flat artifact shape so file-based workflows can transition while Compass's snapshot and store layout remains visible and clearly owned. +## Developer comparison registrations + +Before a new heldout comparison, freeze an independently reviewed exposure +history, explicit task splits/mechanism IDs and hash-bound historical evidence +using the [review protocol](benchmarks/agent_query/BROAD_QUERY_REVIEW.md). +Previously used repositories remain development inputs even when their commit, +task IDs or wording change. Legacy diagnostic registrations remain readable; +they cannot establish an untouched confirmation split. Freeze current capture +and assessor hashes; preserve existing reports with their original verifier. +Native Compass commands and graphs require no migration for this protocol. + +## Opt into ranked implementation answers + +Upgrade ranked JSON consumers to `compass.query.ranked-view/4` before using new +nested code quotations. `nested_callable_code` includes an exact `nestedScope` +anchor; it means lexical inspection context, not execution. Consumers must not +render it as direct-owner code or a graph call. New builds use source profile4 +and AST cache24; run `compass update ` to refresh disposable evidence. +Readers retain old source/ranked versions with their original meanings. + + +JSON query-index validation can now return `query_index_validation_timeout`. +Treat it as a startup timeout and retry or narrow the build scope. The cache is +preserved; no format migration or manual cache removal is required. + +Current updates retain one callable-local identifier site per spelling, counting +omitted repeats separately. AST cache24 refreshes disposable evidence +automatically; no manual cache removal is needed. Historical source artifacts +retain their original sites. Code windows still inspect repeated identifiers +independently of the lexical site budget. + +Use `compass query "where are request errors formatted?" --view ranked` for a +top-five shortlist. Existing query formats and cursors require no migration. +Request `--format json` with this view only when the consumer understands +`compass.query.ranked-view/4`; it has its own typed fields and rejects unknown +major versions. The shortlist has no discovery cursor: narrow with its suggested +source scope to examine other implementations. Do not combine it with +`--cursor`, `--result-envelope`, `--evidence`, `--format agent-json`, or legacy +traversal controls. + +Optional `--include-context` appends bounded one-hop witnessed relationships +after the shortlist; it is disabled by default. + +Run `compass update ` to publish fielded signature/prose/alias postings in +an existing current graph. Older immutable stores remain readable with a +limited-recall warning; historical realizations are not rewritten. Disposable +JSON query indexes rebuild automatically to format 4, isolating their new +operating-system lock from older presence-lock caches. Terminated builders no +longer require manual lock-file removal. Retained lock files alone do not mean a +builder is active. Broad questions with +negative wording may return Low inspection candidates with a warning that the +condition remains unverified. Inspect their source or use a structural query to +verify it. Unquoted `before`, `after`, `during`, and `until` are also reported as +unverified conditions rather than required lexical subjects. Quoted names and +qualified identifiers remain literal. Unfamiliar ordinary words reduce relevance and appear as missing +context; absent explicit names still guard retrieval. A Low partial candidate +does not establish the requested behavior. Disposable response caches use the +new `query-ranker/14-compound-identifier` profile automatically. Hyphenated +compound inspection matches remain Low with explicit reasons; code-based +preferences require a nonempty retained signature without native `…` clipping. +Exact identities and graph facts retain their meaning. No manual cache deletion +or graph rebuild is needed for this ranking-only change. + +Run `compass update ` to obtain the optional immutable `source-query.json` +artifact for callable-body identifiers and comments on Python, Rust, Go and +TypeScript/TSX. AST cache24 rebuilds disposable extraction entries automatically. +Body-dependent matches carry exact source locations and Low inspection relevance. +Old graphs keep their existing evidence with an explicit body-recall warning; +historical realizations are not rewritten. Keep the auxiliary artifact with its +matching graph snapshot. A present corrupt/mismatched artifact fails explicitly; +`compass update --force` reconstructs current source evidence. + +MCP clients opt in with `view: "ranked"` (and `include_context: true` for context). Read its projection from `rankedView` +and the raw discovery shape from `result`. Allow the new optional envelope field +and keep relevance separate from exact identity and structural confidence. + +Broad implementation questions can also recall methods through verified local +call/reference sites. A `code_reference` match is an inspection lead, with a +source citation and Low relevance when it supplies otherwise missing context. +It does not establish the requested behavior. No graph rebuild is required for +this query-only channel when existing relationship postings are available. + +## ECMAScript primary member scopes + +Run `compass update ` to rebuild JavaScript/TypeScript current graphs with +structurally selected member scopes. AST cache24 discards older disposable +entries automatically. JavaScript3 and TypeScript4 remain development producers +marked `Qualifying`; their updated identities do not alter the approved historical +promotion record. Evidence/2, graph/1 and historical realizations retain their +existing contracts. + +## Leading documentation evidence (development) + +Rust integrations constructing `DeclarationFact` values must initialize the new +optional `documentation` field (normally `None`). Serialized absent fields remain +readable. + +Run `compass update ` with the matching new binary to rebuild current +Rust/Go/TypeScript graphs with leading AST documentation. Strict readers of +`compass.languages.evidence/2` must allow the optional declaration +`documentation` object. Graph/1 retains its existing resource, relationship and +provenance fields; no historical realization is modified. Disposable extraction +facts rebuild under extraction/5 and AST cache24. The changed producer versions +remain `Qualifying` until a source audit and new promotion decision are complete. +Treat ranked results as source inspection candidates, not a verified behavioral +answer or a claim of superiority over another tool. + ## Scoped views and freshness No rebuild is required for new scoped views or hotspot ranking. Explicitly @@ -671,3 +786,29 @@ Discovery text cursors now use version 3 because compact pages show relationship before declarations. Restart a query if a saved version 2 cursor is rejected. Natural discovery pages default to 800 tokens. Increase `--text-budget` or follow the printed `--cursor` to retrieve more of the same bounded answer. + + +## Consume ranked source quotations + +Clients of the opt-in ranked JSON projection must accept +`compass.query.ranked-view/4` and its bounded `excerpts` per candidate. The native +reader still accepts immutable version 1 results without quotations; unknown +major versions fail explicitly. No graph/history migration is required. Older +graphs can lack quotation evidence and retain their original retrieval scope. +Quoted comments/docs support source inspection, not verified behavioral answers. + + +## Ranked code excerpts (development) + +Consumers of opt-in ranked JSON must accept `compass.query.ranked-view/4` +and the `callable_code` excerpt kind. Compass still reads immutable ranked views1/2 +and source-query profiles1/2. Run `compass update .` with the updated binary to +retain code windows for current source; old historical artifacts remain unchanged. +Code windows do not change candidate ranking or create graph relationships. + +Python auxiliary scope extraction now requires a named AST node: the unnamed +`lambda` keyword token is not another callable. This corrects scope counts and +can recover later source owners when the per-file scope quota was previously +spent on keyword tokens. The change can affect source-field coverage and +retrieval scores in quota-limited files; it does not change semantic graph +records. AST cache24 rebuilds the affected disposable evidence. diff --git a/PERFORMANCE.md b/PERFORMANCE.md index b85cf8425..daa1f54d5 100644 --- a/PERFORMANCE.md +++ b/PERFORMANCE.md @@ -26,6 +26,44 @@ review and evidence explaining the tradeoff. ## Natural-language answer cost qualification +JSON query-index validation uses a two-second original-file probe and a shared +30-second validation budget. Only eligible rollback-mode indexes from 16 MiB to +768 MiB may use a byte-verified in-memory inspection image after that probe. +Warm file checks normally avoid the image allocation. The image can temporarily +add up to 768 MiB plus SQLite working memory; eligibility is not a memory-use or +end-to-end latency qualification. Graph loading and missing-index construction +are outside this validation-stage deadline. + +An operation-local interrupt timer complements SQLite progress callbacks during +metadata and integrity SQL. It is cancelled and joined before the next operation; +if the thread cannot start, progress checks use an interval of one. This avoids +waiting for a coarse callback interval in long integrity walks. Interruption is +cooperative inside SQLite and cannot cancel a blocked filesystem call. No memory +mapping hint is added to validation. + +The 2026-10-17 bundled-SQLite stage diagnostics in Plan 029 used an exposed +537,133,056-byte Compass index. Two file checks on fresh APFS clone inodes hit a +60-second diagnostic deadline; two image checks completed in 3.883/3.518 seconds. +On the already warm inode, file checks took 0.643/0.647 seconds and image checks +0.773/0.751 seconds, with peak RSS about 5.8 MB and 545.7 MB respectively. These are +small stage diagnostics affected by hashing, shared extents and prior cache +state. They motivated the adaptive candidate; they are neither certified cold +measurements nor a production performance baseline or Graphify comparison. +Native end-to-end and platform qualification of the candidate remain required. + +Fielded broad retrieval retains the shared hard ceilings of 12,801 indexed +candidate records and 291 candidate probes, including reserved routing work. +Its search-stage counters cover posting, documentation-ownership and alias +reads; bounded intent-routing checks are reserved separately. Optional local +LSA has its own limits (32,768 nodes and 64 MiB of corpus data), uses the same +absolute query deadline, and is not included in those indexed-read counters. +Compare complete process wall time and peak RSS when measuring its cost. + +The new broad-query qualification workflow is described in +[`BROAD_QUERY_REVIEW.md`](benchmarks/agent_query/BROAD_QUERY_REVIEW.md). +Development traces and native regression tests establish individual fixes; +they do not establish a current-release superiority or performance claim. + The developer-side [agent query harness](benchmarks/agent_query/README.md) qualifies answer recall and output cost separately from runtime performance. Run `suite_natural.toml` on its five pinned repositories with the same question @@ -1554,3 +1592,23 @@ final verdicts. contains source and artifact pins, suite and binary hashes, the registration and amendment, every oracle observation and every cold/warm timing. These focused query measurements do not replace release-wide performance qualification. + +## Nested callable quotation bounds + +Source profile4 adds lexical context to already retrieved owners without adding +lexical postings, document-frequency contributions or node-score work. Native +preorder parent indexes allow a linear bounded ownership pass. Direct windows +receive publication capacity first; nested code uses the same 16-window owner, +32,768-window and 8 MiB totals, with at most eight nested scopes per owner. The +ranked shortlist keeps two512-byte quotes per candidate. Incomplete nested +context disables reuse of that auxiliary projection on a fact-neutral update; +it is regenerated through the existing build path. This can increase update +cost and must be included in end-to-end qualification. No latency, memory or +Graphify superiority claim follows from these limits. + +Python auxiliary scope extraction now requires a named AST node: the unnamed +`lambda` keyword token is not another callable. This corrects scope counts and +can recover later source owners when the per-file scope quota was previously +spent on keyword tokens. The change can affect source-field coverage and +retrieval scores in quota-limited files; it does not change semantic graph +records. AST cache24 rebuilds the affected disposable evidence. diff --git a/SECURITY.md b/SECURITY.md index 1f0ce2529..83329683d 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -90,6 +90,20 @@ of one size-bounded byte snapshot. Digest equality establishes content identity, not authenticity or semantic correctness. Disposable cache decoding is also byte-bounded; caches do not authorize access to their source graph. +Disposable JSON query-index metadata and full-integrity inspection have a shared +validation deadline. A slow file pager may use an inspection-only image bounded +to 768 MiB, pinned by a read transaction and compared against every original byte +and EOF. WAL-context images and physically different images are ineligible and +fall back to original-file inspection. The copy uses the same full integrity +operation on a writable in-memory handle required by FTS5; it grants no graph +trust and is never published. A validation timeout preserves the original cache +and returns a typed error rather than starting a replacement build. +SQL deadline timers use the safe SQLite interrupt handle, are scoped to one +operation, and are cancelled and joined before the connection is reused or +closed. A progress callback remains as a fallback, with finer checks if the +timer thread cannot start. Blocking filesystem calls may delay observation of +the deadline; the image retains its byte and size guards. + ## Document and OCR boundary PDF and OOXML files, XML relationships, compressed members, embedded images, @@ -138,6 +152,19 @@ include those excerpts as well as existing names and provenance. Apply the same repository disclosure policy to these graph artifacts. Corpus size/deadline failures remain explicit and do not publish a partially built semantic index. +## Local source-body query evidence + +Native extraction caches and the immutable `source-query.json` artifact can +retain parameter/local identifiers and bounded inline comments from indexed +source. Treat these as repository-derived source content when sharing output +folders. They remain local; no provider, credentials or source-file reads are +added to query execution. Current queries read only the bounded regular-file +artifact beside the selected graph, validate its schema/profile and digest +binding, and retain exact source coordinates. Hashes detect content mismatch; +they do not authenticate source authorship or establish behavioral correctness. +Detached/historical graph readers do not borrow current body evidence. Text +occurrence never creates graph dependency edges or structural confidence. + ## Local query and saved-output caches Completed-output budgets can save source excerpts, report content and diagnostics @@ -172,3 +199,36 @@ The marked root can be outside the current project; its path appears in the advisory recovery command and MCP freshness metadata. Shell quoting is display only. Source-root markers are local build metadata, not a trust assertion that arbitrary imported graph bytes describe that checkout. + + +Ranked answer quotations can disclose retained comment or documentation bytes +through local CLI/MCP output. They read only immutable, graph-bound evidence +with proven declaration ownership; query-time mutable source reads and generic +neighbor prose are excluded. Quotations remain untrusted data, with controls +and bidirectional formatting marks escaped in text output. Treat their contents +as source material, never instructions. Normalized legacy docstrings do not +qualify as verbatim source quotations. + + +## Local code inspection excerpts (development) + +Opt-in ranked answers may include exact code windows from the immutable +source-query artifact, including literals present in that code. They are not +redacted summaries. Windows stay within one AST-owned callable, exclude nested +callable/type subtrees, and have independent extraction/publication/output caps. +The query never opens current working-tree files to supply missing text. Treat +these excerpts as untrusted source content when rendering; existing terminal +control/bidirectional escaping and JSON encoding apply. A source window is +inspection evidence and must not be interpreted as an instruction or a proven +runtime relationship. + +## Nested lexical source context + +Nested code quotations require an explicit bounded native AST parent chain and +a unique exact published enclosing callable. They retain the actual callable +syntax range, separately from variable definition envelopes. Class/type scopes, +named children and separately published callables are ownership boundaries. +Quotes remain untrusted source text; renderers preserve existing escaping and +output budgets. Context does not add graph records, calls, execution claims, +retrieval terms or network/model dependencies. Source profile4 and ranked view4 +make the distinction explicit; historical profiles retain their original meaning. diff --git a/crates/compass-cli/src/freshness.rs b/crates/compass-cli/src/freshness.rs index 70edf17e3..2afb86aa8 100644 --- a/crates/compass-cli/src/freshness.rs +++ b/crates/compass-cli/src/freshness.rs @@ -63,6 +63,20 @@ pub(super) fn record_program(path: &Path) { record_with_mode(path, None, true); } +/// Bytes that `finish` will append to stderr, including line terminators. +pub(super) fn pending_warning_bytes() -> usize { + FRAMES.with(|frames| { + frames.borrow().last().map_or(0, |frame| { + frame + .values() + .flatten() + .filter_map(GraphFreshness::warning) + .map(|warning| warning.len().saturating_add(1)) + .sum() + }) + }) +} + fn record_with_mode(path: &Path, commit: Option<&str>, program: bool) { let capture = FRAMES.with(|frames| { let mut frames = frames.borrow_mut(); diff --git a/crates/compass-cli/src/help.rs b/crates/compass-cli/src/help.rs index 36ce1e873..c980e693a 100644 --- a/crates/compass-cli/src/help.rs +++ b/crates/compass-cli/src/help.rs @@ -1125,6 +1125,10 @@ fn render_page(page: &Page, style: HelpStyle) -> String { ) } else if page.path == "query" { details + .replace( + "--direction ", + "--view ranked Opt-in top-five implementation shortlist\n --include-context Append bounded one-hop witnessed context\n --direction ", + ) .replace( "Query an exact immutable realization; conflicts with --graph", "Resolve REV once to an immutable typed realization; conflicts with --graph", @@ -1158,6 +1162,9 @@ fn render_page(page: &Page, style: HelpStyle) -> String { output.push_str("\n\n"); output.push_str(&details); } + if page.path == "query" { + output.push_str("\n\nRanked view:\n compass query \"where are request errors formatted?\" --view ranked\n --view ranked --format json emits compass.query.ranked-view/4.\n --text-budget bounds the whole shortlist, including refinement guidance.\n Ranked view does not accept discovery cursors, result envelopes, evidence,\n or agent-json. Relevance and structural evidence are separate labels.\n --include-context appends bounded one-hop relationships after the shortlist."); + } finish(output, style) } diff --git a/crates/compass-cli/src/lib.rs b/crates/compass-cli/src/lib.rs index 23d4b1074..12e7d5166 100644 --- a/crates/compass-cli/src/lib.rs +++ b/crates/compass-cli/src/lib.rs @@ -5738,10 +5738,33 @@ pub(crate) fn command_natural_query(frontend: Frontend, args: &[String]) -> Outc } .to_owned(); let mut discovery_result_envelope = false; + let mut ranked_view = false; + let mut ranked_context = false; let mut seen_discovery_options = HashSet::new(); let mut index = 1; while index < args.len() { match args[index].as_str() { + "--include-context" => { + ranked_context = true; + discovery_requested = true; + index += 1; + } + "--view" => { + if args.get(index + 1).map(String::as_str) != Some("ranked") { + return Outcome::failure("error: --view requires ranked".to_owned()); + } + ranked_view = true; + discovery_requested = true; + index += 2; + } + value if value.starts_with("--view=") => { + if value != "--view=ranked" { + return Outcome::failure("error: --view requires ranked".to_owned()); + } + ranked_view = true; + discovery_requested = true; + index += 1; + } "--traverse" => { legacy_requested = true; index += 1; @@ -5950,6 +5973,19 @@ pub(crate) fn command_natural_query(frontend: Frontend, args: &[String]) -> Outc .to_owned(), ); } + if ranked_context && !ranked_view { + return Outcome::failure("error: --include-context requires --view ranked".to_owned()); + } + if ranked_view + && (discovery_cursor.is_some() + || discovery_result_envelope + || discovery_format == "agent-json" + || discovery_evidence) + { + return Outcome::failure( + "error: --view ranked supports text or json; omit --cursor, --result-envelope, --evidence, and --format agent-json".to_owned(), + ); + } if !legacy_requested { if matches!(discovery_format.as_str(), "json" | "agent-json") && (discovery_text_pagination_requested || discovery_evidence) @@ -5974,11 +6010,15 @@ pub(crate) fn command_natural_query(frontend: Frontend, args: &[String]) -> Outc let outcome = command_discovery_query( &selection, request, - &discovery_format, - discovery_text_budget, - discovery_cursor.as_deref(), - discovery_result_envelope, - discovery_evidence, + DiscoveryPresentation { + format: &discovery_format, + text_budget: discovery_text_budget, + cursor: discovery_cursor.as_deref(), + result_envelope: discovery_result_envelope, + include_evidence: discovery_evidence, + ranked_view, + include_context: ranked_context, + }, ); if outcome.code == 0 { touch_selected_query_stamp(&selection); @@ -6108,20 +6148,58 @@ fn apply_discovery_limit( Ok(()) } -fn command_discovery_query( - selection: &GraphSelection, - request: DiscoveryQueryRequest, - format: &str, +struct DiscoveryPresentation<'a> { + format: &'a str, text_budget: usize, - cursor: Option<&str>, + cursor: Option<&'a str>, result_envelope: bool, include_evidence: bool, + ranked_view: bool, + include_context: bool, +} + +fn command_discovery_query( + selection: &GraphSelection, + request: DiscoveryQueryRequest, + presentation: DiscoveryPresentation<'_>, ) -> Outcome { + let DiscoveryPresentation { + format, + text_budget, + cursor, + result_envelope, + include_evidence, + ranked_view, + include_context, + } = presentation; let include_heuristic = request.include_heuristic; - let execution = match discovery_query(selection, request) { + let execution = match discovery_query(selection, request, ranked_view, include_context) { Ok(response) => response, Err(error) => return Outcome::failure(format!("error: {error}")), }; + if let Some(ranked) = &execution.ranked { + let view = match compass_output::build_ranked_query_view_with_context( + ranked, + &execution.graph_identity, + &execution.graph_digest, + include_context, + ) { + Ok(view) => view, + Err(error) => return Outcome::failure(format!("error: {error}")), + }; + let rendered = if format == "json" { + serde_json::to_string(&view).map_err(compass_output::OutputError::from) + } else { + let remaining_bytes = text_budget + .saturating_mul(4) + .saturating_sub(freshness::pending_warning_bytes()); + compass_output::render_ranked_query_text(&view, remaining_bytes / 4) + }; + return match rendered { + Ok(output) => Outcome::success(output), + Err(error) => Outcome::failure(format!("error: {error}")), + }; + } if format == "json" { let output = if result_envelope { let envelope = match compass_query::discovery_result_envelope(execution.response) { @@ -6208,6 +6286,7 @@ fn command_discovery_query( struct DiscoveryExecution { response: DiscoveryQueryResponse, + ranked: Option, graph_identity: String, graph_digest: String, } @@ -6215,6 +6294,8 @@ struct DiscoveryExecution { fn discovery_query( selection: &GraphSelection, request: DiscoveryQueryRequest, + ranked_view: bool, + include_context: bool, ) -> Result { match selection { GraphSelection::File(path) => { @@ -6231,11 +6312,22 @@ fn discovery_query( let graph_digest = engine.graph_identity().to_owned(); let engine = engine .with_deadline(Instant::now() + Duration::from_millis(request.limits.timeout_ms)); - let response = engine - .discover(request) - .map_err(|error| error.to_string())?; + let (response, ranked) = if ranked_view { + let ranked = engine + .discover_ranked_with_context(request, include_context) + .map_err(|error| error.to_string())?; + (ranked.discovery.clone(), Some(ranked)) + } else { + ( + engine + .discover(request) + .map_err(|error| error.to_string())?, + None, + ) + }; Ok(DiscoveryExecution { response, + ranked, graph_identity, graph_digest, }) @@ -6266,11 +6358,22 @@ fn discovery_query( let graph_digest = engine.graph_identity().to_owned(); let engine = engine .with_deadline(Instant::now() + Duration::from_millis(request.limits.timeout_ms)); - let response = engine - .discover(request) - .map_err(|error| error.to_string())?; + let (response, ranked) = if ranked_view { + let ranked = engine + .discover_ranked_with_context(request, include_context) + .map_err(|error| error.to_string())?; + (ranked.discovery.clone(), Some(ranked)) + } else { + ( + engine + .discover(request) + .map_err(|error| error.to_string())?, + None, + ) + }; Ok(DiscoveryExecution { response, + ranked, graph_identity, graph_digest, }) @@ -7398,7 +7501,7 @@ fn query_help(frontend: Frontend) -> String { "default: 128; hard maximum: 1000", ); format!( - "{help}\n --result-envelope Wrap JSON with a query-owned semantic digest" + "{help}\n --view ranked Opt-in top-five implementation shortlist (text or ranked-view/4 JSON; no discovery cursor)\n --include-context Append bounded one-hop witnessed context (ranked only)\n --result-envelope Wrap JSON with a query-owned semantic digest" ) } diff --git a/crates/compass-cli/tests/code_query_cli.rs b/crates/compass-cli/tests/code_query_cli.rs index f3e7ab4df..86fc659b6 100644 --- a/crates/compass-cli/tests/code_query_cli.rs +++ b/crates/compass-cli/tests/code_query_cli.rs @@ -11,6 +11,419 @@ use compass_output::{AgentOperation, AgentQueryView}; use compass_store::{STORE_FILE_NAME, STORE_REF_FILE_NAME, SqliteStore}; use serde_json::Value; +#[test] +fn compound_words_prefer_owned_identifiers_over_scattered_mentions() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let root = directory.path().join("source"); + std::fs::create_dir(&root)?; + let mut source = "def partition(fields):\n result = {}\n for field in fields:\n if not field.load_only and not field.dump_only:\n result[field.name] = field\n return result\n\ndef parameter_fields(fields, load_only, dump_only):\n return fields\n\ndef prose_fields(fields):\n \"\"\"Prepare excluded field maps with load-only and dump-only options.\"\"\"\n return fields\n\ndef widget_options(widget):\n return widget.load_only, widget.dump_only\n\n".to_owned(); + for index in 0..7 { + source.push_str(&format!( + "def load_maps_{index}(fields):\n \"\"\"Load maps; excluded fields are omitted and only valid fields returned.\"\"\"\n return fields.load()\n\n" + )); + } + source.push_str("def header_fields(fields, "); + for index in 0..48 { + source.push_str(&format!("padding_{index}=None, ")); + } + source.push_str("load_only=False, dump_only=False):\n return fields\n\n"); + std::fs::write(root.join("sample.py"), source)?; + let mut candidates = None; + for storage in ["json", "sqlite"] { + let output = directory.path().join(storage); + let built = run( + Frontend::Compass, + [ + OsString::from("extract"), + root.as_os_str().to_owned(), + OsString::from("--code-only"), + OsString::from("--no-viz"), + OsString::from("--store"), + OsString::from(storage), + OsString::from("--max-workers"), + OsString::from("1"), + OsString::from("--out"), + output.as_os_str().to_owned(), + ], + ); + assert_eq!(built.code, 0, "{}", built.stderr); + let graph = output.join("compass-out/graph.json"); + let query = |question: &str| { + run( + Frontend::Compass, + [ + OsString::from("query"), + OsString::from(question), + OsString::from("--graph"), + graph.as_os_str().to_owned(), + OsString::from("--view"), + OsString::from("ranked"), + OsString::from("--format"), + OsString::from("json"), + ], + ) + }; + let answer = + query("Where are excluded fields and load-only or dump-only field maps prepared?"); + assert_eq!(answer.code, 0, "{}", answer.stderr); + let answer: Value = serde_json::from_str(&answer.stdout)?; + assert!( + answer["candidates"][0]["name"] + .as_str() + .is_some_and(|name| name.ends_with("partition")), + "{answer}" + ); + assert_eq!(answer["candidates"][0]["relevance"], "low"); + assert!( + answer["candidates"][0]["reasons"] + .to_string() + .contains("Compound identifiers"), + "{answer}" + ); + if let Some(expected) = &candidates { + assert_eq!(&answer["candidates"], expected); + } else { + candidates = Some(answer["candidates"].clone()); + } + for candidate in answer["candidates"] + .as_array() + .ok_or("missing candidates")? + { + if candidate["name"].as_str().is_some_and(|name| { + name.contains("parameter_fields") + || name.contains("prose_fields") + || name.contains("widget_options") + || name.contains("header_fields") + }) { + assert!( + !candidate["reasons"] + .to_string() + .contains("Compound identifiers") + ); + } + } + for question in [ + "Where are load-only field maps prepared without validation?", + "Where are load-only field maps prepared before validation?", + ] { + let answer = query(question); + assert_eq!(answer.code, 0, "{}", answer.stderr); + let answer: Value = serde_json::from_str(&answer.stdout)?; + assert!( + !answer["candidates"] + .as_array() + .ok_or("missing candidates")? + .is_empty() + ); + assert!( + answer["candidates"] + .as_array() + .ok_or("missing candidates")? + .iter() + .all(|candidate| !candidate["reasons"] + .to_string() + .contains("Compound identifiers")) + ); + } + } + Ok(()) +} + +#[test] +fn unindexed_snapshot_vocabulary_returns_tentative_native_source_candidates() +-> Result<(), Box> { + let directory = tempfile::tempdir()?; + let root = directory.path().join("source"); + std::fs::create_dir(&root)?; + let mut source = "def copy_items(cache):\n \"\"\"Copies unexpired items into a new map.\"\"\"\n return {key: item for key, item in cache.items() if not item.expired}\n\n".to_owned(); + for index in 0..7 { + source.push_str(&format!( + "def new_cache_{index}(cache, only, containing):\n return cache\n\n" + )); + } + std::fs::write(root.join("sample.py"), source)?; + let output = directory.path().join("artifacts"); + let built = run( + Frontend::Compass, + [ + OsString::from("extract"), + root.into_os_string(), + OsString::from("--code-only"), + OsString::from("--no-viz"), + OsString::from("--store"), + OsString::from("json"), + OsString::from("--max-workers"), + OsString::from("1"), + OsString::from("--out"), + output.as_os_str().to_owned(), + ], + ); + assert_eq!(built.code, 0, "{}", built.stderr); + let answer = run( + Frontend::Compass, + [ + OsString::from("query"), + OsString::from("Where is a snapshot made containing only live cache entries?"), + OsString::from("--graph"), + output.join("compass-out/graph.json").into_os_string(), + OsString::from("--view"), + OsString::from("ranked"), + OsString::from("--format"), + OsString::from("json"), + ], + ); + assert_eq!(answer.code, 0, "{}", answer.stderr); + let answer: Value = serde_json::from_str(&answer.stdout)?; + assert!( + answer["candidates"][0]["name"] + .as_str() + .is_some_and(|name| name.ends_with("copy_items")), + "{answer}" + ); + assert_eq!(answer["candidates"][0]["relevance"], "low"); + assert!( + answer["candidates"][0]["reasons"] + .to_string() + .contains("Vocabulary fallback") + ); + Ok(()) +} + +#[test] +fn native_body_operation_preference_does_not_borrow_parameter_declarations() +-> Result<(), Box> { + let directory = tempfile::tempdir()?; + let root = directory.path().join("source"); + std::fs::create_dir(&root)?; + let mut source = "def opaque(items):\n return codec.encode(items)\n\n".to_owned(); + for index in 0..7 { + source.push_str(&format!( + "def new_map_{index}(items, encode):\n return items\n\n" + )); + } + std::fs::write(root.join("sample.py"), source)?; + let output = directory.path().join("artifacts"); + let built = run( + Frontend::Compass, + [ + OsString::from("extract"), + root.into_os_string(), + OsString::from("--code-only"), + OsString::from("--no-viz"), + OsString::from("--store"), + OsString::from("json"), + OsString::from("--max-workers"), + OsString::from("1"), + OsString::from("--out"), + output.as_os_str().to_owned(), + ], + ); + assert_eq!(built.code, 0, "{}", built.stderr); + let answer = run( + Frontend::Compass, + [ + OsString::from("query"), + OsString::from("Where are items encoded into a map?"), + OsString::from("--graph"), + output.join("compass-out/graph.json").into_os_string(), + OsString::from("--view"), + OsString::from("ranked"), + OsString::from("--format"), + OsString::from("json"), + ], + ); + assert_eq!(answer.code, 0, "{}", answer.stderr); + let answer: Value = serde_json::from_str(&answer.stdout)?; + assert!( + answer["candidates"][0]["name"] + .as_str() + .is_some_and(|name| name.ends_with("opaque")), + "{answer}" + ); + assert_eq!(answer["candidates"][0]["relevance"], "low"); + for candidate in answer["candidates"] + .as_array() + .ok_or("missing candidates")? + { + if candidate["name"] + .as_str() + .is_some_and(|name| name.contains("new_map_")) + { + assert!( + !candidate["reasons"] + .to_string() + .contains("inspection preference") + ); + } + } + Ok(()) +} + +#[test] +fn broad_qualifications_return_candidates_without_claiming_the_condition() +-> Result<(), Box> { + let directory = tempfile::tempdir()?; + let path = support::write_typed_graph(directory.path())?; + let mut graph = GraphDocument::load(&path)?; + let target = graph + .nodes + .iter_mut() + .find(|node| node.name == "Target") + .ok_or("missing fixture target")?; + target.name = "format_protocol_errors".into(); + target.qualified_name = "Fixture.format_protocol_errors".into(); + std::fs::write(&path, serde_json::to_vec(&graph)?)?; + for question in [ + "where are protocol errors elegantly formatted", + "where are protocol errors formatted without decoration", + "where are redis protocol errors formatted", + "where are protocol errors formatted before decoration", + "where are protocol errors formatted after decoration", + ] { + let output = run( + Frontend::Compass, + [ + OsString::from("query"), + OsString::from(question), + OsString::from("--graph"), + path.as_os_str().to_owned(), + OsString::from("--view"), + OsString::from("ranked"), + OsString::from("--format"), + OsString::from("json"), + ], + ); + assert_eq!(output.code, 0, "{}", output.stderr); + let value: Value = serde_json::from_str(&output.stdout)?; + assert_eq!( + value["candidates"][0]["name"], + "Fixture.format_protocol_errors" + ); + assert_eq!(value["candidates"][0]["relevance"], "low"); + assert!( + value["candidates"][0]["reasons"] + .to_string() + .contains("not established") + ); + if question.contains("without") { + assert!( + value["candidates"][0]["reasons"] + .to_string() + .contains("Negative qualification") + ); + } + let temporal = question.contains("before") || question.contains("after"); + if temporal { + assert_eq!(value["question"], question); + assert!( + value["candidates"][0]["reasons"] + .to_string() + .contains("Temporal qualification") + ); + let text = run( + Frontend::Compass, + [ + OsString::from("query"), + OsString::from(question), + OsString::from("--graph"), + path.as_os_str().to_owned(), + OsString::from("--view"), + OsString::from("ranked"), + OsString::from("--text-budget"), + OsString::from("800"), + ], + ); + assert_eq!(text.code, 0, "{}", text.stderr); + assert!(text.stdout.contains("Temporal qualification")); + assert!(text.stdout.contains("not established")); + assert!(text.stdout.len() <= 3200); + } + let ordinary = run( + Frontend::Compass, + [ + OsString::from("query"), + OsString::from(question), + OsString::from("--graph"), + path.as_os_str().to_owned(), + OsString::from("--format"), + OsString::from("json"), + ], + ); + assert_eq!(ordinary.code, 0, "{}", ordinary.stderr); + let ordinary: Value = serde_json::from_str(&ordinary.stdout)?; + if temporal { + assert!( + ordinary["diagnostics"] + .to_string() + .contains("Temporal qualification") + ); + } + assert!( + ordinary["diagnostics"] + .to_string() + .contains("inspection candidates") + ); + assert!( + ordinary["diagnostics"] + .to_string() + .contains("not established") + ); + } + Ok(()) +} + +#[test] +fn ranked_view_is_opt_in_and_preserves_default_raw_discovery() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let graph = support::write_typed_graph(directory.path())?; + let execute = |extra: &[&str]| { + let mut args = vec![ + OsString::from("query"), + OsString::from("where is Target defined?"), + OsString::from("--graph"), + graph.as_os_str().to_owned(), + ]; + args.extend(extra.iter().map(OsString::from)); + run(Frontend::Compass, args) + }; + let raw = execute(&["--format", "json"]); + assert_eq!(raw.code, 0, "{}", raw.stderr); + let raw: Value = serde_json::from_str(&raw.stdout)?; + assert_eq!(raw["schema"], "compass.query.discovery/1"); + let ranked = execute(&["--view", "ranked", "--format", "json"]); + assert_eq!(ranked.code, 0, "{}", ranked.stderr); + let ranked: Value = serde_json::from_str(&ranked.stdout)?; + assert_eq!(ranked["schema"], "compass.query.ranked-view/4"); + assert_eq!( + ranked["candidates"][0]["name"], "Fixture.Target", + "{ranked}" + ); + assert!(ranked["candidates"][0]["reasons"].is_array()); + assert!(ranked.get("context").is_none()); + let context = execute(&["--view", "ranked", "--include-context", "--format", "json"]); + assert_eq!(context.code, 0, "{}", context.stderr); + let context: Value = serde_json::from_str(&context.stdout)?; + assert_eq!(context["candidates"], ranked["candidates"]); + assert!(context["context"]["edges"].is_array()); + let text = execute(&["--view=ranked", "--text-budget", "800"]); + assert_eq!(text.code, 0, "{}", text.stderr); + assert!(text.stdout.contains("Fixture.Target")); + assert!(text.stdout.contains("relevance=")); + assert!(text.stdout.len() <= 800 * 4); + assert!(text.stdout.len() + text.stderr.len() + 2 <= 800 * 4); + for extra in [ + vec!["--view", "ranked", "--cursor", "old"], + vec!["--view", "ranked", "--format", "agent-json"], + vec!["--view", "ranked", "--traverse"], + vec!["--view", "unknown"], + vec!["--include-context"], + vec!["--view", "ranked", "--text-budget", "1"], + ] { + assert_ne!(execute(&extra).code, 0, "{extra:?}"); + } + Ok(()) +} + #[test] fn ask_preserves_typed_operands_in_agent_and_text_answers() -> Result<(), Box> { let directory = tempfile::tempdir()?; @@ -1237,6 +1650,14 @@ fn natural_query_renders_typed_source_locations() -> Result<(), Box> ); assert_eq!(outcome.code, 0, "{}", outcome.stderr); + assert!( + outcome + .stdout + .lines() + .any(|line| { line.starts_with("SEED Fixture.Target [exact] @ src/lib.rs:1:") }), + "{}", + outcome.stdout + ); assert!( outcome .stdout @@ -3033,3 +3454,417 @@ fn budgeted_machine_answers_fail_intact_and_saved_records_are_readable() assert!(!root.path().join(".compass/project.json").exists()); Ok(()) } + +#[test] +fn broad_documentation_question_after_update_returns_source_owned_candidates() +-> Result<(), Box> { + for (file, source, declaration_line) in [ + ( + "sample.rs", + "struct Store;\nimpl Store {\n /// Protocol diagnostics parsed here.\n fn opaque(&self) {}\n}\n", + 4, + ), + ( + "sample.go", + "package sample\ntype Store struct{}\n// Protocol diagnostics parsed here.\nfunc (s Store) Opaque() {}\n", + 4, + ), + ( + "sample.ts", + "export class Store {\n /** Protocol diagnostics parsed here. */\n opaque() {}\n}\n", + 3, + ), + ] { + for storage in ["json", "sqlite"] { + let directory = tempfile::tempdir()?; + std::fs::write(directory.path().join(file), source)?; + let update = support::compass_command() + .current_dir(directory.path()) + .args([ + "update", + ".", + "--store", + storage, + "--code-only", + "--no-cluster", + "--no-viz", + ]) + .output()?; + assert!( + update.status.success(), + "{file}/{storage}: {}", + String::from_utf8_lossy(&update.stderr) + ); + let graph_path = directory.path().join("compass-out/graph.json"); + let graph = GraphDocument::load(&graph_path)?; + let question = "where are protocol diagnostics parsed"; + for _ in 0..2 { + let query = support::compass_command() + .current_dir(directory.path()) + .args(["query", question, "--view", "ranked", "--format", "json"]) + .output()?; + assert!( + query.status.success(), + "{file}/{storage}: {}", + String::from_utf8_lossy(&query.stderr) + ); + let view = compass_output::RankedQueryView::from_json(&query.stdout)?; + let candidate = view + .candidates + .first() + .ok_or("no broad candidate after update")?; + assert_eq!(candidate.kind, NodeKind::Method); + let anchor = candidate.source.as_ref().ok_or("missing owner citation")?; + assert_eq!(anchor.file, file); + assert_eq!(anchor.start_line, declaration_line); + assert!( + candidate + .matched_fields + .iter() + .any(|field| field == "owned_document") + ); + let excerpt = candidate + .supporting_sources + .first() + .ok_or("missing documentation citation")?; + assert_eq!(excerpt.file, file); + assert_eq!(excerpt.start_line, declaration_line - 1); + assert!(excerpt.end_byte <= anchor.start_byte); + let quote = candidate + .excerpts + .first() + .ok_or("missing raw documentation quote")?; + assert_eq!( + quote.kind, + compass_query::RankedExcerptKind::AstDocumentation + ); + assert_eq!( + source.get(quote.source.start_byte as usize..quote.source.end_byte as usize), + Some(quote.text.as_str()) + ); + assert!(quote.text.contains("Protocol diagnostics")); + let owner = graph + .nodes + .iter() + .find(|node| node.id == candidate.id) + .ok_or("query synthesized an owner")?; + let edge = graph + .links + .iter() + .find(|edge| { + edge.target == owner.id + && edge.occurrence_rule.as_ref().map(|rule| rule.as_str()) + == Some(compass_model::provenance::LEADING_DOCUMENTATION_RULE) + }) + .ok_or("no source ownership relationship")?; + let documentation = graph + .nodes + .iter() + .find(|node| node.id == edge.source) + .ok_or("missing published documentation")?; + assert!(compass_model::provenance::is_proven_leading_documentation( + documentation, + owner, + edge + )); + } + } + } + Ok(()) +} + +#[test] +fn broad_code_reference_and_plural_acronym_questions_work_after_update() +-> Result<(), Box> { + for storage in ["json", "sqlite"] { + let directory = tempfile::tempdir()?; + std::fs::write( + directory.path().join("sample.ts"), + "export class Queue {\n #abortListeners = new Map void>();\n process() { this.#abortListeners.clear(); }\n convert_ttl(ttl: number) { return ttl * 1000; }\n}\n", + )?; + let update = support::compass_command() + .current_dir(directory.path()) + .args([ + "update", + ".", + "--store", + storage, + "--code-only", + "--no-cluster", + "--no-viz", + ]) + .output()?; + assert!( + update.status.success(), + "{}", + String::from_utf8_lossy(&update.stderr) + ); + for (question, expected, field) in [ + ( + "where are abort listeners removed", + ".process", + "code_reference", + ), + ("where are TTLs converted", ".convert_ttl", "name"), + ] { + for _ in 0..2 { + let output = support::compass_command() + .current_dir(directory.path()) + .args(["query", question, "--view", "ranked", "--format", "json"]) + .output()?; + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let view = compass_output::RankedQueryView::from_json(&output.stdout)?; + let candidate = view + .candidates + .iter() + .find(|candidate| candidate.name.ends_with(expected)) + .ok_or_else(|| { + format!( + "{storage}: missing {expected}: {}", + String::from_utf8_lossy(&output.stdout) + ) + })?; + assert!( + candidate + .matched_fields + .iter() + .any(|matched| matched == field) + ); + assert_eq!( + candidate.source.as_ref().map(|source| source.start_line), + Some(if field == "code_reference" { 3 } else { 4 }) + ); + if field == "code_reference" { + assert_eq!(candidate.relevance, compass_query::RelevanceBand::Low); + assert!( + candidate + .supporting_sources + .iter() + .any(|source| source.start_line == 3) + ); + assert!( + candidate + .reasons + .iter() + .any(|reason| reason.contains("not confirmation")) + ); + } + } + } + } + Ok(()) +} + +#[test] +fn broad_local_identifier_ownership_works_after_update() -> Result<(), Box> { + for storage in ["json", "sqlite"] { + let directory = tempfile::tempdir()?; + let mut source = "export function opaque() {\n const annotations = {};\n}\n".to_owned(); + for name in [ + "iris", "maple", "cedar", "birch", "pine", "oak", "elm", "beech", "willow", "ash", + "yew", "palm", + ] { + source.push_str(&format!("export function {name}() {{}}\n")); + } + std::fs::write(directory.path().join("sample.ts"), source)?; + let update = support::compass_command() + .current_dir(directory.path()) + .args([ + "update", + ".", + "--store", + storage, + "--code-only", + "--no-cluster", + "--no-viz", + ]) + .output()?; + assert!( + update.status.success(), + "{}", + String::from_utf8_lossy(&update.stderr) + ); + for question in [ + "Which function owns annotations?", + "Where are annotations declared?", + ] { + let mut first = None; + for _ in 0..2 { + let output = support::compass_command() + .current_dir(directory.path()) + .args(["query", question, "--view", "ranked", "--format", "json"]) + .output()?; + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let view = compass_output::RankedQueryView::from_json(&output.stdout)?; + assert!( + view.candidates + .iter() + .filter(|candidate| candidate.kind.is_callable()) + .all(|candidate| candidate.name.ends_with(".opaque")), + "Generic functions displaced topic evidence: {}", + String::from_utf8_lossy(&output.stdout) + ); + let candidate = view + .candidates + .iter() + .find(|candidate| candidate.name.ends_with(".opaque")) + .ok_or_else(|| { + format!( + "{storage}: owner omitted: {}", + String::from_utf8_lossy(&output.stdout) + ) + })?; + assert!( + candidate + .matched_fields + .iter() + .any(|field| field == "owned_identifier") + ); + assert_eq!(candidate.relevance, compass_query::RelevanceBand::Low); + assert_eq!( + candidate.source.as_ref().map(|source| source.start_line), + Some(1) + ); + assert!( + candidate + .supporting_sources + .iter() + .any(|source| source.start_line == 2) + ); + assert!( + candidate + .reasons + .iter() + .any(|reason| reason.contains("not confirmation")) + ); + if let Some(previous) = &first { + assert_eq!(previous, &output.stdout); + } else { + first = Some(output.stdout); + } + } + } + } + Ok(()) +} + +#[test] +fn broad_body_comments_work_through_native_update_and_query() -> Result<(), Box> { + for storage in ["json", "sqlite"] { + let directory = tempfile::tempdir()?; + std::fs::write( + directory.path().join("sample.py"), + "def opaque(value):\n value = value\n # Handle payload normalization here.\n return value\ndef decoy(value):\n return value\n", + )?; + let update = support::compass_command() + .current_dir(directory.path()) + .args([ + "update", + ".", + "--store", + storage, + "--code-only", + "--no-cluster", + "--no-viz", + ]) + .output()?; + assert!( + update.status.success(), + "{}", + String::from_utf8_lossy(&update.stderr) + ); + let raw = support::compass_command() + .current_dir(directory.path()) + .args(["query", "Where is payload handled?", "--format", "json"]) + .output()?; + assert!( + raw.status.success(), + "{}", + String::from_utf8_lossy(&raw.stderr) + ); + let value: Value = serde_json::from_slice(&raw.stdout)?; + assert_eq!(value["schema"], "compass.query.discovery/1"); + assert!( + value["seeds"] + .as_array() + .is_some_and(|seeds| !seeds.is_empty()), + "{}", + String::from_utf8_lossy(&raw.stdout) + ); + assert!( + value["seeds"].to_string().contains("body_comment"), + "{}", + String::from_utf8_lossy(&raw.stdout) + ); + let mut previous = None; + for _ in 0..2 { + let output = support::compass_command() + .current_dir(directory.path()) + .args([ + "query", + "Where is payload handled?", + "--view", + "ranked", + "--format", + "json", + ]) + .output()?; + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let view = compass_output::RankedQueryView::from_json(&output.stdout)?; + let candidate = view + .candidates + .first() + .ok_or("missing body-only candidate")?; + assert!( + candidate.name.ends_with(".opaque"), + "{}", + String::from_utf8_lossy(&output.stdout) + ); + assert_eq!(candidate.relevance, compass_query::RelevanceBand::Low); + let quote = candidate + .excerpts + .first() + .ok_or("missing callable comment quote")?; + assert_eq!( + quote.kind, + compass_query::RankedExcerptKind::CallableComment + ); + let source = std::fs::read_to_string(directory.path().join("sample.py"))?; + assert_eq!( + source.get(quote.source.start_byte as usize..quote.source.end_byte as usize), + Some(quote.text.as_str()) + ); + assert!(quote.text.contains("payload normalization")); + assert!( + candidate + .matched_fields + .iter() + .any(|field| field == "body_comment") + ); + assert!( + candidate + .supporting_sources + .iter() + .any(|source| source.file == "sample.py" && source.start_line == 3) + ); + if let Some(previous) = &previous { + assert_eq!(previous, &output.stdout); + } else { + previous = Some(output.stdout); + } + } + } + Ok(()) +} diff --git a/crates/compass-cli/tests/store_term_capabilities.rs b/crates/compass-cli/tests/store_term_capabilities.rs new file mode 100644 index 000000000..90c7b94fa --- /dev/null +++ b/crates/compass-cli/tests/store_term_capabilities.rs @@ -0,0 +1,169 @@ +use std::error::Error; +use std::process::Command; + +use compass_files::BuildGuard; +use compass_model::code_graph::GraphDocument; +use compass_store::STORE_REF_FILE_NAME; +use serde_json::Value; + +#[test] +fn source_terms_cannot_collide_with_store_capability_records() -> Result<(), Box> { + let root = tempfile::tempdir()?; + std::fs::write( + root.path().join("lib.rs"), + r#"pub const EXPLANATION: &str = "__compass_cap_identifier_subwords_v1__"; +pub mod __compass_cap_operation_role_terms_v1__ { + pub struct Builder; +} +pub struct __compass_cap_declaration_terms_v1__; +pub fn __compass_cap_relationship_terms_v1__() {} +pub fn __compass_cap_relationship_search_terms_v1__() {} +pub fn sha256_hex() {} +pub fn caller() { + __compass_cap_relationship_terms_v1__(); + __compass_cap_relationship_search_terms_v1__(); +} +"#, + )?; + let build = Command::new(env!("CARGO_BIN_EXE_compass")) + .args([ + "update", + ".", + "--code-only", + "--no-cluster", + "--no-viz", + "--store", + "sqlite", + "--max-workers", + "1", + ]) + .current_dir(root.path()) + .env_remove("COMPASS_OUT") + .output()?; + assert!( + build.status.success(), + "{}", + String::from_utf8_lossy(&build.stderr) + ); + let graph_path = BuildGuard::resolve_artifact(&root.path().join("compass-out"), "graph.json")?; + assert!( + graph_path + .parent() + .ok_or("published graph parent missing")? + .join(STORE_REF_FILE_NAME) + .is_file() + ); + let graph = GraphDocument::load(&graph_path)?; + let target = graph + .nodes + .iter() + .find(|node| node.name == "EXPLANATION") + .ok_or("source constant missing from published graph")?; + let answer = Command::new(env!("CARGO_BIN_EXE_compass")) + .arg("query") + .arg(&target.id) + .arg("--graph") + .arg(&graph_path) + .args(["--view", "ranked", "--format", "json"]) + .current_dir(root.path()) + .env_remove("COMPASS_OUT") + .output()?; + assert!( + answer.status.success(), + "{}", + String::from_utf8_lossy(&answer.stderr) + ); + let value: Value = serde_json::from_slice(&answer.stdout)?; + assert_eq!(value["schema"], "compass.query.ranked-view/4"); + assert_eq!(value["candidates"].as_array().map(Vec::len), Some(1)); + assert_eq!(value["candidates"][0]["id"], target.id); + Ok(()) +} + +#[test] +fn ranked_cli_quotes_published_code_for_both_storage_modes() -> Result<(), Box> { + for storage in ["json", "sqlite"] { + let root = tempfile::tempdir()?; + let source = "def opaque(items):\n return normalize_payload(items)\n"; + std::fs::write(root.path().join("sample.py"), source)?; + let build = Command::new(env!("CARGO_BIN_EXE_compass")) + .args([ + "update", + ".", + "--code-only", + "--no-cluster", + "--no-viz", + "--store", + storage, + "--max-workers", + "1", + ]) + .current_dir(root.path()) + .env_remove("COMPASS_OUT") + .output()?; + assert!( + build.status.success(), + "{}", + String::from_utf8_lossy(&build.stderr) + ); + let graph = BuildGuard::resolve_artifact(&root.path().join("compass-out"), "graph.json")?; + let execute = |extra: &[&str]| { + Command::new(env!("CARGO_BIN_EXE_compass")) + .args(["query", "where is payload normalized", "--view", "ranked"]) + .arg("--graph") + .arg(&graph) + .args(extra) + .current_dir(root.path()) + .env_remove("COMPASS_OUT") + .output() + }; + let answer = execute(&["--format", "json"])?; + assert!( + answer.status.success(), + "{}", + String::from_utf8_lossy(&answer.stderr) + ); + let value: Value = serde_json::from_slice(&answer.stdout)?; + assert_eq!(value["schema"], "compass.query.ranked-view/4"); + let candidate = &value["candidates"][0]; + assert!( + candidate["name"] + .as_str() + .is_some_and(|name| name.contains("opaque")) + ); + let quotes = candidate["excerpts"] + .as_array() + .ok_or("missing code excerpts")?; + assert!(quotes.iter().any(|quote| { + quote["kind"] == "callable_code" + && quote["text"] + .as_str() + .is_some_and(|text| text.contains("normalize_payload(items)")) + })); + for quote in quotes { + let anchor = "e["source"]; + let start = anchor["startByte"].as_u64().ok_or("start byte")? as usize; + let end = anchor["endByte"].as_u64().ok_or("end byte")? as usize; + assert_eq!(source.get(start..end), quote["text"].as_str()); + } + std::fs::write( + root.path().join("sample.py"), + "# source changed after graph publication\n", + )?; + let pinned = execute(&["--format", "json"])?; + assert!(pinned.status.success()); + let pinned: Value = serde_json::from_slice(&pinned.stdout)?; + assert_eq!(pinned["candidates"], value["candidates"]); + let text = execute(&["--text-budget", "1600"])?; + assert!( + text.status.success(), + "{}", + String::from_utf8_lossy(&text.stderr) + ); + let rendered = String::from_utf8(text.stdout)?; + assert!(rendered.contains("Code @")); + assert!(rendered.contains("normalize_payload(items)")); + assert!(rendered.len() < 1600 * 4); + } + Ok(()) +} diff --git a/crates/compass-core/src/build_state.rs b/crates/compass-core/src/build_state.rs index 837c1c2dd..395742f67 100644 --- a/crates/compass-core/src/build_state.rs +++ b/crates/compass-core/src/build_state.rs @@ -22,6 +22,7 @@ fn current_build_fingerprint() -> String { compass_languages::EXTRACTION_SEMANTICS_VERSION, compass_languages::FRAMEWORK_PACK_SEMANTICS_VERSION, compass_files::AST_CACHE_VERSION, + compass_model::source_query::SOURCE_QUERY_PROFILE, ] { digest.update(component.as_bytes()); digest.update([0]); diff --git a/crates/compass-core/src/lib.rs b/crates/compass-core/src/lib.rs index 4d0609091..4aa696c2e 100644 --- a/crates/compass-core/src/lib.rs +++ b/crates/compass-core/src/lib.rs @@ -12,6 +12,7 @@ mod pipeline; mod program; mod raw_guard; mod review; +mod source_query; mod task_context; mod watch; mod watch_scheduler; diff --git a/crates/compass-core/src/pipeline.rs b/crates/compass-core/src/pipeline.rs index 6f38fbe9d..0b8bbe7de 100644 --- a/crates/compass-core/src/pipeline.rs +++ b/crates/compass-core/src/pipeline.rs @@ -99,13 +99,14 @@ const PARALLEL_AST_FACT_DIGEST_MIN_FILES: usize = 32; const SHARED_STORE_GC_MANIFEST_THRESHOLD: usize = 8; const STORE_SNAPSHOT_EXCLUSIONS: [&str; 3] = [STORE_FILE_NAME, "store.sqlite3-wal", "store.sqlite3-shm"]; -const ROOT_ARTIFACTS: [&str; 9] = [ +const ROOT_ARTIFACTS: [&str; 10] = [ "GRAPH_REPORT.md", "orientation.json", "graph-overview.json", "graph.html", "manifest.json", "program.json", + compass_model::source_query::SOURCE_QUERY_FILE, "graph.json", "community-quality.json", "community-hierarchy.json", @@ -993,6 +994,11 @@ impl Serialize for FactDigestDeclaration<'_, '_> { map.serialize_entry("directBasesComplete", &true)?; } map.serialize_entry("variadic", &declaration.variadic)?; + // Leading comments are outside declaration/body hashes. Retain their + // text and ranges so equal-length doc edits cannot reuse stale prose. + if let Some(documentation) = &declaration.documentation { + map.serialize_entry("documentation", documentation)?; + } let file_declaration = self.context.declaration_is_file(&declaration.id); if !file_declaration { if let Some(value) = &declaration.signature_hash { @@ -1429,7 +1435,16 @@ impl Serialize for FactDigestExtraction<'_> { if let Some(error) = &self.extraction.error { map.serialize_entry("error", error)?; } - let mut extensions = self.extraction.extensions.iter().collect::>(); + // Query-only source text must not change structural Fact IDs/digests. + // Its eventual auxiliary artifact has its own source/content identity. + let mut extensions = self + .extraction + .extensions + .iter() + .filter(|(key, _)| { + key.as_str() != compass_languages::source_query::EXTRACTION_EXTENSION + }) + .collect::>(); extensions.sort_unstable_by(|left, right| left.0.cmp(right.0)); for (key, value) in extensions { map.serialize_entry(key, &FactDigestValue(value))?; @@ -1732,6 +1747,143 @@ fn full_file_source_anchor( type FactNeutralDocument = (Option, V1GraphDocument, BTreeSet); +type FileContainerKey = (String, String, String, &'static str); + +struct FileContainerEnvelope { + declaration: SourceAnchor, + envelope: SourceAnchor, + extractor: String, +} + +const fn file_container_kind(kind: NodeKind) -> Option<&'static str> { + match kind { + NodeKind::Module => Some("module"), + NodeKind::Package => Some("package"), + NodeKind::Namespace => Some("namespace"), + _ => None, + } +} + +fn evidence_source_anchor(range: &compass_languages::EvidenceRange, root: &Path) -> SourceAnchor { + SourceAnchor { + file: relative_fact_path(Path::new(&range.source_file), root), + start_byte: range.start_byte, + end_byte: range.end_byte, + start_line: range.start_line, + start_column: range.start_column, + end_line: range.end_line, + end_column: range.end_column, + } +} + +/// A file-scoped language container can own the file envelope without being a +/// file node. Require its directly owned scope and the actual file-root parent; +/// coincident declaration offsets alone never establish that ownership. +fn file_container_envelopes( + extractions: &[Extraction], + root: &Path, + source_digests: &BTreeMap, +) -> BTreeMap> { + let mut envelopes = BTreeMap::new(); + for extraction in extractions { + let Some(batch) = &extraction.semantic_evidence else { + continue; + }; + if !batch.scopes.iter().any(|scope| { + source_digests.contains_key(&relative_fact_path( + Path::new(&scope.range.source_file), + root, + )) + }) { + continue; + } + let declarations = batch + .declarations + .iter() + .map(|declaration| (declaration.id.as_str(), declaration)) + .collect::>(); + let scopes = batch + .scopes + .iter() + .map(|scope| (scope.id.as_str(), scope)) + .collect::>(); + for scope in &batch.scopes { + let Some(declaration) = scope + .owner_declaration_id + .as_deref() + .and_then(|id| declarations.get(id)) + else { + continue; + }; + let kind = match declaration.kind.as_str() { + "module" => "module", + "package" => "package", + "namespace" => "namespace", + _ => continue, + }; + let envelope = evidence_source_anchor(&scope.range, root); + if scope.kind != kind + || scope.language != declaration.language + || scope.parent_scope_id != declaration.scope_id + || scope.range.source_file != declaration.range.source_file + || envelope.start_byte != 0 + || envelope.start_line != 1 + || envelope.start_column != 0 + || source_digests + .get(&envelope.file) + .is_none_or(|file| file.byte_size != envelope.end_byte) + { + continue; + } + if let Some(parent_id) = scope.parent_scope_id.as_deref() { + let Some(parent) = scopes.get(parent_id) else { + continue; + }; + let Some(owner) = parent + .owner_declaration_id + .as_deref() + .and_then(|id| declarations.get(id)) + else { + continue; + }; + if parent.parent_scope_id.is_some() + || !matches!(parent.kind.as_str(), "file" | "module") + || !matches!(owner.kind.as_str(), "file" | "module") + || owner.scope_id.is_some() + || parent.language != declaration.language + || parent.range != scope.range + { + continue; + } + } + let key = ( + envelope.file.clone(), + declaration.language.clone(), + declaration.qualified_name.clone(), + kind, + ); + let proof = FileContainerEnvelope { + declaration: evidence_source_anchor(&declaration.range, root), + envelope, + extractor: format!("compass.languages.{}.universal", declaration.language), + }; + envelopes + .entry(key) + .and_modify(|candidate| *candidate = None) + .or_insert(Some(proof)); + } + } + envelopes +} + +fn is_file_envelope(anchor: &SourceAnchor, source: &str, size: u64) -> bool { + anchor.file == source + && anchor.start_byte == 0 + && anchor.end_byte == size + && anchor.start_line == 1 + && anchor.start_column == 0 +} + fn sort_dedup_serialized(values: &mut Vec) { if values.len() < 2 { return; @@ -1845,6 +1997,7 @@ fn prepare_fact_neutral_document( canonicalize_fact_neutral_metadata(&mut evidence.coverage, &mut evidence.diagnostics); let anchors = extraction_file_anchors(extractions, root); + let containers = file_container_envelopes(extractions, root, source_digests); let previous_file_sizes = current .graph .files @@ -1870,6 +2023,15 @@ fn prepare_fact_neutral_document( if !files.contains_key(source.as_str()) { continue; } + // Refresh an existing file payload from the authoritative inventory. + // Preserve absent optional details rather than synthesizing a new shape. + if let Some(compass_model::code_graph::NodeDetails::File(details)) = node.details.as_mut() + && let Some(file) = files.get(source.as_str()) + { + details.content_digest.clone_from(&file.content_digest); + details.byte_size = file.byte_size; + details.generated = file.generated; + } // The canonical graph-v1 file inventory owns digest, size, and generated // state. Do not synthesize an optional file-node payload only on the // fact-neutral route: full publication intentionally preserves the @@ -1881,40 +2043,66 @@ fn prepare_fact_neutral_document( .cloned() .or_else(|| full_file_source_anchor(&root.join(&source), &source, max_source_bytes)) }); - if let Some(anchor) = refreshed_envelope.flatten() { - let source_was_file_envelope = node.source.as_ref().is_some_and(|previous| { - previous.start_byte == 0 - && previous_file_sizes - .get(&source) - .is_some_and(|size| previous.end_byte == *size) + let previous_size = previous_file_sizes.get(&source).copied(); + let container = file_container_kind(node.kind) + .zip(node.language.as_ref()) + .and_then(|(kind, language)| { + containers.get(&( + source.clone(), + language.clone(), + node.qualified_name.clone(), + kind, + )) }); - if node.kind == NodeKind::File || source_was_file_envelope { - node.source = Some(anchor.clone()); - } + // Multiple owned scopes are not an authorization to select one. The + // full publication route can resolve the current source coherently. + if container.is_some_and(Option::is_none) { + return Ok(None); + } + let proven_container = container.and_then(Option::as_ref).filter(|proof| { + previous_size.is_some_and(|size| { + node.source + .as_ref() + .is_some_and(|anchor| is_file_envelope(anchor, &source, size)) + && node.evidence.iter().any(|provenance| { + provenance.origin == EvidenceOrigin::Ast + && provenance.confidence + == compass_model::provenance::EvidenceConfidence::Exact + && provenance.extractor == proof.extractor + && provenance.anchors.iter().any(|anchor| { + *anchor == proof.declaration + || (proof.declaration == proof.envelope + && is_file_envelope(anchor, &source, size)) + }) + }) + }) + }); + // A declaration may coincide with EOF. Refresh a non-file node only + // when current AST ownership and its original declaration provenance + // prove it is a file-scoped container, rather than widening its body. + let refreshed = if node.kind == NodeKind::File { + refreshed_envelope.flatten() + } else { + proven_container.map(|proof| proof.envelope.clone()) + }; + if let Some(anchor) = refreshed { + node.source = Some(anchor.clone()); for provenance in &mut node.evidence { for candidate in &mut provenance.anchors { - let candidate_was_file_envelope = candidate.file == source - && candidate.start_byte == 0 - && previous_file_sizes - .get(&source) - .is_some_and(|size| candidate.end_byte == *size); if candidate.file == source - && (node.kind == NodeKind::File || candidate_was_file_envelope) + && (node.kind == NodeKind::File + || previous_size + .is_some_and(|size| is_file_envelope(candidate, &source, size))) { candidate.clone_from(&anchor); } } - let wiring_was_file_envelope = - provenance.wiring_site.as_ref().is_some_and(|candidate| { - candidate.file == source - && candidate.start_byte == 0 - && previous_file_sizes - .get(&source) - .is_some_and(|size| candidate.end_byte == *size) - }); - if provenance.wiring_site.is_some() - && (node.kind == NodeKind::File || wiring_was_file_envelope) - { + if provenance.wiring_site.as_ref().is_some_and(|site| { + site.file == source + && (node.kind == NodeKind::File + || previous_size + .is_some_and(|size| is_file_envelope(site, &source, size))) + }) { provenance.wiring_site = Some(anchor.clone()); } } @@ -1959,6 +2147,8 @@ fn publish_fact_neutral_incremental( current: &V1GraphDocument, changed_node_ids: &BTreeSet, fact_state: &AstFactDigestState, + source_query_inputs: crate::source_query::Inputs, + prior_source_query: Option, timings: &mut BuildTimings, ) -> Result { let publish_started = Instant::now(); @@ -2021,6 +2211,18 @@ fn publish_fact_neutral_incremental( }), ) }; + crate::source_query::publish( + &output_dir, + current, + &graph_seal + .as_ref() + .ok_or_else(|| { + CoreError::InvalidBuildState("missing graph seal for source evidence".to_owned()) + })? + .sha256, + &source_query_inputs, + prior_source_query, + )?; if let Some(metrics) = store_metrics { record_store_metrics(timings, metrics); } @@ -2746,12 +2948,13 @@ fn build_graph_inner_unscoped( &root, ); if let Some(early_missing) = early_missing + && let Some(prior_source_query) = crate::source_query::reusable(&output_dir) && let Some(early_batch) = extract_fact_neutral_sources(&early_missing, options, &root, &project_evidence)? && let Some(prior_state) = prior_fact_digest_state.as_ref() { let FactNeutralExtractionBatch { - extractions: early_extractions, + extractions: mut early_extractions, source_digests: early_source_digests, empty_files: early_empty_files, } = early_batch; @@ -2796,6 +2999,7 @@ fn build_graph_inner_unscoped( .collect::>(); cache.write_portable_ast_batch_ref(&cache_sources)?; cache.flush()?; + let source_query_inputs = crate::source_query::collect(&mut early_extractions)?; timings.deterministic_extract = stage_started.elapsed(); let result = publish_fact_neutral_incremental( options, @@ -2813,6 +3017,8 @@ fn build_graph_inner_unscoped( ¤t, &changed_file_node_ids, ¤t_fact_state, + source_query_inputs, + Some(prior_source_query), &mut timings, )?; return Ok((result, None)); @@ -3284,6 +3490,7 @@ fn build_graph_inner_unscoped( cache_result?; let premerge_fact_digests = premerge_digest_result?; drop(ast_cache_sources); + let source_query_inputs = crate::source_query::collect(&mut ordered)?; profile_internal_duration("AST cache streaming publication", cache_elapsed); profile_internal_duration("AST fact digest construction", digest_elapsed); // Declaration/definition merging is project-wide and is not idempotent. @@ -3370,6 +3577,8 @@ fn build_graph_inner_unscoped( ¤t, &changed_file_node_ids, ¤t_fact_state, + source_query_inputs, + None, &mut timings, )?; return Ok((result, None)); @@ -3664,6 +3873,20 @@ fn build_graph_inner_unscoped( "no-cluster graph and store publication", no_cluster_graph_write_started.elapsed(), ); + crate::source_query::publish( + &output_dir, + &published.document, + &graph_seal + .as_ref() + .ok_or_else(|| { + CoreError::InvalidBuildState( + "missing graph seal for source evidence".to_owned(), + ) + })? + .sha256, + &source_query_inputs, + None, + )?; if let Some(metrics) = store_metrics { record_store_metrics(&mut timings, metrics); } @@ -4249,6 +4472,18 @@ fn build_graph_inner_unscoped( if let Some(metrics) = store_metrics { record_store_metrics(&mut timings, metrics); } + crate::source_query::publish( + &output_dir, + &published_document, + &graph_seal + .as_ref() + .ok_or_else(|| { + CoreError::InvalidBuildState("missing graph seal for source evidence".to_owned()) + })? + .sha256, + &source_query_inputs, + None, + )?; let graph_seal_for_quality = graph_seal.as_ref().ok_or_else(|| { CoreError::InvalidBuildState( "graph artifact seal is unavailable for community quality evidence".to_owned(), @@ -4643,7 +4878,10 @@ fn publish_build_state( let metrics = ensure_store_snapshot(output_dir)?; record_store_metrics(timings, metrics); } - let mut required = vec![output_dir.join(OUTPUT_STATS_FILE)]; + let mut required = vec![ + output_dir.join(OUTPUT_STATS_FILE), + output_dir.join(compass_model::source_query::SOURCE_QUERY_FILE), + ]; if publish_store { required.push(output_dir.join(STORE_REF_FILE_NAME)); } @@ -4731,6 +4969,13 @@ fn commit_snapshot( if guard.staging_directory().join("program.json").is_file() { artifacts.push("program.json"); } + if guard + .staging_directory() + .join(compass_model::source_query::SOURCE_QUERY_FILE) + .is_file() + { + artifacts.push(compass_model::source_query::SOURCE_QUERY_FILE); + } if publish_store { let gc = garbage_collect_shared_store(output_container, guard.staging_directory())?; timings.store_gc_deleted_entries = timings @@ -7886,6 +8131,53 @@ mod tests { assert!(!is_php_source_path(Path::new("src/controller"))); } + #[test] + fn source_query_metadata_does_not_change_structural_fact_digests() -> Result<(), Box> + { + let path = Path::new("src/sample.rs"); + let source = "fn opaque(payload: &str) -> &str { // alpha marker\npayload }"; + let mut engine = Engine::default(); + let mut first = + engine.extract_source_graph_only(path, "src/sample.rs", source.as_bytes())?; + let second = engine.extract_source_graph_only( + path, + "src/sample.rs", + source.replace("alpha", "bravo").as_bytes(), + )?; + let first_metadata = first + .extensions + .get(compass_languages::source_query::EXTRACTION_EXTENSION) + .ok_or("missing source-query evidence")?; + let second_metadata = second + .extensions + .get(compass_languages::source_query::EXTRACTION_EXTENSION) + .ok_or("missing updated source-query evidence")?; + assert_ne!( + first_metadata, second_metadata, + "source evidence must notice a same-length comment edit" + ); + let expected = extraction_fact_digest(&first)?; + // Body comments remain part of the existing declaration source hash. + let first_hash = first + .semantic_evidence + .as_ref() + .and_then(|batch| batch.declarations.iter().find(|d| d.name == "opaque")) + .and_then(|d| d.source_hash.as_ref()); + let second_hash = second + .semantic_evidence + .as_ref() + .and_then(|batch| batch.declarations.iter().find(|d| d.name == "opaque")) + .and_then(|d| d.source_hash.as_ref()); + assert!(first_hash.is_some() && second_hash.is_some()); + assert_ne!(first_hash, second_hash); + assert_ne!(expected, extraction_fact_digest(&second)?); + first + .extensions + .remove(compass_languages::source_query::EXTRACTION_EXTENSION); + assert_eq!(expected, extraction_fact_digest(&first)?); + Ok(()) + } + #[test] fn compact_extraction_preserves_nested_ast_facts() -> Result<(), Box> { let mut attributes = Map::with_capacity(64); @@ -8963,6 +9255,138 @@ mod tests { Ok(()) } + #[test] + fn leading_documentation_edits_change_fact_digest_without_body_or_range_changes() + -> Result<(), Box> { + let extract = |source: &[u8]| -> Result> { + let extraction = Engine::default() + .extract_source_combined(Path::new("sample.rs"), "sample.rs", source)? + .graph; + if let Some(error) = &extraction.error { + return Err(error.clone().into()); + } + Ok(extraction) + }; + let original_source = b"/// alpha documentation\npub fn work() -> u8 { 1 }\n"; + let changed_source = b"/// omega documentation\npub fn work() -> u8 { 1 }\n"; + assert_eq!(original_source.len(), changed_source.len()); + let original = extract(original_source)?; + let changed = extract(changed_source)?; + let original_declaration = original + .semantic_evidence + .as_ref() + .and_then(|batch| { + batch + .declarations + .iter() + .find(|declaration| declaration.name == "work") + }) + .ok_or("original declaration missing")?; + let changed_declaration = changed + .semantic_evidence + .as_ref() + .and_then(|batch| { + batch + .declarations + .iter() + .find(|declaration| declaration.name == "work") + }) + .ok_or("changed declaration missing")?; + assert_eq!(original_declaration.range, changed_declaration.range); + assert_eq!( + original_declaration.signature_hash, + changed_declaration.signature_hash + ); + assert_eq!( + original_declaration.implementation_hash, + changed_declaration.implementation_hash + ); + assert_eq!( + original_declaration.source_hash, + changed_declaration.source_hash + ); + assert!(original_declaration.documentation.is_some()); + assert!(changed_declaration.documentation.is_some()); + let original_digest = extraction_fact_digest(&original)?; + assert_ne!(original_digest, extraction_fact_digest(&changed)?); + assert_eq!( + original_digest, + extraction_fact_digest(&extract(original_source)?)? + ); + Ok(()) + } + + #[test] + fn leading_documentation_incremental_matches_cold_and_restores_exact_facts() + -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let root = directory.path().join("source"); + fs::create_dir_all(&root)?; + let source = root.join("sample.rs"); + let original = "/// alpha documentation\npub fn work() -> u8 { 1 }\n"; + let changed = "/// omega documentation\npub fn work() -> u8 { 1 }\n"; + let mut options = BuildOptions::new(&root); + options.output_root = Some(directory.path().join("warm-output")); + options.cache_root = Some(directory.path().join("warm-cache")); + options.no_cluster = true; + options.no_viz = true; + options.purpose = BuildPurpose::Extract; + options.graph_storage = GraphStorage::Json; + let semantic = SemanticLayer { + fragment: json!({"nodes": [], "edges": [], "hyperedges": []}), + refreshed_files: Vec::new(), + partial_files: Vec::new(), + allow_partial: false, + }; + let load = + |result: &BuildResult| V1GraphDocument::load(&result.output_dir.join("graph.json")); + let excerpts = |graph: &V1GraphDocument| { + graph + .nodes + .iter() + .filter_map(|node| match &node.details { + Some(compass_model::code_graph::NodeDetails::Resource(details)) => { + details.content.clone() + } + _ => None, + }) + .collect::>() + }; + fs::write(&source, original)?; + let initial_result = build_graph_with_semantic(&options, &semantic)?; + let initial = load(&initial_result)?; + assert_eq!( + excerpts(&initial), + vec!["/// alpha documentation\n".to_owned()] + ); + let warm_result = build_graph_with_semantic(&options, &semantic)?; + assert_eq!(warm_result.files_extracted, 0); + let warm = load(&warm_result)?; + assert_eq!(warm.nodes, initial.nodes); + assert_eq!(warm.links, initial.links); + fs::write(&source, changed)?; + let changed_result = build_graph_with_semantic(&options, &semantic)?; + assert_eq!(changed_result.files_extracted, 1); + let incremental = load(&changed_result)?; + assert_eq!( + excerpts(&incremental), + vec!["/// omega documentation\n".to_owned()] + ); + let mut cold_options = options.clone(); + cold_options.output_root = Some(directory.path().join("cold-output")); + cold_options.cache_root = Some(directory.path().join("cold-cache")); + let cold = load(&build_graph_with_semantic(&cold_options, &semantic)?)?; + assert_eq!(incremental.nodes, cold.nodes); + assert_eq!(incremental.links, cold.links); + assert_eq!(incremental.graph.files, cold.graph.files); + fs::write(&source, original)?; + let restored = load(&build_graph_with_semantic(&options, &semantic)?)?; + assert_eq!(restored.nodes, initial.nodes); + assert_eq!(restored.links, initial.links); + assert_eq!(restored.graph.files, initial.graph.files); + Ok(()) + } + #[test] fn fact_digest_match_requires_all_cached_source_facts() -> Result<(), Box> { let mut entries = BTreeMap::new(); @@ -9045,7 +9469,108 @@ mod tests { package.source.as_ref().map(|anchor| anchor.end_byte), Some(changed_source.len() as u64) ); + let cold_output = tempfile::tempdir()?; + let mut cold_options = options.clone(); + cold_options.output_root = Some(cold_output.path().join("artifacts")); + cold_options.cache_root = Some(cold_output.path().join("cache")); + let rebuilt = build_local_graph(&cold_options)?; + let rebuilt_graph = V1GraphDocument::load(&rebuilt.output_dir.join("graph.json"))?; + assert_eq!(changed_graph.nodes, rebuilt_graph.nodes); + assert_eq!(changed_graph.links, rebuilt_graph.links); + assert_eq!(changed_graph.graph.files, rebuilt_graph.graph.files); compass_model::validate_code_graph(&changed_graph)?; + fs::write(&source, initial)?; + let restored = build_local_graph(&options)?; + let restored_graph = V1GraphDocument::load(&restored.output_dir.join("graph.json"))?; + assert_eq!(restored_graph.nodes, cold_graph.nodes); + assert_eq!(restored_graph.links, cold_graph.links); + assert_eq!(restored_graph.graph.files, cold_graph.graph.files); + Ok(()) + } + + #[test] + fn file_container_envelopes_reject_inherited_and_ambiguous_scopes() -> Result<(), Box> + { + let source = "package example\nclass Main\n"; + let root = Path::new("."); + let extraction = Engine::default() + .extract_source_combined(Path::new("Main.kt"), "Main.kt", source.as_bytes())? + .graph; + let digests = BTreeMap::from([( + "Main.kt".to_owned(), + SourceDigest { + content_digest: "sha256:fixture".to_owned(), + byte_size: source.len() as u64, + }, + )]); + let proofs = file_container_envelopes(std::slice::from_ref(&extraction), root, &digests); + assert_eq!(proofs.values().filter(|proof| proof.is_some()).count(), 1); + let mut ambiguous = extraction.clone(); + let batch = ambiguous + .semantic_evidence + .as_mut() + .ok_or("missing Kotlin evidence")?; + let mut repeated = batch + .scopes + .iter() + .find(|scope| scope.kind == "package") + .cloned() + .ok_or("missing package scope")?; + repeated.id = "scope:second-package-envelope".to_owned(); + batch.scopes.push(repeated); + let ambiguous_proofs = file_container_envelopes(&[ambiguous], root, &digests); + assert_eq!(ambiguous_proofs.len(), 1); + assert!(ambiguous_proofs.values().all(Option::is_none)); + let mut inherited = extraction; + let batch = inherited + .semantic_evidence + .as_mut() + .ok_or("missing Kotlin evidence")?; + let scope = batch + .scopes + .iter_mut() + .find(|scope| scope.kind == "package") + .ok_or("missing package scope")?; + scope.parent_scope_id = Some("scope:unrelated-block".to_owned()); + assert!(file_container_envelopes(&[inherited], root, &digests).is_empty()); + Ok(()) + } + + #[test] + fn an_explicit_namespace_at_eof_does_not_become_a_file_envelope() -> Result<(), Box> + { + let directory = tempfile::tempdir()?; + let root = directory.path().join("source"); + fs::create_dir(&root)?; + let path = root.join("Main.cs"); + let original = "namespace Example { public class Main {} }"; + fs::write(&path, original)?; + let mut options = BuildOptions::new(&root); + options.no_cluster = true; + options.no_viz = true; + options.output_root = Some(directory.path().join("artifacts")); + options.cache_root = Some(directory.path().join("cache")); + let first = build_local_graph(&options)?; + let initial = V1GraphDocument::load(&first.output_dir.join("graph.json"))?; + let namespace = initial + .nodes + .iter() + .find(|node| node.kind == NodeKind::Namespace) + .ok_or("missing explicit namespace")?; + let expected = namespace.clone(); + assert_eq!( + expected.source.as_ref().map(|anchor| anchor.end_byte), + Some(original.len() as u64) + ); + fs::write(&path, format!("{original}\n// outside the namespace\n"))?; + let next = build_local_graph(&options)?; + assert_eq!(next.timings.graph_assembly, Duration::ZERO); + let changed = V1GraphDocument::load(&next.output_dir.join("graph.json"))?; + assert_eq!( + changed.nodes.iter().find(|node| node.id == expected.id), + Some(&expected) + ); + assert_eq!(changed.links, initial.links); Ok(()) } diff --git a/crates/compass-core/src/source_query.rs b/crates/compass-core/src/source_query.rs new file mode 100644 index 000000000..19ba91303 --- /dev/null +++ b/crates/compass-core/src/source_query.rs @@ -0,0 +1,630 @@ +//! Coherent, graph-bound publication of query-only native source evidence. +use crate::CoreError; +use compass_languages::{ + Extraction, + source_query::{self as raw, SourceQueryExtraction}, +}; +use compass_model::code_graph::{EdgeKind, GraphDocument, NodeRecord}; +use compass_model::provenance::{EvidenceConfidence, EvidenceOrigin, SourceAnchor}; +use compass_model::source_query::*; +use sha2::{Digest, Sha256}; +use std::collections::{BTreeMap, BTreeSet}; +use std::fs::File; +use std::io::{Read, Write}; +use std::path::Path; + +#[derive(Default)] +pub(crate) struct Inputs { + files: BTreeMap, + aggregate_truncated: bool, +} +fn invalid(message: impl Into) -> CoreError { + CoreError::InvalidBuildState(message.into()) +} + +pub(crate) fn collect(extractions: &mut [Extraction]) -> Result { + let mut result = Inputs::default(); + let mut bytes = 0_usize; + let mut code_bytes = 0_usize; + for extraction in extractions { + let Some(value) = extraction.extensions.remove(raw::EXTRACTION_EXTENSION) else { + continue; + }; + let mut file: SourceQueryExtraction = serde_json::from_value(value).map_err(|e| { + invalid(format!( + "invalid cached source-query evidence: {e}; run compass update --force" + )) + })?; + if file.schema != raw::EXTRACTION_SCHEMA + || file.profile != SOURCE_QUERY_PROFILE + || file.projection.scopes.len() > 512 + || file.projection.visits > 200_000 + { + return Err(invalid( + "unsupported or oversized source-query extraction; run compass update --force", + )); + } + if bytes.saturating_add(file.projection.retained_bytes) > SOURCE_QUERY_MAX_TEXT_BYTES + || result.files.len() >= SOURCE_QUERY_MAX_OWNERS + { + file.projection.scopes.clear(); + file.projection.truncated = true; + result.aggregate_truncated = true; + } else { + bytes += file.projection.retained_bytes; + } + if file.projection.scopes.is_empty() { + file.projection.retained_code_bytes = 0; + } + if code_bytes.saturating_add(file.projection.retained_code_bytes) + > SOURCE_QUERY_MAX_CODE_BYTES + { + for scope in &mut file.projection.scopes { + scope.omitted_code_windows = scope + .omitted_code_windows + .saturating_add(scope.code_windows.len() as u64); + scope.code_windows.clear(); + } + file.projection.retained_code_bytes = 0; + } else { + code_bytes += file.projection.retained_code_bytes; + } + if result.files.len() < SOURCE_QUERY_MAX_OWNERS + && result + .files + .insert(file.source_file.clone(), file) + .is_some() + { + return Err(invalid("duplicate source-query file identity")); + } + } + Ok(result) +} + +/// A bounded digest pass binds the old sidecar before a neutral graph rewrite. +fn graph_digest(path: &Path) -> Option { + let mut input = File::open(path) + .ok()? + .take(compass_model::DEFAULT_GRAPH_SIZE_CAP_BYTES + 1); + let mut hasher = Sha256::new(); + let mut buffer = [0_u8; 65536]; + let mut bytes = 0_u64; + loop { + let n = input.read(&mut buffer).ok()?; + if n == 0 { + break; + }; + bytes = bytes.checked_add(n as u64)?; + if bytes > compass_model::DEFAULT_GRAPH_SIZE_CAP_BYTES { + return None; + }; + hasher.update(&buffer[..n]); + } + Some(format!("{:x}", hasher.finalize())) +} + +pub(crate) fn reusable(output: &Path) -> Option { + let bytes = + compass_files::read_bytes_bounded(&output.join(SOURCE_QUERY_FILE), SOURCE_QUERY_MAX_BYTES) + .ok()?; + let artifact: SourceQueryArtifact = serde_json::from_slice(&bytes).ok()?; + artifact.validate().ok()?; + if artifact.aggregate_truncated + || artifact.owners.iter().any(|owner| { + owner.omitted_nested_contexts > 0 + || owner + .nested_contexts + .iter() + .any(|context| context.omitted_code_windows > 0) + }) + || graph_digest(&output.join("graph.json"))?.as_str() != artifact.graph_digest + { + return None; + }; + Some(artifact) +} + +fn enclosed(owner: &SourceAnchor, site: &SourceAnchor) -> bool { + owner.file == site.file + && owner.start_byte <= site.start_byte + && site.end_byte <= owner.end_byte + && (owner.start_line, owner.start_column) <= (site.start_line, site.start_column) + && (site.end_line, site.end_column) <= (owner.end_line, owner.end_column) +} +fn exact(node: &NodeRecord) -> bool { + let Some(source) = &node.source else { + return false; + }; + !node.evidence.is_empty() + && node.evidence.iter().all(|p| { + p.origin != EvidenceOrigin::Heuristic + && p.confidence == EvidenceConfidence::Exact + && p.validate().is_ok() + }) + && node.evidence.iter().any(|p| { + p.origin == EvidenceOrigin::Ast && p.anchors.iter().any(|a| enclosed(source, a)) + }) +} +fn site(file: &str, site: &raw::Site) -> Result { + Ok(SourceQuerySite { + text: site.text.clone(), + source: SourceAnchor { + file: file.to_owned(), + start_byte: site.start as u64, + end_byte: site.end as u64, + start_line: u32::try_from(site.start_point.row) + .ok() + .and_then(|r| r.checked_add(1)) + .ok_or_else(|| invalid("source-query line overflow"))?, + end_line: u32::try_from(site.end_point.row) + .ok() + .and_then(|r| r.checked_add(1)) + .ok_or_else(|| invalid("source-query line overflow"))?, + start_column: u32::try_from(site.start_point.column) + .map_err(|_| invalid("source-query column overflow"))?, + end_column: u32::try_from(site.end_point.column) + .map_err(|_| invalid("source-query column overflow"))?, + }, + }) +} + +pub(crate) fn publish( + output: &Path, + graph: &GraphDocument, + digest: &str, + inputs: &Inputs, + prior: Option, +) -> Result<(), CoreError> { + let files = graph + .graph + .files + .iter() + .map(|f| (f.path.as_str(), f)) + .collect::>(); + let nodes = graph + .nodes + .iter() + .map(|n| (n.id.as_str(), n)) + .collect::>(); + let mut ranges = BTreeMap::<(&str, u64, u64), Vec<&NodeRecord>>::new(); + for node in &graph.nodes { + if node.kind.is_callable() + && let Some(source) = &node.source + { + ranges + .entry((&source.file, source.start_byte, source.end_byte)) + .or_default() + .push(node); + } + } + let mut result = SourceQueryArtifact::empty(digest.to_owned()); + result.aggregate_truncated = inputs.aggregate_truncated; + if let Some(prior) = prior { + for file in prior.files { + if !inputs.files.contains_key(&file.file) + && files.get(file.file.as_str()).is_some_and(|f| { + f.content_digest == file.source_digest && f.byte_size == file.byte_size + }) + { + result.files.push(file); + } + } + let kept = result + .files + .iter() + .map(|f| f.file.as_str()) + .collect::>(); + for owner in prior.owners { + if kept.contains(owner.source.file.as_str()) { + if !nodes.get(owner.node_id.as_str()).is_some_and(|node| { + node.kind.is_callable() + && node.source.as_ref() == Some(&owner.source) + && exact(node) + }) { + return Err(invalid( + "prior source-query owner does not match an exact-AST callable; run compass update --force", + )); + } + result.owners.push(owner); + } + } + } + + let mut retained_sites = result + .owners + .iter() + .map(|o| o.identifiers.len() + o.comments.len()) + .sum::(); + let mut retained_text = result + .owners + .iter() + .flat_map(|o| o.identifiers.iter().chain(&o.comments)) + .map(|s| s.text.len()) + .sum::(); + // Prior nested context is allocated after all direct-owner windows too. + let mut retained_code_sites = result + .owners + .iter() + .map(|owner| owner.code_windows.len()) + .sum::(); + let mut retained_code_bytes = result + .owners + .iter() + .flat_map(|owner| &owner.code_windows) + .map(|window| window.text.len()) + .sum::(); + for input in inputs.files.values() { + let Some(file) = files.get(input.source_file.as_str()) else { + continue; + }; + if file.content_digest != input.source_digest || file.byte_size != input.byte_size { + return Err(invalid( + "source-query extraction digest does not match published file inventory", + )); + } + let mut summary = SourceQueryFile { + file: input.source_file.clone(), + source_digest: input.source_digest.clone(), + byte_size: input.byte_size, + language: input.language.clone(), + truncated: input.projection.truncated || input.omission.is_some(), + syntax_recovery: input.syntax_recovery, + omitted_owners: 0, + }; + for scope in &input.projection.scopes { + let candidates = ranges.get(&( + input.source_file.as_str(), + scope.start as u64, + scope.end as u64, + )); + let node = match candidates.map(Vec::as_slice) { + Some([node]) if exact(node) => *node, + _ => { + summary.omitted_owners += 1; + continue; + } + }; + if scope.names.len() > 256 + || scope.comments.len() > 8 + || scope.code_windows.len() > 16 + || scope.names.values().any(|s| s.len() > 4) + { + return Err(invalid( + "source-query scope exceeds extraction profile limits", + )); + } + let mut identifiers = scope + .names + .values() + .flatten() + .map(|s| site(&input.source_file, s)) + .collect::, _>>()?; + let mut comments = scope + .comments + .iter() + .map(|s| site(&input.source_file, s)) + .collect::, _>>()?; + identifiers.sort_by_key(|s| (s.source.start_byte, s.source.end_byte)); + comments.sort_by_key(|s| (s.source.start_byte, s.source.end_byte)); + let count = identifiers.len() + comments.len(); + let bytes = identifiers + .iter() + .chain(&comments) + .map(|s| s.text.len()) + .sum::(); + if result.owners.len() >= SOURCE_QUERY_MAX_OWNERS + || retained_sites.saturating_add(count) > SOURCE_QUERY_MAX_SITES + || retained_text.saturating_add(bytes) > SOURCE_QUERY_MAX_TEXT_BYTES + { + summary.omitted_owners += 1; + summary.truncated = true; + result.aggregate_truncated = true; + continue; + } + let mut code_windows = scope + .code_windows + .iter() + .map(|s| site(&input.source_file, s)) + .collect::, _>>()?; + code_windows.sort_by_key(|s| (s.source.start_byte, s.source.end_byte)); + let mut omitted_code_windows = scope.omitted_code_windows; + let code_bytes = code_windows.iter().map(|s| s.text.len()).sum::(); + if retained_code_sites.saturating_add(code_windows.len()) + > SOURCE_QUERY_MAX_CODE_WINDOWS + || retained_code_bytes.saturating_add(code_bytes) > SOURCE_QUERY_MAX_CODE_BYTES + { + omitted_code_windows = + omitted_code_windows.saturating_add(code_windows.len() as u64); + code_windows.clear(); + } else { + retained_code_sites += code_windows.len(); + retained_code_bytes += code_bytes; + } + retained_sites += count; + retained_text += bytes; + result.owners.push(SourceQueryOwner { + node_id: node.id.clone(), + source: node + .source + .clone() + .ok_or_else(|| invalid("source-query owner lost its source"))?, + source_digest: input.source_digest.clone(), + identifiers, + comments, + code_windows, + omitted_code_windows, + nested_contexts: Vec::new(), + omitted_nested_contexts: 0, + omitted_sites: scope.omitted_sites as u64, + truncated: scope.truncated, + syntax_recovery: input.syntax_recovery, + }); + } + result.files.push(summary); + } + // Direct-owner windows receive the existing quotas first. Only spare + // shared capacity can retain separate nested lexical context. + let positions = result + .owners + .iter() + .enumerate() + .map(|(at, owner)| (owner.node_id.clone(), at)) + .collect::>(); + for input in inputs.files.values() { + // Scopes arrive in native preorder. Resolve the nearest published + // owner in one bounded pass, never by guessing from range containment. + let mut ancestor_owners = + Vec::>::with_capacity(input.projection.scopes.len()); + for (at, scope) in input.projection.scopes.iter().enumerate() { + if scope.callable_start < scope.start + || scope.callable_end > scope.end + || scope.callable_start >= scope.callable_end + { + return Err(invalid("callable syntax exceeds its definition envelope")); + } + let parent_owner = if let Some(parent) = scope.parent_scope { + if parent >= at { + return Err(invalid("invalid source-query parent scope")); + } + let outer = &input.projection.scopes[parent]; + if !(outer.start <= scope.start && scope.end <= outer.end) { + return Err(invalid("nested callable exceeds explicit parent scope")); + } + ancestor_owners[parent] + } else { + None + }; + let key = ( + input.source_file.as_str(), + scope.start as u64, + scope.end as u64, + ); + let published = ranges.get(&key); + let inherited = match published.map(Vec::as_slice) { + Some([node]) if exact(node) => positions.get(&node.id).copied(), + Some(_) => None, + None if scope.name.is_none() => parent_owner, + None => None, + }; + ancestor_owners.push(inherited); + if scope.name.is_some() + || (scope.code_windows.is_empty() && scope.omitted_code_windows == 0) + || !matches!( + scope.syntax.as_str(), + "lambda" + | "closure_expression" + | "func_literal" + | "arrow_function" + | "function_expression" + | "generator_function" + ) + || published.is_some() + { + continue; + } + let selected = parent_owner; + let Some(index) = selected else { continue }; + let owner = &mut result.owners[index]; + let used = owner.code_windows.len() + + owner + .nested_contexts + .iter() + .map(|c| c.code_windows.len()) + .sum::(); + if owner.nested_contexts.len() >= SOURCE_QUERY_MAX_NESTED_CONTEXTS || used >= 16 { + owner.omitted_nested_contexts = owner.omitted_nested_contexts.saturating_add(1); + continue; + } + let scope_anchor = SourceAnchor { + file: input.source_file.clone(), + start_byte: scope.callable_start as u64, + end_byte: scope.callable_end as u64, + start_line: u32::try_from(scope.start_point.row) + .ok() + .and_then(|r| r.checked_add(1)) + .ok_or_else(|| invalid("nested source line overflow"))?, + end_line: u32::try_from(scope.end_point.row) + .ok() + .and_then(|r| r.checked_add(1)) + .ok_or_else(|| invalid("nested source line overflow"))?, + start_column: u32::try_from(scope.start_point.column) + .map_err(|_| invalid("nested source column overflow"))?, + end_column: u32::try_from(scope.end_point.column) + .map_err(|_| invalid("nested source column overflow"))?, + }; + if !enclosed(&owner.source, &scope_anchor) + || (scope_anchor.start_byte == owner.source.start_byte + && scope_anchor.end_byte == owner.source.end_byte) + { + return Err(invalid( + "nested source context does not match its published parent", + )); + } + let mut context = SourceQueryNestedContext { + source: scope_anchor, + syntax: scope.syntax.clone(), + code_windows: Vec::new(), + omitted_code_windows: scope.omitted_code_windows, + }; + let mut windows = scope.code_windows.iter().collect::>(); + windows.sort_by_key(|w| (w.start, w.end)); + for window in windows { + if used + context.code_windows.len() >= 16 { + context.omitted_code_windows = context.omitted_code_windows.saturating_add(1); + continue; + } + let window = site(&input.source_file, window)?; + if !enclosed(&context.source, &window.source) { + return Err(invalid("nested code window escapes its scope")); + } + context.code_windows.push(window); + } + if context.code_windows.is_empty() { + owner.omitted_nested_contexts = owner.omitted_nested_contexts.saturating_add(1); + } else { + owner.nested_contexts.push(context); + } + } + } + let mut context_order = (0..result.owners.len()).collect::>(); + context_order.sort_by(|left, right| { + let left = &result.owners[*left]; + let right = &result.owners[*right]; + (&left.source.file, left.source.start_byte, &left.node_id).cmp(&( + &right.source.file, + right.source.start_byte, + &right.node_id, + )) + }); + for index in context_order { + let owner = &mut result.owners[index]; + owner + .nested_contexts + .sort_by_key(|context| (context.source.start_byte, context.source.end_byte)); + for context in &mut owner.nested_contexts { + context.code_windows.retain(|window| { + if retained_code_sites >= SOURCE_QUERY_MAX_CODE_WINDOWS + || retained_code_bytes.saturating_add(window.text.len()) + > SOURCE_QUERY_MAX_CODE_BYTES + { + context.omitted_code_windows = context.omitted_code_windows.saturating_add(1); + false + } else { + retained_code_sites += 1; + retained_code_bytes += window.text.len(); + true + } + }); + } + owner.omitted_nested_contexts = owner.omitted_nested_contexts.saturating_add( + owner + .nested_contexts + .iter() + .filter(|context| context.code_windows.is_empty()) + .count() as u64, + ); + owner + .nested_contexts + .retain(|context| !context.code_windows.is_empty()); + } + result.files.sort_by(|a, b| a.file.cmp(&b.file)); + result.owners.sort_by(|a, b| a.node_id.cmp(&b.node_id)); + let summaries = result + .files + .iter() + .map(|f| f.file.as_str()) + .collect::>(); + result.omitted_files = files + .values() + .filter(|f| { + f.language + .as_deref() + .is_some_and(|l| matches!(l, "python" | "rust" | "go" | "typescript" | "tsx")) + && !summaries.contains(f.path.as_str()) + }) + .count() as u64; + result.omitted_files += result + .files + .iter() + .filter(|f| f.truncated || f.syntax_recovery) + .count() as u64; + result.omitted_owners = result.files.iter().map(|f| f.omitted_owners).sum(); + let owner_ids = result + .owners + .iter() + .map(|o| o.node_id.as_str()) + .collect::>(); + let mut aliases = BTreeMap::<&str, Vec<&str>>::new(); + for edge in &graph.links { + if edge.kind == EdgeKind::Aliases && owner_ids.contains(edge.target.as_str()) { + let entries = aliases.entry(&edge.target).or_default(); + if entries.len() < 9 { + entries.push( + if !edge.evidence.is_empty() + && compass_model::search::is_exact_nonheuristic_relationship(edge) + { + nodes + .get(edge.source.as_str()) + .map_or("", |n| n.name.as_str()) + } else { + "" + }, + ); + } + } + } + for owner in &result.owners { + let node = nodes + .get(owner.node_id.as_str()) + .ok_or_else(|| invalid("source-query owner is missing from graph"))?; + let base = compass_model::search::search_document_with_aliases( + node, + aliases + .get(owner.node_id.as_str()) + .into_iter() + .flatten() + .copied(), + ) + .terms(); + for term in owner.document().terms() { + *result + .additional_document_frequencies + .entry(term.clone()) + .or_default() += u64::from(!base.contains(&term)); + } + } + result.validate().map_err(|e| invalid(e.to_string()))?; + let path = output.join(SOURCE_QUERY_FILE); + compass_files::write_atomic_with_digest(&path, |writer| { + let mut bounded = CappedWriter { + inner: writer, + bytes: 0, + }; + serde_json::to_writer(&mut bounded, &result).map_err(|source| { + compass_files::FileError::Json { + path: path.clone(), + source, + } + }) + })?; + Ok(()) +} +struct CappedWriter { + inner: W, + bytes: u64, +} +impl Write for CappedWriter { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + if self.bytes.saturating_add(bytes.len() as u64) > SOURCE_QUERY_MAX_BYTES { + return Err(std::io::Error::other( + "source-query artifact byte limit exceeded", + )); + } + let n = self.inner.write(bytes)?; + self.bytes += n as u64; + Ok(n) + } + fn flush(&mut self) -> std::io::Result<()> { + self.inner.flush() + } +} diff --git a/crates/compass-core/tests/source_query_bounds.rs b/crates/compass-core/tests/source_query_bounds.rs new file mode 100644 index 000000000..c3f0926ca --- /dev/null +++ b/crates/compass-core/tests/source_query_bounds.rs @@ -0,0 +1,93 @@ +use compass_core::{BuildOptions, GraphStorage, build_local_graph}; +use compass_model::query_contract::{ + DiscoveryDirection, DiscoveryLimits, DiscoveryQueryRequest, DiscoveryTraversal, +}; +use compass_query::{EngineSelection, open_with_engine}; +use std::error::Error; +use std::fs; +fn request(question: &str) -> DiscoveryQueryRequest { + DiscoveryQueryRequest { + question: question.to_owned(), + direction: DiscoveryDirection::Auto, + relation_contexts: Vec::new(), + scope: Vec::new(), + traversal: DiscoveryTraversal::Bfs, + include_heuristic: false, + limits: DiscoveryLimits::default(), + } +} +#[test] +fn body_candidate_ceiling_is_explicit_on_both_native_backends() -> Result<(), Box> { + let d = tempfile::tempdir()?; + let root = d.path().join("repo"); + fs::create_dir(&root)?; + let source = (0..70) + .map(|i| format!("fn opaque_{i}() {{ // Handle payload here.\n}}\n")) + .collect::(); + fs::write(root.join("sample.rs"), source)?; + let mut options = BuildOptions::new(&root); + options.output_root = Some(d.path().join("artifacts")); + options.graph_storage = GraphStorage::Sqlite; + options.no_cluster = true; + options.no_viz = true; + options.max_workers = Some(1); + let built = build_local_graph(&options)?; + let path = built.output_dir.join("graph.json"); + for backend in [EngineSelection::Json, EngineSelection::Store] { + let engine = open_with_engine( + &path, + None, + &d.path().join(format!("cache-{backend:?}")), + backend, + )?; + let response = engine.discover_ranked(request("Where is payload handled?"))?; + assert!(response.bounded); + assert_eq!(response.eligible_count, None); + assert!(!response.candidates.is_empty()); + assert!(response.candidates.len() <= 64); + assert!( + response + .discovery + .diagnostics + .iter() + .any(|d| d.message.contains("bound")) + ); + } + Ok(()) +} +#[test] +fn body_comments_on_nested_callables_do_not_nominate_the_outer_function() +-> Result<(), Box> { + let d = tempfile::tempdir()?; + let root = d.path().join("repo"); + fs::create_dir(&root)?; + fs::write( + root.join("sample.py"), + "def opaque(value):\n def inner(value):\n value = value\n # Handle secret messages here.\n return value\n return inner(value)\n", + )?; + let mut options = BuildOptions::new(&root); + options.output_root = Some(d.path().join("artifacts")); + options.no_cluster = true; + options.no_viz = true; + options.max_workers = Some(1); + let built = build_local_graph(&options)?; + for backend in [EngineSelection::Json, EngineSelection::Store] { + let engine = open_with_engine( + &built.output_dir.join("graph.json"), + None, + &d.path().join(format!("cache-{backend:?}")), + backend, + )?; + let response = engine.discover_ranked(request("Where is secret handled?"))?; + assert!(!response.candidates.is_empty()); + assert!( + response + .candidates + .iter() + .all(|row| row.node.name.trim_end_matches("()") == "inner"), + "{:?}", + response.candidates + ); + } + Ok(()) +} diff --git a/crates/compass-core/tests/source_query_integrity.rs b/crates/compass-core/tests/source_query_integrity.rs new file mode 100644 index 000000000..e0466eb6e --- /dev/null +++ b/crates/compass-core/tests/source_query_integrity.rs @@ -0,0 +1,260 @@ +use compass_core::{BuildOptions, GraphStorage, build_local_graph}; +use compass_model::code_graph::{GraphDocument, NodeKind}; +use compass_model::provenance::SourceAnchor; +use compass_model::query_contract::{ + DiscoveryDirection, DiscoveryLimits, DiscoveryQueryRequest, DiscoveryTraversal, +}; +use compass_model::source_query::{ + SOURCE_QUERY_FILE, SourceQueryArtifact, SourceQueryOwner, SourceQuerySite, +}; +use compass_query::{EngineSelection, QueryErrorKind, open_with_engine}; +use std::error::Error; +use std::fs; +use std::path::Path; +fn request() -> DiscoveryQueryRequest { + DiscoveryQueryRequest { + question: "Where is payload handled?".to_owned(), + direction: DiscoveryDirection::Auto, + relation_contexts: Vec::new(), + scope: Vec::new(), + traversal: DiscoveryTraversal::Bfs, + include_heuristic: false, + limits: DiscoveryLimits::default(), + } +} +fn options(root: &Path, output: &Path) -> BuildOptions { + let mut o = BuildOptions::new(root); + o.output_root = Some(output.to_path_buf()); + o.graph_storage = GraphStorage::Json; + o.no_cluster = true; + o.no_viz = true; + o.max_workers = Some(1); + o +} +#[test] +fn outside_edit_does_not_widen_a_declaration_that_ends_at_eof() -> Result<(), Box> { + let d = tempfile::tempdir()?; + let root = d.path().join("repo"); + fs::create_dir(&root)?; + let source = + "fn opaque(value: &str) -> &str {\nlet result = value;\n// Handle payload here.\nresult }"; + fs::write(root.join("sample.rs"), source)?; + let o = options(&root, &d.path().join("artifacts")); + let first = build_local_graph(&o)?; + let a = GraphDocument::load(&first.output_dir.join("graph.json"))?; + let owner = a + .nodes + .iter() + .find(|n| n.kind.is_callable() && n.name.trim_end_matches("()") == "opaque") + .ok_or("missing owner")?; + let expected = owner.source.clone(); + assert_eq!( + expected.as_ref().map(|s| s.end_byte), + Some(source.len() as u64) + ); + fs::write( + root.join("sample.rs"), + format!("{source}\n// outside declaration\n"), + )?; + let second = build_local_graph(&o)?; + let b = GraphDocument::load(&second.output_dir.join("graph.json"))?; + let owner = b + .nodes + .iter() + .find(|n| n.kind.is_callable() && n.name.trim_end_matches("()") == "opaque") + .ok_or("missing owner")?; + assert_eq!( + owner.source, expected, + "a coincident file/declaration range is not a file envelope" + ); + let engine = open_with_engine( + &second.output_dir.join("graph.json"), + None, + &d.path().join("query"), + EngineSelection::Json, + )?; + assert!( + engine + .discover_ranked(request())? + .candidates + .iter() + .any(|c| c.node.id == owner.id) + ); + Ok(()) +} +#[test] +fn schema_valid_non_callable_body_owners_are_rejected() -> Result<(), Box> { + let d = tempfile::tempdir()?; + let root = d.path().join("repo"); + fs::create_dir(&root)?; + let source = "struct Holder {\n// Handle payload.\nvalue: u8\n}\nfn opaque() {}\n"; + fs::write(root.join("sample.rs"), source)?; + let o = options(&root, &d.path().join("artifacts")); + let built = build_local_graph(&o)?; + let path = built.output_dir.join("graph.json"); + let graph = GraphDocument::load(&path)?; + let node = graph + .nodes + .iter() + .find(|n| n.kind == NodeKind::Struct && n.name.trim_end_matches("()") == "Holder") + .ok_or("missing struct")?; + let mut artifact: SourceQueryArtifact = + serde_json::from_slice(&fs::read(built.output_dir.join(SOURCE_QUERY_FILE))?)?; + let text = "// Handle payload."; + let start = source.find(text).ok_or("missing comment")?; + let owner = SourceQueryOwner { + node_id: node.id.clone(), + source: node.source.clone().ok_or("missing struct source")?, + source_digest: artifact.files[0].source_digest.clone(), + identifiers: Vec::new(), + comments: vec![SourceQuerySite { + source: SourceAnchor { + file: "sample.rs".into(), + start_byte: start as u64, + end_byte: (start + text.len()) as u64, + start_line: 2, + start_column: 0, + end_line: 2, + end_column: text.len() as u32, + }, + text: text.into(), + }], + code_windows: Vec::new(), + omitted_code_windows: 0, + nested_contexts: Vec::new(), + omitted_nested_contexts: 0, + omitted_sites: 0, + truncated: false, + syntax_recovery: false, + }; + let base = compass_model::search::search_document(node).terms(); + artifact.additional_document_frequencies = owner + .document() + .terms() + .into_iter() + .map(|term| { + let df = u64::from(!base.contains(&term)); + (term, df) + }) + .collect(); + artifact.owners = vec![owner]; + artifact.validate()?; + fs::write( + built.output_dir.join(SOURCE_QUERY_FILE), + serde_json::to_vec(&artifact)?, + )?; + let engine = open_with_engine(&path, None, &d.path().join("query"), EngineSelection::Json)?; + assert!( + engine.discover_ranked(request()).is_err(), + "callable source evidence cannot be attached to a struct" + ); + Ok(()) +} +#[test] +fn unknown_major_is_a_typed_error_even_without_current_version_fields() -> Result<(), Box> +{ + let d = tempfile::tempdir()?; + let root = d.path().join("repo"); + fs::create_dir(&root)?; + fs::write(root.join("sample.rs"), "fn opaque() {}")?; + let o = options(&root, &d.path().join("artifacts")); + let built = build_local_graph(&o)?; + fs::write( + built.output_dir.join(SOURCE_QUERY_FILE), + br#"{"schema":"compass.source-query/2"}"#, + )?; + let result = open_with_engine( + &built.output_dir.join("graph.json"), + None, + &d.path().join("query"), + EngineSelection::Json, + ); + let error = match result { + Err(error) => error, + Ok(_) => return Err("unknown major was admitted".into()), + }; + assert_eq!(error.kind(), QueryErrorKind::UnsupportedSchema); + assert_eq!(error.code(), "unsupported_source_query_schema"); + Ok(()) +} + +#[test] +fn immutable_legacy_projection_is_readable_without_rewriting_on_both_backends() +-> Result<(), Box> { + use compass_model::source_query::{ + SOURCE_QUERY_PROFILE, SOURCE_QUERY_PROFILE_V1, SOURCE_QUERY_PROFILE_V2, + }; + let d = tempfile::tempdir()?; + let root = d.path().join("repo"); + fs::create_dir(&root)?; + fs::write( + root.join("sample.rs"), + "fn opaque(value: &str) -> &str { let retained = value; // Handle payload here.\n retained }\n", + )?; + let mut o = options(&root, &d.path().join("out")); + o.graph_storage = GraphStorage::Sqlite; + let built = build_local_graph(&o)?; + let graph = built.output_dir.join("graph.json"); + let path = built.output_dir.join(SOURCE_QUERY_FILE); + let original = fs::read(&path)?; + let mut artifact: SourceQueryArtifact = serde_json::from_slice(&original)?; + assert_eq!(artifact.profile, SOURCE_QUERY_PROFILE); + let mut expected = open_with_engine( + &graph, + None, + &d.path().join("current"), + EngineSelection::Json, + )? + .discover_ranked(request())? + .candidates; + assert!(!expected.is_empty()); + for owner in &mut artifact.owners { + owner.code_windows.clear(); + owner.omitted_code_windows = 0; + } + for candidate in &mut expected { + candidate + .excerpts + .retain(|quote| quote.kind != compass_query::RankedExcerptKind::CallableCode); + } + for profile in [SOURCE_QUERY_PROFILE_V1, SOURCE_QUERY_PROFILE_V2] { + artifact.profile = profile.to_owned(); + artifact.validate()?; + let legacy = serde_json::to_vec(&artifact)?; + fs::write(&path, &legacy)?; + for mode in [EngineSelection::Json, EngineSelection::Store] { + let engine = open_with_engine( + &graph, + None, + &d.path().join(format!("legacy-{profile}-{mode:?}")), + mode, + )?; + assert_eq!(engine.discover_ranked(request())?.candidates, expected); + assert_eq!( + fs::read(&path)?, + legacy, + "query rewrote immutable legacy evidence" + ); + } + } + // Version dispatch must precede decoding the current payload fields. + fs::write( + &path, + br#"{"schema":"compass.source-query/1","profile":"source-query/999-future"}"#, + )?; + for mode in [EngineSelection::Json, EngineSelection::Store] { + let result = open_with_engine( + &graph, + None, + &d.path().join(format!("future-{mode:?}")), + mode, + ); + let error = match result { + Err(error) => error, + Ok(_) => return Err("unknown profile was admitted".into()), + }; + assert_eq!(error.kind(), QueryErrorKind::UnsupportedSchema); + assert_eq!(error.code(), "unsupported_source_query_profile"); + } + Ok(()) +} diff --git a/crates/compass-core/tests/source_query_pipeline.rs b/crates/compass-core/tests/source_query_pipeline.rs new file mode 100644 index 000000000..1f79bc97d --- /dev/null +++ b/crates/compass-core/tests/source_query_pipeline.rs @@ -0,0 +1,606 @@ +use compass_core::{BuildOptions, GraphStorage, build_local_graph}; +use compass_model::code_graph::GraphDocument; +use compass_model::query_contract::{ + DiscoveryDirection, DiscoveryLimits, DiscoveryQueryRequest, DiscoveryTraversal, +}; +use compass_model::source_query::{SOURCE_QUERY_FILE, SourceQueryArtifact}; +use compass_query::{EngineSelection, RelevanceBand, open_with_engine}; +use sha2::{Digest, Sha256}; +use std::error::Error; +use std::fs; +use std::path::Path; +fn request(question: &str) -> DiscoveryQueryRequest { + DiscoveryQueryRequest { + question: question.to_owned(), + direction: DiscoveryDirection::Auto, + relation_contexts: Vec::new(), + scope: Vec::new(), + traversal: DiscoveryTraversal::Bfs, + include_heuristic: false, + limits: DiscoveryLimits::default(), + } +} +fn options(root: &Path, output: &Path, storage: GraphStorage) -> BuildOptions { + let mut o = BuildOptions::new(root); + o.output_root = Some(output.to_path_buf()); + o.no_cluster = true; + o.no_viz = true; + o.max_workers = Some(1); + o.graph_storage = storage; + o.built_at_commit = Some("0123456789012345678901234567890123456789".to_owned()); + o +} +fn artifact(output: &Path) -> Result> { + let a: SourceQueryArtifact = + serde_json::from_slice(&fs::read(output.join(SOURCE_QUERY_FILE))?)?; + a.validate()?; + assert_eq!( + a.graph_digest, + format!("{:x}", Sha256::digest(fs::read(output.join("graph.json"))?)) + ); + Ok(a) +} + +#[test] +fn direct_variable_callable_bodies_bind_and_query_on_both_backends() -> Result<(), Box> { + for extension in ["ts", "tsx"] { + for (form, initializer) in [ + ( + "arrow", + "(value) => { value = value; // Handle payload normalization here.\n return value; }", + ), + ( + "named", + "function internal(value) { value = value; // Handle payload normalization here.\n return value; }", + ), + ( + "generator", + "function* (value) { value = value; // Handle payload normalization here.\n yield value; }", + ), + ] { + let d = tempfile::tempdir()?; + let root = d.path().join("repo"); + fs::create_dir(&root)?; + let file = format!("{form}.{extension}"); + let source = format!("export const opaque = {initializer};\n"); + fs::write(root.join(&file), &source)?; + let built = build_local_graph(&options( + &root, + &d.path().join("artifacts"), + GraphStorage::Sqlite, + ))?; + let a = artifact(&built.output_dir)?; + let graph_path = built.output_dir.join("graph.json"); + let graph = GraphDocument::load(&graph_path)?; + let node = graph + .nodes + .iter() + .find(|node| { + node.kind.is_callable() && node.name.trim_end_matches("()") == "opaque" + }) + .ok_or("missing callable binding")?; + let owner = a + .owners + .iter() + .find(|owner| owner.node_id == node.id) + .ok_or("initializer body did not bind")?; + assert_eq!(Some(&owner.source), node.source.as_ref()); + assert_eq!( + &source[owner.source.start_byte as usize..owner.source.end_byte as usize], + format!("opaque = {initializer}") + ); + let mut previous = None; + for mode in [EngineSelection::Json, EngineSelection::Store] { + let engine = open_with_engine( + &graph_path, + None, + &d.path().join(format!("cache-{mode:?}")), + mode, + )?; + let response = engine.discover_ranked(request("Where is payload handled?"))?; + let candidate = response + .candidates + .first() + .ok_or("missing body candidate")?; + assert_eq!(candidate.node.id, node.id, "{file}"); + assert_eq!(candidate.relevance, RelevanceBand::Low); + assert!( + candidate + .matched_fields + .iter() + .any(|field| field == "body_comment") + ); + assert!(candidate.supporting_sources.iter().any(|anchor| { + anchor.file == file + && source + .get(anchor.start_byte as usize..anchor.end_byte as usize) + .is_some_and(|text| text.contains("payload normalization")) + })); + assert!(!candidate.excerpts.is_empty(), "{file}"); + for quote in &candidate.excerpts { + assert!(matches!( + quote.kind, + compass_query::RankedExcerptKind::CallableComment + | compass_query::RankedExcerptKind::CallableCode + )); + assert!(quote.is_coherent()); + assert_eq!( + source + .get(quote.source.start_byte as usize..quote.source.end_byte as usize), + Some(quote.text.as_str()) + ); + assert!(quote.text.contains("payload")); + } + // A newly opened engine must quote the immutable artifact even + // when current source has changed since graph publication. + fs::write(root.join(&file), "// modified after publication\n")?; + let pinned = open_with_engine( + &graph_path, + None, + &d.path().join(format!("cache-mutated-{mode:?}")), + mode, + )?; + assert_eq!( + pinned + .discover_ranked(request("Where is payload handled?"))? + .candidates, + response.candidates + ); + fs::write(root.join(&file), &source)?; + if let Some(previous) = &previous { + assert_eq!(previous, &response.candidates); + } + previous = Some(response.candidates); + } + } + } + Ok(()) +} + +#[test] +fn native_update_body_only_query_is_shared_by_json_and_store() -> Result<(), Box> { + for (file, source) in [ + ( + "sample.py", + "def opaque(value):\n value = value\n # Handle payload normalization here.\n return value\ndef decoy(value):\n return value\n", + ), + ( + "leading_comment.py", + "def opaque(value):\n # Handle payload normalization here.\n return value\ndef decoy(value):\n return value\n", + ), + ( + "sample.go", + "package sample\nfunc opaque(value string) string { // Handle payload normalization here.\nreturn value }\nfunc decoy(value string) string { return value }\n", + ), + ( + "sample.rs", + "fn opaque(value: &str) -> &str { // Handle payload normalization here.\nvalue }\nfn decoy(value: &str) -> &str { value }\n", + ), + ( + "sample.ts", + "export function opaque(value: unknown) { // Handle payload normalization here.\nreturn value; }\nfunction decoy(value: unknown) { return value; }\n", + ), + ( + "sample.tsx", + "export function opaque(value: unknown) { // Handle payload normalization here.\nreturn {String(value)}; }\nfunction decoy(value: unknown) { return value; }\n", + ), + ] { + let d = tempfile::tempdir()?; + let root = d.path().join("repo"); + fs::create_dir(&root)?; + fs::write(root.join(file), source)?; + let o = options(&root, &d.path().join("artifacts"), GraphStorage::Sqlite); + let built = build_local_graph(&o)?; + let a = artifact(&built.output_dir)?; + let graph_path = built.output_dir.join("graph.json"); + let graph = GraphDocument::load(&graph_path)?; + let expected = graph + .nodes + .iter() + .find(|n| n.kind.is_callable() && n.name.trim_end_matches("()") == "opaque") + .ok_or("missing opaque")?; + assert!( + a.owners.iter().any(|owner| owner.node_id == expected.id), + "{file}: {a:?}" + ); + let mut previous = None; + for mode in [EngineSelection::Json, EngineSelection::Store] { + let engine = open_with_engine( + &graph_path, + None, + &d.path().join(format!("cache-{mode:?}")), + mode, + )?; + let response = engine.discover_ranked(request("Where is payload handled?"))?; + let candidate = response.candidates.first().ok_or_else(|| { + format!( + "{file}: no candidates: {:?}", + response.discovery.diagnostics + ) + })?; + assert_eq!(candidate.node.id, expected.id, "{file}"); + assert_eq!( + candidate.relevance, + RelevanceBand::Low, + "{file}: {candidate:?}" + ); + assert!( + candidate.matched_fields.iter().any(|f| f == "body_comment"), + "{file}" + ); + assert!( + candidate.supporting_sources.iter().any(|s| s.file == file + && source + .get(s.start_byte as usize..s.end_byte as usize) + .is_some_and(|t| t.contains("payload"))), + "{file}" + ); + assert!( + response.candidates.iter().all(|r| r.node.id == expected.id), + "{file}: irrelevant candidates" + ); + assert!( + engine + .discover_ranked(request("Where is unicorn handled?"))? + .candidates + .is_empty() + ); + assert_eq!( + response.candidates, + engine + .discover_ranked(request("Where is payload handled?"))? + .candidates + ); + if let Some(previous) = previous { + assert_eq!(response.candidates, previous, "{file}: backend mismatch"); + } + previous = Some(response.candidates); + } + } + Ok(()) +} + +#[test] +fn publication_refreshes_neutral_edits_preserves_other_files_and_repairs_corruption() +-> Result<(), Box> { + for storage in [GraphStorage::Json, GraphStorage::Sqlite] { + let d = tempfile::tempdir()?; + let root = d.path().join("repo"); + fs::create_dir(&root)?; + let original = "fn opaque(value: &str) -> &str { // Handle payload.\nvalue }\n"; + fs::write(root.join("sample.rs"), original)?; + fs::write( + root.join("other.rs"), + "fn other() { // Check invariant.\n}\n", + )?; + let mut o = options(&root, &d.path().join("artifacts"), storage); + let cold = build_local_graph(&o)?; + let first = artifact(&cold.output_dir)?; + let first_bytes = fs::read(cold.output_dir.join(SOURCE_QUERY_FILE))?; + let warm = build_local_graph(&o)?; + assert!(!warm.outputs_changed); + assert_eq!( + first_bytes, + fs::read(warm.output_dir.join(SOURCE_QUERY_FILE))? + ); + fs::write( + root.join("sample.rs"), + format!("{original}// outside declaration\n"), + )?; + let updated = build_local_graph(&o)?; + let after = artifact(&updated.output_dir)?; + assert_ne!(first.graph_digest, after.graph_digest); + assert_eq!( + first + .owners + .iter() + .filter(|o| o.source.file == "other.rs") + .collect::>(), + after + .owners + .iter() + .filter(|o| o.source.file == "other.rs") + .collect::>() + ); + let preserved = after.clone(); + o.force = true; + let rebuilt = build_local_graph(&o)?; + assert_eq!(preserved, artifact(&rebuilt.output_dir)?); + o.force = false; + fs::write(rebuilt.output_dir.join(SOURCE_QUERY_FILE), b"corrupt")?; + assert!( + open_with_engine( + &rebuilt.output_dir.join("graph.json"), + None, + &d.path().join("broken-query"), + EngineSelection::Default + ) + .is_err() + ); + let repaired = build_local_graph(&o)?; + artifact(&repaired.output_dir)?; + fs::remove_file(root.join("other.rs"))?; + let deleted = build_local_graph(&o)?; + let a = artifact(&deleted.output_dir)?; + assert!(a.files.iter().all(|f| f.file != "other.rs")); + assert!(a.owners.iter().all(|f| f.source.file != "other.rs")); + } + Ok(()) +} + +#[test] +fn comment_free_implementation_has_immutable_code_excerpt_on_both_backends() +-> Result<(), Box> { + for (file, source) in [ + ( + "sample.py", + "def opaque(items):\n return normalize_payload(items)\ndef decoy(items):\n return items\n", + ), + ( + "sample.rs", + "fn opaque(items: usize) -> usize { normalize_payload(items) }\nfn decoy(items: usize) -> usize { items }\n", + ), + ( + "sample.go", + "package sample\nfunc opaque(items int) int { return normalize_payload(items) }\nfunc decoy(items int) int { return items }\n", + ), + ( + "sample.ts", + "export function opaque(items: unknown) { return normalize_payload(items); }\nfunction decoy(items: unknown) { return items; }\n", + ), + ( + "sample.tsx", + "export const opaque = (items: unknown) => normalize_payload(items);\nfunction decoy(items: unknown) { return items; }\n", + ), + ] { + let d = tempfile::tempdir()?; + let root = d.path().join("repo"); + fs::create_dir(&root)?; + fs::write(root.join(file), source)?; + let built = build_local_graph(&options( + &root, + &d.path().join("artifacts"), + GraphStorage::Sqlite, + ))?; + let graph_path = built.output_dir.join("graph.json"); + let mut previous = None; + for mode in [EngineSelection::Json, EngineSelection::Store] { + let engine = open_with_engine( + &graph_path, + None, + &d.path().join(format!("cache-code-{mode:?}")), + mode, + )?; + let response = engine.discover_ranked(request("where is payload normalized"))?; + let candidate = response.candidates.first().ok_or("no candidate")?; + assert!(candidate.node.qualified_name.contains("opaque")); + assert!( + candidate.excerpts.iter().any(|quote| { + serde_json::to_value(quote) + .ok() + .is_some_and(|value| value["kind"] == "callable_code") + && quote.text.contains("normalize_payload(items)") + }), + "comment-free implementation must show retained code: {:?}", + candidate.excerpts + ); + for quote in &candidate.excerpts { + assert!(quote.is_coherent()); + assert_eq!( + source.get(quote.source.start_byte as usize..quote.source.end_byte as usize), + Some(quote.text.as_str()) + ); + } + fs::write(root.join(file), "# current source is different\n")?; + let pinned = open_with_engine( + &graph_path, + None, + &d.path().join(format!("cache-code-pinned-{mode:?}")), + mode, + )?; + assert_eq!( + pinned + .discover_ranked(request("where is payload normalized"))? + .candidates, + response.candidates + ); + fs::write(root.join(file), source)?; + if let Some(previous) = &previous { + assert_eq!(previous, &response.candidates); + } + previous = Some(response.candidates); + } + } + Ok(()) +} + +#[test] +fn nested_formatter_context_is_exact_separate_and_equal_on_json_and_store() +-> Result<(), Box> { + let directory = tempfile::tempdir()?; + let root = directory.path().join("repo"); + fs::create_dir(&root)?; + let source = "fn opaque(diagnostics: &[Diagnostic]) -> Vec {\n diagnostics.iter().map(|diagnostic| {\n format!(\"{}: {}\", diagnostic.code(), diagnostic.message())\n }).collect()\n}\n"; + fs::write(root.join("formatter.rs"), source)?; + let built = build_local_graph(&options( + &root, + &directory.path().join("artifacts"), + GraphStorage::Sqlite, + ))?; + let a = artifact(&built.output_dir)?; + let graph_path = built.output_dir.join("graph.json"); + let graph = GraphDocument::load(&graph_path)?; + let target = graph + .nodes + .iter() + .find(|node| node.kind.is_callable() && node.name.trim_end_matches("()") == "opaque") + .ok_or("missing formatter")?; + let owner = a + .owners + .iter() + .find(|owner| owner.node_id == target.id) + .ok_or("missing owner")?; + assert!(!owner.nested_contexts.is_empty()); + assert!( + owner + .code_windows + .iter() + .all(|window| !window.text.contains("format!")) + ); + assert!( + owner + .nested_contexts + .iter() + .flat_map(|context| &context.code_windows) + .any(|window| window.text.contains("format!(")) + ); + let mut prior = None; + for mode in [EngineSelection::Json, EngineSelection::Store] { + let engine = open_with_engine( + &graph_path, + None, + &directory.path().join(format!("query-cache-{mode:?}")), + mode, + )?; + let response = + engine.discover_ranked(request("Where are diagnostics formatted in `opaque`?"))?; + let candidate = response + .candidates + .iter() + .find(|candidate| candidate.node.id == target.id) + .ok_or("formatter not retrieved")?; + let quote = candidate + .excerpts + .first() + .ok_or("missing implementation quote")?; + assert_eq!( + quote.kind, + compass_query::RankedExcerptKind::NestedCallableCode + ); + assert!( + quote.text.contains("format!(") && quote.text.contains("diagnostic.message()"), + "{}", + quote.text + ); + let scope = quote.nested_scope.as_ref().ok_or("missing lexical scope")?; + assert!(target.source.as_ref().is_some_and( + |parent| parent.start_byte < scope.start_byte && scope.end_byte < parent.end_byte + )); + assert_eq!( + &source[quote.source.start_byte as usize..quote.source.end_byte as usize], + quote.text + ); + if let Some(old) = prior { + assert_eq!(old, candidate.excerpts); + } + prior = Some(candidate.excerpts.clone()); + } + Ok(()) +} + +#[test] +fn nested_context_respects_named_published_and_type_boundaries() -> Result<(), Box> { + let d = tempfile::tempdir()?; + let root = d.path().join("repo"); + fs::create_dir(&root)?; + let source = "export function opaque(payload: number) {\n const callback = (item: number) => secret_named(item);\n function named() { secret_function(); }\n class Nested { run() { return (() => secret_type(payload))(); } }\n return payload;\n}\n"; + fs::write(root.join("sample.ts"), source)?; + let built = build_local_graph(&options( + &root, + &d.path().join("artifacts"), + GraphStorage::Sqlite, + ))?; + let a = artifact(&built.output_dir)?; + let graph = GraphDocument::load(&built.output_dir.join("graph.json"))?; + let callback = graph + .nodes + .iter() + .find(|node| node.kind.is_callable() && node.name.trim_end_matches("()") == "callback") + .ok_or("fixture requires a published callback binding")?; + assert!(a.owners.iter().any(|owner| owner.node_id == callback.id)); + let target = graph + .nodes + .iter() + .find(|node| node.kind.is_callable() && node.name.trim_end_matches("()") == "opaque") + .ok_or("missing parent")?; + let owner = a + .owners + .iter() + .find(|owner| owner.node_id == target.id) + .ok_or("missing owner")?; + assert!( + owner.nested_contexts.is_empty(), + "named/published children and class scopes cannot lend code to this parent" + ); + Ok(()) +} + +#[test] +fn nested_context_uses_only_spare_direct_window_capacity() -> Result<(), Box> { + let d = tempfile::tempdir()?; + let root = d.path().join("repo"); + fs::create_dir(&root)?; + let mut source = "fn opaque(payload: usize) {\n".to_owned(); + for at in 0..24 { + source.push_str(&format!(" normalize_payload_{at}(payload);\n")); + } + source.push_str(" let unused = || hidden_payload_error(payload);\n}\n"); + fs::write(root.join("sample.rs"), source)?; + let built = build_local_graph(&options( + &root, + &d.path().join("artifacts"), + GraphStorage::Sqlite, + ))?; + let a = artifact(&built.output_dir)?; + let graph = GraphDocument::load(&built.output_dir.join("graph.json"))?; + let target = graph + .nodes + .iter() + .find(|node| node.kind.is_callable() && node.name.trim_end_matches("()") == "opaque") + .ok_or("missing parent")?; + let owner = a + .owners + .iter() + .find(|owner| owner.node_id == target.id) + .ok_or("missing owner")?; + assert_eq!(owner.code_windows.len(), 16); + assert!(owner.nested_contexts.is_empty()); + assert!(owner.omitted_nested_contexts > 0); + Ok(()) +} + +#[test] +fn unused_nested_code_is_only_lexical_context_and_survives_neutral_rebuilds() +-> Result<(), Box> { + let d = tempfile::tempdir()?; + let root = d.path().join("repo"); + fs::create_dir(&root)?; + let source = "fn opaque(payload: usize) { let unused = || hidden_payload_error(payload); }\nfn hidden_payload_error(payload: usize) -> usize { payload }\n"; + fs::write(root.join("sample.rs"), source)?; + let o = options(&root, &d.path().join("incremental"), GraphStorage::Sqlite); + let built = build_local_graph(&o)?; + let first = artifact(&built.output_dir)?; + assert!( + first + .owners + .iter() + .flat_map(|owner| &owner.nested_contexts) + .any(|context| context + .code_windows + .iter() + .any(|window| window.text.contains("hidden_payload_error"))) + ); + fs::write(root.join("sample.rs"), format!("{source}\n"))?; + let incremental = build_local_graph(&o)?; + let mut incremental = artifact(&incremental.output_dir)?; + let clean = build_local_graph(&options( + &root, + &d.path().join("clean"), + GraphStorage::Sqlite, + ))?; + let mut clean = artifact(&clean.output_dir)?; + incremental.graph_digest = "a".repeat(64); + clean.graph_digest = "a".repeat(64); + assert_eq!(incremental, clean); + Ok(()) +} diff --git a/crates/compass-core/tests/source_query_snapshot_contract.rs b/crates/compass-core/tests/source_query_snapshot_contract.rs new file mode 100644 index 000000000..ed418cff0 --- /dev/null +++ b/crates/compass-core/tests/source_query_snapshot_contract.rs @@ -0,0 +1,139 @@ +use compass_core::{BuildOptions, GraphStorage, build_local_graph}; +use compass_model::code_graph::GraphDocument; +use compass_model::query_contract::{ + DiscoveryDirection, DiscoveryLimits, DiscoveryQueryRequest, DiscoveryTraversal, +}; +use compass_model::source_query::SOURCE_QUERY_FILE; +use compass_query::{ + EngineSelection, QueryEngineCache, open_with_engine, open_with_verified_document, +}; +use std::error::Error; +use std::fs; +use std::path::Path; +fn request() -> DiscoveryQueryRequest { + DiscoveryQueryRequest { + question: "Where is payload handled?".to_owned(), + direction: DiscoveryDirection::Auto, + relation_contexts: Vec::new(), + scope: Vec::new(), + traversal: DiscoveryTraversal::Bfs, + include_heuristic: false, + limits: DiscoveryLimits::default(), + } +} +fn build(root: &Path, output: &Path, topic: &str) -> Result> { + fs::create_dir_all(root)?; + fs::write( + root.join("sample.rs"), + format!( + "fn opaque(value: &str) -> &str {{\nlet returned = value;\n// Handle {topic} here.\nreturned }}\n" + ), + )?; + let mut o = BuildOptions::new(root); + o.output_root = Some(output.to_path_buf()); + o.graph_storage = GraphStorage::Sqlite; + o.no_cluster = true; + o.no_viz = true; + o.max_workers = Some(1); + Ok(build_local_graph(&o)?.output_dir) +} +#[test] +fn current_engine_caches_detect_auxiliary_corruption_and_content_restoration() +-> Result<(), Box> { + let d = tempfile::tempdir()?; + let output = build(&d.path().join("repo"), &d.path().join("out"), "payload")?; + let graph = output.join("graph.json"); + let path = output.join(SOURCE_QUERY_FILE); + let original = fs::read(&path)?; + let cache = QueryEngineCache::default(); + let first = cache.open_published_store(&graph)?; + assert!( + !first + .lock() + .map_err(|_| "poisoned engine")? + .discover_ranked(request())? + .candidates + .is_empty() + ); + fs::write(&path, b"corrupt")?; + assert!( + cache.open_published_store(&graph).is_err(), + "cache must not return a stale body snapshot" + ); + fs::write(&path, &original)?; + let restored = cache.open_published_store(&graph)?; + assert!(std::sync::Arc::ptr_eq(&first, &restored)); + let (document, identity) = GraphDocument::load_with_artifact_digest(&graph)?; + let json_cache = QueryEngineCache::default(); + let first = json_cache.open_verified_document( + &document, + &identity, + &graph, + &d.path().join("json-cache"), + )?; + fs::write(&path, b"corrupt")?; + assert!( + json_cache + .open_verified_document(&document, &identity, &graph, &d.path().join("json-cache")) + .is_err() + ); + fs::write(&path, &original)?; + let restored = json_cache.open_verified_document( + &document, + &identity, + &graph, + &d.path().join("json-cache"), + )?; + assert!(std::sync::Arc::ptr_eq(&first, &restored)); + Ok(()) +} +#[test] +fn mismatched_or_unknown_artifacts_fail_current_queries_but_detached_graphs_stay_isolated() +-> Result<(), Box> { + let d = tempfile::tempdir()?; + let a = build(&d.path().join("a"), &d.path().join("out-a"), "payload")?; + let b = build(&d.path().join("b"), &d.path().join("out-b"), "transport")?; + let graph = a.join("graph.json"); + let path = a.join(SOURCE_QUERY_FILE); + let original = fs::read(&path)?; + fs::copy(b.join(SOURCE_QUERY_FILE), &path)?; + for mode in [EngineSelection::Json, EngineSelection::Store] { + assert!(open_with_engine(&graph, None, &d.path().join("bad-cache"), mode).is_err()); + } + let (document, identity) = GraphDocument::load_with_artifact_digest(&graph)?; + let detached = + open_with_verified_document(document, identity, &graph, None, &d.path().join("detached"))?; + assert!( + detached.discover_ranked(request())?.candidates.is_empty(), + "a detached graph must not borrow current body evidence" + ); + let mut value: serde_json::Value = serde_json::from_slice(&original)?; + value["schema"] = "compass.source-query/2".into(); + fs::write(&path, serde_json::to_vec(&value)?)?; + assert!( + open_with_engine( + &graph, + None, + &d.path().join("version-cache"), + EngineSelection::Default + ) + .is_err() + ); + fs::remove_file(&path)?; + let older = open_with_engine( + &graph, + None, + &d.path().join("older-cache"), + EngineSelection::Default, + )?; + let answer = older.discover_ranked(request())?; + assert!(answer.candidates.is_empty()); + assert!( + answer + .discovery + .diagnostics + .iter() + .any(|d| d.message.contains("no immutable callable-body index")) + ); + Ok(()) +} diff --git a/crates/compass-files/src/cache.rs b/crates/compass-files/src/cache.rs index aafccebca..9701c1514 100644 --- a/crates/compass-files/src/cache.rs +++ b/crates/compass-files/src/cache.rs @@ -13,7 +13,7 @@ use sha2::{Digest, Sha256}; use crate::{FileError, StatHashIndex, file_hash, io_error, write_bytes_atomic, write_json_atomic}; /// Changes whenever cached extraction semantics change, even if the wire encoding does not. -pub const AST_CACHE_VERSION: &str = "16"; +pub const AST_CACHE_VERSION: &str = "24"; /// Portable cache encoding version used in the on-disk namespace. pub const CACHE_ENCODING_VERSION: u32 = 1; const MESSAGEPACK_EXTENSION: &str = "msgpack"; diff --git a/crates/compass-graph/src/snapshot.rs b/crates/compass-graph/src/snapshot.rs index a9a1c1602..d8c00379c 100644 --- a/crates/compass-graph/src/snapshot.rs +++ b/crates/compass-graph/src/snapshot.rs @@ -1953,17 +1953,27 @@ impl<'a, S: Store + ?Sized> GraphSnapshotReader<'a, S> { } /// Read the first graph diagnostic for a stable code through a point - /// projection. This avoids scanning large omission sets when a query only - /// needs the publication summary. + /// projection. Older snapshots without that projection use the bounded + /// diagnostic supplement; an incomplete scan cannot prove absence. pub fn graph_diagnostic_by_code( &self, code: &str, ) -> Result, SnapshotError> { let key = encode_graph_index_key(IndexKind::Metadata, &[b"diagnostic-code", code.as_bytes()])?; - self.lookup(IndexKind::Metadata, &key)? - .map(|value| decode_json::(&value)) - .transpose() + if let Some(value) = self.lookup(IndexKind::Metadata, &key)? { + return decode_json::(&value).map(Some); + } + let (diagnostics, truncated) = self.graph_diagnostics(SnapshotReadLimits::default())?; + if let Some(diagnostic) = diagnostics.into_iter().find(|item| item.code == code) { + return Ok(Some(diagnostic)); + } + if truncated { + return Err(SnapshotError::Limit(format!( + "diagnostic lookup for {code} has no point projection and reached its scan limit; rebuild with compass update --force or narrow the build scope" + ))); + } + Ok(None) } pub fn get_node(&self, id: &str) -> Result, SnapshotError> { @@ -2278,12 +2288,28 @@ impl<'a, S: Store + ?Sized> GraphSnapshotReader<'a, S> { Ok((nodes, truncated, work)) } + fn explicit_term_capability(&self, capability: &str) -> Result { + let Some(value) = self.lookup(IndexKind::Terms, &term_capability_key(capability)?)? else { + return Ok(false); + }; + let version = decode_json::(&value)?; + if version != 1 { + return Err(SnapshotError::Corrupt(format!( + "unsupported term capability version {version} for {capability}" + ))); + } + Ok(true) + } + /// Whether this snapshot includes raw identifier-subword term postings. /// - /// The sentinel is an ordinary empty posting so readers predating this - /// capability continue to accept the additive index entry. + /// Current snapshots use a separate capability record. Legacy empty + /// postings remain readable; a real source term is never replaced by one. pub fn supports_identifier_subwords(&self) -> Result { let capability = IDENTIFIER_SUBWORD_INDEX_CAPABILITY_V1; + if self.explicit_term_capability(capability)? { + return Ok(true); + } let posting_prefix = capability.get(..3).unwrap_or(capability); let key = encode_graph_index_key( IndexKind::Terms, @@ -2301,10 +2327,43 @@ impl<'a, S: Store + ?Sized> GraphSnapshotReader<'a, S> { Ok(posting.term == capability && posting.node_ids.is_empty()) } + /// Source-owned field projection document frequencies. Absence identifies + /// an older immutable index; never rewrite historical snapshots to add it. + pub fn fielded_document_frequency(&self, term: &str) -> Result, SnapshotError> { + let capability = + encode_graph_index_key(IndexKind::Terms, &[b"fielded_df_v1", b"capability"])?; + let Some(capability) = self.lookup(IndexKind::Terms, &capability)? else { + return Ok(None); + }; + if decode_json::(&capability)? != 1 { + return Err(SnapshotError::Corrupt( + "unsupported fielded document-frequency capability".to_owned(), + )); + } + let key = encode_graph_index_key( + IndexKind::Terms, + &[b"fielded_df_v1", b"term", term.as_bytes()], + )?; + let frequency = self + .lookup(IndexKind::Terms, &key)? + .map(|bytes| decode_json::(&bytes)) + .transpose()? + .unwrap_or(0); + if frequency > self.manifest.node_count { + return Err(SnapshotError::Corrupt( + "fielded term frequency exceeds node count".to_owned(), + )); + } + Ok(Some(frequency)) + } + /// Whether this snapshot includes exact term postings restricted to /// source-backed operation-role declarations. pub fn supports_operation_role_terms(&self) -> Result { let capability = OPERATION_ROLE_TERM_INDEX_CAPABILITY_V1; + if self.explicit_term_capability(capability)? { + return Ok(true); + } let key = encode_graph_index_key( IndexKind::Terms, &[b"operation_role", capability.as_bytes(), b"00000000"], @@ -2320,6 +2379,9 @@ impl<'a, S: Store + ?Sized> GraphSnapshotReader<'a, S> { /// source-backed type declarations. pub fn supports_declaration_terms(&self) -> Result { let capability = DECLARATION_TERM_INDEX_CAPABILITY_V1; + if self.explicit_term_capability(capability)? { + return Ok(true); + } let key = encode_graph_index_key( IndexKind::Terms, &[b"declaration", capability.as_bytes(), b"00000000"], @@ -2450,6 +2512,9 @@ impl<'a, S: Store + ?Sized> GraphSnapshotReader<'a, S> { /// Whether this snapshot includes exact direct-caller concept postings. pub fn supports_relationship_terms(&self) -> Result { let capability = RELATIONSHIP_TERM_INDEX_CAPABILITY_V1; + if self.explicit_term_capability(capability)? { + return Ok(true); + } let posting_prefix = capability.get(..3).unwrap_or(capability); let key = encode_graph_index_key( IndexKind::Terms, @@ -2472,6 +2537,9 @@ impl<'a, S: Store + ?Sized> GraphSnapshotReader<'a, S> { /// direct-call postings in the query layer. pub fn supports_relationship_search_terms(&self) -> Result { let capability = RELATIONSHIP_SEARCH_TERM_INDEX_CAPABILITY_V1; + if self.explicit_term_capability(capability)? { + return Ok(true); + } let posting_prefix = capability.get(..3).unwrap_or(capability); let key = encode_graph_index_key( IndexKind::Terms, @@ -3289,13 +3357,13 @@ fn canonical_graph_document_with_generation( } } -fn build_term_postings(graph: &GraphDocument) -> BTreeMap> { +fn alias_names_by_target(graph: &GraphDocument) -> BTreeMap<&str, Vec<&str>> { let node_by_id = graph .nodes .iter() .map(|node| (node.id.as_str(), node)) .collect::>(); - let mut aliases_by_target = BTreeMap::<&str, BTreeSet<&str>>::new(); + let mut aliases_by_target = BTreeMap::<&str, Vec<&str>>::new(); for edge in &graph.links { if edge.kind == compass_model::code_graph::EdgeKind::Aliases && let Some(alias) = node_by_id.get(edge.source.as_str()) @@ -3303,13 +3371,63 @@ fn build_term_postings(graph: &GraphDocument) -> BTreeMap> { aliases_by_target .entry(edge.target.as_str()) .or_default() - .insert(alias.name.as_str()); + .push(alias.name.as_str()); } } + for aliases in aliases_by_target.values_mut() { + aliases.sort_unstable(); + } + aliases_by_target +} +fn fielded_alias_names_by_target(graph: &GraphDocument) -> BTreeMap<&str, Vec<&str>> { + let node_by_id = graph + .nodes + .iter() + .map(|node| (node.id.as_str(), node)) + .collect::>(); + let mut aliases_by_target = BTreeMap::<&str, Vec<&str>>::new(); + for edge in &graph.links { + if edge.kind != EdgeKind::Aliases { + continue; + } + // Preserve raw relationship multiplicity so filtering cannot reach a + // ninth exact alias beyond the bounded query-time read window. + let name = if !edge.evidence.is_empty() + && compass_model::search::is_exact_nonheuristic_relationship(edge) + { + node_by_id + .get(edge.source.as_str()) + .map_or("", |alias| alias.name.as_str()) + } else { + "" + }; + aliases_by_target + .entry(edge.target.as_str()) + .or_default() + .push(name); + } + for aliases in aliases_by_target.values_mut() { + aliases.sort_unstable(); + } + aliases_by_target +} + +fn build_term_postings(graph: &GraphDocument) -> BTreeMap> { + let aliases_by_target = alias_names_by_target(graph); + let fielded_aliases_by_target = fielded_alias_names_by_target(graph); let mut term_postings = BTreeMap::>::new(); for node in &graph.nodes { let mut terms = searchable_node_terms(node); + let document = compass_model::search::search_document_with_aliases( + node, + fielded_aliases_by_target + .get(node.id.as_str()) + .into_iter() + .flat_map(|aliases| aliases.iter().copied()), + ); + terms.extend(document.terms()); + terms.extend(document.primary_terms(node.kind)); for alias in aliases_by_target .get(node.id.as_str()) .into_iter() @@ -3331,6 +3449,8 @@ fn build_term_postings(graph: &GraphDocument) -> BTreeMap> { fn searchable_node_terms(node: &NodeRecord) -> BTreeSet { let mut terms = compass_model::search::document_search_terms(node); + terms.extend(compass_model::search::search_document(node).terms()); + terms.extend(compass_model::search::primary_evidence_terms(node)); terms.extend(search_terms(&node.name)); terms.extend(search_terms(&node.qualified_name)); terms.extend(compass_model::search::identifier_search_terms(&node.name)); @@ -3428,14 +3548,12 @@ fn build_index( )?, diagnostic, )?; - if diagnostic.code == "publication_omission_summary" { - let key = encode_graph_index_key( - IndexKind::Metadata, - &[b"diagnostic-code", diagnostic.code.as_bytes()], - )?; - let value = encode_json(diagnostic)?; - entries.entry(key).or_insert(value); - } + let key = encode_graph_index_key( + IndexKind::Metadata, + &[b"diagnostic-code", diagnostic.code.as_bytes()], + )?; + let value = encode_json(diagnostic)?; + entries.entry(key).or_insert(value); } insert_json( &mut entries, @@ -3476,6 +3594,36 @@ fn build_index( IndexKind::Terms => { let term_postings = term_postings .ok_or_else(|| SnapshotError::Corrupt("term postings are missing".to_owned()))?; + let mut frequencies = BTreeMap::::new(); + let fielded_aliases_by_target = fielded_alias_names_by_target(graph); + for node in &graph.nodes { + for term in compass_model::search::search_document_with_aliases( + node, + fielded_aliases_by_target + .get(node.id.as_str()) + .into_iter() + .flat_map(|aliases| aliases.iter().copied()), + ) + .terms() + { + *frequencies.entry(term).or_default() += 1; + } + } + insert_json( + &mut entries, + encode_graph_index_key(IndexKind::Terms, &[b"fielded_df_v1", b"capability"])?, + &1_u64, + )?; + for (term, frequency) in frequencies { + insert_json( + &mut entries, + encode_graph_index_key( + IndexKind::Terms, + &[b"fielded_df_v1", b"term", term.as_bytes()], + )?, + &frequency, + )?; + } for (term, node_ids) in term_postings { let prefix_length = term.len().min(3); let prefix = term.get(..prefix_length).unwrap_or(term.as_str()); @@ -3503,8 +3651,9 @@ fn build_index( } let capability = IDENTIFIER_SUBWORD_INDEX_CAPABILITY_V1; let prefix = capability.get(..3).unwrap_or(capability); - insert_json( + insert_term_capability( &mut entries, + capability, encode_graph_index_key( IndexKind::Terms, &[ @@ -3514,10 +3663,6 @@ fn build_index( b"00000000", ], )?, - &TermPostingChunk { - term: capability.to_owned(), - node_ids: Vec::new(), - }, )?; for (term, node_ids) in operation_role_term_postings(graph) { for (chunk_index, chunk) in @@ -3538,8 +3683,9 @@ fn build_index( } } let operation_capability = OPERATION_ROLE_TERM_INDEX_CAPABILITY_V1; - insert_json( + insert_term_capability( &mut entries, + operation_capability, encode_graph_index_key( IndexKind::Terms, &[ @@ -3548,10 +3694,6 @@ fn build_index( b"00000000", ], )?, - &TermPostingChunk { - term: operation_capability.to_owned(), - node_ids: Vec::new(), - }, )?; for (term, node_ids) in declaration_term_postings(graph, term_postings) { for (chunk_index, chunk) in @@ -3572,8 +3714,9 @@ fn build_index( } } let declaration_capability = DECLARATION_TERM_INDEX_CAPABILITY_V1; - insert_json( + insert_term_capability( &mut entries, + declaration_capability, encode_graph_index_key( IndexKind::Terms, &[ @@ -3582,10 +3725,6 @@ fn build_index( b"00000000", ], )?, - &TermPostingChunk { - term: declaration_capability.to_owned(), - node_ids: Vec::new(), - }, )?; let direct_relationship_postings = compass_model::search::direct_call_source_identifier_postings(graph); @@ -3645,8 +3784,9 @@ fn build_index( let relationship_prefix = relationship_capability .get(..3) .unwrap_or(relationship_capability); - insert_json( + insert_term_capability( &mut entries, + relationship_capability, encode_graph_index_key( IndexKind::Terms, &[ @@ -3656,10 +3796,6 @@ fn build_index( b"00000000", ], )?, - &TermPostingChunk { - term: relationship_capability.to_owned(), - node_ids: Vec::new(), - }, )?; let relationship_search_postings = @@ -3724,8 +3860,9 @@ fn build_index( let relationship_search_prefix = relationship_search_capability .get(..3) .unwrap_or(relationship_search_capability); - insert_json( + insert_term_capability( &mut entries, + relationship_search_capability, encode_graph_index_key( IndexKind::Terms, &[ @@ -3735,10 +3872,6 @@ fn build_index( b"00000000", ], )?, - &TermPostingChunk { - term: relationship_search_capability.to_owned(), - node_ids: Vec::new(), - }, )?; for node in &graph.nodes { for (kind, value, canonical) in @@ -4175,6 +4308,38 @@ fn update_index_tree( } } +// Capability metadata has a disjoint namespace from every source posting. +fn term_capability_key(capability: &str) -> Result, SnapshotError> { + encode_graph_index_key(IndexKind::Terms, &[b"capability", capability.as_bytes()]) +} + +fn insert_term_capability( + entries: &mut BTreeMap, Vec>, + capability: &str, + legacy_key: Vec, +) -> Result<(), SnapshotError> { + insert_json(entries, term_capability_key(capability)?, &1_u64)?; + if let Some(value) = entries.get(&legacy_key) { + let posting = decode_json::(value)?; + if posting.term != capability { + return Err(SnapshotError::Corrupt( + "legacy capability key contains a different source term".to_owned(), + )); + } + // Real source postings retain every ID and chunk. Older readers may + // report this capability unavailable; current readers use the record above. + return Ok(()); + } + insert_json( + entries, + legacy_key, + &TermPostingChunk { + term: capability.to_owned(), + node_ids: Vec::new(), + }, + ) +} + fn insert_json( entries: &mut BTreeMap, Vec>, key: Vec, @@ -4947,6 +5112,107 @@ mod tests { ); } + #[test] + fn fielded_aliases_count_empty_provenance_without_promoting_legacy_names() + -> Result<(), SnapshotError> { + let mut graph = cache_fixture_graph(); + let mut alias = graph + .nodes + .first() + .cloned() + .ok_or_else(|| SnapshotError::Corrupt("test declaration is missing".to_owned()))?; + alias.id = "alias".to_owned(); + alias.kind = NodeKind::TypeAlias; + alias.name = "renewal".to_owned(); + alias.qualified_name = "crate::renewal".to_owned(); + let site = alias.source.clone(); + graph.nodes.push(alias); + let id = + compass_model::identity::edge_id("alias", EdgeKind::Aliases, "a", site.as_ref(), None); + graph.links.push(EdgeRecord { + id: id.clone(), + key: id, + source: "alias".to_owned(), + target: "a".to_owned(), + kind: EdgeKind::Aliases, + occurrence_rule: None, + relationship_site: site, + details: None, + evidence: Vec::new(), + weight: None, + context: None, + deferred: false, + diagnostics: Vec::new(), + }); + // Exercise the projection directly: empty provenance cannot be newly + // published, but the legacy relationship predicate accepts it. + assert_eq!(alias_names_by_target(&graph)["a"], ["renewal"]); + assert_eq!(fielded_alias_names_by_target(&graph)["a"], [""]); + Ok(()) + } + + #[test] + fn fielded_frequency_rejects_malformed_capabilities_and_impossible_counts() + -> Result<(), SnapshotError> { + for (name, capability, frequency) in [ + ("legacy", None, &b"1"[..]), + ("current", Some(&b"1"[..]), &b"1"[..]), + ("unknown-capability", Some(&b"2"[..]), &b"1"[..]), + ("malformed-capability", Some(&b"{}"[..]), &b"1"[..]), + ("impossible-frequency", Some(&b"1"[..]), &b"2"[..]), + ("negative-frequency", Some(&b"1"[..]), &b"-1"[..]), + ] { + let graph = cache_fixture_graph(); + let store = MemoryStore::default(); + let builder = GraphSnapshotBuilder::new(); + let mut content = builder.prepare_content(&store, &graph)?; + let postings = build_term_postings(&graph); + let mut entries = build_index(&graph, IndexKind::Terms, Some(&postings))?; + let capability_key = + encode_graph_index_key(IndexKind::Terms, &[b"fielded_df_v1", b"capability"])?; + if let Some(value) = capability { + entries.insert(capability_key, value.to_vec()); + } else { + entries.remove(&capability_key); + } + entries.insert( + encode_graph_index_key( + IndexKind::Terms, + &[b"fielded_df_v1", b"term", b"function"], + )?, + frequency.to_vec(), + ); + let entry_count = entries.len() as u64; + let mut writer = ObjectWriter::new(&store)?; + let digest = build_index_tree(&mut writer, IndexKind::Terms, entries)?; + let _ = writer.finish()?; + let term_root = content + .roots + .iter_mut() + .find(|root| root.index == IndexKind::Terms) + .ok_or_else(|| SnapshotError::Corrupt("test term root missing".to_owned()))?; + term_root.digest = digest; + term_root.entry_count = entry_count; + let (graph_digest, graph_bytes) = digest_canonical_graph(&graph, false)?; + let prepared = builder.finish_content(&store, content, graph_digest, graph_bytes)?; + builder.activate(&store, &prepared)?; + let reader = GraphSnapshotReader::open_active(&store)? + .ok_or_else(|| SnapshotError::Corrupt("test active snapshot missing".to_owned()))?; + match name { + "legacy" => assert_eq!(reader.fielded_document_frequency("function")?, None), + "current" => assert_eq!(reader.fielded_document_frequency("function")?, Some(1)), + _ => assert!( + matches!( + reader.fielded_document_frequency("function"), + Err(SnapshotError::Corrupt(_) | SnapshotError::Decode(_)) + ), + "{name}" + ), + } + } + Ok(()) + } + #[test] fn canonical_digest_counter_has_no_two_gibibyte_cutoff() -> Result<(), std::io::Error> { let mut writer = DigestWriter::new(); @@ -5102,6 +5368,134 @@ mod tests { graph } + fn diagnostic(code: &str, message: &str) -> GraphDiagnostic { + GraphDiagnostic { + severity: compass_model::code_graph::DiagnosticSeverity::Warning, + code: code.to_owned(), + message: message.to_owned(), + anchor: None, + related_ids: Vec::new(), + } + } + + fn legacy_diagnostic_store(graph: &GraphDocument) -> Result { + let store = MemoryStore::default(); + let builder = GraphSnapshotBuilder::new(); + let mut content = builder.prepare_content(&store, graph)?; + let mut metadata_entries = build_index(graph, IndexKind::Metadata, None)?; + for diagnostic in &graph.graph.diagnostics { + metadata_entries.remove(&encode_graph_index_key( + IndexKind::Metadata, + &[b"diagnostic-code", diagnostic.code.as_bytes()], + )?); + } + let entry_count = metadata_entries.len() as u64; + let mut writer = ObjectWriter::new(&store)?; + let digest = build_index_tree(&mut writer, IndexKind::Metadata, metadata_entries)?; + let _ = writer.finish()?; + let root = content + .roots + .iter_mut() + .find(|root| root.index == IndexKind::Metadata) + .ok_or_else(|| SnapshotError::Corrupt("metadata root is missing".to_owned()))?; + root.digest = digest; + root.entry_count = entry_count; + let (digest, bytes) = digest_canonical_graph(graph, false)?; + let prepared = builder.finish_content(&store, content, digest, bytes)?; + builder.activate(&store, &prepared)?; + Ok(store) + } + + #[test] + fn diagnostic_code_projection_preserves_first_occurrence_and_export() + -> Result<(), SnapshotError> { + let mut graph = cache_fixture_graph(); + graph.graph.diagnostics = vec![ + diagnostic("leading_documentation_omission_summary", "first omission"), + diagnostic("leading_documentation_omission_summary", "second omission"), + diagnostic("other", "unrelated warning"), + ]; + let store = MemoryStore::default(); + let builder = GraphSnapshotBuilder::new(); + let prepared = builder.prepare(&store, &graph)?; + builder.activate(&store, &prepared)?; + let reader = GraphSnapshotReader::open_active(&store)? + .ok_or_else(|| SnapshotError::Corrupt("missing test snapshot".to_owned()))?; + let key = encode_graph_index_key( + IndexKind::Metadata, + &[ + b"diagnostic-code", + b"leading_documentation_omission_summary", + ], + )?; + assert!(reader.lookup(IndexKind::Metadata, &key)?.is_some()); + assert_eq!( + reader.graph_diagnostic_by_code("leading_documentation_omission_summary")?, + graph.graph.diagnostics.first().cloned() + ); + assert!(reader.graph_diagnostic_by_code("absent")?.is_none()); + assert_eq!( + reader.export_graph()?.graph.diagnostics, + graph.graph.diagnostics + ); + Ok(()) + } + + #[test] + fn legacy_diagnostic_lookup_reads_immutable_supplements_without_rewriting() + -> Result<(), SnapshotError> { + let mut graph = cache_fixture_graph(); + graph.graph.diagnostics = vec![ + diagnostic("other", "unrelated warning"), + diagnostic("leading_documentation_omission_summary", "first omission"), + diagnostic("leading_documentation_omission_summary", "second omission"), + ]; + let store = legacy_diagnostic_store(&graph)?; + let reader = GraphSnapshotReader::open_active(&store)? + .ok_or_else(|| SnapshotError::Corrupt("missing test snapshot".to_owned()))?; + let selector = reader.selector().clone(); + assert_eq!( + reader.graph_diagnostic_by_code("leading_documentation_omission_summary")?, + graph.graph.diagnostics.get(1).cloned() + ); + assert!(reader.graph_diagnostic_by_code("absent")?.is_none()); + assert_eq!( + reader.export_graph()?.graph.diagnostics, + graph.graph.diagnostics + ); + assert_eq!( + GraphSnapshotReader::open_active(&store)? + .ok_or_else(|| SnapshotError::Corrupt("missing test snapshot".to_owned()))? + .selector(), + &selector + ); + Ok(()) + } + + #[test] + fn legacy_diagnostic_lookup_does_not_report_absence_after_a_scan_limit() + -> Result<(), SnapshotError> { + let mut graph = cache_fixture_graph(); + graph.graph.diagnostics = (0..=MAX_SCAN_ITEMS) + .map(|_| diagnostic("early", "warning")) + .collect(); + graph.graph.diagnostics.push(diagnostic("late", "omission")); + let store = legacy_diagnostic_store(&graph)?; + let reader = GraphSnapshotReader::open_active(&store)? + .ok_or_else(|| SnapshotError::Corrupt("missing test snapshot".to_owned()))?; + assert_eq!( + reader.graph_diagnostic_by_code("early")?, + graph.graph.diagnostics.first().cloned() + ); + for code in ["late", "absent"] { + assert!(matches!( + reader.graph_diagnostic_by_code(code), + Err(SnapshotError::Limit(message)) if message.contains("scan limit") + )); + } + Ok(()) + } + #[test] fn decoded_tree_cache_is_bounded_and_retains_branches_over_leaf_lru() { let mut cache = TreeObjectCache::default(); @@ -5621,6 +6015,15 @@ mod tests { b"00000000", ], )?); + for capability in [ + IDENTIFIER_SUBWORD_INDEX_CAPABILITY_V1, + OPERATION_ROLE_TERM_INDEX_CAPABILITY_V1, + DECLARATION_TERM_INDEX_CAPABILITY_V1, + RELATIONSHIP_TERM_INDEX_CAPABILITY_V1, + RELATIONSHIP_SEARCH_TERM_INDEX_CAPABILITY_V1, + ] { + term_entries.remove(&term_capability_key(capability)?); + } let term_entry_count = term_entries.len() as u64; let mut writer = ObjectWriter::new(&store)?; let term_digest = build_index_tree(&mut writer, IndexKind::Terms, term_entries)?; @@ -5888,4 +6291,278 @@ mod tests { assert_eq!(decode_tree_object(&encoded)?, compressible); Ok(()) } + + fn publish_test_term_entries<'a>( + store: &'a MemoryStore, + graph: &GraphDocument, + entries: BTreeMap, Vec>, + ) -> Result, SnapshotError> { + let builder = GraphSnapshotBuilder::new(); + let mut content = builder.prepare_content(store, graph)?; + let count = entries.len() as u64; + let mut writer = ObjectWriter::new(store)?; + let digest = build_index_tree(&mut writer, IndexKind::Terms, entries)?; + let _ = writer.finish()?; + let root = content + .roots + .iter_mut() + .find(|root| root.index == IndexKind::Terms) + .ok_or_else(|| SnapshotError::Corrupt("test term root missing".to_owned()))?; + root.digest = digest; + root.entry_count = count; + let (graph_digest, graph_bytes) = digest_canonical_graph(graph, false)?; + let prepared = builder.finish_content(store, content, graph_digest, graph_bytes)?; + builder.activate(store, &prepared)?; + GraphSnapshotReader::open_active(store)? + .ok_or_else(|| SnapshotError::Corrupt("test active snapshot missing".to_owned())) + } + + #[test] + fn legacy_empty_capability_postings_remain_readable() -> Result<(), SnapshotError> { + let graph = cache_fixture_graph(); + let postings = build_term_postings(&graph); + let mut entries = build_index(&graph, IndexKind::Terms, Some(&postings))?; + for capability in [ + IDENTIFIER_SUBWORD_INDEX_CAPABILITY_V1, + OPERATION_ROLE_TERM_INDEX_CAPABILITY_V1, + DECLARATION_TERM_INDEX_CAPABILITY_V1, + RELATIONSHIP_TERM_INDEX_CAPABILITY_V1, + RELATIONSHIP_SEARCH_TERM_INDEX_CAPABILITY_V1, + ] { + entries.remove(&term_capability_key(capability)?); + } + let store = MemoryStore::default(); + let reader = publish_test_term_entries(&store, &graph, entries)?; + assert!(reader.supports_identifier_subwords()?); + assert!(reader.supports_operation_role_terms()?); + assert!(reader.supports_declaration_terms()?); + assert!(reader.supports_relationship_terms()?); + assert!(reader.supports_relationship_search_terms()?); + assert_eq!(reader.export_graph()?, graph); + Ok(()) + } + + #[test] + fn explicit_capability_records_do_not_fall_back_when_malformed() -> Result<(), SnapshotError> { + for capability in [ + IDENTIFIER_SUBWORD_INDEX_CAPABILITY_V1, + OPERATION_ROLE_TERM_INDEX_CAPABILITY_V1, + DECLARATION_TERM_INDEX_CAPABILITY_V1, + RELATIONSHIP_TERM_INDEX_CAPABILITY_V1, + RELATIONSHIP_SEARCH_TERM_INDEX_CAPABILITY_V1, + ] { + for invalid in [b"2".as_slice(), b"{}", b"null"] { + let graph = cache_fixture_graph(); + let postings = build_term_postings(&graph); + let mut entries = build_index(&graph, IndexKind::Terms, Some(&postings))?; + entries.insert(term_capability_key(capability)?, invalid.to_vec()); + let store = MemoryStore::default(); + let reader = publish_test_term_entries(&store, &graph, entries)?; + let result = match capability { + IDENTIFIER_SUBWORD_INDEX_CAPABILITY_V1 => reader.supports_identifier_subwords(), + OPERATION_ROLE_TERM_INDEX_CAPABILITY_V1 => { + reader.supports_operation_role_terms() + } + DECLARATION_TERM_INDEX_CAPABILITY_V1 => reader.supports_declaration_terms(), + RELATIONSHIP_TERM_INDEX_CAPABILITY_V1 => reader.supports_relationship_terms(), + _ => reader.supports_relationship_search_terms(), + }; + assert!( + matches!( + result, + Err(SnapshotError::Corrupt(_) | SnapshotError::Decode(_)) + ), + "{capability}: {invalid:?}" + ); + } + } + Ok(()) + } + + #[test] + fn capability_named_source_terms_keep_all_posting_chunks() -> Result<(), SnapshotError> { + let mut graph = cache_fixture_graph(); + let template = graph + .nodes + .pop() + .ok_or_else(|| SnapshotError::Corrupt("test node missing".to_owned()))?; + let count = GRAPH_TERM_POSTING_CHUNK_ITEMS * 2 + 1; + let capability = IDENTIFIER_SUBWORD_INDEX_CAPABILITY_V1; + for at in 0..count { + let mut node = template.clone(); + node.id = format!("marker:{at:04}"); + node.name = format!("record_{at:04}"); + node.qualified_name = format!("crate::{capability}::record_{at:04}"); + graph.nodes.push(node); + } + let store = MemoryStore::default(); + let builder = GraphSnapshotBuilder::new(); + let prepared = builder.prepare(&store, &graph)?; + builder.activate(&store, &prepared)?; + let reader = GraphSnapshotReader::open_active(&store)? + .ok_or_else(|| SnapshotError::Corrupt("test active snapshot missing".to_owned()))?; + let limits = SnapshotReadLimits { + max_items: count + GRAPH_TERM_POSTING_CHUNK_ITEMS, + ..SnapshotReadLimits::default() + }; + let (nodes, truncated) = reader.nodes_for_terms(&[capability.to_owned()], limits)?; + assert!(!truncated); + assert_eq!(nodes.len(), count); + assert_eq!( + nodes.iter().map(|node| &node.id).collect::>(), + graph.nodes.iter().map(|node| &node.id).collect::>() + ); + assert!(reader.supports_identifier_subwords()?); + assert_eq!(reader.export_graph()?, graph); + Ok(()) + } + + #[test] + fn capability_markers_preserve_user_postings() -> Result<(), SnapshotError> { + for (capability, mode) in [ + (IDENTIFIER_SUBWORD_INDEX_CAPABILITY_V1, 0), + (OPERATION_ROLE_TERM_INDEX_CAPABILITY_V1, 1), + (DECLARATION_TERM_INDEX_CAPABILITY_V1, 2), + (RELATIONSHIP_TERM_INDEX_CAPABILITY_V1, 3), + (RELATIONSHIP_SEARCH_TERM_INDEX_CAPABILITY_V1, 4), + ] { + let mut graph = cache_fixture_graph(); + let owner = graph + .nodes + .first_mut() + .ok_or_else(|| SnapshotError::Corrupt("test owner missing".to_owned()))?; + match mode { + 0 => { + owner.kind = NodeKind::Constant; + owner.details = Some(NodeDetails::Symbol( + compass_model::code_graph::SymbolNodeDetails { + signature: Some(format!("const MARKER: &str = \"{capability}\";")), + modifiers: Vec::new(), + overload_discriminator: None, + declaring_type: None, + signature_digest: None, + implementation_digest: None, + source_digest: None, + call_sites: None, + }, + )); + } + 1 => { + owner.kind = NodeKind::Struct; + owner.name = "Builder".to_owned(); + owner.qualified_name = format!("crate::{capability}::Builder"); + } + 2 => { + owner.kind = NodeKind::Struct; + owner.name = capability.to_owned(); + owner.qualified_name = format!("crate::{capability}"); + } + _ => { + let mut target = owner.clone(); + target.id = "b".to_owned(); + target.name = capability.to_owned(); + target.qualified_name = format!("crate::{capability}"); + let site = owner.source.clone(); + let evidence = owner.evidence.clone(); + graph.nodes.push(target); + let id = compass_model::identity::edge_id( + "a", + EdgeKind::Calls, + "b", + site.as_ref(), + None, + ); + graph.links.push(EdgeRecord { + id: id.clone(), + key: id, + source: "a".to_owned(), + target: "b".to_owned(), + kind: EdgeKind::Calls, + occurrence_rule: None, + relationship_site: site, + details: None, + evidence, + weight: None, + context: None, + deferred: false, + diagnostics: Vec::new(), + }); + } + } + let store = MemoryStore::default(); + let builder = GraphSnapshotBuilder::new(); + let prepared = builder.prepare(&store, &graph)?; + builder.activate(&store, &prepared)?; + let reader = GraphSnapshotReader::open_active(&store)? + .ok_or_else(|| SnapshotError::Corrupt("test active snapshot missing".to_owned()))?; + assert_eq!(reader.export_graph()?, graph, "{capability}"); + assert!(reader.supports_identifier_subwords()?, "{capability}"); + assert!(reader.supports_operation_role_terms()?, "{capability}"); + assert!(reader.supports_declaration_terms()?, "{capability}"); + assert!(reader.supports_relationship_terms()?, "{capability}"); + assert!(reader.supports_relationship_search_terms()?, "{capability}"); + let limits = SnapshotReadLimits { + max_items: 1024, + ..SnapshotReadLimits::default() + }; + let (nodes, truncated) = reader.nodes_for_terms(&[capability.to_owned()], limits)?; + assert!(!truncated, "{capability}"); + let expected = if mode >= 3 { "b" } else { "a" }; + assert_eq!( + nodes + .iter() + .map(|node| node.id.as_str()) + .collect::>(), + [expected], + "{capability}" + ); + match mode { + 1 => { + let (nodes, truncated, _) = reader + .operation_role_nodes_for_terms_bounded_work( + &[capability.to_owned()], + limits, + )?; + assert!(!truncated); + assert_eq!( + nodes + .iter() + .map(|node| node.id.as_str()) + .collect::>(), + ["a"] + ); + } + 2 => { + let (nodes, truncated, _) = reader.declaration_nodes_for_terms_bounded_work( + &[capability.to_owned()], + limits, + )?; + assert!(!truncated); + assert_eq!( + nodes + .iter() + .map(|node| node.id.as_str()) + .collect::>(), + ["a"] + ); + } + 3 => { + let (ids, truncated, _) = reader + .source_ids_for_exact_relationship_term_bounded_work(capability, limits)?; + assert!(!truncated); + assert_eq!(ids, ["a"]); + } + 4 => { + let (ids, truncated, _) = reader + .source_ids_for_exact_relationship_search_term_bounded_work( + capability, limits, + )?; + assert!(!truncated); + assert_eq!(ids, ["a"]); + } + _ => {} + } + } + Ok(()) + } } diff --git a/crates/compass-graph/src/v1.rs b/crates/compass-graph/src/v1.rs index b11ba4f8a..be21b7b2b 100644 --- a/crates/compass-graph/src/v1.rs +++ b/crates/compass-graph/src/v1.rs @@ -23,6 +23,7 @@ use compass_model::identity::{ use compass_model::provenance::{ COALESCED_NODE_EVIDENCE_ATTRIBUTE, CONSUME_INCREMENTAL_ENDPOINT_REMAP_ATTRIBUTE, ENDPOINT_REWRITE_RULES_ATTRIBUTE, EndpointRewriteRule, EvidenceConfidence, EvidenceOrigin, + LEADING_DOCUMENTATION_OWNER_ANCHOR_ATTRIBUTE, LEADING_DOCUMENTATION_RULE, NODE_PROVENANCE_ANCHOR_ATTRIBUTE, OCCURRENCE_RULE_ATTRIBUTE, OccurrenceRule, Provenance, ResolutionCandidate, ResolutionState, SEMANTIC_LAYER_EXTRACTOR, SourceAnchor, TRUSTED_EDGE_RECORD_ATTRIBUTE, TRUSTED_NODE_RECORD_ATTRIBUTE, @@ -38,7 +39,7 @@ use crate::inference::{InferenceLevel, prefilter_extraction_inference}; use crate::quarantine::{PublicationOutcome, QuarantineCollector}; /// Normalization/publication semantics used by `compass.graph/1`. -pub const V1_PUBLICATION_SEMANTICS_VERSION: &str = "compass.graph.publication/2"; +pub const V1_PUBLICATION_SEMANTICS_VERSION: &str = "compass.graph.publication/3"; const MAX_DOCUMENT_REFERENCE_CANDIDATES: usize = 20; const MAX_DOCUMENT_REFERENCE_PROBES: usize = 100_000; use sha2::{Digest, Sha256}; @@ -4693,6 +4694,27 @@ fn normalize_provenance( } else if let Some(anchor) = anchor { provenance.anchors.push(anchor); } + if let Some(value) = attributes.get(LEADING_DOCUMENTATION_OWNER_ANCHOR_ATTRIBUTE) { + if provenance.rule.as_deref() != Some(LEADING_DOCUMENTATION_RULE) + || provenance.origin != EvidenceOrigin::Ast + || provenance.confidence != EvidenceConfidence::Exact + || provenance.anchors.len() != 1 + { + return Err(raw_error( + record, + "invalid leading documentation owner payload", + )); + } + let mut owner = structured_source_anchor(value)?; + owner.file = portable_path(&owner.file, root)?; + if !owner.is_valid() { + return Err(raw_error( + record, + "invalid leading documentation owner anchor", + )); + } + provenance.anchors.push(owner); + } provenance .validate() .map_err(|error| raw_error(record, &error.to_string()))?; diff --git a/crates/compass-languages/src/engine.rs b/crates/compass-languages/src/engine.rs index 73169e335..5f23149e7 100644 --- a/crates/compass-languages/src/engine.rs +++ b/crates/compass-languages/src/engine.rs @@ -440,6 +440,13 @@ impl Engine { if pipeline.is_some() && extraction.semantic_evidence.is_some() { extraction.raw_calls = None; } + crate::source_query::attach( + &mut extraction, + evidence_source_file, + spec.name, + source, + root, + ); if root.has_error() { extraction.extensions.insert( EXTRACTION_QUALITY_EXTENSION.to_owned(), diff --git a/crates/compass-languages/src/evidence/build.rs b/crates/compass-languages/src/evidence/build.rs index 92d4e4383..3a3cc8390 100644 --- a/crates/compass-languages/src/evidence/build.rs +++ b/crates/compass-languages/src/evidence/build.rs @@ -8,8 +8,9 @@ use tree_sitter::Node; use crate::{EXTRACTION_SEMANTICS_VERSION, UniversalEvidencePipeline, file_stem, make_id}; use super::model::{ - BindingFact, BindingKind, CandidateRelation, DeclarationFact, EvidenceDiagnostic, - EvidenceRange, HierarchyConstraint, OccurrenceFact, ReceiverDispatchStrategy, + BindingFact, BindingKind, CandidateRelation, DECLARATION_DOCUMENTATION_OMISSION_CODE, + DeclarationDocumentationFact, DeclarationFact, EvidenceDiagnostic, EvidenceRange, + HierarchyConstraint, MAX_BATCH_DOCUMENTATION_BYTES, OccurrenceFact, ReceiverDispatchStrategy, RelationshipCandidate, ResolutionConstraint, ScopeFact, SemanticEvidenceBatch, SemanticRole, SymbolNamespace, UniversalEvidenceIdentity, }; @@ -29,6 +30,8 @@ pub struct EvidenceBuilder { batch: SemanticEvidenceBatch, source_file: String, limits: EvidenceLimits, + documentation_bytes: usize, + documentation_omission_reported: bool, } #[derive(Default)] @@ -41,6 +44,7 @@ struct DeclarationMetadata { signature_hash: Option, implementation_hash: Option, source_hash: Option, + documentation: Option, } impl EvidenceBuilder { @@ -83,6 +87,8 @@ impl EvidenceBuilder { }, source_file: source_file.into(), limits, + documentation_bytes: 0, + documentation_omission_reported: false, } } @@ -154,9 +160,38 @@ impl EvidenceBuilder { namespace: Option, signature: Option<&str>, range: EvidenceRange, + ) -> Result { + self.declare_with_signature_and_documentation( + kind, + graph_node_id, + name, + qualified_name, + module_or_package, + scope_id, + namespace, + signature, + None, + range, + ) + } + + #[allow(clippy::too_many_arguments)] + pub(crate) fn declare_with_signature_and_documentation( + &mut self, + kind: &str, + graph_node_id: &str, + name: &str, + qualified_name: &str, + module_or_package: Option<&str>, + scope_id: Option<&str>, + namespace: Option, + signature: Option<&str>, + documentation: Option, + range: EvidenceRange, ) -> Result { let metadata = DeclarationMetadata { signature: signature.map(str::to_owned), + documentation, ..DeclarationMetadata::default() }; self.declare_with_metadata_and_namespace( @@ -289,13 +324,28 @@ impl EvidenceBuilder { scope_id: Option<&str>, range: EvidenceRange, namespace: Option, - metadata: DeclarationMetadata, + mut metadata: DeclarationMetadata, ) -> Result { ensure_capacity( "declarations", self.batch.declarations.len(), self.limits.declarations, )?; + if let Some(documentation) = metadata.documentation.take() { + match self + .documentation_bytes + .checked_add(documentation.content.len()) + { + Some(next) if next <= MAX_BATCH_DOCUMENTATION_BYTES => { + self.documentation_bytes = next; + metadata.documentation = Some(documentation); + } + _ => self.record_documentation_omission( + documentation.range, + "the per-file aggregate documentation byte ceiling was reached", + )?, + } + } let start_byte = range.start_byte.to_string(); let end_byte = range.end_byte.to_string(); let mut identity = vec![ @@ -329,12 +379,49 @@ impl EvidenceBuilder { signature_hash: metadata.signature_hash, implementation_hash: metadata.implementation_hash, source_hash: metadata.source_hash, + documentation: metadata.documentation, definition_start_byte: None, range, }); Ok(id) } + pub(crate) fn leading_documentation( + &mut self, + node: Node<'_>, + source: &[u8], + ) -> Result, EvidenceError> { + match super::documentation::leading_documentation( + &self.source_file, + source, + node, + &self.batch.pipeline.language, + ) { + Ok(documentation) => Ok(documentation), + Err(omission) => { + self.record_documentation_omission(omission.range, omission.message)?; + Ok(None) + } + } + } + + fn record_documentation_omission( + &mut self, + range: EvidenceRange, + message: &str, + ) -> Result<(), EvidenceError> { + if !self.documentation_omission_reported { + self.diagnose( + DECLARATION_DOCUMENTATION_OMISSION_CODE, + None, + Some(range), + &format!("Some leading documentation was omitted: {message}. Other groups may also be absent"), + )?; + self.documentation_omission_reported = true; + } + Ok(()) + } + pub fn open_scope( &mut self, kind: &str, @@ -1227,6 +1314,7 @@ impl<'source> DirectEvidenceState<'source> { .source .get(node.start_byte()..node.end_byte()) .map(evidence_normalized_source_hash), + documentation: None, } } @@ -4180,6 +4268,7 @@ impl<'source> DirectEvidenceState<'source> { let graph_node_id = self.unique_graph_id(make_id(&[&self.module_or_package, &qualified_name]), node); let mut metadata = self.declaration_metadata(node); + metadata.documentation = self.builder.leading_documentation(node, self.source)?; if kind == "trait" { metadata.direct_bases_complete = !self.overlaps_parser_error(node); } @@ -4438,7 +4527,8 @@ impl<'source> DirectEvidenceState<'source> { let qualified_name = rust_join_qualified(&qualified_owner, &name); let graph_node_id = self.unique_graph_id(make_id(&[&self.module_or_package, &qualified_name]), node); - let metadata = self.declaration_metadata(node); + let mut metadata = self.declaration_metadata(node); + metadata.documentation = self.builder.leading_documentation(node, self.source)?; let parent_scope = active_impl.map_or(owner.scope_id.as_str(), |value| value.scope_id.as_str()); let fact_id = self.builder.declare_with_metadata( @@ -7440,7 +7530,8 @@ impl<'source> DirectEvidenceState<'source> { } else { "type_alias" }; - let metadata = self.declaration_metadata(node); + let mut metadata = self.declaration_metadata(node); + metadata.documentation = self.builder.leading_documentation(node, self.source)?; let fact_id = self.builder.declare_with_metadata( kind, &graph_node_id, @@ -7510,6 +7601,7 @@ impl<'source> DirectEvidenceState<'source> { }; let mut metadata = self.declaration_metadata(node); let (parameter_count, variadic) = go_parameter_signature(node); + metadata.documentation = self.builder.leading_documentation(node, self.source)?; metadata.parameter_count = Some(parameter_count); metadata.variadic = variadic; let fact_id = self.builder.declare_with_metadata( diff --git a/crates/compass-languages/src/evidence/documentation.rs b/crates/compass-languages/src/evidence/documentation.rs new file mode 100644 index 000000000..f2ecc68b9 --- /dev/null +++ b/crates/compass-languages/src/evidence/documentation.rs @@ -0,0 +1,184 @@ +//! Language-specific AST attachment for bounded leading documentation. +use tree_sitter::Node; + +use super::build::range_for_node; +use super::model::{ + DeclarationDocumentationFact, DeclarationDocumentationKind, EvidenceRange, + MAX_DECLARATION_DOCUMENTATION_BYTES, +}; + +const MAX_ATTACHMENT_NODES: usize = 1024; +const MAX_ATTACHMENT_SPAN_BYTES: usize = 64 * 1024; + +#[derive(Debug)] +pub(super) struct DocumentationOmission { + pub range: EvidenceRange, + pub message: &'static str, +} + +pub(super) fn leading_documentation( + source_file: &str, + source: &[u8], + declaration: Node<'_>, + language: &str, +) -> Result, DocumentationOmission> { + let kind = match language { + "rust" => DeclarationDocumentationKind::RustOuter, + "go" => DeclarationDocumentationKind::GoLeading, + "typescript" => DeclarationDocumentationKind::TypeScriptJsDoc, + _ => return Ok(None), + }; + if declaration.has_error() || declaration.is_missing() { + return Ok(None); + } + let mut owner = declaration; + if language == "go" && matches!(owner.kind(), "type_spec" | "type_alias") { + let Some(parent) = owner + .parent() + .filter(|parent| parent.kind() == "type_declaration") + else { + return Ok(None); + }; + let mut cursor = parent.walk(); + let count = parent + .named_children(&mut cursor) + .filter(|child| matches!(child.kind(), "type_spec" | "type_alias")) + .take(2) + .count(); + if count != 1 { + return Ok(None); + } + owner = parent; + } + if language == "typescript" + && let Some(parent) = owner + .parent() + .filter(|parent| parent.kind() == "export_statement") + && parent + .child_by_field_name("declaration") + .is_some_and(|child| child.id() == owner.id()) + { + owner = parent; + } + if owner.has_error() || owner.is_missing() { + return Ok(None); + } + let mut comments = Vec::new(); + let mut crossed_attributes_after_comments = false; + let mut boundary = owner; + let mut previous = boundary.prev_named_sibling(); + let mut steps = 0; + while let Some(node) = previous { + steps += 1; + if steps > MAX_ATTACHMENT_NODES + || owner.start_byte().saturating_sub(node.start_byte()) > MAX_ATTACHMENT_SPAN_BYTES + { + return Err(DocumentationOmission { + range: range_for_node(source_file, declaration), + message: "leading documentation attachment exceeded its bounded AST walk", + }); + } + if node.has_error() || node.is_missing() { + break; + } + let Some(gap) = source.get(node.end_byte()..boundary.start_byte()) else { + return Err(DocumentationOmission { + range: range_for_node(source_file, declaration), + message: "leading documentation attachment has an invalid source interval", + }); + }; + if !gap.iter().all(u8::is_ascii_whitespace) { + break; + } + let attribute = (language == "rust" && node.kind() == "attribute_item") + || (language == "typescript" && node.kind() == "decorator"); + if attribute { + if !comments.is_empty() { + crossed_attributes_after_comments = true; + } + boundary = node; + previous = node.prev_named_sibling(); + continue; + } + if !is_documentation(node, language, source) { + break; + } + if crossed_attributes_after_comments { + return Err(DocumentationOmission { + range: range_for_node(source_file, declaration), + message: "multiple documentation groups separated by attributes are outside this attachment profile", + }); + } + if language == "go" + && (node.end_position().row.checked_add(1) != Some(boundary.start_position().row) + || !starts_own_line(node, source)) + { + break; + } + comments.push(node); + boundary = node; + previous = node.prev_named_sibling(); + } + let (Some(first), Some(last)) = (comments.last(), comments.first()) else { + return Ok(None); + }; + let start = first.start_byte(); + let end = last.end_byte(); + let mut range = range_for_node(source_file, *first); + let last_range = range_for_node(source_file, *last); + range.end_byte = last_range.end_byte; + range.end_line = last_range.end_line; + range.end_column = last_range.end_column; + if end.saturating_sub(start) > MAX_DECLARATION_DOCUMENTATION_BYTES { + return Err(DocumentationOmission { + range, + message: "leading documentation exceeds the 4096-byte source excerpt limit", + }); + } + let Some(bytes) = source.get(start..end) else { + return Err(DocumentationOmission { + range, + message: "leading documentation source range is unavailable", + }); + }; + let Ok(content) = std::str::from_utf8(bytes) else { + return Err(DocumentationOmission { + range, + message: "leading documentation is not valid UTF-8 source text", + }); + }; + Ok(Some(DeclarationDocumentationFact { + kind, + range, + content: content.to_owned(), + })) +} + +fn is_documentation(node: Node<'_>, language: &str, source: &[u8]) -> bool { + match language { + "rust" => { + matches!(node.kind(), "line_comment" | "block_comment") + && node.child_by_field_name("outer").is_some() + && node.child_by_field_name("inner").is_none() + } + "go" => node.kind() == "comment", + "typescript" => { + node.kind() == "comment" + && source + .get(node.start_byte()..node.end_byte()) + .is_some_and(|text| text.starts_with(b"/**") && text.ends_with(b"*/")) + } + _ => false, + } +} + +fn starts_own_line(node: Node<'_>, source: &[u8]) -> bool { + let Some(prefix) = source.get(..node.start_byte()) else { + return false; + }; + let start = prefix + .iter() + .rposition(|byte| *byte == b'\n') + .map_or(0, |position| position.saturating_add(1)); + prefix[start..].iter().all(u8::is_ascii_whitespace) +} diff --git a/crates/compass-languages/src/evidence/mod.rs b/crates/compass-languages/src/evidence/mod.rs index 40decb565..078a1138f 100644 --- a/crates/compass-languages/src/evidence/mod.rs +++ b/crates/compass-languages/src/evidence/mod.rs @@ -1,6 +1,7 @@ mod build; mod csharp; mod dart; +mod documentation; mod groovy; mod kotlin; mod model; @@ -17,8 +18,10 @@ pub const UNIVERSAL_EVIDENCE_SCHEMA: &str = "compass.languages.evidence/2"; pub(crate) use build::extract_tree_evidence; pub use build::{EvidenceBuilder, range_for_node}; pub use model::{ - BindingFact, BindingKind, CandidateRelation, DeclarationFact, EvidenceDiagnostic, - EvidenceRange, HierarchyConstraint, LanguageCapability, OccurrenceFact, + BindingFact, BindingKind, CandidateRelation, DECLARATION_DOCUMENTATION_OMISSION_CODE, + DeclarationDocumentationFact, DeclarationDocumentationKind, DeclarationFact, + EvidenceDiagnostic, EvidenceRange, HierarchyConstraint, LanguageCapability, + MAX_BATCH_DOCUMENTATION_BYTES, MAX_DECLARATION_DOCUMENTATION_BYTES, OccurrenceFact, ReceiverDispatchStrategy, RelationshipCandidate, ResolutionConstraint, ScopeFact, SemanticEvidenceBatch, SemanticRole, SymbolNamespace, UniversalEvidenceIdentity, }; diff --git a/crates/compass-languages/src/evidence/model.rs b/crates/compass-languages/src/evidence/model.rs index 7bb9dffdb..a331b743a 100644 --- a/crates/compass-languages/src/evidence/model.rs +++ b/crates/compass-languages/src/evidence/model.rs @@ -201,6 +201,33 @@ impl CandidateRelation { } } +/// Maximum retained source text for one declaration's leading documentation. +pub const MAX_DECLARATION_DOCUMENTATION_BYTES: usize = 4_096; +/// Aggregate retained documentation ceiling for one per-file evidence batch. +pub const MAX_BATCH_DOCUMENTATION_BYTES: usize = 1024 * 1024; +/// One bounded per-file warning marks omitted leading documentation. +pub const DECLARATION_DOCUMENTATION_OMISSION_CODE: &str = "leading_documentation_omitted"; + +/// Syntax attachment established by a native language producer. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum DeclarationDocumentationKind { + RustOuter, + GoLeading, + #[serde(rename = "typescript_jsdoc")] + TypeScriptJsDoc, +} + +/// One contiguous, byte-exact leading documentation group. Its owner is the +/// enclosing declaration, never a separately resolved spelling or nearby name. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct DeclarationDocumentationFact { + pub kind: DeclarationDocumentationKind, + pub range: EvidenceRange, + pub content: String, +} + /// One source declaration that may become a graph node. #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] @@ -240,6 +267,10 @@ pub struct DeclarationFact { pub implementation_hash: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub source_hash: Option, + /// Optional AST-attached source documentation. Historical batches without + /// this additive metadata retain their original meaning. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub documentation: Option, /// Full syntax-node start for legacy callable identities. The canonical /// evidence range may intentionally begin at the identifier, while the /// public graph identity historically used the enclosing declaration diff --git a/crates/compass-languages/src/evidence/typescript.rs b/crates/compass-languages/src/evidence/typescript.rs index 29386bc07..e5fa2f287 100644 --- a/crates/compass-languages/src/evidence/typescript.rs +++ b/crates/compass-languages/src/evidence/typescript.rs @@ -1404,7 +1404,16 @@ impl<'source, 'tree> CandidateState<'source, 'tree> { let graph_node_id = stable_graph_id(self.source_file, kind, name, node.start_byte()); let signature = typescript_declaration_signature(node, kind, self.source) .map(|signature| self.canonicalize_declaration_signature(scope_id, kind, &signature)); - let declaration_id = self.builder.declare_with_signature( + let documentation = if self.language == "typescript" + && matches!( + kind, + "function" | "method" | "class" | "interface" | "enum" | "type_alias" + ) { + self.builder.leading_documentation(node, self.source)? + } else { + None + }; + let declaration_id = self.builder.declare_with_signature_and_documentation( kind, &graph_node_id, name, @@ -1413,6 +1422,7 @@ impl<'source, 'tree> CandidateState<'source, 'tree> { Some(scope_id), Some(symbol_namespace(namespace)), signature.as_deref(), + documentation, range_for_node(self.source_file, range_node), )?; self.definition_start_bytes @@ -9877,11 +9887,21 @@ impl<'source, 'tree> CandidateState<'source, 'tree> { } fn scope_for_owner_declaration(&self, declaration_id: &str) -> Option { - self.scope_owners + let declaration = self.declarations.get(declaration_id)?; + // Nested blocks and callable types inherit this owner as well. Only + // the directly owned child of the declaration's binding scope is its + // member scope; hashed IDs cannot choose between structural scopes. + let mut scopes = self + .scope_owners .iter() - .filter(|(_, owner)| owner.as_str() == declaration_id) - .map(|(scope, _)| scope.clone()) - .min() + .filter(|(scope, owner)| { + owner.as_str() == declaration_id + && self.scope_parents.get(*scope).and_then(Option::as_ref) + == Some(&declaration.scope_id) + }) + .map(|(scope, _)| scope); + let scope = scopes.next()?; + scopes.next().is_none().then(|| scope.clone()) } fn scope_for_qualified_nominal(&self, qualified_name: &str) -> Option { @@ -10222,6 +10242,11 @@ fn declaration_shape<'tree>( }; let name = node .child_by_field_name("name") + .or_else(|| { + (node.kind() == "field_definition") + .then(|| node.child_by_field_name("property")) + .flatten() + }) .or_else(|| { matches!(node.kind(), "pair" | "enum_assignment") .then(|| node.child_by_field_name("key")) diff --git a/crates/compass-languages/src/evidence/validate.rs b/crates/compass-languages/src/evidence/validate.rs index 215d47956..f9cc7ddb5 100644 --- a/crates/compass-languages/src/evidence/validate.rs +++ b/crates/compass-languages/src/evidence/validate.rs @@ -5,9 +5,10 @@ use ahash::AHashMap; use serde::{Deserialize, Serialize}; use super::model::{ - BindingFact, CandidateRelation, DeclarationFact, EvidenceRange, HierarchyConstraint, - LanguageCapability, OccurrenceFact, RelationshipCandidate, ScopeFact, SemanticEvidenceBatch, - SemanticRole, SymbolNamespace, + BindingFact, CandidateRelation, DeclarationDocumentationKind, DeclarationFact, EvidenceRange, + HierarchyConstraint, LanguageCapability, MAX_BATCH_DOCUMENTATION_BYTES, + MAX_DECLARATION_DOCUMENTATION_BYTES, OccurrenceFact, RelationshipCandidate, ScopeFact, + SemanticEvidenceBatch, SemanticRole, SymbolNamespace, }; /// Hard resource ceilings for a single semantic evidence batch. @@ -264,6 +265,33 @@ fn validate_limits( format!("{name} count {count} exceeds limit {limit}"), )); } + let mut documentation_bytes = 0_usize; + for documentation in batch + .declarations + .iter() + .filter_map(|declaration| declaration.documentation.as_ref()) + { + if documentation.content.len() > MAX_DECLARATION_DOCUMENTATION_BYTES { + return Err(EvidenceError::new( + EvidenceErrorCode::ResourceLimit, + "declaration documentation exceeds the 4096-byte source excerpt limit", + )); + } + documentation_bytes = documentation_bytes + .checked_add(documentation.content.len()) + .ok_or_else(|| { + EvidenceError::new( + EvidenceErrorCode::ResourceLimit, + "documentation byte count overflows usize", + ) + })?; + if documentation_bytes > MAX_BATCH_DOCUMENTATION_BYTES { + return Err(EvidenceError::new( + EvidenceErrorCode::ResourceLimit, + "documentation exceeds the per-file aggregate byte limit", + )); + } + } Ok(()) } @@ -312,6 +340,7 @@ fn validate_fact( return Err(invalid_fact(&fact.id, "declaration identity is empty")); } require_optional_reference(&fact.id, "scope", fact.scope_id.as_deref(), scopes)?; + validate_declaration_documentation(fact)?; validate_callable_types( &fact.id, &fact.parameter_types, @@ -813,6 +842,87 @@ fn validate_language( Ok(()) } +fn validate_declaration_documentation(fact: &DeclarationFact) -> Result<(), EvidenceError> { + let Some(documentation) = &fact.documentation else { + return Ok(()); + }; + validate_range(&documentation.range, &fact.id, false)?; + let language_matches = matches!( + (documentation.kind, fact.language.as_str()), + (DeclarationDocumentationKind::RustOuter, "rust") + | (DeclarationDocumentationKind::GoLeading, "go") + | (DeclarationDocumentationKind::TypeScriptJsDoc, "typescript") + ); + if !language_matches { + return Err(invalid_fact( + &fact.id, + "documentation attachment kind does not match its declaration language", + )); + } + if !matches!( + fact.kind.as_str(), + "function" + | "method" + | "module" + | "class" + | "struct" + | "interface" + | "trait" + | "enum" + | "type_alias" + ) { + return Err(invalid_fact( + &fact.id, + "leading documentation requires a supported named declaration", + )); + } + let range = &documentation.range; + if range.source_file != fact.range.source_file + || range.end_byte > fact.range.start_byte + || (range.end_line, range.end_column) > (fact.range.start_line, fact.range.start_column) + { + return Err(invalid_fact( + &fact.id, + "leading documentation must precede its declaration in the same source file", + )); + } + if documentation.content.is_empty() + || range.end_byte.saturating_sub(range.start_byte) != documentation.content.len() as u64 + { + return Err(invalid_fact( + &fact.id, + "documentation text must be the nonempty byte-exact recorded source range", + )); + } + let newline_count = documentation + .content + .bytes() + .filter(|byte| *byte == b'\n') + .count(); + let newline_count = u32::try_from(newline_count) + .map_err(|_| invalid_fact(&fact.id, "documentation line count overflows u32"))?; + let last_column_bytes = documentation + .content + .rsplit_once('\n') + .map_or(documentation.content.len(), |(_, last)| last.len()); + let last_column = u32::try_from(last_column_bytes) + .map_err(|_| invalid_fact(&fact.id, "documentation column overflows u32"))?; + let expected_end_column = if newline_count == 0 { + range.start_column.checked_add(last_column) + } else { + Some(last_column) + }; + if range.start_line.checked_add(newline_count) != Some(range.end_line) + || expected_end_column != Some(range.end_column) + { + return Err(invalid_fact( + &fact.id, + "documentation text does not match its recorded line and byte-column extent", + )); + } + Ok(()) +} + fn validate_range( range: &EvidenceRange, id: &str, diff --git a/crates/compass-languages/src/evidence_pipeline.rs b/crates/compass-languages/src/evidence_pipeline.rs index 13863a565..d756c05c3 100644 --- a/crates/compass-languages/src/evidence_pipeline.rs +++ b/crates/compass-languages/src/evidence_pipeline.rs @@ -449,11 +449,11 @@ const UNIVERSAL_EVIDENCE_PIPELINES: &[UniversalEvidencePipeline] = &[ producer: UniversalEvidenceProducer { id: "compass.go", language: "go", - version: 1, + version: 2, evidence_schema: crate::UNIVERSAL_EVIDENCE_SCHEMA, capabilities: GO_CAPABILITIES, }, - qualification: UniversalEvidenceQualification::Qualified, + qualification: UniversalEvidenceQualification::Qualifying, }, GROOVY_EVIDENCE_PIPELINE, UniversalEvidencePipeline { @@ -470,11 +470,11 @@ const UNIVERSAL_EVIDENCE_PIPELINES: &[UniversalEvidencePipeline] = &[ producer: UniversalEvidenceProducer { id: "compass.javascript", language: "javascript", - version: 1, + version: 3, evidence_schema: crate::UNIVERSAL_EVIDENCE_SCHEMA, capabilities: JAVASCRIPT_CAPABILITIES, }, - qualification: UniversalEvidenceQualification::Qualified, + qualification: UniversalEvidenceQualification::Qualifying, }, UniversalEvidencePipeline { producer: UniversalEvidenceProducer { @@ -511,11 +511,11 @@ const UNIVERSAL_EVIDENCE_PIPELINES: &[UniversalEvidencePipeline] = &[ producer: UniversalEvidenceProducer { id: "compass.rust", language: "rust", - version: 2, + version: 3, evidence_schema: crate::UNIVERSAL_EVIDENCE_SCHEMA, capabilities: RUST_CAPABILITIES, }, - qualification: UniversalEvidenceQualification::Qualified, + qualification: UniversalEvidenceQualification::Qualifying, }, SCALA_EVIDENCE_PIPELINE, SWIFT_EVIDENCE_PIPELINE, @@ -523,10 +523,10 @@ const UNIVERSAL_EVIDENCE_PIPELINES: &[UniversalEvidencePipeline] = &[ producer: UniversalEvidenceProducer { id: "compass.typescript", language: "typescript", - version: 1, + version: 4, evidence_schema: crate::UNIVERSAL_EVIDENCE_SCHEMA, capabilities: TYPESCRIPT_CAPABILITIES, }, - qualification: UniversalEvidenceQualification::Qualified, + qualification: UniversalEvidenceQualification::Qualifying, }, ]; diff --git a/crates/compass-languages/src/lib.rs b/crates/compass-languages/src/lib.rs index 43cf30947..4c232ed84 100644 --- a/crates/compass-languages/src/lib.rs +++ b/crates/compass-languages/src/lib.rs @@ -17,7 +17,7 @@ mod fortran; pub mod frameworks; /// Version of the extraction contract consumed by graph publication. -pub const EXTRACTION_SEMANTICS_VERSION: &str = "compass.languages.extraction/4"; +pub const EXTRACTION_SEMANTICS_VERSION: &str = "compass.languages.extraction/5"; mod go; mod html; mod ids; @@ -37,6 +37,7 @@ mod r; mod registry; mod scip; mod semantic; +pub mod source_query; mod sql; mod templates; mod terraform; @@ -47,9 +48,11 @@ mod zig; #[doc(hidden)] pub use builtins::{is_language_builtin_global, is_language_builtin_qualified_target}; pub use evidence::{ - BindingFact, BindingKind, CandidateRelation, DeclarationFact, EvidenceBuilder, + BindingFact, BindingKind, CandidateRelation, DECLARATION_DOCUMENTATION_OMISSION_CODE, + DeclarationDocumentationFact, DeclarationDocumentationKind, DeclarationFact, EvidenceBuilder, EvidenceDiagnostic, EvidenceError, EvidenceErrorCode, EvidenceLimits, EvidenceRange, - HierarchyConstraint, LanguageCapability, OccurrenceFact, ReceiverDispatchStrategy, + HierarchyConstraint, LanguageCapability, MAX_BATCH_DOCUMENTATION_BYTES, + MAX_DECLARATION_DOCUMENTATION_BYTES, OccurrenceFact, ReceiverDispatchStrategy, RelationshipCandidate, ResolutionConstraint, ScopeFact, SemanticEvidenceBatch, SemanticRole, SymbolNamespace, UNIVERSAL_EVIDENCE_SCHEMA, UniversalEvidenceIdentity, range_for_node, validate_evidence, diff --git a/crates/compass-languages/src/source_query.rs b/crates/compass-languages/src/source_query.rs new file mode 100644 index 000000000..c8bb3dcbc --- /dev/null +++ b/crates/compass-languages/src/source_query.rs @@ -0,0 +1,1086 @@ +//! Bounded lexical evidence from native callable scopes. No name resolution or graph edges. +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::collections::BTreeMap; +use tree_sitter::Node; + +/// Reserved per-file extraction metadata. This is query evidence, not semantic facts. +pub const EXTRACTION_EXTENSION: &str = "_compass_source_query_evidence"; +pub const EXTRACTION_SCHEMA: &str = "compass.source-query.extraction/1"; +pub const PROJECTION_PROFILE: &str = "source-query/4-nested-callable-context"; + +/// Byte columns and zero-based rows, matching Tree-sitter's source positions. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct Point { + pub row: usize, + pub column: usize, +} +impl From for Point { + fn from(point: tree_sitter::Point) -> Self { + Self { + row: point.row, + column: point.column, + } + } +} + +/// Portable cached producer output. Consumers must bind it to the exact source +/// digest and a unique published callable definition range before indexing it. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct SourceQueryExtraction { + pub schema: String, + pub profile: String, + pub source_file: String, + pub source_digest: String, + pub byte_size: u64, + pub language: String, + pub syntax_recovery: bool, + pub omission: Option, + pub projection: Projection, +} + +pub(crate) fn attach( + extraction: &mut crate::Extraction, + source_file: &str, + language: &str, + source: &[u8], + root: Node<'_>, +) { + if !matches!(language, "python" | "rust" | "go" | "typescript" | "tsx") { + return; + } + let (projection, omission) = match project(root, source, language, Limits::default()) { + Ok(projection) => (projection, None), + Err(reason) => ( + Projection { + truncated: true, + ..Projection::default() + }, + Some(reason.to_owned()), + ), + }; + let evidence = SourceQueryExtraction { + schema: EXTRACTION_SCHEMA.to_owned(), + profile: PROJECTION_PROFILE.to_owned(), + source_file: source_file.to_owned(), + source_digest: format!("sha256:{:x}", Sha256::digest(source)), + byte_size: u64::try_from(source.len()).unwrap_or(u64::MAX), + language: language.to_owned(), + syntax_recovery: root.has_error(), + omission, + projection, + }; + match serde_json::to_value(evidence) { + Ok(value) => { + extraction + .extensions + .insert(EXTRACTION_EXTENSION.to_owned(), value); + } + Err(error) => { + extraction.error = Some(format!( + "source-query evidence serialization failed: {error}" + )); + } + } +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct Limits { + pub file_bytes: usize, + pub visits: usize, + pub depth: usize, + pub scopes: usize, + pub names_per_scope: usize, + pub sites_per_name: usize, + pub comments_per_scope: usize, + pub comment_bytes: usize, + pub retained_bytes: usize, + pub code_visits: usize, +} +impl Default for Limits { + fn default() -> Self { + Self { + file_bytes: 4 * 1024 * 1024, + visits: 200_000, + depth: 512, + scopes: 512, + names_per_scope: 256, + // Recall needs each spelling once, not repeated occurrences that + // spend the publication site budget without adding searchable terms. + // Code windows are collected independently before this site limit. + sites_per_name: 1, + comments_per_scope: 8, + comment_bytes: 4096, + retained_bytes: 1024 * 1024, + code_visits: 200_000, + } + } +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct Site { + pub start: usize, + pub end: usize, + pub start_point: Point, + pub end_point: Point, + pub text: String, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct Scope { + pub name: Option, + pub syntax: String, + pub start: usize, + pub end: usize, + /// Explicit native AST ancestry; class/type barriers leave no parent. + pub parent_scope: Option, + /// Actual callable syntax, distinct from a variable definition envelope. + pub callable_start: usize, + pub callable_end: usize, + pub start_point: Point, + pub end_point: Point, + pub names: BTreeMap>, + pub comments: Vec, + #[serde(default)] + pub code_windows: Vec, + #[serde(default)] + pub omitted_code_windows: u64, + pub omitted_sites: usize, + pub truncated: bool, + #[serde(skip)] + name_span: Option<(usize, usize)>, +} +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct Projection { + pub scopes: Vec, + pub visits: usize, + pub retained_bytes: usize, + pub truncated: bool, + #[serde(default)] + pub retained_code_bytes: usize, +} +fn callable(language: &str, kind: &str) -> bool { + match language { + "python" => matches!(kind, "function_definition" | "lambda"), + "rust" => matches!( + kind, + "function_item" | "function_signature_item" | "closure_expression" + ), + "go" => matches!( + kind, + "function_declaration" | "method_declaration" | "func_literal" + ), + "typescript" | "tsx" => matches!( + kind, + "function_declaration" + | "function_signature" + | "method_definition" + | "arrow_function" + | "function_expression" + | "generator_function_declaration" + | "generator_function" + ), + _ => false, + } +} + +fn definition_node<'tree>(language: &str, node: Node<'tree>) -> Node<'tree> { + if !matches!(language, "typescript" | "tsx") { + return node; + } + let Some(parent) = node.parent().filter(|parent| { + parent.kind() == "variable_declarator" + && !parent.has_error() + && !parent.is_missing() + && parent.end_byte() == node.end_byte() + && parent + .child_by_field_name("value") + .is_some_and(|value| value.id() == node.id()) + && parent.child_by_field_name("name").is_some_and(|name| { + name.kind() == "identifier" && name.start_byte() == parent.start_byte() + }) + }) else { + return node; + }; + parent +} + +fn barrier(kind: &str) -> bool { + matches!( + kind, + "class_definition" + | "class_declaration" + | "interface_declaration" + | "trait_item" + | "impl_item" + | "enum_item" + | "struct_item" + | "type_declaration" + | "type_alias_declaration" + | "type_item" + ) +} +fn identifier(kind: &str) -> bool { + matches!( + kind, + "identifier" + | "field_identifier" + | "property_identifier" + | "private_property_identifier" + | "type_identifier" + | "shorthand_property_identifier" + | "shorthand_property_identifier_pattern" + ) +} +fn comment(kind: &str) -> bool { + matches!(kind, "comment" | "line_comment" | "block_comment") +} +fn site(node: Node<'_>, source: &[u8], maximum: usize) -> Option { + let raw = std::str::from_utf8(source.get(node.byte_range())?).ok()?; + let mut end = raw.len().min(maximum); + while !raw.is_char_boundary(end) { + end = end.checked_sub(1)?; + } + if end == 0 { + return None; + } + let text = raw.get(..end)?.to_owned(); + let mut point = Point::from(node.start_position()); + for byte in text.bytes() { + if byte == b'\n' { + point.row += 1; + point.column = 0; + } else { + point.column += 1; + } + } + Some(Site { + start: node.start_byte(), + end: node.start_byte() + end, + start_point: Point::from(node.start_position()), + end_point: point, + text, + }) +} +/// Largest small enclosing syntax node whose entire subtree belongs to this +/// callable. A nested callable/type, parser recovery, or scan limit rejects the +/// larger window. Exact bytes come from extraction, never a query-time file. +fn code_window( + node: Node<'_>, + source: &[u8], + language: &str, + visits_left: &mut usize, +) -> Option { + let mut selected = None; + let mut parent = node.parent(); + for _ in 0..8 { + let Some(candidate) = parent else { break }; + if callable(language, candidate.kind()) + || barrier(candidate.kind()) + || candidate.end_byte().saturating_sub(candidate.start_byte()) > 512 + { + break; + } + let mut cursor = candidate.walk(); + let mut safe = true; + let mut local_visits = 0; + loop { + if *visits_left == 0 || local_visits >= 128 { + safe = false; + break; + } + *visits_left -= 1; + local_visits += 1; + let child = cursor.node(); + if child.has_error() + || child.is_missing() + || callable(language, child.kind()) + || barrier(child.kind()) + { + safe = false; + break; + } + if cursor.goto_first_child() { + continue; + } + loop { + if cursor.goto_next_sibling() { + break; + } + if !cursor.goto_parent() { + break; + } + } + if cursor.node() == candidate { + break; + } + } + if !safe { + break; + } + selected = site(candidate, source, 512); + parent = candidate.parent(); + } + selected +} + +/// Input and traversal limits are independent of the graph's dependency model. +/// Each nested callable has its own owner; class/type scopes block inheritance. +/// The caller supplies the already parsed native tree and original source bytes. +fn project( + root: Node<'_>, + source: &[u8], + language: &str, + limits: Limits, +) -> Result { + if source.len() > limits.file_bytes { + return Err("source-query file byte limit exceeded"); + } + if std::str::from_utf8(source).is_err() { + return Err("source-query evidence requires decoded UTF-8 source"); + } + if !matches!(language, "python" | "rust" | "go" | "typescript" | "tsx") { + return Err("unsupported source-query projection language"); + } + if root.end_byte() > source.len() { + return Err("source-query AST range exceeds original source"); + } + let mut result = Projection::default(); + let mut code_visits_left = limits.code_visits; + let mut cursor = root.walk(); + let mut owner = None::; + let mut ancestors = Vec::>::new(); + loop { + if result.visits >= limits.visits { + result.truncated = true; + break; + } + result.visits += 1; + let node = cursor.node(); + let mut child_owner = owner; + if barrier(node.kind()) { + child_owner = None; + } + if node.is_named() && callable(language, node.kind()) { + child_owner = None; + if node.has_error() || node.is_missing() || result.scopes.len() >= limits.scopes { + result.truncated = true; + } else { + let name_node = node.child_by_field_name("name"); + let raw_name = name_node + .and_then(|name| std::str::from_utf8(source.get(name.byte_range())?).ok()); + let name_too_large = raw_name.is_some_and(|name| name.len() > 256); + let name = raw_name.filter(|name| name.len() <= 256).map(str::to_owned); + let definition = definition_node(language, node); + let at = result.scopes.len(); + result.scopes.push(Scope { + name, + syntax: node.kind().to_owned(), + start: definition.start_byte(), + end: definition.end_byte(), + parent_scope: owner, + callable_start: node.start_byte(), + callable_end: node.end_byte(), + start_point: Point::from(node.start_position()), + end_point: Point::from(node.end_position()), + names: BTreeMap::new(), + comments: Vec::new(), + code_windows: Vec::new(), + omitted_code_windows: 0, + omitted_sites: 0, + truncated: name_too_large, + name_span: name_node.map(|name| (name.start_byte(), name.end_byte())), + }); + child_owner = Some(at); + } + } + if let Some(at) = child_owner { + let scope = &mut result.scopes[at]; + if identifier(node.kind()) + && scope.name_span != Some((node.start_byte(), node.end_byte())) + { + // Code windows have independent quotas and never consume the + // lexical budget or contribute terms/statistics to retrieval. + let covered = scope.code_windows.iter().any(|window| { + window.start <= node.start_byte() && node.end_byte() <= window.end + }); + if !covered { + if scope.code_windows.len() >= 16 + || result.retained_code_bytes >= 256 * 1024 + || code_visits_left == 0 + { + scope.omitted_code_windows = scope.omitted_code_windows.saturating_add(1); + } else { + let window = code_window(node, source, language, &mut code_visits_left); + if let Some(window) = window + && window.start >= scope.start + && window.end <= scope.end + && !scope + .code_windows + .iter() + .any(|prior| prior.start == window.start && prior.end == window.end) + { + if result.retained_code_bytes.saturating_add(window.text.len()) + <= 256 * 1024 + { + result.retained_code_bytes += window.text.len(); + scope.code_windows.push(window); + } else { + scope.omitted_code_windows = + scope.omitted_code_windows.saturating_add(1); + } + } else if code_visits_left == 0 { + scope.omitted_code_windows = + scope.omitted_code_windows.saturating_add(1); + } + } + } + if let Some(value) = site(node, source, 256) { + // Never keep a truncated identifier as a different spelling. + if value.end != node.end_byte() + || (!scope.names.contains_key(&value.text) + && scope.names.len() >= limits.names_per_scope) + { + scope.truncated = true; + scope.omitted_sites += 1; + } else if scope + .names + .get(&value.text) + .is_some_and(|sites| sites.len() >= limits.sites_per_name) + { + scope.omitted_sites += 1; + } else if result.retained_bytes.saturating_add(value.text.len()) + > limits.retained_bytes + { + scope.truncated = true; + scope.omitted_sites += 1; + result.truncated = true; + } else { + result.retained_bytes += value.text.len(); + scope + .names + .entry(value.text.clone()) + .or_default() + .push(value); + } + } + } else if comment(node.kind()) { + if scope.comments.len() >= limits.comments_per_scope { + scope.truncated = true; + scope.omitted_sites += 1; + } else if let Some(value) = site(node, source, limits.comment_bytes) { + if result.retained_bytes.saturating_add(value.text.len()) + > limits.retained_bytes + { + scope.truncated = true; + scope.omitted_sites += 1; + result.truncated = true; + } else { + scope.truncated |= value.end != node.end_byte(); + result.retained_bytes += value.text.len(); + scope.comments.push(value); + } + } + } + } + if ancestors.len() < limits.depth && cursor.goto_first_child() { + ancestors.push(owner); + owner = child_owner; + continue; + } + if ancestors.len() >= limits.depth && node.child_count() > 0 { + result.truncated = true; + } + loop { + if cursor.goto_next_sibling() { + break; + } + if !cursor.goto_parent() { + if result.truncated { + for scope in &mut result.scopes { + scope.truncated = true; + } + } + return Ok(result); + } + owner = ancestors.pop().flatten(); + } + } + for scope in &mut result.scopes { + scope.truncated = true; + } + Ok(result) +} + +#[cfg(test)] +mod tests { + use super::{Limits, project}; + fn parse( + source: &str, + language: &str, + ) -> Result> { + let mut parser = tree_sitter::Parser::new(); + parser.set_language(&tree_sitter_language_pack::get_language(language)?)?; + let tree = parser.parse(source, None).ok_or("no syntax tree")?; + if tree.root_node().has_error() { + return Err(format!("malformed test fixture: {}", tree.root_node().to_sexp()).into()); + } + Ok(tree) + } + #[test] + fn default_occurrence_budget_preserves_terms_and_code_without_duplicate_sites() + -> Result<(), Box> { + for (language, source) in [ + ( + "python", + "def opaque(payload):\n value = payload\n # retained context\n return value + payload + value + payload\n", + ), + ( + "rust", + "fn opaque(payload: i32) -> i32 { let value = payload; // retained context\nvalue + payload + value + payload }", + ), + ( + "go", + "package p\nfunc opaque(payload int) int { value := payload; // retained context\nreturn value + payload + value + payload }", + ), + ( + "typescript", + "function opaque(payload: number) { const value = payload; // retained context\nreturn value + payload + value + payload; }", + ), + ( + "tsx", + "function opaque(payload: number) { const value = payload; // retained context\nreturn value + payload + value + payload; }", + ), + ] { + let tree = parse(source, language)?; + let compact = project( + tree.root_node(), + source.as_bytes(), + language, + Limits::default(), + ) + .map_err(std::io::Error::other)?; + let repeated = project( + tree.root_node(), + source.as_bytes(), + language, + Limits { + sites_per_name: 4, + ..Limits::default() + }, + ) + .map_err(std::io::Error::other)?; + let owner = compact + .scopes + .iter() + .find(|s| s.name.as_deref() == Some("opaque")) + .ok_or("missing compact owner")?; + let prior = repeated + .scopes + .iter() + .find(|s| s.name.as_deref() == Some("opaque")) + .ok_or("missing repeated owner")?; + assert!( + owner.names.values().all(|sites| sites.len() == 1), + "{language}" + ); + assert_eq!( + owner.names.keys().collect::>(), + prior.names.keys().collect::>(), + "{language}" + ); + for (name, sites) in &owner.names { + assert_eq!( + sites.first(), + prior.names[name].first(), + "{language}: {name}" + ); + } + assert!( + compact.retained_bytes < repeated.retained_bytes, + "{language}" + ); + assert!(owner.omitted_sites > prior.omitted_sites, "{language}"); + assert!(!owner.truncated && !compact.truncated, "{language}"); + assert_eq!(owner.comments, prior.comments, "{language}"); + assert_eq!(owner.code_windows, prior.code_windows, "{language}"); + assert_eq!( + owner.omitted_code_windows, prior.omitted_code_windows, + "{language}" + ); + } + Ok(()) + } + #[test] + fn native_scopes_keep_parameters_and_comments_without_borrowing_nested_bodies() + -> Result<(), Box> { + for (language, source, outer) in [ + ( + "python", + "# outside_marker\ndef opaque(payload):\n annotations = payload\n # retained marker\n def inner():\n return secret\n return annotations\n", + "opaque", + ), + ( + "rust", + "// outside_marker\nfn opaque(payload: &str) { let annotations = payload; // retained marker\nfn inner() { let secret = 0; } }", + "opaque", + ), + ( + "go", + "package p\n// outside_marker\nfunc Opaque(payload string) { annotations := payload; // retained marker\ninner := func() { secret() }; _ = annotations; _ = inner }", + "Opaque", + ), + ( + "typescript", + "// outside_marker\nexport function opaque(payload: unknown) { const annotations = payload; // retained marker\nfunction inner() { const secret = 0; } return annotations; }", + "opaque", + ), + ] { + let tree = parse(source, language)?; + let projection = project( + tree.root_node(), + source.as_bytes(), + language, + Limits::default(), + ) + .map_err(std::io::Error::other)?; + assert!(!projection.truncated, "{language}"); + let scope = projection + .scopes + .iter() + .find(|scope| scope.name.as_deref() == Some(outer)) + .ok_or("missing outer scope")?; + assert!(scope.names.contains_key("payload"), "{language}"); + assert!(scope.names.contains_key("annotations"), "{language}"); + assert!(!scope.names.contains_key("secret"), "{language}"); + assert!( + !scope.names.contains_key(outer), + "do not duplicate declaration name" + ); + assert!( + scope + .comments + .iter() + .any(|site| site.text.contains("retained marker")), + "{language}" + ); + assert!( + !scope + .comments + .iter() + .any(|site| site.text.contains("outside_marker")), + "{language}" + ); + assert!( + projection + .scopes + .iter() + .any(|scope| scope.names.contains_key("secret")), + "nested owner missing: {language}" + ); + for scope in &projection.scopes { + for site in scope.names.values().flatten().chain(&scope.comments) { + assert_eq!(source.get(site.start..site.end), Some(site.text.as_str())); + assert!(scope.start <= site.start && site.end <= scope.end); + } + } + } + Ok(()) + } + #[test] + fn covered_sites_do_not_count_as_omitted_when_scan_budget_ends() + -> Result<(), Box> { + let source = "def opaque(items):\n return items + items\n"; + let tree = parse(source, "python")?; + // This fixture has three physical occurrences. Checking only retained + // representative sites would mistake term coverage for code coverage. + let occurrences = source + .match_indices("items") + .map(|(start, text)| (start, start + text.len())) + .collect::>(); + assert_eq!(occurrences.len(), 3); + let mut complete_cases = 0; + let mut representative_only_cases = 0; + for code_visits in 0..=256 { + let projection = project( + tree.root_node(), + source.as_bytes(), + "python", + Limits { + code_visits, + ..Limits::default() + }, + ) + .map_err(std::io::Error::other)?; + let owner = projection.scopes.first().ok_or("owner")?; + let covered = occurrences.iter().all(|&(start, end)| { + owner + .code_windows + .iter() + .any(|window| window.start <= start && end <= window.end) + }); + if covered { + complete_cases += 1; + assert_eq!( + owner.omitted_code_windows, 0, + "all sites covered with scan budget {code_visits}" + ); + } else if owner.names.values().flatten().all(|site| { + owner + .code_windows + .iter() + .any(|window| window.start <= site.start && site.end <= window.end) + }) { + representative_only_cases += 1; + assert!(owner.omitted_code_windows > 0); + } + } + assert!(complete_cases > 0); + assert!(representative_only_cases > 0); + Ok(()) + } + + #[test] + fn code_window_quota_does_not_drop_lexical_identifiers() + -> Result<(), Box> { + let body = (0..80) + .map(|at| format!(" normalize_payload_{at}(items)\n")) + .collect::(); + let source = format!("def opaque(items):\n{body}"); + let tree = parse(&source, "python")?; + let projection = project( + tree.root_node(), + source.as_bytes(), + "python", + Limits::default(), + ) + .map_err(std::io::Error::other)?; + let owner = projection.scopes.first().ok_or("owner")?; + assert_eq!(owner.code_windows.len(), 16); + assert!(owner.omitted_code_windows > 0); + assert!(owner.names.contains_key("normalize_payload_79")); + assert!(!owner.truncated); + assert!(!projection.truncated); + Ok(()) + } + + #[test] + fn code_windows_are_exact_and_cannot_borrow_nested_or_sibling_implementations() + -> Result<(), Box> { + for (language, source, outer) in [ + ( + "python", + "def opaque(items):\n result = normalize_payload(items)\n def nested():\n secret_nested()\n return result\ndef sibling():\n secret_sibling()\n", + "opaque", + ), + ( + "rust", + "fn opaque(items: usize) { normalize_payload(items); fn nested() { secret_nested(); } } fn sibling() { secret_sibling(); }", + "opaque", + ), + ( + "go", + "package p\nfunc Opaque(items int) { normalize_payload(items); nested := func() { secret_nested() }; _ = nested }\nfunc sibling() { secret_sibling() }", + "Opaque", + ), + ( + "typescript", + "function opaque(items: unknown) { normalize_payload(items); class Nested { run() { secret_nested(); } } } function sibling() { secret_sibling(); }", + "opaque", + ), + ] { + let tree = parse(source, language)?; + let projection = project( + tree.root_node(), + source.as_bytes(), + language, + Limits::default(), + ) + .map_err(std::io::Error::other)?; + let scope = projection + .scopes + .iter() + .find(|scope| scope.name.as_deref() == Some(outer)) + .ok_or("missing owner")?; + assert!( + scope + .code_windows + .iter() + .any(|window| window.text.contains("normalize_payload(items)")), + "{language}" + ); + for window in &scope.code_windows { + assert_eq!( + source.get(window.start..window.end), + Some(window.text.as_str()) + ); + assert!(scope.start <= window.start && window.end <= scope.end); + assert!( + !window.text.contains("secret_nested"), + "{language}: {}", + window.text + ); + assert!(!window.text.contains("secret_sibling")); + assert!(window.text.len() <= 512); + } + assert!(projection.retained_code_bytes <= 256 * 1024); + } + Ok(()) + } + + #[test] + fn classes_block_outer_ownership_and_strings_are_not_lexical_identifiers() + -> Result<(), Box> { + let source = "function opaque(payload: unknown) { const annotations = 'secret_literal'; class Nested { x = classOnly; meth() { methodOnly(); } } return payload; }"; + let tree = parse(source, "typescript")?; + let p = project( + tree.root_node(), + source.as_bytes(), + "typescript", + Limits::default(), + ) + .map_err(std::io::Error::other)?; + let outer = p + .scopes + .iter() + .find(|scope| scope.name.as_deref() == Some("opaque")) + .ok_or("missing scope")?; + for term in ["secret_literal", "classOnly", "methodOnly", "Nested"] { + assert!(!outer.names.contains_key(term), "{term}"); + } + assert!( + p.scopes + .iter() + .any(|scope| scope.name.as_deref() == Some("meth") + && scope.names.contains_key("methodOnly")) + ); + Ok(()) + } + #[test] + fn unicode_ranges_and_quotas_remain_explicit() -> Result<(), Box> { + let source = "function opaque(payload: unknown) { const café = payload; // 雪 retained\nreturn café; }"; + let tree = parse(source, "typescript")?; + let p = project( + tree.root_node(), + source.as_bytes(), + "typescript", + Limits::default(), + ) + .map_err(std::io::Error::other)?; + assert!(p.scopes[0].names.contains_key("café")); + for site in p.scopes[0] + .names + .values() + .flatten() + .chain(&p.scopes[0].comments) + { + assert_eq!(source.get(site.start..site.end), Some(site.text.as_str())); + } + let p = project( + tree.root_node(), + source.as_bytes(), + "typescript", + Limits { + visits: 1, + ..Limits::default() + }, + ) + .map_err(std::io::Error::other)?; + assert!(p.truncated && p.visits == 1); + let p = project( + tree.root_node(), + source.as_bytes(), + "typescript", + Limits { + names_per_scope: 1, + ..Limits::default() + }, + ) + .map_err(std::io::Error::other)?; + assert!(p.scopes[0].truncated && p.scopes[0].names.len() == 1); + assert!( + project( + tree.root_node(), + source.as_bytes(), + "typescript", + Limits { + file_bytes: 1, + ..Limits::default() + } + ) + .is_err() + ); + Ok(()) + } + #[test] + fn independent_projection_limits_disclose_omissions() -> Result<(), Box> + { + let source = "function opaque(payload: unknown) { // retained marker\nreturn payload + payload; } function other() {}"; + let tree = parse(source, "typescript")?; + for limits in [ + Limits { + depth: 1, + ..Limits::default() + }, + Limits { + scopes: 1, + ..Limits::default() + }, + Limits { + retained_bytes: 1, + ..Limits::default() + }, + ] { + let projection = project(tree.root_node(), source.as_bytes(), "typescript", limits) + .map_err(std::io::Error::other)?; + assert!(projection.truncated); + } + let projection = project( + tree.root_node(), + source.as_bytes(), + "typescript", + Limits { + comments_per_scope: 0, + ..Limits::default() + }, + ) + .map_err(std::io::Error::other)?; + assert!(projection.scopes[0].truncated && projection.scopes[0].omitted_sites > 0); + assert!(projection.scopes[0].comments.is_empty()); + let projection = project( + tree.root_node(), + source.as_bytes(), + "typescript", + Limits { + sites_per_name: 1, + ..Limits::default() + }, + ) + .map_err(std::io::Error::other)?; + assert_eq!(projection.scopes[0].names["payload"].len(), 1); + assert!(projection.scopes[0].omitted_sites >= 2); + // Only repeated source occurrences were omitted; term coverage survives. + assert!(!projection.scopes[0].truncated); + Ok(()) + } + + #[test] + fn native_parent_facts_and_callable_ranges_are_explicit_in_five_languages() + -> Result<(), Box> { + for (language, source) in [ + ( + "python", + "def opaque(payload):\n callback = lambda item: (lambda other: payload_error(other))(item)\n return callback\ndef sibling():\n return 0\n", + ), + ( + "rust", + "fn opaque(payload: usize) { let callback = |item: usize| { let inner = |other: usize| payload_error(other); inner(item) }; } fn sibling() {}", + ), + ( + "go", + "package p\nfunc opaque(payload int) { callback := func(item int) { inner := func(other int) { payload_error(other) }; _ = inner }; _ = callback }\nfunc sibling() {}", + ), + ( + "typescript", + "function opaque(payload: number) { const callback = (item: number) => { const inner = (other: number) => payload_error(other); return inner(item); }; } function sibling() {}", + ), + ( + "tsx", + "function opaque(payload: number) { const callback = (item: number) => { const inner = (other: number) => payload_error(other); return inner(item); }; } function sibling() {}", + ), + ] { + let mut parser = tree_sitter::Parser::new(); + parser.set_language(&tree_sitter_language_pack::get_language(language)?)?; + let tree = parser.parse(source, None).ok_or("no tree")?; + assert!(!tree.root_node().has_error(), "{language}"); + let projection = project( + tree.root_node(), + source.as_bytes(), + language, + Limits::default(), + ) + .map_err(std::io::Error::other)?; + assert_eq!(projection.scopes.len(), 4, "{language}"); + if language == "python" { + let bounded = project( + tree.root_node(), + source.as_bytes(), + language, + Limits { + scopes: 4, + ..Limits::default() + }, + ) + .map_err(std::io::Error::other)?; + assert!(!bounded.truncated); + assert!( + bounded + .scopes + .iter() + .any(|scope| scope.name.as_deref() == Some("sibling")) + ); + assert!( + bounded + .scopes + .iter() + .all(|scope| source.get(scope.callable_start..scope.callable_end) + != Some("lambda")) + ); + } + + assert_eq!(projection.scopes[0].parent_scope, None); + assert_eq!(projection.scopes[1].parent_scope, Some(0)); + assert_eq!(projection.scopes[2].parent_scope, Some(1)); + assert_eq!(projection.scopes[3].parent_scope, None); + for scope in &projection.scopes { + assert!(scope.start <= scope.callable_start && scope.callable_end <= scope.end); + let prefix = &source.as_bytes()[..scope.callable_start]; + assert_eq!( + scope.start_point.row, + prefix.iter().filter(|byte| **byte == b'\n').count() + ); + let column = prefix + .iter() + .rposition(|byte| *byte == b'\n') + .map_or(prefix.len(), |at| prefix.len() - at - 1); + assert_eq!(scope.start_point.column, column); + for window in &scope.code_windows { + assert_eq!( + source.get(window.start..window.end), + Some(window.text.as_str()) + ); + } + } + if matches!(language, "typescript" | "tsx") { + assert!(projection.scopes[1].start < projection.scopes[1].callable_start); + } + assert!(!projection.scopes[0].names.contains_key("payload_error")); + } + Ok(()) + } + #[test] + fn type_barriers_do_not_supply_an_outer_callable_parent() + -> Result<(), Box> { + let source = + "function outer() { class Nested { run() { return (() => secret_payload())(); } } }"; + let mut parser = tree_sitter::Parser::new(); + parser.set_language(&tree_sitter_language_pack::get_language("typescript")?)?; + let tree = parser.parse(source, None).ok_or("no tree")?; + let projection = project( + tree.root_node(), + source.as_bytes(), + "typescript", + Limits::default(), + ) + .map_err(std::io::Error::other)?; + assert_eq!(projection.scopes.len(), 3); + assert_eq!(projection.scopes[0].name.as_deref(), Some("outer")); + assert_eq!(projection.scopes[1].name.as_deref(), Some("run")); + assert_eq!(projection.scopes[1].parent_scope, None); + assert_eq!(projection.scopes[2].parent_scope, Some(1)); + Ok(()) + } +} diff --git a/crates/compass-languages/tests/callable_initializer_definitions.rs b/crates/compass-languages/tests/callable_initializer_definitions.rs new file mode 100644 index 000000000..8aac16ea3 --- /dev/null +++ b/crates/compass-languages/tests/callable_initializer_definitions.rs @@ -0,0 +1,77 @@ +use std::error::Error; + +use compass_languages::{ + Engine, + source_query::{EXTRACTION_EXTENSION, SourceQueryExtraction}, +}; +use serde::Deserialize; + +#[derive(Deserialize)] +struct Case { + extensions: Vec, + source: String, + definition: String, +} + +#[test] +fn direct_initializer_projection_uses_the_exact_declarator_envelope() -> Result<(), Box> +{ + let cases: Vec = serde_json::from_str(include_str!( + "../../../fixtures/callable_initializer_definitions.json" + ))?; + for (index, case) in cases.iter().enumerate() { + for extension in case + .extensions + .iter() + .filter(|value| value.starts_with("ts")) + { + for seed in ["alpha", "zeta"] { + let file = format!("{seed}_{index}.{extension}"); + let directory = tempfile::tempdir()?; + let path = directory.path().join(&file); + std::fs::write(&path, &case.source)?; + let extraction = Engine::default().extract_source_graph_only( + &path, + &file, + case.source.as_bytes(), + )?; + assert!(extraction.error.is_none(), "{file}: {:?}", extraction.error); + let raw: SourceQueryExtraction = serde_json::from_value( + extraction + .extensions + .get(EXTRACTION_EXTENSION) + .ok_or("missing source projection")? + .clone(), + )?; + let start = case + .source + .find(&case.definition) + .ok_or("missing definition")?; + let end = start + case.definition.len(); + let matches = raw + .projection + .scopes + .iter() + .filter(|scope| scope.start == start && scope.end == end) + .collect::>(); + assert_eq!( + matches.len(), + 1, + "{file}: missing exact declarator envelope" + ); + assert!(matches[0].names.contains_key("payload"), "{file}"); + assert!( + !matches[0].names.contains_key("opaque"), + "{file}: binding name is outside its body" + ); + if case.source.contains("item =>") { + assert!( + !matches[0].names.contains_key("item"), + "{file}: callback body leaked" + ); + } + } + } + } + Ok(()) +} diff --git a/crates/compass-languages/tests/leading_documentation.rs b/crates/compass-languages/tests/leading_documentation.rs new file mode 100644 index 000000000..85e6bb074 --- /dev/null +++ b/crates/compass-languages/tests/leading_documentation.rs @@ -0,0 +1,392 @@ +use std::collections::BTreeSet; +use std::error::Error; +use std::path::Path; + +use compass_languages::{ + DECLARATION_DOCUMENTATION_OMISSION_CODE, DeclarationDocumentationKind, Engine, EvidenceLimits, + MAX_BATCH_DOCUMENTATION_BYTES, MAX_DECLARATION_DOCUMENTATION_BYTES, SemanticEvidenceBatch, + UniversalEvidenceQualification, validate_evidence, +}; + +fn extract(file: &str, source: &str) -> Result> { + let extraction = Engine::default() + .extract_source_combined(Path::new(file), file, source.as_bytes())? + .graph; + if let Some(error) = extraction.error { + return Err(error.into()); + } + let batch = extraction + .semantic_evidence + .ok_or("missing direct evidence")?; + validate_evidence(&batch, EvidenceLimits::default())?; + Ok(batch) +} + +fn assert_documentation( + batch: &SemanticEvidenceBatch, + name: &str, + source: &str, + expected: &str, + kind: DeclarationDocumentationKind, +) -> Result<(), Box> { + let declaration = batch + .declarations + .iter() + .find(|declaration| { + declaration.name == name + && matches!( + declaration.kind.as_str(), + "function" + | "method" + | "class" + | "struct" + | "interface" + | "trait" + | "enum" + | "type_alias" + ) + }) + .ok_or_else(|| format!("missing declaration {name}"))?; + let documentation = declaration + .documentation + .as_ref() + .ok_or_else(|| format!("missing documentation for {name}"))?; + assert_eq!(documentation.kind, kind); + assert_eq!(documentation.content, expected); + let start = usize::try_from(documentation.range.start_byte)?; + let end = usize::try_from(documentation.range.end_byte)?; + assert_eq!(source.get(start..end), Some(expected)); + assert!(documentation.range.end_byte <= declaration.range.start_byte); + assert_eq!( + documentation.range.source_file, + declaration.range.source_file + ); + Ok(()) +} + +#[test] +fn rust_outer_documentation_follows_ast_markers_and_attributes() -> Result<(), Box> { + for (source, expected) in [ + ( + "/// Preserve error context.\n#[inline]\npub fn work() {}\n", + "/// Preserve error context.\n", + ), + ( + "#[inline]\n/// Preserve error context.\npub fn work() {}\n", + "/// Preserve error context.\n", + ), + ("/// café\r\npub fn work() {}\r\n", "/// café\r\n"), + ( + "/** Preserve error context. */\npub fn work() {}\n", + "/** Preserve error context. */", + ), + ( + " /** café\r\n * 文 */\r\n pub fn work() {}\r\n", + "/** café\r\n * 文 */", + ), + ] { + let batch = extract("src/sample.rs", source)?; + assert_documentation( + &batch, + "work", + source, + expected, + DeclarationDocumentationKind::RustOuter, + )?; + assert_eq!(batch.pipeline.version, 3); + assert_eq!( + batch.pipeline.qualification, + UniversalEvidenceQualification::Qualifying + ); + } + Ok(()) +} + +#[test] +fn rust_inner_ordinary_and_other_owner_comments_do_not_attach() -> Result<(), Box> { + for source in [ + "//! Inner module documentation.\npub fn work() {}\n", + "// Ordinary comment.\npub fn work() {}\n", + "//// Ordinary slash comment.\npub fn work() {}\n", + "/// First declaration.\nfn first() {}\npub fn work() {}\n", + "struct Store;\n/// Implementation documentation.\nimpl Store { fn work() {} }\n", + ] { + let batch = extract("src/sample.rs", source)?; + let declaration = batch + .declarations + .iter() + .find(|declaration| declaration.name == "work") + .ok_or("missing work declaration")?; + assert!(declaration.documentation.is_none(), "{source}"); + } + Ok(()) +} + +#[test] +fn go_leading_groups_attach_to_exact_function_method_and_single_type() -> Result<(), Box> +{ + let source = "package sample\n\n// Store keeps records.\ntype Store struct{}\n\n// Read preserves errors.\n// It retains their context.\nfunc (s Store) Read() {}\n\n/* Flush sends records. */\nfunc Flush() {}\n"; + let batch = extract("sample.go", source)?; + for (name, expected) in [ + ("Store", "// Store keeps records."), + ( + "Read", + "// Read preserves errors.\n// It retains their context.", + ), + ("Flush", "/* Flush sends records. */"), + ] { + assert_documentation( + &batch, + name, + source, + expected, + DeclarationDocumentationKind::GoLeading, + )?; + } + assert_eq!(batch.pipeline.version, 2); + assert_eq!( + batch.pipeline.qualification, + UniversalEvidenceQualification::Qualifying + ); + Ok(()) +} + +#[test] +fn go_blank_trailing_and_ambiguous_group_comments_do_not_attach() -> Result<(), Box> { + for source in [ + "package sample\n// Detached documentation.\n\nfunc Work() {}\n", + "package sample\nfunc First() {} // Trailing comment.\nfunc Work() {}\n", + ] { + let batch = extract("sample.go", source)?; + let work = batch + .declarations + .iter() + .find(|declaration| declaration.name == "Work") + .ok_or("missing Work declaration")?; + assert!(work.documentation.is_none()); + } + let batch = extract( + "sample.go", + "package sample\n// Shared group description.\ntype (\n First struct{}\n Second struct{}\n)\n", + )?; + for declaration in batch + .declarations + .iter() + .filter(|declaration| matches!(declaration.name.as_str(), "First" | "Second")) + { + assert!(declaration.documentation.is_none()); + } + Ok(()) +} + +#[test] +fn typescript_exports_decorators_getters_and_setters_keep_separate_owners() +-> Result<(), Box> { + let source = "declare function decorate(value: unknown): any;\n/** Work preserves errors. */\nexport async function work(): Promise {}\n/** Widget retains state. */\n@decorate\nexport class Widget {\n /** Getter reads state. */\n get value(): number { return 1; }\n /** Setter writes state. */\n set value(input: number) {}\n}\n"; + let batch = extract("src/sample.ts", source)?; + assert_documentation( + &batch, + "work", + source, + "/** Work preserves errors. */", + DeclarationDocumentationKind::TypeScriptJsDoc, + )?; + assert_documentation( + &batch, + "Widget", + source, + "/** Widget retains state. */", + DeclarationDocumentationKind::TypeScriptJsDoc, + )?; + let accessors = batch + .declarations + .iter() + .filter(|declaration| declaration.name == "value") + .filter_map(|declaration| declaration.documentation.as_ref()) + .map(|documentation| documentation.content.as_str()) + .collect::>(); + assert_eq!( + accessors, + BTreeSet::from(["/** Getter reads state. */", "/** Setter writes state. */"]) + ); + assert!( + batch + .declarations + .iter() + .filter(|declaration| declaration.kind == "export") + .all(|declaration| declaration.documentation.is_none()) + ); + assert_eq!(batch.pipeline.version, 4); + assert_eq!( + batch.pipeline.qualification, + UniversalEvidenceQualification::Qualifying + ); + Ok(()) +} + +#[test] +fn typescript_overloads_and_tsx_preserve_each_exact_document_range() -> Result<(), Box> { + let source = "/** Number overload. */\nexport function convert(input: number): number;\n/** String overload. */\nexport function convert(input: string): string;\n/** Runtime implementation. */\nexport function convert(input: unknown): unknown { return input; }\n"; + let batch = extract("src/sample.ts", source)?; + let declarations = batch + .declarations + .iter() + .filter(|declaration| declaration.name == "convert" && declaration.kind == "function") + .collect::>(); + assert_eq!(declarations.len(), 3); + let mut ranges = BTreeSet::new(); + for declaration in declarations { + let documentation = declaration + .documentation + .as_ref() + .ok_or("missing overload docs")?; + assert!(ranges.insert(documentation.range.start_byte)); + assert_eq!( + source.get( + usize::try_from(documentation.range.start_byte)? + ..usize::try_from(documentation.range.end_byte)? + ), + Some(documentation.content.as_str()) + ); + } + let source = + "/** Renders a label. */\nexport function Label() { return label; }\n"; + let batch = extract("src/sample.tsx", source)?; + assert_documentation( + &batch, + "Label", + source, + "/** Renders a label. */", + DeclarationDocumentationKind::TypeScriptJsDoc, + )?; + Ok(()) +} + +#[test] +fn ordinary_typescript_comments_and_javascript_emit_no_new_documentation() +-> Result<(), Box> { + for source in [ + "// Ordinary comment.\nexport function work() {}\n", + "/* Ordinary block. */\nexport function work() {}\n", + ] { + let batch = extract("src/sample.ts", source)?; + assert!( + batch + .declarations + .iter() + .all(|declaration| declaration.documentation.is_none()) + ); + } + let batch = extract( + "src/sample.js", + "/** Existing JavaScript source. */\nexport function work() {}\n", + )?; + assert!( + batch + .declarations + .iter() + .all(|declaration| declaration.documentation.is_none()) + ); + assert_eq!(batch.pipeline.version, 3); + assert_eq!( + batch.pipeline.qualification, + UniversalEvidenceQualification::Qualifying + ); + Ok(()) +} + +#[test] +fn oversized_and_interleaved_documentation_preserves_structural_facts_with_warning() +-> Result<(), Box> { + for source in [ + format!( + "/** {} */\npub fn work() {{}}\n", + "x".repeat(MAX_DECLARATION_DOCUMENTATION_BYTES) + ), + "/// First group.\n#[inline]\n/// Second group.\npub fn work() {}\n".to_owned(), + ] { + let batch = extract("src/sample.rs", &source)?; + let work = batch + .declarations + .iter() + .find(|declaration| declaration.name == "work") + .ok_or("bounded docs lost the declaration")?; + assert!(work.documentation.is_none()); + assert_eq!( + batch + .diagnostics + .iter() + .filter(|diagnostic| diagnostic.code == DECLARATION_DOCUMENTATION_OMISSION_CODE) + .count(), + 1 + ); + } + Ok(()) +} + +#[test] +fn aggregate_documentation_budget_retains_all_declarations_and_one_warning() +-> Result<(), Box> { + // Rust line-comment AST ranges include the terminating LF. Count its byte + // as part of the exact source excerpt, so each retained group is 4096 bytes. + let comment = format!( + "/// {}", + "x".repeat(MAX_DECLARATION_DOCUMENTATION_BYTES - 5) + ); + let retained = MAX_BATCH_DOCUMENTATION_BYTES / MAX_DECLARATION_DOCUMENTATION_BYTES; + let mut source = String::new(); + for index in 0..retained + 4 { + source.push_str(&format!("{comment}\npub fn work_{index}() {{}}\n")); + } + let batch = extract("src/sample.rs", &source)?; + assert_eq!( + batch + .declarations + .iter() + .filter(|declaration| declaration.kind == "function") + .count(), + retained + 4 + ); + assert_eq!( + batch + .declarations + .iter() + .filter(|declaration| declaration.documentation.is_some()) + .count(), + retained + ); + assert_eq!( + batch + .diagnostics + .iter() + .filter(|diagnostic| diagnostic.code == DECLARATION_DOCUMENTATION_OMISSION_CODE) + .count(), + 1 + ); + Ok(()) +} + +#[test] +fn malformed_declarations_do_not_gain_leading_documentation() -> Result<(), Box> { + for (file, source) in [ + ("sample.rs", "/// Claimed behavior.\npub fn work( { }\n"), + ( + "sample.go", + "package sample\n// Claimed behavior.\nfunc Work( {\n", + ), + ( + "sample.ts", + "/** Claimed behavior. */\nexport function work( {\n", + ), + ] { + let batch = extract(file, source)?; + assert!( + batch + .declarations + .iter() + .all(|declaration| declaration.documentation.is_none()), + "{file}: {batch:?}" + ); + } + Ok(()) +} diff --git a/crates/compass-languages/tests/source_query_evidence.rs b/crates/compass-languages/tests/source_query_evidence.rs new file mode 100644 index 000000000..23c1e7438 --- /dev/null +++ b/crates/compass-languages/tests/source_query_evidence.rs @@ -0,0 +1,121 @@ +use compass_languages::Engine; +use serde_json::Value; +use std::error::Error; +use std::path::Path; + +const EXTENSION: &str = "_compass_source_query_evidence"; +fn extract(path: &str, source: &str) -> Result> { + let graph = + Engine::default().extract_source_graph_only(Path::new(path), path, source.as_bytes())?; + graph + .extensions + .get(EXTENSION) + .cloned() + .ok_or_else(|| "missing source-query projection".into()) +} + +#[test] +fn native_extraction_keeps_owned_body_vocabulary_and_exact_sites() -> Result<(), Box> { + for (path, source, name) in [ + ( + "src/sample.py", + "# outside_marker\ndef opaque(payload):\n annotations = payload\n # retained marker\n def inner():\n return secret\n return annotations\n", + "opaque", + ), + ( + "src/sample.rs", + "// outside_marker\nfn opaque(payload: &str) { let annotations = payload; // retained marker\nfn inner() { let secret = 0; } }", + "opaque", + ), + ( + "sample.go", + "package p\n// outside_marker\nfunc Opaque(payload string) { annotations := payload; // retained marker\ninner := func() { secret() }; _ = annotations; _ = inner }", + "Opaque", + ), + ( + "src/sample.ts", + "// outside_marker\nexport function opaque(payload: unknown) { const annotations = payload; // retained marker\nfunction inner() { const secret = 0; } return annotations; }", + "opaque", + ), + ] { + let value = extract(path, source)?; + assert_eq!(value["schema"], "compass.source-query.extraction/1"); + assert_eq!(value["sourceFile"], path); + assert_eq!(value["byteSize"], source.len()); + assert_eq!(value["syntaxRecovery"], false); + let p = &value["projection"]; + assert_eq!(p["truncated"], false, "{path}"); + let scopes = p["scopes"].as_array().ok_or("missing scopes")?; + let owner = scopes + .iter() + .find(|s| s["name"] == name) + .ok_or("missing owner")?; + let names = owner["names"].as_object().ok_or("missing names")?; + assert!( + names.contains_key("payload") && names.contains_key("annotations"), + "{path}" + ); + assert!( + !names.contains_key("secret") && !names.contains_key(name), + "{path}" + ); + assert!(scopes.iter().any(|s| s["names"].get("secret").is_some())); + let comments = owner["comments"].as_array().ok_or("missing comments")?; + assert!(comments.iter().any(|s| { + s["text"] + .as_str() + .is_some_and(|t| t.contains("retained marker")) + })); + assert!(!comments.iter().any(|s| { + s["text"] + .as_str() + .is_some_and(|t| t.contains("outside_marker")) + })); + for scope in scopes { + let start = scope["start"].as_u64().ok_or("scope start")?; + let end = scope["end"].as_u64().ok_or("scope end")?; + let names = scope["names"].as_object().ok_or("scope names")?; + let sites = names + .values() + .flat_map(|v| v.as_array().into_iter().flatten()) + .chain(scope["comments"].as_array().into_iter().flatten()); + for site in sites { + let a = site["start"].as_u64().ok_or("site start")?; + let b = site["end"].as_u64().ok_or("site end")?; + assert!(a >= start && b <= end); + assert_eq!( + source.get(usize::try_from(a)?..usize::try_from(b)?), + site["text"].as_str() + ); + } + } + // The payload survives the real extraction cache's serde representation. + let decoded: Value = serde_json::from_slice(&serde_json::to_vec(&value)?)?; + assert_eq!(value, decoded); + } + Ok(()) +} + +#[test] +fn class_boundaries_strings_and_unicode_are_preserved() -> Result<(), Box> { + let source = "function opaque(payload: unknown) { const café = 'secret_literal'; class Nested { x = classOnly; meth() { methodOnly(); } } // 雪 retained\nreturn café; }"; + let value = extract("src/sample.ts", source)?; + let scopes = value["projection"]["scopes"] + .as_array() + .ok_or("missing scopes")?; + let owner = scopes + .iter() + .find(|s| s["name"] == "opaque") + .ok_or("missing owner")?; + let names = owner["names"].as_object().ok_or("missing names")?; + assert!(names.contains_key("café")); + for term in ["secret_literal", "classOnly", "methodOnly", "Nested"] { + assert!(!names.contains_key(term), "{term}"); + } + assert!( + scopes + .iter() + .any(|s| s["name"] == "meth" && s["names"].get("methodOnly").is_some()) + ); + Ok(()) +} diff --git a/crates/compass-languages/tests/typescript_member_scopes.rs b/crates/compass-languages/tests/typescript_member_scopes.rs new file mode 100644 index 000000000..ad37cb300 --- /dev/null +++ b/crates/compass-languages/tests/typescript_member_scopes.rs @@ -0,0 +1,116 @@ +use std::error::Error; +use std::path::Path; + +use compass_languages::{CandidateRelation, Engine}; +use serde::Deserialize; + +#[derive(Deserialize)] +struct Expected { + kind: String, + line: u32, +} + +#[derive(Deserialize)] +struct Case { + prefix: String, + kind: String, + owner: String, + target: String, + relation: String, + expected: Option, + typescript: String, + javascript: String, +} + +fn check_cases(ambiguous: bool) -> Result<(), Box> { + let cases: Vec = serde_json::from_str(include_str!( + "../../../fixtures/typescript_member_scopes.json" + ))?; + let mut engine = Engine::default(); + for case in cases { + if (case.kind == "ambiguous") != ambiguous { + continue; + } + for (extension, source) in [("ts", &case.typescript), ("js", &case.javascript)] { + for seed in 0..24 { + let file = format!("{}{:02}.{extension}", case.prefix, seed); + let batch = engine.extract_source_universal_evidence( + Path::new(&file), + &file, + source.as_bytes(), + )?; + let relation = match case.relation.as_str() { + "calls" => CandidateRelation::Calls, + "accesses_member" => CandidateRelation::AccessesMember, + _ => return Err("unsupported fixture relation".into()), + }; + let candidates = batch + .candidates + .iter() + .filter(|candidate| { + candidate.relation == relation + && candidate.target_spelling == case.target + && batch.declarations.iter().any(|declaration| { + declaration.id == candidate.source_declaration_id + && declaration.name == case.owner + }) + }) + .collect::>(); + assert_eq!(candidates.len(), 1, "{file}"); + let candidate = candidates[0]; + if let Some(expected) = &case.expected { + let target = candidate + .constraints + .exact_target_declaration_id + .as_ref() + .and_then(|id| batch.declarations.iter().find(|d| &d.id == id)) + .ok_or_else(|| format!("{file}: missing proven member target"))?; + assert_eq!(target.name, case.target, "{file}"); + assert_eq!(target.kind, expected.kind, "{file}"); + assert_eq!(target.range.start_line, expected.line, "{file}"); + } else { + assert!( + candidate.constraints.exact_target_declaration_id.is_none(), + "{file}: a hash-selected child scope must not prove a target" + ); + } + } + } + } + Ok(()) +} + +#[test] +fn ambiguous_accessor_targets_do_not_depend_on_filename_hashes() -> Result<(), Box> { + check_cases(true) +} + +#[test] +fn primary_scope_preserves_members_nested_calls_and_shadowing() -> Result<(), Box> { + check_cases(false) +} + +#[test] +fn javascript_field_declarations_keep_the_property_name_anchor() -> Result<(), Box> { + let source = "class Owner { declared = 1; callback = () => {}; #hidden = 2; }"; + let batch = Engine::default().extract_source_universal_evidence( + Path::new("fields.js"), + "fields.js", + source.as_bytes(), + )?; + for name in ["declared", "callback", "#hidden"] { + let fields = batch + .declarations + .iter() + .filter(|declaration| declaration.kind == "property" && declaration.name == name) + .collect::>(); + assert_eq!(fields.len(), 1, "missing or duplicated field {name}"); + let range = &fields[0].range; + assert_eq!(range.source_file, "fields.js"); + assert_eq!( + source.get(usize::try_from(range.start_byte)?..usize::try_from(range.end_byte)?), + Some(name) + ); + } + Ok(()) +} diff --git a/crates/compass-languages/tests/typescript_universal_evidence.rs b/crates/compass-languages/tests/typescript_universal_evidence.rs index 215258fe2..ab501d233 100644 --- a/crates/compass-languages/tests/typescript_universal_evidence.rs +++ b/crates/compass-languages/tests/typescript_universal_evidence.rs @@ -29,7 +29,7 @@ new App(); let batch = universal_evidence("src/app.ts", source); validate_evidence(&batch, EvidenceLimits::default()).expect("valid evidence"); assert_eq!(batch.pipeline.language, "typescript"); - assert_eq!(batch.pipeline.version, 1); + assert_eq!(batch.pipeline.version, 4); assert_eq!(batch.pipeline.dialect.as_deref(), Some("ts")); assert!( batch @@ -176,7 +176,7 @@ export async function load() { return import("./lazy.js"); } "#; let batch = universal_evidence("src/render.jsx", source); assert_eq!(batch.pipeline.language, "javascript"); - assert_eq!(batch.pipeline.version, 1); + assert_eq!(batch.pipeline.version, 3); assert_eq!(batch.pipeline.dialect.as_deref(), Some("jsx")); assert!( batch diff --git a/crates/compass-languages/tests/universal_evidence.rs b/crates/compass-languages/tests/universal_evidence.rs index 66f6c5130..b87a3ad3f 100644 --- a/crates/compass-languages/tests/universal_evidence.rs +++ b/crates/compass-languages/tests/universal_evidence.rs @@ -60,6 +60,7 @@ fn valid_batch() -> SemanticEvidenceBatch { signature_hash: None, implementation_hash: None, source_hash: None, + documentation: None, definition_start_byte: None, range: range(0, 6), }], @@ -168,6 +169,143 @@ fn unknown_fields_are_rejected_at_nested_boundaries() { assert!(error.to_string().contains("unknown field")); } +fn documentation_batch(content: &str) -> SemanticEvidenceBatch { + use compass_languages::{DeclarationDocumentationFact, DeclarationDocumentationKind}; + let mut batch = valid_batch(); + batch.pipeline.language = "rust".to_owned(); + batch.bindings.clear(); + batch.occurrences.clear(); + batch.candidates.clear(); + batch.scopes.clear(); + let declaration = &mut batch.declarations[0]; + declaration.language = "rust".to_owned(); + declaration.range = EvidenceRange { + source_file: "src/example.rs".to_owned(), + start_byte: content.len() as u64 + 4, + end_byte: content.len() as u64 + 10, + start_line: content.bytes().filter(|byte| *byte == b'\n').count() as u32 + 2, + start_column: 3, + end_line: content.bytes().filter(|byte| *byte == b'\n').count() as u32 + 2, + end_column: 9, + }; + declaration.documentation = Some(DeclarationDocumentationFact { + kind: DeclarationDocumentationKind::RustOuter, + range: EvidenceRange { + source_file: "src/example.rs".to_owned(), + start_byte: 0, + end_byte: content.len() as u64, + start_line: 1, + start_column: 0, + end_line: content.bytes().filter(|byte| *byte == b'\n').count() as u32 + 1, + end_column: content.rsplit('\n').next().map_or(0, str::len) as u32, + }, + content: content.to_owned(), + }); + batch +} + +#[test] +fn declaration_documentation_round_trips_without_changing_historical_missing_fields() { + let historical = serde_json::to_value(valid_batch()).expect("serialize historical fixture"); + assert!(historical["declarations"][0].get("documentation").is_none()); + assert_eq!( + serde_json::from_value::(historical).expect("old field absence"), + valid_batch() + ); + for content in ["/// documentation", "/// café\r\n/// 文"] { + let batch = documentation_batch(content); + validate_evidence(&batch, EvidenceLimits::default()).expect("exact source metadata"); + let encoded = serde_json::to_value(&batch).expect("serialize docs"); + assert_eq!( + encoded["declarations"][0]["documentation"]["kind"], + "rust_outer" + ); + assert_eq!( + serde_json::from_value::(encoded).expect("docs round trip"), + batch + ); + } +} + +#[test] +fn declaration_documentation_rejects_wrong_file_language_owner_and_extent() { + use compass_languages::DeclarationDocumentationKind; + let good = documentation_batch("/// documentation"); + let mut wrong_file = good.clone(); + wrong_file.declarations[0] + .documentation + .as_mut() + .expect("docs") + .range + .source_file = "src/unrelated.rs".to_owned(); + assert_code(&wrong_file, EvidenceErrorCode::InvalidFact); + let mut wrong_language = good.clone(); + wrong_language.declarations[0] + .documentation + .as_mut() + .expect("docs") + .kind = DeclarationDocumentationKind::GoLeading; + assert_code(&wrong_language, EvidenceErrorCode::InvalidFact); + let mut wrong_owner_kind = good.clone(); + wrong_owner_kind.declarations[0].kind = "parameter".to_owned(); + assert_code(&wrong_owner_kind, EvidenceErrorCode::InvalidFact); + let mut overlapping = good.clone(); + overlapping.declarations[0].range.start_byte = 1; + assert_code(&overlapping, EvidenceErrorCode::InvalidFact); + let mut wrong_bytes = good.clone(); + wrong_bytes.declarations[0] + .documentation + .as_mut() + .expect("docs") + .range + .end_byte -= 1; + assert_code(&wrong_bytes, EvidenceErrorCode::InvalidFact); + let mut wrong_columns = documentation_batch("/// café\r\n/// 文"); + wrong_columns.declarations[0] + .documentation + .as_mut() + .expect("docs") + .range + .end_column -= 1; + assert_code(&wrong_columns, EvidenceErrorCode::InvalidFact); +} + +#[test] +fn declaration_documentation_has_closed_attachment_and_content_schema() { + let batch = documentation_batch("/// documentation"); + let mut unknown_field = serde_json::to_value(&batch).expect("serialize docs"); + unknown_field["declarations"][0]["documentation"]["summary"] = serde_json::json!("invented"); + assert!(serde_json::from_value::(unknown_field).is_err()); + let mut unknown_kind = serde_json::to_value(&batch).expect("serialize docs"); + unknown_kind["declarations"][0]["documentation"]["kind"] = serde_json::json!("nearby_comment"); + assert!(serde_json::from_value::(unknown_kind).is_err()); +} + +#[test] +fn declaration_documentation_bounds_each_excerpt_and_aggregate_bytes() { + use compass_languages::{MAX_BATCH_DOCUMENTATION_BYTES, MAX_DECLARATION_DOCUMENTATION_BYTES}; + let content = format!( + "/// {}", + "x".repeat(MAX_DECLARATION_DOCUMENTATION_BYTES - 4) + ); + let mut batch = documentation_batch(&content); + validate_evidence(&batch, EvidenceLimits::default()).expect("exact excerpt ceiling"); + assert_code( + &documentation_batch(&format!("{content}x")), + EvidenceErrorCode::ResourceLimit, + ); + let declaration = batch.declarations[0].clone(); + batch.declarations = (0..=MAX_BATCH_DOCUMENTATION_BYTES / MAX_DECLARATION_DOCUMENTATION_BYTES) + .map(|index| { + let mut value = declaration.clone(); + value.id = format!("decl:{index}"); + value.graph_node_id = format!("graph:{index}"); + value + }) + .collect(); + assert_code(&batch, EvidenceErrorCode::ResourceLimit); +} + #[test] fn type_only_bindings_require_type_or_namespace_identity() { let mut batch = valid_batch(); @@ -503,7 +641,7 @@ fn universal_evidence_pipelines_are_unique_sorted_and_truthful() { })); assert_eq!( UniversalEvidenceRegistry::pipeline("go").map(|pipeline| pipeline.producer.version), - Some(1) + Some(2) ); assert_eq!( pipelines @@ -527,17 +665,17 @@ fn universal_evidence_pipelines_are_unique_sorted_and_truthful() { ); assert_eq!( UniversalEvidenceRegistry::pipeline("rust").map(|pipeline| pipeline.producer.version), - Some(2) + Some(3) ); assert_eq!( UniversalEvidenceRegistry::pipeline("javascript") .map(|pipeline| (pipeline.producer.version, pipeline.qualification)), - Some((1, UniversalEvidenceQualification::Qualified)) + Some((3, UniversalEvidenceQualification::Qualifying)) ); assert_eq!( UniversalEvidenceRegistry::pipeline("typescript") .map(|pipeline| (pipeline.producer.version, pipeline.qualification)), - Some((1, UniversalEvidenceQualification::Qualified)) + Some((4, UniversalEvidenceQualification::Qualifying)) ); } @@ -709,7 +847,7 @@ fn typescript_javascript_pipeline_is_wired_into_production_extraction() { assert_eq!(evidence.pipeline.dialect.as_deref(), Some(dialect)); assert_eq!( evidence.pipeline.qualification, - UniversalEvidenceQualification::Qualified + UniversalEvidenceQualification::Qualifying ); assert_eq!(evidence.pipeline.id, format!("compass.{language}")); } diff --git a/crates/compass-mcp/src/code_query.rs b/crates/compass-mcp/src/code_query.rs index 5ae475d05..65216d205 100644 --- a/crates/compass-mcp/src/code_query.rs +++ b/crates/compass-mcp/src/code_query.rs @@ -235,6 +235,9 @@ pub(super) fn invoke_with_engine( pub(super) fn has_discovery_arguments(arguments: &Map) -> bool { [ + "view", + "text_budget", + "include_context", "direction", "relation_contexts", "scope", @@ -258,6 +261,9 @@ pub(super) fn validate_query_graph_arguments( ) -> Result<(), super::InvocationError> { const ALLOWED: &[&str] = &[ "question", + "view", + "text_budget", + "include_context", "semantic_search", "project_path", "mode", @@ -289,6 +295,40 @@ pub(super) fn validate_query_graph_arguments( let legacy = ["mode", "depth", "token_budget", "context_filter"] .iter() .any(|name| arguments.contains_key(*name)); + if let Some(view) = arguments.get("view") + && view.as_str() != Some("ranked") + { + return Err(super::InvocationError::InvalidParams( + "view must be ranked when provided".to_owned(), + )); + } + if legacy && arguments.contains_key("view") { + return Err(super::InvocationError::InvalidParams( + "view requires bounded discovery; omit legacy traversal controls".to_owned(), + )); + } + if let Some(budget) = arguments.get("text_budget") { + if !arguments.contains_key("view") { + return Err(super::InvocationError::InvalidParams( + "text_budget requires view=ranked".to_owned(), + )); + } + if !budget + .as_u64() + .is_some_and(|budget| (1..=65_536).contains(&budget)) + { + return Err(super::InvocationError::InvalidParams( + "text_budget must be an integer between 1 and 65536".to_owned(), + )); + } + } + if let Some(include_context) = arguments.get("include_context") + && (!arguments.contains_key("view") || !include_context.is_boolean()) + { + return Err(super::InvocationError::InvalidParams( + "include_context must be a boolean and requires view=ranked".to_owned(), + )); + } if legacy && arguments.contains_key("semantic_search") { return Err(super::InvocationError::InvalidParams( "semantic_search requires bounded discovery; omit legacy traversal controls".to_owned(), @@ -306,6 +346,26 @@ pub(super) fn invoke_discovery_with_engine( arguments: &Map, engine: &CodeQueryEngine, ) -> Result { + engine + .discover(discovery_request(arguments)?) + .map_err(query_invocation_error) +} + +pub(super) fn invoke_ranked_with_engine( + arguments: &Map, + engine: &CodeQueryEngine, +) -> Result { + engine + .discover_ranked_with_context( + discovery_request(arguments)?, + boolean(arguments, "include_context")?, + ) + .map_err(query_invocation_error) +} + +fn discovery_request( + arguments: &Map, +) -> Result { let defaults = DiscoveryLimits::default(); let request = DiscoveryQueryRequest { question: required_string(arguments, "question")?, @@ -343,7 +403,7 @@ pub(super) fn invoke_discovery_with_engine( timeout_ms: u64_value(arguments, "timeout_ms", defaults.timeout_ms)?, }, }; - engine.discover(request).map_err(query_invocation_error) + Ok(request) } fn query_invocation_error(error: compass_query::QueryError) -> super::InvocationError { diff --git a/crates/compass-mcp/src/lib.rs b/crates/compass-mcp/src/lib.rs index b2307528f..b5f3d5a1f 100644 --- a/crates/compass-mcp/src/lib.rs +++ b/crates/compass-mcp/src/lib.rs @@ -1312,6 +1312,50 @@ fn invoke_discovery_tool( }) })?; engine.set_semantic_search(semantic); + if arguments.get("view").and_then(Value::as_str) == Some("ranked") { + let response = code_query::invoke_ranked_with_engine(arguments, &engine)?; + let include_context = arguments + .get("include_context") + .and_then(Value::as_bool) + .unwrap_or(false); + let view = compass_output::build_ranked_query_view_with_context( + &response, + engine.build_generation_identity(), + engine.graph_identity(), + include_context, + ) + .map_err(|error| InvocationError::Internal(error.to_string()))?; + let budget = arguments + .get("text_budget") + .and_then(Value::as_u64) + .unwrap_or(800); + let digest = compass_query::discovery_response_digest(&response.discovery) + .map_err(|error| InvocationError::Internal(error.to_string()))?; + let mut envelope = transport_envelope_with_digest( + serde_json::to_value(&response.discovery) + .map_err(|error| InvocationError::Internal(error.to_string()))?, + Some(&digest), + )?; + envelope["rankedView"] = serde_json::to_value(&view) + .map_err(|error| InvocationError::Internal(error.to_string()))?; + let mut invocation = decorate_freshness( + ToolInvocation { + text: String::new(), + structured_content: Some(validate_transport_envelope(envelope)?), + }, + engine.graph_path(), + engine.source_commit(), + )?; + // Freshness warnings share the whole-answer budget with the shortlist. + let remaining_bytes = usize::try_from(budget) + .unwrap_or(usize::MAX) + .saturating_mul(4) + .saturating_sub(invocation.text.len()); + let text = compass_output::render_ranked_query_text(&view, remaining_bytes / 4) + .map_err(|error| InvocationError::InvalidParams(error.to_string()))?; + invocation.text.push_str(&text); + return Ok(invocation); + } let response = code_query::invoke_discovery_with_engine(arguments, &engine)?; let question = arguments .get("question") @@ -1514,6 +1558,9 @@ fn tool_specs() -> Vec { "Run bounded structured discovery for natural-language questions; explicit legacy traversal fields preserve compatibility text context.", json!({"type":"object","additionalProperties":false,"properties":{ "question":{"type":"string","minLength":1,"maxLength":MAX_DISCOVERY_QUESTION_BYTES,"description":"Natural language question or keyword search"}, + "view":{"type":"string","enum":["ranked"],"description":"Opt-in implementation shortlist in compass.query.ranked-view/4; result retains the raw discovery contract."}, + "text_budget":{"type":"integer","minimum":1,"maximum":65536,"default":800,"description":"Whole ranked text budget (UTF-8 bytes divided by four); requires view=ranked."}, + "include_context":{"type":"boolean","default":false,"description":"Append bounded one-hop witnessed context after the ranked shortlist; requires view=ranked."}, "mode":{"type":"string","enum":["bfs","dfs"],"description":"Explicit legacy mode; selects compatibility traversal"}, "depth":{"type":"integer","minimum":0,"maximum":6,"description":"Explicit legacy traversal depth (1-6)"}, "token_budget":{"type":"integer","minimum":0,"description":"Explicit legacy text token budget"}, diff --git a/crates/compass-mcp/tests/code_query_tools.rs b/crates/compass-mcp/tests/code_query_tools.rs index 7db566acd..d888d10a0 100644 --- a/crates/compass-mcp/tests/code_query_tools.rs +++ b/crates/compass-mcp/tests/code_query_tools.rs @@ -21,6 +21,125 @@ use rmcp::ServiceExt; use rmcp::model::CallToolRequestParams; use serde_json::{Map, Value, json}; +#[test] +fn broad_partial_context_is_disclosed_in_default_and_ranked_tools() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let path = write_typed_graph(directory.path())?; + let mut graph = GraphDocument::load(&path)?; + let node = graph + .nodes + .iter_mut() + .find(|node| node.name == "Target") + .ok_or("missing fixture target")?; + node.name = "format_protocol_errors".into(); + node.qualified_name = "Fixture.format_protocol_errors".into(); + fs::write(&path, serde_json::to_vec(&graph)?)?; + let server = CompassMcp::new(&path); + for question in [ + "where are redis protocol errors formatted", + "where are protocol errors formatted without decoration", + "where are protocol errors formatted before decoration", + "where are protocol errors formatted after decoration", + ] { + for view in [None, Some("ranked")] { + let mut arguments = json!({"question":question}); + if let Some(view) = view { + arguments["view"] = json!(view); + } + let output = server.invoke( + "query_graph", + arguments.as_object().cloned().ok_or("arguments")?, + ); + let envelope: Value = serde_json::from_str(&output)?; + let diagnostics = envelope["result"]["diagnostics"].to_string(); + assert!(diagnostics.contains("inspection candidates"), "{output}"); + assert!(diagnostics.contains("not established"), "{output}"); + if question.contains("before") || question.contains("after") { + assert_eq!(envelope["result"]["question"], question); + assert!(diagnostics.contains("Temporal qualification"), "{output}"); + if view.is_some() { + assert!( + envelope["rankedView"]["candidates"][0]["reasons"] + .to_string() + .contains("Temporal qualification"), + "{output}" + ); + } + } + if view.is_some() { + assert_eq!(envelope["rankedView"]["candidates"][0]["relevance"], "low"); + } + } + } + Ok(()) +} + +#[test] +fn ranked_discovery_is_a_separate_typed_projection() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let graph = write_typed_graph(directory.path())?; + let server = CompassMcp::new(&graph); + let arguments = + json!({"question":"where is Target defined?", "view":"ranked", "text_budget":800}); + let output = server.invoke( + "query_graph", + arguments.as_object().cloned().ok_or("arguments")?, + ); + let envelope: Value = serde_json::from_str(&output)?; + assert_eq!(envelope["schema"], "compass.mcp.tool-result/1"); + assert_eq!(envelope["result"]["schema"], "compass.query.discovery/1"); + assert_eq!( + envelope["rankedView"]["schema"], + "compass.query.ranked-view/4" + ); + assert_eq!( + envelope["rankedView"]["candidates"][0]["name"], + "Fixture.Target" + ); + assert!(envelope["rankedView"].get("context").is_none()); + let context: Value = serde_json::from_str( + &server.invoke( + "query_graph", + json!({"question":"where is Target defined?", "view":"ranked", "include_context":true}) + .as_object() + .cloned() + .ok_or("arguments")?, + ), + )?; + assert_eq!( + context["rankedView"]["candidates"], + envelope["rankedView"]["candidates"] + ); + assert!(context["rankedView"]["context"]["edges"].is_array()); + let plain = server.invoke( + "query_graph", + json!({"question":"where is Target defined?"}) + .as_object() + .cloned() + .ok_or("arguments")?, + ); + let plain: Value = serde_json::from_str(&plain)?; + assert!(plain.get("rankedView").is_none()); + assert!(plain.get("agentView").is_some()); + let invalid = CompassMcp::new(directory.path().join("missing.json")); + for arguments in [ + json!({"question":"Target", "view":"other"}), + json!({"question":"Target", "view":"ranked", "mode":"bfs"}), + json!({"question":"Target", "view":"ranked", "text_budget":-1}), + json!({"question":"Target", "text_budget":800}), + json!({"question":"Target", "include_context":true}), + json!({"question":"Target", "view":"ranked", "include_context":"true"}), + ] { + let error = invalid.invoke( + "query_graph", + arguments.as_object().cloned().ok_or("arguments")?, + ); + assert!(!error.contains("missing.json"), "{error}"); + assert!(error.starts_with("Error executing"), "{error}"); + } + Ok(()) +} + fn write_typed_graph(root: &Path) -> Result> { let graph_path = root.join("graph.json"); fs::create_dir_all(root.join("src"))?; @@ -486,6 +605,11 @@ fn query_graph_schema_exposes_typed_discovery_controls() -> Result<(), Box Result< } Ok(()) } + +#[test] +fn broad_body_evidence_survives_native_update_and_mcp_current_snapshot_caches() +-> Result<(), Box> { + for storage in [ + compass_core::GraphStorage::Json, + compass_core::GraphStorage::Sqlite, + ] { + let directory = tempfile::tempdir()?; + let root = directory.path().join("repo"); + fs::create_dir(&root)?; + fs::write( + root.join("sample.rs"), + "fn opaque(value: &str) -> &str {\nlet returned = value;\n// Handle payload normalization here.\nreturned }\nfn decoy() {}\n", + )?; + let mut options = compass_core::BuildOptions::new(&root); + options.output_root = Some(directory.path().join("artifacts")); + options.graph_storage = storage; + options.no_cluster = true; + options.no_viz = true; + options.max_workers = Some(1); + let built = compass_core::build_local_graph(&options)?; + let server = CompassMcp::new(built.output_dir.join("graph.json")); + for _ in 0..2 { + for view in [None, Some("ranked")] { + let mut args = json!({"question":"Where is payload handled?"}); + if let Some(view) = view { + args["view"] = json!(view); + } + let output = + server.invoke("query_graph", args.as_object().cloned().ok_or("arguments")?); + let envelope: Value = serde_json::from_str(&output)?; + assert!( + envelope["result"]["seeds"] + .as_array() + .is_some_and(|seeds| !seeds.is_empty()), + "{output}" + ); + if view.is_some() { + let candidate = &envelope["rankedView"]["candidates"][0]; + assert!( + candidate["name"] + .as_str() + .is_some_and(|name| name.contains("opaque")), + "{output}" + ); + assert_eq!(candidate["relevance"], "low"); + let view = compass_output::RankedQueryView::from_json(&serde_json::to_vec( + &envelope["rankedView"], + )?)?; + let quote = view + .candidates + .first() + .and_then(|candidate| candidate.excerpts.first()) + .ok_or("missing MCP callable quote")?; + assert_eq!( + quote.kind, + compass_query::RankedExcerptKind::CallableComment + ); + let source = fs::read_to_string(root.join("sample.rs"))?; + assert_eq!( + source + .get(quote.source.start_byte as usize..quote.source.end_byte as usize), + Some(quote.text.as_str()) + ); + assert!(quote.text.contains("payload normalization")); + assert!( + candidate["matchedFields"] + .as_array() + .is_some_and(|fields| fields + .iter() + .any(|field| field == "body_comment")), + "{output}" + ); + } + } + } + } + Ok(()) +} diff --git a/crates/compass-model/src/lib.rs b/crates/compass-model/src/lib.rs index 33e3b4c1a..ed7dca1f6 100644 --- a/crates/compass-model/src/lib.rs +++ b/crates/compass-model/src/lib.rs @@ -15,6 +15,7 @@ pub mod provenance; pub mod query_contract; mod query_index; pub mod search; +pub mod source_query; mod validation; pub use artifact::GraphArtifact; diff --git a/crates/compass-model/src/provenance.rs b/crates/compass-model/src/provenance.rs index 47ed9a7ed..8ca8c0afa 100644 --- a/crates/compass-model/src/provenance.rs +++ b/crates/compass-model/src/provenance.rs @@ -11,6 +11,109 @@ pub const NODE_PROVENANCE_ANCHOR_ATTRIBUTE: &str = "_node_provenance_anchor"; pub const TRUSTED_EDGE_RECORD_ATTRIBUTE: &str = "_compass_v1_edge_record"; pub const TRUSTED_NODE_RECORD_ATTRIBUTE: &str = "_compass_v1_node_record"; pub const SEMANTIC_LAYER_EXTRACTOR: &str = "compass.semantic"; +/// AST attachment rule for a bounded leading excerpt and its exact declaration. +pub const LEADING_DOCUMENTATION_RULE: &str = "universal-leading-documentation-v1"; +/// Internal raw projection payload; publication retains it as a provenance anchor. +pub const LEADING_DOCUMENTATION_OWNER_ANCHOR_ATTRIBUTE: &str = + "_compass_leading_documentation_owner_anchor"; + +/// Verify the cross-record ownership proof used by leading-document retrieval. +/// +/// Generic `documents` relationships and same-name/distant resources do not +/// establish ownership. The producer must retain both exact source intervals +/// together under the known AST attachment rule. Source auditing remains +/// necessary to establish that a producer observed the syntax it claims. +#[must_use] +pub fn is_proven_leading_documentation( + document: &crate::code_graph::NodeRecord, + owner: &crate::code_graph::NodeRecord, + edge: &crate::code_graph::EdgeRecord, +) -> bool { + use crate::code_graph::{EdgeKind, NodeDetails, NodeKind, ResourceKind}; + + if edge.kind != EdgeKind::Documents + || edge.source != document.id + || edge.target != owner.id + || document.kind != NodeKind::Resource + || !matches!( + owner.kind, + NodeKind::Function + | NodeKind::Method + | NodeKind::Module + | NodeKind::Class + | NodeKind::Struct + | NodeKind::Interface + | NodeKind::Trait + | NodeKind::Enum + | NodeKind::TypeAlias + ) + || edge.occurrence_rule.as_ref().map(OccurrenceRule::as_str) + != Some(LEADING_DOCUMENTATION_RULE) + { + return false; + } + let Some(NodeDetails::Resource(details)) = &document.details else { + return false; + }; + if details.resource_kind != ResourceKind::Rationale + || !details + .content + .as_ref() + .is_some_and(|content| !content.is_empty() && content.len() <= 4096) + { + return false; + } + let (Some(excerpt), Some(definition)) = (&document.source, &owner.source) else { + return false; + }; + if edge.relationship_site.as_ref() != Some(excerpt) + || !excerpt.is_valid() + || !definition.is_valid() + || excerpt.file != definition.file + || excerpt.end_byte > definition.start_byte + || (excerpt.end_line, excerpt.end_column) > (definition.start_line, definition.start_column) + || details.content.as_ref().map(|content| content.len() as u64) + != excerpt.end_byte.checked_sub(excerpt.start_byte) + { + return false; + } + let exact_nonheuristic = |evidence: &Provenance| { + evidence.origin != EvidenceOrigin::Heuristic + && evidence.confidence == EvidenceConfidence::Exact + && evidence.validate().is_ok() + }; + if !edge.evidence.iter().all(exact_nonheuristic) + || !document.evidence.iter().all(exact_nonheuristic) + || !document.evidence.iter().any(|evidence| { + evidence.origin == EvidenceOrigin::Ast && evidence.anchors.contains(excerpt) + }) + || !owner.evidence.iter().any(|evidence| { + exact_nonheuristic(evidence) + && evidence.origin == EvidenceOrigin::Ast + && evidence.anchors.iter().any(|anchor| { + anchor.file == definition.file + && definition.start_byte <= anchor.start_byte + && anchor.end_byte <= definition.end_byte + && (definition.start_line, definition.start_column) + <= (anchor.start_line, anchor.start_column) + && (anchor.end_line, anchor.end_column) + <= (definition.end_line, definition.end_column) + }) + }) + { + return false; + } + edge.evidence.iter().any(|proof| { + proof.origin == EvidenceOrigin::Ast + && proof.confidence == EvidenceConfidence::Exact + && proof.rule.as_deref() == Some(LEADING_DOCUMENTATION_RULE) + && proof.anchors.len() == 2 + && proof.anchors.contains(excerpt) + && proof.anchors.contains(definition) + && proof.wiring_site.is_none() + && proof.validate().is_ok() + }) +} /// Immutable producer rule used to distinguish relationship occurrences. #[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] diff --git a/crates/compass-model/src/search.rs b/crates/compass-model/src/search.rs index 3268973da..3695a64bc 100644 --- a/crates/compass-model/src/search.rs +++ b/crates/compass-model/src/search.rs @@ -3,12 +3,164 @@ use std::collections::{BTreeMap, BTreeSet}; use unicode_normalization::UnicodeNormalization; use unicode_normalization::char::is_combining_mark; -use crate::code_graph::{EdgeKind, GraphDocument}; +use crate::code_graph::{EdgeKind, GraphDocument, NodeKind, NodeRecord}; use crate::provenance::{EvidenceConfidence, EvidenceOrigin}; pub const OPERATION_ROLE_TOKENS: &[&str] = &[ "builder", "factory", "handler", "manager", "provider", "service", ]; +pub const SEARCH_FIELD_MAX_BYTES: usize = 8192; +pub const SEARCH_FIELD_MAX_TERMS: usize = 512; +pub const SEARCH_MAX_ALIASES: usize = 8; + +/// Bounded source-evidence projection shared by native query backends. +/// Fields describe recall evidence, never declaration aliases or graph edges. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SearchDocument { + pub fields: BTreeMap<&'static str, BTreeSet>, + pub truncated: bool, +} + +impl SearchDocument { + #[must_use] + pub fn terms(&self) -> BTreeSet { + self.fields + .values() + .flat_map(|terms| terms.iter().cloned()) + .collect() + } + + /// Reserved implementation channel, using exactly the same enriched + /// projection as document-frequency statistics and relevance scoring. + #[must_use] + pub fn primary_terms(&self, kind: NodeKind) -> BTreeSet { + if is_primary_search_kind(kind) { + self.terms() + .into_iter() + .map(|term| format!("__compass_primary_v1_{term}")) + .collect() + } else { + BTreeSet::new() + } + } +} + +/// Declaration kinds that can own implementation-search evidence. This is a +/// retrieval category and never changes graph kind or structural confidence. +#[must_use] +pub fn is_primary_search_kind(kind: NodeKind) -> bool { + kind.is_callable() + || matches!( + kind, + NodeKind::Class + | NodeKind::Struct + | NodeKind::Interface + | NodeKind::Trait + | NodeKind::Protocol + | NodeKind::Enum + | NodeKind::TypeAlias + | NodeKind::ConfigKey + ) +} + +/// Project only metadata already published with its source owner. General +/// comments are deliberately not associated by proximity at query time. +#[must_use] +pub fn search_document(node: &NodeRecord) -> SearchDocument { + search_document_with_aliases(node, std::iter::empty()) +} + +/// Enrich a declaration with names from its incoming alias relationships. +/// Read at most nine entries. More than eight aliases or 8 KiB of alias text +/// omits the whole alias field and reports truncation, rather than selecting +/// an order-dependent subset. Duplicate alias relationships count toward the +/// work bound; callers must preserve their multiplicity on both backends. +#[must_use] +pub fn search_document_with_aliases<'a>( + node: &NodeRecord, + aliases: impl IntoIterator, +) -> SearchDocument { + use crate::code_graph::NodeDetails; + let mut document = SearchDocument::default(); + let mut field = |name, text: &str| { + let mut end = text.len().min(SEARCH_FIELD_MAX_BYTES); + while !text.is_char_boundary(end) { + end -= 1; + } + let bounded = &text[..end]; + let terms = identifier_search_terms(bounded) + .into_iter() + .map(crate::canonical_code_token) + .collect::>(); + document.truncated |= end < text.len() || terms.len() > SEARCH_FIELD_MAX_TERMS; + document.fields.insert( + name, + terms.into_iter().take(SEARCH_FIELD_MAX_TERMS).collect(), + ); + }; + field("name", &node.name); + field("qualified_name", &node.qualified_name); + field("path", node.source_file().unwrap_or_default()); + field("kind", node.kind.as_str()); + field("language", node.language.as_deref().unwrap_or_default()); + let roles = node + .roles + .iter() + .map(|role| format!("{role:?}")) + .collect::>(); + field("roles", &roles.into_iter().collect::>().join(" ")); + match &node.details { + Some(NodeDetails::Symbol(details)) => { + field( + "signature", + details.signature.as_deref().unwrap_or_default(), + ); + field( + "declaring_type", + details.declaring_type.as_deref().unwrap_or_default(), + ); + } + Some(NodeDetails::Document(details)) => { + field("document", details.content.as_deref().unwrap_or_default()) + } + Some(NodeDetails::Resource(details)) => { + field("document", details.content.as_deref().unwrap_or_default()) + } + _ => {} + } + let aliases = aliases + .into_iter() + .take(SEARCH_MAX_ALIASES + 1) + .collect::>(); + let alias_bytes = aliases + .iter() + .fold(0_usize, |total, alias| total.saturating_add(alias.len())); + if aliases.len() > SEARCH_MAX_ALIASES || alias_bytes > SEARCH_FIELD_MAX_BYTES { + document.truncated = true; + } else if !aliases.is_empty() { + // Sorting before normalization makes the projection independent of + // graph insertion order and of alias occurrence ordering. + let aliases = aliases.into_iter().collect::>(); + let terms = aliases + .into_iter() + .flat_map(identifier_search_terms) + .map(crate::canonical_code_token) + .collect::>(); + document.truncated |= terms.len() > SEARCH_FIELD_MAX_TERMS; + document.fields.insert( + "alias", + terms.into_iter().take(SEARCH_FIELD_MAX_TERMS).collect(), + ); + } + document +} + +/// Reserved native recall channel for implementations and type declarations. +/// A common term's parameter leaves cannot exhaust this posting quota. +#[must_use] +pub fn primary_evidence_terms(node: &NodeRecord) -> BTreeSet { + search_document(node).primary_terms(node.kind) +} /// Structural relationships that can establish a source-backed dependency /// on a named target. The structural relationship commands use the bounded @@ -257,10 +409,90 @@ mod tests { use crate::provenance::{EvidenceConfidence, EvidenceOrigin, Provenance, SourceAnchor}; use super::{ - direct_call_source_identifier_postings, direct_call_source_identifier_targets, - identifier_search_terms, relationship_source_identifier_postings, + SEARCH_FIELD_MAX_BYTES, SEARCH_FIELD_MAX_TERMS, direct_call_source_identifier_postings, + direct_call_source_identifier_targets, identifier_search_terms, + relationship_source_identifier_postings, search_document_with_aliases, }; + fn projection_node() -> NodeRecord { + NodeRecord { + id: "target".to_owned(), + kind: NodeKind::Function, + roles: vec![ + crate::code_graph::NodeRole::RouteHandler, + crate::code_graph::NodeRole::Service, + ], + name: "execute".to_owned(), + qualified_name: "app::execute".to_owned(), + language: None, + framework: None, + source: None, + details: None, + evidence: Vec::new(), + coverage: Vec::new(), + diagnostics: Vec::new(), + community: None, + } + } + + #[test] + fn fielded_projection_preserves_alias_origin_kind_roles_and_canonical_order() { + let mut node = projection_node(); + let a = search_document_with_aliases(&node, ["RenewalGateway", "PaymentEndpoint"]); + node.roles.reverse(); + let b = search_document_with_aliases(&node, ["PaymentEndpoint", "RenewalGateway"]); + assert_eq!(a, b); + assert!(!a.truncated); + assert!(a.fields["alias"].contains("renewal")); + assert!(a.fields["kind"].contains("function")); + assert!(a.fields["roles"].contains("route")); + assert!(a.fields["roles"].contains("handler")); + assert!(!a.fields["name"].contains("renewal")); + assert!( + a.primary_terms(NodeKind::Function) + .contains("__compass_primary_v1_renewal") + ); + assert!(a.primary_terms(NodeKind::Parameter).is_empty()); + } + + #[test] + fn alias_projection_omits_overflow_instead_of_selecting_an_arbitrary_subset() { + let node = projection_node(); + let aliases = (0..9) + .map(|index| format!("Alias{index}")) + .collect::>(); + let a = search_document_with_aliases(&node, aliases.iter().map(String::as_str)); + let b = search_document_with_aliases(&node, aliases.iter().rev().map(String::as_str)); + assert_eq!(a, b); + assert!(a.truncated); + assert!(!a.fields.contains_key("alias")); + let oversized = "x".repeat(SEARCH_FIELD_MAX_BYTES + 1); + let too_large = search_document_with_aliases(&node, [oversized.as_str()]); + assert!(too_large.truncated); + assert!(!too_large.fields.contains_key("alias")); + let read_count = std::cell::Cell::new(0); + let unbounded = + std::iter::repeat("alias").inspect(|_| read_count.set(read_count.get() + 1)); + assert!(search_document_with_aliases(&node, unbounded).truncated); + assert_eq!(read_count.get(), 9); + } + + #[test] + fn source_fields_bound_utf8_bytes_and_unique_terms_independently() { + let mut node = projection_node(); + node.name = format!("{}🙂beyondlimit", "é".repeat(4095)); + let document = super::search_document(&node); + assert!(document.truncated); + assert!(!document.fields["name"].contains("beyondlimit")); + let many_terms = (0..600) + .map(|index| format!("token{index:03}")) + .collect::>() + .join(" "); + let document = search_document_with_aliases(&projection_node(), [many_terms.as_str()]); + assert!(document.truncated); + assert_eq!(document.fields["alias"].len(), SEARCH_FIELD_MAX_TERMS); + } + #[test] fn preserves_full_tokens_and_adds_identifier_subwords() { assert_eq!( diff --git a/crates/compass-model/src/source_query.rs b/crates/compass-model/src/source_query.rs new file mode 100644 index 000000000..22bbe189a --- /dev/null +++ b/crates/compass-model/src/source_query.rs @@ -0,0 +1,419 @@ +//! Immutable auxiliary lexical evidence. It never adds semantic graph records. +use crate::provenance::SourceAnchor; +use crate::search::{ + SEARCH_FIELD_MAX_BYTES, SEARCH_FIELD_MAX_TERMS, SearchDocument, identifier_search_terms, +}; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; + +pub const SOURCE_QUERY_SCHEMA: &str = "compass.source-query/1"; +pub const SOURCE_QUERY_PROFILE_V1: &str = "source-query/1-ast-callable-lexemes"; +pub const SOURCE_QUERY_PROFILE_V2: &str = "source-query/2-callable-definition-envelopes"; +pub const SOURCE_QUERY_PROFILE_V3: &str = "source-query/3-owned-code-windows"; +pub const SOURCE_QUERY_PROFILE: &str = "source-query/4-nested-callable-context"; +pub const SOURCE_QUERY_MAX_NESTED_CONTEXTS: usize = 8; +pub const SOURCE_QUERY_MAX_CODE_BYTES: usize = 8 * 1024 * 1024; +pub const SOURCE_QUERY_MAX_CODE_WINDOWS: usize = 32_768; +pub const SOURCE_QUERY_FILE: &str = "source-query.json"; +pub const SOURCE_QUERY_MAX_BYTES: u64 = 64 * 1024 * 1024; +pub const SOURCE_QUERY_MAX_OWNERS: usize = 32_768; +pub const SOURCE_QUERY_MAX_SITES: usize = 65_536; +pub const SOURCE_QUERY_MAX_TEXT_BYTES: usize = 16 * 1024 * 1024; + +#[must_use] +pub fn is_supported_source_query_profile(profile: &str) -> bool { + matches!( + profile, + SOURCE_QUERY_PROFILE_V1 + | SOURCE_QUERY_PROFILE_V2 + | SOURCE_QUERY_PROFILE_V3 + | SOURCE_QUERY_PROFILE + ) +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct SourceQuerySite { + pub source: SourceAnchor, + pub text: String, +} + +/// Lexically nested anonymous-callable inspection; not a call relationship. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct SourceQueryNestedContext { + pub source: SourceAnchor, + pub syntax: String, + pub code_windows: Vec, + pub omitted_code_windows: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct SourceQueryOwner { + pub node_id: String, + pub source: SourceAnchor, + pub source_digest: String, + pub identifiers: Vec, + pub comments: Vec, + /// Exact AST-owned inspection text; excluded from retrieval statistics. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub code_windows: Vec, + #[serde(default, skip_serializing_if = "is_zero")] + pub omitted_code_windows: u64, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub nested_contexts: Vec, + #[serde(default, skip_serializing_if = "is_zero")] + pub omitted_nested_contexts: u64, + pub omitted_sites: u64, + pub truncated: bool, + pub syntax_recovery: bool, +} + +/// Per-file coverage survives deterministic fact-neutral incremental updates. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct SourceQueryFile { + pub file: String, + pub source_digest: String, + pub byte_size: u64, + pub language: String, + pub truncated: bool, + pub syntax_recovery: bool, + pub omitted_owners: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct SourceQueryArtifact { + pub schema: String, + pub profile: String, + pub graph_digest: String, + pub files: Vec, + pub owners: Vec, + pub aggregate_truncated: bool, + pub omitted_owners: u64, + pub omitted_files: u64, + /// Count only owner documents where the term is absent from the complete + /// graph-only field projection. This adds to the same backend's corpus DF. + pub additional_document_frequencies: BTreeMap, +} + +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +#[error("invalid source-query artifact: {0}")] +pub struct SourceQueryError(pub String); + +fn is_zero(value: &u64) -> bool { + *value == 0 +} + +fn digest(value: &str) -> bool { + value.len() == 64 + && value + .bytes() + .all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase()) +} +fn contained(parent: &SourceAnchor, child: &SourceAnchor) -> bool { + parent.file == child.file + && parent.start_byte <= child.start_byte + && child.end_byte <= parent.end_byte + && (parent.start_line, parent.start_column) <= (child.start_line, child.start_column) + && (child.end_line, child.end_column) <= (parent.end_line, parent.end_column) +} +fn safe_source(source: &SourceAnchor) -> bool { + source.is_valid() + && source.start_byte < source.end_byte + && source.file.len() <= 4096 + && !source + .file + .split('/') + .any(|part| part.is_empty() || matches!(part, "." | "..")) + && !source.file.contains([':', '\0']) +} + +impl SourceQueryOwner { + /// Exactly the same bounded body projection is used for recall, scoring + /// and document-frequency increments on every backend. + #[must_use] + pub fn document(&self) -> SearchDocument { + let mut result = SearchDocument { + truncated: self.truncated, + ..SearchDocument::default() + }; + for (field, sites) in [ + ("body_identifier", &self.identifiers), + ("body_comment", &self.comments), + ] { + let mut bytes = 0_usize; + let mut terms = BTreeSet::new(); + for site in sites { + let remaining = SEARCH_FIELD_MAX_BYTES.saturating_sub(bytes); + let mut end = site.text.len().min(remaining); + while !site.text.is_char_boundary(end) { + end -= 1; + } + bytes = bytes.saturating_add(end); + result.truncated |= end < site.text.len(); + terms.extend( + identifier_search_terms(&site.text[..end]) + .into_iter() + .map(crate::canonical_code_token), + ); + } + result.truncated |= terms.len() > SEARCH_FIELD_MAX_TERMS; + if !terms.is_empty() { + result.fields.insert( + field, + terms.into_iter().take(SEARCH_FIELD_MAX_TERMS).collect(), + ); + } + } + result + } +} + +impl SourceQueryArtifact { + pub fn empty(graph_digest: String) -> Self { + Self { + schema: SOURCE_QUERY_SCHEMA.to_owned(), + profile: SOURCE_QUERY_PROFILE.to_owned(), + graph_digest, + files: Vec::new(), + owners: Vec::new(), + aggregate_truncated: false, + omitted_owners: 0, + omitted_files: 0, + additional_document_frequencies: BTreeMap::new(), + } + } + pub fn validate(&self) -> Result<(), SourceQueryError> { + let fail = |message: &str| SourceQueryError(message.to_owned()); + if self.schema != SOURCE_QUERY_SCHEMA || !is_supported_source_query_profile(&self.profile) { + return Err(fail( + "unsupported schema or projection profile; update Compass", + )); + } + if !digest(&self.graph_digest) { + return Err(fail("invalid graph digest")); + } + if self.files.len() > SOURCE_QUERY_MAX_OWNERS || self.owners.len() > SOURCE_QUERY_MAX_OWNERS + { + return Err(fail("owner limit exceeded")); + } + let mut previous_file = None::<&str>; + let mut files = BTreeMap::new(); + for file in &self.files { + if file.file.is_empty() + || file.file.len() > 4096 + || previous_file.is_some_and(|p| p >= file.file.as_str()) + || !file + .source_digest + .strip_prefix("sha256:") + .is_some_and(digest) + || !matches!( + file.language.as_str(), + "python" | "rust" | "go" | "typescript" | "tsx" + ) + { + return Err(fail("invalid or noncanonical source-file inventory")); + } + previous_file = Some(&file.file); + files.insert(file.file.as_str(), file); + } + let mut previous = None::<&str>; + let mut total_code_sites = 0_usize; + let mut total_code_text = 0_usize; + let mut total_sites = 0_usize; + let mut total_text = 0_usize; + let mut body_frequencies = BTreeMap::::new(); + for owner in &self.owners { + if owner.node_id.is_empty() + || owner.node_id.len() > 4096 + || previous.is_some_and(|p| p >= owner.node_id.as_str()) + { + return Err(fail( + "owners must have bounded unique IDs in canonical order", + )); + } + previous = Some(&owner.node_id); + if !safe_source(&owner.source) + || !owner + .source_digest + .strip_prefix("sha256:") + .is_some_and(digest) + { + return Err(fail("invalid owner source or digest")); + } + if !files.get(owner.source.file.as_str()).is_some_and(|file| { + file.source_digest == owner.source_digest && owner.source.end_byte <= file.byte_size + }) { + return Err(fail("owner does not match the source-file inventory")); + } + if owner.identifiers.len() > 1024 + || owner.comments.len() > 8 + || owner.code_windows.len() > 16 + { + return Err(fail("per-owner site limit exceeded")); + } + if !matches!( + self.profile.as_str(), + SOURCE_QUERY_PROFILE_V3 | SOURCE_QUERY_PROFILE + ) && (!owner.code_windows.is_empty() || owner.omitted_code_windows > 0) + { + return Err(fail("legacy profile does not support code windows")); + } + for (sites, cap, code) in [ + (&owner.identifiers, 256_usize, false), + (&owner.comments, 4096, false), + (&owner.code_windows, 512, true), + ] { + let mut previous = None; + for site in sites { + let key = (site.source.start_byte, site.source.end_byte); + if previous.is_some_and(|p| p >= key) { + return Err(fail("sites must be distinct and in source order")); + } + previous = Some(key); + if !safe_source(&site.source) + || !contained(&owner.source, &site.source) + || site.text.is_empty() + || site.text.len() > cap + || site.source.end_byte - site.source.start_byte != site.text.len() as u64 + { + return Err(fail("invalid source excerpt")); + } + let mut line = site.source.start_line; + let mut column = site.source.start_column; + for byte in site.text.bytes() { + if byte == b'\n' { + line = line + .checked_add(1) + .ok_or_else(|| fail("source line overflow"))?; + column = 0; + } else { + column = column + .checked_add(1) + .ok_or_else(|| fail("source column overflow"))?; + } + } + if (line, column) != (site.source.end_line, site.source.end_column) { + return Err(fail("excerpt coordinates disagree with UTF-8 bytes")); + } + if code { + total_code_sites = total_code_sites.saturating_add(1); + total_code_text = total_code_text.saturating_add(site.text.len()); + if total_code_sites > SOURCE_QUERY_MAX_CODE_WINDOWS + || total_code_text > SOURCE_QUERY_MAX_CODE_BYTES + { + return Err(fail("aggregate code window limit exceeded")); + } + continue; + } + total_sites = total_sites.saturating_add(1); + total_text = total_text.saturating_add(site.text.len()); + if total_sites > SOURCE_QUERY_MAX_SITES + || total_text > SOURCE_QUERY_MAX_TEXT_BYTES + { + return Err(fail("aggregate source evidence limit exceeded")); + } + } + } + if owner.nested_contexts.len() > SOURCE_QUERY_MAX_NESTED_CONTEXTS + || (self.profile != SOURCE_QUERY_PROFILE + && (!owner.nested_contexts.is_empty() || owner.omitted_nested_contexts > 0)) + { + return Err(fail( + "nested context is oversized or unsupported by this profile", + )); + } + let mut previous_context = None; + let mut owner_code_sites = owner.code_windows.len(); + for context in &owner.nested_contexts { + let key = (context.source.start_byte, context.source.end_byte); + if previous_context.is_some_and(|prior| prior >= key) + || !safe_source(&context.source) + || !contained(&owner.source, &context.source) + || (context.source.start_byte == owner.source.start_byte + && context.source.end_byte == owner.source.end_byte) + || context.code_windows.is_empty() + { + return Err(fail("invalid or noncanonical nested callable context")); + } + let supported_syntax = files.get(owner.source.file.as_str()).is_some_and(|file| { + matches!( + (file.language.as_str(), context.syntax.as_str()), + ("python", "lambda") + | ("rust", "closure_expression") + | ("go", "func_literal") + | ( + "typescript" | "tsx", + "arrow_function" | "function_expression" | "generator_function" + ) + ) + }); + if !supported_syntax { + return Err(fail( + "nested callable syntax does not match the source language", + )); + } + previous_context = Some(key); + let mut previous_window = None; + for window in &context.code_windows { + let key = (window.source.start_byte, window.source.end_byte); + if previous_window.is_some_and(|prior| prior >= key) + || !safe_source(&window.source) + || !contained(&context.source, &window.source) + || window.text.is_empty() + || window.text.len() > 512 + || window.source.end_byte - window.source.start_byte + != window.text.len() as u64 + { + return Err(fail("invalid nested code quotation")); + } + previous_window = Some(key); + let mut line = window.source.start_line; + let mut column = window.source.start_column; + for byte in window.text.bytes() { + if byte == b'\n' { + line = line + .checked_add(1) + .ok_or_else(|| fail("source line overflow"))?; + column = 0; + } else { + column = column + .checked_add(1) + .ok_or_else(|| fail("source column overflow"))?; + } + } + if (line, column) != (window.source.end_line, window.source.end_column) { + return Err(fail("nested quotation coordinates disagree with bytes")); + } + owner_code_sites += 1; + total_code_sites = total_code_sites.saturating_add(1); + total_code_text = total_code_text.saturating_add(window.text.len()); + if owner_code_sites > 16 + || total_code_sites > SOURCE_QUERY_MAX_CODE_WINDOWS + || total_code_text > SOURCE_QUERY_MAX_CODE_BYTES + { + return Err(fail("nested code exceeds shared code-window limits")); + } + } + } + for term in owner.document().terms() { + *body_frequencies.entry(term).or_default() += 1; + } + } + if !self + .additional_document_frequencies + .keys() + .eq(body_frequencies.keys()) + || self + .additional_document_frequencies + .iter() + .any(|(term, df)| *df > body_frequencies.get(term).copied().unwrap_or(0)) + { + return Err(fail("invalid additional document frequency")); + } + Ok(()) + } +} diff --git a/crates/compass-model/src/validation.rs b/crates/compass-model/src/validation.rs index cdf97d4a2..9cb6b8cd4 100644 --- a/crates/compass-model/src/validation.rs +++ b/crates/compass-model/src/validation.rs @@ -517,6 +517,15 @@ fn validate_code_graph_edge( site )); } + if edge.occurrence_rule.as_ref().map(|rule| rule.as_str()) + == Some(crate::provenance::LEADING_DOCUMENTATION_RULE) + && !crate::provenance::is_proven_leading_documentation(source, target, edge) + { + errors.push(format!( + "edge {} has an invalid leading documentation ownership proof", + edge.id + )); + } (!errors.is_empty()).then(|| RecordValidationErrors { id: edge.id.clone(), errors, diff --git a/crates/compass-model/tests/leading_documentation.rs b/crates/compass-model/tests/leading_documentation.rs new file mode 100644 index 000000000..d5707748a --- /dev/null +++ b/crates/compass-model/tests/leading_documentation.rs @@ -0,0 +1,250 @@ +use compass_model::code_graph::{ + EdgeKind, EdgeRecord, NodeDetails, NodeKind, NodeRecord, ResourceKind, ResourceNodeDetails, +}; +use compass_model::identity::edge_id; +use compass_model::provenance::{ + EvidenceConfidence, EvidenceOrigin, LEADING_DOCUMENTATION_RULE, OccurrenceRule, Provenance, + SourceAnchor, is_proven_leading_documentation, +}; + +fn records() -> (NodeRecord, NodeRecord, EdgeRecord) { + let excerpt = SourceAnchor { + file: "src/lib.rs".to_owned(), + start_byte: 0, + end_byte: 8, + start_line: 1, + start_column: 0, + end_line: 2, + end_column: 0, + }; + let definition = SourceAnchor { + file: "src/lib.rs".to_owned(), + start_byte: 8, + end_byte: 25, + start_line: 2, + start_column: 0, + end_line: 2, + end_column: 17, + }; + let proof = Provenance { + origin: EvidenceOrigin::Ast, + extractor: "compass.resolve.rust.universal".to_owned(), + confidence: EvidenceConfidence::Exact, + rule: Some(LEADING_DOCUMENTATION_RULE.to_owned()), + anchors: vec![excerpt.clone(), definition.clone()], + wiring_site: None, + score: None, + candidates: Vec::new(), + }; + let document = NodeRecord { + id: "documentation".to_owned(), + kind: NodeKind::Resource, + roles: Vec::new(), + name: "Documentation for work".to_owned(), + qualified_name: "sample.work::doc".to_owned(), + language: Some("rust".to_owned()), + framework: None, + source: Some(excerpt.clone()), + details: Some(NodeDetails::Resource(ResourceNodeDetails { + resource_kind: ResourceKind::Rationale, + uri: None, + media_type: None, + content: Some("/// doc\n".to_owned()), + })), + evidence: vec![proof.clone()], + coverage: Vec::new(), + diagnostics: Vec::new(), + community: None, + }; + let mut owner = document.clone(); + owner.id = "owner".to_owned(); + owner.kind = NodeKind::Function; + owner.name = "work".to_owned(); + owner.qualified_name = "sample.work".to_owned(); + owner.source = Some(definition); + owner.details = None; + let id = edge_id( + &document.id, + EdgeKind::Documents, + &owner.id, + Some(&excerpt), + Some(LEADING_DOCUMENTATION_RULE), + ); + let edge = EdgeRecord { + id: id.clone(), + key: id, + source: document.id.clone(), + target: owner.id.clone(), + kind: EdgeKind::Documents, + occurrence_rule: OccurrenceRule::new(LEADING_DOCUMENTATION_RULE), + relationship_site: Some(excerpt), + details: None, + evidence: vec![proof], + weight: None, + context: None, + deferred: false, + diagnostics: Vec::new(), + }; + (document, owner, edge) +} + +#[test] +fn two_exact_ast_anchors_establish_leading_ownership() { + let (document, owner, mut edge) = records(); + assert!(is_proven_leading_documentation(&document, &owner, &edge)); + edge.evidence[0].anchors.reverse(); + assert!(is_proven_leading_documentation(&document, &owner, &edge)); +} + +#[test] +fn generic_heuristic_incomplete_and_wrong_owner_proofs_are_rejected() { + let (document, owner, edge) = records(); + let mut variants = Vec::new(); + let mut other = edge.clone(); + other.occurrence_rule = None; + variants.push(other); + let mut other = edge.clone(); + other.evidence[0].rule = None; + variants.push(other); + let mut other = edge.clone(); + other.evidence[0].origin = EvidenceOrigin::Heuristic; + variants.push(other); + let mut other = edge.clone(); + other.evidence[0].confidence = EvidenceConfidence::Inferred; + variants.push(other); + let mut other = edge.clone(); + other.evidence[0].anchors.pop(); + variants.push(other); + let mut other = edge.clone(); + other.evidence[0].anchors[1].start_byte += 1; + variants.push(other); + let mut other = edge.clone(); + other.relationship_site = None; + variants.push(other); + let mut other = edge.clone(); + other.source = "other".to_owned(); + variants.push(other); + let mut other = edge.clone(); + other.target = "same_name_other_owner".to_owned(); + variants.push(other); + for variant in variants { + assert!(!is_proven_leading_documentation( + &document, &owner, &variant + )); + } + let mut wrong_owner = owner.clone(); + wrong_owner.source = document.source.clone(); + assert!(!is_proven_leading_documentation( + &document, + &wrong_owner, + &edge + )); + let mut wrong_content = document.clone(); + if let Some(NodeDetails::Resource(details)) = &mut wrong_content.details { + details.content = Some("short".to_owned()); + } + assert!(!is_proven_leading_documentation( + &wrong_content, + &owner, + &edge + )); +} + +#[test] +fn strict_graph_validation_rejects_a_claimed_rule_without_owner_proof() +-> Result<(), Box> { + use compass_model::code_graph::{BuildMetadata, ExtractionStatus, FileRecord, GraphDocument}; + let (document, owner, edge) = records(); + let mut graph = GraphDocument::empty_v1(BuildMetadata { + builder_version: "test".to_owned(), + schema_fingerprint: "schema".to_owned(), + source_tree_digest: "tree".to_owned(), + configuration_digest: "config".to_owned(), + generation_id: "generation".to_owned(), + source_commit: None, + }); + graph.graph.files.push(FileRecord { + id: compass_model::identity::file_id("src/lib.rs"), + path: "src/lib.rs".to_owned(), + language: Some("rust".to_owned()), + content_digest: "sha256:test".to_owned(), + byte_size: 100, + generated: false, + extraction_status: ExtractionStatus::Extracted, + extractor_versions: vec!["test".to_owned()], + coverage: Vec::new(), + diagnostics: Vec::new(), + }); + graph.nodes = vec![document, owner]; + graph.links = vec![edge]; + compass_model::validate_code_graph(&graph)?; + graph.links[0].evidence[0].anchors.pop(); + let report = compass_model::validate_code_graph_records(&graph); + assert!(!report.is_valid()); + assert!( + report + .edge_errors + .iter() + .flat_map(|record| &record.errors) + .any(|error| error.contains("invalid leading documentation ownership proof")) + ); + Ok(()) +} + +#[test] +fn contradictory_or_missing_node_evidence_cannot_be_promoted_by_an_exact_edge() { + let (document, owner, edge) = records(); + let mut documents = Vec::new(); + let mut other = document.clone(); + other.evidence.clear(); + documents.push(other); + let mut other = document.clone(); + other.evidence[0].confidence = EvidenceConfidence::Inferred; + documents.push(other); + let mut other = document.clone(); + other.evidence[0].origin = EvidenceOrigin::Config; + documents.push(other); + let mut other = document.clone(); + other.evidence[0].anchors.remove(0); + documents.push(other); + for other in documents { + assert!(!is_proven_leading_documentation(&other, &owner, &edge)); + } + let mut owners = Vec::new(); + let mut other = owner.clone(); + other.evidence.clear(); + owners.push(other); + let mut other = owner.clone(); + other.evidence[0].origin = EvidenceOrigin::Config; + owners.push(other); + let mut other = owner.clone(); + other.evidence[0].confidence = EvidenceConfidence::Inferred; + owners.push(other); + let mut other = owner.clone(); + other.evidence[0].anchors.pop(); + owners.push(other); + for other in owners { + assert!(!is_proven_leading_documentation(&document, &other, &edge)); + } + let mut heuristic = edge.evidence[0].clone(); + heuristic.origin = EvidenceOrigin::Heuristic; + heuristic.confidence = EvidenceConfidence::Inferred; + heuristic.rule = Some("semantic-text-supplement".to_owned()); + heuristic.wiring_site = document.source.clone(); + heuristic.anchors.clear(); + assert!(heuristic.validate().is_ok()); + let mut mixed_document = document.clone(); + mixed_document.evidence.push(heuristic.clone()); + assert!(!is_proven_leading_documentation( + &mixed_document, + &owner, + &edge + )); + let mut mixed_edge = edge; + mixed_edge.evidence.push(heuristic); + assert!(!is_proven_leading_documentation( + &document, + &owner, + &mixed_edge + )); +} diff --git a/crates/compass-model/tests/source_query.rs b/crates/compass-model/tests/source_query.rs new file mode 100644 index 000000000..f5f282e69 --- /dev/null +++ b/crates/compass-model/tests/source_query.rs @@ -0,0 +1,257 @@ +use compass_model::provenance::SourceAnchor; +use compass_model::source_query::{ + SourceQueryArtifact, SourceQueryFile, SourceQueryOwner, SourceQuerySite, +}; + +fn artifact() -> SourceQueryArtifact { + let source = SourceAnchor { + file: "src/sample.rs".to_owned(), + start_byte: 0, + end_byte: 100, + start_line: 1, + start_column: 0, + end_line: 4, + end_column: 1, + }; + let site = SourceQuerySite { + source: SourceAnchor { + file: source.file.clone(), + start_byte: 12, + end_byte: 17, + start_line: 2, + start_column: 4, + end_line: 2, + end_column: 9, + }, + text: "café".to_owned(), + }; + let owner = SourceQueryOwner { + node_id: "owner".to_owned(), + source, + source_digest: format!("sha256:{}", "b".repeat(64)), + identifiers: vec![site], + comments: Vec::new(), + code_windows: Vec::new(), + omitted_code_windows: 0, + nested_contexts: Vec::new(), + omitted_nested_contexts: 0, + omitted_sites: 0, + truncated: false, + syntax_recovery: false, + }; + let mut artifact = SourceQueryArtifact::empty("a".repeat(64)); + artifact.additional_document_frequencies = owner + .document() + .terms() + .into_iter() + .map(|term| (term, 1)) + .collect(); + artifact.files.push(SourceQueryFile { + file: owner.source.file.clone(), + source_digest: owner.source_digest.clone(), + byte_size: 100, + language: "rust".to_owned(), + truncated: false, + syntax_recovery: false, + omitted_owners: 0, + }); + artifact.owners.push(owner); + artifact +} + +#[test] +fn immutable_v1_projection_stays_readable_and_unknown_profiles_fail() +-> Result<(), Box> { + use compass_model::source_query::{ + SOURCE_QUERY_PROFILE, SOURCE_QUERY_PROFILE_V1, SOURCE_QUERY_PROFILE_V2, + SOURCE_QUERY_PROFILE_V3, + }; + for profile in [ + SOURCE_QUERY_PROFILE_V1, + SOURCE_QUERY_PROFILE_V2, + SOURCE_QUERY_PROFILE_V3, + SOURCE_QUERY_PROFILE, + ] { + let mut value = artifact(); + value.profile = profile.to_owned(); + let decoded: SourceQueryArtifact = serde_json::from_slice(&serde_json::to_vec(&value)?)?; + decoded.validate()?; + assert_eq!(decoded, value); + } + let mut value = artifact(); + value.profile = "source-query/999-future".to_owned(); + assert!(value.validate().is_err()); + Ok(()) +} +#[test] +fn round_trip_keeps_source_sites_and_shared_field_terms() -> Result<(), Box> +{ + let artifact = artifact(); + artifact.validate()?; + assert!(!artifact.owners[0].document().fields["body_identifier"].is_empty()); + let bytes = serde_json::to_vec(&artifact)?; + let decoded: SourceQueryArtifact = serde_json::from_slice(&bytes)?; + decoded.validate()?; + assert_eq!(decoded, artifact); + assert_eq!(serde_json::to_vec(&decoded)?, bytes); + Ok(()) +} +#[test] +fn invalid_sources_coordinates_order_and_versions_fail_explicitly() { + let mut a = artifact(); + a.schema = "compass.source-query/2".to_owned(); + assert!(a.validate().is_err()); + let mut a = artifact(); + a.profile = "unknown".to_owned(); + assert!(a.validate().is_err()); + let mut a = artifact(); + a.graph_digest = "x".repeat(64); + assert!(a.validate().is_err()); + let mut a = artifact(); + a.owners[0].source.file = "../outside.rs".to_owned(); + assert!(a.validate().is_err()); + let mut a = artifact(); + a.owners[0].identifiers[0].source.end_column = 8; + assert!(a.validate().is_err()); + let mut a = artifact(); + a.owners[0].identifiers[0].source.end_byte = 101; + assert!(a.validate().is_err()); + let mut a = artifact(); + a.owners.push(a.owners[0].clone()); + assert!(a.validate().is_err()); + let mut a = artifact(); + let duplicate = a.owners[0].identifiers[0].clone(); + a.owners[0].identifiers.push(duplicate); + assert!(a.validate().is_err()); + let mut a = artifact(); + a.additional_document_frequencies.clear(); + assert!(a.validate().is_err()); + let mut a = artifact(); + for df in a.additional_document_frequencies.values_mut() { + *df = 2; + } + assert!(a.validate().is_err()); +} +#[test] +fn field_caps_report_omissions_without_changing_literal_excerpts() { + let mut a = artifact(); + let owner = &mut a.owners[0]; + owner.comments = (0..8) + .map(|_| SourceQuerySite { + source: owner.source.clone(), + text: "a".repeat(4096), + }) + .collect(); + let document = owner.document(); + assert!(document.truncated); + assert_eq!(owner.comments[0].text.len(), 4096); + assert!(document.fields["body_comment"].len() <= 512); +} + +#[test] +fn code_windows_are_bounded_inspection_only_and_legacy_profiles_reject_them() +-> Result<(), Box> { + let mut value = artifact(); + let document = value.owners[0].document(); + let mut window = value.owners[0].identifiers[0].clone(); + window.text = "unindexed_literal".to_owned(); + window.source.end_byte = window.source.start_byte + window.text.len() as u64; + window.source.end_column = window.source.start_column + window.text.len() as u32; + value.owners[0].code_windows = vec![window]; + value.validate()?; + assert_eq!(document, value.owners[0].document()); + let encoded = serde_json::to_vec(&value)?; + assert_eq!( + serde_json::from_slice::(&encoded)?, + value + ); + let mut historical = value.clone(); + historical.profile = compass_model::source_query::SOURCE_QUERY_PROFILE_V3.to_owned(); + historical.validate()?; + let mut bad = value.clone(); + bad.profile = compass_model::source_query::SOURCE_QUERY_PROFILE_V2.to_owned(); + assert!(bad.validate().is_err()); + let mut bad = value.clone(); + bad.owners[0].code_windows[0].source.file = "other.rs".to_owned(); + assert!(bad.validate().is_err()); + let mut bad = value.clone(); + bad.owners[0].code_windows[0].text = "x".repeat(513); + assert!(bad.validate().is_err()); + let mut bad = value; + bad.owners[0].code_windows[0].source.end_column += 1; + assert!(bad.validate().is_err()); + Ok(()) +} + +#[test] +fn nested_context_is_versioned_bounded_and_cannot_change_retrieval_terms() +-> Result<(), Box> { + use compass_model::source_query::{SOURCE_QUERY_PROFILE_V3, SourceQueryNestedContext}; + let mut value = artifact(); + let document = value.owners[0].document(); + let window = value.owners[0].identifiers[0].clone(); + let mut source = value.owners[0].source.clone(); + source.start_byte = 10; + source.end_byte = 40; + source.start_line = 2; + source.start_column = 0; + source.end_line = 3; + source.end_column = 0; + value.owners[0] + .nested_contexts + .push(SourceQueryNestedContext { + source, + syntax: "closure_expression".to_owned(), + code_windows: vec![window], + omitted_code_windows: 0, + }); + value.validate()?; + assert_eq!(document, value.owners[0].document()); + let roundtrip: SourceQueryArtifact = serde_json::from_slice(&serde_json::to_vec(&value)?)?; + assert_eq!(roundtrip, value); + let mut bad = value.clone(); + bad.profile = SOURCE_QUERY_PROFILE_V3.to_owned(); + assert!(bad.validate().is_err()); + let mut bad = value.clone(); + bad.owners[0].nested_contexts[0].source = bad.owners[0].source.clone(); + assert!(bad.validate().is_err()); + let mut bad = value.clone(); + bad.owners[0].nested_contexts[0].source.file = "other.rs".to_owned(); + assert!(bad.validate().is_err()); + let mut bad = value.clone(); + bad.files[0].language = "python".to_owned(); + assert!(bad.validate().is_err()); + let mut bad = value.clone(); + bad.owners[0].nested_contexts[0].syntax = "function_item".to_owned(); + assert!(bad.validate().is_err()); + let mut bad = value.clone(); + bad.owners[0].nested_contexts[0].code_windows[0] + .source + .end_column += 1; + assert!(bad.validate().is_err()); + let mut bad = value.clone(); + bad.owners[0].nested_contexts[0].code_windows[0] + .source + .end_byte = 41; + assert!(bad.validate().is_err()); + let mut bad = value.clone(); + bad.owners[0] + .nested_contexts + .resize(9, value.owners[0].nested_contexts[0].clone()); + assert!(bad.validate().is_err()); + let mut bad = value; + let template = bad.owners[0].nested_contexts[0].code_windows[0].clone(); + bad.owners[0].nested_contexts[0].code_windows = (0..17) + .map(|at| { + let mut site = template.clone(); + site.text = "x".to_owned(); + site.source.start_byte = 20 + at; + site.source.end_byte = 21 + at; + site.source.start_column = 12 + at as u32; + site.source.end_column = 13 + at as u32; + site + }) + .collect(); + assert!(bad.validate().is_err()); + Ok(()) +} diff --git a/crates/compass-output/src/lib.rs b/crates/compass-output/src/lib.rs index e97aaaf9c..5ffdecfd1 100644 --- a/crates/compass-output/src/lib.rs +++ b/crates/compass-output/src/lib.rs @@ -17,6 +17,7 @@ mod lenses; mod obsidian; mod overview; mod palette; +mod ranked_query; mod report; mod review; mod svg; @@ -27,6 +28,11 @@ mod wiki; mod workbench; pub use overview::render_hotspots_text; +pub use ranked_query::{ + RANKED_QUERY_VIEW_SCHEMA, RANKED_VIEW_MAX_CANDIDATES, RankedNameMatch, RankedQueryCandidate, + RankedQueryContext, RankedQueryView, RankedRetrievalKind, build_ranked_query_view, + build_ranked_query_view_with_context, render_ranked_query_text, +}; pub use text_budget::{BudgetedText, render_budgeted_text}; pub use agent_query::{ diff --git a/crates/compass-output/src/ranked_query.rs b/crates/compass-output/src/ranked_query.rs new file mode 100644 index 000000000..349b0bf15 --- /dev/null +++ b/crates/compass-output/src/ranked_query.rs @@ -0,0 +1,787 @@ +//! A bounded candidate shortlist, separate from graph traversal ledgers. +use std::collections::{BTreeMap, BTreeSet}; + +use compass_model::code_graph::NodeKind; +use compass_model::provenance::SourceAnchor; +use compass_model::query_contract::{ + DiscoveryEdge, DiscoveryLimits, DiscoveryScope, QueryDiagnosticCode, QueryEvidence, QueryNode, +}; +use compass_query::{ + RankedDiscoveryResponse, RankedExcerptKind, RankedSourceExcerpt, RelevanceBand, +}; +use serde::{Deserialize, Serialize}; + +use crate::{AgentCoverage, AgentEvidence, OutputError}; + +pub const RANKED_QUERY_VIEW_SCHEMA: &str = "compass.query.ranked-view/4"; +const CODE_RANKED_QUERY_VIEW_SCHEMA: &str = "compass.query.ranked-view/3"; +const COMMENT_RANKED_QUERY_VIEW_SCHEMA: &str = "compass.query.ranked-view/2"; +const LEGACY_RANKED_QUERY_VIEW_SCHEMA: &str = "compass.query.ranked-view/1"; +pub const RANKED_VIEW_MAX_CANDIDATES: usize = 5; +const MAX_VIEW_BYTES: usize = 128 * 1024; +const MAX_REASON_CHARS: usize = 240; +const COMPACT_REASON_CHARS: usize = 160; +const MAX_CONTEXT_EDGES: usize = 8; +const MAX_CONTEXT_NODES: usize = MAX_CONTEXT_EDGES * 2; + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RankedNameMatch { + ExactId, + ExactName, + Approximate, +} + +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RankedRetrievalKind { + Identity, + Lexical, + Signature, + Document, + SemanticLsa, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct RankedQueryCandidate { + pub rank: u32, + pub id: String, + pub name: String, + pub kind: NodeKind, + pub source: Option, + pub supporting_sources: Vec, + pub omitted_supporting_sources: u64, + pub relevance: RelevanceBand, + pub name_match: RankedNameMatch, + pub evidence: AgentEvidence, + pub retrieval_kinds: Vec, + pub reasons: Vec, + pub matched_terms: Vec, + pub matched_fields: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub excerpts: Vec, +} + +/// Witnessed relationships adjacent to displayed candidates. Relevance never +/// creates a relationship; endpoint direction and occurrence evidence survive. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct RankedQueryContext { + pub nodes: Vec, + pub edges: Vec, + pub omitted_edges: u64, + pub truncated: bool, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct RankedQueryView { + pub schema: String, + pub question: String, + pub graph_identity: String, + pub graph_digest: String, + pub candidates: Vec, + pub eligible_count: Option, + pub retained_count: u64, + pub more_available: bool, + pub candidate_recall_bounded: bool, + pub response_truncated: bool, + pub graph_coverage: AgentCoverage, + pub scope: Vec, + pub limits: DiscoveryLimits, + pub narrowing: String, + pub caveats: Vec, + #[serde(skip_serializing_if = "Option::is_none", default)] + pub context: Option, +} + +impl RankedQueryView { + pub fn from_json(bytes: &[u8]) -> Result { + if bytes.len() > MAX_VIEW_BYTES { + return Err(invalid("ranked view exceeds 128 KiB")); + } + #[derive(Deserialize)] + struct Header { + schema: String, + } + let header: Header = serde_json::from_slice(bytes)?; + if !matches!( + header.schema.as_str(), + RANKED_QUERY_VIEW_SCHEMA + | CODE_RANKED_QUERY_VIEW_SCHEMA + | COMMENT_RANKED_QUERY_VIEW_SCHEMA + | LEGACY_RANKED_QUERY_VIEW_SCHEMA + ) { + return Err(invalid("unsupported ranked query view schema")); + } + if header.schema == LEGACY_RANKED_QUERY_VIEW_SCHEMA { + let legacy: serde_json::Value = serde_json::from_slice(bytes)?; + if legacy + .get("candidates") + .and_then(serde_json::Value::as_array) + .is_some_and(|candidates| { + candidates + .iter() + .any(|candidate| candidate.get("excerpts").is_some()) + }) + { + return Err(invalid("ranked-view/1 does not support source quotations")); + } + } + let view: Self = serde_json::from_slice(bytes)?; + view.validate()?; + Ok(view) + } + + fn validate(&self) -> Result<(), OutputError> { + if !matches!( + self.schema.as_str(), + RANKED_QUERY_VIEW_SCHEMA + | CODE_RANKED_QUERY_VIEW_SCHEMA + | COMMENT_RANKED_QUERY_VIEW_SCHEMA + | LEGACY_RANKED_QUERY_VIEW_SCHEMA + ) { + return Err(invalid("unsupported ranked query view schema")); + } + if self.graph_identity.is_empty() || self.graph_digest.is_empty() { + return Err(invalid( + "ranked graph identity and digest must not be empty", + )); + } + if self.candidates.len() > RANKED_VIEW_MAX_CANDIDATES + || self.retained_count < self.candidates.len() as u64 + || self + .eligible_count + .is_some_and(|count| count < self.retained_count) + { + return Err(invalid("inconsistent ranked candidate counts")); + } + let mut identities = BTreeSet::new(); + for (index, candidate) in self.candidates.iter().enumerate() { + if candidate.id.is_empty() + || candidate.rank as usize != index + 1 + || !identities.insert(&candidate.id) + { + return Err(invalid( + "ranked candidates must have unique IDs and consecutive ranks", + )); + } + if candidate + .source + .iter() + .chain(&candidate.supporting_sources) + .any(|source| !source.is_valid()) + { + return Err(invalid("ranked candidate source anchor is invalid")); + } + } + for candidate in &self.candidates { + if candidate.excerpts.len() > 2 + || (self.schema == LEGACY_RANKED_QUERY_VIEW_SCHEMA + && !candidate.excerpts.is_empty()) + || candidate.excerpts.iter().any(|quote| { + !quote.is_coherent() + || (self.schema == COMMENT_RANKED_QUERY_VIEW_SCHEMA + && matches!( + quote.kind, + RankedExcerptKind::CallableCode + | RankedExcerptKind::NestedCallableCode + )) + || (self.schema == CODE_RANKED_QUERY_VIEW_SCHEMA + && quote.kind == RankedExcerptKind::NestedCallableCode) + || candidate.source.as_ref().is_none_or(|source| { + source.file != quote.source.file + || match quote.kind { + RankedExcerptKind::CallableComment + | RankedExcerptKind::CallableCode + | RankedExcerptKind::NestedCallableCode => { + quote.source.start_byte < source.start_byte + || quote.source.end_byte > source.end_byte + || (quote.source.start_line, quote.source.start_column) + < (source.start_line, source.start_column) + || (quote.source.end_line, quote.source.end_column) + > (source.end_line, source.end_column) + || quote.nested_scope.as_ref().is_some_and(|scope| { + scope.file != source.file + || (scope.start_byte == source.start_byte + && scope.end_byte == source.end_byte) + || scope.start_byte < source.start_byte + || scope.end_byte > source.end_byte + || (scope.start_line, scope.start_column) + < (source.start_line, source.start_column) + || (scope.end_line, scope.end_column) + > (source.end_line, source.end_column) + }) + } + RankedExcerptKind::AstDocumentation => { + quote.source.end_byte > source.start_byte + || (quote.source.end_line, quote.source.end_column) + > (source.start_line, source.start_column) + } + } + }) + }) + { + return Err(invalid( + "ranked source quotations exceed bounds or lack coherent source anchors", + )); + } + } + if let Some(context) = &self.context { + if context.nodes.len() > MAX_CONTEXT_NODES || context.edges.len() > MAX_CONTEXT_EDGES { + return Err(invalid("ranked context exceeds its node or edge limit")); + } + let nodes = context + .nodes + .iter() + .map(|node| &node.id) + .collect::>(); + if nodes.len() != context.nodes.len() + || context.nodes.iter().any(|node| { + node.source + .as_ref() + .is_some_and(|source| !source.is_valid()) + }) + || context.edges.iter().any(|edge| { + !nodes.contains(&edge.source) + || !nodes.contains(&edge.target) + || (!identities.contains(&edge.source) + && !identities.contains(&edge.target)) + || edge + .relationship_site + .as_ref() + .is_some_and(|source| !source.is_valid()) + }) + { + return Err(invalid( + "ranked context must retain witnessed endpoints adjacent to candidates", + )); + } + } + let byte_limit = MAX_VIEW_BYTES + .min(usize::try_from(self.limits.max_response_bytes).unwrap_or(usize::MAX)); + if serde_json::to_vec(self)?.len() > byte_limit { + return Err(invalid( + "ranked view exceeds its response byte limit (at most 128 KiB)", + )); + } + Ok(()) + } +} + +pub fn build_ranked_query_view( + response: &RankedDiscoveryResponse, + graph_identity: &str, + graph_digest: &str, +) -> Result { + build_ranked_query_view_with_context(response, graph_identity, graph_digest, false) +} + +pub fn build_ranked_query_view_with_context( + response: &RankedDiscoveryResponse, + graph_identity: &str, + graph_digest: &str, + include_context: bool, +) -> Result { + if response.candidates.len() > compass_model::query_contract::MAX_DISCOVERY_NODES as usize { + return Err(invalid( + "ranked candidate input exceeds the discovery node limit", + )); + } + let mut seen = BTreeSet::new(); + let mut candidates = Vec::new(); + for candidate in &response.candidates { + if !candidate.score.is_finite() { + return Err(invalid("non-finite ranked candidate score")); + } + if candidate.excerpts.len() > 2 + || candidate.excerpts.iter().any(|quote| !quote.is_coherent()) + { + return Err(invalid( + "ranked source quotations exceed bounded input limits", + )); + } + // A declaration can be reached through several passages. Its reasons + // are already fused by the query owner; display the owner once. + if !seen.insert(candidate.node.id.clone()) { + continue; + } + if candidates.len() == RANKED_VIEW_MAX_CANDIDATES { + continue; + } + let fields = &candidate.matched_fields; + let name_match = if fields.iter().any(|field| field == "exact_id") { + RankedNameMatch::ExactId + } else if fields.iter().any(|field| field == "exact_name") { + RankedNameMatch::ExactName + } else { + RankedNameMatch::Approximate + }; + let evidence = structural_evidence(&candidate.node.evidence); + let mut retrieval_kinds = BTreeSet::new(); + for field in fields { + retrieval_kinds.insert(match field.as_str() { + "exact_id" | "exact_name" => RankedRetrievalKind::Identity, + "document" | "document_link" | "owned_document" => RankedRetrievalKind::Document, + "semantic_lsa" => RankedRetrievalKind::SemanticLsa, + "signature" => RankedRetrievalKind::Signature, + _ => RankedRetrievalKind::Lexical, + }); + } + candidates.push(RankedQueryCandidate { + rank: u32::try_from(candidates.len() + 1).unwrap_or(u32::MAX), + id: candidate.node.id.clone(), + name: candidate.node.qualified_name.clone(), + kind: candidate.node.kind, + source: candidate.node.source.clone(), + supporting_sources: candidate + .supporting_sources + .iter() + .take(4) + .cloned() + .collect(), + omitted_supporting_sources: candidate.supporting_sources.len().saturating_sub(4) as u64, + relevance: candidate.relevance, + name_match, + evidence, + retrieval_kinds: retrieval_kinds.into_iter().collect(), + reasons: candidate + .reasons + .iter() + .take(3) + .map(|reason| scalar(reason, MAX_REASON_CHARS)) + .collect(), + matched_terms: candidate.matched_terms.clone(), + matched_fields: fields.clone(), + excerpts: candidate.excerpts.clone(), + }); + } + let retained_count = u64::try_from(seen.len()).unwrap_or(u64::MAX); + let displayed = u64::try_from(candidates.len()).unwrap_or(u64::MAX); + let narrowing = candidates + .first() + .and_then(|candidate| candidate.source.as_ref()) + .map_or_else( + || { + "Narrow the question with a concrete subject or add --scope source:." + .to_owned() + }, + |source| { + format!( + "Narrow with --scope {} or name a specific behavior.", + shell_quote(&format!("source:{}", source.file)) + ) + }, + ); + let context = if include_context { + if response.discovery.nodes.len() + > compass_model::query_contract::MAX_DISCOVERY_NODES as usize + || response.discovery.edges.len() + > compass_model::query_contract::MAX_DISCOVERY_EDGES as usize + { + return Err(invalid("ranked context input exceeds discovery limits")); + } + let ids = candidates + .iter() + .map(|candidate| candidate.id.as_str()) + .collect::>(); + let nodes = response + .discovery + .nodes + .iter() + .map(|node| (node.id.as_str(), node)) + .collect::>(); + let adjacent = response + .discovery + .edges + .iter() + .filter(|edge| ids.contains(edge.source.as_str()) || ids.contains(edge.target.as_str())) + .collect::>(); + let edges = adjacent + .iter() + .take(MAX_CONTEXT_EDGES) + .map(|edge| (*edge).clone()) + .collect::>(); + let mut endpoints = BTreeMap::new(); + for edge in &edges { + for id in [&edge.source, &edge.target] { + let node = nodes + .get(id.as_str()) + .ok_or_else(|| invalid("ranked context relationship lacks an endpoint"))?; + endpoints.insert(id.clone(), (*node).clone()); + } + } + let omitted_edges = adjacent.len().saturating_sub(edges.len()) as u64; + Some(RankedQueryContext { + nodes: endpoints.into_values().collect(), + edges, + omitted_edges, + truncated: response.discovery.truncated || omitted_edges > 0, + }) + } else { + None + }; + let view = RankedQueryView { + schema: RANKED_QUERY_VIEW_SCHEMA.to_owned(), + question: response.discovery.question.clone(), + graph_identity: graph_identity.to_owned(), + graph_digest: graph_digest.to_owned(), + candidates, + eligible_count: response.eligible_count, + retained_count, + more_available: response.bounded + || retained_count > displayed + || response + .eligible_count + .is_some_and(|count| count > displayed), + candidate_recall_bounded: response.bounded, + response_truncated: response.discovery.truncated, + graph_coverage: if response + .discovery + .diagnostics + .iter() + .any(|diagnostic| diagnostic.code == QueryDiagnosticCode::IncompleteCoverage) + { + AgentCoverage::Incomplete + } else { + AgentCoverage::Unknown + }, + scope: response.discovery.scope.clone(), + limits: response.discovery.limits.clone(), + narrowing, + caveats: response + .discovery + .diagnostics + .iter() + .take(8) + .map(|diagnostic| scalar(&diagnostic.message, MAX_REASON_CHARS)) + .collect(), + context, + }; + view.validate()?; + Ok(view) +} + +/// Try abbreviated reasons before dropping whole candidate rows, reserving +/// uncertainty, metadata and refinement in the same UTF-8 byte budget. +/// No cursor contract is reused. +pub fn render_ranked_query_text( + view: &RankedQueryView, + token_budget: usize, +) -> Result { + view.validate()?; + let limit = token_budget.saturating_mul(4).saturating_sub(1); + let total = view + .eligible_count + .map_or_else(|| "unknown".to_owned(), |count| count.to_string()); + for shown in (0..=view.candidates.len()).rev() { + if shown == 0 && !view.candidates.is_empty() { + break; + } + // Preserve the ranked shortlist before preserving lengthy prose. + // JSON retains the original bounded reasons; text makes elision explicit. + for (compact_reasons, first_excerpt_chars, other_excerpt_chars) in [ + (false, 180, 180), + (true, 180, 180), + (true, 180, 80), + (true, 180, 0), + (true, 80, 80), + (true, 0, 0), + ] { + let mut text = if view.candidates.is_empty() { + "No supporting indexed evidence found. This does not prove the behavior is absent.\n" + .to_owned() + } else { + format!("Ranked candidates: {shown} shown; {total} eligible.\n") + }; + for candidate in view.candidates.iter().take(shown) { + let excerpt_chars = if candidate.rank == 1 { + first_excerpt_chars + } else { + other_excerpt_chars + }; + let source = candidate.source.as_ref().map_or_else( + || "source unavailable".to_owned(), + |source| format!("{}:{}", scalar(&source.file, 4096), source.start_line), + ); + let reasons = candidate + .reasons + .iter() + .map(|reason| scalar(reason, MAX_REASON_CHARS)) + .collect::>() + .join("; "); + let reasons = if compact_reasons { + scalar(&reasons, COMPACT_REASON_CHARS) + } else { + reasons + }; + text.push_str(&format!( + "{}. {} @ {} [relevance={}; name={}; evidence={}; via={}; kind={}]\n {}\n", + candidate.rank, + scalar(&candidate.name, 512), + source, + band(candidate.relevance), + match_name(candidate.name_match), + evidence_name(candidate.evidence), + candidate + .retrieval_kinds + .iter() + .map(|kind| retrieval_name(*kind)) + .collect::>() + .join(","), + candidate.kind.as_str(), + reasons + )); + for quote in candidate + .excerpts + .iter() + .take(usize::from(excerpt_chars > 0)) + { + text.push_str(&format!( + " {} @ {}:{}: \"{}\"{}\n", + match quote.kind { + RankedExcerptKind::CallableComment => "Comment", + RankedExcerptKind::CallableCode => "Code", + RankedExcerptKind::NestedCallableCode => + "Nested lexical code (execution unverified)", + RankedExcerptKind::AstDocumentation => "Documentation", + }, + scalar("e.source.file, 4096), + quote.source.start_line, + quote_scalar("e.text, excerpt_chars), + if quote.clipped || quote.text.chars().count() > excerpt_chars { + " [excerpt clipped]" + } else { + "" + } + )); + } + if let Some(source) = candidate.supporting_sources.first() { + text.push_str(&format!( + " supporting passage: {}:{}\n", + scalar(&source.file, 4096), + source.start_line, + )); + } + } + if view + .candidates + .iter() + .take(shown) + .any(|candidate| !candidate.excerpts.is_empty()) + { + text.push_str(if first_excerpt_chars == 0 { + "Quotes omitted for budget; see --format json.\n" + } else { + "Comments/documentation are inspection evidence; behavior remains unverified.\n" + }); + if first_excerpt_chars > 0 + && other_excerpt_chars == 0 + && view + .candidates + .iter() + .take(shown) + .skip(1) + .any(|candidate| !candidate.excerpts.is_empty()) + { + text.push_str("Lower-ranked quotes omitted for budget; see --format json.\n"); + } + } + if compact_reasons && shown > 0 { + text.push_str("Reasons abbreviated; --format json shows more detail.\n"); + if view + .candidates + .iter() + .take(shown) + .any(|candidate| candidate.relevance == RelevanceBand::Low) + { + text.push_str( + "Low matches are inspection leads; the requested behavior is unverified.\n", + ); + } + } + text.push_str(&format!( + "Coverage: {}; candidate recall {}; response {}.\n", + if view.graph_coverage == AgentCoverage::Incomplete { + "incomplete" + } else { + "unknown" + }, + if view.candidate_recall_bounded { + "bounded" + } else { + "complete within indexed scope" + }, + if view.response_truncated { + "truncated" + } else { + "within limits" + } + )); + if view.candidate_recall_bounded { + text.push_str( + "Additional candidates may exist outside these bounds; refine the scope.\n", + ); + } else if view.more_available || shown < view.candidates.len() { + text.push_str("More available; refine the scope to inspect other candidates.\n"); + } + if let Some(caveat) = view.caveats.first() { + text.push_str(&format!("Note: {}\n", scalar(caveat, MAX_REASON_CHARS))); + } + if let Some(context) = &view.context { + let shown_ids = view + .candidates + .iter() + .take(shown) + .map(|candidate| candidate.id.as_str()) + .collect::>(); + let edges = context + .edges + .iter() + .filter(|edge| { + shown_ids.contains(edge.source.as_str()) + || shown_ids.contains(edge.target.as_str()) + }) + .collect::>(); + for edge_count in (0..=edges.len()).rev() { + let mut with_context = text.clone(); + let omitted = context + .omitted_edges + .saturating_add(context.edges.len().saturating_sub(edge_count) as u64); + with_context.push_str(&format!( + "Context (one hop): {edge_count} edges shown; {omitted} omitted{}.\n", + if context.truncated { "; bounded" } else { "" } + )); + for edge in edges.iter().take(edge_count) { + let name = |id: &str| { + context.nodes.iter().find(|node| node.id == id).map_or_else( + || scalar(id, 256), + |node| scalar(&node.qualified_name, 256), + ) + }; + let source = edge.relationship_site.as_ref().map_or_else( + || "source unavailable".to_owned(), + |source| { + format!("{}:{}", scalar(&source.file, 4096), source.start_line) + }, + ); + with_context.push_str(&format!( + " {} --{}--> {} @ {} [evidence={}]\n", + name(&edge.source), + edge.kind.as_str(), + name(&edge.target), + source, + evidence_name(structural_evidence(&edge.evidence)) + )); + } + with_context.push_str(&scalar(&view.narrowing, 4096)); + if with_context.len() <= limit { + return Ok(with_context); + } + } + } else { + text.push_str(&scalar(&view.narrowing, 4096)); + if text.len() <= limit { + return Ok(text); + } + } + } + } + Err(invalid( + "ranked answer metadata and one candidate exceed --text-budget; increase the budget", + )) +} + +fn structural_evidence(evidence: &[QueryEvidence]) -> AgentEvidence { + use compass_model::provenance::{EvidenceConfidence, ResolutionState}; + if evidence.iter().any(|item| { + item.confidence == EvidenceConfidence::Ambiguous + || item.resolution == ResolutionState::Ambiguous + }) { + return AgentEvidence::Ambiguous; + } + let exact = evidence + .iter() + .any(|item| item.confidence == EvidenceConfidence::Exact); + let inferred = evidence + .iter() + .any(|item| item.confidence != EvidenceConfidence::Exact); + match (exact, inferred) { + (true, true) => AgentEvidence::Mixed, + (true, false) => AgentEvidence::Exact, + (false, true) => AgentEvidence::Inferred, + (false, false) => AgentEvidence::None, + } +} + +fn invalid(message: &str) -> OutputError { + OutputError::InvalidAgentQuery(message.to_owned()) +} +fn quote_scalar(value: &str, chars: usize) -> String { + value + .chars() + .take(chars) + .flat_map(|character| { + if character.is_control() + || matches!(character, '\u{202a}'..='\u{202e}' | '\u{2066}'..='\u{2069}') + { + character.escape_unicode().collect::>() + } else if matches!(character, '"' | '\\') { + character.escape_default().collect::>() + } else { + vec![character] + } + }) + .collect() +} +fn scalar(value: &str, chars: usize) -> String { + value + .chars() + .take(chars) + .flat_map(|character| { + if character.is_control() { + character.escape_default().collect::>() + } else { + vec![character] + } + }) + .collect() +} +fn shell_quote(value: &str) -> String { + format!("'{}'", scalar(value, 4096).replace('\'', "'\"'\"'")) +} +fn band(value: RelevanceBand) -> &'static str { + match value { + RelevanceBand::High => "high", + RelevanceBand::Medium => "medium", + RelevanceBand::Low => "low", + } +} +fn evidence_name(value: AgentEvidence) -> &'static str { + match value { + AgentEvidence::Exact => "exact", + AgentEvidence::Inferred => "inferred", + AgentEvidence::Mixed => "mixed", + AgentEvidence::Ambiguous => "ambiguous", + AgentEvidence::None => "none", + } +} + +fn match_name(value: RankedNameMatch) -> &'static str { + match value { + RankedNameMatch::ExactId => "exact_id", + RankedNameMatch::ExactName => "exact_name", + RankedNameMatch::Approximate => "approximate", + } +} + +fn retrieval_name(value: RankedRetrievalKind) -> &'static str { + match value { + RankedRetrievalKind::Identity => "identity", + RankedRetrievalKind::Lexical => "lexical", + RankedRetrievalKind::Signature => "signature", + RankedRetrievalKind::Document => "document", + RankedRetrievalKind::SemanticLsa => "semantic_lsa", + } +} diff --git a/crates/compass-output/src/text_budget.rs b/crates/compass-output/src/text_budget.rs index bce1b487a..b00f2f772 100644 --- a/crates/compass-output/src/text_budget.rs +++ b/crates/compass-output/src/text_budget.rs @@ -67,7 +67,7 @@ mod tests { while start < input.len() { let page = render_budgeted_text(&input, start, 64, |end| format!("More: --offset {end}"))?; - assert!(page.text.len() + 1 <= 256); + assert!(page.text.len() < 256); assert!(page.end > start); restored.push_str(&input[start..page.end]); start = page.end; diff --git a/crates/compass-output/tests/agent_query.rs b/crates/compass-output/tests/agent_query.rs index 2cc348700..23e2a9a6c 100644 --- a/crates/compass-output/tests/agent_query.rs +++ b/crates/compass-output/tests/agent_query.rs @@ -1147,7 +1147,7 @@ fn compact_one_hop_and_no_match_answers_cost_only_the_facts() -> Result<(), Box< }; let compact = compass_output::render_compact_code_query_text_page(&response, context.clone(), options)?; - assert!(compact.text.len() + 1 <= 400, "{}", compact.text); + assert!(compact.text.len() < 400, "{}", compact.text); assert!( compact.text.contains("Caller --calls--> Fixture.Target"), "{}", @@ -1201,7 +1201,7 @@ fn compact_unicode_pages_never_overflow_the_budget_or_skip_entries() -> Result<( cursor: cursor.as_deref(), }, )?; - assert!(page.text.len() + 1 <= 1024); + assert!(page.text.len() < 1024); count += page.entry_end - page.entry_start; cursor = page.next_cursor; if cursor.is_none() { diff --git a/crates/compass-output/tests/ranked_query.rs b/crates/compass-output/tests/ranked_query.rs new file mode 100644 index 000000000..7d7e47abe --- /dev/null +++ b/crates/compass-output/tests/ranked_query.rs @@ -0,0 +1,516 @@ +use std::error::Error; + +use compass_model::code_graph::{EdgeKind, NodeKind}; +use compass_model::provenance::SourceAnchor; +use compass_model::query_contract::{ + DiscoveryDirection, DiscoveryDirectionSource, DiscoveryEdge, DiscoveryLimits, + DiscoveryQueryResponse, DiscoveryTraversal, QueryNode, +}; +use compass_output::{ + AgentCoverage, AgentEvidence, RANKED_QUERY_VIEW_SCHEMA, RankedNameMatch, + build_ranked_query_view, build_ranked_query_view_with_context, render_ranked_query_text, +}; +use compass_query::{RankedCandidate, RankedDiscoveryResponse, RelevanceBand}; + +fn response() -> RankedDiscoveryResponse { + let candidates = (0..7) + .map(|index| RankedCandidate { + // Reverse identity order deliberately: display order is relevance order. + node: QueryNode { + id: format!("node:{}", 9 - index), + kind: NodeKind::Function, + roles: Vec::new(), + name: format!("implementation_{index}"), + qualified_name: format!("app::implementation_{index}"), + language: Some("rust".to_owned()), + framework: None, + source: Some(SourceAnchor { + file: format!("src/impl_{index}.rs"), + start_byte: 0, + end_byte: 1, + start_line: 1, + start_column: 0, + end_line: 1, + end_column: 1, + }), + details: None, + evidence: Vec::new(), + }, + score: (10 - index) as f64, + relevance: RelevanceBand::High, + matched_terms: vec!["implementation".to_owned()], + matched_fields: vec!["signature".to_owned()], + reasons: vec!["Request subject and action match the signature.".to_owned()], + supporting_sources: Vec::new(), + excerpts: Vec::new(), + }) + .collect::>(); + RankedDiscoveryResponse { + discovery: DiscoveryQueryResponse { + schema: "compass.query.discovery/1".to_owned(), + question: "where is the request implementation".to_owned(), + selected_direction: DiscoveryDirection::Both, + direction_source: DiscoveryDirectionSource::Neutral, + relation_contexts: Vec::new(), + scope: Vec::new(), + traversal: DiscoveryTraversal::Bfs, + seeds: Vec::new(), + nodes: candidates + .iter() + .take(3) + .map(|candidate| candidate.node.clone()) + .collect(), + edges: Vec::new(), + diagnostics: Vec::new(), + limits: DiscoveryLimits::default(), + stats: Default::default(), + omissions: Default::default(), + truncated: false, + }, + candidates, + eligible_count: Some(7), + bounded: false, + } +} + +#[test] +fn candidate_bounds_do_not_claim_that_a_traversal_was_truncated() -> Result<(), Box> { + let mut input = response(); + input.bounded = true; + input.discovery.truncated = true; + input.eligible_count = None; + assert!(input.discovery.edges.is_empty()); + let view = build_ranked_query_view(&input, "generation", "digest")?; + assert!(view.candidate_recall_bounded); + assert!(view.response_truncated); + let encoded = serde_json::to_value(&view)?; + assert_eq!(encoded["responseTruncated"], true); + assert!(encoded.get("traversalTruncated").is_none()); + let text = render_ranked_query_text(&view, 800)?; + assert!(text.contains("candidate recall bounded; response truncated")); + assert!(!text.contains("traversal truncated")); + Ok(()) +} + +#[test] +fn shortlist_preserves_ranks_four_and_five_independent_of_graph_seeds() -> Result<(), Box> +{ + let input = response(); + let view = build_ranked_query_view(&input, "generation", "digest")?; + assert_eq!(view.schema, RANKED_QUERY_VIEW_SCHEMA); + assert_eq!(view.candidates.len(), 5); + assert_eq!( + view.candidates + .iter() + .map(|candidate| candidate.id.as_str()) + .collect::>(), + ["node:9", "node:8", "node:7", "node:6", "node:5"] + ); + assert_eq!(view.retained_count, 7); + assert_eq!(view.eligible_count, Some(7)); + assert!(view.more_available); + assert_eq!(view.graph_coverage, AgentCoverage::Unknown); + // High relevance must not imply an exact name or proven structural evidence. + assert_eq!(view.candidates[0].name_match, RankedNameMatch::Approximate); + assert_eq!(view.candidates[0].evidence, AgentEvidence::None); + let text = render_ranked_query_text(&view, 800)?; + assert!(text.contains("app::implementation_4")); + assert!(text.contains("src/impl_4.rs:1")); + assert!(text.len() < 800 * 4); + assert!(text.contains("--scope 'source:src/impl_0.rs'")); + let encoded = serde_json::to_vec(&view)?; + assert_eq!(compass_output::RankedQueryView::from_json(&encoded)?, view); + let mut future = view.clone(); + future.schema = "compass.query.ranked-view/99".to_owned(); + assert!(compass_output::RankedQueryView::from_json(&serde_json::to_vec(&future)?).is_err()); + let mut invalid_anchor = view.clone(); + invalid_anchor.candidates[0] + .source + .as_mut() + .ok_or("source")? + .start_line = 0; + assert!( + compass_output::RankedQueryView::from_json(&serde_json::to_vec(&invalid_anchor)?).is_err() + ); + let mut too_small = input.clone(); + too_small.discovery.limits.max_response_bytes = 64; + assert!(build_ranked_query_view(&too_small, "generation", "digest").is_err()); + Ok(()) +} + +#[test] +fn default_budget_compacts_explanations_before_dropping_the_fifth_candidate() +-> Result<(), Box> { + let mut input = response(); + input.bounded = true; + input.discovery.truncated = true; + input.eligible_count = None; + for (at, candidate) in input.candidates.iter_mut().enumerate() { + candidate.node.qualified_name = + format!("src.marshmallow.schema.Schema::implementation_{at}"); + let source = candidate.node.source.as_mut().ok_or("missing source")?; + source.file = "src/marshmallow/schema.py".to_owned(); + source.start_line = 815 + u32::try_from(at)?; + source.end_line = source.start_line; + candidate.relevance = RelevanceBand::Low; + candidate.reasons = vec![ + "Negative qualification is not established by lexical evidence. These are inspection candidates, not confirmation of the requested condition".to_owned(), + "Indexed code_reference, name, owned_document, path, qualified_name, signature evidence matches 7 of 9 query concepts".to_owned(), + "Partial indexed evidence; missing owner-specific context: dictionary, error, callback. The requested behavior is not established by this candidate".to_owned(), + ]; + candidate.supporting_sources = candidate.node.source.clone().into_iter().collect(); + } + let view = build_ranked_query_view(&input, "generation", "digest")?; + let original = view.clone(); + let text = render_ranked_query_text(&view, 800)?; + assert!(text.contains("Ranked candidates: 5 shown")); + for at in 0..5 { + assert!(text.contains(&format!( + "src.marshmallow.schema.Schema::implementation_{at}" + ))); + assert!(text.contains(&format!("src/marshmallow/schema.py:{}", 815 + at))); + } + assert!(text.contains("relevance=low; name=approximate; evidence=none")); + assert!(text.contains("Negative qualification is not established")); + assert!(text.contains("Reasons abbreviated")); + assert!(text.contains("--format json")); + assert!(text.contains("Coverage: unknown; candidate recall bounded; response truncated")); + assert!(text.contains("Narrow with")); + assert!(text.len() < 800 * 4); + assert_eq!(view, original); + let json = serde_json::to_string(&view)?; + assert!(json.contains("dictionary, error, callback")); + Ok(()) +} + +#[test] +fn tight_budget_preserves_first_quote_before_lower_ranked_quotes() -> Result<(), Box> { + use compass_query::{RankedExcerptKind, RankedSourceExcerpt}; + let mut input = response(); + input.bounded = true; + input.discovery.truncated = true; + input.eligible_count = None; + let quote = "format!(\n \"{}: {}\\n --> {source_name}:{line}:{column}{help}\",\n diagnostic.code(),\n diagnostic.message()\n )"; + for (at, candidate) in input.candidates.iter_mut().enumerate() { + candidate.node.qualified_name = + format!("application::diagnostic_commands::format_diagnostic_{at}"); + candidate.relevance = RelevanceBand::Low; + candidate.reasons = vec!["Nested lexical code context was omitted by shared inspection limits; no execution is inferred; Indexed body_identifier, name, path, qualified_name, signature evidence matches query concepts".to_owned()]; + let source = candidate.node.source.as_mut().ok_or("source")?; + source.file = "crates/application-cli/src/diagnostic_commands.rs".to_owned(); + source.end_byte = quote.len() as u64 + 2; + source.end_line = 5; + source.end_column = 14; + let mut quoted = source.clone(); + quoted.start_byte = 1; + quoted.start_column = 1; + quoted.end_byte -= 1; + quoted.end_column -= 1; + candidate.excerpts = vec![RankedSourceExcerpt { + kind: RankedExcerptKind::NestedCallableCode, + source: quoted.clone(), + text: quote.to_owned(), + clipped: false, + nested_scope: Some(quoted), + }]; + } + let view = build_ranked_query_view(&input, "generation", "digest")?; + let original = view.clone(); + let text = render_ranked_query_text(&view, 800)?; + assert!(text.contains("Ranked candidates: 5 shown"), "{text}"); + let first = text.split("2. application::").next().ok_or("first row")?; + assert!(first.contains("diagnostic.message()"), "{text}"); + assert!(!first.contains("[excerpt clipped]"), "{text}"); + assert!( + text.contains("Lower-ranked quotes omitted for budget"), + "{text}" + ); + assert!(text.len() < 3200); + assert_eq!(view, original); + Ok(()) +} + +#[test] +fn budgets_keep_rows_whole_and_reserve_limits_and_refinement() -> Result<(), Box> { + let mut input = response(); + input.candidates[0].node.qualified_name = "é東京🙂".repeat(80); + let view = build_ranked_query_view(&input, "generation", "digest")?; + for budget in [500, 650, 800] { + let text = render_ranked_query_text(&view, budget)?; + assert!(text.len() < budget * 4); + assert!(text.contains("Coverage:")); + assert!(text.contains("Narrow with")); + assert!(text.contains("relevance=high; name=approximate; evidence=none")); + } + assert!(render_ranked_query_text(&view, 1).is_err()); + Ok(()) +} + +#[test] +fn empty_results_abstain_without_claiming_absence() -> Result<(), Box> { + let mut input = response(); + input.candidates.clear(); + input.eligible_count = None; + input.bounded = true; + let view = build_ranked_query_view(&input, "generation", "digest")?; + let text = render_ranked_query_text(&view, 800)?; + assert!(text.contains("No supporting indexed evidence found")); + assert!(text.contains("does not prove the behavior is absent")); + assert!(view.candidate_recall_bounded); + assert!(view.more_available); + Ok(()) +} + +#[test] +fn optional_context_preserves_direction_occurrences_and_source_anchors() +-> Result<(), Box> { + let mut input = response(); + let site = input.candidates[0].node.source.clone(); + input.discovery.edges = (0..10) + .map(|index| DiscoveryEdge { + id: Some(format!("edge:{index}")), + source: "node:8".to_owned(), + target: "node:9".to_owned(), + kind: EdgeKind::Calls, + occurrence_rule: None, + relationship_site: site.clone(), + details: None, + evidence: Vec::new(), + context: Some("call".to_owned()), + }) + .collect(); + let plain = build_ranked_query_view(&input, "generation", "digest")?; + assert!(plain.context.is_none()); + let view = build_ranked_query_view_with_context(&input, "generation", "digest", true)?; + assert_eq!(view.candidates, plain.candidates); + let context = view.context.as_ref().ok_or("context")?; + assert_eq!(context.edges.len(), 8); + assert_eq!(context.omitted_edges, 2); + assert!(context.truncated); + assert_eq!(context.edges[0].source, "node:8"); + assert_eq!(context.edges[0].target, "node:9"); + assert_eq!(context.edges[0].relationship_site, site); + assert_ne!(context.edges[0].id, context.edges[1].id); + let text = render_ranked_query_text(&view, 800)?; + assert!( + text.find("5.").ok_or("fifth candidate")? + < text.find("Context (one hop)").ok_or("context")? + ); + assert!(text.len() < 800 * 4); + assert!(text.contains("--calls-->")); + let encoded = serde_json::to_vec(&view)?; + assert_eq!(compass_output::RankedQueryView::from_json(&encoded)?, view); + Ok(()) +} + +#[test] +fn document_support_is_bounded_and_identity_is_explicit() -> Result<(), Box> { + let mut input = response(); + let source = input.candidates[0].node.source.clone().ok_or("source")?; + input.candidates[0].supporting_sources = vec![source.clone(); 6]; + input.candidates[0] + .matched_fields + .push("owned_document".to_owned()); + input.candidates[0] + .matched_fields + .push("exact_name".to_owned()); + let view = build_ranked_query_view(&input, "generation", "digest")?; + assert_eq!(view.candidates[0].name_match, RankedNameMatch::ExactName); + assert_eq!(view.candidates[0].supporting_sources, vec![source; 4]); + assert_eq!(view.candidates[0].omitted_supporting_sources, 2); + assert!( + view.candidates[0] + .retrieval_kinds + .contains(&compass_output::RankedRetrievalKind::Document) + ); + assert!(render_ranked_query_text(&view, 800)?.contains("supporting passage: src/impl_0.rs:1")); + Ok(()) +} + +#[test] +fn candidate_text_distinguishes_resource_matches_from_implementation_declarations() +-> Result<(), Box> { + let mut input = response(); + input.candidates[0].node.kind = NodeKind::Resource; + input.candidates[0].node.name = "Cookie reference".to_owned(); + input.candidates[0].node.qualified_name = "Cookie reference".to_owned(); + input.candidates[0].matched_fields = vec!["document".to_owned()]; + let view = build_ranked_query_view(&input, "generation", "digest")?; + let text = render_ranked_query_text(&view, 800)?; + assert!(text.starts_with("Ranked candidates: 5 shown; 7 eligible.")); + assert!(text.contains("via=document; kind=resource]")); + assert!(text.contains("via=signature; kind=function]")); + assert_eq!(view.candidates[0].kind, NodeKind::Resource); + assert!(text.len() < 800 * 4); + Ok(()) +} + +#[test] +fn source_quotes_are_bounded_escaped_and_do_not_displace_the_shortlist() +-> Result<(), Box> { + use compass_query::{RankedExcerptKind, RankedSourceExcerpt}; + let mut input = response(); + for candidate in &mut input.candidates { + let text = format!("// payload \u{202e} \x1b {}", "x".repeat(470)); + let mut source = candidate.node.source.clone().ok_or("source")?; + source.end_byte = u64::try_from(text.len())?; + source.end_column = u32::try_from(text.len())?; + candidate.node.source = Some(source.clone()); + candidate.excerpts = vec![RankedSourceExcerpt { + kind: RankedExcerptKind::CallableComment, + source, + text, + clipped: false, + nested_scope: None, + }]; + } + let view = build_ranked_query_view(&input, "generation", "digest")?; + let text = render_ranked_query_text(&view, 800)?; + assert!(text.contains("Ranked candidates: 5 shown")); + assert!(text.len() < 3200); + assert!(!text.contains('\u{202e}')); + assert!(!text.contains('\x1b')); + assert!(text.contains("Comment @") || text.contains("Quotes omitted for budget")); + let encoded = serde_json::to_vec(&view)?; + assert_eq!(compass_output::RankedQueryView::from_json(&encoded)?, view); + let mut invalid = view.clone(); + invalid.candidates[0].excerpts[0].source.start_byte = 1; + assert!(compass_output::RankedQueryView::from_json(&serde_json::to_vec(&invalid)?).is_err()); + let mut outside = view.clone(); + let quote = &mut outside.candidates[0].excerpts[0]; + quote.source.start_byte += 1; + quote.source.end_byte += 1; + quote.source.start_column += 1; + quote.source.end_column += 1; + assert!(quote.is_coherent()); + assert!(compass_output::RankedQueryView::from_json(&serde_json::to_vec(&outside)?).is_err()); + let mut foreign = view.clone(); + foreign.candidates[0].excerpts[0].source.file = "src/neighbor.rs".to_owned(); + assert!(compass_output::RankedQueryView::from_json(&serde_json::to_vec(&foreign)?).is_err()); + let mut normalized = view.clone(); + normalized.candidates[0].excerpts[0].text = "normalized".to_owned(); + assert!(compass_output::RankedQueryView::from_json(&serde_json::to_vec(&normalized)?).is_err()); + Ok(()) +} + +#[test] +fn legacy_ranked_views_remain_readable_and_unknown_major_is_explicit() -> Result<(), Box> +{ + let mut legacy = build_ranked_query_view(&response(), "generation", "digest")?; + legacy.schema = "compass.query.ranked-view/1".to_owned(); + assert_eq!( + compass_output::RankedQueryView::from_json(&serde_json::to_vec(&legacy)?)?, + legacy + ); + let mut legacy_with_quotes = serde_json::to_value(&legacy)?; + legacy_with_quotes["candidates"][0]["excerpts"] = serde_json::json!([]); + assert!( + compass_output::RankedQueryView::from_json(&serde_json::to_vec(&legacy_with_quotes)?) + .is_err() + ); + let error = + compass_output::RankedQueryView::from_json(br#"{"schema":"compass.query.ranked-view/99"}"#) + .err() + .ok_or("future major accepted")?; + assert!( + error + .to_string() + .contains("unsupported ranked query view schema") + ); + Ok(()) +} + +#[test] +fn code_quotes_require_view_three_and_legacy_comment_views_stay_readable() +-> Result<(), Box> { + use compass_query::{RankedExcerptKind, RankedSourceExcerpt}; + let mut input = response(); + let candidate = &mut input.candidates[0]; + let text = "normalize_payload(items)".to_owned(); + let mut source = candidate.node.source.clone().ok_or("source")?; + source.end_byte = source.start_byte + text.len() as u64; + source.end_line = source.start_line; + source.end_column = source.start_column + text.len() as u32; + candidate.node.source = Some(source.clone()); + candidate.excerpts = vec![RankedSourceExcerpt { + kind: RankedExcerptKind::CallableCode, + source, + text, + clipped: false, + nested_scope: None, + }]; + let mut view = build_ranked_query_view(&input, "generation", "digest")?; + assert_eq!(view.schema, "compass.query.ranked-view/4"); + assert_eq!( + compass_output::RankedQueryView::from_json(&serde_json::to_vec(&view)?)?, + view + ); + assert!(render_ranked_query_text(&view, 1600)?.contains("Code @")); + view.schema = "compass.query.ranked-view/2".to_owned(); + assert!(compass_output::RankedQueryView::from_json(&serde_json::to_vec(&view)?).is_err()); + view.candidates[0].excerpts[0].kind = RankedExcerptKind::CallableComment; + assert_eq!( + compass_output::RankedQueryView::from_json(&serde_json::to_vec(&view)?)?, + view + ); + Ok(()) +} + +#[test] +fn nested_code_is_separate_lexical_context_and_legacy_views_reject_it() -> Result<(), Box> +{ + use compass_query::{RankedExcerptKind, RankedSourceExcerpt}; + let mut input = response(); + let candidate = &mut input.candidates[0]; + let mut parent = candidate.node.source.clone().ok_or("source")?; + parent.end_byte = parent.start_byte + 100; + parent.end_line = parent.start_line; + parent.end_column = parent.start_column + 100; + candidate.node.source = Some(parent.clone()); + let mut scope = parent.clone(); + scope.start_byte += 10; + scope.end_byte -= 10; + scope.start_column += 10; + scope.end_column -= 10; + let text = "payload_error(item)".to_owned(); + let mut source = scope.clone(); + source.start_byte += 1; + source.start_column += 1; + source.end_byte = source.start_byte + text.len() as u64; + source.end_column = source.start_column + text.len() as u32; + candidate.excerpts = vec![RankedSourceExcerpt { + kind: RankedExcerptKind::NestedCallableCode, + source, + text, + clipped: false, + nested_scope: Some(scope), + }]; + let view = build_ranked_query_view(&input, "generation", "digest")?; + assert_eq!( + compass_output::RankedQueryView::from_json(&serde_json::to_vec(&view)?)?, + view + ); + let text = render_ranked_query_text(&view, 800)?; + assert!(text.contains("Nested lexical code (execution unverified)")); + for schema in [ + "compass.query.ranked-view/1", + "compass.query.ranked-view/2", + "compass.query.ranked-view/3", + ] { + let mut bad = view.clone(); + bad.schema = schema.to_owned(); + assert!(compass_output::RankedQueryView::from_json(&serde_json::to_vec(&bad)?).is_err()); + } + let mut bad = view.clone(); + bad.candidates[0].excerpts[0].nested_scope = None; + assert!(compass_output::RankedQueryView::from_json(&serde_json::to_vec(&bad)?).is_err()); + let mut bad = view.clone(); + bad.candidates[0].excerpts[0].nested_scope = bad.candidates[0].source.clone(); + assert!(compass_output::RankedQueryView::from_json(&serde_json::to_vec(&bad)?).is_err()); + let mut legacy = view; + legacy.schema = "compass.query.ranked-view/3".to_owned(); + legacy.candidates[0].excerpts[0].kind = RankedExcerptKind::CallableCode; + legacy.candidates[0].excerpts[0].nested_scope = None; + compass_output::RankedQueryView::from_json(&serde_json::to_vec(&legacy)?)?; + Ok(()) +} diff --git a/crates/compass-query/Cargo.toml b/crates/compass-query/Cargo.toml index d283733e1..3877c82b7 100644 --- a/crates/compass-query/Cargo.toml +++ b/crates/compass-query/Cargo.toml @@ -15,7 +15,7 @@ categories.workspace = true compass-agent-graph = { path = "../compass-agent-graph", version = "0.4.2" } compass-files = { path = "../compass-files", version = "0.4.2" } regex.workspace = true -rusqlite.workspace = true +rusqlite = { workspace = true, features = ["hooks", "serialize"] } serde.workspace = true serde_json.workspace = true sha2.workspace = true diff --git a/crates/compass-query/examples/broad_retrieval_trace.rs b/crates/compass-query/examples/broad_retrieval_trace.rs new file mode 100644 index 000000000..5a24b7315 --- /dev/null +++ b/crates/compass-query/examples/broad_retrieval_trace.rs @@ -0,0 +1,39 @@ +//! Developer stage tracing against an explicitly selected graph and cache. +//! This observes a known target; it never changes retrieval or its ordering. +use std::error::Error; +use std::io::{Error as IoError, ErrorKind}; +use std::path::Path; + +use compass_model::query_contract::{ + DiscoveryDirection, DiscoveryLimits, DiscoveryQueryRequest, DiscoveryTraversal, +}; +use compass_query::{EngineSelection, open_with_engine}; + +fn main() -> Result<(), Box> { + let arguments = std::env::args().skip(1).take(5).collect::>(); + if arguments.len() != 4 { + return Err(IoError::new( + ErrorKind::InvalidInput, + "usage: broad_retrieval_trace GRAPH CACHE QUESTION EXPECTED_ID", + ) + .into()); + } + let engine = open_with_engine( + Path::new(&arguments[0]), + None, + Path::new(&arguments[1]), + EngineSelection::Json, + )?; + let request = DiscoveryQueryRequest { + question: arguments[2].clone(), + direction: DiscoveryDirection::Auto, + relation_contexts: Vec::new(), + scope: Vec::new(), + traversal: DiscoveryTraversal::Bfs, + include_heuristic: false, + limits: DiscoveryLimits::default(), + }; + let trace = engine.trace_ranked(request, &arguments[3])?; + serde_json::to_writer(std::io::stdout().lock(), &trace)?; + Ok(()) +} diff --git a/crates/compass-query/src/broad.rs b/crates/compass-query/src/broad.rs new file mode 100644 index 000000000..74d5080e6 --- /dev/null +++ b/crates/compass-query/src/broad.rs @@ -0,0 +1,2305 @@ +//! Source-owned fielded retrieval for broad implementation questions. +//! Candidate reads are quota-bounded before fusion. Scores are retrieval +//! evidence, never a structural assertion or a calibrated probability. +use std::collections::{BTreeMap, BTreeSet, VecDeque}; +use std::time::{Duration, Instant}; + +use compass_model::code_graph::{EdgeKind, NodeKind, NodeRecord, NodeRole}; +use compass_model::provenance::{EvidenceConfidence, EvidenceOrigin, SourceAnchor}; +use compass_model::query_contract::{ + CodeQueryLimits, CodeQueryOperation, CodeQueryResponse, ConceptMatch, ConceptMatchMethod, + DISCOVERY_QUERY_SCHEMA_V1, DiscoveryDirection, DiscoveryOmissions, DiscoveryQueryRequest, + DiscoveryQueryResponse, DiscoveryScope, DiscoveryScoreTier, DiscoverySeed, DiscoverySeedSource, + DiscoveryStats, MAX_INDEXED_CANDIDATE_NODES_READ, MAX_INDEXED_CANDIDATE_PROBES, + QueryDiagnostic, QueryDiagnosticCode, QueryEdge, QueryNode, SearchHit, SearchRequest, + discovery_scope_matches, +}; +use compass_model::search::{ + SearchDocument, identifier_search_terms, is_primary_search_kind, search_document, + search_document_with_aliases, +}; +use rusqlite::{OptionalExtension, params}; +use serde::{Deserialize, Serialize}; + +use crate::code_query::{ + PinnedDiscoveryBackend, normalize_symbol, query_edge, query_node, search_query_terms, +}; +use crate::intent::{NaturalQueryIntent, plan_natural_query}; +use crate::{CodeQueryEngine, QueryError, QueryErrorKind}; + +mod excerpts; +pub use excerpts::{RankedExcerptKind, RankedSourceExcerpt}; +mod compounds; +mod ownership; +mod relationships; + +const POSTING_QUOTA: usize = 256; +// At most three public routing checks each perform eight bounded exact-name +// lookups (nine decoded records including lookahead). Reserve that work from +// the shared hard ceiling; search-phase counters describe the work below. +const FIELD_READ_CEILING: u64 = MAX_INDEXED_CANDIDATE_NODES_READ - 216; +const FIELD_PROBE_CEILING: u64 = MAX_INDEXED_CANDIDATE_PROBES - 24; +// An ordinal implementation-inspection preference, below exact name/ID tiers. +// It is not additional confidence or evidence of execution. +const IMPLEMENTATION_ACTION_PREFERENCE: f64 = 10_000.0; + +fn is_retrieval_action(term: &str) -> bool { + crate::ranking::canonical_predicate_token(term).is_some() + || matches!(term, "encode" | "serialize") +} + +fn test_context(node: &NodeRecord) -> bool { + if node.roles.contains(&NodeRole::Test) + || node.roles.contains(&NodeRole::Fixture) + || node.name.starts_with("test_") + { + return true; + } + let mut parts = node.source_file().unwrap_or_default().rsplit(['/', '\\']); + let file = parts.next().unwrap_or_default().to_ascii_lowercase(); + file.starts_with("test_") + || file.ends_with("_test.go") + || file.ends_with("_test.py") + || file.ends_with("_test.rs") + || file.contains(".test.") + || file.contains(".spec.") + || parts.any(|part| { + matches!( + part.to_ascii_lowercase().as_str(), + "test" | "tests" | "fixtures" | "__tests__" + ) + }) +} + +fn is_query_scaffolding(word: &str) -> bool { + matches!( + word.to_ascii_lowercase().as_str(), + "where" + | "what" + | "which" + | "when" + | "who" + | "how" + | "does" + | "can" + | "find" + | "show" + | "explain" + | "please" + | "locate" + | "list" + | "describe" + | "identify" + | "tell" + | "give" + | "not" + | "never" + | "without" + | "and" + | "or" + ) +} + +fn explicit_identifiers(question: &str) -> BTreeSet { + let mut contexts = BTreeSet::new(); + let content = terms(question); + for word in question.split_whitespace() { + let word = word.trim_matches(|character: char| { + !character.is_alphanumeric() && !matches!(character, '_' | '/' | '\\') + }); + let qualified = word.contains("::") || word.contains(['.', '/', '\\']); + let named_casing = word.chars().filter(|c| c.is_uppercase()).count() > 1 + && plural_acronym_stem(word).is_none() + && (word.chars().any(char::is_lowercase) + || content.contains(&compass_model::canonical_code_token(word.to_lowercase()))) + && !is_query_scaffolding(word); + if (qualified || word.contains('_') || named_casing) + && word.chars().all(|character| { + character.is_alphanumeric() || matches!(character, '_' | ':' | '.' | '/' | '\\') + }) + && word.chars().any(char::is_alphabetic) + && word + .split([':', '.', '/', '\\']) + .any(|part| part.len() >= 2) + { + contexts.insert(word.to_owned()); + } + } + for delimiter in ['`', '"'] { + for quoted in question.split(delimiter).skip(1).step_by(2) { + if quoted.split_whitespace().count() == 1 { + contexts.insert(quoted.trim().to_owned()); + } + } + } + contexts +} + +fn named_contexts(question: &str, content: &[String]) -> BTreeSet { + question + .split(|character: char| !character.is_alphanumeric() && character != '_') + .filter(|word| word.len() >= 3 && word.chars().next().is_some_and(char::is_uppercase)) + .filter(|word| content.contains(&compass_model::canonical_code_token(word.to_lowercase()))) + .filter(|word| !is_query_scaffolding(word)) + .filter(|word| plural_acronym_stem(word).is_none()) + .map(str::to_owned) + .chain(explicit_identifiers(question)) + .flat_map(|word| compass_model::identifier_tokens(&word.replace('_', " "))) + .map(compass_model::canonical_code_token) + .collect() +} + +fn has_explicit_identity(node: &NodeRecord, identifier: &str) -> bool { + let identifier = normalize_symbol(identifier).replace("::", "."); + [&node.name, &node.qualified_name].into_iter().any(|name| { + let name = normalize_symbol(name).replace("::", "."); + name == identifier + || name.starts_with(&format!("{identifier}.")) + || name.ends_with(&format!(".{identifier}")) + || name.contains(&format!(".{identifier}.")) + }) || node.source_file().is_some_and(|path| { + let path = path.replace('\\', "/"); + let identifier = identifier.replace('\\', "/"); + path == identifier || path.ends_with(&format!("/{identifier}")) + }) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RelevanceBand { + High, + Medium, + Low, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct RankedCandidate { + pub node: QueryNode, + pub score: f64, + pub relevance: RelevanceBand, + pub matched_terms: Vec, + pub matched_fields: Vec, + pub reasons: Vec, + pub supporting_sources: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub excerpts: Vec, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct RankedDiscoveryResponse { + pub discovery: DiscoveryQueryResponse, + pub candidates: Vec, + pub eligible_count: Option, + pub bounded: bool, +} + +/// Developer evidence for the earliest missing stage, not a relevance oracle. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct RankedRetrievalTrace { + pub schema: String, + pub graph_present: bool, + pub admitted: bool, + pub rank: Option, + pub seed: bool, + pub first_page: bool, + pub bounded: bool, + pub target_details: Option, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RankedFilterReason { + MissingNamedContext, + MissingSubjectEvidence, + NoMatchedConcept, + InsufficientSupportedCoverage, + RepresentedOwnedDocument, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct RankedTermTrace { + pub term: String, + pub variants: Vec, + pub document_frequency: u64, + pub inverse_document_frequency: f64, + pub matched_fields: Vec, + pub action: bool, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct RankedTargetTrace { + pub terms: Vec, + pub field_lengths: BTreeMap, + pub supported_concepts: usize, + pub supported_coverage: f64, + pub exact_id: bool, + pub exact_name: bool, + pub semantic_score: f64, + pub score: Option, + pub filter_reason: Option, +} + +struct Retrieval { + ranked: Vec, + admitted: BTreeSet, + bounded: bool, + eligible: usize, + reads: u64, + probes: u64, + missing_subjects: Vec, + negative_qualification: bool, + temporal_qualification: BTreeSet, + legacy: bool, + relationship_legacy: bool, + ranks: BTreeMap, + recall_time: Duration, + ranking_time: Duration, + concept_matches: Vec, + witness_nodes: Vec, + witness_edges: Vec, + target_details: Option, +} + +pub(crate) fn is_broad_question(query: &str) -> bool { + query.split_whitespace().count() >= 3 + && search_query_terms(query).is_ok_and(|terms| !terms.is_empty()) + && plan_natural_query(query).is_ok_and(|plan| { + matches!( + plan.intent(), + NaturalQueryIntent::Search | NaturalQueryIntent::Fallback + ) + }) +} + +fn alternatives(term: &str) -> Vec { + let related: &[&str] = match term { + "error" | "exception" | "diagnostic" => &["error", "exception", "diagnostic"], + "configuration" | "config" => &["configuration", "config", "setting"], + "save" | "persist" | "store" => &["save", "persist", "store"], + "render" => &["render"], + "server" => &["server", "serve"], + "start" => &["start", "begin", "run"], + "enforc" | "enforce" => &["enforc", "enforce"], + _ => &[], + }; + std::iter::once(term.to_owned()) + .chain(related.iter().map(|s| (*s).to_owned())) + .collect::>() + .into_iter() + .collect() +} + +fn plural_acronym_stem(word: &str) -> Option<&str> { + let stem = word.strip_suffix('s')?; + (stem.len() >= 2 && stem.bytes().all(|byte| byte.is_ascii_uppercase())).then_some(stem) +} + +/// Lexical matches cannot prove ordering. Keep that condition in the original +/// request and diagnostics, rather than treating the connective as a subject. +/// Quoted and qualified identities retain their literal search terms. +fn retrieval_content(question: &str) -> (String, BTreeSet) { + let identifier_character = |character: char| { + character.is_alphanumeric() || matches!(character, '_' | ':' | '.' | '/' | '\\' | '-') + }; + let mut content = String::with_capacity(question.len()); + let mut conditions = BTreeSet::new(); + let mut quoted = None; + for part in question.split_inclusive(|character: char| !identifier_character(character)) { + let word = part.trim_end_matches(|character: char| !identifier_character(character)); + let lower = word.to_ascii_lowercase(); + if quoted.is_none() && matches!(lower.as_str(), "before" | "after" | "during" | "until") { + conditions.insert(lower); + content.push(' '); + } else { + content.push_str(word); + } + let suffix = &part[word.len()..]; + content.push_str(suffix); + for delimiter in suffix + .chars() + .filter(|character| matches!(character, '`' | '"')) + { + if quoted == Some(delimiter) { + quoted = None; + } else if quoted.is_none() { + quoted = Some(delimiter); + } + } + } + (content, conditions) +} + +fn terms(question: &str) -> Vec { + // Natural prose uses plural acronyms such as APIs and TTLs. Code's camel + // case splitter would otherwise turn TTLs into `tt` and `ls`. Explicit + // quoted/qualified identities are still checked against the original text. + let normalized = question + .split_inclusive(|character: char| !character.is_alphanumeric()) + .map(|part| { + let word = part.trim_end_matches(|character: char| !character.is_alphanumeric()); + plural_acronym_stem(word).map_or_else( + || part.to_owned(), + |stem| format!("{}{}", stem.to_ascii_lowercase(), &part[word.len()..]), + ) + }) + .collect::(); + crate::query_terms(&normalized) + .into_iter() + .filter(|term| { + !matches!( + term.as_str(), + "builtin" + | "built" + | "cli" + | "output" + | "code" + | "implement" + | "implementation" + | "define" + | "defin" + | "defined" + | "definition" + | "declaration" + | "declar" + | "located" + | "locat" + | "location" + | "happen" + | "please" + ) + }) + .collect::>() + .into_iter() + .collect() +} + +fn frequency( + engine: &CodeQueryEngine, + backend: &PinnedDiscoveryBackend<'_>, + term: &str, +) -> Result, QueryError> { + let frequency = match backend { + PinnedDiscoveryBackend::Store(reader) => reader + .fielded_document_frequency(term) + .map(|frequency| frequency.map(|df| (reader.manifest().node_count, df))) + .map_err(crate::code_query::snapshot_error), + PinnedDiscoveryBackend::Materialized { graph, .. } => { + let connection = engine.connection.as_ref().ok_or_else(|| { + QueryError::new( + QueryErrorKind::Internal, + "query_index_missing", + "fielded search requires the native search index", + ) + })?; + let df = connection + .query_row( + "SELECT frequency FROM fielded_stats WHERE term=?1", + params![term], + |row| row.get::<_, u64>(0), + ) + .optional() + .map_err(crate::code_query::sql_error)? + .unwrap_or(0); + if df > graph.nodes.len() as u64 { + return Err(QueryError::new( + QueryErrorKind::CorruptArtifact, + "invalid_fielded_frequency", + "Fielded document frequency exceeds the graph node count; rebuild the query cache", + )); + } + Ok(Some((graph.nodes.len() as u64, df))) + } + }?; + frequency + .map(|(n, df)| { + let extra = engine + .source_query + .as_ref() + .and_then(|index| index.frequencies.get(term)) + .copied() + .unwrap_or(0); + let df = df.checked_add(extra).filter(|df| *df <= n).ok_or_else(|| { + QueryError::new( + QueryErrorKind::CorruptArtifact, + "invalid_source_document_frequency", + "Combined fielded frequency exceeds graph document count", + ) + })?; + Ok((n, df)) + }) + .transpose() +} + +fn matching(document: &SearchDocument, variants: &[String]) -> Vec<&'static str> { + document + .fields + .iter() + .filter(|(_, field)| variants.iter().any(|term| field.contains(term))) + .map(|(name, _)| *name) + .collect() +} + +fn field_weight(name: &str) -> f64 { + match name { + "name" => 4.0, + "qualified_name" => 1.5, + "signature" => 2.0, + "declaring_type" => 1.0, + "document" => 1.0, + "document_link" => 0.35, + "alias" => 1.5, + _ => 0.5, + } +} + +fn enforce_ranked_response_budget(response: &RankedDiscoveryResponse) -> Result<(), QueryError> { + struct ByteBudget { + bytes: usize, + maximum: usize, + exceeded: bool, + } + impl std::io::Write for ByteBudget { + fn write(&mut self, buffer: &[u8]) -> std::io::Result { + self.bytes = self.bytes.saturating_add(buffer.len()); + if self.bytes > self.maximum { + self.exceeded = true; + return Err(std::io::Error::other("ranked response byte limit")); + } + Ok(buffer.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let mut budget = ByteBudget { + bytes: 0, + maximum: response.discovery.limits.max_response_bytes as usize, + exceeded: false, + }; + let encoded = serde_json::to_writer(&mut budget, response); + if budget.exceeded { + return Err(QueryError::new( + QueryErrorKind::InvalidParameter, + "ranked_response_limit", + "Ranked evidence or context exceeds the response byte bound; narrow scope, reduce max-nodes, or omit context", + )); + } + encoded.map_err(|error| { + QueryError::new( + QueryErrorKind::Internal, + "ranked_serialization", + error.to_string(), + ) + }) +} + +impl CodeQueryEngine { + pub(crate) fn is_fielded_question(&self, question: &str) -> Result { + if !is_broad_question(question) { + return Ok(false); + } + let backend = self.backend.pin_discovery()?; + let subject = plan_natural_query(question) + .ok() + .filter(|plan| { + plan.intent() == NaturalQueryIntent::Search + && plan.routes_to_typed_query() + && plan.operands().len() == 1 + }) + .map(|plan| crate::ranking::normalize_symbol_name(&plan.operands()[0])); + let mut seen = BTreeSet::new(); + for literal in subject + .iter() + .cloned() + .chain(crate::discovery::literal_identifier_terms(question)) + .filter(|literal| seen.insert(literal.clone())) + .take(8) + { + self.check_deadline()?; + let (nodes, truncated) = backend.nodes_by_normalized_name(&literal, 8)?; + if truncated { + return Ok(false); + } + if nodes.iter().any(|node| { + let allowed_kind = if subject.as_deref() == Some(literal.as_str()) { + node.kind != NodeKind::Resource + } else { + !matches!( + node.kind, + NodeKind::Resource + | NodeKind::File + | NodeKind::Module + | NodeKind::Package + | NodeKind::Namespace + ) + }; + allowed_kind + && (crate::ranking::normalize_symbol_name(&node.name) == literal + || crate::ranking::normalize_symbol_name(&node.qualified_name) == literal) + }) { + return Ok(false); + } + } + Ok(true) + } + + fn retrieve_broad( + &self, + request: &SearchRequest, + scopes: &[DiscoveryScope], + check: &dyn Fn() -> Result<(), QueryError>, + admit: &dyn Fn(&NodeRecord) -> bool, + trace_target: Option<&str>, + retain_excerpts: bool, + ) -> Result { + let started = Instant::now(); + crate::code_query::validate_limits(&request.limits)?; + let _ = search_query_terms(&request.query)?; + let negative_qualification = request + .query + .split(|character: char| !character.is_alphanumeric() && character != '_') + .any(|word| { + matches!( + word.to_ascii_lowercase().as_str(), + "not" | "never" | "without" + ) + }); + check()?; + let backend = self.backend.pin_discovery()?; + let (content_question, temporal_qualification) = retrieval_content(&request.query); + let original_terms = terms(&content_question); + let named_contexts = named_contexts(&request.query, &original_terms); + let explicit_identifiers = explicit_identifiers(&request.query); + let compounds = if explicit_identifiers.is_empty() { + compounds::Compounds::parse(&content_question) + } else { + compounds::Compounds::default() + }; + if original_terms.len() > compass_model::query_contract::MAX_INDEXED_QUERY_TERMS { + return Err(QueryError::new( + QueryErrorKind::ExpansionLimit, + "broad_concept_limit", + "Broad retrieval supports at most 32 normalized content concepts; narrow the question rather than silently dropping its meaning", + )); + } + let wants_documents = original_terms.iter().any(|term| { + matches!( + term.as_str(), + "document" | "documentation" | "design" | "rationale" | "readme" | "comment" + ) + }); + let mut action_terms = original_terms + .iter() + .filter(|term| is_retrieval_action(term)) + .cloned() + .collect::>(); + let mut semantic_groups = BTreeMap::>::new(); + let mut covered = BTreeSet::new(); + let expansions = crate::concepts::expansions(&request.query); + for (phrase, alternative) in &expansions { + let phrase_terms = terms(phrase); + if phrase_terms.is_empty() + || !phrase_terms + .iter() + .all(|term| original_terms.contains(term)) + { + continue; + } + covered.extend(phrase_terms.iter().cloned()); + let entry = semantic_groups.entry(phrase.clone()).or_default(); + entry.extend(phrase_terms); + entry.extend( + identifier_search_terms(alternative) + .into_iter() + .map(compass_model::canonical_code_token), + ); + } + for term in &original_terms { + if !covered.contains(term) { + semantic_groups.insert(term.clone(), alternatives(term).into_iter().collect()); + } + } + // Conservative spelling alternatives preserve silent-e verbs and + // nouns such as cookies without rewriting strict/historical tokens. + for raw in identifier_search_terms(&request.query) { + if raw.is_ascii() && raw.len() > 4 { + let canonical = compass_model::canonical_code_token(raw.clone()); + if let Some(variants) = semantic_groups.get_mut(&canonical) { + if let Some(stem) = raw.strip_suffix('s') { + variants.insert(compass_model::canonical_code_token(stem.to_owned())); + } + if let Some(stem) = raw.strip_suffix("ed").or_else(|| raw.strip_suffix("ing")) { + let alternate = format!("{stem}e"); + if is_retrieval_action(&alternate) { + action_terms.insert(canonical); + } + variants.insert(alternate); + } + } + } + } + let mut groups = Vec::new(); + let mut vocabulary_fallbacks = BTreeMap::, Vec)>::new(); + let mut literal_supported_terms = BTreeSet::new(); + let mut missing_subjects = Vec::new(); + let mut legacy = false; + for (term, variants) in semantic_groups { + check()?; + let mut variants = variants.into_iter().collect::>(); + let mut df = 0; + let mut count = 1; + let mut group_legacy = false; + for variant in &variants { + if let Some((n, frequency)) = frequency(self, &backend, variant)? { + count = n.max(1); + df = df.max(frequency); + } else { + legacy = true; + group_legacy = true; + } + } + // Only corpus-supported terms can justify an implementation. An + // absent concrete subject is not replaced by a ubiquitous verb. + if df == 0 && !legacy && !action_terms.contains(&term) { + missing_subjects.push(term.clone()); + } + if df > 0 { + literal_supported_terms.insert(term.clone()); + } + // Keep known indexed words and explicit identities literal. Only + // a verified zero-frequency ordinary word can use these hints. + if df == 0 + && !group_legacy + && !wants_documents + && named_contexts.is_empty() + && explicit_identifiers.is_empty() + { + let literal = variants.clone(); + let mut aliases = Vec::new(); + let mut alias_df = 0; + for alias in crate::concepts::unindexed_vocabulary_alternatives(&term) { + check()?; + match frequency(self, &backend, alias)? { + Some((n, frequency)) if frequency > 0 => { + count = n.max(1); + alias_df = alias_df.max(frequency); + aliases.push((*alias).to_owned()); + } + Some(_) => {} + None => group_legacy = true, + } + } + legacy |= group_legacy; + if !group_legacy && !aliases.is_empty() { + variants.extend(aliases.iter().cloned()); + variants.sort(); + variants.dedup(); + df = alias_df; + vocabulary_fallbacks.insert(term.clone(), (literal, aliases)); + } + } + let idf = ((count as f64 + 1.0) / (df as f64 + 1.0)).ln() + 1.0; + groups.push((term, variants, idf, df)); + } + let mut missing_named_context = false; + for named in &named_contexts { + check()?; + if frequency(self, &backend, named)?.is_some_and(|(_, df)| df == 0) { + missing_subjects.push(named.clone()); + missing_named_context = true; + } + } + missing_subjects.sort(); + missing_subjects.dedup(); + groups.sort_by(|a, b| a.3.cmp(&b.3).then_with(|| a.0.cmp(&b.0))); + let mut nodes = BTreeMap::::new(); + let mut reads = 0_u64; + let mut probes = 2_u64; // exact ID and exact normalized-name lookups + let mut bounded = false; + // Exact identity remains authoritative even for a phrase-shaped name. + if let Some(node) = backend.node_by_id(&request.query)? { + reads += 1; + if discovery_scope_matches(&node, scopes) { + nodes.insert(node.id.clone(), node); + } + } + let (exact, exact_bounded) = + backend.nodes_by_normalized_name(&normalize_symbol(&request.query), POSTING_QUOTA)?; + bounded |= exact_bounded; + reads = reads.saturating_add(exact.len() as u64 + u64::from(exact_bounded)); + for node in exact { + if discovery_scope_matches(&node, scopes) { + nodes.insert(node.id.clone(), node); + } + } + if let Some(index) = &self.source_query { + bounded |= index.bounded; + let mut source_queries = Vec::new(); + for offset in 0..groups.iter().map(|g| g.1.len()).max().unwrap_or(0) { + for (_, variants, _, _) in &groups { + if let Some(term) = variants.get(offset) { + source_queries.push(term); + } + } + } + bounded |= source_queries.len() > 8; + let mut hydrated = BTreeSet::new(); + for term in source_queries.into_iter().take(8) { + check()?; + probes += 1; + let (owners, limited) = index.candidates(term, 64); + bounded |= limited; + reads += owners.len() as u64; + for owner in owners { + check()?; + if hydrated.contains(&owner.node_id) { + continue; + } + if hydrated.len() >= 64 { + bounded = true; + break; + } + hydrated.insert(owner.node_id.clone()); + reads += 1; + let node = backend.node_by_id(&owner.node_id)?.ok_or_else(|| { + QueryError::new( + QueryErrorKind::CorruptArtifact, + "source_query_owner_missing", + "Source-body owner is absent from its bound graph", + ) + })?; + if !node.kind.is_callable() + || node.source.as_ref() != Some(&owner.source) + || !relationships::exact_ast(&node) + { + return Err(QueryError::new( + QueryErrorKind::CorruptArtifact, + "source_query_owner_mismatch", + "Source-body owner does not match its exact AST graph record", + )); + } + if discovery_scope_matches(&node, scopes) && admit(&node) { + nodes.entry(node.id.clone()).or_insert(node); + } + } + } + } + let mut relationships = relationships::recall(&backend, &groups, scopes, check, admit)?; + reads = reads.saturating_add(relationships.reads); + probes = probes.saturating_add(relationships.probes); + bounded |= relationships.bounded; + nodes.append(&mut relationships.nodes); + let mut queries = Vec::new(); + // Reserve independent quotas for primary declarations and prose/other + // evidence, rarest concepts first. Conjunction coverage is checked on + // source-owned fields after fusion, not on incompatible SQL/store + // posting intersections. Neither backend scans the entire corpus. + for (_, variants, _, _) in &groups { + for variant in variants { + queries.push(vec![format!("__compass_primary_v1_{variant}")]); + } + for variant in variants { + queries.push(vec![variant.clone()]); + } + } + let mut ownership_candidates = BTreeMap::new(); + for query in queries { + check()?; + if probes + 8 + ownership::PROBE_RESERVE >= FIELD_PROBE_CEILING + || reads.saturating_add(POSTING_QUOTA as u64 * 4 + ownership::READ_RESERVE) + > FIELD_READ_CEILING + { + bounded = true; + break; + } + let read = self.discovery_term_candidates(&backend, &query, POSTING_QUOTA)?; + reads = reads.saturating_add(read.node_ids_decoded); + probes += 1 + read.chunks_decoded; + bounded |= read.truncated; + for node in read.nodes { + // Context may lie outside a node-only answer scope. It can + // nominate only a uniquely witnessed, in-scope callable. + if ownership::is_local(&node) { + ownership_candidates + .entry(node.id.clone()) + .or_insert_with(|| node.clone()); + } + if discovery_scope_matches(&node, scopes) { + nodes.entry(node.id.clone()).or_insert(node); + } + } + } + let mut ownership = ownership::recall( + &backend, + &ownership_candidates, + &groups, + scopes, + check, + admit, + )?; + reads = reads.saturating_add(ownership.reads); + probes = probes.saturating_add(ownership.probes); + bounded |= ownership.bounded; + nodes.append(&mut ownership.nodes); + let mut owned_passages = BTreeMap::)>>::new(); + let mut owned_excerpts = BTreeMap::>::new(); + let mut excerpt_terms = groups + .iter() + .filter(|group| !action_terms.contains(&group.0)) + .flat_map(|group| group.1.iter().cloned()) + .collect::>(); + if excerpt_terms.is_empty() { + excerpt_terms.extend(groups.iter().flat_map(|group| group.1.iter().cloned())); + } + let mut owned_document_ids = BTreeMap::>::new(); + let mut linked_passages = BTreeMap::>::new(); + let mut linked_sources = BTreeMap::>::new(); + let mut concept_matches = Vec::new(); + let mut witness_nodes = relationships + .targets + .into_iter() + .map(|(id, node)| (id, query_node(&node))) + .collect::>(); + let mut witness_edges = relationships + .witnesses + .into_iter() + .map(|(id, edge)| (id, query_edge(&edge))) + .collect::>(); + witness_nodes.extend( + ownership + .children + .into_iter() + .map(|(id, node)| (id, query_node(&node))), + ); + witness_edges.extend( + ownership + .witnesses + .into_iter() + .map(|(id, edge)| (id, query_edge(&edge))), + ); + let mut document_ids = nodes + .values() + .filter(|node| search_document(node).fields.contains_key("document")) + .map(|node| { + let document = search_document(node); + let relevance = groups + .iter() + .filter(|(_, variants, _, _)| !matching(&document, variants).is_empty()) + .map(|(_, _, idf, _)| *idf) + .sum::(); + (node.id.clone(), relevance) + }) + .collect::>(); + document_ids.sort_by(|a, b| b.1.total_cmp(&a.1).then_with(|| a.0.cmp(&b.0))); + bounded |= document_ids.len() > 16; + for (id, _) in document_ids.into_iter().take(16) { + check()?; + if reads.saturating_add(16) > FIELD_READ_CEILING { + bounded = true; + break; + } + let Some(passage) = nodes.get(&id).cloned() else { + continue; + }; + let Some(anchor) = passage.source.clone() else { + continue; + }; + // A published Markdown mention may suggest an implementation, + // but is never an ownership assertion. Require both the subject + // and the target's actual name in the source-backed passage. + let passage_document = search_document(&passage); + let passage_terms = passage_document + .fields + .get("document") + .cloned() + .unwrap_or_default(); + if groups + .iter() + .filter(|(term, _, _, _)| !action_terms.contains(term)) + .all(|(_, variants, _, _)| variants.iter().any(|term| passage_terms.contains(term))) + { + let mut pending = + VecDeque::from([(passage.clone(), vec![id.clone()], Vec::new(), 0_usize)]); + let mut visited = BTreeSet::new(); + while let Some((current, via, path_edges, depth)) = pending.pop_front() { + check()?; + if !visited.insert(current.id.clone()) { + continue; + } + if visited.len() > 16 + || reads.saturating_add(81) > FIELD_READ_CEILING + || probes >= FIELD_PROBE_CEILING + { + bounded = true; + break; + } + probes += 1; + let (outgoing, limited) = backend.matching_bounded( + ¤t.id, + false, + &[ + EdgeKind::Contains, + EdgeKind::Documents, + EdgeKind::References, + ], + true, + 16, + )?; + bounded |= limited; + reads = reads.saturating_add(outgoing.len() as u64 + u64::from(limited)); + for edge in outgoing { + if crate::code_query::is_heuristic(&edge) { + continue; + } + reads += 1; + let Some(target) = backend.node_by_id(&edge.target)? else { + continue; + }; + let target_document = search_document(&target); + if edge.kind == EdgeKind::Contains + && target_document.fields.contains_key("document") + { + if depth < 2 { + let mut next_via = via.clone(); + next_via.push(target.id.clone()); + let mut next_edges = path_edges.clone(); + next_edges.push(edge); + pending.push_back((target, next_via, next_edges, depth + 1)); + } else { + bounded = true; + } + } else if edge.kind != EdgeKind::Contains + && !target_document.fields.contains_key("document") + && discovery_scope_matches(&target, scopes) + && admit(&target) + { + let name_terms = identifier_search_terms(&target.name) + .into_iter() + .map(compass_model::canonical_code_token) + .collect::>(); + if name_terms.is_empty() || !name_terms.is_subset(&passage_terms) { + continue; + } + linked_passages + .entry(target.id.clone()) + .or_default() + .extend(passage_terms.iter().cloned()); + linked_sources + .entry(target.id.clone()) + .or_default() + .insert(anchor.clone()); + concept_matches.push(ConceptMatch { + node_id: target.id.clone(), + method: ConceptMatchMethod::DocumentLink, + query: request.query.clone(), + matched: target.qualified_name.clone(), + via: via.clone(), + }); + for record in [passage.clone(), current.clone()] { + witness_nodes.insert(record.id.clone(), query_node(&record)); + } + // Intermediate document endpoints are looked up + // from the published graph, never synthesized. + for step in &via { + reads += 1; + if let Some(record) = backend.node_by_id(step)? { + witness_nodes.insert(step.clone(), query_node(&record)); + } + } + for record in path_edges.iter().chain(std::iter::once(&edge)) { + witness_edges.insert(record.id.clone(), query_edge(record)); + } + nodes.entry(target.id.clone()).or_insert(target); + } + } + } + } + check()?; + if reads.saturating_add(17) > FIELD_READ_CEILING || probes >= FIELD_PROBE_CEILING { + bounded = true; + break; + } + probes += 1; + let (edges, limited) = + backend.matching_bounded(&id, false, &[EdgeKind::Documents], true, 8)?; + bounded |= limited; + reads = reads.saturating_add(edges.len() as u64 + u64::from(limited)); + for edge in edges { + if !compass_model::search::is_exact_nonheuristic_relationship(&edge) { + continue; + } + if !edge.evidence.iter().any(|evidence| { + evidence.origin == EvidenceOrigin::Ast + && evidence.confidence == EvidenceConfidence::Exact + }) { + continue; + } + reads += 1; + let Some(owner) = backend.node_by_id(&edge.target)? else { + continue; + }; + let Some(source) = &owner.source else { + continue; + }; + // Enclosed docstrings keep their existing ownership rule. + // Leading excerpts require the versioned two-anchor AST proof; + // a generic document link or matching name cannot substitute. + let enclosed = source.file == anchor.file + && source.start_line <= anchor.start_line + && source.end_line >= anchor.end_line + && source.start_byte <= anchor.start_byte + && source.end_byte >= anchor.end_byte; + if !is_primary_search_kind(owner.kind) + || !(enclosed + || compass_model::provenance::is_proven_leading_documentation( + &passage, &owner, &edge, + )) + { + continue; + } + // Only the raw-retention leading-doc producer profile proves a + // verbatim excerpt. Generic enclosed/normalized docstrings keep + // their retrieval pointer but cannot substitute for this proof. + if retain_excerpts + && compass_model::provenance::is_proven_leading_documentation( + &passage, &owner, &edge, + ) + && let Some(compass_model::code_graph::NodeDetails::Resource(details)) = + &passage.details + && let Some(text) = &details.content + && let Some(excerpt) = excerpts::select( + RankedExcerptKind::AstDocumentation, + &anchor, + text, + &excerpt_terms, + 512, + ) + { + let quotes = owned_excerpts.entry(owner.id.clone()).or_default(); + if quotes.len() < 2 && !quotes.contains(&excerpt) { + quotes.push(excerpt); + } + } + let entry = owned_passages.entry(owner.id.clone()).or_default(); + if entry.len() < 2 { + owned_document_ids + .entry(owner.id.clone()) + .or_default() + .insert(id.clone()); + entry.push(( + anchor.clone(), + search_document(&passage) + .fields + .get("document") + .cloned() + .unwrap_or_default(), + )); + } else { + bounded = true; + } + if discovery_scope_matches(&owner, scopes) { + nodes.entry(owner.id.clone()).or_insert(owner); + } + } + } + let mut semantic = BTreeMap::new(); + if self.semantic_search && !missing_named_context { + for (node, similarity) in self.semantic_candidates_guarded( + &request.query, + request.limits.max_candidates, + check, + )? { + check()?; + if discovery_scope_matches(&node, scopes) + && similarity.is_finite() + && similarity > 0.0 + { + semantic.insert(node.id.clone(), similarity); + nodes.entry(node.id.clone()).or_insert(node); + } + } + } + nodes.retain(|_, node| admit(node)); + let admitted = nodes.keys().cloned().collect(); + let recall_time = started.elapsed(); + let ranking_started = Instant::now(); + let wants_tests = original_terms + .iter() + .any(|term| matches!(term.as_str(), "test" | "fixture")); + let supported_groups = groups + .iter() + .filter(|(_, _, _, df)| legacy || *df > 0) + .count(); + let literal_supported_groups = if legacy { + groups.len() + } else { + literal_supported_terms.len() + }; + let mut target_details = None; + let mut ranked = Vec::new(); + for node in nodes.into_values() { + check()?; + let mut document = search_document(&node); + // Only inspect aliases when a recalled concept has no explanation + // in the plain projection. This keeps enrichment inside the same + // global read envelope as candidate recall. + if groups + .iter() + .any(|(_, variants, _, _)| matching(&document, variants).is_empty()) + { + if reads.saturating_add(20) <= FIELD_READ_CEILING && probes < FIELD_PROBE_CEILING { + probes += 1; + let (aliases, limited) = + backend.matching_bounded(&node.id, true, &[EdgeKind::Aliases], true, 9)?; + bounded |= limited; + reads = reads.saturating_add(aliases.len() as u64 + u64::from(limited)); + let mut names = Vec::new(); + for alias in aliases { + if !alias.evidence.is_empty() + && compass_model::search::is_exact_nonheuristic_relationship(&alias) + { + reads += 1; + names.push( + backend + .node_by_id(&alias.source)? + .map(|source| source.name) + .unwrap_or_default(), + ); + } else { + names.push(String::new()); + } + } + document = + search_document_with_aliases(&node, names.iter().map(String::as_str)); + } else { + bounded = true; + } + } + let passages = owned_passages.remove(&node.id).unwrap_or_default(); + if let Some(linked) = linked_passages.get(&node.id) { + document.fields.insert("document_link", linked.clone()); + } + if !passages.is_empty() { + document.fields.insert( + "owned_document", + passages + .iter() + .flat_map(|(_, terms)| terms.iter().cloned()) + .collect(), + ); + } + // Only terminal names on verified outgoing code relationships, + // never the target's namespace, path or documentation. + let relationship_terms = relationships.terms.remove(&node.id).unwrap_or_default(); + let relationship_needed = groups.iter().any(|(_, variants, _, _)| { + matching(&document, variants).is_empty() + && variants + .iter() + .any(|term| relationship_terms.contains(term)) + }); + if !relationship_terms.is_empty() { + document.fields.insert("code_reference", relationship_terms); + } + let identifier_terms = ownership.terms.remove(&node.id).unwrap_or_default(); + let identifier_needed = groups.iter().any(|(_, variants, _, _)| { + matching(&document, variants).is_empty() + && variants.iter().any(|term| identifier_terms.contains(term)) + }); + if !identifier_terms.is_empty() { + document.fields.insert("owned_identifier", identifier_terms); + } + let mut body_needed = false; + let mut body_sources = BTreeSet::new(); + let source_owner = self + .source_query + .as_ref() + .and_then(|index| index.owner(&node.id)); + if let Some(owner) = source_owner { + if !node.kind.is_callable() + || node.source.as_ref() != Some(&owner.source) + || !relationships::exact_ast(&node) + { + return Err(QueryError::new( + QueryErrorKind::CorruptArtifact, + "source_query_owner_mismatch", + "Source evidence no longer matches the pinned graph", + )); + } + let body = owner.document(); + body_needed = groups.iter().any(|(_, variants, _, _)| { + // Some existing header projections include leading body + // comments. Repeated signature/body terms are not independent + // corroboration of the requested behavior. + matching(&document, variants) + .iter() + .all(|field| *field == "signature") + && !matching(&body, variants).is_empty() + }); + document.truncated |= body.truncated; + document.fields.extend(body.fields); + let terms = groups + .iter() + .flat_map(|g| g.1.iter().cloned()) + .collect::>(); + let (sources, limited) = crate::source_query::matching_sources(owner, &terms); + body_sources = sources; + bounded |= limited; + } + bounded |= document.truncated; + let inspection_signature = compounds::inspection_signature(match &node.details { + Some(compass_model::code_graph::NodeDetails::Symbol(details)) => { + details.signature.as_deref() + } + _ => None, + }); + let compound_matches = if document.truncated { + compounds::Matches::default() + } else { + compounds.matches( + &node.name, + inspection_signature, + source_owner.map_or(&[], |owner| owner.identifiers.as_slice()), + ) + }; + let exact_id = node.id == request.query; + let exact_name = normalize_symbol(&node.name) == normalize_symbol(&request.query) + || normalize_symbol(&node.qualified_name) == normalize_symbol(&request.query); + let mut vocabulary_matches = Vec::new(); + for (term, (literal, aliases)) in &vocabulary_fallbacks { + if matching(&document, literal).is_empty() { + for alias in aliases { + if !matching(&document, std::slice::from_ref(alias)).is_empty() { + vocabulary_matches.push((term.clone(), alias.clone())); + } + } + } + } + let vocabulary_needed = !exact_id && !exact_name && !vocabulary_matches.is_empty(); + let literal_topic = groups.iter().any(|(term, variants, _, _)| { + !action_terms.contains(term) + && !matches!(term.as_str(), "all" | "only" | "new" | "made") + && matching( + &document, + vocabulary_fallbacks + .get(term) + .map_or(variants, |(literal, _)| literal), + ) + .iter() + .any(|field| { + matches!( + *field, + "name" + | "signature" + | "declaring_type" + | "owned_document" + | "body_identifier" + | "body_comment" + | "code_reference" + | "owned_identifier" + ) + }) + }); + // Observe one requested target using already hydrated, bounded + // fields. Normal queries allocate no trace and perform no extra + // lookups; tracing does not affect admission, scores or ordering. + let trace = (trace_target == Some(node.id.as_str())).then(|| { + let terms = groups + .iter() + .map(|(term, variants, idf, df)| RankedTermTrace { + term: term.clone(), + variants: variants.clone(), + document_frequency: *df, + inverse_document_frequency: *idf, + matched_fields: matching(&document, variants) + .into_iter() + .map(str::to_owned) + .collect(), + action: action_terms.contains(term), + }) + .collect::>(); + let matched = terms + .iter() + .filter(|term| !term.matched_fields.is_empty()) + .count(); + RankedTargetTrace { + terms, + field_lengths: document + .fields + .iter() + .map(|(field, terms)| ((*field).to_owned(), terms.len())) + .collect(), + supported_concepts: supported_groups, + supported_coverage: matched as f64 / supported_groups.max(1) as f64, + exact_id, + exact_name, + semantic_score: semantic.get(&node.id).copied().unwrap_or(0.0), + score: None, + filter_reason: None, + } + }); + // Unknown ordinary words may be a paraphrase of indexed behavior. + // Retain tentative candidates and disclose their missing context; + // a missing explicitly named subject remains a retrieval veto. + if !exact_id && !exact_name && missing_named_context { + if let Some(mut details) = trace { + details.filter_reason = Some(RankedFilterReason::MissingNamedContext); + target_details = Some(details); + } + continue; + } + let mut matched_terms = Vec::new(); + let mut matched_fields = BTreeSet::new(); + let mut score = 0.0; + let mut action_matched = false; + let mut subject_matched = false; + let mut implementation_action = false; + let mut implementation_subject = false; + let mut compound_subject = false; + for (term, variants, idf, _) in &groups { + let fields = matching(&document, variants); + if fields.is_empty() { + continue; + } + matched_terms.push(term.clone()); + // Prose, signatures and namespaces can mention an operation + // without identifying its implementation. Prefer a lexical + // operation signal in the callable's own name, identifiers or + // exact witnessed outgoing references, with separate topic + // evidence. Comments alone never provide this preference. + // Callable identifier postings include parameters. Retained + // signature terms therefore cannot independently supply the + // code signal; missing/clipped signatures keep ordinary ranking. + implementation_action |= action_terms.contains(term) + && (fields.contains(&"name") + || (!document.truncated + && inspection_signature.is_some() + && !fields.contains(&"signature") + && fields.iter().any(|field| { + matches!(*field, "body_identifier" | "code_reference") + }))); + implementation_subject |= !action_terms.contains(term) + && fields.iter().any(|field| { + matches!( + *field, + "name" + | "signature" + | "declaring_type" + | "owned_document" + | "body_identifier" + | "body_comment" + | "code_reference" + | "owned_identifier" + ) + }); + compound_subject |= !compound_matches.phrases.is_empty() + && !action_terms.contains(term) + && !compounds.contains_component_group(term) + && !matches!(term.as_str(), "all" | "only" | "new" | "made") + && fields.iter().any(|field| { + matches!( + *field, + "name" + | "signature" + | "declaring_type" + | "owned_document" + | "body_identifier" + | "body_comment" + | "code_reference" + | "owned_identifier" + ) + }); + subject_matched |= !action_terms.contains(term) + && fields + .iter() + .any(|field| !matches!(*field, "kind" | "language")); + let field_score = fields + .iter() + .map(|field| { + let length = + document.fields.get(field).map_or(1, BTreeSet::len).max(1) as f64; + // BM25F-style saturation/length normalization uses corpus + // DF and bounded per-field unique-token frequencies. + field_weight(field) / (0.75 + 0.25 * length / 16.0) + }) + .sum::(); + score += idf * field_score * 2.2 / (1.2 + field_score); + action_matched |= action_terms.contains(term) && fields.contains(&"name"); + matched_fields.extend(fields); + } + let semantic_score = semantic.get(&node.id).copied().unwrap_or(0.0); + if vocabulary_needed && !literal_topic && semantic_score == 0.0 { + if let Some(mut details) = trace { + details.filter_reason = Some(RankedFilterReason::MissingSubjectEvidence); + target_details = Some(details); + } + continue; + } + // A pure kind/language listing needs no topic. Once the question + // asks for any other subject, a facet cannot stand in for it. + // Keep unfamiliar wording tentative only when this candidate has + // real subject evidence outside those two descriptive fields. + let facet_only_subjects = groups + .iter() + .filter(|(term, _, _, _)| !action_terms.contains(term)) + .all(|(_, variants, _, _)| { + matching(&document, variants) + .iter() + .any(|field| matches!(*field, "kind" | "language")) + }); + if !subject_matched + && !facet_only_subjects + && groups + .iter() + .any(|(term, _, _, _)| !action_terms.contains(term)) + && !exact_id + && !exact_name + && semantic_score == 0.0 + { + if let Some(mut details) = trace { + details.filter_reason = Some(RankedFilterReason::MissingSubjectEvidence); + target_details = Some(details); + } + continue; + } + if matched_terms.is_empty() && !exact_id && !exact_name && semantic_score == 0.0 { + if let Some(mut details) = trace { + details.filter_reason = Some(RankedFilterReason::NoMatchedConcept); + target_details = Some(details); + } + continue; + } + let coverage = matched_terms.len() as f64 / groups.len().max(1) as f64; + // Unknown paraphrase words still lower score and relevance. They + // must not also veto otherwise well-supported inspection pointers. + // Legacy statistics are unknown, so retain the full denominator. + let supported_coverage = matched_terms.len() as f64 / supported_groups.max(1) as f64; + // Additional alias frequencies must not veto an unchanged literal + // candidate that passed the original corpus-supported screen. + let preserves_literal_support = !vocabulary_needed + && matched_terms.len() as f64 / literal_supported_groups.max(1) as f64 >= 0.4; + if supported_coverage < 0.4 + && !preserves_literal_support + && !exact_id + && !exact_name + && semantic_score == 0.0 + { + if let Some(mut details) = trace { + details.filter_reason = Some(RankedFilterReason::InsufficientSupportedCoverage); + target_details = Some(details); + } + continue; + } + score *= coverage * coverage; + if semantic_score > 0.0 { + score += semantic_score.min(1.0) * 0.25; + matched_fields.insert("semantic_lsa"); + } + if action_matched && node.kind.is_callable() { + score *= 1.5; + } + if !passages.is_empty() { + score *= 1.25; + } + if matches!( + node.kind, + NodeKind::Parameter | NodeKind::Field | NodeKind::Property | NodeKind::Variable + ) { + score *= 0.35; + } + if document.fields.contains_key("document") && !wants_documents { + // Prose remains a fallback candidate and a supporting passage. + // For implementation questions it should not displace the + // declaration that owns or is explicitly named by that prose. + score *= 0.35; + } + if test_context(&node) != wants_tests { + score *= 0.45; + } + let preferred_implementation = implementation_action + && implementation_subject + && !vocabulary_needed + // A lexical preference cannot establish the relationship + // between multiple actions (e.g. stopping new task starts). + && action_terms.len() == 1 + && !negative_qualification + && temporal_qualification.is_empty() + && node.kind.is_callable() + && !wants_documents + && test_context(&node) == wants_tests; + let preferred_compound = !compound_matches.phrases.is_empty() + && compound_subject + && !vocabulary_needed + && node.kind.is_callable() + && !negative_qualification + && temporal_qualification.is_empty() + && !wants_documents + && test_context(&node) == wants_tests; + if preferred_compound { + // Bounded lexical cohesion outranks an isolated operation word, + // while exact identity keeps its existing higher tier. + score += 20_000.0 + compound_matches.phrases.len() as f64 * 100.0; + let mut selected = compound_matches.sources.clone(); + for source in &body_sources { + if selected.len() >= 8 { + bounded |= !selected.contains(source); + } else { + selected.insert(source.clone()); + } + } + body_sources = selected; + } else if preferred_implementation { + score += IMPLEMENTATION_ACTION_PREFERENCE; + } + if exact_id { + score += 1_000_000.0; + matched_fields.insert("exact_id"); + } + if exact_name { + score += 500_000.0; + matched_fields.insert("exact_name"); + } + let mut missing_context = groups + .iter() + .filter(|(term, variants, _, _)| { + !action_terms.contains(term) && matching(&document, variants).is_empty() + }) + .map(|(term, _, _, _)| term.clone()) + .collect::>(); + let evidence_terms = document.terms(); + missing_context.extend( + named_contexts + .iter() + .filter(|term| !evidence_terms.contains(*term)) + .cloned(), + ); + missing_context.extend( + explicit_identifiers + .iter() + .filter(|identifier| !has_explicit_identity(&node, identifier)) + .map(|identifier| format!("explicit identity `{identifier}`")), + ); + let relevance = if vocabulary_needed + || (preferred_compound && !exact_id && !exact_name) + || negative_qualification + || !temporal_qualification.is_empty() + || ((relationship_needed || identifier_needed || body_needed) + && !exact_id + && !exact_name) + { + RelevanceBand::Low + } else if exact_id + || exact_name + || (missing_context.is_empty() + && coverage >= 0.8 + && action_matched + && node.kind.is_callable()) + { + RelevanceBand::High + } else if missing_context.is_empty() && coverage >= 0.5 { + RelevanceBand::Medium + } else { + RelevanceBand::Low + }; + let mut reasons = Vec::new(); + if vocabulary_needed { + reasons.push(format!("Vocabulary fallback for unindexed words: {}. These are inspection alternatives; equivalence and behavior remain unverified", vocabulary_matches.iter().map(|(term, alias)| format!("{term} -> {alias}")).collect::>().join(", "))); + } + if preferred_compound { + reasons.push(format!("Compound identifiers match requested words together: {}. Separate owner-topic evidence gives an inspection preference; equivalence and behavior remain unverified", compound_matches.phrases.into_iter().collect::>().join(", "))); + } else if preferred_implementation { + reasons.push("Implementation-local operation identifiers and separate topic evidence receive an inspection preference; this does not establish behavior or execution".to_owned()); + } + if negative_qualification { + reasons.push("Negative qualification is not established by lexical evidence. These are inspection candidates, not confirmation of the requested condition".to_owned()); + } + if !temporal_qualification.is_empty() { + reasons.push(format!( + "Temporal qualification ({}) is not established by lexical evidence. Inspect the candidate source to verify the requested ordering or condition", + temporal_qualification.iter().cloned().collect::>().join(", ") + )); + } + reasons.push(format!( + "Indexed {} evidence matches {} of {} query concepts", + matched_fields + .iter() + .copied() + .collect::>() + .join(", "), + matched_terms.len(), + groups.len() + )); + if !missing_context.is_empty() && !exact_id && !exact_name { + reasons.push(format!("Partial indexed evidence; missing owner-specific context: {}. The requested behavior is not established by this candidate", missing_context.into_iter().collect::>().join(", "))); + } + if document.fields.contains_key("document_link") { + reasons.push("Source-backed prose names this candidate through a published link; this does not establish ownership or implementation equivalence".to_owned()); + } + if matched_fields.contains("code_reference") { + reasons.push("Exact AST call/reference sites inside this declaration mention matching identifiers. This supports inspection, not confirmation of the requested behavior".to_owned()); + } + if matched_fields.contains("owned_identifier") { + reasons.push("Exact AST containment inside this declaration owns matching local identifiers. This supports inspection, not confirmation of the requested behavior".to_owned()); + } + if matched_fields.contains("body_identifier") || matched_fields.contains("body_comment") + { + reasons.push("Source-bound AST lexical evidence inside this callable matches query terms. Identifier/comment occurrence supports inspection; it does not establish the requested behavior".to_owned()); + } + if vocabulary_needed { + for (term, alias) in &vocabulary_matches { + concept_matches.push(ConceptMatch { + node_id: node.id.clone(), + method: ConceptMatchMethod::Synonym, + query: term.clone(), + matched: alias.clone(), + via: Vec::new(), + }); + } + } + for (concept, alternative) in &expansions { + let alternative_terms = identifier_search_terms(alternative) + .into_iter() + .map(compass_model::canonical_code_token) + .collect::>(); + if !alternative_terms.is_empty() && alternative_terms.is_subset(&document.terms()) { + concept_matches.push(ConceptMatch { + node_id: node.id.clone(), + method: ConceptMatchMethod::Synonym, + query: concept.clone(), + matched: alternative.clone(), + via: Vec::new(), + }); + } + } + if let Some(mut details) = trace { + details.score = Some(score); + target_details = Some(details); + } + ranked.push(RankedCandidate { + node: query_node(&node), + excerpts: Vec::new(), + score, + relevance, + matched_terms, + matched_fields: matched_fields.into_iter().map(str::to_owned).collect(), + reasons, + supporting_sources: passages + .into_iter() + .map(|(source, _)| source) + .chain(linked_sources.remove(&node.id).unwrap_or_default()) + .chain(relationships.sources.remove(&node.id).unwrap_or_default()) + .chain(ownership.sources.remove(&node.id).unwrap_or_default()) + .chain(body_sources) + .collect::>() + .into_iter() + .collect(), + }); + } + if !wants_documents { + let ranked_ids = ranked + .iter() + .map(|candidate| candidate.node.id.as_str()) + .collect::>(); + let represented_documents = owned_document_ids + .iter() + .filter(|(owner, _)| ranked_ids.contains(owner.as_str())) + .flat_map(|(_, documents)| documents.iter().cloned()) + .collect::>(); + if trace_target.is_some_and(|id| represented_documents.contains(id)) + && let Some(details) = &mut target_details + { + details.filter_reason = Some(RankedFilterReason::RepresentedOwnedDocument); + } + ranked.retain(|candidate| !represented_documents.contains(&candidate.node.id)); + } + ranked.sort_by(|a, b| { + b.score + .total_cmp(&a.score) + .then_with(|| a.node.id.cmp(&b.node.id)) + }); + let eligible = ranked.len(); + let ranks = ranked + .iter() + .enumerate() + .map(|(at, candidate)| (candidate.node.id.clone(), at + 1)) + .collect(); + let cap = request.limits.max_candidates.min(request.limits.max_nodes) as usize; + bounded |= eligible > cap; + ranked.truncate(cap); + let quotation_groups = groups + .iter() + .map(|group| group.1.clone()) + .collect::>(); + let quotation_actions = groups + .iter() + .filter(|group| action_terms.contains(&group.0)) + .map(|group| group.1.clone()) + .collect::>(); + let quotation_terms = quotation_groups + .iter() + .flat_map(|group| group.iter().cloned()) + .collect::>(); + // Quotation selection never feeds admission, score or candidate order. + // Only the displayed shortlist retains at most 1024 bytes per owner. + for candidate in ranked.iter_mut().take(if retain_excerpts { 5 } else { 0 }) { + check()?; + candidate.excerpts = owned_excerpts + .remove(&candidate.node.id) + .unwrap_or_default(); + let owner = self + .source_query + .as_ref() + .and_then(|index| index.owner(&candidate.node.id)); + if let Some(owner) = owner { + // The owner was already checked against the pinned exact-AST + // declaration during scoring. Never read current source here. + for comment in &owner.comments { + if let Some(excerpt) = excerpts::select( + RankedExcerptKind::CallableComment, + &comment.source, + &comment.text, + "ation_terms, + 512, + ) { + candidate.excerpts.push(excerpt); + } + } + if owner.omitted_code_windows > 0 { + candidate.reasons.push("Code inspection windows were omitted by extraction/publication quotas; retained text is not the complete implementation".to_owned()); + } + for window in &owner.code_windows { + if let Some(excerpt) = excerpts::select( + RankedExcerptKind::CallableCode, + &window.source, + &window.text, + "ation_terms, + 512, + ) { + candidate.excerpts.push(excerpt); + } + } + if owner.omitted_nested_contexts > 0 + || owner + .nested_contexts + .iter() + .any(|context| context.omitted_code_windows > 0) + { + candidate.reasons.insert(0,"Nested lexical code context was omitted by shared inspection limits; no execution is inferred".to_owned()); + } + for context in &owner.nested_contexts { + for window in &context.code_windows { + if let Some(mut excerpt) = excerpts::select( + RankedExcerptKind::NestedCallableCode, + &window.source, + &window.text, + "ation_terms, + 512, + ) { + excerpt.nested_scope = Some(context.source.clone()); + candidate.excerpts.push(excerpt); + } + } + } + } + candidate.excerpts.sort_by_cached_key(|quote| { + let text = excerpts::ranking_text( + quote, + owner.map_or(&[], |owner| owner.comments.as_slice()), + ); + ( + std::cmp::Reverse(excerpts::coverage(&text, "ation_actions)), + std::cmp::Reverse(excerpts::coverage(&text, "ation_groups)), + std::cmp::Reverse(matches!( + quote.kind, + RankedExcerptKind::CallableCode | RankedExcerptKind::NestedCallableCode + )), + quote.source.clone(), + ) + }); + candidate.excerpts.dedup(); + candidate.excerpts.truncate(2); + } + Ok(Retrieval { + ranked, + admitted, + bounded, + eligible, + reads, + probes, + missing_subjects, + negative_qualification, + temporal_qualification, + legacy, + relationship_legacy: relationships.legacy, + ranks, + recall_time, + ranking_time: ranking_started.elapsed(), + concept_matches, + witness_nodes: witness_nodes.into_values().collect(), + witness_edges: witness_edges.into_values().collect(), + target_details, + }) + } + + pub(crate) fn broad_search( + &self, + request: SearchRequest, + instrumentation: &mut crate::telemetry::QueryInstrumentation, + admit: &dyn Fn(&NodeRecord) -> bool, + ) -> Result { + let deadline = Instant::now() + Duration::from_secs(30); + let check = || { + self.check_deadline()?; + if Instant::now() >= deadline { + return Err(QueryError::new( + QueryErrorKind::Timeout, + "broad_query_timeout", + "Broad retrieval exceeded its timeout; narrow the question or scope", + )); + } + Ok(()) + }; + let retrieval = self.retrieve_broad(&request, &[], &check, admit, None, false)?; + instrumentation.recall += retrieval.recall_time; + instrumentation.ranking += retrieval.ranking_time; + instrumentation.work.candidates_read += retrieval.reads; + instrumentation.work.postings_decoded += retrieval.reads; + let mut response = CodeQueryResponse::empty(CodeQueryOperation::Search, request.limits); + response.truncated = retrieval.bounded; + response.nodes = retrieval + .ranked + .iter() + .map(|candidate| candidate.node.clone()) + .collect(); + response.results = retrieval + .ranked + .iter() + .map(|candidate| SearchHit { + node_id: candidate.node.id.clone(), + score: candidate.score, + matched_fields: candidate.matched_fields.clone(), + }) + .collect(); + response.diagnostics = self.broad_diagnostics(&retrieval); + let mut retained = response + .nodes + .iter() + .map(|node| node.id.clone()) + .collect::>(); + for node in &retrieval.witness_nodes { + if retained.len() < response.limits.max_nodes as usize { + if retained.insert(node.id.clone()) { + response.nodes.push(node.clone()); + } + } else if !retained.contains(&node.id) { + response.truncated = true; + } + } + let mut witness_edges = retrieval + .witness_edges + .into_iter() + .filter(|edge| retained.contains(&edge.source) && retained.contains(&edge.target)) + .collect::>(); + response.truncated |= witness_edges.len() > response.limits.max_edges as usize; + witness_edges.truncate(response.limits.max_edges as usize); + response.edges = witness_edges; + response.concept_matches = retrieval + .concept_matches + .into_iter() + .filter(|item| { + retained.contains(&item.node_id) && item.via.iter().all(|id| retained.contains(id)) + }) + .collect(); + response.sort_stable(); + crate::code_query::enforce_response_size(&mut response)?; + Ok(response) + } + + fn broad_diagnostics(&self, retrieval: &Retrieval) -> Vec { + let mut diagnostics = Vec::new(); + let mut push = |code, message| { + diagnostics.push(QueryDiagnostic { + code, + message, + node_id: None, + path: None, + }) + }; + // Put the actionable recovery before generic truncation. Bounded text + // projections may have room for only the first diagnostic. + if retrieval.legacy { + push(QueryDiagnosticCode::IncompleteCoverage, "This immutable index predates fielded evidence statistics. Run compass update on the current checkout for signature/prose recall; historical snapshots remain unchanged".to_owned()); + } + if retrieval.relationship_legacy { + push(QueryDiagnosticCode::IncompleteCoverage, "This immutable index predates broad relationship postings; method recall through relationships is limited to direct calls. Run compass update on the current checkout; historical snapshots remain unchanged".to_owned()); + } + if retrieval.negative_qualification { + push(QueryDiagnosticCode::IncompleteCoverage, "Negative qualification is not established by lexical retrieval. Inspect the candidate source or use a structural query to verify the requested condition".to_owned()); + } + if !retrieval.temporal_qualification.is_empty() { + push( + QueryDiagnosticCode::IncompleteCoverage, + format!( + "Temporal qualification ({}) is not established by lexical retrieval. Inspect the candidate source to verify the requested ordering or condition", + retrieval + .temporal_qualification + .iter() + .cloned() + .collect::>() + .join(", ") + ), + ); + } + if retrieval + .ranked + .iter() + .any(|candidate| candidate.relevance == RelevanceBand::Low) + { + let missing = if retrieval.missing_subjects.is_empty() { + "Some candidates lack owner-specific query context".to_owned() + } else { + format!( + "Missing indexed query context: {}", + retrieval.missing_subjects.join(", ") + ) + }; + push( + QueryDiagnosticCode::IncompleteCoverage, + format!( + "These are inspection candidates; the requested behavior is not established. {missing}. Use ranked view for per-candidate relevance and reasons" + ), + ); + } + if retrieval.ranked.is_empty() { + push( + QueryDiagnosticCode::NoMatch, + format!( + "No supporting indexed evidence found. Missing subject evidence: {}. Narrow the question or check indexed scope; absence is not proof the behavior does not exist.", + retrieval.missing_subjects.join(", ") + ), + ); + } + if retrieval.bounded { + push(QueryDiagnosticCode::BoundedTruncation, "Fielded retrieval reached a candidate, source-field, or output bound; available candidates are not a complete repository census".to_owned()); + } + if self.source_query.is_none() { + push(QueryDiagnosticCode::IncompleteCoverage, "This graph has no immutable callable-body index. Broad recall uses published names, signatures and graph evidence; run compass update on the current checkout for body identifiers/comments. Historical snapshots remain unchanged".to_owned()); + } else if let Some(message) = self + .source_query + .as_ref() + .and_then(|index| index.coverage_message.as_ref()) + { + push(QueryDiagnosticCode::IncompleteCoverage, message.clone()); + } + if let Some(message) = &self.partial_graph_message { + push(QueryDiagnosticCode::IncompleteCoverage, message.clone()); + } + diagnostics + } + + /// Ranked source evidence independent of the three-seed graph expansion. + pub fn discover_ranked( + &self, + request: DiscoveryQueryRequest, + ) -> Result { + crate::discovery::validate_request(&request)?; + let guard = + crate::discovery::DiscoveryGuard::new(request.limits.timeout_ms, None, self.deadline); + self.discover_ranked_guarded(request, &guard, true) + } + + fn discover_ranked_guarded( + &self, + request: DiscoveryQueryRequest, + guard: &crate::discovery::DiscoveryGuard<'_>, + enforce_ranked_budget: bool, + ) -> Result { + guard.check()?; + let fielded = self.is_fielded_question(&request.question)?; + guard.check()?; + if !fielded + || request.direction != DiscoveryDirection::Auto + || !request.relation_contexts.is_empty() + { + let exact_subject = request.question.trim().to_owned(); + let discovery = self.discover_guarded(request, guard)?; + let has_exact_id = discovery + .seeds + .iter() + .any(|seed| seed.node_id == exact_subject); + // A literal existing ID identifies the inspection target. Lexical + // matches for an ID prefix belong to neither its shortlist nor its + // structural evidence; retain raw discovery/context separately. + let candidates = discovery.seeds.iter().filter(|seed| !has_exact_id || seed.node_id == exact_subject).filter_map(|seed| discovery.nodes.iter().find(|node| node.id == seed.node_id).map(|node| RankedCandidate { + node: node.clone(), score: seed.score.parse().unwrap_or(0.0), relevance: RelevanceBand::Low, + matched_terms: seed.matched_terms.clone(), matched_fields: seed.matched_fields.clone(), reasons: vec!["Structural query seed; relevance is not inferred from graph confidence".to_owned()], + supporting_sources: Vec::new(), excerpts: Vec::new(), + })).collect(); + let response = RankedDiscoveryResponse { + eligible_count: None, + bounded: discovery.truncated, + discovery, + candidates, + }; + if enforce_ranked_budget { + enforce_ranked_response_budget(&response)?; + } + guard.check()?; + return Ok(response); + } + let search = SearchRequest { + query: request.question.clone(), + limits: CodeQueryLimits { + max_candidates: request.limits.max_candidates, + max_nodes: request.limits.max_nodes, + max_response_bytes: request.limits.max_response_bytes, + ..CodeQueryLimits::default() + }, + }; + let scopes = self.ranked_scopes(&request, guard)?; + let retrieval = + self.retrieve_broad(&search, &scopes, &|| guard.check(), &|_| true, None, true)?; + guard.check()?; + let seeds = retrieval + .ranked + .iter() + .take(request.limits.max_seeds as usize) + .enumerate() + .map(|(rank, candidate)| DiscoverySeed { + node_id: candidate.node.id.clone(), + score: format!("{:.6}", candidate.score), + score_tier: DiscoveryScoreTier::Lexical, + rank: rank as u32 + 1, + matched_terms: candidate.matched_terms.clone(), + matched_fields: candidate.matched_fields.clone(), + source: candidate.node.source.clone(), + candidate_source: DiscoverySeedSource::TermIndex, + alternatives: Vec::new(), + ambiguous: false, + }) + .collect(); + let (selected_direction, direction_source) = + crate::discovery::infer_discovery_direction(&request.question); + let discovery = DiscoveryQueryResponse { + schema: DISCOVERY_QUERY_SCHEMA_V1.to_owned(), + question: request.question, + selected_direction, + direction_source, + relation_contexts: request.relation_contexts, + scope: scopes, + traversal: request.traversal, + seeds, + nodes: retrieval + .ranked + .iter() + .map(|candidate| candidate.node.clone()) + .collect(), + edges: Vec::new(), + diagnostics: self.broad_diagnostics(&retrieval), + limits: request.limits, + stats: DiscoveryStats { + candidate_probes: retrieval.probes, + candidate_nodes: retrieval.reads, + candidates_admitted: retrieval.ranked.len() as u64, + returned_nodes: retrieval.ranked.len() as u64, + ..DiscoveryStats::default() + }, + omissions: DiscoveryOmissions::default(), + truncated: retrieval.bounded, + }; + let response = RankedDiscoveryResponse { + discovery, + eligible_count: (!retrieval.bounded).then_some(retrieval.eligible as u64), + bounded: retrieval.bounded, + candidates: retrieval.ranked, + }; + guard.check()?; + if enforce_ranked_budget { + enforce_ranked_response_budget(&response)?; + } + guard.check()?; + Ok(response) + } + + /// Optional witnessed one-hop context. Candidate rank is unaffected by + /// adjacency degree and no context node becomes an implementation answer. + pub fn discover_ranked_with_context( + &self, + request: DiscoveryQueryRequest, + include_context: bool, + ) -> Result { + crate::discovery::validate_request(&request)?; + let guard = + crate::discovery::DiscoveryGuard::new(request.limits.timeout_ms, None, self.deadline); + self.discover_ranked_with_context_guarded(request, include_context, &guard, true) + } + + pub(crate) fn discover_ranked_with_context_guarded( + &self, + request: DiscoveryQueryRequest, + include_context: bool, + guard: &crate::discovery::DiscoveryGuard<'_>, + enforce_ranked_budget: bool, + ) -> Result { + guard.check()?; + let mut response = + self.discover_ranked_guarded(request.clone(), guard, enforce_ranked_budget)?; + guard.check()?; + if !include_context { + return Ok(response); + } + let backend = self.backend.pin_discovery()?; + let context_roots = if self.is_fielded_question(&request.question)? + && request.direction == DiscoveryDirection::Auto + && request.relation_contexts.is_empty() + { + response + .candidates + .iter() + .take(5) + .map(|candidate| candidate.node.clone()) + .collect::>() + } else { + response.discovery.nodes.clone() + }; + let mut nodes = context_roots + .into_iter() + .map(|node| (node.id.clone(), node)) + .collect::>(); + let mut edges = response + .discovery + .edges + .iter() + .cloned() + .enumerate() + .map(|(at, edge)| { + ( + edge.id.clone().unwrap_or_else(|| format!("anonymous:{at}")), + edge, + ) + }) + .collect::>(); + let per_seed = + (request.limits.max_edges as usize / response.discovery.seeds.len().max(1)).max(1); + let mut bounded = false; + let mut expanded = 0_u64; + for seed in &response.discovery.seeds { + guard.check()?; + let remaining_work = request + .limits + .max_expanded_relationships + .saturating_sub(expanded); + if remaining_work == 0 { + bounded = true; + break; + } + let (incident, limited) = backend.incident_bounded( + &seed.node_id, + request.include_heuristic, + per_seed.min(remaining_work as usize), + )?; + expanded = expanded.saturating_add(incident.len() as u64); + bounded |= limited; + for edge in incident { + guard.check()?; + if !crate::discovery::edge_matches_context( + &edge, + &response.discovery.relation_contexts, + ) { + continue; + } + if (response.discovery.selected_direction == DiscoveryDirection::Incoming + && edge.target != seed.node_id) + || (response.discovery.selected_direction == DiscoveryDirection::Outgoing + && edge.source != seed.node_id) + { + continue; + } + if edges.len() >= request.limits.max_edges as usize { + bounded = true; + break; + } + let neighbor = if edge.source == seed.node_id { + &edge.target + } else { + &edge.source + }; + if !nodes.contains_key(neighbor) { + if nodes.len() >= request.limits.max_nodes as usize { + bounded = true; + continue; + } + let Some(node) = backend.node_by_id(neighbor)? else { + continue; + }; + if !discovery_scope_matches(&node, &response.discovery.scope) { + continue; + } + nodes.insert(node.id.clone(), query_node(&node)); + } + edges.insert(edge.id.clone(), crate::discovery::discovery_edge(&edge)); + } + } + response.discovery.nodes = nodes.into_values().collect(); + response.discovery.edges = edges.into_values().collect(); + response.discovery.stats.returned_nodes = response.discovery.nodes.len() as u64; + response.discovery.stats.returned_edges = response.discovery.edges.len() as u64; + response.discovery.stats.expanded_relationships = expanded; + response.discovery.stats.visited_nodes = response.discovery.nodes.len() as u64; + response.discovery.truncated |= bounded; + if bounded { + response.discovery.diagnostics.push(QueryDiagnostic { code: QueryDiagnosticCode::BoundedTruncation, message: "One-hop ranked context reached a node/edge bound; shortlist ranking is unchanged".to_owned(), node_id: None, path: None }); + } + guard.check()?; + if enforce_ranked_budget { + enforce_ranked_response_budget(&response)?; + } + guard.check()?; + Ok(response) + } + + pub fn trace_ranked( + &self, + request: DiscoveryQueryRequest, + expected_id: &str, + ) -> Result { + crate::discovery::validate_request(&request)?; + let guard = + crate::discovery::DiscoveryGuard::new(request.limits.timeout_ms, None, self.deadline); + guard.check()?; + let graph_present = self.backend.node_by_id(expected_id)?.is_some(); + let scopes = self.ranked_scopes(&request, &guard)?; + let retrieval = self.retrieve_broad( + &SearchRequest { + query: request.question, + limits: CodeQueryLimits { + max_candidates: request.limits.max_candidates, + max_nodes: request.limits.max_nodes, + ..CodeQueryLimits::default() + }, + }, + &scopes, + &|| guard.check(), + &|_| true, + Some(expected_id), + false, + )?; + guard.check()?; + let rank = retrieval.ranks.get(expected_id).copied(); + Ok(RankedRetrievalTrace { + schema: "compass.query.retrieval-trace/1".to_owned(), + graph_present, + admitted: retrieval.admitted.contains(expected_id), + rank, + seed: rank.is_some_and(|rank| rank <= request.limits.max_seeds as usize), + first_page: rank.is_some_and(|rank| rank <= 5), + bounded: retrieval.bounded, + target_details: retrieval.target_details, + }) + } +} diff --git a/crates/compass-query/src/broad/compounds.rs b/crates/compass-query/src/broad/compounds.rs new file mode 100644 index 000000000..8dc7a65a4 --- /dev/null +++ b/crates/compass-query/src/broad/compounds.rs @@ -0,0 +1,208 @@ +//! Ranking-only lexical cohesion; no recall postings or behavioral inference. +use std::collections::BTreeSet; + +use compass_model::provenance::SourceAnchor; +use compass_model::source_query::SourceQuerySite; + +#[derive(Default)] +pub(super) struct Compounds { + phrases: BTreeSet>, + components: BTreeSet, +} + +#[derive(Default)] +pub(super) struct Matches { + pub phrases: BTreeSet, + pub sources: BTreeSet, +} + +fn words(value: &str) -> Vec { + compass_model::identifier_tokens(&value.replace(['_', '-'], " ")) + .into_iter() + .map(compass_model::canonical_code_token) + .collect() +} + +fn together(haystack: &[String], needle: &[String]) -> bool { + !needle.is_empty() && haystack.windows(needle.len()).any(|part| part == needle) +} + +pub(super) fn inspection_signature(signature: Option<&str>) -> Option<&str> { + signature.filter(|text| !text.trim().is_empty() && !text.trim_end().ends_with('…')) +} + +impl Compounds { + pub fn parse(question: &str) -> Self { + let mut result = Self::default(); + for raw in question.split_whitespace() { + let raw = raw.trim_matches(|c: char| !c.is_alphanumeric() && c != '-'); + if !raw.contains('-') + || raw + .split('-') + .any(|part| part.is_empty() || !part.chars().all(char::is_alphanumeric)) + { + continue; + } + let phrase = words(raw); + if !(2..=4).contains(&phrase.len()) { + continue; + } + result.components.extend(phrase.iter().cloned()); + result.phrases.insert(phrase); + // Keep ordinary ranking for oversized compound requests rather + // than selecting a wording/order-dependent subset. Recall is unchanged. + if result.phrases.len() > 8 { + return Self::default(); + } + } + result + } + + pub fn contains_component_group(&self, term: &str) -> bool { + let terms = words(term); + !terms.is_empty() && terms.iter().all(|term| self.components.contains(term)) + } + + /// The caller validates owner identity/provenance and excludes clipped + /// projections. Parameter/type identifiers in retained signatures cannot + /// supply a code signal. A declaration's own name remains a direct signal. + pub fn matches( + &self, + name: &str, + signature: Option<&str>, + identifiers: &[SourceQuerySite], + ) -> Matches { + let mut result = Matches::default(); + if self.phrases.is_empty() { + return result; + } + let name = words(name); + let signature = inspection_signature(signature); + let header = signature + .into_iter() + .flat_map(|text| text.split(|c: char| !c.is_alphanumeric() && c != '_')) + .filter(|word| !word.is_empty()) + .map(words) + .collect::>(); + let sites = identifiers + .iter() + .filter(|site| { + signature.is_some() + && !site.text.is_empty() + && site.text.chars().all(|c| c.is_alphanumeric() || c == '_') + }) + .map(|site| (words(&site.text), &site.source)) + .collect::>(); + for phrase in &self.phrases { + let label = phrase.join("-"); + if together(&name, phrase) { + result.phrases.insert(label); + } else if !header.iter().any(|word| together(word, phrase)) + && let Some((_, source)) = sites.iter().find(|(tokens, _)| together(tokens, phrase)) + { + result.phrases.insert(label); + result.sources.insert((*source).clone()); + } + } + result + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn site(text: &str) -> SourceQuerySite { + SourceQuerySite { + source: SourceAnchor { + file: "sample.py".into(), + start_byte: 10, + end_byte: 10 + text.len() as u64, + start_line: 2, + end_line: 2, + start_column: 0, + end_column: text.len() as u32, + }, + text: text.into(), + } + } + + #[test] + fn compound_words_require_contiguous_identifier_terms() { + let request = Compounds::parse("Where are read-only fields prepared?"); + for name in ["read_only", "readOnly", "isReadOnlyField"] { + let item = site(name); + let found = request.matches( + "opaque", + Some("fn(fields: Fields)"), + std::slice::from_ref(&item), + ); + assert_eq!(found.phrases, BTreeSet::from(["read-only".into()])); + assert_eq!(found.sources, BTreeSet::from([item.source])); + } + for items in [ + vec![site("read"), site("only")], + vec![site("only_read")], + vec![site("already_only")], + vec![site("read only")], + vec![site("cfg.readOnly")], + ] { + assert!( + request + .matches("opaque", Some("fn(fields: Fields)"), &items) + .phrases + .is_empty() + ); + } + assert!(request.contains_component_group("read")); + assert!(request.contains_component_group("read only")); + assert!(!request.contains_component_group("field")); + } + + #[test] + fn retained_header_overlap_and_missing_signatures_do_not_supply_code_preference() { + let request = Compounds::parse("Where are read-only fields prepared?"); + let items = [site("read_only")]; + for signature in [ + None, + Some(""), + Some("fn(fields: Fields, …"), + Some("fn(fields: Fields, read_only: Mode)"), + Some("fn(fields: ReadOnlyFields)"), + ] { + assert!( + request + .matches("opaque", signature, &items) + .phrases + .is_empty() + ); + } + assert!( + !request + .matches( + "opaque", + Some("fn(fields: Fields, read: X, only: Y)"), + &items + ) + .phrases + .is_empty() + ); + assert!( + !request + .matches("prepare_read_only_fields", None, &[]) + .phrases + .is_empty() + ); + } + + #[test] + fn compound_limits_retain_ordinary_ranking_without_an_arbitrary_subset() { + assert!(Compounds::parse("plain words").phrases.is_empty()); + assert!(Compounds::parse("a-b-c-d-e").phrases.is_empty()); + let question = (0..9) + .map(|i| format!("mode{i}-only")) + .collect::>() + .join(" "); + assert!(Compounds::parse(&question).phrases.is_empty()); + } +} diff --git a/crates/compass-query/src/broad/excerpts.rs b/crates/compass-query/src/broad/excerpts.rs new file mode 100644 index 000000000..c324e510a --- /dev/null +++ b/crates/compass-query/src/broad/excerpts.rs @@ -0,0 +1,407 @@ +//! Exact retained source quotations. Callers must establish ownership first. +use compass_model::provenance::SourceAnchor; +use compass_model::source_query::SourceQuerySite; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeSet; + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RankedExcerptKind { + CallableComment, + CallableCode, + NestedCallableCode, + AstDocumentation, +} + +/// A quotation is inspection evidence, not a behavioral assertion. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct RankedSourceExcerpt { + pub kind: RankedExcerptKind, + pub source: SourceAnchor, + pub text: String, + pub clipped: bool, + /// Full lexical anonymous-callable scope, never an execution assertion. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub nested_scope: Option, +} + +impl RankedSourceExcerpt { + /// Check bounded quote/anchor coherence. Ownership is established by the + /// query producer against its pinned graph, not by this shape check. + #[must_use] + pub fn is_coherent(&self) -> bool { + self.text.len() <= 512 + && self.source.file.len() <= 4096 + && exact_shape(&self.source, &self.text) + && match self.kind { + RankedExcerptKind::NestedCallableCode => { + self.nested_scope.as_ref().is_some_and(|scope| { + scope.is_valid() + && scope.file.len() <= 4096 + && scope.file == self.source.file + && scope.start_byte <= self.source.start_byte + && self.source.end_byte <= scope.end_byte + && (scope.start_line, scope.start_column) + <= (self.source.start_line, self.source.start_column) + && (self.source.end_line, self.source.end_column) + <= (scope.end_line, scope.end_column) + }) + } + _ => self.nested_scope.is_none(), + } + } +} + +fn advance(mut point: (u32, u32), text: &str) -> Option<(u32, u32)> { + for byte in text.bytes() { + if byte == b'\n' { + point.0 = point.0.checked_add(1)?; + point.1 = 0; + } else { + point.1 = point.1.checked_add(1)?; + } + } + Some(point) +} + +/// Check anchor/text coherence, not source-byte truth. The caller must establish +/// raw retention and ownership through the versioned producer contract. +pub(crate) fn exact_shape(source: &SourceAnchor, text: &str) -> bool { + source.is_valid() + && !text.is_empty() + && source.end_byte.checked_sub(source.start_byte) == Some(text.len() as u64) + && advance((source.start_line, source.start_column), text) + == Some((source.end_line, source.end_column)) +} + +fn first_match(text: &str, terms: &BTreeSet) -> Option { + let mut start = None; + for (at, character) in text + .char_indices() + .chain(std::iter::once((text.len(), ' '))) + { + if character.is_alphanumeric() || character == '_' { + start.get_or_insert(at); + } else if let Some(begin) = start.take() + && compass_model::search::identifier_search_terms(&text[begin..at]) + .into_iter() + .map(compass_model::canonical_code_token) + .any(|term| terms.contains(&term)) + { + return Some(begin); + } + } + None +} + +/// Choose an exact, UTF-8-safe window around the first matching identifier term. +/// No match means no quotation. Source coordinates describe retained bytes only. +pub(crate) fn select( + kind: RankedExcerptKind, + source: &SourceAnchor, + text: &str, + terms: &BTreeSet, + max_bytes: usize, +) -> Option { + if !exact_shape(source, text) || max_bytes == 0 || text.len() > 4096 { + return None; + } + let max_bytes = max_bytes.min(512); + let matched = first_match(text, terms)?; + let mut start = matched.saturating_sub(max_bytes / 3); + while !text.is_char_boundary(start) { + start = start.checked_sub(1)?; + } + let mut end = start.saturating_add(max_bytes).min(text.len()); + while !text.is_char_boundary(end) { + end = end.checked_sub(1)?; + } + if start >= end || matched >= end { + return None; + } + let retained = &text[start..end]; + // A matched identifier subword may lie outside this clipped window. + // Retained bytes must themselves support the quotation's relevance. + first_match(retained, terms)?; + let begin = advance((source.start_line, source.start_column), &text[..start])?; + let finish = advance(begin, retained)?; + Some(RankedSourceExcerpt { + kind, + source: SourceAnchor { + file: source.file.clone(), + start_byte: source.start_byte.checked_add(start as u64)?, + end_byte: source.start_byte.checked_add(end as u64)?, + start_line: begin.0, + start_column: begin.1, + end_line: finish.0, + end_column: finish.1, + }, + text: retained.to_owned(), + clipped: start > 0 || end < text.len(), + nested_scope: None, + }) +} + +/// Quotation selection uses the whole question, including action terms. This +/// affects displayed evidence only, never retrieval admission or node scores. +pub(crate) fn coverage(text: &str, groups: &[Vec]) -> usize { + let terms = compass_model::search::identifier_search_terms(text) + .into_iter() + .map(compass_model::canonical_code_token) + .collect::>(); + groups + .iter() + .filter(|group| group.iter().any(|term| terms.contains(term))) + .count() +} + +/// A retained code window can contain comments. Do not credit their terms as +/// code evidence when choosing between the exact comment and its enclosing +/// window. This scoring copy never replaces the displayed source quotation. +pub(crate) fn ranking_text(excerpt: &RankedSourceExcerpt, comments: &[SourceQuerySite]) -> String { + if !matches!( + excerpt.kind, + RankedExcerptKind::CallableCode | RankedExcerptKind::NestedCallableCode + ) { + return excerpt.text.clone(); + } + let mut text = String::with_capacity(excerpt.text.len()); + let mut cursor = 0; + for comment in comments { + if comment.source.file != excerpt.source.file + || comment.source.end_byte <= excerpt.source.start_byte + || comment.source.start_byte >= excerpt.source.end_byte + { + continue; + } + let start = comment + .source + .start_byte + .saturating_sub(excerpt.source.start_byte) + .min(excerpt.text.len() as u64) as usize; + let end = comment + .source + .end_byte + .saturating_sub(excerpt.source.start_byte) + .min(excerpt.text.len() as u64) as usize; + if start > cursor { + let Some(retained) = excerpt.text.get(cursor..start) else { + return String::new(); + }; + text.push_str(retained); + } + // Separate identifiers on either side of a removed comment. + text.push(' '); + cursor = cursor.max(end); + } + if let Some(retained) = excerpt.text.get(cursor..) { + text.push_str(retained); + } + text +} + +#[cfg(test)] +mod tests { + use super::*; + + fn anchor(text: &str) -> Option { + let finish = advance((3, 4), text)?; + Some(SourceAnchor { + file: "src/example.rs".to_owned(), + start_byte: 20, + end_byte: 20 + text.len() as u64, + start_line: 3, + start_column: 4, + end_line: finish.0, + end_column: finish.1, + }) + } + + #[test] + fn quoted_window_is_the_original_utf8_slice_with_exact_coordinates() + -> Result<(), Box> { + let text = format!( + "// {}\r\n// HandlePayload guards the error.\n// tail", + "é".repeat(400) + ); + let source = anchor(&text).ok_or("source")?; + let terms = BTreeSet::from(["payload".to_owned()]); + let quote = select( + RankedExcerptKind::CallableComment, + &source, + &text, + &terms, + 100, + ) + .ok_or("quote")?; + assert!(quote.clipped); + assert!(quote.text.contains("HandlePayload")); + let start = usize::try_from(quote.source.start_byte - source.start_byte)?; + let end = usize::try_from(quote.source.end_byte - source.start_byte)?; + assert_eq!(quote.text, text[start..end]); + assert_eq!(quote.source.start_line, 3); + assert_eq!(quote.source.end_line, 5); + assert!(exact_shape("e.source, "e.text)); + Ok(()) + } + + #[test] + fn normalized_or_unmatched_text_is_not_reported_as_a_source_quote() + -> Result<(), Box> { + let text = "// Handle payload."; + let source = anchor(text).ok_or("source")?; + let terms = BTreeSet::from(["payload".to_owned()]); + assert!( + select( + RankedExcerptKind::AstDocumentation, + &source, + "Handle payload.", + &terms, + 100 + ) + .is_none() + ); + let mut wrong = source.clone(); + wrong.end_column += 1; + assert!( + select( + RankedExcerptKind::AstDocumentation, + &wrong, + text, + &terms, + 100 + ) + .is_none() + ); + assert!( + select( + RankedExcerptKind::CallableComment, + &source, + text, + &BTreeSet::from(["cookie".to_owned()]), + 100 + ) + .is_none() + ); + assert!(select(RankedExcerptKind::CallableComment, &source, text, &terms, 0).is_none()); + Ok(()) + } + + #[test] + fn clipping_never_keeps_only_an_unmatched_identifier_prefix() + -> Result<(), Box> { + let text = format!("// {}Payload.", "x".repeat(800)); + let source = anchor(&text).ok_or("source")?; + let terms = BTreeSet::from(["payload".to_owned()]); + // The full identifier is a lexical match, but its displayed prefix is + // not. Never quote unrelated bytes as matching source evidence. + assert!(first_match(&text, &terms).is_some()); + assert!( + select( + RankedExcerptKind::CallableComment, + &source, + &text, + &terms, + 100 + ) + .is_none() + ); + Ok(()) + } + + #[test] + fn quote_relevance_counts_concept_groups_once_and_retains_action_evidence() { + let subject = ["diagnostic", "diagnostics"] + .into_iter() + .map(str::to_owned) + .map(compass_model::canonical_code_token) + .collect(); + let action = ["format", "formatted"] + .into_iter() + .map(str::to_owned) + .map(compass_model::canonical_code_token) + .collect(); + let groups = vec![subject, action]; + let parameter = "diagnostics: &Diagnostic"; + let implementation = "format!(\"{}\", diagnostic.message())"; + assert_eq!(super::coverage(parameter, &groups), 1); + assert_eq!(super::coverage(implementation, &groups), 2); + assert_eq!(super::coverage(parameter, &groups[1..]), 0); + assert_eq!(super::coverage(implementation, &groups[1..]), 1); + } + + #[test] + fn enclosing_code_does_not_borrow_comment_terms_for_quote_priority() + -> Result<(), Box> { + let text = "value = value\n # Handle payload normalization here.\n return value"; + let source = anchor(text).ok_or("source")?; + let comment_text = "# Handle payload normalization here."; + let terms = BTreeSet::from(["handle".to_owned(), "payload".to_owned()]); + let code = + select(RankedExcerptKind::CallableCode, &source, text, &terms, 512).ok_or("code")?; + let begin = text.find('#').ok_or("comment start")?; + let point = advance((source.start_line, source.start_column), &text[..begin]) + .ok_or("comment point")?; + let comment_source = SourceAnchor { + file: source.file.clone(), + start_byte: source.start_byte + begin as u64, + end_byte: source.start_byte + (begin + comment_text.len()) as u64, + start_line: point.0, + start_column: point.1, + end_line: point.0, + end_column: point.1 + comment_text.len() as u32, + }; + let comment = SourceQuerySite { + source: comment_source, + text: comment_text.to_owned(), + }; + let groups = vec![vec!["handle".to_owned()], vec!["payload".to_owned()]]; + assert_eq!(coverage(&code.text, &groups), 2); + assert_eq!( + coverage( + &ranking_text(&code, std::slice::from_ref(&comment)), + &groups + ), + 0 + ); + assert_eq!(code.text, text); + let quote = select( + RankedExcerptKind::CallableComment, + &comment.source, + &comment.text, + &terms, + 512, + ) + .ok_or("comment quote")?; + assert_eq!(coverage(&ranking_text("e, &[comment]), &groups), 2); + Ok(()) + } + + #[test] + fn comment_mask_preserves_code_terms_and_separates_adjacent_identifiers() + -> Result<(), Box> { + let text = "handle/* payload */payload"; + let source = anchor(text).ok_or("source")?; + let mut comment_source = source.clone(); + comment_source.start_byte += 6; + comment_source.start_column += 6; + comment_source.end_byte -= 7; + comment_source.end_column -= 7; + let comments = [SourceQuerySite { + source: comment_source, + text: "/* payload */".to_owned(), + }]; + let quote = RankedSourceExcerpt { + kind: RankedExcerptKind::CallableCode, + source, + text: text.to_owned(), + clipped: false, + nested_scope: None, + }; + let groups = vec![vec!["handle".to_owned()], vec!["payload".to_owned()]]; + assert_eq!(ranking_text("e, &comments), "handle payload"); + assert_eq!(coverage(&ranking_text("e, &comments), &groups), 2); + Ok(()) + } +} diff --git a/crates/compass-query/src/broad/ownership.rs b/crates/compass-query/src/broad/ownership.rs new file mode 100644 index 000000000..436d82d5d --- /dev/null +++ b/crates/compass-query/src/broad/ownership.rs @@ -0,0 +1,165 @@ +//! Local declaration names are inspection evidence for their unique callable +//! owner. Containment does not establish behavior or resolve a reference. +use super::*; +use compass_model::code_graph::EdgeRecord; + +const MAX_CHILDREN: usize = 32; +const MAX_PARENTS: usize = 4; +pub(super) const READ_RESERVE: u64 = (MAX_CHILDREN * (MAX_PARENTS * 2 + 1)) as u64; +pub(super) const PROBE_RESERVE: u64 = MAX_CHILDREN as u64; + +pub(super) fn is_local(node: &NodeRecord) -> bool { + matches!( + node.kind, + NodeKind::Parameter + | NodeKind::Variable + | NodeKind::Field + | NodeKind::Property + | NodeKind::Constant + ) +} + +#[derive(Default)] +pub(super) struct OwnershipRecall { + pub nodes: BTreeMap, + pub terms: BTreeMap>, + pub sources: BTreeMap>, + pub witnesses: BTreeMap, + pub children: BTreeMap, + pub reads: u64, + pub probes: u64, + pub bounded: bool, +} + +pub(super) fn recall( + backend: &PinnedDiscoveryBackend<'_>, + candidates: &BTreeMap, + groups: &[(String, Vec, f64, u64)], + scopes: &[DiscoveryScope], + check: &dyn Fn() -> Result<(), QueryError>, + admit: &dyn Fn(&NodeRecord) -> bool, +) -> Result { + let mut result = OwnershipRecall::default(); + let mut children = Vec::new(); + for child in candidates.values() { + check()?; + if !is_local(child) || !relationships::exact_ast(child) { + continue; + } + let document = search_document(child); + result.bounded |= document.truncated; + let terms = document + .fields + .get("name") + .cloned() + .unwrap_or_default() + .into_iter() + .filter(|term| { + groups + .iter() + .any(|(_, variants, _, _)| variants.contains(term)) + }) + .collect::>(); + if terms.is_empty() { + continue; + } + let weight = groups + .iter() + .filter(|(_, variants, _, _)| variants.iter().any(|term| terms.contains(term))) + .map(|(_, _, idf, _)| idf) + .sum::(); + children.push((child, terms, weight)); + } + children.sort_by(|a, b| b.2.total_cmp(&a.2).then_with(|| a.0.id.cmp(&b.0.id))); + result.bounded |= children.len() > MAX_CHILDREN; + for (child, terms, _) in children.into_iter().take(MAX_CHILDREN) { + check()?; + result.probes += 1; + let (edges, truncated, examined) = backend.matching_bounded_counted( + &child.id, + true, + &[EdgeKind::Contains], + true, + MAX_PARENTS, + )?; + result.reads += examined as u64; + // A hidden parent can invalidate uniqueness. Never choose from a + // truncated prefix, even when every visible parent agrees. + if truncated { + result.bounded = true; + continue; + } + let mut owners = BTreeMap::)>::new(); + for edge in edges { + check()?; + if edge.target != child.id + || !compass_model::search::is_exact_nonheuristic_relationship(&edge) + || edge.evidence.iter().any(|proof| proof.validate().is_err()) + { + continue; + } + let (Some(child_source), Some(site)) = (&child.source, &edge.relationship_site) else { + continue; + }; + if !relationships::encloses(child_source, site) + || !edge.evidence.iter().any(|proof| { + proof.origin == EvidenceOrigin::Ast && proof.anchors.contains(site) + }) + { + continue; + } + result.reads += 1; + let Some(owner) = backend.node_by_id(&edge.source)? else { + continue; + }; + if !owner.kind.is_callable() + || !relationships::exact_ast(&owner) + || !owner + .source + .as_ref() + .is_some_and(|source| relationships::encloses(source, child_source)) + { + continue; + } + owners + .entry(owner.id.clone()) + .or_insert_with(|| (owner, Vec::new())) + .1 + .push(edge); + } + // Scope is applied after uniqueness: excluding one ambiguous parent + // must never manufacture unique ownership of the other. + if owners.len() != 1 { + continue; + } + let Some((owner, edges)) = owners.into_values().next() else { + continue; + }; + if !discovery_scope_matches(&owner, scopes) || !admit(&owner) { + continue; + } + let retained = result.terms.entry(owner.id.clone()).or_default(); + retained.extend(terms); + if retained.len() > compass_model::search::SEARCH_FIELD_MAX_TERMS { + result.bounded = true; + *retained = retained + .iter() + .take(compass_model::search::SEARCH_FIELD_MAX_TERMS) + .cloned() + .collect(); + } + for edge in edges { + if let Some(site) = &edge.relationship_site { + result + .sources + .entry(owner.id.clone()) + .or_default() + .insert(site.clone()); + } + result.witnesses.insert(edge.id.clone(), edge); + } + result.children.insert(child.id.clone(), child.clone()); + result.nodes.insert(owner.id.clone(), owner); + } + Ok(result) +} diff --git a/crates/compass-query/src/broad/relationships.rs b/crates/compass-query/src/broad/relationships.rs new file mode 100644 index 000000000..9c922f2a5 --- /dev/null +++ b/crates/compass-query/src/broad/relationships.rs @@ -0,0 +1,212 @@ +//! Relationship postings recall identifiers, not behavioral assertions. +//! Only a witnessed outgoing call/reference inside a callable contributes its +//! target's terminal name. Namespace, path and neighboring prose are excluded. +use super::*; +use compass_model::code_graph::EdgeRecord; + +const MAX_POSTING_PROBES: usize = 8; +const MAX_OWNERS: usize = 64; +const MAX_OUTGOING: usize = 32; +const MAX_READS: u64 = 6400; +const MAX_PROBES: u64 = 96; + +#[derive(Default)] +pub(super) struct RelationshipRecall { + pub nodes: BTreeMap, + pub terms: BTreeMap>, + pub sources: BTreeMap>, + pub witnesses: BTreeMap, + pub targets: BTreeMap, + pub reads: u64, + pub probes: u64, + pub bounded: bool, + pub legacy: bool, +} + +pub(super) fn encloses(owner: &SourceAnchor, site: &SourceAnchor) -> bool { + owner.is_valid() + && site.is_valid() + && site.start_byte < site.end_byte + && owner.file == site.file + && owner.start_byte <= site.start_byte + && site.end_byte <= owner.end_byte + && (owner.start_line, owner.start_column) <= (site.start_line, site.start_column) + && (site.end_line, site.end_column) <= (owner.end_line, owner.end_column) +} + +pub(super) fn exact_ast(node: &NodeRecord) -> bool { + let Some(source) = &node.source else { + return false; + }; + !node.evidence.is_empty() + && node.evidence.iter().all(|proof| { + proof.origin != EvidenceOrigin::Heuristic + && proof.confidence == EvidenceConfidence::Exact + && proof.validate().is_ok() + }) + && node.evidence.iter().any(|proof| { + proof.origin == EvidenceOrigin::Ast + && proof.anchors.iter().any(|anchor| encloses(source, anchor)) + }) +} + +fn verified_site<'a>(owner: &NodeRecord, edge: &'a EdgeRecord) -> Option<&'a SourceAnchor> { + let (Some(source), Some(site)) = (&owner.source, &edge.relationship_site) else { + return None; + }; + (edge.source == owner.id + && matches!(edge.kind, EdgeKind::Calls | EdgeKind::References) + && encloses(source, site) + && !edge.evidence.is_empty() + && edge.evidence.iter().all(|proof| { + proof.origin != EvidenceOrigin::Heuristic + && proof.confidence == EvidenceConfidence::Exact + && proof.validate().is_ok() + }) + && edge + .evidence + .iter() + .any(|proof| proof.origin == EvidenceOrigin::Ast && proof.anchors.contains(site))) + .then_some(site) +} + +pub(super) fn recall( + backend: &PinnedDiscoveryBackend<'_>, + groups: &[(String, Vec, f64, u64)], + scopes: &[DiscoveryScope], + check: &dyn Fn() -> Result<(), QueryError>, + admit: &dyn Fn(&NodeRecord) -> bool, +) -> Result { + let mut result = RelationshipRecall::default(); + if let PinnedDiscoveryBackend::Store(reader) = backend { + result.probes += 1; + result.legacy = !reader + .supports_relationship_search_terms() + .map_err(crate::code_query::snapshot_error)?; + result.bounded |= result.legacy; + } + let mut sources = BTreeMap::::new(); + // One variant per concept before alternative spellings; a single expanded + // concept must not consume every independent recall quota. + let mut queries = Vec::new(); + for offset in 0..groups.iter().map(|group| group.1.len()).max().unwrap_or(0) { + for (_, variants, idf, _) in groups { + if let Some(variant) = variants.get(offset) { + queries.push((variant, *idf)); + } + } + } + result.bounded |= queries.len() > MAX_POSTING_PROBES; + for (variant, idf) in queries.into_iter().take(MAX_POSTING_PROBES) { + check()?; + if result.reads.saturating_add(POSTING_QUOTA as u64 * 4) > MAX_READS + || result.probes.saturating_add(8) > MAX_PROBES + { + result.bounded = true; + break; + } + // The older discovery posting intentionally contains direct calls + // only. Use the separately versioned broad relationship posting. + let read = backend.relationship_sources_for_term(variant, POSTING_QUOTA)?; + result.reads = result.reads.saturating_add(read.node_ids_decoded); + // Store reads also check the capability marker before choosing their + // broad or legacy posting. Count that logical lookup, too. + result.probes = result.probes.saturating_add( + 1 + read.chunks_decoded + + u64::from(matches!(backend, PinnedDiscoveryBackend::Store(_))), + ); + result.bounded |= read.truncated; + for id in read.source_ids { + *sources.entry(id).or_default() += idf; + } + } + let mut sources = sources.into_iter().collect::>(); + sources.sort_by(|a, b| b.1.total_cmp(&a.1).then_with(|| a.0.cmp(&b.0))); + result.bounded |= sources.len() > MAX_OWNERS; + for (id, _) in sources.into_iter().take(MAX_OWNERS) { + check()?; + if result.reads.saturating_add((MAX_OUTGOING * 2 + 2) as u64) > MAX_READS + || result.probes >= MAX_PROBES + { + result.bounded = true; + break; + } + result.reads += 1; + let Some(owner) = backend.node_by_id(&id)? else { + continue; + }; + if !owner.kind.is_callable() + || !exact_ast(&owner) + || !discovery_scope_matches(&owner, scopes) + || !admit(&owner) + { + continue; + } + result.probes += 1; + let (edges, truncated, examined) = backend.matching_bounded_counted( + &id, + false, + &[EdgeKind::Calls, EdgeKind::References], + true, + MAX_OUTGOING, + )?; + result.reads = result.reads.saturating_add(examined as u64); + result.bounded |= truncated; + for edge in edges { + check()?; + let Some(site) = verified_site(&owner, &edge) else { + continue; + }; + result.reads += 1; + let Some(target) = backend.node_by_id(&edge.target)? else { + continue; + }; + if !exact_ast(&target) { + continue; + } + // Shared projections enforce byte/token bounds even on adversarial + // terminal names. Do not bypass those bounds with raw tokenization. + let document = search_document(&target); + let terms = document + .fields + .get("name") + .cloned() + .unwrap_or_default() + .into_iter() + .filter(|term| { + groups + .iter() + .any(|(_, variants, _, _)| variants.contains(term)) + }) + .collect::>(); + result.bounded |= document.truncated; + if !groups + .iter() + .any(|(_, variants, _, _)| variants.iter().any(|term| terms.contains(term))) + { + continue; + } + let retained = result.terms.entry(id.clone()).or_default(); + retained.extend(terms); + if retained.len() > compass_model::search::SEARCH_FIELD_MAX_TERMS { + result.bounded = true; + *retained = retained + .iter() + .take(compass_model::search::SEARCH_FIELD_MAX_TERMS) + .cloned() + .collect(); + } + result + .sources + .entry(id.clone()) + .or_default() + .insert(site.clone()); + result.targets.insert(target.id.clone(), target); + result.witnesses.insert(edge.id.clone(), edge); + } + if result.terms.contains_key(&id) { + result.nodes.insert(id, owner); + } + } + Ok(result) +} diff --git a/crates/compass-query/src/code_query.rs b/crates/compass-query/src/code_query.rs index 3a5cf532c..15b6401d5 100644 --- a/crates/compass-query/src/code_query.rs +++ b/crates/compass-query/src/code_query.rs @@ -361,6 +361,7 @@ pub struct CodeQueryEngine { pub(crate) semantic_search: bool, pub(crate) semantic_index: Mutex>, pub(crate) response_cache: crate::response_cache::ResponseCache, + pub(crate) source_query: Option, } #[derive(Clone, Debug, Eq, PartialEq)] @@ -1251,13 +1252,14 @@ impl PinnedDiscoveryBackend<'_> { } match self { Self::Materialized { lookup, .. } => { - let indexed = lookup - .relationship_source_ids_by_term - .get(&normalized) + let indexed = lookup.relationship_source_ids_by_term.get(&normalized); + let truncated = indexed.is_some_and(|posting| posting.len() > limit); + let source_ids = indexed + .into_iter() + .flatten() + .take(limit) .cloned() - .unwrap_or_default(); - let truncated = indexed.len() > limit; - let source_ids = indexed.into_iter().take(limit).collect::>(); + .collect::>(); Ok(RelationshipCandidateRead { node_ids_decoded: u64::try_from(source_ids.len()).unwrap_or(u64::MAX), source_ids, @@ -1433,7 +1435,15 @@ impl PinnedDiscoveryBackend<'_> { }; (nodes, exact_truncated, exact_work) }; - if nodes.is_empty() && !truncated { + // Reserved declaration-channel keys are exact postings. Falling back + // to their common `__c` prefix decodes unrelated synthetic terms and + // reports truncation even for an absent term in a tiny graph. + if nodes.is_empty() + && !truncated + && !concepts + .iter() + .any(|term| term.starts_with("__compass_primary_v1_")) + { (nodes, truncated, work) = reader .nodes_for_terms_bounded_work(concepts, read_limits) .map_err(snapshot_error)?; @@ -1541,7 +1551,7 @@ fn snapshot_limits(max_items: usize) -> Result { }) } -fn snapshot_error(error: compass_graph::SnapshotError) -> QueryError { +pub(crate) fn snapshot_error(error: compass_graph::SnapshotError) -> QueryError { QueryError::new( QueryErrorKind::CorruptArtifact, "store_graph_snapshot_failed", @@ -2343,7 +2353,15 @@ impl CodeQueryEngine { .collect::>() .join(" AND "); let prefix_query = fts_query_from_terms(concepts); - for query in [exact_query, prefix_query] { + let queries = if concepts + .iter() + .any(|term| term.starts_with("__compass_primary_v1_")) + { + vec![exact_query] + } else { + vec![exact_query, prefix_query] + }; + for query in queries { let mut statement = connection .prepare( "SELECT n.id @@ -4418,7 +4436,7 @@ fn add_admitted_candidates( } } -fn is_heuristic(edge: &EdgeRecord) -> bool { +pub(crate) fn is_heuristic(edge: &EdgeRecord) -> bool { edge.evidence .iter() .any(|evidence| evidence.origin == compass_model::provenance::EvidenceOrigin::Heuristic) @@ -4966,7 +4984,7 @@ pub(crate) fn enforce_response_size(response: &mut CodeQueryResponse) -> Result< Ok(()) } -fn sql_error(error: rusqlite::Error) -> QueryError { +pub(crate) fn sql_error(error: rusqlite::Error) -> QueryError { QueryError::new( QueryErrorKind::CorruptArtifact, "query_index_error", diff --git a/crates/compass-query/src/concepts.rs b/crates/compass-query/src/concepts.rs index dff6c5bed..c385ce8ec 100644 --- a/crates/compass-query/src/concepts.rs +++ b/crates/compass-query/src/concepts.rs @@ -16,6 +16,17 @@ const MAX_CONCEPT_POSTINGS: usize = 768; const MAX_DOCUMENT_ROOTS: usize = 8; const MAX_DOCUMENT_PROBES: usize = 64; +/// Small, directional recall hints for ordinary words absent from the index. +/// Broad retrieval checks literal context and keeps dependent matches tentative; +/// these do not change strict names or the shared concept-equivalence table. +pub(crate) fn unindexed_vocabulary_alternatives(term: &str) -> &'static [&'static str] { + match term { + "entry" => &["item", "element"], + "snapshot" => &["copy", "clone"], + _ => &[], + } +} + // These are recall alternatives, never assertions of equivalent code meaning. const SYNONYMS: &[(&[&str], &[&str])] = &[ ( @@ -397,6 +408,9 @@ impl CodeQueryEngine { request: SearchRequest, instrumentation: &mut QueryInstrumentation, ) -> Result { + if self.is_fielded_question(&request.query)? { + return self.broad_search(request, instrumentation, &|_| true); + } self.search_concepts_instrumented(request, instrumentation, &|_| true) } diff --git a/crates/compass-query/src/discovery.rs b/crates/compass-query/src/discovery.rs index 2eda420f3..1251f2408 100644 --- a/crates/compass-query/src/discovery.rs +++ b/crates/compass-query/src/discovery.rs @@ -93,21 +93,28 @@ struct DiscoveryCandidateSelection { truncated: bool, } -struct DiscoveryGuard<'a> { +pub(crate) struct DiscoveryGuard<'a> { deadline: Instant, cancelled: Option<&'a AtomicBool>, } impl<'a> DiscoveryGuard<'a> { - fn new(timeout_ms: u64, cancelled: Option<&'a AtomicBool>) -> Self { + pub(crate) fn new( + timeout_ms: u64, + cancelled: Option<&'a AtomicBool>, + engine_deadline: Option, + ) -> Self { let started = Instant::now(); Self { - deadline: started + Duration::from_millis(timeout_ms), + deadline: engine_deadline + .map_or(started + Duration::from_millis(timeout_ms), |deadline| { + deadline.min(started + Duration::from_millis(timeout_ms)) + }), cancelled, } } - fn check(&self) -> Result<(), QueryError> { + pub(crate) fn check(&self) -> Result<(), QueryError> { if self .cancelled .is_some_and(|cancelled| cancelled.load(Ordering::Relaxed)) @@ -395,15 +402,37 @@ impl CodeQueryEngine { cancelled: Option<&AtomicBool>, ) -> Result { validate_request(&request)?; - let guard = DiscoveryGuard::new(request.limits.timeout_ms, cancelled); + let guard = DiscoveryGuard::new(request.limits.timeout_ms, cancelled, self.deadline); + self.discover_guarded(request, &guard) + } + + pub(crate) fn discover_guarded( + &self, + request: DiscoveryQueryRequest, + guard: &DiscoveryGuard<'_>, + ) -> Result { guard.check()?; - if cancelled.is_none() + let fielded = self.is_fielded_question(&request.question)?; + guard.check()?; + if fielded + && request.direction == DiscoveryDirection::Auto + && request.relation_contexts.is_empty() + { + let mut response = self + .discover_ranked_with_context_guarded(request, true, guard, false)? + .discovery; + finish_response(guard, &mut response)?; + return Ok(response); + } + // The typed adapter does not carry a caller-owned cancellation flag. + // Cancelable requests retain the neighborhood executor's shared guard. + if guard.cancelled.is_none() && let Some(mut response) = self.structured_discovery_answer(&request)? { - finish_response(&guard, &mut response)?; + finish_response(guard, &mut response)?; return Ok(response); } - self.discover_neighborhood(request, &guard) + self.discover_neighborhood(request, guard) } fn discover_neighborhood( @@ -577,6 +606,14 @@ impl CodeQueryEngine { path: None, }); }; + if let Some(message) = &self.partial_graph_message { + response.diagnostics.push(QueryDiagnostic { + code: QueryDiagnosticCode::IncompleteCoverage, + message: message.clone(), + node_id: None, + path: None, + }); + } finish_response(guard, &mut response)?; return Ok(response); } @@ -602,6 +639,18 @@ impl CodeQueryEngine { Ok(response) } + pub(crate) fn ranked_scopes( + &self, + request: &DiscoveryQueryRequest, + guard: &DiscoveryGuard<'_>, + ) -> Result, QueryError> { + guard.check()?; + let backend = self.backend.pin_discovery()?; + let scopes = self.resolve_scopes(&backend, &request.scope, guard)?; + guard.check()?; + Ok(scopes) + } + fn resolve_scopes( &self, backend: &PinnedDiscoveryBackend<'_>, @@ -635,6 +684,7 @@ impl CodeQueryEngine { true, )); } + guard.check()?; let canonical = canonical_resolved_scope(&requested_scope, &values)?; resolved.push(canonical); } @@ -1933,7 +1983,7 @@ fn is_explicit_binary_path_question( >= 2 } -fn literal_identifier_terms(question: &str) -> BTreeSet { +pub(crate) fn literal_identifier_terms(question: &str) -> BTreeSet { // Standalone names already use the full-query exact lookup. In prose, // underscores and mixed-case internal capitals distinguish code spellings // from ordinary words such as "Explain", "close", or "HTTP". This is a @@ -2143,7 +2193,9 @@ fn intersection_token_bytes(concepts: &[String]) -> usize { concepts.iter().map(String::len).sum() } -fn infer_discovery_direction(question: &str) -> (DiscoveryDirection, DiscoveryDirectionSource) { +pub(crate) fn infer_discovery_direction( + question: &str, +) -> (DiscoveryDirection, DiscoveryDirectionSource) { let normalized = question .chars() .map(|character| { @@ -2350,7 +2402,7 @@ fn scope_resolution_error( ) } -fn validate_request(request: &DiscoveryQueryRequest) -> Result<(), QueryError> { +pub(crate) fn validate_request(request: &DiscoveryQueryRequest) -> Result<(), QueryError> { if !request.limits.is_valid() { return Err(QueryError::new( QueryErrorKind::InvalidParameter, @@ -2550,7 +2602,7 @@ fn edges_for_direction( } } -fn edge_matches_context(edge: &EdgeRecord, contexts: &[String]) -> bool { +pub(crate) fn edge_matches_context(edge: &EdgeRecord, contexts: &[String]) -> bool { contexts.is_empty() || edge .context @@ -2558,7 +2610,7 @@ fn edge_matches_context(edge: &EdgeRecord, contexts: &[String]) -> bool { .is_some_and(|context| contexts.iter().any(|candidate| candidate == context)) } -fn discovery_edge(edge: &EdgeRecord) -> DiscoveryEdge { +pub(crate) fn discovery_edge(edge: &EdgeRecord) -> DiscoveryEdge { let projected = query_edge(edge); DiscoveryEdge { id: (!projected.id.is_empty()).then_some(projected.id), @@ -2871,6 +2923,7 @@ mod tests { connection .execute_batch( r#"CREATE TABLE nodes(id TEXT PRIMARY KEY); + CREATE TABLE fielded_stats(term TEXT PRIMARY KEY, frequency INTEGER NOT NULL); CREATE VIRTUAL TABLE node_fts USING fts5( node_id UNINDEXED, name, qualified_name, aliases, kind, roles, language, framework, normalized_path, source_file, community_id, @@ -2891,6 +2944,8 @@ mod tests { let identifier_terms = [node.name.as_str(), node.qualified_name.as_str()] .into_iter() .flat_map(compass_model::search::identifier_search_terms) + .chain(compass_model::search::search_document(node).terms()) + .chain(compass_model::search::primary_evidence_terms(node)) .collect::>() .into_iter() .collect::>() @@ -2923,6 +2978,20 @@ mod tests { ) .unwrap_or_else(|_| std::process::abort()); } + let mut frequencies = std::collections::BTreeMap::::new(); + for node in &graph.nodes { + for term in compass_model::search::search_document(node).terms() { + *frequencies.entry(term).or_default() += 1; + } + } + for (term, frequency) in frequencies { + connection + .execute( + "INSERT INTO fielded_stats VALUES(?1,?2)", + rusqlite::params![term, frequency], + ) + .unwrap_or_else(|_| std::process::abort()); + } for (term, source_ids) in compass_model::search::direct_call_source_identifier_postings(&graph) { @@ -2966,6 +3035,7 @@ mod tests { semantic_search: false, semantic_index: std::sync::Mutex::new(None), response_cache: crate::response_cache::ResponseCache::disabled(), + source_query: None, } } @@ -4913,6 +4983,89 @@ mod tests { assert_eq!(error.code(), "discovery_cancelled"); } + #[test] + fn cancelable_typed_questions_retain_guarded_neighborhood_routing() + -> Result<(), Box> { + let engine = engine( + vec![node("n:alpha", "alpha"), node("n:caller", "caller")], + vec![edge("e:call", "n:caller", "n:alpha")], + ); + let mut query = request(DiscoveryDirection::Auto); + query.question = "who calls alpha".to_owned(); + let structured = engine + .structured_discovery_answer(&query)? + .ok_or("typed fixture must exercise the structured adapter")?; + let cancelled = AtomicBool::new(false); + let guard = super::DiscoveryGuard::new(query.limits.timeout_ms, Some(&cancelled), None); + let expected = engine.discover_neighborhood(query.clone(), &guard)?; + // Distinct seed metadata proves this fixture distinguishes the routes; + // it does not depend on a scheduler race or elapsed wall-clock time. + assert_ne!(structured.seeds, expected.seeds); + let actual = engine.discover_with_cancellation(query, Some(&cancelled))?; + assert_eq!(actual, expected); + Ok(()) + } + + #[test] + fn explicit_code_definitions_preserve_name_ambiguity() -> Result<(), Box> + { + for name in ["Target", "close", "quoted method", "pkg"] { + let mut first = anchored_node("n:first", name, "src/first.rs", 1); + let mut second = anchored_node("n:second", name, "src/second.rs", 1); + let operand = if name == "pkg" { + first.kind = NodeKind::Module; + second.kind = NodeKind::Module; + name.to_owned() + } else if name.contains(' ') { + first.kind = NodeKind::Method; + second.kind = NodeKind::Method; + format!("\"{name}\"") + } else { + name.to_owned() + }; + let engine = engine(vec![first, second], Vec::new()); + for question in [ + format!("where is {operand} defined"), + format!("find definition of {operand}"), + ] { + let mut query = request(DiscoveryDirection::Auto); + query.question = question; + assert!(!engine.is_fielded_question(&query.question)?); + let ordinary = engine.discover_with_cancellation(query.clone(), None)?; + let cancelled = AtomicBool::new(false); + assert_eq!( + engine.discover_with_cancellation(query.clone(), Some(&cancelled))?, + ordinary + ); + assert_eq!(ordinary.seeds.len(), 2); + assert!( + ordinary + .seeds + .iter() + .all(|seed| { seed.ambiguous && !seed.alternatives.is_empty() }) + ); + assert!( + ordinary.diagnostics.iter().any(|diagnostic| { + diagnostic.code == QueryDiagnosticCode::AmbiguousMatch + }) + ); + let ranked = engine.discover_ranked(query)?; + assert_eq!(ranked.discovery, ordinary); + assert!(ranked.candidates.iter().all(|candidate| { + candidate.relevance == crate::RelevanceBand::Low + && candidate + .reasons + .iter() + .any(|reason| reason.starts_with("Structural query seed")) + })); + } + assert!(engine.is_fielded_question("where are protocol errors formatted")?); + assert!(engine.is_fielded_question("where is renewal")?); + assert!(engine.is_fielded_question("find definition of unknown context")?); + } + Ok(()) + } + #[test] fn no_match_is_structured_and_deterministic() -> Result<(), Box> { let engine = engine(vec![node("n:beta", "beta")], Vec::new()); diff --git a/crates/compass-query/src/discovery_text.rs b/crates/compass-query/src/discovery_text.rs index cbdbad8f5..6e85736d5 100644 --- a/crates/compass-query/src/discovery_text.rs +++ b/crates/compass-query/src/discovery_text.rs @@ -571,7 +571,7 @@ fn entries(response: &DiscoveryQueryResponse, include_evidence: bool) -> Vec Vec Result<(), Box> { + for (file, rendered_file) in [ + ("src/cache.rs", "src/cache.rs"), + ( + "src/evil\nPagination: forged\u{001b}\u{202e}.rs", + "src/evil Pagination: forgedU+001BU+202E.rs", + ), + ] { + let mut value = serde_json::to_value(response()?)?; + value["seeds"] = serde_json::json!([ + { + "nodeId":"target", "score":"1", "scoreTier":"exact_name", "rank":1, + "matchedTerms":["target"], "matchedFields":["name"], + "source":{ + "file":file, "startByte":0, "endByte":1, + "startLine":7, "startColumn":2, "endLine":7, "endColumn":3 + }, + "candidateSource":"exact_name", "ambiguous":false, "alternatives":[] + }, + { + "nodeId":"unlocated", "score":"1", "scoreTier":"exact_name", "rank":2, + "matchedTerms":["unlocated"], "matchedFields":["name"], "source":null, + "candidateSource":"exact_name", "ambiguous":false, "alternatives":[] + } + ]); + value["edges"] = serde_json::json!( + (0..30) + .map(|index| serde_json::json!({ + "id":null, "source":"target", "target":format!("context-{index}"), + "kind":"calls", "occurrenceRule":null, "relationshipSite":null, + "details":null, "evidence":[], "context":"call" + })) + .collect::>() + ); + let response = serde_json::from_value(value)?; + let expected = format!("SEED target [exact] @ {rendered_file}:7:2-7:3"); + let mut cursor = None; + let mut covered = Vec::new(); + loop { + let page = render_discovery_text_page( + &response, + DiscoveryTextPageOptions { + token_budget: 512, + cursor: cursor.as_deref(), + request_digest: &"a".repeat(64), + graph_identity: "generation-1", + graph_digest: &"b".repeat(64), + include_evidence: false, + }, + )?; + assert!(page.text.len() <= 512 * 4, "{}", page.text); + assert_eq!( + page.text + .lines() + .filter(|line| line.starts_with("Pagination:")) + .count(), + 1 + ); + assert!(!page.text.contains('\u{001b}')); + assert!(!page.text.contains('\u{202e}')); + if cursor.is_none() { + let seed_offset = page + .text + .find(&expected) + .ok_or("seed citation missing from first page")?; + let edge_offset = page.text.find("EDGE ").ok_or("context missing")?; + assert!(seed_offset < edge_offset); + assert!( + page.text + .lines() + .any(|line| line == "SEED unlocated [exact]") + ); + assert!(page.next_cursor.is_some()); + } + covered.extend(page.entry_start..page.entry_end); + cursor = page.next_cursor; + if cursor.is_none() { + assert_eq!(covered, (0..page.entry_total).collect::>()); + break; + } + } + } + Ok(()) + } + #[test] fn request_digest_normalizes_set_like_contexts_and_scopes() -> Result<(), Box> { diff --git a/crates/compass-query/src/graph_engine.rs b/crates/compass-query/src/graph_engine.rs index 03a6d5019..2c38fc4e6 100644 --- a/crates/compass-query/src/graph_engine.rs +++ b/crates/compass-query/src/graph_engine.rs @@ -403,21 +403,31 @@ pub(crate) fn open_local_store_snapshot( .build; let source_commit = build.source_commit; let build_generation_identity = build.generation_id; - let partial_graph_message = reader - .graph_diagnostic_by_code("publication_omission_summary") - .map_err(|error| { + let mut partial_messages = Vec::new(); + for code in [ + "publication_omission_summary", + "leading_documentation_omission_summary", + ] { + if let Some(diagnostic) = reader.graph_diagnostic_by_code(code).map_err(|error| { QueryError::new( - QueryErrorKind::CorruptArtifact, + if matches!(error, compass_graph::SnapshotError::Limit(_)) { + QueryErrorKind::MemoryLimit + } else { + QueryErrorKind::CorruptArtifact + }, "store_graph_snapshot_failed", error.to_string(), ) - })? - .map(|diagnostic| { - format!( - "Published graph coverage is incomplete: {}", - diagnostic.message - ) - }); + })? { + partial_messages.push(diagnostic.message); + } + } + let partial_graph_message = (!partial_messages.is_empty()).then(|| { + format!( + "Published graph coverage is incomplete: {}", + partial_messages.join(" ") + ) + }); drop(reader); Ok(LocalStoreSnapshot { store, diff --git a/crates/compass-query/src/index.rs b/crates/compass-query/src/index.rs index 415532fce..e79871394 100644 --- a/crates/compass-query/src/index.rs +++ b/crates/compass-query/src/index.rs @@ -1,11 +1,10 @@ use std::collections::{BTreeMap, HashMap, VecDeque}; use std::fs::{self, File, OpenOptions}; -use std::io::Write; use std::path::Path; use std::path::PathBuf; use std::sync::{Arc, Mutex}; use std::thread; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use compass_graph::SnapshotSelector; use compass_ir::{PROGRAM_SCHEMA, ProgramBundle}; @@ -16,7 +15,7 @@ use compass_model::search::{ identifier_search_terms, }; use compass_store::Store; -use rusqlite::{Connection, OpenFlags, OptionalExtension, params}; +use rusqlite::{Connection, OpenFlags, params}; use sha2::{Digest, Sha256}; use crate::CodeQueryEngine; @@ -27,7 +26,8 @@ use crate::graph_engine::{ read_store_ref, }; -const INDEX_FORMAT_VERSION: &str = "compass-code-index/2"; +// Separate the OS-lock protocol from older presence-lock builders. +const INDEX_FORMAT_VERSION: &str = "compass-code-index/4"; /// Selects the source used to hydrate the typed query engine. #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -59,9 +59,11 @@ enum QueryEngineIdentity { snapshot_id: String, manifest_digest: String, graph_digest: String, + source_identity: Option, }, VerifiedDocument { graph_identity: String, + source_identity: Option, }, } @@ -122,6 +124,7 @@ impl QueryEngineCache { snapshot_id: reference.snapshot_id, manifest_digest: reference.manifest_digest, graph_digest: reference.graph_digest, + source_identity: crate::source_query::identity(&path)?, }; let mut state = self .state @@ -160,6 +163,7 @@ impl QueryEngineCache { })?; let identity = QueryEngineIdentity::VerifiedDocument { graph_identity: graph_identity.to_owned(), + source_identity: crate::source_query::identity(&path)?, }; let mut state = self .state @@ -168,7 +172,7 @@ impl QueryEngineCache { if let Some(engine) = state.get(&path, &identity) { return Ok(engine); } - let engine = Arc::new(Mutex::new(open_with_verified_document( + let engine = Arc::new(Mutex::new(open_with_verified_publication( document.clone(), graph_identity.to_owned(), &path, @@ -272,7 +276,8 @@ pub fn open_with_engine( return open_from_local_store(graph_path, program_path, cache_root); } let graph_engine = open_graph_engine(graph_path, selection)?; - open_from_graph_engine(graph_path, program_path, cache_root, graph_engine) + let engine = open_from_graph_engine(graph_path, program_path, cache_root, graph_engine)?; + crate::source_query::attach(engine, graph_path, cache_root) } /// Hydrate the typed query engine from any common-contract store adapter. @@ -330,6 +335,20 @@ pub fn open_with_verified_document( open_from_graph_engine(graph_path, program_path, cache_root, graph_engine) } +/// Open a verified current publication, including its graph-bound source evidence. +/// Detached/historical callers should use `open_with_verified_document`. +pub fn open_with_verified_publication( + graph: GraphDocument, + graph_identity: String, + graph_path: &Path, + program_path: Option<&Path>, + cache_root: &Path, +) -> Result { + let engine = + open_with_verified_document(graph, graph_identity, graph_path, program_path, cache_root)?; + crate::source_query::attach(engine, graph_path, cache_root) +} + fn open_from_graph_engine( graph_path: &Path, program_path: Option<&Path>, @@ -350,11 +369,7 @@ fn open_from_graph_engine( let index_dir = cache_root.join("code-query").join(&key); fs::create_dir_all(&index_dir).map_err(|error| io_error("create_index_dir", error))?; let index_path = index_dir.join("index.sqlite3"); - if !valid_index(&index_path, &key) { - if index_path.exists() { - fs::remove_file(&index_path) - .map_err(|error| io_error("remove_invalid_index", error))?; - } + if !valid_index(&index_path, &key)? { build_with_lock(&index_path, &key, &graph, program.as_ref())?; } // FTS5's integrity-check virtual-table operation needs a writable handle, @@ -364,17 +379,26 @@ fn open_from_graph_engine( .map_err(sql_error)?; let adjacency = crate::code_query::CodeAdjacencyIndex::build(&graph); let lookup = crate::code_query::CodeLookupIndex::build(&graph); - let partial_graph_message = graph - .graph - .diagnostics - .iter() - .rfind(|diagnostic| diagnostic.code == "publication_omission_summary") - .map(|diagnostic| { - format!( - "Published graph coverage is incomplete: {}", - diagnostic.message - ) - }); + let partial_messages = [ + "publication_omission_summary", + "leading_documentation_omission_summary", + ] + .into_iter() + .filter_map(|code| { + graph + .graph + .diagnostics + .iter() + .rfind(|diagnostic| diagnostic.code == code) + }) + .map(|diagnostic| diagnostic.message.as_str()) + .collect::>(); + let partial_graph_message = (!partial_messages.is_empty()).then(|| { + format!( + "Published graph coverage is incomplete: {}", + partial_messages.join(" ") + ) + }); let response_cache = crate::response_cache::ResponseCache::open(cache_root, &graph_identity, program.as_ref()); Ok(CodeQueryEngine { @@ -398,6 +422,7 @@ fn open_from_graph_engine( semantic_search: false, semantic_index: std::sync::Mutex::new(None), response_cache, + source_query: None, }) } @@ -415,7 +440,7 @@ fn open_from_local_store( let index_path = snapshot.store_path.clone(); let response_cache = crate::response_cache::ResponseCache::open(cache_root, &graph_identity, program.as_ref()); - Ok(CodeQueryEngine { + let engine = CodeQueryEngine { backend: CodeGraphBackend::Store(Box::new(snapshot)), program, connection: None, @@ -432,7 +457,9 @@ fn open_from_local_store( semantic_search: false, semantic_index: std::sync::Mutex::new(None), response_cache, - }) + source_query: None, + }; + crate::source_query::attach(engine, graph_path, cache_root) } fn load_program( @@ -483,38 +510,50 @@ fn build_with_lock( program: Option<&ProgramBundle>, ) -> Result<(), QueryError> { let lock_path = index_path.with_extension("lock"); - let mut lock = None; - for _ in 0..100 { - match OpenOptions::new() - .create_new(true) - .write(true) - .open(&lock_path) - { - Ok(file) => { - lock = Some(file); - break; - } - Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => { - if valid_index(index_path, key) { - return Ok(()); + let _lock = acquire_index_lock(&lock_path, Duration::from_secs(5))?; + // A different builder may have published while this request waited. Never + // remove or replace its completed index based on the pre-lock observation. + if valid_index(index_path, key)? { + return Ok(()); + } + if index_path.exists() { + fs::remove_file(index_path).map_err(|error| io_error("remove_invalid_index", error))?; + } + build_index(index_path, key, graph, program) +} + +fn acquire_index_lock(lock_path: &Path, wait: Duration) -> Result { + // Keep this inode in place. Unlinking a held lock can let another process + // acquire a different file under the same path. The OS releases ownership + // when the handle closes, including when its process is terminated. + let file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(lock_path) + .map_err(|error| io_error("create_index_lock", error))?; + let started = Instant::now(); + loop { + match file.try_lock() { + Ok(()) => return Ok(file), + Err(std::fs::TryLockError::WouldBlock) => { + if started.elapsed() >= wait { + return Err(QueryError::new( + QueryErrorKind::Internal, + "query_index_lock_timeout", + "timed out waiting for query index builder; retry after the active query finishes", + )); } - thread::sleep(Duration::from_millis(50)); + thread::sleep( + Duration::from_millis(50).min(wait.saturating_sub(started.elapsed())), + ); + } + Err(std::fs::TryLockError::Error(error)) => { + return Err(io_error("acquire_index_lock", error)); } - Err(error) => return Err(io_error("create_index_lock", error)), } } - let Some(mut lock) = lock else { - return Err(QueryError::new( - QueryErrorKind::Internal, - "query_index_lock_timeout", - "timed out waiting for query index builder", - )); - }; - let result = build_index(index_path, key, graph, program); - let _ = lock.flush(); - drop(lock); - let _ = fs::remove_file(lock_path); - result } fn build_index( @@ -546,6 +585,7 @@ fn build_index( CREATE TABLE files(path TEXT PRIMARY KEY, digest TEXT NOT NULL, json TEXT NOT NULL); CREATE TABLE evidence(owner_type TEXT NOT NULL, owner_id TEXT NOT NULL, position INTEGER NOT NULL, json TEXT NOT NULL); CREATE TABLE aliases(node_id TEXT NOT NULL, alias TEXT NOT NULL); + CREATE TABLE fielded_stats(term TEXT PRIMARY KEY, frequency INTEGER NOT NULL); CREATE TABLE relationship_terms( term TEXT NOT NULL, source_id TEXT NOT NULL, PRIMARY KEY(term, source_id) @@ -576,18 +616,51 @@ fn build_index( .iter() .map(|node| (node.id.as_str(), node)) .collect::>(); - let mut aliases_by_target = HashMap::<&str, Vec<&str>>::new(); + let mut aliases_by_target = BTreeMap::<&str, Vec<&str>>::new(); + let mut fielded_aliases_by_target = BTreeMap::<&str, Vec<&str>>::new(); for edge in &graph.links { - if edge.kind == compass_model::code_graph::EdgeKind::Aliases - && let Some(alias) = node_by_id.get(edge.source.as_str()) - { + if edge.kind != compass_model::code_graph::EdgeKind::Aliases { + continue; + } + let alias = node_by_id.get(edge.source.as_str()); + if let Some(alias) = alias { aliases_by_target .entry(edge.target.as_str()) .or_default() .push(alias.name.as_str()); } + // Count every relationship against the shared eight-alias work + // bound, including aliases that cannot support source relevance. + let name = if !edge.evidence.is_empty() + && compass_model::search::is_exact_nonheuristic_relationship(edge) + { + alias.map_or("", |alias| alias.name.as_str()) + } else { + "" + }; + fielded_aliases_by_target + .entry(edge.target.as_str()) + .or_default() + .push(name); + } + for aliases in aliases_by_target.values_mut() { + aliases.sort_unstable(); + } + for aliases in fielded_aliases_by_target.values_mut() { + aliases.sort_unstable(); } + let mut fielded_stats = BTreeMap::::new(); for node in &graph.nodes { + let document = compass_model::search::search_document_with_aliases( + node, + fielded_aliases_by_target + .get(node.id.as_str()) + .into_iter() + .flat_map(|aliases| aliases.iter().copied()), + ); + for term in document.terms() { + *fielded_stats.entry(term).or_default() += 1; + } let roles = node .roles .iter() @@ -629,6 +702,8 @@ fn build_index( ) .flat_map(identifier_search_terms) .chain(compass_model::search::document_search_terms(node)) + .chain(document.terms()) + .chain(document.primary_terms(node.kind)) .collect::>() .into_iter() .collect::>() @@ -786,6 +861,14 @@ fn build_index( transaction .execute("UPDATE metadata SET value='1' WHERE key='complete'", []) .map_err(sql_error)?; + for (term, frequency) in fielded_stats { + transaction + .execute( + "INSERT INTO fielded_stats VALUES(?1,?2)", + params![term, frequency], + ) + .map_err(sql_error)?; + } transaction.commit().map_err(sql_error)?; } let integrity: String = connection @@ -812,40 +895,8 @@ fn build_index( Ok(()) } -fn valid_index(path: &Path, key: &str) -> bool { - if !path.is_file() { - return false; - } - Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_WRITE) - .ok() - .is_some_and(|connection| valid_connection(&connection, key)) -} - -fn valid_connection(connection: &Connection, key: &str) -> bool { - let metadata = connection - .query_row( - "SELECT - (SELECT value FROM metadata WHERE key='format'), - (SELECT value FROM metadata WHERE key='key'), - (SELECT value FROM metadata WHERE key='complete')", - [], - |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, String>(1)?, - row.get::<_, String>(2)?, - )) - }, - ) - .optional() - .ok() - .flatten(); - let integrity = connection - .query_row("PRAGMA integrity_check", [], |row| row.get::<_, String>(0)) - .ok(); - metadata.is_some_and(|(format, found_key, complete)| { - format == INDEX_FORMAT_VERSION && found_key == key && complete == "1" - }) && integrity.as_deref() == Some("ok") +fn valid_index(path: &Path, key: &str) -> Result { + crate::index_validation::validate(path, key, INDEX_FORMAT_VERSION) } fn hex_digest(bytes: &[u8]) -> String { @@ -871,3 +922,199 @@ fn json_error(error: serde_json::Error) -> QueryError { error.to_string(), ) } + +#[cfg(test)] +mod tests { + use std::error::Error; + use std::io::Read; + use std::process::{Child, Command, Stdio}; + use std::sync::{Arc, Barrier}; + + use compass_model::code_graph::BuildMetadata; + + use super::*; + + fn empty_graph() -> GraphDocument { + GraphDocument::empty_v1(BuildMetadata { + builder_version: "index-lock-test".to_owned(), + schema_fingerprint: "sha256:schema".to_owned(), + source_tree_digest: "sha256:tree".to_owned(), + configuration_digest: "sha256:config".to_owned(), + generation_id: "sha256:generation".to_owned(), + source_commit: None, + }) + } + + struct ChildGuard(Child); + + impl Drop for ChildGuard { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } + } + + // A test subprocess acquires the production lock, then acknowledges actual + // ownership. No environment flag means the ordinary test run is a no-op. + #[test] + fn index_lock_child() -> Result<(), Box> { + let Some(directory) = std::env::var_os("COMPASS_QUERY_INDEX_LOCK_TEST_DIR") else { + return Ok(()); + }; + let directory = PathBuf::from(directory); + let _guard = acquire_index_lock(&directory.join("index.lock"), Duration::from_secs(5))?; + fs::write(directory.join("ready"), b"locked")?; + // The parent owns the pipe lifetime; termination also closes it. + let _ = std::io::stdin().read(&mut [0_u8; 1])?; + Ok(()) + } + + #[test] + fn public_open_preserves_live_builder_index_and_recovers_after_process_death() + -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let graph_path = directory.path().join("graph.json"); + fs::write(&graph_path, serde_json::to_vec(&empty_graph())?)?; + let cache = directory.path().join("cache"); + let engine = open(&graph_path, None, &cache)?; + let index = engine.index_path().to_path_buf(); + drop(engine); + let index_directory = index.parent().ok_or("index directory missing")?; + let key = index_directory + .file_name() + .and_then(|name| name.to_str()) + .ok_or("index key missing")?; + let lock = index.with_extension("lock"); + fs::write( + &index, + b"invalid index must survive a live competing builder", + )?; + let original = fs::read(&index)?; + let mut child = ChildGuard( + Command::new(std::env::current_exe()?) + .args(["--exact", "index::tests::index_lock_child", "--nocapture"]) + .env("COMPASS_QUERY_INDEX_LOCK_TEST_DIR", index_directory) + .stdin(Stdio::piped()) + .spawn()?, + ); + let start = Instant::now(); + while !index_directory.join("ready").is_file() { + if let Some(status) = child.0.try_wait()? { + return Err(format!("lock child exited before readiness: {status}").into()); + } + if start.elapsed() > Duration::from_secs(10) { + return Err("lock child did not acknowledge ownership".into()); + } + thread::sleep(Duration::from_millis(10)); + } + let error = match open(&graph_path, None, &cache) { + Ok(_) => return Err("public open stole a live builder's lock".into()), + Err(error) => error, + }; + assert_eq!(error.code(), "query_index_lock_timeout"); + assert_eq!(fs::read(&index)?, original); + assert!(lock.is_file()); + child.0.kill()?; + child.0.wait()?; + let reopened = open(&graph_path, None, &cache)?; + assert_eq!(reopened.index_path(), index); + assert!(valid_index(&index, key)?); + assert!( + lock.is_file(), + "the persistent coordination inode must remain" + ); + Ok(()) + } + + #[test] + fn index_build_rechecks_validity_under_lock_and_preserves_marker() -> Result<(), Box> + { + let directory = tempfile::tempdir()?; + let index = directory.path().join("index.sqlite3"); + let graph = empty_graph(); + build_with_lock(&index, "test-key", &graph, None)?; + let connection = Connection::open(&index)?; + connection.execute( + "INSERT INTO metadata VALUES('build_marker', 'preserved')", + [], + )?; + drop(connection); + build_with_lock(&index, "test-key", &graph, None)?; + let connection = Connection::open(&index)?; + let marker: String = connection.query_row( + "SELECT value FROM metadata WHERE key='build_marker'", + [], + |row| row.get(0), + )?; + assert_eq!(marker, "preserved"); + Ok(()) + } + + #[test] + fn concurrent_index_builders_publish_one_valid_index() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let index = Arc::new(directory.path().join("index.sqlite3")); + let barrier = Arc::new(Barrier::new(2)); + let mut workers = Vec::new(); + for _ in 0..2 { + let index = Arc::clone(&index); + let barrier = Arc::clone(&barrier); + workers.push(thread::spawn(move || { + barrier.wait(); + build_with_lock(&index, "test-key", &empty_graph(), None) + })); + } + for worker in workers { + worker.join().map_err(|_| "index builder thread failed")??; + } + assert!(valid_index(&index, "test-key")?); + assert!(index.with_extension("lock").is_file()); + Ok(()) + } + + #[test] + fn index_build_error_releases_lock_for_next_attempt() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let index = directory.path().join("index.sqlite3"); + // Removing a directory as a file deterministically fails on all native + // platforms, after acquisition but before any index can be published. + fs::create_dir(&index)?; + let graph = empty_graph(); + let error = match build_with_lock(&index, "test-key", &graph, None) { + Ok(()) => return Err("directory unexpectedly replaced as an index".into()), + Err(error) => error, + }; + assert_eq!(error.code(), "remove_invalid_index"); + fs::remove_dir(&index)?; + build_with_lock(&index, "test-key", &graph, None)?; + assert!(valid_index(&index, "test-key")?); + Ok(()) + } + + #[test] + fn public_open_metadata_deadline_preserves_existing_cache() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let graph_path = directory.path().join("graph.json"); + fs::write(&graph_path, serde_json::to_vec(&empty_graph())?)?; + let cache = directory.path().join("cache"); + let engine = open(&graph_path, None, &cache)?; + let index = engine.index_path().to_path_buf(); + drop(engine); + Connection::open(&index)?.execute_batch( + "ALTER TABLE metadata RENAME TO metadata_retained; + CREATE VIEW metadata AS + WITH RECURSIVE n(x) AS (VALUES(0) UNION ALL SELECT x+1 FROM n) + SELECT CASE WHEN x<0 THEN 'format' ELSE 'irrelevant' END AS key, + 'untrusted' AS value FROM n;", + )?; + let original = fs::read(&index)?; + let error = match open(&graph_path, None, &cache) { + Ok(_) => return Err("unbounded metadata view unexpectedly accepted".into()), + Err(error) => error, + }; + assert_eq!(error.kind(), QueryErrorKind::Timeout); + assert_eq!(error.code(), "query_index_validation_timeout"); + assert_eq!(fs::read(&index)?, original); + Ok(()) + } +} diff --git a/crates/compass-query/src/index_validation.rs b/crates/compass-query/src/index_validation.rs new file mode 100644 index 000000000..a77679044 --- /dev/null +++ b/crates/compass-query/src/index_validation.rs @@ -0,0 +1,649 @@ +//! Full validation of a disposable JSON query index, including slow file pagers. +//! Images are inspection-only and must preserve the exact original file bytes. +use std::fs::File; +use std::io::{Read, Seek}; +use std::path::Path; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc::{self, RecvTimeoutError, Sender}; +use std::thread::{self, JoinHandle}; +use std::time::{Duration, Instant}; + +use rusqlite::serialize::Data; +use rusqlite::{Connection, DatabaseName, OpenFlags}; + +use crate::cql::{QueryError, QueryErrorKind}; + +#[derive(Clone, Copy)] +struct Limits { + file_probe: Duration, + total: Duration, + minimum_image_bytes: u64, + maximum_image_bytes: u64, +} + +impl Default for Limits { + fn default() -> Self { + Self { + file_probe: Duration::from_secs(2), + total: Duration::from_secs(30), + minimum_image_bytes: 16 * 1024 * 1024, + maximum_image_bytes: 768 * 1024 * 1024, + } + } +} + +#[derive(Debug, Eq, PartialEq)] +enum Validation { + File, + Image, + Invalid, +} + +enum Check { + Complete(bool), + Interrupted, +} + +fn timeout() -> QueryError { + QueryError::new( + QueryErrorKind::Timeout, + "query_index_validation_timeout", + "local query index validation exceeded its startup budget; retry, or build a narrower graph with compass update ", + ) +} + +fn within(deadline: Instant) -> Result<(), QueryError> { + if Instant::now() >= deadline { + Err(timeout()) + } else { + Ok(()) + } +} + +/// Cancel and join before the connection can start another operation. In +/// particular, no late interrupt may escape into image inspection or queries. +struct DeadlineGuard<'a> { + connection: &'a Connection, + cancel: Sender<()>, + worker: Option>, +} + +impl Drop for DeadlineGuard<'_> { + fn drop(&mut self) { + self.connection.progress_handler(0, None:: bool>); + let _ = self.cancel.send(()); + if let Some(worker) = self.worker.take() { + let _ = worker.join(); + } + } +} + +fn bounded( + connection: &Connection, + deadline: Instant, + run: impl FnOnce(&Connection) -> rusqlite::Result, +) -> Result, QueryError> { + within(deadline)?; + let interrupted = Arc::new(AtomicBool::new(false)); + let timer_flag = Arc::clone(&interrupted); + let interrupt = connection.get_interrupt_handle(); + let (cancel, cancellation) = mpsc::channel(); + let worker = thread::Builder::new() + .name("compass-index-deadline".to_owned()) + .spawn(move || { + let remaining = deadline.saturating_duration_since(Instant::now()); + if matches!( + cancellation.recv_timeout(remaining), + Err(RecvTimeoutError::Timeout) + ) { + timer_flag.store(true, Ordering::Relaxed); + // SQLite checks this flag within its B-tree integrity walk, + // independently of the coarser VM/page progress interval. + interrupt.interrupt(); + } + }) + .ok(); + // If an OS thread cannot be started, retain deadline checks at each + // progress step rather than silently losing the independent timer. + let progress_interval = if worker.is_some() { 1000 } else { 1 }; + let guard = DeadlineGuard { + connection, + cancel, + worker, + }; + let flag = Arc::clone(&interrupted); + connection.progress_handler( + progress_interval, + Some(move || { + let expired = Instant::now() >= deadline; + if expired { + flag.store(true, Ordering::Relaxed); + } + expired + }), + ); + within(deadline)?; + let result = run(connection); + drop(guard); + if interrupted.load(Ordering::Relaxed) || Instant::now() >= deadline { + Err(timeout()) + } else { + Ok(result) + } +} + +fn full_check(connection: &Connection, deadline: Instant) -> Check { + match bounded(connection, deadline, |connection| { + connection.query_row("PRAGMA integrity_check", [], |row| row.get::<_, String>(0)) + }) { + Ok(result) => Check::Complete(result.is_ok_and(|text| text == "ok")), + Err(_) => Check::Interrupted, + } +} + +fn metadata_matches( + connection: &Connection, + key: &str, + format: &str, + deadline: Instant, +) -> Result { + bounded(connection, deadline, |connection| { + connection.query_row( + "SELECT + (SELECT value FROM metadata WHERE key='format'), + (SELECT value FROM metadata WHERE key='key'), + (SELECT value FROM metadata WHERE key='complete')", + [], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + )) + }, + ) + }) + .map(|result| { + result.is_ok_and(|(found_format, found_key, complete)| { + found_format == format && found_key == key && complete == "1" + }) + }) +} + +/// `None` means the original file must still be validated. Ineligibility or a +/// different serialized image is never evidence that the original is corrupt. +fn image_check( + connection: &Connection, + path: &Path, + limits: Limits, + deadline: Instant, +) -> Result, QueryError> { + within(deadline)?; + let Some(mut file) = File::open(path).ok() else { + return Ok(None); + }; + let Some(metadata) = file.metadata().ok() else { + return Ok(None); + }; + let size = metadata.len(); + if !metadata.is_file() + || size < limits.minimum_image_bytes.max(100) + || size > limits.maximum_image_bytes + { + return Ok(None); + } + let mut header = [0_u8; 100]; + if file.read_exact(&mut header).is_err() + || &header[..16] != b"SQLite format 3\0" + // A byte-equal WAL image does not preserve the original pager context. + || header[18] != 1 + || header[19] != 1 + { + return Ok(None); + } + let pages = connection.query_row("PRAGMA page_count", [], |row| row.get::<_, u64>(0)); + let page_size = connection.query_row("PRAGMA page_size", [], |row| row.get::<_, u64>(0)); + let (Ok(pages), Ok(page_size)) = (pages, page_size) else { + return Ok(None); + }; + if !(512..=65536).contains(&page_size) + || !page_size.is_power_of_two() + || pages == 0 + || pages.checked_mul(page_size) != Some(size) + { + return Ok(None); + } + within(deadline)?; + // This safe API allocates at most the checked page_count * page_size for + // the read transaction. Never construct OwnedData from raw pointers. + let Ok(image) = connection.serialize(DatabaseName::Main) else { + return Ok(None); + }; + within(deadline)?; + if u64::try_from(image.len()).ok() != Some(size) || file.rewind().is_err() { + return Ok(None); + } + let mut buffer = [0_u8; 65536]; + let mut start = 0; + while start < image.len() { + within(deadline)?; + let end = start.saturating_add(buffer.len()).min(image.len()); + let bytes = &mut buffer[..end - start]; + if file.read_exact(bytes).is_err() || image.get(start..end) != Some(bytes) { + return Ok(None); + } + start = end; + } + // Serialize may normalize trailing pages or zero-fill read failures. Exact + // physical equality, including EOF, is required before inspecting the copy. + if file.read(&mut [0_u8; 1]).ok() != Some(0) { + return Ok(None); + } + let Data::Owned(image) = image else { + return Ok(None); + }; + let Ok(mut memory) = Connection::open_in_memory() else { + return Ok(None); + }; + // FTS5's integrity operation needs a writable handle even for inspection. + if memory + .deserialize(DatabaseName::Main, image, false) + .is_err() + { + return Ok(None); + } + match full_check(&memory, deadline) { + Check::Complete(valid) => Ok(Some(valid)), + Check::Interrupted => Err(timeout()), + } +} + +pub(crate) fn validate(path: &Path, key: &str, format: &str) -> Result { + validate_with_limits(path, key, format, Limits::default()) + .map(|result| result != Validation::Invalid) +} + +fn validate_with_limits( + path: &Path, + key: &str, + format: &str, + limits: Limits, +) -> Result { + if !path.is_file() { + return Ok(Validation::Invalid); + } + let started = Instant::now(); + let deadline = started.checked_add(limits.total).ok_or_else(timeout)?; + within(deadline)?; + let Ok(connection) = Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_WRITE) + else { + return Ok(Validation::Invalid); + }; + // Pin one database snapshot across metadata, interrupted file inspection, + // serialization and byte comparison. Validation never publishes the image. + let Ok(transaction) = connection.unchecked_transaction() else { + return Ok(Validation::Invalid); + }; + if !metadata_matches(&transaction, key, format, deadline)? { + return Ok(Validation::Invalid); + } + within(deadline)?; + let probe_deadline = started + .checked_add(limits.file_probe) + .unwrap_or(deadline) + .min(deadline); + match full_check(&transaction, probe_deadline) { + Check::Complete(true) => return Ok(Validation::File), + Check::Complete(false) => return Ok(Validation::Invalid), + Check::Interrupted => {} + } + within(deadline)?; + if let Some(valid) = image_check(&transaction, path, limits, deadline)? { + return Ok(if valid { + Validation::Image + } else { + Validation::Invalid + }); + } + match full_check(&transaction, deadline) { + Check::Complete(true) => Ok(Validation::File), + Check::Complete(false) => Ok(Validation::Invalid), + Check::Interrupted => Err(timeout()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::error::Error; + use std::fs; + + const FORMAT: &str = "test-index/1"; + const KEY: &str = "test-key"; + + fn fixture(path: &Path, page_size: u32, wal: bool) -> Result<(), Box> { + let connection = Connection::open(path)?; + connection.pragma_update(None, "page_size", page_size)?; + if wal { + connection.pragma_update(None, "journal_mode", "WAL")?; + } + connection.execute_batch( + "CREATE TABLE metadata(key TEXT PRIMARY KEY,value TEXT); + CREATE TABLE payload(id INTEGER PRIMARY KEY,body BLOB); + CREATE INDEX payload_body ON payload(body); + CREATE VIRTUAL TABLE words USING fts5(text); + WITH RECURSIVE rows(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM rows WHERE n<2000) + INSERT INTO payload SELECT n,zeroblob(100) FROM rows; + INSERT INTO words VALUES('exact source identifier fixture');", + )?; + for (key, value) in [("format", FORMAT), ("key", KEY), ("complete", "1")] { + connection.execute("INSERT INTO metadata VALUES(?1,?2)", [key, value])?; + } + Ok(()) + } + + fn forced_image() -> Limits { + Limits { + file_probe: Duration::ZERO, + total: Duration::from_secs(10), + minimum_image_bytes: 0, + maximum_image_bytes: 2 * 1024 * 1024, + } + } + + #[test] + fn adaptive_image_retains_fts_integrity_and_original_bytes() -> Result<(), Box> { + for page_size in [4096, 8192, 65536] { + let directory = tempfile::tempdir()?; + let path = directory.path().join("index.sqlite3"); + fixture(&path, page_size, false)?; + let original = fs::read(&path)?; + assert_eq!( + validate_with_limits(&path, KEY, FORMAT, Limits::default())?, + Validation::File + ); + assert_eq!( + validate_with_limits(&path, KEY, FORMAT, forced_image())?, + Validation::Image + ); + assert_eq!(fs::read(&path)?, original); + } + Ok(()) + } + + #[test] + fn byte_equal_image_preserves_corruption_rejection() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let path = directory.path().join("index.sqlite3"); + fixture(&path, 4096, false)?; + let root_page: usize = Connection::open(&path)?.query_row( + "SELECT rootpage FROM sqlite_schema WHERE name='payload'", + [], + |row| row.get(0), + )?; + let mut bytes = fs::read(&path)?; + *bytes.get_mut((root_page - 1) * 4096).ok_or("root page")? = 0; + fs::write(&path, &bytes)?; + let connection = Connection::open(&path)?; + assert!(matches!( + full_check(&connection, Instant::now() + Duration::from_secs(10)), + Check::Complete(false) + )); + assert_eq!( + image_check( + &connection, + &path, + forced_image(), + Instant::now() + Duration::from_secs(10) + )?, + Some(false) + ); + drop(connection); + assert_eq!( + validate_with_limits(&path, KEY, FORMAT, forced_image())?, + Validation::Invalid + ); + assert_eq!(fs::read(&path)?, bytes); + Ok(()) + } + + #[test] + fn physical_size_mismatch_falls_back_to_original_full_check() -> Result<(), Box> { + for truncate in [false, true] { + let directory = tempfile::tempdir()?; + let path = directory.path().join("index.sqlite3"); + fixture(&path, 4096, false)?; + let mut bytes = fs::read(&path)?; + if truncate { + bytes.truncate(bytes.len().saturating_sub(41)); + } else { + bytes.extend_from_slice(&[0_u8; 4096]); + } + fs::write(&path, &bytes)?; + let connection = Connection::open(&path)?; + let valid = matches!( + full_check(&connection, Instant::now() + Duration::from_secs(10)), + Check::Complete(true) + ); + assert_eq!( + image_check( + &connection, + &path, + forced_image(), + Instant::now() + Duration::from_secs(10) + )?, + None + ); + drop(connection); + assert_eq!( + validate_with_limits(&path, KEY, FORMAT, forced_image())?, + if valid { + Validation::File + } else { + Validation::Invalid + } + ); + assert_eq!(fs::read(&path)?, bytes); + } + Ok(()) + } + + #[test] + fn wal_and_image_size_limits_keep_original_validation() -> Result<(), Box> { + for wal in [false, true] { + let directory = tempfile::tempdir()?; + let path = directory.path().join("index.sqlite3"); + fixture(&path, 4096, wal)?; + let bytes = fs::read(&path)?; + let limits = if wal { + forced_image() + } else { + Limits { + maximum_image_bytes: 1, + ..forced_image() + } + }; + assert_eq!( + image_check( + &Connection::open(&path)?, + &path, + limits, + Instant::now() + Duration::from_secs(10) + )?, + None + ); + assert_eq!( + validate_with_limits(&path, KEY, FORMAT, limits)?, + Validation::File + ); + assert_eq!(fs::read(&path)?, bytes); + } + Ok(()) + } + + #[test] + fn metadata_guards_apply_before_image_acceptance() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let path = directory.path().join("index.sqlite3"); + fixture(&path, 4096, false)?; + assert!(!validate(&path, "wrong-key", FORMAT)?); + assert!(!validate(&path, KEY, "unknown-format")?); + Connection::open(&path)? + .execute("UPDATE metadata SET value='0' WHERE key='complete'", [])?; + assert_eq!( + validate_with_limits(&path, KEY, FORMAT, forced_image())?, + Validation::Invalid + ); + Ok(()) + } + + #[test] + fn deadline_is_a_typed_error_and_never_invalidates_the_index() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let path = directory.path().join("index.sqlite3"); + fixture(&path, 4096, false)?; + let bytes = fs::read(&path)?; + let error = validate_with_limits( + &path, + KEY, + FORMAT, + Limits { + total: Duration::ZERO, + ..forced_image() + }, + ) + .err() + .ok_or("deadline unexpectedly accepted")?; + assert_eq!(error.kind(), QueryErrorKind::Timeout); + assert_eq!(error.code(), "query_index_validation_timeout"); + assert_eq!(fs::read(&path)?, bytes); + assert!(validate(&path, KEY, FORMAT)?); + Ok(()) + } + + #[test] + fn byte_guard_rejects_a_different_physical_database() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let first = directory.path().join("first.sqlite3"); + let second = directory.path().join("second.sqlite3"); + fixture(&first, 4096, false)?; + fs::copy(&first, &second)?; + Connection::open(&second)? + .execute("UPDATE metadata SET value='new-value' WHERE key='key'", [])?; + assert_eq!(fs::metadata(&first)?.len(), fs::metadata(&second)?.len()); + assert_eq!( + image_check( + &Connection::open(&first)?, + &second, + forced_image(), + Instant::now() + Duration::from_secs(10) + )?, + None + ); + Ok(()) + } + + #[test] + fn interrupted_integrity_retains_the_read_snapshot_for_image_validation() + -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let path = directory.path().join("index.sqlite3"); + fixture(&path, 4096, false)?; + let bytes = fs::read(&path)?; + let connection = Connection::open(&path)?; + let transaction = connection.unchecked_transaction()?; + assert!(metadata_matches( + &transaction, + KEY, + FORMAT, + Instant::now() + Duration::from_secs(10) + )?); + // Force an actual SQLite VM interruption, independently of wall timing. + transaction.progress_handler(1, Some(|| true)); + let interrupted = + transaction.query_row("PRAGMA integrity_check", [], |row| row.get::<_, String>(0)); + transaction.progress_handler(0, None:: bool>); + assert!( + matches!(interrupted, Err(rusqlite::Error::SqliteFailure(error, _)) if error.code == rusqlite::ErrorCode::OperationInterrupted) + ); + assert!(!transaction.is_autocommit()); + assert_eq!( + image_check( + &transaction, + &path, + forced_image(), + Instant::now() + Duration::from_secs(10) + )?, + Some(true) + ); + drop(transaction); + drop(connection); + assert_eq!(fs::read(&path)?, bytes); + Ok(()) + } + + #[test] + fn deadline_interrupts_sql_without_progress_callbacks() -> Result<(), Box> { + let connection = Connection::open_in_memory()?; + let started = Instant::now(); + let result = bounded( + &connection, + started + Duration::from_millis(20), + |connection| { + // A long SQLite operation can spend substantial time between + // progress callbacks. Exercise the independent interrupt path. + connection.progress_handler(0, None:: bool>); + connection.query_row( + "WITH RECURSIVE rows(n) AS ( + VALUES(1) UNION ALL SELECT n+1 FROM rows WHERE n<50000000 + ) SELECT sum(n) FROM rows", + [], + |row| row.get::<_, i64>(0), + ) + }, + ); + let error = result.err().ok_or("deadline unexpectedly accepted")?; + assert_eq!(error.code(), "query_index_validation_timeout"); + assert!(started.elapsed() < Duration::from_secs(2)); + assert_eq!( + connection.query_row("SELECT 1", [], |row| row.get::<_, i64>(0))?, + 1 + ); + Ok(()) + } + + #[test] + fn cancelled_deadline_cannot_interrupt_the_next_sql_operation() -> Result<(), Box> { + let connection = Connection::open_in_memory()?; + let value = bounded( + &connection, + Instant::now() + Duration::from_millis(200), + |connection| connection.query_row("SELECT 1", [], |row| row.get::<_, i64>(0)), + )??; + assert_eq!(value, 1); + // This query crosses the first operation's deadline. A detached or late + // interrupt must not escape into subsequent work on this connection. + let mut delay_once = true; + connection.progress_handler( + 1, + Some(move || { + if delay_once { + delay_once = false; + std::thread::sleep(Duration::from_millis(250)); + } + false + }), + ); + let result = connection.query_row( + "WITH RECURSIVE rows(n) AS ( + VALUES(1) UNION ALL SELECT n+1 FROM rows WHERE n<100 + ) SELECT sum(n) FROM rows", + [], + |row| row.get::<_, i64>(0), + ); + connection.progress_handler(0, None:: bool>); + assert_eq!(result?, 5050); + Ok(()) + } +} diff --git a/crates/compass-query/src/intent.rs b/crates/compass-query/src/intent.rs index 181d44d0b..cc74fc192 100644 --- a/crates/compass-query/src/intent.rs +++ b/crates/compass-query/src/intent.rs @@ -6,7 +6,7 @@ use std::time::Instant; use crate::code_query::{CodeQueryEngine, validate_limits}; use crate::cql::{QueryError, QueryErrorKind}; -use crate::ranking::QUERY_RANKER_PROFILE_V1; +use crate::ranking::QUERY_RANKER_PROFILE_V10; use crate::telemetry::{ProfiledCodeQueryResponse, QueryInstrumentation}; pub const QUERY_PLANNER_PROFILE_V1: &str = "query-planner/1"; @@ -111,7 +111,7 @@ impl CodeQueryEngine { response, total_started.elapsed(), QUERY_PLANNER_PROFILE_V2, - QUERY_RANKER_PROFILE_V1, + QUERY_RANKER_PROFILE_V10, ), selections, )) diff --git a/crates/compass-query/src/lib.rs b/crates/compass-query/src/lib.rs index 28d502766..abc1888e1 100644 --- a/crates/compass-query/src/lib.rs +++ b/crates/compass-query/src/lib.rs @@ -5,6 +5,7 @@ use sha2::Digest as _; mod affected; mod benchmark; mod bm25; +mod broad; mod code_query; mod concepts; mod cql; @@ -14,6 +15,7 @@ mod explanation_members; mod export_binding; mod graph_engine; mod index; +mod index_validation; mod intent; mod natural_answers; mod neighbors; @@ -26,6 +28,7 @@ mod response_cache; mod score; mod semantic_search; mod source; +mod source_query; mod telemetry; mod text; mod text_cursor; @@ -33,6 +36,10 @@ mod traversal; pub use affected::{DEFAULT_AFFECTED_RELATIONS, affected_nodes, format_affected, resolve_seed}; pub use benchmark::{BenchmarkQuestion, BenchmarkResult, format_benchmark, run_benchmark}; +pub use broad::{ + RankedCandidate, RankedDiscoveryResponse, RankedExcerptKind, RankedFilterReason, + RankedRetrievalTrace, RankedSourceExcerpt, RankedTargetTrace, RankedTermTrace, RelevanceBand, +}; pub use code_query::{ CodeQueryEngine, ExactSearchFilter, normalize_symbol as normalize_code_query_symbol, }; @@ -58,7 +65,7 @@ pub use index::{ CachedQueryEngine, DEFAULT_QUERY_ENGINE_CACHE_CAPACITY, EngineSelection, MAX_QUERY_ENGINE_CACHE_CAPACITY, QueryEngineCache, QueryEngineKind, has_published_store, open, open_with_document, open_with_engine, open_with_store, open_with_store_selector, - open_with_verified_document, + open_with_verified_document, open_with_verified_publication, }; pub use intent::{ NaturalQueryIntent, NaturalQueryPlan, NaturalQueryRequest, QUERY_PLANNER_PROFILE_V1, @@ -73,7 +80,13 @@ pub use overview::{ MAX_OVERVIEW_NODES, OverviewError, OverviewScope, ScopeSelection, hotspots, scoped_document, }; pub use program_join::join_program_evidence; -pub use ranking::QUERY_RANKER_PROFILE_V1; +pub use ranking::{ + QUERY_RANKER_PROFILE_V1, QUERY_RANKER_PROFILE_V2, QUERY_RANKER_PROFILE_V3, + QUERY_RANKER_PROFILE_V4, QUERY_RANKER_PROFILE_V5, QUERY_RANKER_PROFILE_V6, + QUERY_RANKER_PROFILE_V7, QUERY_RANKER_PROFILE_V8, QUERY_RANKER_PROFILE_V9, + QUERY_RANKER_PROFILE_V10, QUERY_RANKER_PROFILE_V11, QUERY_RANKER_PROFILE_V12, + QUERY_RANKER_PROFILE_V13, QUERY_RANKER_PROFILE_V14, +}; pub use relevance::{ EdgeIdentity, EdgeJudgment, IdJudgment, IntentMetrics, JudgedQuery, JudgmentCorpus, MAX_JUDGMENTS_PER_QUERY, MAX_QUESTIONS, MAX_TEXT_BYTES, MetricValue, ObservedEdge, diff --git a/crates/compass-query/src/ranking.rs b/crates/compass-query/src/ranking.rs index 90e49c3b6..a96214f50 100644 --- a/crates/compass-query/src/ranking.rs +++ b/crates/compass-query/src/ranking.rs @@ -9,6 +9,19 @@ use crate::recall::{CandidateSource, SearchCandidate}; use crate::text::{canonical_query_token, search_tokens, strip_diacritics}; pub const QUERY_RANKER_PROFILE_V1: &str = "query-ranker/1"; +pub const QUERY_RANKER_PROFILE_V2: &str = "query-ranker/2-fielded"; +pub const QUERY_RANKER_PROFILE_V3: &str = "query-ranker/3-partial-evidence"; +pub const QUERY_RANKER_PROFILE_V4: &str = "query-ranker/4-leading-documentation"; +pub const QUERY_RANKER_PROFILE_V5: &str = "query-ranker/5-code-reference-recall"; +pub const QUERY_RANKER_PROFILE_V6: &str = "query-ranker/6-owned-identifier-recall"; +pub const QUERY_RANKER_PROFILE_V7: &str = "query-ranker/7-topic-evidence"; +pub const QUERY_RANKER_PROFILE_V8: &str = "query-ranker/8-native-body-evidence"; +pub const QUERY_RANKER_PROFILE_V9: &str = "query-ranker/9-test-context"; +pub const QUERY_RANKER_PROFILE_V10: &str = "query-ranker/10-temporal-qualification"; +pub const QUERY_RANKER_PROFILE_V11: &str = "query-ranker/11-implementation-action"; +pub const QUERY_RANKER_PROFILE_V12: &str = "query-ranker/12-single-operation-inspection"; +pub const QUERY_RANKER_PROFILE_V13: &str = "query-ranker/13-unindexed-vocabulary"; +pub const QUERY_RANKER_PROFILE_V14: &str = "query-ranker/14-compound-identifier"; #[derive(Clone, Debug)] pub(crate) struct RankedSearchResult { @@ -937,11 +950,14 @@ pub(crate) fn canonical_predicate_token(token: &str) -> Option<&'static str> { "decode" => Some("decode"), "delete" => Some("delete"), "discover" => Some("discover"), + "document" => Some("document"), "dispatch" => Some("dispatch"), "detect" => Some("detect"), "drain" => Some("drain"), "extract" => Some("extract"), + "enforc" | "enforce" => Some("enforce"), "find" => Some("find"), + "format" => Some("format"), "generate" | "generated" => Some("generate"), "freeze" => Some("freeze"), "get" | "read" | "select" => Some("read"), @@ -1002,6 +1018,7 @@ pub(crate) fn is_explicit_operation_predicate(token: &str) -> bool { | "generate" | "generated" | "find" + | "format" | "freeze" | "get" | "handle" diff --git a/crates/compass-query/src/response_cache.rs b/crates/compass-query/src/response_cache.rs index 98bcd574a..856c047e1 100644 --- a/crates/compass-query/src/response_cache.rs +++ b/crates/compass-query/src/response_cache.rs @@ -33,7 +33,16 @@ impl ResponseCache { digest.update(graph.as_bytes()); digest.update(env!("CARGO_PKG_VERSION").as_bytes()); digest.update(crate::QUERY_PLANNER_PROFILE_V2.as_bytes()); - digest.update(crate::QUERY_RANKER_PROFILE_V1.as_bytes()); + digest.update(crate::QUERY_RANKER_PROFILE_V14.as_bytes()); + digest.update(b"compound-identifier/2"); + digest.update(b"unindexed-vocabulary/1"); + // Invalidate experimental single-action answers lacking context vetoes. + digest.update(b"single-operation-context/1"); + // Projection epoch: old cached responses + // must not silently omit retained ranked source quotations. + digest.update(b"ranked-source-excerpts/3"); + // Historical local-store responses may lack documentation omissions. + digest.update(b"graph-diagnostic-projection/1"); if let Some(program) = program { // Hash through a writer, without constructing another IR-sized buffer. if serde_json::to_writer(DigestWriter(&mut digest), program).is_err() { diff --git a/crates/compass-query/src/semantic_search.rs b/crates/compass-query/src/semantic_search.rs index 8492e443f..5f1dd72f1 100644 --- a/crates/compass-query/src/semantic_search.rs +++ b/crates/compass-query/src/semantic_search.rs @@ -124,9 +124,11 @@ impl SemanticIndex { (term, (index, idf)) }) .collect::>(); + check()?; let matrix = documents .into_iter() .map(|terms| { + check()?; let mut row = terms .into_iter() .filter_map(|term| vocabulary.get(&term).copied()) @@ -137,9 +139,9 @@ impl SemanticIndex { *value /= size; } } - row + Ok(row) }) - .collect::>(); + .collect::, QueryError>>()?; let mut directions: Vec> = Vec::new(); // Power iteration of A^T A with reorthogonalization. Integer seed // arithmetic and ordered sparse rows make equivalent inputs stable. @@ -147,6 +149,7 @@ impl SemanticIndex { .min(vocabulary.len()) .min(nodes.len().isqrt().max(1)) { + check()?; let mut vector = (0..vocabulary.len()) .map(|index| { let seed = (index as u64 + 1) @@ -163,6 +166,7 @@ impl SemanticIndex { check()?; let mut next = vec![0.0; vocabulary.len()]; for row in &matrix { + check()?; let dot = row .iter() .map(|(index, weight)| vector[*index] * weight) @@ -180,7 +184,12 @@ impl SemanticIndex { // Degenerate directions do not supply arbitrary similarity. let energy = matrix .iter() - .map(|row| row.iter().map(|(i, w)| vector[*i] * w).sum::().powi(2)) + .map(|row| { + check()?; + Ok(row.iter().map(|(i, w)| vector[*i] * w).sum::().powi(2)) + }) + .collect::, QueryError>>()? + .into_iter() .sum::(); if energy <= 1e-8 { continue; @@ -190,14 +199,16 @@ impl SemanticIndex { let embeddings = matrix .iter() .map(|row| { + check()?; let mut embedding = directions .iter() .map(|basis| row.iter().map(|(i, w)| basis[*i] * w).sum()) .collect::>(); norm(&mut embedding); - embedding + Ok(embedding) }) - .collect(); + .collect::, QueryError>>()?; + check()?; Ok(Self { nodes, vocabulary, @@ -207,56 +218,66 @@ impl SemanticIndex { }) } - fn search(&self, query: &str, max_nodes: u32) -> Vec<(NodeRecord, f64)> { + fn search( + &self, + query: &str, + max_nodes: u32, + check: &dyn Fn() -> Result<(), QueryError>, + ) -> Result, QueryError> { + check()?; let terms = crate::query_terms(query) .into_iter() .filter_map(|term| self.vocabulary.get(&term).copied()) .collect::>(); let query_indices = terms.keys().copied().collect::>(); - let bridges = self - .rows - .iter() - .filter(|row| row.iter().any(|(index, _)| query_indices.contains(index))) - .flat_map(|row| row.iter().map(|(index, _)| *index)) - .collect::>(); + let mut bridges = BTreeSet::new(); + for row in &self.rows { + check()?; + if row.iter().any(|(index, _)| query_indices.contains(index)) { + bridges.extend(row.iter().map(|(index, _)| *index)); + } + } let mut vector = self .directions .iter() .map(|basis| terms.iter().map(|(i, w)| basis[*i] * w).sum()) .collect::>(); if !norm(&mut vector) { - return Vec::new(); + check()?; + return Ok(Vec::new()); } - let mut scores = self - .embeddings - .iter() - .enumerate() - .filter_map(|(index, embedding)| { - if !self.rows[index] - .iter() - .any(|(index, _)| bridges.contains(index)) - { - return None; - } - let similarity = vector - .iter() - .zip(embedding) - .map(|(a, b)| a * b) - .sum::(); - let similarity = (similarity.clamp(-1.0, 1.0) * 1_000_000.0).round() / 1_000_000.0; - (similarity >= MIN_SIMILARITY).then_some((index, similarity)) - }) - .collect::>(); + let mut scores = Vec::new(); + for (index, embedding) in self.embeddings.iter().enumerate() { + check()?; + if !self.rows[index] + .iter() + .any(|(index, _)| bridges.contains(index)) + { + continue; + } + let similarity = vector + .iter() + .zip(embedding) + .map(|(a, b)| a * b) + .sum::(); + let similarity = (similarity.clamp(-1.0, 1.0) * 1_000_000.0).round() / 1_000_000.0; + if similarity >= MIN_SIMILARITY { + scores.push((index, similarity)); + } + } + check()?; scores.sort_by(|(a, ascore), (b, bscore)| { bscore .total_cmp(ascore) .then_with(|| self.nodes[*a].id.cmp(&self.nodes[*b].id)) }); - scores + let result = scores .into_iter() .take(usize::try_from(max_nodes).unwrap_or(usize::MAX).min(8)) .map(|(index, score)| (self.nodes[index].clone(), score)) - .collect() + .collect(); + check()?; + Ok(result) } } @@ -266,11 +287,26 @@ impl CodeQueryEngine { query: &str, max_nodes: u32, ) -> Result, QueryError> { - self.check_deadline()?; - let mut cache = self - .semantic_index - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); + self.semantic_candidates_guarded(query, max_nodes, &|| self.check_deadline()) + } + + pub(crate) fn semantic_candidates_guarded( + &self, + query: &str, + max_nodes: u32, + check: &dyn Fn() -> Result<(), QueryError>, + ) -> Result, QueryError> { + check()?; + // A query waiting behind another index build retains its own absolute + // deadline/cancellation guard instead of blocking in Mutex::lock. + let mut cache = loop { + check()?; + match self.semantic_index.try_lock() { + Ok(cache) => break cache, + Err(std::sync::TryLockError::Poisoned(error)) => break error.into_inner(), + Err(std::sync::TryLockError::WouldBlock) => std::thread::yield_now(), + } + }; if cache.is_none() { let mut nodes = match &self.backend { CodeGraphBackend::Materialized { graph, .. } => { @@ -291,15 +327,18 @@ impl CodeQueryEngine { }) .map_err(|error| limit(&format!("cannot read bounded corpus: {error}")))?, }; + check()?; nodes.sort_by(|a, b| a.id.cmp(&b.id)); + check()?; // Publish the cache only after successful, bounded construction. - let index = SemanticIndex::build(nodes, || self.check_deadline())?; - self.check_deadline()?; + let index = SemanticIndex::build(nodes, check)?; + check()?; *cache = Some(index); } - Ok(cache - .as_ref() - .map_or_else(Vec::new, |index| index.search(query, max_nodes))) + match cache.as_ref() { + Some(index) => index.search(query, max_nodes, check), + None => Ok(Vec::new()), + } } } @@ -335,13 +374,13 @@ mod tests { node("billing", "invoice payment refund"), ]; let index = SemanticIndex::build(corpus.clone(), || Ok(()))?; - let result = index.search("orchard", 3); + let result = index.search("orchard", 3, &|| Ok(()))?; assert!( result.iter().any(|(node, _)| node.id == "fruit"), "{result:?}" ); assert!(result.iter().all(|(node, _)| node.id != "billing")); - assert!(index.search("absentvocabulary", 3).is_empty()); + assert!(index.search("absentvocabulary", 3, &|| Ok(()))?.is_empty()); let repeated = SemanticIndex::build(corpus, || Ok(()))?; assert_eq!(index.directions, repeated.directions); assert_eq!(index.embeddings, repeated.embeddings); @@ -378,4 +417,53 @@ mod tests { }); assert!(matches!(error, Err(error) if error.kind() == QueryErrorKind::Timeout)); } + #[test] + fn cached_semantic_search_observes_guard_while_scanning_rows() -> Result<(), QueryError> { + let index = SemanticIndex::build( + (0..64) + .map(|at| node(&format!("n{at}"), "fruit apples harvest")) + .collect(), + || Ok(()), + )?; + let checks = std::cell::Cell::new(0); + let error = index.search("fruit", 5, &|| { + checks.set(checks.get() + 1); + if checks.get() >= 12 { + Err(QueryError::new( + QueryErrorKind::Cancelled, + "test_cancelled", + "cancelled during cached semantic search", + )) + } else { + Ok(()) + } + }); + assert!(matches!(error, Err(error) if error.kind() == QueryErrorKind::Cancelled)); + assert_eq!(checks.get(), 12); + Ok(()) + } + + #[test] + fn semantic_build_checks_guard_after_tokenization_before_publishing() { + let checks = std::cell::Cell::new(0); + let result = SemanticIndex::build( + (0..32) + .map(|at| node(&format!("n{at}"), "fruit apples harvest")) + .collect(), + || { + checks.set(checks.get() + 1); + if checks.get() >= 48 { + Err(QueryError::new( + QueryErrorKind::Timeout, + "test_timeout", + "expired in matrix construction", + )) + } else { + Ok(()) + } + }, + ); + assert!(matches!(result, Err(error) if error.kind() == QueryErrorKind::Timeout)); + assert_eq!(checks.get(), 48); + } } diff --git a/crates/compass-query/src/source_query.rs b/crates/compass-query/src/source_query.rs new file mode 100644 index 000000000..08b53a50f --- /dev/null +++ b/crates/compass-query/src/source_query.rs @@ -0,0 +1,186 @@ +//! Bounded immutable source-body recall shared by JSON and local-store queries. +use crate::{CodeQueryEngine, QueryError, QueryErrorKind}; +use compass_model::provenance::SourceAnchor; +use compass_model::source_query::*; +use sha2::{Digest, Sha256}; +use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; + +pub(crate) struct Index { + pub digest: String, + pub owners: Vec, + by_id: BTreeMap, + postings: BTreeMap>, + pub frequencies: BTreeMap, + pub bounded: bool, + pub coverage_message: Option, +} +fn corrupt(message: impl Into) -> QueryError { + QueryError::new( + QueryErrorKind::CorruptArtifact, + "source_query_artifact_invalid", + format!( + "{}; run compass update --force to rebuild current source evidence", + message.into() + ), + ) +} +fn bytes(graph: &Path) -> Result>, QueryError> { + let path = graph + .parent() + .unwrap_or_else(|| Path::new(".")) + .join(SOURCE_QUERY_FILE); + match std::fs::symlink_metadata(&path) { + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(e) => return Err(corrupt(e.to_string())), + Ok(m) if !m.file_type().is_file() => { + return Err(corrupt("source-query artifact must be a regular file")); + } + Ok(_) => {} + } + compass_files::read_bytes_bounded(&path, SOURCE_QUERY_MAX_BYTES) + .map(Some) + .map_err(|e| corrupt(e.to_string())) +} +pub(crate) fn identity(graph: &Path) -> Result, QueryError> { + Ok(bytes(graph)?.map(|b| format!("{:x}", Sha256::digest(&b)))) +} +impl Index { + fn open(graph: &Path, graph_digest: &str) -> Result, QueryError> { + let Some(bytes) = bytes(graph)? else { + return Ok(None); + }; + let digest = format!("{:x}", Sha256::digest(&bytes)); + // Inspect the bounded version envelope before decoding current fields. + // A future major need not contain any of the current payload shape. + #[derive(serde::Deserialize)] + struct Header { + schema: String, + #[serde(default)] + profile: Option, + } + let header: Header = serde_json::from_slice(&bytes).map_err(|e| corrupt(e.to_string()))?; + if header.schema != SOURCE_QUERY_SCHEMA { + return Err(QueryError::new( + QueryErrorKind::UnsupportedSchema, + "unsupported_source_query_schema", + format!( + "Unsupported source-query schema {}; update Compass or rebuild current source evidence", + header.schema + ), + )); + } + if header + .profile + .as_deref() + .is_some_and(|profile| !is_supported_source_query_profile(profile)) + { + return Err(QueryError::new( + QueryErrorKind::UnsupportedSchema, + "unsupported_source_query_profile", + "Unsupported source-query projection profile; update Compass or rebuild current source evidence", + )); + } + let artifact: SourceQueryArtifact = + serde_json::from_slice(&bytes).map_err(|e| corrupt(e.to_string()))?; + artifact.validate().map_err(|e| corrupt(e.to_string()))?; + if artifact.graph_digest != graph_digest { + return Err(corrupt( + "source-query artifact is bound to a different graph", + )); + } + let bounded = artifact.aggregate_truncated + || artifact.omitted_files > 0 + || artifact.omitted_owners > 0; + let omitted_files = artifact.omitted_files; + let omitted_owners = artifact.omitted_owners; + let mut clipped_owners = 0_usize; + let mut result = Self { + digest, + owners: artifact.owners, + by_id: BTreeMap::new(), + postings: BTreeMap::new(), + frequencies: artifact.additional_document_frequencies, + bounded, + coverage_message: None, + }; + let mut count = 0_usize; + for (at, owner) in result.owners.iter().enumerate() { + result.by_id.insert(owner.node_id.clone(), at); + let document = owner.document(); + clipped_owners += usize::from(document.truncated); + result.bounded |= document.truncated; + for term in document.terms() { + count += 1; + if count > 262_144 + || result.postings.len() >= 65_536 && !result.postings.contains_key(&term) + { + return Err(QueryError::new( + QueryErrorKind::MemoryLimit, + "source_query_index_limit", + "Source-body query index exceeds its bounded posting/term capacity; narrow the build scope", + )); + } + result.postings.entry(term).or_default().push(at); + } + } + if result.bounded { + result.coverage_message = Some(format!( + "Callable-body recall is bounded: {omitted_files} source files are omitted or partial, {omitted_owners} AST scopes have no indexed owner, and {clipped_owners} owner projections reached source/field limits (8192 bytes and 512 terms per field). Narrow the build scope to reduce retained evidence" + )); + } + Ok(Some(result)) + } + pub fn owner(&self, id: &str) -> Option<&SourceQueryOwner> { + self.by_id.get(id).and_then(|at| self.owners.get(*at)) + } + pub fn candidates<'a>(&'a self, term: &str, limit: usize) -> (Vec<&'a SourceQueryOwner>, bool) { + let Some(posting) = self.postings.get(term) else { + return (Vec::new(), false); + }; + ( + posting + .iter() + .take(limit) + .filter_map(|at| self.owners.get(*at)) + .collect(), + posting.len() > limit, + ) + } +} +pub(crate) fn attach( + mut engine: CodeQueryEngine, + graph: &Path, + cache: &Path, +) -> Result { + engine.source_query = Index::open(graph, &engine.graph_identity)?; + if let Some(index) = &engine.source_query { + engine.response_cache = crate::response_cache::ResponseCache::open( + cache, + &format!("{}:{}", engine.graph_identity, index.digest), + engine.program.as_ref(), + ); + } + Ok(engine) +} +pub(crate) fn matching_sources( + owner: &SourceQueryOwner, + terms: &BTreeSet, +) -> (BTreeSet, bool) { + let mut sources = BTreeSet::new(); + let mut bounded = false; + for site in owner.identifiers.iter().chain(&owner.comments) { + if compass_model::search::identifier_search_terms(&site.text) + .into_iter() + .map(compass_model::canonical_code_token) + .any(|t| terms.contains(&t)) + { + if sources.len() >= 8 { + bounded = true; + break; + } + sources.insert(site.source.clone()); + } + } + (sources, bounded) +} diff --git a/crates/compass-query/tests/broad_retrieval.rs b/crates/compass-query/tests/broad_retrieval.rs new file mode 100644 index 000000000..386ad6c3d --- /dev/null +++ b/crates/compass-query/tests/broad_retrieval.rs @@ -0,0 +1,2156 @@ +mod support; + +use compass_graph::GraphSnapshotBuilder; +use compass_model::code_graph::{GraphDocument, NodeDetails, NodeKind}; +use compass_model::query_contract::{ + CodeQueryLimits, DiscoveryDirection, DiscoveryLimits, DiscoveryQueryRequest, + DiscoveryTraversal, QueryDiagnosticCode, SearchRequest, +}; +use compass_query::{open, open_with_store}; +use compass_store::SqliteStore; + +fn request(question: &str) -> DiscoveryQueryRequest { + DiscoveryQueryRequest { + question: question.to_owned(), + direction: DiscoveryDirection::Auto, + relation_contexts: Vec::new(), + scope: Vec::new(), + traversal: DiscoveryTraversal::Bfs, + include_heuristic: false, + limits: DiscoveryLimits::default(), + } +} + +fn fixture(path: &std::path::Path) -> Result> { + support::write_graph(path)?; + let mut graph = GraphDocument::load(path)?; + graph.nodes.clear(); + graph.links.clear(); + for at in 0..6 { + let mut node = support::node( + &format!("z:formatter:{at}"), + NodeKind::Function, + &format!("format_message_{at}"), + &format!("protocol::format_message_{at}"), + ); + node.details = Some(serde_json::from_value::( + serde_json::json!({"type":"symbol","data":{"signature":"fn(errors: ProtocolDiagnostics) -> String"}}), + )?); + graph.nodes.push(node); + } + // Common-word parameter leaves must not displace implementation evidence. + for at in 0..600 { + graph.nodes.push(support::node( + &format!("a:noise:{at:04}"), + NodeKind::Parameter, + "errors", + "unrelated::errors", + )); + } + Ok(graph) +} + +#[test] +fn unindexed_collection_words_require_literal_context_and_stay_tentative() +-> Result<(), Box> { + use compass_query::RelevanceBand; + let directory = tempfile::tempdir()?; + let path = directory.path().join("graph.json"); + support::write_graph(&path)?; + let mut graph = GraphDocument::load(&path)?; + graph.nodes.clear(); + graph.links.clear(); + for (id, name, qualified, signature) in [ + ( + "target", + "copy_items", + "fixture::copy_items", + "fn(cache: Cache) -> Map", + ), + ( + "namespace-only", + "clone_items", + "cache::clone_items", + "fn(widget: Widget)", + ), + ] { + let mut node = support::node(id, NodeKind::Function, name, qualified); + node.details = Some(serde_json::from_value(serde_json::json!({ + "type":"symbol", "data":{"signature":signature} + }))?); + graph.nodes.push(node); + } + for index in 0..7 { + let mut node = support::node( + &format!("noise:{index}"), + NodeKind::Function, + "new_cache", + "fixture::new_cache", + ); + node.details = Some(serde_json::from_value(serde_json::json!({ + "type":"symbol", "data":{"signature":"fn(cache: Cache, only: Mode, containing: Flags)"} + }))?); + graph.nodes.push(node); + } + std::fs::write(&path, serde_json::to_vec(&graph)?)?; + let json = open(&path, None, &directory.path().join("json-cache"))?; + let store = SqliteStore::open(directory.path().join("store.sqlite3"))?; + let prepared = GraphSnapshotBuilder::new().prepare(&store, &graph)?; + GraphSnapshotBuilder::new().activate(&store, &prepared)?; + let stored = open_with_store(&store, &path, None, &directory.path().join("store-cache"))?; + let question = request("Where is a snapshot made containing only live cache entries?"); + let answer = json.discover_ranked(question.clone())?; + assert_eq!( + answer.candidates, + stored.discover_ranked(question)?.candidates + ); + let first = answer + .candidates + .first() + .ok_or("missing approximate candidate")?; + assert_eq!(first.node.id, "target", "{answer:?}"); + assert_eq!(first.relevance, RelevanceBand::Low); + assert!(first.matched_terms.iter().any(|term| term == "entry")); + assert!(first.matched_terms.iter().any(|term| term == "snapshot")); + assert!( + first + .reasons + .iter() + .any(|reason| reason.contains("Vocabulary fallback")) + ); + assert!( + !answer + .candidates + .iter() + .any(|candidate| candidate.node.id == "namespace-only") + ); + for unsupported in [ + "Where are rocket snapshots made?", + "Where are `entries` copied?", + "Where is cache.snapshot made?", + ] { + assert!( + json.discover_ranked(request(unsupported))? + .candidates + .is_empty(), + "{unsupported}" + ); + } + // A bare existing ID remains authoritative even if its spelling is an + // otherwise unindexed vocabulary word. + graph.nodes[0].id = "snapshot".to_owned(); + std::fs::write(&path, serde_json::to_vec(&graph)?)?; + let identified = open(&path, None, &directory.path().join("identity-cache"))?; + let exact = identified.discover_ranked(request("snapshot"))?; + assert_eq!(exact.candidates.len(), 1); + assert_eq!(exact.candidates[0].node.id, "snapshot"); + assert!( + !exact.candidates[0] + .reasons + .iter() + .any(|reason| reason.contains("Vocabulary fallback")) + ); + + // A new alias-supported concept cannot evict an otherwise eligible + // literal inspection lead by changing the coverage denominator. + graph.nodes.push(support::node( + "literal-inspection", + NodeKind::Function, + "cache", + "fixture::cache", + )); + graph.nodes.push(support::node( + "reset-widget", + NodeKind::Function, + "reset_widget", + "fixture::reset_widget", + )); + std::fs::write(&path, serde_json::to_vec(&graph)?)?; + let admission = open(&path, None, &directory.path().join("admission-cache"))?; + let trace = admission.trace_ranked( + request("Where are cache snapshots reset?"), + "literal-inspection", + )?; + let details = trace + .target_details + .ok_or("missing literal inspection trace")?; + assert!(details.supported_coverage < 0.4); + assert_eq!(details.filter_reason, None); + + // Once the corpus actually contains the literal word, it is not replaced + // by recall alternatives, even on a different candidate. + graph.nodes.push(support::node( + "literal-snapshot", + NodeKind::Function, + "snapshot", + "fixture::snapshot", + )); + std::fs::write(&path, serde_json::to_vec(&graph)?)?; + let literal = open(&path, None, &directory.path().join("literal-cache"))?; + let trace = literal.trace_ranked(request("Where is a cache snapshot made?"), "snapshot")?; + let details = trace.target_details.ok_or("missing literal-word trace")?; + let concept = details + .terms + .iter() + .find(|term| term.term == "snapshot") + .ok_or("missing snapshot concept")?; + assert!( + !concept + .variants + .iter() + .any(|term| matches!(term.as_str(), "copy" | "clone")) + ); + Ok(()) +} + +#[test] +fn implementation_operation_evidence_precedes_signature_mentions() +-> Result<(), Box> { + use compass_model::code_graph::EdgeKind; + use compass_query::RelevanceBand; + for (operation, inflected) in [ + ("encode", "encoded"), + ("decode", "decoded"), + ("parse", "parsed"), + ] { + let directory = tempfile::tempdir()?; + let path = directory.path().join("graph.json"); + support::write_graph(&path)?; + let mut graph = GraphDocument::load(&path)?; + let template = graph.links.first().cloned().ok_or("missing edge")?; + graph.nodes.clear(); + graph.links.clear(); + for index in 0..7 { + let mut node = support::node( + &format!("a:context:{index}"), + NodeKind::Function, + "new_map", + "fixture::new_map", + ); + node.details = Some(serde_json::from_value(serde_json::json!({ + "type":"symbol", "data":{"signature":format!("fn(items: ItemMap, {operation}: Codec)")} + }))?); + graph.nodes.push(node); + } + for (id, signature) in [ + ("z:implementation", "fn(items: Items)"), + ("wrong-subject", "fn(widget: Widget)"), + ("clipped-signature", "fn(items: Items, …"), + ] { + let mut node = support::node(id, NodeKind::Function, "opaque", "fixture::opaque"); + node.details = Some(serde_json::from_value(serde_json::json!({ + "type":"symbol", "data":{"signature":signature} + }))?); + graph.nodes.push(node); + let mut edge = template.clone(); + edge.source = id.to_owned(); + edge.target = "operation".to_owned(); + edge.kind = EdgeKind::Calls; + edge.id = compass_model::identity::edge_id( + &edge.source, + edge.kind, + &edge.target, + edge.relationship_site.as_ref(), + None, + ); + edge.key = edge.id.clone(); + graph.links.push(edge); + } + graph.nodes.push(support::node( + "operation", + NodeKind::Function, + operation, + &format!("codec::{operation}"), + )); + std::fs::write(&path, serde_json::to_vec(&graph)?)?; + let json = open(&path, None, &directory.path().join("json-cache"))?; + let store = SqliteStore::open(directory.path().join("store.sqlite3"))?; + let prepared = GraphSnapshotBuilder::new().prepare(&store, &graph)?; + GraphSnapshotBuilder::new().activate(&store, &prepared)?; + let stored = open_with_store(&store, &path, None, &directory.path().join("store-cache"))?; + let question = request(&format!("Where are items {inflected} into a map?")); + let answer = json.discover_ranked(question.clone())?; + assert_eq!( + answer.candidates, + stored.discover_ranked(question)?.candidates + ); + assert_eq!( + answer + .candidates + .first() + .map(|candidate| candidate.node.id.as_str()), + Some("z:implementation"), + "{operation}: {answer:?}" + ); + assert_eq!(answer.candidates[0].relevance, RelevanceBand::Low); + let clipped = answer + .candidates + .iter() + .find(|row| row.node.id == "clipped-signature") + .ok_or("clipped candidate missing")?; + assert!( + clipped + .reasons + .iter() + .all(|reason| !reason.contains("inspection preference")) + ); + assert!( + !answer + .candidates + .iter() + .any(|candidate| candidate.node.id == "wrong-subject") + ); + // An explicit identity remains authoritative even for a contextual mention. + assert_eq!( + json.discover_ranked(request("a:context:0"))?.candidates[0] + .node + .id, + "a:context:0" + ); + assert!( + json.discover_ranked(request("Where are `MissingItems` encoded?"))? + .candidates + .is_empty() + ); + let second = if operation == "encode" { + "decoded" + } else { + "encoded" + }; + let compound = json.discover_ranked(request(&format!( + "Where are items {inflected} and {second} into a map?" + )))?; + assert!(compound.candidates.iter().all(|candidate| { + candidate + .reasons + .iter() + .all(|reason| !reason.contains("inspection preference")) + })); + for condition in ["without decoration", "before midnight"] { + let qualified = json.discover_ranked(request(&format!( + "Where are items {inflected} {condition}?" + )))?; + assert!(!qualified.candidates.is_empty()); + assert!(qualified.candidates.iter().all(|candidate| { + candidate.relevance == RelevanceBand::Low + && candidate + .reasons + .iter() + .all(|reason| !reason.contains("inspection preference")) + })); + } + } + Ok(()) +} + +#[test] +fn ordinary_plural_acronyms_do_not_become_missing_symbol_names() +-> Result<(), Box> { + let directory = tempfile::tempdir()?; + let path = directory.path().join("graph.json"); + let mut graph = fixture(&path)?; + graph.nodes.clear(); + for (id, name) in [ + ("ttl", "convert_ttl"), + ("api", "register_api"), + ("url", "parse_url"), + ] { + graph + .nodes + .push(support::node(id, NodeKind::Function, name, name)); + } + std::fs::write(&path, serde_json::to_vec(&graph)?)?; + let engine = open(&path, None, &directory.path().join("cache"))?; + for (plural, singular, expected) in [ + ( + "Where are TTLs converted?", + "Where is ttl converted?", + "ttl", + ), + ( + "Where are APIs registered?", + "Where is api registered?", + "api", + ), + ("Where are URLs parsed?", "Where is url parsed?", "url"), + ] { + let answer = engine.discover_ranked(request(plural))?; + let singular_answer = engine.discover_ranked(request(singular))?; + assert_eq!(answer.candidates, singular_answer.candidates, "{plural}"); + assert_eq!( + answer.candidates.first().map(|row| row.node.id.as_str()), + Some(expected) + ); + } + // Explicitly quoted names still require that exact identity, even when + // their spelling resembles an ordinary plural acronym. + let answer = engine.discover_ranked(request("Where are `TTLs` converted?"))?; + assert!(answer.candidates.is_empty()); + Ok(()) +} + +#[test] +fn broad_method_recall_requires_witnessed_local_reference_not_target_namespace() +-> Result<(), Box> { + use compass_model::code_graph::EdgeKind; + use compass_model::provenance::{EvidenceConfidence, EvidenceOrigin, SourceAnchor}; + use sha2::{Digest, Sha256}; + let directory = tempfile::tempdir()?; + let path = directory.path().join("graph.json"); + support::write_graph(&path)?; + let mut graph = GraphDocument::load(&path)?; + let template = graph.links.first().cloned().ok_or("missing edge fixture")?; + graph.nodes.clear(); + graph.links.clear(); + std::fs::write(directory.path().join("src/lib.rs"), b"codecode")?; + let file = graph + .graph + .files + .iter_mut() + .find(|file| file.path == "src/lib.rs") + .ok_or("missing source file")?; + file.byte_size = 8; + file.content_digest = format!("sha256:{:x}", Sha256::digest(b"codecode")); + graph.nodes.push(support::node( + "listeners", + NodeKind::Property, + "abortListeners", + "queue.abortListeners", + )); + graph.nodes.push(support::node( + "namespace-only", + NodeKind::Property, + "checkpoint", + "abortListeners.checkpoint", + )); + for (id, kind) in [ + ("good-reference", EdgeKind::References), + ("good-call", EdgeKind::Calls), + ("wrong-kind", EdgeKind::Renders), + ("no-ast-proof", EdgeKind::References), + ("inferred-proof", EdgeKind::References), + ("outside-owner", EdgeKind::References), + ("wrong-direction", EdgeKind::References), + ("namespace-borrower", EdgeKind::References), + ("no-ast-owner-proof", EdgeKind::References), + ] { + let mut owner = support::node( + id, + NodeKind::Method, + "process", + &format!("queue.{id}.process"), + ); + if id == "no-ast-owner-proof" { + for proof in &mut owner.evidence { + proof.origin = EvidenceOrigin::Config; + } + } + graph.nodes.push(owner); + let mut edge = template.clone(); + edge.source = id.to_owned(); + edge.target = if id == "namespace-borrower" { + "namespace-only" + } else { + "listeners" + } + .to_owned(); + edge.kind = kind; + match id { + "no-ast-proof" => { + for proof in &mut edge.evidence { + proof.origin = EvidenceOrigin::Config; + } + } + "inferred-proof" => { + for proof in &mut edge.evidence { + proof.confidence = EvidenceConfidence::Inferred; + } + } + "outside-owner" => { + let site = SourceAnchor { + start_byte: 4, + end_byte: 8, + start_column: 4, + end_column: 8, + ..edge.relationship_site.clone().ok_or("missing site")? + }; + edge.relationship_site = Some(site.clone()); + for proof in &mut edge.evidence { + proof.anchors = vec![site.clone()]; + } + } + "wrong-direction" => std::mem::swap(&mut edge.source, &mut edge.target), + _ => {} + } + edge.id = compass_model::identity::edge_id( + &edge.source, + edge.kind, + &edge.target, + edge.relationship_site.as_ref(), + edge.occurrence_rule.as_ref().map(|rule| rule.as_str()), + ); + edge.key = edge.id.clone(); + graph.links.push(edge); + } + std::fs::write(&path, serde_json::to_vec(&graph)?)?; + let json = open(&path, None, &directory.path().join("json-cache"))?; + let store = SqliteStore::open(directory.path().join("store.sqlite3"))?; + let prepared = GraphSnapshotBuilder::new().prepare(&store, &graph)?; + GraphSnapshotBuilder::new().activate(&store, &prepared)?; + let stored = open_with_store(&store, &path, None, &directory.path().join("store-cache"))?; + let question = request("where are abort listeners removed"); + let reference = json.discover_ranked(question.clone())?; + assert!( + reference + .candidates + .first() + .is_some_and(|candidate| matches!( + candidate.node.id.as_str(), + "good-reference" | "good-call" + )) + ); + assert_eq!( + reference.candidates, + stored.discover_ranked(question.clone())?.candidates + ); + graph.nodes.reverse(); + graph.links.reverse(); + let reordered_path = directory.path().join("reordered.json"); + std::fs::write(&reordered_path, serde_json::to_vec(&graph)?)?; + let reordered = open( + &reordered_path, + None, + &directory.path().join("reordered-cache"), + )?; + assert_eq!( + reference.candidates, + reordered.discover_ranked(question.clone())?.candidates + ); + for id in ["good-reference", "good-call"] { + let candidate = reference + .candidates + .iter() + .find(|candidate| candidate.node.id == id) + .ok_or_else(|| { + format!( + "method {id} not recalled; candidates={:?}; trace={:?}", + reference.candidates, + json.trace_ranked(question.clone(), id) + ) + })?; + assert_eq!(candidate.relevance, compass_query::RelevanceBand::Low); + assert!( + candidate + .matched_fields + .iter() + .any(|field| field == "code_reference") + ); + assert_eq!(candidate.supporting_sources.len(), 1); + assert!( + candidate + .reasons + .iter() + .any(|reason| reason.contains("not confirmation")) + ); + } + for id in [ + "wrong-kind", + "no-ast-proof", + "inferred-proof", + "outside-owner", + "wrong-direction", + "namespace-borrower", + "no-ast-owner-proof", + ] { + assert!( + !reference + .candidates + .iter() + .any(|candidate| candidate.node.id == id), + "{id}" + ); + } + let search = json.search(SearchRequest { + query: question.question, + limits: CodeQueryLimits::default(), + })?; + assert!( + search + .edges + .iter() + .any(|edge| edge.kind == EdgeKind::References && edge.source == "good-reference") + ); + assert!( + search + .edges + .iter() + .any(|edge| edge.kind == EdgeKind::Calls && edge.source == "good-call") + ); + Ok(()) +} + +#[test] +fn dense_relationship_recall_reports_bounds_and_keeps_shared_work_ceilings() +-> Result<(), Box> { + use compass_model::code_graph::EdgeKind; + use compass_model::query_contract::{ + MAX_INDEXED_CANDIDATE_NODES_READ, MAX_INDEXED_CANDIDATE_PROBES, + }; + let directory = tempfile::tempdir()?; + let path = directory.path().join("graph.json"); + support::write_graph(&path)?; + let mut graph = GraphDocument::load(&path)?; + let template = graph.links.first().cloned().ok_or("missing edge")?; + graph.nodes.clear(); + graph.links.clear(); + for at in 0..40 { + graph.nodes.push(support::node( + &format!("target:{at}"), + NodeKind::Property, + "abortListeners", + &format!("queue{at}.abortListeners"), + )); + } + for at in 0..300 { + let id = format!("owner:{at:04}"); + graph.nodes.push(support::node( + &id, + NodeKind::Method, + "process", + &format!("queue{at}.process"), + )); + for target in 0..40 { + let mut edge = template.clone(); + edge.source = id.clone(); + edge.target = format!("target:{target}"); + edge.kind = EdgeKind::References; + edge.id = compass_model::identity::edge_id( + &edge.source, + edge.kind, + &edge.target, + edge.relationship_site.as_ref(), + None, + ); + edge.key = edge.id.clone(); + graph.links.push(edge); + } + } + std::fs::write(&path, serde_json::to_vec(&graph)?)?; + let json = open(&path, None, &directory.path().join("json-cache"))?; + let store = SqliteStore::open(directory.path().join("store.sqlite3"))?; + let prepared = GraphSnapshotBuilder::new().prepare(&store, &graph)?; + GraphSnapshotBuilder::new().activate(&store, &prepared)?; + let stored = open_with_store(&store, &path, None, &directory.path().join("store-cache"))?; + let question = request("where are abort listeners removed"); + let reference = json.discover_ranked(question.clone())?; + for engine in [&json, &stored] { + let answer = engine.discover_ranked(question.clone())?; + assert!(answer.bounded && answer.eligible_count.is_none()); + assert!(answer.discovery.stats.candidate_nodes <= MAX_INDEXED_CANDIDATE_NODES_READ); + assert!(answer.discovery.stats.candidate_probes <= MAX_INDEXED_CANDIDATE_PROBES); + assert!( + answer + .discovery + .diagnostics + .iter() + .any(|diagnostic| diagnostic.code == QueryDiagnosticCode::BoundedTruncation) + ); + assert_eq!(answer.candidates, reference.candidates); + assert!( + answer + .candidates + .iter() + .any(|candidate| candidate.node.id.starts_with("owner:")) + ); + } + Ok(()) +} + +#[test] +fn broad_signature_recall_precedes_common_term_caps_and_preserves_rank_four_and_five() +-> Result<(), Box> { + let directory = tempfile::tempdir()?; + let path = directory.path().join("graph.json"); + let graph = fixture(&path)?; + std::fs::write(&path, serde_json::to_vec(&graph)?)?; + let engine = open(&path, None, &directory.path().join("cache"))?; + let query = request("where are protocol errors formatted"); + let ranked = engine.discover_ranked(query.clone())?; + assert_eq!(ranked.discovery.seeds.len(), 3); + assert!(ranked.candidates.len() >= 6); + for candidate in ranked.candidates.iter().take(6) { + assert!(candidate.node.id.starts_with("z:formatter:")); + assert!( + candidate + .matched_fields + .iter() + .any(|field| field == "signature") + ); + assert!( + !candidate + .matched_fields + .iter() + .any(|field| field == "exact_name") + ); + } + let trace = engine.trace_ranked(query.clone(), "z:formatter:4")?; + assert!(trace.graph_present && trace.admitted && trace.first_page); + assert_eq!(trace.rank, Some(5)); + assert!(!trace.seed); + assert_eq!(trace.schema, "compass.query.retrieval-trace/1"); + let details = trace + .target_details + .as_ref() + .ok_or("missing target trace")?; + assert_eq!(details.filter_reason, None); + assert_eq!(details.score, Some(ranked.candidates[4].score)); + assert!(details.supported_coverage >= 0.4); + assert!( + details + .terms + .iter() + .any(|term| { term.matched_fields.iter().any(|field| field == "signature") }) + ); + // The diagnostic observes the same ranking and does not alter later calls. + assert_eq!( + ranked.candidates, + engine.discover_ranked(query.clone())?.candidates + ); + let default = engine.discover(query.clone())?; + assert_eq!(default.seeds[0].node_id, ranked.candidates[0].node.id); + let search = engine.search(SearchRequest { + query: query.question, + limits: CodeQueryLimits::default(), + })?; + assert_eq!(search.results[0].node_id, ranked.candidates[0].node.id); + Ok(()) +} + +#[test] +fn target_trace_names_the_supported_coverage_filter() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let path = directory.path().join("graph.json"); + let mut graph = fixture(&path)?; + graph.nodes = vec![ + support::node( + "target", + NodeKind::Function, + "retain_payload", + "retain_payload", + ), + support::node( + "other", + NodeKind::Function, + "compress_encrypt_persist", + "compress_encrypt_persist", + ), + ]; + std::fs::write(&path, serde_json::to_vec(&graph)?)?; + let engine = open(&path, None, &directory.path().join("cache"))?; + let query = request("where is payload compressed encrypted persisted"); + let before = engine.discover_ranked(query.clone())?; + let trace = engine.trace_ranked(query.clone(), "target")?; + assert!(trace.graph_present && trace.admitted); + assert_eq!(trace.rank, None); + let details = trace.target_details.ok_or("missing target trace")?; + assert_eq!( + details.filter_reason, + Some(compass_query::RankedFilterReason::InsufficientSupportedCoverage) + ); + assert!(details.supported_coverage < 0.4); + assert_eq!(details.score, None); + assert!( + details + .terms + .iter() + .any(|term| !term.matched_fields.is_empty()) + ); + assert!( + details + .terms + .iter() + .any(|term| term.matched_fields.is_empty()) + ); + assert_eq!(before.candidates, engine.discover_ranked(query)?.candidates); + Ok(()) +} + +#[test] +fn unsupported_subjects_do_not_borrow_generic_operation_evidence() +-> Result<(), Box> { + let directory = tempfile::tempdir()?; + let path = directory.path().join("graph.json"); + let mut graph = fixture(&path)?; + graph.nodes.push(support::node( + "run", + NodeKind::Function, + "run", + "server::run", + )); + std::fs::write(&path, serde_json::to_vec(&graph)?)?; + let engine = open(&path, None, &directory.path().join("cache"))?; + for question in [ + "where does the built-in Redis cluster failover run", + "where does the built-in redis cluster failover run", + "where does the built-in REDIS cluster failover run", + "where does the built-in `redis` cluster failover run", + "where does the built-in redis_cluster failover run", + ] { + let answer = engine.discover_ranked(request(question))?; + assert!(answer.candidates.is_empty(), "{question}"); + assert!(answer.discovery.seeds.is_empty(), "{question}"); + assert!( + answer + .discovery + .diagnostics + .iter() + .any(|d| d.code == QueryDiagnosticCode::NoMatch + && d.message.contains("supporting indexed evidence")) + ); + } + Ok(()) +} + +#[test] +fn unfamiliar_ordinary_context_returns_tentative_source_candidates() +-> Result<(), Box> { + use compass_query::RelevanceBand; + let directory = tempfile::tempdir()?; + let path = directory.path().join("graph.json"); + std::fs::write(&path, serde_json::to_vec(&fixture(&path)?)?)?; + let engine = open(&path, None, &directory.path().join("cache"))?; + for question in [ + "where are protocol errors elegantly formatted", + "where are redis protocol errors formatted", + "where are protocol errors elegantly decorated before presenting readable messages", + ] { + let answer = engine.discover_ranked(request(question))?; + assert!(!answer.candidates.is_empty(), "{question}"); + let default = engine.discover(request(question))?; + let search = engine.search(SearchRequest { + query: question.to_owned(), + limits: CodeQueryLimits::default(), + })?; + for diagnostics in [ + &answer.discovery.diagnostics, + &default.diagnostics, + &search.diagnostics, + ] { + assert!(diagnostics.iter().any(|diagnostic| { + diagnostic.code == QueryDiagnosticCode::IncompleteCoverage + && diagnostic.message.contains("inspection candidates") + && diagnostic.message.contains("not established") + })); + } + for candidate in &answer.candidates { + assert!(candidate.node.id.starts_with("z:formatter:")); + assert_eq!(candidate.relevance, RelevanceBand::Low); + assert!( + candidate + .reasons + .iter() + .any(|reason| reason.contains("not established")) + ); + } + } + for question in [ + "where are Redis protocol errors formatted", + "where are `redis` protocol errors formatted", + "where are redis_cache protocol errors formatted", + "where are redis.client protocol errors formatted", + "where are redis::client protocol errors formatted", + "where are redis/client.py protocol errors formatted", + ] { + assert!( + engine + .discover_ranked(request(question))? + .candidates + .is_empty() + ); + } + Ok(()) +} + +#[test] +fn explicit_composite_identity_cannot_be_assembled_from_separate_name_parts() +-> Result<(), Box> { + use compass_query::RelevanceBand; + let directory = tempfile::tempdir()?; + let path = directory.path().join("graph.json"); + let mut graph = fixture(&path)?; + let mut correct = graph.nodes.first().cloned().ok_or("missing formatter")?; + correct.id = "correct".into(); + correct.name = "format_errors".into(); + correct.qualified_name = "redis::client::format_errors".into(); + let mut wrong = correct.clone(); + wrong.id = "wrong-owner".into(); + wrong.qualified_name = "client::redis::format_errors".into(); + graph.nodes.extend([correct, wrong]); + std::fs::write(&path, serde_json::to_vec(&graph)?)?; + let engine = open(&path, None, &directory.path().join("cache"))?; + for qualified in ["redis::client", "redis.client"] { + let answer = engine.discover_ranked(request(&format!( + "where are {qualified} protocol errors formatted" + )))?; + let correct = answer + .candidates + .iter() + .find(|candidate| candidate.node.id == "correct") + .ok_or("matching owner missing")?; + assert_eq!(correct.relevance, RelevanceBand::High); + let wrong = answer + .candidates + .iter() + .find(|candidate| candidate.node.id == "wrong-owner") + .ok_or("wrong-owner inspection pointer missing")?; + assert_eq!(wrong.relevance, RelevanceBand::Low); + assert!(wrong.reasons.iter().any( + |reason| reason.contains("explicit identity") && reason.contains("not established") + )); + } + let absent = + engine.discover_ranked(request("where are redis_client protocol errors formatted"))?; + assert!(!absent.candidates.is_empty()); + assert!( + absent + .candidates + .iter() + .all(|candidate| candidate.relevance == RelevanceBand::Low) + ); + Ok(()) +} + +#[test] +fn plural_subjects_recall_singular_identifiers_without_changing_strict_lookup() +-> Result<(), Box> { + let directory = tempfile::tempdir()?; + let path = directory.path().join("graph.json"); + let mut graph = fixture(&path)?; + graph.nodes.push(support::node( + "cookie", + NodeKind::Function, + "store_cookie", + "session::store_cookie", + )); + std::fs::write(&path, serde_json::to_vec(&graph)?)?; + let engine = open(&path, None, &directory.path().join("cache"))?; + let answer = engine.discover_ranked(request("where are cookies stored"))?; + assert_eq!( + answer + .candidates + .first() + .map(|candidate| candidate.node.id.as_str()), + Some("cookie") + ); + let exact = engine.search_exact( + SearchRequest { + query: "cookies".into(), + limits: CodeQueryLimits::default(), + }, + Default::default(), + )?; + assert!(exact.nodes.is_empty()); + Ok(()) +} + +#[test] +fn a_subject_elsewhere_cannot_establish_a_candidates_missing_context() +-> Result<(), Box> { + use compass_query::RelevanceBand; + let directory = tempfile::tempdir()?; + let path = directory.path().join("graph.json"); + let mut graph = fixture(&path)?; + graph.nodes.push(support::node( + "redis-unrelated", + NodeKind::Function, + "redis_ping", + "cache::redis_ping", + )); + let mut target = graph + .nodes + .first() + .cloned() + .ok_or("formatter fixture missing")?; + target.id = "redis-formatter".into(); + target.qualified_name = "redis::format_errors".into(); + target.name = "format_errors".into(); + graph.nodes.push(target); + std::fs::write(&path, serde_json::to_vec(&graph)?)?; + let engine = open(&path, None, &directory.path().join("cache"))?; + let answer = engine.discover_ranked(request("where are Redis protocol errors formatted"))?; + assert_eq!( + answer + .candidates + .first() + .map(|candidate| candidate.node.id.as_str()), + Some("redis-formatter") + ); + assert_eq!(answer.candidates[0].relevance, RelevanceBand::High); + for generic in answer + .candidates + .iter() + .filter(|candidate| candidate.node.id.starts_with("z:formatter:")) + { + assert_eq!(generic.relevance, RelevanceBand::Low); + assert!( + generic + .reasons + .iter() + .any(|reason| reason.contains("redis") && reason.contains("not established")) + ); + } + Ok(()) +} + +#[test] +fn documentation_ownership_requires_ast_evidence_and_byte_enclosure() +-> Result<(), Box> { + use compass_model::code_graph::{EdgeKind, ResourceKind, ResourceNodeDetails}; + use compass_model::provenance::EvidenceOrigin; + for case in [ + "owned", + "owned_class", + "owned_struct", + "empty", + "config", + "outside_bytes", + ] { + let directory = tempfile::tempdir()?; + let path = directory.path().join("graph.json"); + support::write_graph(&path)?; + let mut graph = GraphDocument::load(&path)?; + let mut edge = graph.links.first().cloned().ok_or("fixture has no edge")?; + let mut owner = support::node( + "owner", + match case { + "owned_class" => NodeKind::Class, + "owned_struct" => NodeKind::Struct, + _ => NodeKind::Function, + }, + "opaque", + "implementation::opaque", + ); + if case == "outside_bytes" { + let source = owner.source.as_mut().ok_or("missing source")?; + source.start_byte = 1; + source.start_column = 1; + } + let mut prose = support::node("prose", NodeKind::Resource, "Notes", "Notes"); + prose.details = Some(NodeDetails::Resource(ResourceNodeDetails { + resource_kind: ResourceKind::Rationale, + uri: None, + media_type: None, + content: Some("protocol diagnostics parse".into()), + })); + let source = prose.source.as_mut().ok_or("missing prose source")?; + source.end_byte = 1; + source.end_column = 1; + edge.source = prose.id.clone(); + edge.target = owner.id.clone(); + edge.kind = EdgeKind::Documents; + edge.id = compass_model::identity::edge_id( + &edge.source, + edge.kind, + &edge.target, + edge.relationship_site.as_ref(), + None, + ); + edge.key = edge.id.clone(); + if case == "empty" { + edge.evidence.clear(); + } + if case == "config" { + for evidence in &mut edge.evidence { + evidence.origin = EvidenceOrigin::Config; + } + } + graph.nodes = vec![owner, prose]; + graph.links = vec![edge]; + std::fs::write(&path, serde_json::to_vec(&graph)?)?; + if case == "empty" { + assert!( + open(&path, None, &directory.path().join("cache")).is_err(), + "provenance-free relationships must fail graph validation" + ); + continue; + } + let engine = open(&path, None, &directory.path().join("cache"))?; + let answer = engine.discover_ranked(request("where are protocol diagnostics parsed"))?; + let owner = answer + .candidates + .iter() + .find(|candidate| candidate.node.id == "owner"); + assert_eq!( + owner.is_some(), + case.starts_with("owned"), + "{case}: {answer:?}" + ); + if let Some(owner) = owner { + assert!( + owner.excerpts.is_empty(), + "enclosure alone does not establish verbatim raw retention" + ); + assert!( + owner + .matched_fields + .iter() + .any(|field| field == "owned_document") + ); + assert_eq!(owner.supporting_sources.len(), 1); + assert!( + !answer + .candidates + .iter() + .any(|candidate| candidate.node.id == "prose"), + "owned prose should be supporting evidence, not a duplicate candidate" + ); + } + } + Ok(()) +} + +#[test] +fn negation_is_not_silently_rewritten_into_positive_evidence() +-> Result<(), Box> { + let directory = tempfile::tempdir()?; + let path = directory.path().join("graph.json"); + let graph = fixture(&path)?; + std::fs::write(&path, serde_json::to_vec(&graph)?)?; + let engine = open(&path, None, &directory.path().join("cache"))?; + for question in [ + "where are protocol errors not formatted", + "WHERE are protocol errors NOT formatted", + "where are protocol errors formatted without decoration", + ] { + let answer = engine.discover_ranked(request(question))?; + assert!(!answer.candidates.is_empty()); + assert!(answer.discovery.diagnostics.iter().any(|diagnostic| { + diagnostic.code == QueryDiagnosticCode::IncompleteCoverage + && diagnostic.message.contains("Negative qualification") + })); + for candidate in answer.candidates { + assert_eq!(candidate.relevance, compass_query::RelevanceBand::Low); + assert!(candidate.reasons.iter().any(|reason| { + reason.contains("Negative qualification") && reason.contains("not established") + })); + } + } + Ok(()) +} + +#[test] +fn temporal_qualifications_remain_visible_without_becoming_lexical_subjects() +-> Result<(), Box> { + let directory = tempfile::tempdir()?; + let path = directory.path().join("graph.json"); + let mut graph = fixture(&path)?; + graph.nodes = vec![ + support::node( + "parser", + NodeKind::Function, + "parse_payload", + "parse_payload", + ), + support::node( + "noise", + NodeKind::Function, + "before_after_during_until", + "before_after_during_until", + ), + ]; + std::fs::write(&path, serde_json::to_vec(&graph)?)?; + let json = open(&path, None, &directory.path().join("json-cache"))?; + let store = SqliteStore::open(directory.path().join("store.sqlite3"))?; + let prepared = GraphSnapshotBuilder::new().prepare(&store, &graph)?; + GraphSnapshotBuilder::new().activate(&store, &prepared)?; + let stored = open_with_store(&store, &path, None, &directory.path().join("store-cache"))?; + for condition in ["before", "after", "during", "until", "BEFORE"] { + let question = format!("where are payloads parsed {condition}"); + let answer = json.discover_ranked(request(&question))?; + assert_eq!(answer.discovery.question, question); + assert_eq!( + answer.candidates, + stored.discover_ranked(request(&question))?.candidates + ); + let target = answer.candidates.first().ok_or("missing payload parser")?; + assert_eq!(target.node.id, "parser"); + assert_eq!(target.relevance, compass_query::RelevanceBand::Low); + assert!(target.reasons.iter().any(|reason| { + reason.contains("Temporal qualification") && reason.contains("not established") + })); + assert!(answer.discovery.diagnostics.iter().any(|diagnostic| { + diagnostic.code == QueryDiagnosticCode::IncompleteCoverage + && diagnostic.message.contains("Temporal qualification") + && diagnostic.message.contains(&condition.to_ascii_lowercase()) + })); + let trace = json.trace_ranked(request(&question), "parser")?; + let details = trace.target_details.ok_or("missing trace details")?; + assert!( + !details + .terms + .iter() + .any(|term| { ["before", "after", "dur", "until"].contains(&term.term.as_str()) }) + ); + } + for question in [ + "where are `before` helpers", + "where are \"during\" helpers", + "where are before_after helpers", + "where are src/before.ts helpers", + ] { + let answer = json.discover_ranked(request(question))?; + assert!( + !answer + .discovery + .diagnostics + .iter() + .any(|diagnostic| { diagnostic.message.contains("Temporal qualification") }), + "{question}" + ); + if question.contains('`') || question.contains('"') { + let trace = json.trace_ranked(request(question), "noise")?; + let details = trace + .target_details + .ok_or("missing quoted identity trace")?; + let expected = if question.contains("during") { + "dur" + } else { + "before" + }; + assert!( + details.terms.iter().any(|term| term.term == expected), + "{question}" + ); + } + } + Ok(()) +} + +#[test] +fn an_identifier_containing_without_is_not_a_negative_condition() +-> Result<(), Box> { + let directory = tempfile::tempdir()?; + let path = directory.path().join("graph.json"); + let mut graph = fixture(&path)?; + let node = graph.nodes.first_mut().ok_or("missing formatter")?; + node.name = "format_protocol_errors_without_cache".into(); + node.qualified_name = "protocol::format_protocol_errors_without_cache".into(); + let target = node.id.clone(); + std::fs::write(&path, serde_json::to_vec(&graph)?)?; + let engine = open(&path, None, &directory.path().join("cache"))?; + let answer = + engine.discover_ranked(request("where are protocol errors without_cache formatted"))?; + assert_eq!( + answer + .candidates + .first() + .map(|candidate| &candidate.node.id), + Some(&target) + ); + assert!( + !answer + .discovery + .diagnostics + .iter() + .any(|diagnostic| { diagnostic.message.contains("Negative qualification") }) + ); + Ok(()) +} + +#[test] +fn fielded_rank_and_reasons_are_identical_across_json_store_and_reordered_input() +-> Result<(), Box> { + let directory = tempfile::tempdir()?; + let path = directory.path().join("graph.json"); + let graph = fixture(&path)?; + std::fs::write(&path, serde_json::to_vec(&graph)?)?; + let json = open(&path, None, &directory.path().join("json-cache"))?; + let store = SqliteStore::open(directory.path().join("store.sqlite3"))?; + let prepared = GraphSnapshotBuilder::new().prepare(&store, &graph)?; + GraphSnapshotBuilder::new().activate(&store, &prepared)?; + let stored = open_with_store(&store, &path, None, &directory.path().join("store-cache"))?; + let mut reversed = graph; + reversed.nodes.reverse(); + std::fs::write(&path, serde_json::to_vec(&reversed)?)?; + let reordered = open(&path, None, &directory.path().join("reverse-cache"))?; + for question in [ + "where are protocol errors formatted", + "where are protocol errors elegantly formatted", + "where are protocol errors formatted without decoration", + "where are `redis` protocol errors formatted", + ] { + let query = request(question); + let expected = json.discover_ranked(query.clone())?; + let actual = stored.discover_ranked(query.clone())?; + assert_eq!(actual.candidates, expected.candidates, "{question}"); + assert_eq!( + actual.discovery.seeds, expected.discovery.seeds, + "{question}" + ); + assert_eq!(actual.bounded, expected.bounded, "{question}"); + assert_eq!( + actual.discovery.diagnostics, expected.discovery.diagnostics, + "{question}" + ); + assert_eq!( + reordered.discover_ranked(query)?.candidates, + expected.candidates, + "{question}" + ); + } + Ok(()) +} + +#[test] +fn cancellation_flag_false_preserves_broad_routing_and_true_is_typed() +-> Result<(), Box> { + use std::sync::atomic::AtomicBool; + let directory = tempfile::tempdir()?; + let path = directory.path().join("graph.json"); + let graph = fixture(&path)?; + std::fs::write(&path, serde_json::to_vec(&graph)?)?; + let engine = open(&path, None, &directory.path().join("cache"))?; + let question = request("where are protocol errors formatted"); + let ordinary = engine.discover_with_cancellation(question.clone(), None)?; + let unset = AtomicBool::new(false); + let cancelable = engine.discover_with_cancellation(question.clone(), Some(&unset))?; + assert_eq!(ordinary.seeds, cancelable.seeds); + assert_eq!(ordinary.nodes, cancelable.nodes); + assert_eq!(ordinary.edges, cancelable.edges); + assert_eq!(ordinary.diagnostics, cancelable.diagnostics); + let set = AtomicBool::new(true); + let result = engine.discover_with_cancellation(question, Some(&set)); + assert!( + matches!(result, Err(error) if error.kind() == compass_query::QueryErrorKind::Cancelled) + ); + Ok(()) +} + +#[test] +fn ranked_public_paths_share_an_already_expired_engine_deadline() +-> Result<(), Box> { + let directory = tempfile::tempdir()?; + let path = directory.path().join("graph.json"); + let graph = fixture(&path)?; + std::fs::write(&path, serde_json::to_vec(&graph)?)?; + let engine = open(&path, None, &directory.path().join("cache"))? + .with_deadline(std::time::Instant::now()); + let question = request("where are protocol errors formatted"); + for result in [ + engine.discover_ranked(question.clone()), + engine.discover_ranked_with_context(question.clone(), true), + engine.discover_ranked_with_context(question.clone(), false), + ] { + assert!( + matches!(result, Err(error) if error.kind() == compass_query::QueryErrorKind::Timeout) + ); + } + assert!( + matches!(engine.trace_ranked(question.clone(), "z:formatter:4"), Err(error) if error.kind() == compass_query::QueryErrorKind::Timeout) + ); + let unset = std::sync::atomic::AtomicBool::new(false); + assert!( + matches!(engine.discover_with_cancellation(question, Some(&unset)), Err(error) if error.kind() == compass_query::QueryErrorKind::Timeout) + ); + Ok(()) +} + +#[test] +fn ordinary_broad_discovery_uses_its_own_byte_budget_and_trims_coherently() +-> Result<(), Box> { + let directory = tempfile::tempdir()?; + let path = directory.path().join("graph.json"); + support::write_graph(&path)?; + let mut edge = GraphDocument::load(&path)? + .links + .into_iter() + .next() + .ok_or("edge fixture missing")?; + let mut graph = fixture(&path)?; + edge.source = "z:formatter:0".to_owned(); + edge.target = "z:formatter:5".to_owned(); + edge.kind = compass_model::code_graph::EdgeKind::Calls; + edge.id = compass_model::identity::edge_id( + &edge.source, + edge.kind, + &edge.target, + edge.relationship_site.as_ref(), + edge.occurrence_rule.as_ref().map(|rule| rule.as_str()), + ); + edge.key = edge.id.clone(); + graph.links.push(edge); + std::fs::write(&path, serde_json::to_vec(&graph)?)?; + let engine = open(&path, None, &directory.path().join("cache"))?; + let question = request("where are protocol errors formatted"); + let ordinary = engine.discover(question.clone())?; + assert!(!ordinary.edges.is_empty()); + let raw_bytes = serde_json::to_vec(&ordinary)?.len(); + let ranked = engine.discover_ranked_with_context(question.clone(), true)?; + let fits_raw = raw_bytes as u64 + 128; + assert!(serde_json::to_vec(&ranked)?.len() as u64 > fits_raw); + + let mut limited = question.clone(); + limited.limits.max_response_bytes = fits_raw; + let fits = engine.discover(limited.clone())?; + assert_eq!(fits.nodes, ordinary.nodes); + assert_eq!(fits.edges, ordinary.edges); + assert!(serde_json::to_vec(&fits)?.len() as u64 <= fits_raw); + for include_context in [false, true] { + assert!( + matches!(engine.discover_ranked_with_context(limited.clone(), include_context), Err(error) if error.code() == "ranked_response_limit") + ); + } + + let mut without_edges = ordinary.clone(); + without_edges.edges.clear(); + limited.limits.max_response_bytes = serde_json::to_vec(&without_edges)? + .len() + .saturating_sub(256) as u64; + let trimmed = engine.discover(limited)?; + assert!(trimmed.truncated); + assert!(trimmed.nodes.len() < ordinary.nodes.len()); + assert_eq!(trimmed.seeds, ordinary.seeds); + assert!(serde_json::to_vec(&trimmed)?.len() as u64 <= trimmed.limits.max_response_bytes); + let retained = trimmed + .nodes + .iter() + .map(|node| node.id.as_str()) + .collect::>(); + assert!( + trimmed + .seeds + .iter() + .all(|seed| retained.contains(seed.node_id.as_str())) + ); + assert!(trimmed.edges.iter().all(|edge| { + retained.contains(edge.source.as_str()) && retained.contains(edge.target.as_str()) + })); + Ok(()) +} + +#[test] +fn ranked_exact_id_does_not_admit_identifier_prefix_matches() +-> Result<(), Box> { + let directory = tempfile::tempdir()?; + let path = directory.path().join("graph.json"); + let mut graph = fixture(&path)?; + let exact_id = format!("sha256:{}", "0123456789abcdef".repeat(4)); + graph.nodes = vec![ + support::node( + &exact_id, + NodeKind::Function, + "render_message", + "output::render_message", + ), + support::node( + "digest-helper", + NodeKind::Function, + "sha256_hex", + "crypto::sha256_hex", + ), + support::node( + "digest-wrapper", + NodeKind::Function, + "sha256_digest", + "crypto::sha256_digest", + ), + ]; + std::fs::write(&path, serde_json::to_vec(&graph)?)?; + let json = open(&path, None, &directory.path().join("json-cache"))?; + let store = SqliteStore::open(directory.path().join("store.sqlite3"))?; + let prepared = GraphSnapshotBuilder::new().prepare(&store, &graph)?; + GraphSnapshotBuilder::new().activate(&store, &prepared)?; + let stored = open_with_store(&store, &path, None, &directory.path().join("store-cache"))?; + for engine in [&json, &stored] { + let ordinary = engine.discover(request(&exact_id))?; + let ranked = engine.discover_ranked(request(&exact_id))?; + assert_eq!(ranked.discovery.seeds, ordinary.seeds); + assert_eq!(ranked.discovery.nodes, ordinary.nodes); + assert_eq!(ranked.discovery.edges, ordinary.edges); + assert_eq!( + ranked + .candidates + .iter() + .map(|candidate| candidate.node.id.as_str()) + .collect::>(), + vec![exact_id.as_str()] + ); + let broad = engine.discover_ranked(request("where is sha256 computed"))?; + assert!(broad.candidates.iter().any(|candidate| { + matches!( + candidate.node.id.as_str(), + "digest-helper" | "digest-wrapper" + ) + })); + } + Ok(()) +} + +#[test] +fn ranked_structural_fallback_bounds_its_wrapper_independently_of_raw_discovery() +-> Result<(), Box> { + let directory = tempfile::tempdir()?; + let path = directory.path().join("graph.json"); + support::write_graph(&path)?; + let engine = open(&path, None, &directory.path().join("cache"))?; + let question = request("who calls callee"); + let ordinary = engine.discover(question.clone())?; + let ranked = engine.discover_ranked(question.clone())?; + assert!(!ranked.candidates.is_empty()); + assert!(ranked.candidates.iter().all(|candidate| { + candidate + .reasons + .iter() + .any(|reason| reason.starts_with("Structural query seed")) + })); + let mut limited = question; + limited.limits.max_response_bytes = serde_json::to_vec(&ordinary)?.len() as u64 + 64; + assert!(serde_json::to_vec(&ranked)?.len() as u64 > limited.limits.max_response_bytes); + let raw = engine.discover(limited.clone())?; + assert_eq!(raw.nodes, ordinary.nodes); + assert_eq!(raw.edges, ordinary.edges); + assert!(serde_json::to_vec(&raw)?.len() as u64 <= raw.limits.max_response_bytes); + for result in [ + engine.discover_ranked(limited.clone()), + engine.discover_ranked_with_context(limited, false), + ] { + assert!(matches!(result, Err(error) if error.code() == "ranked_response_limit")); + } + Ok(()) +} + +#[test] +fn broad_search_marks_witness_edges_omitted_by_the_edge_cap() +-> Result<(), Box> { + use compass_model::code_graph::{EdgeKind, ResourceKind, ResourceNodeDetails}; + let directory = tempfile::tempdir()?; + let path = directory.path().join("graph.json"); + support::write_graph(&path)?; + let mut graph = GraphDocument::load(&path)?; + let template = graph.links.first().cloned().ok_or("edge fixture missing")?; + graph.nodes.clear(); + graph.links.clear(); + let mut paragraph = support::node("paragraph", NodeKind::Resource, "Notes", "Notes"); + paragraph.details = Some(NodeDetails::Resource(ResourceNodeDetails { + resource_kind: ResourceKind::Document, + uri: None, + media_type: Some("text/markdown".to_owned()), + content: Some("OrchardOne and OrchardTwo implement orchard harvesting".to_owned()), + })); + graph.nodes.push(paragraph); + for name in ["OrchardOne", "OrchardTwo"] { + graph + .nodes + .push(support::node(name, NodeKind::Class, name, name)); + let mut edge = template.clone(); + edge.source = "paragraph".to_owned(); + edge.target = name.to_owned(); + edge.kind = EdgeKind::References; + edge.id = compass_model::identity::edge_id( + &edge.source, + edge.kind, + &edge.target, + edge.relationship_site.as_ref(), + None, + ); + edge.key = edge.id.clone(); + graph.links.push(edge); + } + std::fs::write(&path, serde_json::to_vec(&graph)?)?; + let json = open(&path, None, &directory.path().join("json-cache"))?; + let store = SqliteStore::open(directory.path().join("store.sqlite3"))?; + let prepared = GraphSnapshotBuilder::new().prepare(&store, &graph)?; + GraphSnapshotBuilder::new().activate(&store, &prepared)?; + let stored = open_with_store(&store, &path, None, &directory.path().join("store-cache"))?; + for engine in [&json, &stored] { + let search = SearchRequest { + query: "where is orchard harvesting".to_owned(), + limits: CodeQueryLimits::default(), + }; + let complete = engine.search(search.clone())?; + assert!(!complete.truncated, "{complete:?}"); + assert_eq!(complete.edges.len(), 2); + let mut limited = search; + limited.limits.max_edges = 1; + let bounded = engine.search(limited)?; + assert!(bounded.truncated); + assert_eq!(bounded.results, complete.results); + assert_eq!(bounded.edges.len(), 1); + assert!(complete.edges.contains(&bounded.edges[0])); + assert_eq!(bounded.edges[0].source, "paragraph"); + assert!( + bounded + .nodes + .iter() + .any(|node| node.id == bounded.edges[0].target) + ); + } + Ok(()) +} + +#[test] +fn leading_document_proof_retrieves_opaque_owner_with_json_store_parity() +-> Result<(), Box> { + use compass_model::code_graph::{ + DiagnosticSeverity, EdgeKind, GraphDiagnostic, ResourceKind, ResourceNodeDetails, + }; + use compass_model::provenance::{LEADING_DOCUMENTATION_RULE, OccurrenceRule, SourceAnchor}; + for proven in [true, false] { + let directory = tempfile::tempdir()?; + let path = directory.path().join("graph.json"); + support::write_graph(&path)?; + let mut graph = GraphDocument::load(&path)?; + let content = "/// protocol diagnostics parsed\n"; + let excerpt = SourceAnchor { + file: "src/lib.rs".to_owned(), + start_byte: 0, + end_byte: content.len() as u64, + start_line: 1, + start_column: 0, + end_line: 2, + end_column: 0, + }; + let definition = SourceAnchor { + file: "src/lib.rs".to_owned(), + start_byte: excerpt.end_byte, + end_byte: excerpt.end_byte + 14, + start_line: 2, + start_column: 0, + end_line: 2, + end_column: 14, + }; + let actual_source = format!("{content}fn opaque() {{}}"); + std::fs::write(directory.path().join("src/lib.rs"), &actual_source)?; + use sha2::Digest; + let source_file = graph + .graph + .files + .iter_mut() + .find(|file| file.path == "src/lib.rs") + .ok_or("missing source file")?; + source_file.byte_size = actual_source.len() as u64; + source_file.content_digest = format!( + "sha256:{:x}", + sha2::Sha256::digest(actual_source.as_bytes()) + ); + let mut owner = support::node( + "owner", + NodeKind::Function, + "opaque", + "implementation::opaque", + ); + owner.source = Some(definition.clone()); + owner.evidence[0].anchors = vec![definition.clone()]; + let mut prose = support::node("prose", NodeKind::Resource, "Notes", "Notes"); + prose.source = Some(excerpt.clone()); + prose.evidence[0].anchors = vec![excerpt.clone()]; + prose.details = Some(NodeDetails::Resource(ResourceNodeDetails { + resource_kind: ResourceKind::Rationale, + uri: None, + media_type: None, + content: Some(content.to_owned()), + })); + let mut edge = graph.links.first().cloned().ok_or("missing fixture edge")?; + edge.kind = EdgeKind::Documents; + edge.source = prose.id.clone(); + edge.target = owner.id.clone(); + edge.relationship_site = Some(excerpt.clone()); + edge.occurrence_rule = if proven { + OccurrenceRule::new(LEADING_DOCUMENTATION_RULE) + } else { + None + }; + edge.evidence[0].anchors = vec![excerpt]; + edge.evidence[0].rule = None; + if proven { + edge.evidence[0].anchors.push(definition); + edge.evidence[0].rule = Some(LEADING_DOCUMENTATION_RULE.to_owned()); + } + edge.id = compass_model::identity::edge_id( + &edge.source, + edge.kind, + &edge.target, + edge.relationship_site.as_ref(), + edge.occurrence_rule.as_ref().map(|rule| rule.as_str()), + ); + edge.key = edge.id.clone(); + graph.nodes = vec![owner, prose]; + graph.links = vec![edge]; + graph.graph.diagnostics.push(GraphDiagnostic { + severity: DiagnosticSeverity::Warning, + code: "leading_documentation_omission_summary".to_owned(), + message: "An additional oversized documentation group was omitted.".to_owned(), + anchor: None, + related_ids: Vec::new(), + }); + std::fs::write(&path, serde_json::to_vec(&graph)?)?; + let json = open(&path, None, &directory.path().join("json-cache"))?; + let store = SqliteStore::open(directory.path().join("store.sqlite3"))?; + let prepared = GraphSnapshotBuilder::new().prepare(&store, &graph)?; + GraphSnapshotBuilder::new().activate(&store, &prepared)?; + let stored = open_with_store(&store, &path, None, &directory.path().join("store-cache"))?; + let question = request("where are protocol diagnostics parsed"); + let expected = json.discover_ranked(question.clone())?; + let actual = stored.discover_ranked(question)?; + assert_eq!(actual.candidates, expected.candidates); + assert_eq!(actual.discovery.diagnostics, expected.discovery.diagnostics); + assert!( + actual + .discovery + .diagnostics + .iter() + .any(|diagnostic| diagnostic.message.contains("oversized documentation group")) + ); + let owner = actual + .candidates + .iter() + .find(|candidate| candidate.node.id == "owner"); + assert_eq!(owner.is_some(), proven, "{actual:?}"); + if let Some(owner) = owner { + assert!( + owner + .matched_fields + .iter() + .any(|field| field == "owned_document") + ); + assert_eq!(owner.supporting_sources.len(), 1); + assert_eq!(owner.supporting_sources[0].start_line, 1); + let quote = owner + .excerpts + .first() + .ok_or("missing proven raw documentation quote")?; + assert_eq!( + quote.kind, + compass_query::RankedExcerptKind::AstDocumentation + ); + assert_eq!(quote.text, content); + assert_eq!(quote.source, owner.supporting_sources[0]); + assert_eq!(quote.source.start_byte, 0); + assert_eq!(quote.source.end_byte, content.len() as u64); + assert!(!quote.clipped); + assert!( + !actual + .candidates + .iter() + .any(|candidate| candidate.node.id == "prose") + ); + } + } + Ok(()) +} + +#[test] +fn broad_local_identifier_recall_requires_unique_exact_callable_ownership() +-> Result<(), Box> { + use compass_model::code_graph::EdgeKind; + use compass_model::provenance::{EvidenceConfidence, EvidenceOrigin}; + use compass_model::query_contract::{DiscoveryScope, DiscoveryScopeKind}; + let directory = tempfile::tempdir()?; + let path = directory.path().join("graph.json"); + support::write_graph(&path)?; + let mut graph = GraphDocument::load(&path)?; + let template = graph.links.first().cloned().ok_or("missing edge")?; + graph.nodes.clear(); + graph.links.clear(); + for case in [ + "good", + "ambiguous", + "no-ast", + "inferred", + "outside", + "namespace", + "nested", + "truncated", + "no-owner-ast", + ] { + let owner_id = format!("owner:{case}"); + let child_id = format!("child:{case}"); + let mut owner = support::node( + &owner_id, + NodeKind::Function, + "opaque", + &format!("sample.{case}.opaque"), + ); + if case == "no-owner-ast" { + for proof in &mut owner.evidence { + proof.origin = EvidenceOrigin::Config; + } + } + let mut child = support::node( + &child_id, + if case == "nested" { + NodeKind::Function + } else { + NodeKind::Variable + }, + if case == "namespace" { + "checkpoint" + } else { + "annotations" + }, + &format!("sample.annotations.{case}"), + ); + if case == "outside" { + // Both ranges remain individually valid, but the child is in a + // different source file and cannot be owned by this declaration. + let anchor = child.source.as_mut().ok_or("missing child source")?; + anchor.file = "src/payments/gateway.rs".to_owned(); + for proof in &mut child.evidence { + proof.anchors = vec![anchor.clone()]; + } + } + graph.nodes.push(owner); + graph.nodes.push(child.clone()); + let parent_count = match case { + "ambiguous" => 2, + "truncated" => 5, + _ => 1, + }; + for parent in 0..parent_count { + let parent_id = if parent == 0 { + owner_id.clone() + } else { + format!("{owner_id}:{parent}") + }; + if parent > 0 { + graph.nodes.push(support::node( + &parent_id, + NodeKind::Function, + "opaque", + &format!("sample.{case}.other{parent}"), + )); + } + let mut edge = template.clone(); + edge.source = parent_id; + edge.target = child_id.clone(); + edge.kind = EdgeKind::Contains; + edge.relationship_site = child.source.clone(); + for proof in &mut edge.evidence { + proof.anchors = vec![child.source.clone().ok_or("missing source")?]; + if case == "no-ast" { + proof.origin = EvidenceOrigin::Config; + } + if case == "inferred" { + proof.confidence = EvidenceConfidence::Inferred; + } + } + edge.id = compass_model::identity::edge_id( + &edge.source, + edge.kind, + &edge.target, + edge.relationship_site.as_ref(), + None, + ); + edge.key = edge.id.clone(); + graph.links.push(edge); + } + } + std::fs::write(&path, serde_json::to_vec(&graph)?)?; + let json = open(&path, None, &directory.path().join("json-cache"))?; + let store = SqliteStore::open(directory.path().join("store.sqlite3"))?; + let prepared = GraphSnapshotBuilder::new().prepare(&store, &graph)?; + GraphSnapshotBuilder::new().activate(&store, &prepared)?; + let stored = open_with_store(&store, &path, None, &directory.path().join("store-cache"))?; + let question = request("Which function owns annotations?"); + let reference = json.discover_ranked(question.clone())?; + let owner = reference + .candidates + .iter() + .find(|candidate| candidate.node.id == "owner:good") + .ok_or("unique owner not recalled")?; + assert!( + owner + .matched_fields + .iter() + .any(|field| field == "owned_identifier") + ); + assert_eq!(owner.relevance, compass_query::RelevanceBand::Low); + assert_eq!(owner.supporting_sources.len(), 1); + assert!( + owner + .reasons + .iter() + .any(|reason| reason.contains("not confirmation")) + ); + assert!( + reference + .candidates + .iter() + .filter(|candidate| candidate + .matched_fields + .iter() + .any(|field| field == "owned_identifier")) + .all(|candidate| candidate.node.id == "owner:good") + ); + assert!( + reference.bounded, + "hidden ownership parents must be disclosed" + ); + for engine in [&json, &stored] { + assert_eq!( + reference.candidates, + engine.discover_ranked(question.clone())?.candidates + ); + let mut scoped = question.clone(); + scoped.scope.push(DiscoveryScope { + kind: DiscoveryScopeKind::Node, + value: "owner:good".to_owned(), + }); + let answer = engine.discover_ranked(scoped)?; + assert_eq!(answer.candidates.len(), 1); + assert_eq!(answer.candidates[0].node.id, "owner:good"); + assert!( + answer.candidates[0] + .matched_fields + .iter() + .any(|field| field == "owned_identifier") + ); + for case in [ + "ambiguous", + "no-ast", + "inferred", + "outside", + "namespace", + "nested", + "truncated", + "no-owner-ast", + ] { + let mut negative = question.clone(); + negative.scope.push(DiscoveryScope { + kind: DiscoveryScopeKind::Node, + value: format!("owner:{case}"), + }); + let answer = engine.discover_ranked(negative)?; + assert!( + answer.candidates.iter().all(|candidate| !candidate + .matched_fields + .iter() + .any(|field| field == "owned_identifier")), + "{case}" + ); + } + assert!( + reference.discovery.stats.candidate_nodes + <= compass_model::query_contract::MAX_INDEXED_CANDIDATE_NODES_READ + ); + assert!( + reference.discovery.stats.candidate_probes + <= compass_model::query_contract::MAX_INDEXED_CANDIDATE_PROBES + ); + let search = engine.search(SearchRequest { + query: question.question.clone(), + limits: CodeQueryLimits::default(), + })?; + assert!( + search + .edges + .iter() + .any(|edge| edge.kind == EdgeKind::Contains + && edge.source == "owner:good" + && edge.target == "child:good") + ); + assert!(search.nodes.iter().any(|node| node.id == "child:good")); + } + graph.nodes.reverse(); + graph.links.reverse(); + let reordered_path = directory.path().join("reordered.json"); + std::fs::write(&reordered_path, serde_json::to_vec(&graph)?)?; + let reordered = open( + &reordered_path, + None, + &directory.path().join("reordered-cache"), + )?; + assert_eq!( + reference.candidates, + reordered.discover_ranked(question)?.candidates + ); + Ok(()) +} + +#[test] +fn broad_local_identifier_recall_bounds_children_before_parent_reads() +-> Result<(), Box> { + use compass_model::code_graph::EdgeKind; + let directory = tempfile::tempdir()?; + let path = directory.path().join("graph.json"); + support::write_graph(&path)?; + let mut graph = GraphDocument::load(&path)?; + let template = graph.links.first().cloned().ok_or("missing edge")?; + graph.nodes = vec![support::node( + "owner", + NodeKind::Function, + "opaque", + "sample.opaque", + )]; + graph.links.clear(); + for at in 0..40 { + let id = format!("child:{at:02}"); + graph.nodes.push(support::node( + &id, + NodeKind::Variable, + "annotations", + &format!("sample.local{at}.annotations"), + )); + let mut edge = template.clone(); + edge.source = "owner".to_owned(); + edge.target = id; + edge.kind = EdgeKind::Contains; + edge.id = compass_model::identity::edge_id( + &edge.source, + edge.kind, + &edge.target, + edge.relationship_site.as_ref(), + None, + ); + edge.key = edge.id.clone(); + graph.links.push(edge); + } + std::fs::write(&path, serde_json::to_vec(&graph)?)?; + let json = open(&path, None, &directory.path().join("json-cache"))?; + let store = SqliteStore::open(directory.path().join("store.sqlite3"))?; + let prepared = GraphSnapshotBuilder::new().prepare(&store, &graph)?; + GraphSnapshotBuilder::new().activate(&store, &prepared)?; + let stored = open_with_store(&store, &path, None, &directory.path().join("store-cache"))?; + let question = request("Which function owns annotations?"); + let expected = json.discover_ranked(question.clone())?; + for engine in [&json, &stored] { + let answer = engine.discover_ranked(question.clone())?; + assert_eq!(answer.candidates, expected.candidates); + assert!(answer.bounded && answer.eligible_count.is_none()); + assert!(answer.candidates.iter().any(|candidate| { + candidate.node.id == "owner" + && candidate + .matched_fields + .iter() + .any(|field| field == "owned_identifier") + })); + assert!( + answer.discovery.stats.candidate_nodes + <= compass_model::query_contract::MAX_INDEXED_CANDIDATE_NODES_READ + ); + assert!( + answer.discovery.stats.candidate_probes + <= compass_model::query_contract::MAX_INDEXED_CANDIDATE_PROBES + ); + } + Ok(()) +} + +#[test] +fn generic_facets_cannot_replace_subject_evidence_but_facet_listings_remain_valid() +-> Result<(), Box> { + let directory = tempfile::tempdir()?; + let path = directory.path().join("graph.json"); + support::write_graph(&path)?; + let mut graph = GraphDocument::load(&path)?; + graph.nodes.clear(); + graph.links.clear(); + let mut python_file = graph + .graph + .files + .first() + .cloned() + .ok_or("missing source file")?; + python_file.path = "src/sample.py".to_owned(); + python_file.id = compass_model::identity::file_id(&python_file.path); + python_file.language = Some("python".to_owned()); + std::fs::write(directory.path().join(&python_file.path), b"code")?; + graph.graph.files.push(python_file); + for (id, name, language) in [ + ("payload", "process_payload", "rust"), + ("generic-action", "handle", "rust"), + ("generic-kind", "spin", "rust"), + ("python", "opaque", "python"), + ] { + let mut node = support::node(id, NodeKind::Function, name, &format!("sample.{name}")); + node.language = Some(language.to_owned()); + if language == "python" { + node.source.as_mut().ok_or("missing source")?.file = "src/sample.py".to_owned(); + for proof in &mut node.evidence { + for anchor in &mut proof.anchors { + anchor.file = "src/sample.py".to_owned(); + } + } + } + graph.nodes.push(node); + } + std::fs::write(&path, serde_json::to_vec(&graph)?)?; + let json = open(&path, None, &directory.path().join("json-cache"))?; + let store = SqliteStore::open(directory.path().join("store.sqlite3"))?; + let prepared = GraphSnapshotBuilder::new().prepare(&store, &graph)?; + GraphSnapshotBuilder::new().activate(&store, &prepared)?; + let stored = open_with_store(&store, &path, None, &directory.path().join("store-cache"))?; + for engine in [&json, &stored] { + for question in [ + "Which function owns payload?", + "Which function handles payload?", + "Which function mysteriously handles payload?", + ] { + let answer = engine.discover_ranked(request(question))?; + assert_eq!( + answer.candidates.len(), + 1, + "{question}: {:?}", + answer.candidates + ); + assert_eq!(answer.candidates[0].node.id, "payload"); + assert!( + answer.candidates[0] + .matched_fields + .iter() + .any(|field| !matches!(field.as_str(), "kind" | "language")) + ); + if question.contains("mysteriously") { + assert_eq!( + answer.candidates[0].relevance, + compass_query::RelevanceBand::Low + ); + } + } + assert!( + engine + .discover_ranked(request("Which function handles ethereal flux?"))? + .candidates + .is_empty() + ); + let all = engine.discover_ranked(request("Which functions are defined?"))?; + assert_eq!(all.candidates.len(), 4); + let python = engine.discover_ranked(request("Which Python functions are defined?"))?; + assert_eq!(python.candidates.len(), 1); + assert_eq!(python.candidates[0].node.id, "python"); + let exact = engine.discover_ranked(request("opaque"))?; + assert_eq!( + exact + .candidates + .first() + .map(|candidate| candidate.node.id.as_str()), + Some("python") + ); + } + for question in [ + "Which function owns payload?", + "Which functions are defined?", + "Which Python functions are defined?", + ] { + assert_eq!( + json.discover_ranked(request(question))?.candidates, + stored.discover_ranked(request(question))?.candidates + ); + } + graph.nodes.reverse(); + let reordered_path = directory.path().join("reordered.json"); + std::fs::write(&reordered_path, serde_json::to_vec(&graph)?)?; + let reordered = open( + &reordered_path, + None, + &directory.path().join("reordered-cache"), + )?; + assert_eq!( + json.discover_ranked(request("Which function owns payload?"))? + .candidates, + reordered + .discover_ranked(request("Which function owns payload?"))? + .candidates + ); + Ok(()) +} diff --git a/crates/compass-query/tests/broad_test_context.rs b/crates/compass-query/tests/broad_test_context.rs new file mode 100644 index 000000000..a177cfd63 --- /dev/null +++ b/crates/compass-query/tests/broad_test_context.rs @@ -0,0 +1,119 @@ +mod support; + +use std::error::Error; + +use compass_graph::GraphSnapshotBuilder; +use compass_model::code_graph::{GraphDocument, NodeKind, NodeRole}; +use compass_model::identity::file_id; +use compass_model::query_contract::{ + DiscoveryDirection, DiscoveryLimits, DiscoveryQueryRequest, DiscoveryTraversal, +}; +use compass_query::{open, open_with_store}; +use compass_store::SqliteStore; + +fn request(question: &str) -> DiscoveryQueryRequest { + DiscoveryQueryRequest { + question: question.to_owned(), + direction: DiscoveryDirection::Auto, + relation_contexts: Vec::new(), + scope: Vec::new(), + traversal: DiscoveryTraversal::Bfs, + include_heuristic: false, + limits: DiscoveryLimits::default(), + } +} + +#[test] +fn implementation_questions_demote_test_context_but_keep_explicit_test_intent() +-> Result<(), Box> { + for (production_path, test_path, test_role) in [ + ("src/codec.live.ts", "src/codec.test.ts", None), + ("src/codec.live.ts", "src/codec.spec.ts", None), + ("src/codec_live.go", "src/codec_test.go", None), + ("src/codec_live.py", "src/test_codec.py", None), + ("src/contest.ts", "src/codec.test.ts", None), + ("src/latest.ts", "src/codec.spec.ts", None), + ( + "src/codec.live.ts", + "src/codec.flagged.ts", + Some(NodeRole::Test), + ), + ( + "src/codec.live.ts", + "src/codec.fixture.ts", + Some(NodeRole::Fixture), + ), + ] { + let directory = tempfile::tempdir()?; + let path = directory.path().join("graph.json"); + support::write_graph(&path)?; + let mut graph = GraphDocument::load(&path)?; + let template = graph.graph.files.first().cloned().ok_or("missing file")?; + graph.nodes.clear(); + graph.links.clear(); + for (id, file, role) in [ + ("z:production", production_path, None), + ("a:test", test_path, test_role), + ] { + let source = directory.path().join(file); + std::fs::create_dir_all(source.parent().ok_or("missing parent")?)?; + std::fs::write(source, b"code")?; + let mut record = template.clone(); + record.id = file_id(file); + record.path = file.to_owned(); + graph.graph.files.push(record); + let mut node = + support::node(id, NodeKind::Function, "format_payload", "format_payload"); + node.source.as_mut().ok_or("missing source")?.file = file.to_owned(); + for proof in &mut node.evidence { + for anchor in &mut proof.anchors { + anchor.file = file.to_owned(); + } + } + if let Some(role) = role { + node.roles.push(role); + } + graph.nodes.push(node); + } + std::fs::write(&path, serde_json::to_vec(&graph)?)?; + let json = open(&path, None, &directory.path().join("json-cache"))?; + let store = SqliteStore::open(directory.path().join("store.sqlite3"))?; + let prepared = GraphSnapshotBuilder::new().prepare(&store, &graph)?; + GraphSnapshotBuilder::new().activate(&store, &prepared)?; + let stored = open_with_store(&store, &path, None, &directory.path().join("store-cache"))?; + let explicit = if test_role == Some(NodeRole::Fixture) { + "where are payload formatting fixtures" + } else { + "where are payload formatting tests" + }; + for (question, expected) in [ + ("where are payloads formatted", "z:production"), + (explicit, "a:test"), + ] { + let response = json.discover_ranked(request(question))?; + assert_eq!( + response.candidates.first().map(|row| row.node.id.as_str()), + Some(expected), + "{test_path}: {question}" + ); + assert_eq!( + response.candidates, + stored.discover_ranked(request(question))?.candidates, + "{test_path}: backend parity" + ); + assert!( + response + .candidates + .iter() + .any(|row| row.node.id == "a:test") + ); + assert!( + response + .candidates + .iter() + .any(|row| row.node.id == "z:production") + ); + } + } + Ok(()) +} diff --git a/crates/compass-query/tests/fielded_projection.rs b/crates/compass-query/tests/fielded_projection.rs new file mode 100644 index 000000000..c6fe26740 --- /dev/null +++ b/crates/compass-query/tests/fielded_projection.rs @@ -0,0 +1,309 @@ +mod support; + +use std::error::Error; +use std::path::Path; + +use compass_graph::{GraphSnapshotBuilder, GraphSnapshotReader, SnapshotReadLimits}; +use compass_model::code_graph::{EdgeKind, GraphDocument, NodeKind, NodeRole}; +use compass_model::provenance::{EvidenceConfidence, EvidenceOrigin}; +use compass_model::query_contract::{ + CodeQueryLimits, DiscoveryDirection, DiscoveryLimits, DiscoveryQueryRequest, + DiscoveryTraversal, SearchRequest, +}; +use compass_query::{open, open_with_store}; +use compass_store::SqliteStore; +use rusqlite::{Connection, params}; + +fn fixture(path: &Path, aliases: usize) -> Result> { + support::write_graph(path)?; + let mut graph = GraphDocument::load(path)?; + let template = graph.links.first().cloned().ok_or("edge fixture missing")?; + graph.nodes.clear(); + graph.links.clear(); + let mut target = support::node("owner", NodeKind::Function, "execute", "app::execute"); + target.roles = vec![NodeRole::RouteHandler]; + graph.nodes.push(target); + for at in 0..aliases { + let id = format!("alias:{at}"); + let name = if aliases == 1 { + "renewal".to_owned() + } else { + format!("renewal_{at}") + }; + graph.nodes.push(support::node( + &id, + NodeKind::TypeAlias, + &name, + &format!("app::{name}"), + )); + let mut edge = template.clone(); + edge.source = id; + edge.target = "owner".to_owned(); + edge.kind = EdgeKind::Aliases; + edge.occurrence_rule = None; + edge.context = None; + edge.details = None; + edge.id = compass_model::identity::edge_id( + &edge.source, + edge.kind, + &edge.target, + edge.relationship_site.as_ref(), + None, + ); + edge.key = edge.id.clone(); + graph.links.push(edge); + } + Ok(graph) +} + +fn question() -> DiscoveryQueryRequest { + DiscoveryQueryRequest { + question: "where are renewal route handlers".to_owned(), + direction: DiscoveryDirection::Auto, + relation_contexts: Vec::new(), + scope: Vec::new(), + traversal: DiscoveryTraversal::Bfs, + include_heuristic: false, + limits: DiscoveryLimits::default(), + } +} + +#[test] +fn alias_kind_and_role_projection_matches_persisted_statistics_and_primary_postings() +-> Result<(), Box> { + let directory = tempfile::tempdir()?; + let path = directory.path().join("graph.json"); + let mut graph = fixture(&path, 1)?; + std::fs::write(&path, serde_json::to_vec(&graph)?)?; + let json = open(&path, None, &directory.path().join("json-cache"))?; + let connection = Connection::open(json.index_path())?; + let store = SqliteStore::open(directory.path().join("store.sqlite3"))?; + let prepared = GraphSnapshotBuilder::new().prepare(&store, &graph)?; + GraphSnapshotBuilder::new().activate(&store, &prepared)?; + let reader = GraphSnapshotReader::open_active(&store)?.ok_or("snapshot missing")?; + for (term, expected) in [ + ("renewal", 2_u64), + ("route", 1), + ("handler", 1), + ("function", 1), + ] { + let json_df: u64 = connection.query_row( + "SELECT frequency FROM fielded_stats WHERE term=?1", + params![term], + |row| row.get(0), + )?; + assert_eq!(json_df, expected, "{term}"); + assert_eq!( + reader.fielded_document_frequency(term)?, + Some(expected), + "{term}" + ); + } + let reserved = "__compass_primary_v1_renewal"; + let json_primary: Vec = connection + .prepare("SELECT node_id FROM node_fts WHERE node_fts MATCH ?1 ORDER BY node_id")? + .query_map(params![format!("\"{reserved}\"")], |row| row.get(0))? + .collect::>()?; + assert_eq!(json_primary, ["alias:0", "owner"]); + let (stored_primary, truncated, _) = + reader.nodes_for_exact_term_bounded_work(reserved, SnapshotReadLimits::default())?; + assert!(!truncated); + assert_eq!( + stored_primary + .iter() + .map(|node| node.id.as_str()) + .collect::>(), + ["alias:0", "owner"] + ); + let stored = open_with_store(&store, &path, None, &directory.path().join("store-cache"))?; + let expected = json.discover_ranked(question())?; + assert!(expected.candidates.iter().any(|candidate| { + candidate.node.id == "owner" + && candidate + .matched_fields + .iter() + .any(|field| field == "alias") + && candidate + .matched_fields + .iter() + .any(|field| field == "roles") + })); + assert_eq!( + stored.discover_ranked(question())?.candidates, + expected.candidates + ); + graph.nodes.reverse(); + graph.links.reverse(); + std::fs::write(&path, serde_json::to_vec(&graph)?)?; + let reordered = open(&path, None, &directory.path().join("reverse-cache"))?; + assert_eq!( + reordered.discover_ranked(question())?.candidates, + expected.candidates + ); + Ok(()) +} + +#[test] +fn alias_overflow_is_visible_and_omits_primary_owner_alias_terms_on_both_backends() +-> Result<(), Box> { + let directory = tempfile::tempdir()?; + let path = directory.path().join("graph.json"); + let graph = fixture(&path, 9)?; + std::fs::write(&path, serde_json::to_vec(&graph)?)?; + let json = open(&path, None, &directory.path().join("json-cache"))?; + let connection = Connection::open(json.index_path())?; + let json_df: u64 = connection.query_row( + "SELECT frequency FROM fielded_stats WHERE term='renewal'", + [], + |row| row.get(0), + )?; + assert_eq!(json_df, 9); + let store = SqliteStore::open(directory.path().join("store.sqlite3"))?; + let prepared = GraphSnapshotBuilder::new().prepare(&store, &graph)?; + GraphSnapshotBuilder::new().activate(&store, &prepared)?; + let reader = GraphSnapshotReader::open_active(&store)?.ok_or("snapshot missing")?; + assert_eq!(reader.fielded_document_frequency("renewal")?, Some(9)); + let reserved = "__compass_primary_v1_renewal"; + let json_primary: Vec = connection + .prepare("SELECT node_id FROM node_fts WHERE node_fts MATCH ?1 ORDER BY node_id")? + .query_map(params![format!("\"{reserved}\"")], |row| row.get(0))? + .collect::>()?; + let aliases: Vec = (0..9).map(|at| format!("alias:{at}")).collect(); + assert_eq!(json_primary, aliases); + let (stored_primary, truncated, _) = + reader.nodes_for_exact_term_bounded_work(reserved, SnapshotReadLimits::default())?; + assert!(!truncated); + assert_eq!( + stored_primary + .iter() + .map(|node| node.id.clone()) + .collect::>(), + aliases + ); + let stored = open_with_store(&store, &path, None, &directory.path().join("store-cache"))?; + let expected = json.discover_ranked(question())?; + assert!(expected.bounded); + assert_eq!(expected.eligible_count, None); + assert_eq!( + stored.discover_ranked(question())?.candidates, + expected.candidates + ); + assert!(stored.discover_ranked(question())?.bounded); + Ok(()) +} + +#[test] +fn fielded_aliases_filter_provenance_with_raw_bounds_and_preserve_legacy_lookup() +-> Result<(), Box> { + for (aliases, exact_aliases) in [(3, 0), (8, 1), (9, 1)] { + let directory = tempfile::tempdir()?; + let path = directory.path().join("graph.json"); + let mut graph = fixture(&path, aliases)?; + for (at, edge) in graph + .links + .iter_mut() + .enumerate() + .take(aliases - exact_aliases) + { + let mut weaker = edge.evidence.first().cloned().ok_or("alias evidence")?; + if at % 3 == 0 { + weaker.origin = EvidenceOrigin::Heuristic; + weaker.rule = Some("fixture-alias".to_owned()); + weaker.wiring_site = edge.relationship_site.clone(); + // Even exact confidence cannot promote heuristic provenance. + edge.evidence = vec![weaker]; + } else { + weaker.confidence = EvidenceConfidence::Inferred; + if at % 3 == 1 { + edge.evidence = vec![weaker]; + } else { + // One exact entry cannot hide another inferred entry. + edge.evidence.push(weaker); + } + } + } + std::fs::write(&path, serde_json::to_vec(&graph)?)?; + let json = open(&path, None, &directory.path().join("json-cache"))?; + let connection = Connection::open(json.index_path())?; + let store = SqliteStore::open(directory.path().join("store.sqlite3"))?; + let prepared = GraphSnapshotBuilder::new().prepare(&store, &graph)?; + GraphSnapshotBuilder::new().activate(&store, &prepared)?; + let reader = GraphSnapshotReader::open_active(&store)?.ok_or("snapshot missing")?; + let owner_has_alias = exact_aliases > 0 && aliases <= 8; + let frequency = aliases as u64 + u64::from(owner_has_alias); + let json_df: u64 = connection.query_row( + "SELECT frequency FROM fielded_stats WHERE term='renewal'", + [], + |row| row.get(0), + )?; + assert_eq!(json_df, frequency, "{aliases} raw aliases"); + assert_eq!( + reader.fielded_document_frequency("renewal")?, + Some(frequency) + ); + let reserved = "__compass_primary_v1_renewal"; + let json_primary: Vec = connection + .prepare("SELECT node_id FROM node_fts WHERE node_fts MATCH ?1 ORDER BY node_id")? + .query_map(params![format!("\"{reserved}\"")], |row| row.get(0))? + .collect::>()?; + let (stored_primary, truncated, _) = + reader.nodes_for_exact_term_bounded_work(reserved, SnapshotReadLimits::default())?; + assert!(!truncated); + assert_eq!( + stored_primary + .iter() + .map(|node| node.id.clone()) + .collect::>(), + json_primary + ); + assert_eq!(json_primary.iter().any(|id| id == "owner"), owner_has_alias); + + let stored = open_with_store(&store, &path, None, &directory.path().join("store-cache"))?; + for engine in [&json, &stored] { + for at in 0..aliases { + let strict = engine.search(SearchRequest { + query: format!("renewal_{at}"), + limits: CodeQueryLimits::default(), + })?; + assert!(strict.results.iter().any(|hit| hit.node_id == "owner")); + } + } + let mut request = question(); + request.question = "where is renewal".to_owned(); + let expected = json.discover_ranked(request.clone())?; + let owner = expected + .candidates + .iter() + .find(|candidate| candidate.node.id == "owner"); + assert_eq!(owner.is_some(), owner_has_alias, "{aliases} raw aliases"); + if let Some(owner) = owner { + assert!(owner.matched_fields.iter().any(|field| field == "alias")); + } + assert_eq!( + stored.discover_ranked(request.clone())?.candidates, + expected.candidates + ); + if aliases > 8 { + assert!(expected.bounded); + assert_eq!(expected.eligible_count, None); + } + request.include_heuristic = true; + assert_eq!( + json.discover_ranked(request.clone())?.candidates, + expected.candidates + ); + assert_eq!( + stored.discover_ranked(request.clone())?.candidates, + expected.candidates + ); + graph.nodes.reverse(); + graph.links.reverse(); + std::fs::write(&path, serde_json::to_vec(&graph)?)?; + let reordered = open(&path, None, &directory.path().join("reverse-cache"))?; + assert_eq!( + reordered.discover_ranked(request)?.candidates, + expected.candidates + ); + } + Ok(()) +} diff --git a/crates/compass-query/tests/natural_intent.rs b/crates/compass-query/tests/natural_intent.rs index 3081ca6a2..48fda335a 100644 --- a/crates/compass-query/tests/natural_intent.rs +++ b/crates/compass-query/tests/natural_intent.rs @@ -12,7 +12,7 @@ use compass_model::query_contract::{ }; use compass_query::{ EngineSelection, NaturalQueryIntent, NaturalQueryRequest, ProfiledCodeQueryResponse, - QUERY_EXECUTION_PROFILE_V1, QUERY_PLANNER_PROFILE_V2, QUERY_RANKER_PROFILE_V1, QueryErrorKind, + QUERY_EXECUTION_PROFILE_V1, QUERY_PLANNER_PROFILE_V2, QUERY_RANKER_PROFILE_V10, QueryErrorKind, open_with_engine, plan_natural_query, }; use compass_store::{STORE_FILE_NAME, STORE_REF_FILE_NAME, SqliteStore}; @@ -359,7 +359,7 @@ fn profiled_natural_queries_report_real_stage_work_without_changing_the_response assert_eq!(profiled.response, repeated.response); assert_eq!(profiled.profile.schema, QUERY_EXECUTION_PROFILE_V1); assert_eq!(profiled.profile.planner_profile, QUERY_PLANNER_PROFILE_V2); - assert_eq!(profiled.profile.ranker_profile, QUERY_RANKER_PROFILE_V1); + assert_eq!(profiled.profile.ranker_profile, QUERY_RANKER_PROFILE_V10); assert!(profiled.profile.work.candidates_read > 0); assert_eq!( profiled.profile.work.candidates_read, diff --git a/crates/compass-query/tests/store_engine.rs b/crates/compass-query/tests/store_engine.rs index 1c2149518..b2b5ca5e9 100644 --- a/crates/compass-query/tests/store_engine.rs +++ b/crates/compass-query/tests/store_engine.rs @@ -7,7 +7,8 @@ use std::thread; use compass_graph::{GraphSnapshotBuilder, GraphSnapshotReader}; use compass_model::code_graph::{ - CODE_GRAPH_SCHEMA_V1, EdgeKind, GraphDocument, NodeKind, NodeRecord, + CODE_GRAPH_SCHEMA_V1, DiagnosticSeverity, EdgeKind, GraphDiagnostic, GraphDocument, NodeKind, + NodeRecord, }; use compass_model::identity::{edge_id, file_id}; use compass_model::provenance::OccurrenceRule; @@ -75,6 +76,57 @@ fn assert_discovery_semantically_equal( Ok(()) } +#[test] +fn local_store_discovery_preserves_documentation_coverage_diagnostics() +-> Result<(), Box> { + let directory = tempfile::tempdir()?; + let graph_path = directory.path().join("graph.json"); + support::write_graph(&graph_path)?; + let mut graph = GraphDocument::load(&graph_path)?; + graph.graph.diagnostics.push(GraphDiagnostic { + severity: DiagnosticSeverity::Warning, + code: "leading_documentation_omission_summary".to_owned(), + message: "An oversized documentation group was omitted.".to_owned(), + anchor: None, + related_ids: Vec::new(), + }); + fs::write(&graph_path, serde_json::to_vec(&graph)?)?; + let _store = publish_phase2_snapshot(directory.path(), &graph_path)?; + let json = open_with_engine( + &graph_path, + None, + &directory.path().join("json-cache"), + EngineSelection::Json, + )?; + let local = open_with_engine( + &graph_path, + None, + &directory.path().join("local-cache"), + EngineSelection::Store, + )?; + for question in ["UserService.list", "Where are errors handled?"] { + let request = discovery_request(question); + let expected = json.discover(request.clone())?; + let actual = local.discover(request)?; + for response in [&expected, &actual] { + assert!( + response + .diagnostics + .iter() + .any(|diagnostic| diagnostic.message.contains("oversized documentation group")) + ); + } + if question == "UserService.list" { + assert_discovery_semantically_equal(&actual, &expected)?; + } else { + // Backend work limits may differ for a negative neighborhood query. + // Both outcomes must retain coverage rather than imply a census. + assert!(actual.seeds.is_empty() && expected.seeds.is_empty()); + } + } + Ok(()) +} + #[test] fn discovery_is_identical_for_json_store_direct_document_and_immutable_selectors() -> Result<(), Box> { @@ -1578,7 +1630,8 @@ fn explicit_json_selection_survives_a_corrupt_store_sidecar() } #[test] -fn unknown_json_index_is_rebuilt_to_current_format() -> Result<(), Box> { +fn older_and_unknown_json_indexes_are_rebuilt_to_current_format() +-> Result<(), Box> { let directory = tempfile::tempdir()?; let graph_path = directory.path().join("graph.json"); support::write_graph(&graph_path)?; @@ -1587,21 +1640,30 @@ fn unknown_json_index_is_rebuilt_to_current_format() -> Result<(), Box 0, "rebuild must populate fielded statistics"); + } Ok(()) } diff --git a/crates/compass-resolve/src/evidence/index/builder.rs b/crates/compass-resolve/src/evidence/index/builder.rs index 04a205058..c1acda792 100644 --- a/crates/compass-resolve/src/evidence/index/builder.rs +++ b/crates/compass-resolve/src/evidence/index/builder.rs @@ -76,6 +76,30 @@ impl<'a> IndexBuilder<'a> { } } +const MAX_AGGREGATE_DOCUMENTATION_BYTES: usize = 128 * 1024 * 1024; + +fn checked_documentation_bytes( + batches: &[SemanticEvidenceBatch], + limit: usize, +) -> Result { + let mut total = 0_usize; + for documentation in batches + .iter() + .flat_map(|batch| &batch.declarations) + .filter_map(|declaration| declaration.documentation.as_ref()) + { + total = total + .checked_add(documentation.content.len()) + .ok_or_else(|| "universal aggregate documentation bytes overflow usize".to_owned())?; + if total > limit { + return Err(format!( + "universal aggregate documentation bytes {total} exceed limit {limit}" + )); + } + } + Ok(total) +} + impl UniversalResolutionIndex { pub fn new( batches: &[SemanticEvidenceBatch], @@ -90,6 +114,7 @@ impl UniversalResolutionIndex { root: &Path, limits: UniversalResolutionLimits, ) -> Result { + checked_documentation_bytes(batches, MAX_AGGREGATE_DOCUMENTATION_BYTES)?; Self::new_with_inventory_owned(batches.to_vec(), inventory_nodes, root, limits) } @@ -162,6 +187,7 @@ impl IndexBuilder<'_> { validate_batches, admission, } = self; + checked_documentation_bytes(&batches, MAX_AGGREGATE_DOCUMENTATION_BYTES)?; let languages = LanguagePresence::detect(&batches, inventory_nodes); let mut profile_started = Instant::now(); if validate_batches { @@ -1258,3 +1284,35 @@ mod tests { } } } + +#[cfg(test)] +mod documentation_limit_tests { + use super::checked_documentation_bytes; + use compass_languages::Engine; + + #[test] + fn aggregate_documentation_is_checked_at_the_exact_byte_boundary() + -> Result<(), Box> { + let batch = Engine::default() + .extract_source_combined( + std::path::Path::new("sample.rs"), + "sample.rs", + b"/// documented\nfn work() {}\n", + )? + .graph + .semantic_evidence + .ok_or("missing evidence")?; + let size = batch + .declarations + .iter() + .filter_map(|declaration| declaration.documentation.as_ref()) + .map(|documentation| documentation.content.len()) + .sum::(); + assert!(size > 0); + let batches = vec![batch.clone(), batch]; + assert_eq!(checked_documentation_bytes(&batches, size * 2)?, size * 2); + assert!(checked_documentation_bytes(&batches, size * 2 - 1).is_err()); + assert_eq!(checked_documentation_bytes(&[], 0)?, 0); + Ok(()) + } +} diff --git a/crates/compass-resolve/src/evidence/mod.rs b/crates/compass-resolve/src/evidence/mod.rs index 2226378e6..2a7f1432a 100644 --- a/crates/compass-resolve/src/evidence/mod.rs +++ b/crates/compass-resolve/src/evidence/mod.rs @@ -28,6 +28,7 @@ mod resolve; use projection::is_deferred_receiver; pub(crate) use projection::is_replaced_relation; +pub(crate) use projection::leading_documentation_diagnostics; pub use api::{ ResolutionDecision, ResolutionEvidence, ResolutionRule, UNIVERSAL_RESOLUTION_REPORT_EXTENSION, @@ -624,13 +625,13 @@ fn unique_definition_ranges( let Some(declaration) = declarations.get(owner_id) else { continue; }; - if !range_contains(&scope.range, &declaration.range) || ambiguous.contains(owner_id) { + if ambiguous.contains(owner_id) { continue; } - if ranges - .insert(owner_id.clone(), scope.range.clone()) - .is_some() - { + let Some(range) = definition_range_for_scope(declaration, scope) else { + continue; + }; + if ranges.insert(owner_id.clone(), range).is_some() { ranges.remove(owner_id); ambiguous.insert(owner_id.clone()); } @@ -638,6 +639,44 @@ fn unique_definition_ranges( ranges } +/// A directly owned ECMAScript initializer may begin after its binding name. +/// The producer records both the declaration start and the function scope's +/// exact owner/parent. Those facts establish the definition envelope without +/// borrowing an inherited callback scope or selecting by spelling/proximity. +fn definition_range_for_scope( + declaration: &DeclarationFact, + scope: &compass_languages::ScopeFact, +) -> Option { + if scope.owner_declaration_id.as_deref() != Some(declaration.id.as_str()) + || scope.language != declaration.language + { + return None; + } + if range_contains(&scope.range, &declaration.range) { + return Some(scope.range.clone()); + } + if !matches!(declaration.language.as_str(), "javascript" | "typescript") + || declaration.kind != "function" + || scope.kind != "function" + || declaration.scope_id.is_none() + || scope.parent_scope_id != declaration.scope_id + || declaration.definition_start_byte != Some(declaration.range.start_byte) + || scope.range.source_file != declaration.range.source_file + || declaration.range.start_byte >= declaration.range.end_byte + || declaration.range.end_byte > scope.range.start_byte + || scope.range.start_byte >= scope.range.end_byte + || (declaration.range.end_line, declaration.range.end_column) + > (scope.range.start_line, scope.range.start_column) + { + return None; + } + let mut range = declaration.range.clone(); + range.end_byte = scope.range.end_byte; + range.end_line = scope.range.end_line; + range.end_column = scope.range.end_column; + Some(range) +} + fn range_contains(outer: &EvidenceRange, inner: &EvidenceRange) -> bool { outer.source_file == inner.source_file && outer.start_byte <= inner.start_byte @@ -645,3 +684,67 @@ fn range_contains(outer: &EvidenceRange, inner: &EvidenceRange) -> bool { && (outer.start_line, outer.start_column) <= (inner.start_line, inner.start_column) && (inner.end_line, inner.end_column) <= (outer.end_line, outer.end_column) } + +#[cfg(test)] +mod definition_envelope_tests { + use super::*; + + #[test] + fn initializer_envelopes_reject_borrowed_scopes_and_ambiguous_owners() + -> Result<(), Box> { + let range = |start, end| { + serde_json::json!({ + "sourceFile":"sample.ts", "startByte":start, "endByte":end, + "startLine":1, "startColumn":start, "endLine":1, "endColumn":end + }) + }; + let declaration: DeclarationFact = serde_json::from_value(serde_json::json!({ + "id":"declaration:opaque", "language":"typescript", "graphNodeId":"opaque", + "kind":"function", "name":"opaque", "qualifiedName":"sample.opaque", + "scopeId":"scope:module", "definitionStartByte":0, "range":range(0,6) + }))?; + let primary: compass_languages::ScopeFact = serde_json::from_value(serde_json::json!({ + "id":"scope:primary", "language":"typescript", "kind":"function", + "ownerDeclarationId":declaration.id, "parentScopeId":"scope:module", + "range":range(9,40) + }))?; + let envelope = definition_range_for_scope(&declaration, &primary) + .ok_or("direct initializer lost its envelope")?; + assert_eq!((envelope.start_byte, envelope.end_byte), (0, 40)); + for case in 0..8 { + let mut scope = primary.clone(); + let mut owner = declaration.clone(); + match case { + 0 => scope.parent_scope_id = Some(primary.id.clone()), + 1 => scope.owner_declaration_id = Some("declaration:other".to_owned()), + 2 => scope.kind = "block".to_owned(), + 3 => scope.range.source_file = "other.ts".to_owned(), + 4 => owner.definition_start_byte = None, + 5 => owner.scope_id = None, + 6 => { + scope.language = "go".to_owned(); + owner.language = "go".to_owned(); + } + _ => owner.kind = "variable".to_owned(), + } + assert!( + definition_range_for_scope(&owner, &scope).is_none(), + "case {case}" + ); + } + let declarations = FactTable::from_values(vec![declaration.clone()])?; + let mut callback = primary.clone(); + callback.id = "scope:callback".to_owned(); + callback.parent_scope_id = Some(primary.id.clone()); + let scopes = FactTable::from_values(vec![callback, primary.clone()])?; + assert_eq!( + unique_definition_ranges(&declarations, &scopes).get(&declaration.id), + Some(&envelope) + ); + let mut conflict = primary.clone(); + conflict.id = "scope:conflict".to_owned(); + let scopes = FactTable::from_values(vec![conflict, primary])?; + assert!(unique_definition_ranges(&declarations, &scopes).is_empty()); + Ok(()) + } +} diff --git a/crates/compass-resolve/src/evidence/projection/documentation.rs b/crates/compass-resolve/src/evidence/projection/documentation.rs new file mode 100644 index 000000000..ae67b9219 --- /dev/null +++ b/crates/compass-resolve/src/evidence/projection/documentation.rs @@ -0,0 +1,188 @@ +//! Source-proven leading documentation; no spelling or proximity resolution. +use super::*; +use compass_model::provenance::{ + LEADING_DOCUMENTATION_OWNER_ANCHOR_ATTRIBUTE, LEADING_DOCUMENTATION_RULE, +}; + +impl UniversalResolutionIndex { + pub(super) fn materialize_leading_documentation( + &self, + nodes: &mut Vec, + edges: &mut Vec, + graph_ids: &AHashMap, + ) { + let mut declarations = self + .facts + .declarations + .values() + .filter(|declaration| declaration.documentation.is_some()) + .collect::>(); + if declarations.is_empty() { + return; + } + declarations.sort_unstable_by(|left, right| left.id.cmp(&right.id)); + let mut present = nodes + .iter() + .map(|node| node.id.clone()) + .collect::>(); + for declaration in declarations { + let (Some(documentation), Some(definition), Some(owner_id)) = ( + &declaration.documentation, + self.facts.definition_ranges.get(&declaration.id), + graph_ids.get(&declaration.id), + ) else { + continue; + }; + if !present.contains(owner_id) { + continue; + } + let owner_range = published_declaration_source_range(declaration, Some(definition)); + let range = &documentation.range; + if range.source_file != owner_range.source_file + || range.end_byte > owner_range.start_byte + || (range.end_line, range.end_column) + > (owner_range.start_line, owner_range.start_column) + { + continue; + } + let identity = format!("{}:{}", range.start_byte, range.end_byte); + let id = make_id(&[ + LEADING_DOCUMENTATION_RULE, + owner_id, + &range.source_file, + &identity, + ]); + if !present.insert(id.clone()) { + continue; + } + let mut attributes = Map::from_iter([ + ( + "label".to_owned(), + Value::String(format!("Documentation for {}", declaration.name)), + ), + ( + "qualified_name".to_owned(), + Value::String(format!( + "{}::leading_documentation@{identity}", + declaration.qualified_name + )), + ), + ( + "symbol_kind".to_owned(), + Value::String("resource".to_owned()), + ), + ( + "file_type".to_owned(), + Value::String("rationale".to_owned()), + ), + ( + "resource_kind".to_owned(), + Value::String("rationale".to_owned()), + ), + ( + "resource_content".to_owned(), + Value::String(documentation.content.clone()), + ), + ( + "source_file".to_owned(), + Value::String(range.source_file.clone()), + ), + ("source_anchor".to_owned(), source_anchor_value(range)), + ( + "source_location".to_owned(), + Value::String(format!("L{}", range.start_line)), + ), + ( + "language".to_owned(), + Value::String(declaration.language.clone()), + ), + ( + "extractor".to_owned(), + Value::String(format!( + "compass.languages.{}.universal", + declaration.language + )), + ), + ( + "confidence".to_owned(), + Value::String("EXTRACTED".to_owned()), + ), + ("_origin".to_owned(), Value::String("ast".to_owned())), + ]); + nodes.push(NodeRecord { + id: id.clone(), + attributes: attributes.clone(), + }); + attributes.remove("label"); + attributes.remove("qualified_name"); + attributes.remove("symbol_kind"); + attributes.remove("file_type"); + attributes.remove("resource_kind"); + attributes.remove("resource_content"); + attributes.insert("relation".to_owned(), Value::String("documents".to_owned())); + attributes.insert( + "rule".to_owned(), + Value::String(LEADING_DOCUMENTATION_RULE.to_owned()), + ); + attributes.insert( + OCCURRENCE_RULE_ATTRIBUTE.to_owned(), + Value::String(LEADING_DOCUMENTATION_RULE.to_owned()), + ); + attributes.insert( + LEADING_DOCUMENTATION_OWNER_ANCHOR_ATTRIBUTE.to_owned(), + source_anchor_value(owner_range), + ); + edges.push(EdgeRecord { + source: id, + target: owner_id.clone(), + attributes, + }); + } + } +} + +/// A bounded summary travels with the graph instead of disappearing when the +/// transient per-file evidence batches are consumed by the resolver. +pub(crate) fn leading_documentation_diagnostics( + batches: &[SemanticEvidenceBatch], +) -> Vec { + use compass_languages::DECLARATION_DOCUMENTATION_OMISSION_CODE; + use compass_model::code_graph::{DiagnosticSeverity, GraphDiagnostic}; + let extraction_omissions = batches + .iter() + .flat_map(|batch| &batch.diagnostics) + .filter(|diagnostic| diagnostic.code == DECLARATION_DOCUMENTATION_OMISSION_CODE) + .count(); + let declarations = batches + .iter() + .flat_map(|batch| &batch.declarations) + .filter(|declaration| declaration.documentation.is_some()) + .collect::>(); + let ranges = batch_definition_ranges(batches, &declarations); + let projection_omissions = declarations + .iter() + .filter(|declaration| { + let Some(documentation) = &declaration.documentation else { + return false; + }; + ranges.get(&declaration.id).is_none_or(|definition| { + documentation.range.source_file != definition.source_file + || documentation.range.end_byte > definition.start_byte + || (documentation.range.end_line, documentation.range.end_column) + > (definition.start_line, definition.start_column) + }) + }) + .count(); + if extraction_omissions == 0 && projection_omissions == 0 { + return Vec::new(); + } + vec![GraphDiagnostic { + severity: DiagnosticSeverity::Warning, + code: "leading_documentation_omission_summary".to_owned(), + message: format!( + "Leading documentation is incomplete: {extraction_omissions} extraction warning records (at most one per file) report excerpt/attachment limits; {projection_omissions} excerpts lack a unique, non-overlapping declaration definition range. Structural declarations are retained. Shorten oversized groups or use a supported contiguous AST attachment; ambiguous definition ownership requires extractor qualification." + ), + anchor: None, + related_ids: Vec::new(), + }] +} diff --git a/crates/compass-resolve/src/evidence/projection/mod.rs b/crates/compass-resolve/src/evidence/projection/mod.rs index c7aee3673..788b6b179 100644 --- a/crates/compass-resolve/src/evidence/projection/mod.rs +++ b/crates/compass-resolve/src/evidence/projection/mod.rs @@ -3,9 +3,11 @@ use super::*; use crate::ResolutionAdmission; +mod documentation; mod edges; mod nodes; +pub(crate) use documentation::leading_documentation_diagnostics; pub(crate) use edges::is_replaced_relation; use edges::*; pub(in crate::evidence) use nodes::is_deferred_receiver; @@ -127,6 +129,7 @@ impl UniversalResolutionIndex { } } profile_internal("universal declaration projection", &mut profile_started); + self.materialize_leading_documentation(nodes, edges, &graph_ids); self.materialize_python_call_sites(nodes, &graph_ids); let inventory_kinds = nodes .iter() @@ -744,13 +747,13 @@ fn batch_definition_ranges( let Some(declaration) = declarations.get(owner_id.as_str()) else { continue; }; - if !range_contains(&scope.range, &declaration.range) || ambiguous.contains(owner_id) { + if ambiguous.contains(owner_id) { continue; } - if ranges - .insert(owner_id.clone(), scope.range.clone()) - .is_some() - { + let Some(range) = definition_range_for_scope(declaration, scope) else { + continue; + }; + if ranges.insert(owner_id.clone(), range).is_some() { ranges.remove(owner_id); ambiguous.insert(owner_id.clone()); } diff --git a/crates/compass-resolve/src/evidence/projection/nodes.rs b/crates/compass-resolve/src/evidence/projection/nodes.rs index ef03497ea..a80415492 100644 --- a/crates/compass-resolve/src/evidence/projection/nodes.rs +++ b/crates/compass-resolve/src/evidence/projection/nodes.rs @@ -170,10 +170,11 @@ pub(super) fn declaration_node( if declaration.language == "python" && declaration.range.source_file.ends_with(".pyi") { attributes.insert("source_kind".to_owned(), Value::String("stub".to_owned())); } - if let Some(definition_range) = definition_range.filter(|range| *range != &declaration.range) { + let published_range = published_declaration_source_range(declaration, definition_range); + if published_range != &declaration.range { attributes.insert( "source_anchor".to_owned(), - source_anchor_value(definition_range), + source_anchor_value(published_range), ); attributes.insert( NODE_PROVENANCE_ANCHOR_ATTRIBUTE.to_owned(), @@ -186,6 +187,13 @@ pub(super) fn declaration_node( } } +pub(super) fn published_declaration_source_range<'a>( + declaration: &'a DeclarationFact, + definition_range: Option<&'a EvidenceRange>, +) -> &'a EvidenceRange { + definition_range.unwrap_or(&declaration.range) +} + /// Preserve the public callable spelling emitted by pre-universal extractors. /// Universal evidence keeps its richer module-qualified name for resolution, /// while framework route contracts continue to identify source callables as diff --git a/crates/compass-resolve/src/lib.rs b/crates/compass-resolve/src/lib.rs index a22096ee7..cee87602d 100644 --- a/crates/compass-resolve/src/lib.rs +++ b/crates/compass-resolve/src/lib.rs @@ -1034,7 +1034,10 @@ fn finish_resolution( } = mode; let mut profile_started = Instant::now(); let canonical_root = std::fs::canonicalize(root).unwrap_or_else(|_| root.to_path_buf()); - let stub_diagnostics = prepare_python_source_stubs(&mut evidence_batches); + let mut stub_diagnostics = prepare_python_source_stubs(&mut evidence_batches); + stub_diagnostics.extend(evidence::leading_documentation_diagnostics( + &evidence_batches, + )); if !stub_diagnostics.is_empty() { let mut diagnostics = merged .extensions diff --git a/crates/compass-resolve/tests/callable_initializer_definitions.rs b/crates/compass-resolve/tests/callable_initializer_definitions.rs new file mode 100644 index 000000000..4ead767d4 --- /dev/null +++ b/crates/compass-resolve/tests/callable_initializer_definitions.rs @@ -0,0 +1,112 @@ +use std::{collections::HashMap, error::Error}; + +use compass_graph::{BuildEvidence, normalize_v1}; +use compass_languages::Engine; +use compass_model::code_graph::NodeKind; +use compass_model::provenance::EvidenceOrigin; +use compass_resolve::resolve_with_root; +use serde::Deserialize; + +#[derive(Deserialize)] +struct Case { + extensions: Vec, + source: String, + definition: String, +} + +#[test] +fn initializer_source_envelopes_preserve_identity_and_relationship_records() +-> Result<(), Box> { + let cases: Vec = serde_json::from_str(include_str!( + "../../../fixtures/callable_initializer_definitions.json" + ))?; + for (index, case) in cases.iter().enumerate() { + for extension in &case.extensions { + for seed in ["alpha", "zeta"] { + let file = format!("{seed}_{index}.{extension}"); + let directory = tempfile::tempdir()?; + let path = directory.path().join(&file); + std::fs::write(&path, &case.source)?; + let extraction = Engine::default().extract_source_graph_only( + &path, + &file, + case.source.as_bytes(), + )?; + assert!(extraction.error.is_none(), "{file}: {:?}", extraction.error); + let sources = + HashMap::from([(path.to_string_lossy().into_owned(), case.source.clone())]); + let resolved = resolve_with_root(&[extraction], &sources, directory.path()); + let mut previous = resolved.clone(); + for node in &mut previous.nodes { + node.attributes.remove("source_anchor"); + node.attributes + .remove(compass_model::provenance::NODE_PROVENANCE_ANCHOR_ATTRIBUTE); + } + let publish = |raw: compass_languages::Extraction| -> Result<_, Box> { + let evidence = BuildEvidence::from_extraction( + directory.path(), + &raw, + "sha256:initializer-fixture", + )?; + let graph = normalize_v1(raw, evidence)?; + compass_model::validate_code_graph(&graph)?; + Ok(graph) + }; + let graph = publish(resolved)?; + let previous = publish(previous)?; + let targets = graph + .nodes + .iter() + .filter(|node| { + node.kind == NodeKind::Function + && node.name.trim_end_matches("()") == "opaque" + }) + .collect::>(); + assert_eq!(targets.len(), 1, "{file}: ambiguous fixture declaration"); + let target = targets[0]; + let source = target.source.as_ref().ok_or("missing callable source")?; + let start = case + .source + .find(&case.definition) + .ok_or("missing definition")?; + assert_eq!(source.start_byte as usize, start, "{file}"); + assert_eq!( + source.end_byte as usize, + start + case.definition.len(), + "{file}: missing initializer body" + ); + assert_eq!( + &case.source[source.start_byte as usize..source.end_byte as usize], + case.definition, + "{file}" + ); + assert!( + target + .evidence + .iter() + .any(|proof| proof.origin == EvidenceOrigin::Ast + && proof + .anchors + .iter() + .any(|anchor| anchor.start_byte as usize == start + && anchor.end_byte as usize == start + "opaque".len())), + "{file}: binding provenance changed" + ); + assert_eq!( + graph.nodes.iter().map(|node| &node.id).collect::>(), + previous + .nodes + .iter() + .map(|node| &node.id) + .collect::>(), + "{file}: definition envelope changed stable identities" + ); + assert_eq!( + graph.links, previous.links, + "{file}: definition envelope changed relationship occurrences" + ); + } + } + } + Ok(()) +} diff --git a/crates/compass-resolve/tests/evidence_resolution_contract.rs b/crates/compass-resolve/tests/evidence_resolution_contract.rs index b36dae917..0ae113fc8 100644 --- a/crates/compass-resolve/tests/evidence_resolution_contract.rs +++ b/crates/compass-resolve/tests/evidence_resolution_contract.rs @@ -53,6 +53,7 @@ fn declaration( signature_hash: None, implementation_hash: None, source_hash: None, + documentation: None, definition_start_byte: None, range: range(start, start.saturating_add(6)), } diff --git a/crates/compass-resolve/tests/inference_admission.rs b/crates/compass-resolve/tests/inference_admission.rs index ce4108a9e..3fed21d2f 100644 --- a/crates/compass-resolve/tests/inference_admission.rs +++ b/crates/compass-resolve/tests/inference_admission.rs @@ -55,6 +55,7 @@ fn external_call_batch() -> SemanticEvidenceBatch { signature_hash: None, implementation_hash: None, source_hash: None, + documentation: None, definition_start_byte: None, range: range(0, 10), }], diff --git a/crates/compass-resolve/tests/leading_documentation.rs b/crates/compass-resolve/tests/leading_documentation.rs new file mode 100644 index 000000000..f68ccef40 --- /dev/null +++ b/crates/compass-resolve/tests/leading_documentation.rs @@ -0,0 +1,194 @@ +use std::collections::HashMap; +use std::error::Error; + +use compass_graph::{BuildEvidence, normalize_v1}; +use compass_languages::Engine; +use compass_model::code_graph::EdgeKind; +use compass_model::provenance::{LEADING_DOCUMENTATION_RULE, is_proven_leading_documentation}; +use compass_resolve::resolve_with_root; + +#[test] +fn leading_documents_survive_real_extraction_resolution_and_strict_publication() +-> Result<(), Box> { + for (file, source, expected) in [ + ( + "src/lib.rs", + "/// Retains protocol error context.\npub fn work() {}\n", + 1, + ), + ( + "sample.go", + "package sample\n// Retains protocol error context.\nfunc Work() {}\n", + 1, + ), + ( + "sample.ts", + "/** Retains protocol error context. */\nexport function work() {}\n", + 1, + ), + ( + "sample.ts", + "/** Number overload. */\nexport function convert(input: number): number;\n/** String overload. */\nexport function convert(input: string): string;\n/** Retains protocol error context. */\nexport function convert(input: unknown): unknown { return input; }\n", + 3, + ), + ] { + let directory = tempfile::tempdir()?; + let path = directory.path().join(file); + std::fs::create_dir_all(path.parent().ok_or("missing parent")?)?; + std::fs::write(&path, source)?; + let extraction = Engine::default().extract(&path)?; + let resolved = resolve_with_root( + &[extraction], + &HashMap::from([(path.to_string_lossy().into_owned(), source.to_owned())]), + directory.path(), + ); + let build = BuildEvidence::from_extraction( + directory.path(), + &resolved, + "sha256:leading-documentation-test", + )?; + let graph = normalize_v1(resolved, build)?; + compass_model::validate_code_graph(&graph)?; + let edges = graph + .links + .iter() + .filter(|edge| { + edge.occurrence_rule.as_ref().map(|rule| rule.as_str()) + == Some(LEADING_DOCUMENTATION_RULE) + }) + .collect::>(); + assert_eq!( + edges.len(), + expected, + "{file}: {}", + serde_json::to_string(&graph)? + ); + for edge in edges { + assert_eq!(edge.kind, EdgeKind::Documents); + let document = graph + .nodes + .iter() + .find(|node| node.id == edge.source) + .ok_or("missing document")?; + let owner = graph + .nodes + .iter() + .find(|node| node.id == edge.target) + .ok_or("missing owner")?; + assert!(is_proven_leading_documentation(document, owner, edge)); + let anchor = document.source.as_ref().ok_or("missing excerpt")?; + assert_eq!(anchor.file, file); + let excerpt = source + .get(usize::try_from(anchor.start_byte)?..usize::try_from(anchor.end_byte)?) + .ok_or("invalid excerpt")?; + let Some(compass_model::code_graph::NodeDetails::Resource(details)) = &document.details + else { + return Err("not a resource".into()); + }; + assert_eq!(details.content.as_deref(), Some(excerpt)); + assert!( + owner + .source + .as_ref() + .is_some_and(|definition| definition.start_byte >= anchor.end_byte) + ); + } + } + Ok(()) +} + +#[test] +fn excerpt_limits_publish_a_specific_omission_warning() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let path = directory.path().join("sample.rs"); + let source = format!("/** {} */\npub fn work() {{}}\n", "x".repeat(4096)); + std::fs::write(&path, &source)?; + let extraction = Engine::default().extract(&path)?; + let resolved = resolve_with_root( + &[extraction], + &HashMap::from([(path.to_string_lossy().into_owned(), source)]), + directory.path(), + ); + let build = BuildEvidence::from_extraction( + directory.path(), + &resolved, + "sha256:omitted-documentation-test", + )?; + let graph = normalize_v1(resolved, build)?; + assert!(graph.nodes.iter().any(|node| node.name == "work()")); + assert!( + graph + .graph + .diagnostics + .iter() + .any( + |diagnostic| diagnostic.code == "leading_documentation_omission_summary" + && diagnostic.message.contains("1 extraction warning records") + ) + ); + assert!(graph.links.iter().all( + |edge| edge.occurrence_rule.as_ref().map(|rule| rule.as_str()) + != Some(LEADING_DOCUMENTATION_RULE) + )); + Ok(()) +} + +#[test] +fn missing_or_ambiguous_definition_scope_omits_docs_with_warning() -> Result<(), Box> { + for ambiguous in [false, true] { + let directory = tempfile::tempdir()?; + let path = directory.path().join("sample.rs"); + let source = "/// Protocol diagnostics parsed here.\npub fn work() {}\n"; + std::fs::write(&path, source)?; + let mut extraction = Engine::default().extract(&path)?; + let batch = extraction.semantic_evidence.as_mut().ok_or("no evidence")?; + let owner = batch + .declarations + .iter() + .find(|declaration| declaration.name == "work") + .ok_or("no work")? + .id + .clone(); + if ambiguous { + let mut duplicate = batch + .scopes + .iter() + .find(|scope| scope.owner_declaration_id.as_deref() == Some(&owner)) + .ok_or("no definition scope")? + .clone(); + duplicate.id.push_str(":duplicate"); + batch.scopes.push(duplicate); + } else { + batch + .scopes + .retain(|scope| scope.owner_declaration_id.as_deref() != Some(&owner)); + } + let resolved = resolve_with_root( + &[extraction], + &HashMap::from([(path.to_string_lossy().into_owned(), source.to_owned())]), + directory.path(), + ); + let build = BuildEvidence::from_extraction( + directory.path(), + &resolved, + "sha256:missing-doc-owner-test", + )?; + let graph = normalize_v1(resolved, build)?; + assert!(graph.nodes.iter().any(|node| node.name == "work()")); + assert!( + graph + .graph + .diagnostics + .iter() + .any( + |diagnostic| diagnostic.code == "leading_documentation_omission_summary" + && diagnostic.message.contains("1 excerpts lack a unique") + ) + ); + assert!(graph.links.iter().all(|edge| { + edge.occurrence_rule.as_ref().map(|rule| rule.as_str()) + != Some(LEADING_DOCUMENTATION_RULE) + })); + } + Ok(()) +} diff --git a/crates/compass-resolve/tests/source_query_evidence.rs b/crates/compass-resolve/tests/source_query_evidence.rs new file mode 100644 index 000000000..322ca8b88 --- /dev/null +++ b/crates/compass-resolve/tests/source_query_evidence.rs @@ -0,0 +1,59 @@ +use compass_graph::{BuildEvidence, canonical_graph_json, normalize_v1}; +use compass_languages::{Engine, source_query::EXTRACTION_EXTENSION}; +use compass_resolve::resolve_with_root; +use std::collections::HashMap; +use std::error::Error; + +#[test] +fn lexical_body_metadata_does_not_change_published_code_records() -> Result<(), Box> { + for (file, source) in [ + ( + "src/sample.py", + "def helper(payload):\n return payload\ndef opaque(payload):\n # handle annotations\n return helper(payload)\n", + ), + ( + "src/sample.rs", + "fn helper(payload: &str) -> &str { payload }\nfn opaque(payload: &str) -> &str { // handle annotations\nhelper(payload) }\n", + ), + ( + "sample.go", + "package sample\nfunc helper(payload string) string { return payload }\nfunc Opaque(payload string) string { // handle annotations\nreturn helper(payload) }\n", + ), + ( + "src/sample.ts", + "function helper(payload: unknown) { return payload; }\nexport function opaque(payload: unknown) { // handle annotations\nreturn helper(payload); }\n", + ), + ] { + let directory = tempfile::tempdir()?; + let path = directory.path().join(file); + std::fs::create_dir_all(path.parent().ok_or("missing parent")?)?; + std::fs::write(&path, source)?; + let extraction = + Engine::default().extract_source_graph_only(&path, file, source.as_bytes())?; + assert!(extraction.error.is_none(), "{file}: {:?}", extraction.error); + let mut without = extraction.clone(); + assert!( + without.extensions.remove(EXTRACTION_EXTENSION).is_some(), + "{file}" + ); + let sources = HashMap::from([(path.to_string_lossy().into_owned(), source.to_owned())]); + let mut canonical = Vec::new(); + for input in [extraction, without] { + let resolved = resolve_with_root(&[input], &sources, directory.path()); + let evidence = BuildEvidence::from_extraction( + directory.path(), + &resolved, + "sha256:source-query-invariance", + )?; + let graph = normalize_v1(resolved, evidence)?; + compass_model::validate_code_graph(&graph)?; + assert!(!graph.nodes.is_empty() && !graph.links.is_empty(), "{file}"); + canonical.push(canonical_graph_json(&graph)?); + } + assert_eq!( + canonical[0], canonical[1], + "query-only evidence changed graph records: {file}" + ); + } + Ok(()) +} diff --git a/crates/compass-resolve/tests/typescript_member_scopes.rs b/crates/compass-resolve/tests/typescript_member_scopes.rs new file mode 100644 index 000000000..fdd6e57cf --- /dev/null +++ b/crates/compass-resolve/tests/typescript_member_scopes.rs @@ -0,0 +1,120 @@ +use std::collections::HashMap; +use std::error::Error; + +use compass_graph::{BuildEvidence, normalize_v1}; +use compass_languages::Engine; +use compass_model::code_graph::EdgeKind; +use compass_model::provenance::{EvidenceConfidence, EvidenceOrigin}; +use compass_resolve::resolve_with_root; +use serde::Deserialize; + +#[derive(Deserialize)] +struct Expected { + kind: String, + line: u32, +} + +#[derive(Deserialize)] +struct Case { + prefix: String, + kind: String, + owner: String, + target: String, + relation: String, + expected: Option, + typescript: String, + javascript: String, +} + +fn terminal(name: &str) -> &str { + name.trim_start_matches('.').trim_end_matches("()") +} + +fn check_graphs(ambiguous: bool) -> Result<(), Box> { + let cases: Vec = serde_json::from_str(include_str!( + "../../../fixtures/typescript_member_scopes.json" + ))?; + let directory = tempfile::tempdir()?; + let mut engine = Engine::default(); + for case in cases { + if (case.kind == "ambiguous") != ambiguous { + continue; + } + for (extension, source) in [("ts", &case.typescript), ("js", &case.javascript)] { + for seed in 0..24 { + let file = format!("{}{:02}.{extension}", case.prefix, seed); + let path = directory.path().join(&file); + std::fs::write(&path, source)?; + let extraction = + engine.extract_source_graph_only(&path, &file, source.as_bytes())?; + assert!(extraction.error.is_none(), "{file}: {:?}", extraction.error); + let sources = + HashMap::from([(path.to_string_lossy().into_owned(), source.clone())]); + let resolved = resolve_with_root(&[extraction], &sources, directory.path()); + let evidence = BuildEvidence::from_extraction( + directory.path(), + &resolved, + "sha256:primary-member-scope-fixture", + )?; + let graph = normalize_v1(resolved, evidence)?; + compass_model::validate_code_graph(&graph)?; + let kind = match case.relation.as_str() { + "calls" => EdgeKind::Calls, + "accesses_member" => EdgeKind::References, + _ => return Err("unsupported fixture relation".into()), + }; + let matches = graph + .links + .iter() + .filter(|edge| edge.kind == kind) + .filter_map(|edge| { + let owner = graph.nodes.iter().find(|node| node.id == edge.source)?; + let target = graph.nodes.iter().find(|node| node.id == edge.target)?; + (terminal(&owner.name) == case.owner + && terminal(&target.name) == case.target) + .then_some((owner, edge, target)) + }) + .collect::>(); + let Some(expected) = &case.expected else { + assert!(matches.is_empty(), "{file}: invented accessor target"); + continue; + }; + assert_eq!(matches.len(), 1, "{file}: occurrence multiplicity"); + let (owner, edge, target) = matches[0]; + assert_eq!(target.kind.as_str(), expected.kind, "{file}"); + let target_source = target.source.as_ref().ok_or("missing target source")?; + assert_eq!(target_source.file, file); + assert_eq!(target_source.start_line, expected.line, "{file}"); + let site = edge + .relationship_site + .as_ref() + .ok_or("missing occurrence")?; + let owner_source = owner.source.as_ref().ok_or("missing owner source")?; + assert_eq!(site.file, file); + assert!(owner_source.start_byte <= site.start_byte); + assert!(site.end_byte <= owner_source.end_byte); + let snippet = source + .get(usize::try_from(site.start_byte)?..usize::try_from(site.end_byte)?) + .ok_or("invalid occurrence bytes")?; + assert!(snippet.contains(&case.target), "{file}: {snippet}"); + assert!(edge.occurrence_rule.is_some()); + assert!(edge.evidence.iter().any(|proof| { + proof.origin == EvidenceOrigin::Ast + && proof.confidence == EvidenceConfidence::Exact + })); + } + } + } + Ok(()) +} + +#[test] +fn ambiguous_accessors_never_publish_hash_selected_references() -> Result<(), Box> { + check_graphs(true) +} + +#[test] +fn primary_member_scopes_preserve_published_call_and_reference_proofs() -> Result<(), Box> +{ + check_graphs(false) +} diff --git a/docs/design/language-architecture.md b/docs/design/language-architecture.md index 4007969b7..bf6a6fd57 100644 --- a/docs/design/language-architecture.md +++ b/docs/design/language-architecture.md @@ -29,10 +29,10 @@ This architecture is transitioning one language at a time. The status labels bel | Status | Behavior | | --- | --- | | Available now | The vendored package supplies 37 pinned static Tree-sitter grammars | -| Available now | Python, Go, Rust, Java, PHP, Kotlin, Ruby, TypeScript, JavaScript, Swift, Dart, Scala, and Groovy are registered hard-cut `Qualified` evidence pipelines: they emit semantic evidence and use shared resolution and projection | -| Available now | Rust is a quality-gated, hard-cut version-1 `Qualified` pipeline; version 1 preserves bounded multi-stage method-result chains across files while retaining source-proven fallbacks when project-wide result evidence is absent, alongside the earlier associated-type, generic-parameter, re-export, and lexical-call safeguards; replaced publisher and collection resolution branches remain removed | +| Available now | All registered hard-cut evidence pipelines emit semantic evidence through shared resolution and projection; current Go2, Rust3, JavaScript 3 and TypeScript 4 development producers are `Qualifying` for leading-documentation and primary-member-scope changes, while unchanged approved producers remain `Qualified` | +| Available now | Rust is a hard-cut version-3 `Qualifying` development pipeline for leading documentation; its existing bounded source semantics preserve bounded multi-stage method-result chains across files while retaining source-proven fallbacks when project-wide result evidence is absent, alongside the earlier associated-type, generic-parameter, re-export, and lexical-call safeguards; replaced publisher and collection resolution branches remain removed | | Available now | Java is a hard-cut version-1 `Qualified` pipeline; its replaced publisher and Java member resolver are removed, and post-cutover pinned-corpus qualification is complete | -| Available now | TypeScript and JavaScript are hard-cut `Qualified` pipelines; TSX uses the TypeScript identity, both share the bounded ECMAScript producer, and their replaced generic publisher is removed | +| Available now | TypeScript 4 and JavaScript 3 are `Qualifying` for primary member scopes, with TypeScript retaining leading documentation; TSX uses the TypeScript identity, both share the bounded ECMAScript producer, and their replaced generic publisher is removed | | Available now | PHP is a hard-cut version-1 `Qualified` pipeline with explicit case-insensitive type/function/method identity, bounded Composer PSR-4 evidence, conservative trait/inheritance dispatch, and universal Laravel/Drupal source packs; Drupal configuration and Blade template extraction remain available | | Available now | Kotlin is a hard-cut version-1 `Qualified` pipeline with packages, imports, nominal and companion declarations, constructors, functions and extensions, properties, annotations, generic and nullable types, and named/default argument evidence; compiler-only behavior remains explicitly unresolved | | Available now | Ruby is a hard-cut version-1 `Qualified` pipeline; its dedicated producer, method-space-aware resolver policy, replaced Ruby member publisher, and Rails `rails-ruby` universal pack are active under the release promotion decision | @@ -102,13 +102,24 @@ The hard-cut route is selected by `UniversalEvidenceRegistry`. Presence in the s registry or availability of a grammar does not select it. On the current branch, the registry contains C#, Dart, Go, Groovy, Java, JavaScript, Kotlin, PHP, Python, Ruby, Rust, Scala, Swift, and TypeScript. All registered universal -evidence producers are at version 1. `Qualified` records that +evidence producer versions are recorded individually in the runtime registry and +release-promotion decision. Changed development versions remain `Qualifying`. +`Qualified` records that the release promotion gates passed; it does not claim compiler-grade completeness or re-enable the removed direct route. Producer-version changes invalidate cached evidence for only the changed language. Go identities retain the repository-relative directory prefix and use the parsed package clause to distinguish external test packages. +Direct variable-bound ECMAScript functions retain their binding anchors for +identity and provenance while publishing a uniquely owned function definition +range. TypeScript/TSX lexical body evidence uses the exact declarator envelope; +inherited callbacks and ambiguous scopes cannot supply it. This source metadata +change does not invent semantic relationships. Source-query profile 2 keeps +immutable profile 1 artifacts readable; JavaScript body-query evidence is not +added by this change. The updated producers remain `Qualifying` pending source +audit and release promotion. + ## Grammar substrate The vendored `compass-tree-sitter-language-pack` package owns parser availability and reproducibility. diff --git a/docs/reference/commands.md b/docs/reference/commands.md index d4dd0c080..c31366889 100644 --- a/docs/reference/commands.md +++ b/docs/reference/commands.md @@ -302,6 +302,68 @@ quality artifact are the same as `cluster-only`. ### `query` +Current `compass update` builds also retain bounded callable-body identifiers +and comments for Python, Rust, Go and TypeScript/TSX in `source-query.json`. +Broad questions can use this evidence without naming a symbol exactly. Matches +that depend on body terms remain Low inspection leads with source locations; +a word occurring in a function does not prove the requested behavior. Keep this +artifact with its matching graph snapshot. Missing older evidence is reported; +present corrupt/mismatched artifacts fail explicitly and can be rebuilt with +`compass update --force`. Historical realizations remain unchanged. + +For an implementation shortlist, opt into the ranked view: + +```bash +compass query "where are request errors formatted?" --view ranked +compass query "where are request errors formatted?" --view ranked --format json +compass query "where are request errors formatted?" --view ranked --include-context +``` + +This view presents up to five implementation owners with source locations, +reasons, ordinal relevance and separate identity/evidence labels. It reports +known eligible counts or bounded recall and suggests a narrower scope. +`--text-budget` bounds the whole text answer, including metadata. Ranked JSON +uses `compass.query.ranked-view/4`. Do not combine ranked view with discovery +cursors, result envelopes, evidence, `agent-json`, or legacy traversal controls. +Default query output keeps its existing format and cursor contracts. +An existing node ID supplied as the whole question selects that admitted seed +alone for the shortlist. Optional structural context remains available. + +Broad lexical matches are inspection candidates. Unfamiliar ordinary wording +can lower relevance without suppressing every result; missing indexed context +is disclosed. Explicitly named, quoted or qualified subjects retain guards +for absent indexed terms. Negative conditions such as “without” are unverified: +the ranked candidates are Low and the response carries a qualification warning. +Inspect source to establish the behavior. Default discovery and search also +carry incomplete-coverage diagnostics for tentative candidates. + +When explanations would crowd out a ranked candidate, text tries a shorter +explanation layout first and marks the abbreviation. Use `--format json` for +more detail; relevance, uncertainty, citations and the total budget are retained. + +Broad method candidates may use exact AST call/reference sites inside their +source ranges. The `code_reference` field matches the referenced identifier, +with a supporting source citation. A result needing this evidence is a Low +inspection lead, not proof of the requested behavior. Target namespaces, paths +and prose cannot provide this field. Reached recall quotas remain visible. + +A generic kind or language match cannot satisfy an additional requested topic. +For example, “which function owns annotations” requires evidence for annotations; +the fact that an unrelated node is a function does not qualify it. Simple kind +and language listings remain available. + +Broad queries can also find the unique enclosing function through an exact AST +`contains` relationship to a matching local declaration. `owned_identifier` +retains the declaration-site citation. Such evidence supplies a Low inspection +lead; it does not establish the requested behavior. Ambiguous or truncated +parent lists never select a parent. + +MCP `query_graph` accepts the typed equivalent `view: "ranked"` and an optional +`text_budget` (default 800). Add `include_context: true` for bounded one-hop +witnessed context after the shortlist; CLI uses `--include-context`. Context is +disabled by default and keeps candidate order. Its `rankedView` field contains +the shortlist while `result` retains the raw discovery shape. + Natural-language graph discovery: ```text @@ -1319,6 +1381,16 @@ Managed integration/update probe. - `COMPASS_OUT` can change the default output root for several compatible command families; explicit `--out` is clearer in automation. +When a typed read uses a disposable JSON query index, its metadata and full +integrity validation share a 30-second startup budget. A slow eligible file check +can use a byte-identical, size-bounded inspection image; warm reads keep the +original file path. `query_index_validation_timeout` is an error, leaves the +index in place, and is not an empty result or a request to delete the cache. +Retry, or rebuild a narrower source scope with `compass update `. This +budget covers index validation, not graph loading or a new index build. +Metadata and integrity SQL use operation-local interruption in addition to +progress callbacks. Blocking filesystem calls may delay timeout reporting. + ## Store health and recovery ```text @@ -1461,3 +1533,25 @@ test. Exact-only lookup and source-bearing exploration remain native indexed queries; the response cache does not promise acceleration for those paths. + + +## Ranked implementation code windows (development) + +`compass query "where is payload normalized" --view ranked` can show a +`Code @ path:line` quotation even when a retrieved implementation has no comment. +The JSON form uses `compass.query.ranked-view/4` and excerpt kind `callable_code`. +Quotes come from the graph's immutable source artifact, at most two512byte +excerpts per displayed candidate within the whole-answer text budget. Code windows +can omit surrounding logic; inspect the full implementation before making a +behavioral claim. Omissions are counted independently of lexical search coverage. +Relevance remains a heuristic band, separate from graph evidence confidence. + +### Nested lexical inspection in ranked queries + +An already retrieved implementation can include exact `nested_callable_code` +quotations. The text label is “Nested lexical code (execution unverified)”; JSON +also provides the full `nestedScope` anchor. These are retained anonymous-callable +source windows linked by native AST ancestry. Named or separately published +callables and class/type boundaries remain separate. No call or runtime behavior +is inferred. Shared code-window and first-page limits can omit context, with an +explicit omission reason. Rank and graph relationships are unchanged. diff --git a/docs/reference/outputs.md b/docs/reference/outputs.md index dfb1b5073..05c7ebeb2 100644 --- a/docs/reference/outputs.md +++ b/docs/reference/outputs.md @@ -4,6 +4,157 @@ Compass outputs range from the current `compass-out/` directory to versioned CompassQL results and immutable history exports. This reference describes consumer responsibilities and authority. +## Compact discovery citations + +Default discovery pages show `SEED [exact|approximate|auto-picked]` +followed by ` @ ::-:` when the retained seed +has a source anchor. These citations precede relationship context, so a caller +can inspect selected candidates even when the node inventory falls on a later +page. Source-less seeds do not acquire a location. Paths use the same control +escaping as other source locations. Citations count toward `--text-budget`; +entry ordering, continuation positions and raw JSON remain unchanged. + +## Ranked query answers + +`query --view ranked --format json` produces `compass.query.ranked-view/4`. +MCP `query_graph` with `view: "ranked"` returns the same projection in its +`rankedView` envelope field, alongside the raw discovery `result`. + +Each displayed candidate may retain up to two `excerpts`, totaling at most 1,024 +UTF-8 bytes. A quotation includes `kind` (`callable_comment` or +`ast_documentation`), exact `source` coordinates, raw retained `text`, and +`clipped`. A retained window must still contain matching query-term evidence; +if clipping removes every match, omit the quotation and keep the ranked +candidate. Quotations come from the immutable graph-bound body index or the +versioned raw leading-documentation ownership proof. Generic document links, +normalized legacy docstrings and identifier-only sites do not supply quotations. +Current mutable source files are not read. Comment/documentation text is +inspection evidence; it does not establish runtime behavior or raise relevance. + +Text escapes controls, quotes, backslashes and bidirectional formatting marks. +It shortens or omits quotations before removing a ranked candidate to meet +`--text-budget`; omissions and display clipping are labeled. JSON retains the +bounded exact excerpts. The native reader also accepts ranked-view/1 without +excerpts; a version 1 payload containing quotations and unknown major versions +fail explicitly. + +The projection retains up to five ordered candidates independently of the graph +seed limit. Each candidate has a rank, stable graph ID, qualified name, +kind, source anchor when available, concise reasons, matched fields/terms, +an ordinal `relevance` (`high`, `medium`, `low`), and separate `nameMatch`, +`evidence`, and `retrievalKinds` values. Relevant source passages are grouped +under their owner. `supportingSources` retains up to four supporting passage +anchors separately from the declaration anchor; `omittedSupportingSources` +reports the remainder. A relevant approximate match can have exact structural +provenance; neither label proves implementation equivalence. Text labels each +candidate's node kind so resource/document matches are distinguishable from +implementation declarations. + +Low candidates can retain partial topic evidence when ordinary query words +are absent from the index. Their reasons name missing context instead of +claiming the behavior is established. Negative query conditions always keep +lexical candidates Low with an explicit unverified-condition reason. + +When known index statistics have no literal match for ordinary `entry` or +`snapshot` wording, broad retrieval may inspect indexed `item`/`element` or +`copy`/`clone` alternatives. A dependent candidate stays Low and its reason +names the translations and the unverified equivalence/behavior. Lexical-only +candidates need another literal non-action topic in source-owned fields; +paths, namespaces and generic facets do not supply that context. Existing +explicit semantic-channel admission remains separate. Named/quoted/qualified +identities, document requests and unavailable statistics do not use this +fallback. Index absence does not prove source absence. Existing literal-only +candidates retain their admission screen, and fallback candidates receive no +operation preference. No graph relationship or behavioral fact is inferred. + +The underlying discovery/search response also carries an `incomplete_coverage` +diagnostic so clients that do not request the ranked view retain the caveat. +JSON and local SQLite readers also retain graph publication and documentation +omission summaries. Older snapshots without diagnostic point projections use a +bounded supplement read; reaching that limit cannot be reported as complete +coverage or an empty diagnostic result. Discovery retains graph coverage +warnings even when no matching seed is found. + +For an implementation question with one recognized operation, +operation identifiers in a callable's name or local code can give it an +inspection preference over contextual mentions. Separate topic evidence is +required. Signature-overlapping identifiers cannot supply the code signal, +because parameter declarations are also indexed; missing/empty signatures, +native display signatures ending in `…`, or clipped search-field projections +keep ordinary ranking for code-based signals. This does not classify tokens +as runtime body operations. Compound, negative and temporal conditions retain ordinary +ranking. This preference is not execution proof and does not raise relevance or +structural confidence. Missing query context and Low labels remain visible. + +Hyphenated compounds such as `read-only` or `load-only` can prefer a callable +whose own name or one retained owner-local identifier contains those normalized +words contiguously. Separate identifiers, documentation, paths and namespaces +cannot supply that cohesion. A separate owner-field topic outside the compound +is required; a compound alone is insufficient. Code signals use the same +signature guards above. The preference covers up to eight compounds with two +to four words each; larger compound requests retain ordinary ranking. Explicit +identities, document/test intent, recognized negative/temporal conditions and +vocabulary-dependent matches retain their existing guards. Dependent candidates +stay Low and name the matched compounds, with behavior unverified. Matching +identifier anchors receive priority within the existing eight body-source cap; +omissions stay visible. This changes ranking/citation selection, not recall, +graph facts, source ownership or the ranked-view schema. + +The open `matchedFields` vocabulary includes `code_reference` for identifiers +on witnessed outgoing call/reference sites inside a callable. These matches map +to lexical retrieval and preserve the actual site in `supportingSources`. +Unquoted `before`, `after`, `during`, and `until` identify temporal conditions +that lexical retrieval cannot establish. Such candidates have Low relevance +and an explicit warning; the connectives do not contribute subject coverage. +Quoted names and qualified identifiers retain their literal terms. +Target namespaces, paths and documentation are excluded. When this evidence +supplies a concept absent from the owner's fields, relevance is Low (unless the +query matches its exact ID or full name). A reference demonstrates use of an +identifier, not implementation of the requested behavior. + +`kind` and `language` describe facets. They do not supply missing topic evidence +for a broad question. Removing facet-only candidates can shorten a shortlist or +produce no supporting indexed evidence; it does not establish that the requested +behavior is absent. Exact identity and simple facet listings retain their meaning. + +`owned_identifier` is another lexical matched field: a matching local name +belongs to one uniquely witnessed callable through exact AST containment. +`supportingSources` retains the local declaration site. Namespaces, paths and +nested callable names cannot supply it. Relevance is Low when this field +supplies an otherwise missing concept, unless the ID or full name is exact. +This evidence establishes ownership, not implementation of a behavior. + +`eligibleCount` is the total supported set when known; `null` means the bounded +retrieval could not establish a complete count. `retainedCount` records the +retained unique owners before the top-five projection. `moreAvailable`, +`candidateRecallBounded`, `responseTruncated`, `graphCoverage`, `scope`, and +`limits` expose limits and coverage. `responseTruncated` is discovery's aggregate +bound, including candidate, source-field and context limits; it does not assert +that traversal itself was truncated. Optional context's `truncated` flag also +retains upstream response bounds as well as omitted context rows. +`narrowing` gives a concrete source scope +or asks for a specific subject. Empty results abstain from asserting absence. + +The text renderer tries abbreviated explanations before removing a ranked +candidate. It marks this with `Reasons abbreviated`; `--format json` exposes +more detail. Source locations, relevance/identity/evidence labels, recall bounds +and uncertainty remain visible. This changes text packing, not ranking or the +JSON projection. + +The shared text renderer reserves header, limits and refinement guidance inside +the requested UTF-8 byte budget (approximately four bytes per token), drops +whole candidate rows when needed, and fails if one candidate plus metadata +cannot fit. Ranked views are bounded to 128 KiB; oversized machine projections +fail intact. This view uses no discovery cursor and does not change default +raw schemas or cursor semantics. + +`--include-context` / MCP `include_context: true` adds an optional `context` +object after the shortlist. It contains at most eight one-hop witnessed edges +and sixteen endpoint nodes, preserving edge direction, occurrence IDs, source +anchors and provenance. `omittedEdges` and `truncated` expose context limits. +Text drops whole context rows before candidate rows when the answer budget +requires it. Context is disabled by default and does not reorder candidates. + ## Current output directory Default: @@ -1490,3 +1641,23 @@ when the original request selected an immutable snapshot path. Freshness is chec answers. Standalone and historical artifacts without a source-root marker have no advisory check. Program IR currently has no build commit; its own marked artifact reports unknown freshness rather than borrowing a graph's revision. + + +## Ranked view version3 code excerpts (development) + +Ranked JSON now emits `compass.query.ranked-view/4`. Its `excerpts[].kind` +may be `callable_code`, `callable_comment`, or `ast_documentation`. Code and +comment anchors are contained in the owning candidate; attached documentation +precedes it. Text remains exact UTF-8 source with byte/line/column anchors. +A code window need not cover the full callable. Versions1/2 remain readable; +version2 rejects `callable_code` rather than interpreting it as a comment. + +### Ranked nested callable code + +Ranked view4 adds excerpt kind `nested_callable_code` and the optional +`nestedScope` source anchor. The field is required for that kind and forbidden +for direct code, comments and documentation. The scope must be strictly inside +the candidate's callable definition; the quoted slice must be inside that scope. +The label describes lexical nesting only. Existing byte, quotation-count and +text-budget limits still apply; omissions are explicit. Readers reject this +kind under views1–3 while continuing to read their original quotation kinds. diff --git a/docs/reference/universal-semantic-evidence.md b/docs/reference/universal-semantic-evidence.md index 47df287bd..e225d49f5 100644 --- a/docs/reference/universal-semantic-evidence.md +++ b/docs/reference/universal-semantic-evidence.md @@ -4,20 +4,77 @@ Compass resolves source relationships through a language-neutral evidence contract. Every hard-cut language uses the same production route. The release promotion decision in `tests/qualification/universal-evidence-promotion.json` records all 14 -registered pipelines as `Qualified`; they do not retain a second direct graph -publisher. Swift, Dart, Scala, and Groovy have complete independent audit +registered producer versions as `Qualified`; they do not retain a second direct +graph publisher. New development versions below are not covered by that decision. Swift, Dart, Scala, and Groovy have complete independent audit artifacts captured in the mounted qualification target for that decision. This is a hard-cutover interface. It has no raw-fact translation layer, shadow mode, terminal-name fallback, or runtime dependency on Graphify. +## In-progress leading documentation + +The development registry now marks Go2, Rust3 and TypeScript4 as `Qualifying`. +They emit optional `DeclarationFact.documentation` metadata under evidence/2: +an AST attachment kind, exact source range and raw UTF-8 comment text. The first +slice supports one contiguous leading group on supported named declarations; +it does not infer owners from nearby names. Python's existing rationale/docstring +path and JavaScript's documentation extraction remain unchanged. Publication/3 projects these +as rationale resources with a `documents` edge under the versioned +`universal-leading-documentation-v1` rule. The existing provenance vector +retains both the excerpt and unique full declaration range. Graph validation +rejects an inconsistent claimed proof; query ranker/4 uses the same verifier. +No comparative recall or release qualification is claimed for this metadata. + +Each excerpt is at most 4KiB, with at most 1MiB retained per file. Omitted groups +produce one bounded `leading_documentation_omitted` evidence diagnostic per file. +The resolver publishes a bounded `leading_documentation_omission_summary` +for extraction warnings or excerpts without a unique, non-overlapping full +definition range. JSON and store queries expose that summary as incomplete +coverage. Aggregate documentation is capped at 128MiB per resolver index. +New readers accept old absent metadata. Older strict readers reject the new +present field and must update; this is not transparent old-reader compatibility. +Extraction/5 and AST-cache20 invalidate prior disposable facts. The existing +release-promotion record and its strong matching test remain unchanged pending +a new source audit and reviewed decision. + +## In-progress primary ECMAScript member scopes + +JavaScript3 and TypeScript4 are `Qualifying` for a shared receiver-scope +correction. A member scope must be the unique owned child directly below its +declaration's binding scope. Nested blocks and callable-type scopes may share +the owner but cannot be selected by the lexical order of their hashed IDs. +Ambiguous accessors remain unresolved in the regression fixtures; direct fields, +methods, nested calls and shadowing retain their exact occurrence proofs. +JavaScript class-field names use the grammar's `property` child; their exact +declaration anchors are retained instead of being lost before a constructor write. +AST-cache20 rebuilds current disposable facts. The extraction/5 ID seed and +evidence/2 wire shape are unchanged, and the approved historical promotion +record below is not updated by these development checks. + +## In-progress callable initializer definition ranges + +JavaScript3 and TypeScript4 also retain the full definition range of a direct +variable-bound function. The binding declaration keeps its original name anchor +for identity and provenance. An extended definition requires one directly owned +function scope whose parent is the binding scope; inherited callbacks and +ambiguous scopes cannot supply that range. This changes source metadata without +inventing call or reference edges. + +For TypeScript and TSX, source-query projection/2 uses the same exact declarator +envelope while collecting lexical evidence from the callable body itself. +Nested callback bodies keep their own ownership. JavaScript body-query evidence +is not added by this change. Immutable projection/1 artifacts remain readable; +AST-cache20 rebuilds disposable current extraction facts. These checks do not +replace independent graph qualification or the promotion decision. + ## Promotion decision (2026-09-15) The checked-in decision record is the machine-readable source of truth: [`tests/qualification/universal-evidence-promotion.json`](../../tests/qualification/universal-evidence-promotion.json). It pins the evidence schema, producer versions, canonical dialect aliases, and -the release gate policy. The runtime registry and Rust conformance test must -match it exactly. +the release gate policy. Release-qualified registry versions must match it +exactly. The current development versions above intentionally do not pass that +promotion gate until a new audit and reviewed decision exist. | Canonical language | Producer | Version | Status | | --- | --- | ---: | --- | @@ -58,7 +115,7 @@ invalidates older extraction facts. ## Evidence contract The serialized evidence schema is `compass.languages.evidence/2`. The -extraction cache identity is `compass.languages.extraction/4`; changing either +extraction cache identity is `compass.languages.extraction/5`; changing either major version invalidates older artifacts instead of attempting an implicit translation. diff --git a/fixtures/callable_initializer_definitions.json b/fixtures/callable_initializer_definitions.json new file mode 100644 index 000000000..96097be23 --- /dev/null +++ b/fixtures/callable_initializer_definitions.json @@ -0,0 +1,42 @@ +[ + { + "extensions": ["ts", "tsx", "js", "jsx"], + "source": "export const opaque = (payload) => { return payload; };\n", + "definition": "opaque = (payload) => { return payload; }" + }, + { + "extensions": ["ts", "tsx", "js", "jsx"], + "source": "const opaque = function internal(payload) { return payload; };\n", + "definition": "opaque = function internal(payload) { return payload; }" + }, + { + "extensions": ["ts", "tsx", "js", "jsx"], + "source": "const opaque = function* (payload) { yield payload; };\n", + "definition": "opaque = function* (payload) { yield payload; }" + }, + { + "extensions": ["ts", "tsx", "js", "jsx"], + "source": "const opaque = (payload) => { const nested = () => payload; return [payload].map(item => nested(item)); };\n", + "definition": "opaque = (payload) => { const nested = () => payload; return [payload].map(item => nested(item)); }" + }, + { + "extensions": ["ts", "tsx", "js", "jsx"], + "source": "export const earlier = () => 0, opaque = (payload) => payload;\n", + "definition": "opaque = (payload) => payload" + }, + { + "extensions": ["ts", "tsx", "js", "jsx"], + "source": "// λ has two UTF-8 bytes.\nconst opaque = (payload) => payload;\n", + "definition": "opaque = (payload) => payload" + }, + { + "extensions": ["ts", "tsx"], + "source": "const opaque: (value: unknown) => unknown = (payload: unknown) => payload;\n", + "definition": "opaque: (value: unknown) => unknown = (payload: unknown) => payload" + }, + { + "extensions": ["tsx"], + "source": "export const opaque = (payload: unknown) =>
{payload}
;\n", + "definition": "opaque = (payload: unknown) =>
{payload}
" + } +] diff --git a/fixtures/typescript_member_scopes.json b/fixtures/typescript_member_scopes.json new file mode 100644 index 000000000..c96968e41 --- /dev/null +++ b/fixtures/typescript_member_scopes.json @@ -0,0 +1,64 @@ +[ + { + "prefix": "sample", + "kind": "ambiguous", + "owner": "constructor", + "target": "limit", + "relation": "accesses_member", + "expected": null, + "typescript": "export class Queue {\n callback?: () => void;\n constructor(value: number) { this.limit = value; }\n get limit(): number { return 1; }\n set limit(value: number) {}\n}\n", + "javascript": "export class Queue {\n callback = () => {};\n constructor(value) { this.limit = value; }\n get limit() { return 1; }\n set limit(value) {}\n}\n" + }, + { + "prefix": "field", + "kind": "positive", + "owner": "constructor", + "target": "limit", + "relation": "accesses_member", + "expected": { + "kind": "property", + "line": 3 + }, + "typescript": "export class Queue {\n callback?: () => void;\n limit = 0;\n constructor(value: number) { this.limit = value; }\n}\n", + "javascript": "export class Queue {\n callback = () => {};\n limit = 0;\n constructor(value) { this.limit = value; }\n}\n" + }, + { + "prefix": "method", + "kind": "positive", + "owner": "run", + "target": "perform", + "relation": "calls", + "expected": { + "kind": "method", + "line": 3 + }, + "typescript": "export class Queue {\n callback?: () => void;\n perform(value: number) { return value; }\n run(value: number) { return this.perform(value); }\n}\n", + "javascript": "export class Queue {\n callback = () => {};\n perform(value) { return value; }\n run(value) { return this.perform(value); }\n}\n" + }, + { + "prefix": "nested", + "kind": "positive", + "owner": "perform", + "target": "helper", + "relation": "calls", + "expected": { + "kind": "function", + "line": 1 + }, + "typescript": "function helper() { return 1; }\nexport class Queue {\n callback?: () => void;\n perform() { if (true) { return helper(); } return 0; }\n}\n", + "javascript": "function helper() { return 1; }\nexport class Queue {\n callback = () => {};\n perform() { if (true) { return helper(); } return 0; }\n}\n" + }, + { + "prefix": "shadow", + "kind": "positive", + "owner": "run", + "target": "perform", + "relation": "calls", + "expected": { + "kind": "method", + "line": 4 + }, + "typescript": "function perform() { return 0; }\nexport class Queue {\n callback?: () => void;\n perform() { return 1; }\n run() { return this.perform(); }\n}\n", + "javascript": "function perform() { return 0; }\nexport class Queue {\n callback = () => {};\n perform() { return 1; }\n run() { return this.perform(); }\n}\n" + } +] From b0f3385a75a4fbd760628d829a721663d72a9ba5 Mon Sep 17 00:00:00 2001 From: forhappy Date: Sat, 17 Oct 2026 23:36:26 -0700 Subject: [PATCH 02/38] test(query): add source-first quality evaluation Add bounded capture, source-oracle, review, exposure-lineage, and prospective-power tooling for broad questions. Preserve frozen development registrations and mark their reused sources ineligible for confirmation. Keep comparator execution external to the native product and tests. Independent query/graph qualification and a fresh latest-release comparison remain incomplete. --- benchmarks/agent_query/BROAD_QUERY_CAPTURE.md | 167 + benchmarks/agent_query/BROAD_QUERY_REVIEW.md | 518 + .../agent_query/broad_query_build_spec.json | 145 + benchmarks/agent_query/broad_query_capture.py | 589 + benchmarks/agent_query/broad_query_combine.py | 176 + .../agent_query/broad_query_combine_tests.py | 49 + benchmarks/agent_query/broad_query_corpus.py | 459 + .../broad_query_expansion_corpus.py | 473 + .../broad_query_expansion_corrections.json | 55 + .../broad_query_expansion_oracle.py | 120 + ...broad_query_expansion_review_manifest.json | 325 + .../broad_query_expansion_sources.json | 136 + .../broad_query_occurrence_supplement.json | 29 + benchmarks/agent_query/broad_query_oracle.py | 328 + .../agent_query/broad_query_oracle_tests.py | 75 + benchmarks/agent_query/broad_query_panels.py | 352 + benchmarks/agent_query/broad_query_power.py | 103 + .../broad_query_power_registration.json | 720 + .../broad_query_power_registration_v2.json | 715 + .../agent_query/broad_query_registration.json | 9408 +++++ ...broad_query_registration_candidate_v3.json | 9740 +++++ .../broad_query_registration_expanded.json | 29936 ++++++++++++++++ ..._registration_expanded_review_binding.json | 27 + .../broad_query_registration_reviewed.json | 9787 +++++ .../broad_query_registration_v2.json | 9644 +++++ benchmarks/agent_query/broad_query_review.py | 1409 + .../broad_query_source_corrections.json | 38 + .../agent_query/broad_query_sources.json | 42 + .../agent_query/broad_query_workflow.md | 296 + .../tests/test_broad_query_capture.py | 534 + .../tests/test_broad_query_exposure.py | 163 + .../tests/test_broad_query_panels.py | 306 + .../tests/test_broad_query_review.py | 618 + benchmarks/performance/compass/go_oracle.py | 392 + benchmarks/performance/compass/occurrences.py | 16 + benchmarks/performance/oracles/README.md | 70 +- .../performance/oracles/go-source-oracle.go | 529 + .../oracles/typescript-source-oracle.mjs | 22 +- .../performance/tests/test_correctness.py | 26 + .../performance/tests/test_go_oracle.py | 241 + 40 files changed, 78767 insertions(+), 11 deletions(-) create mode 100644 benchmarks/agent_query/BROAD_QUERY_CAPTURE.md create mode 100644 benchmarks/agent_query/BROAD_QUERY_REVIEW.md create mode 100644 benchmarks/agent_query/broad_query_build_spec.json create mode 100644 benchmarks/agent_query/broad_query_capture.py create mode 100644 benchmarks/agent_query/broad_query_combine.py create mode 100644 benchmarks/agent_query/broad_query_combine_tests.py create mode 100644 benchmarks/agent_query/broad_query_corpus.py create mode 100644 benchmarks/agent_query/broad_query_expansion_corpus.py create mode 100644 benchmarks/agent_query/broad_query_expansion_corrections.json create mode 100644 benchmarks/agent_query/broad_query_expansion_oracle.py create mode 100644 benchmarks/agent_query/broad_query_expansion_review_manifest.json create mode 100644 benchmarks/agent_query/broad_query_expansion_sources.json create mode 100644 benchmarks/agent_query/broad_query_occurrence_supplement.json create mode 100644 benchmarks/agent_query/broad_query_oracle.py create mode 100644 benchmarks/agent_query/broad_query_oracle_tests.py create mode 100644 benchmarks/agent_query/broad_query_panels.py create mode 100644 benchmarks/agent_query/broad_query_power.py create mode 100644 benchmarks/agent_query/broad_query_power_registration.json create mode 100644 benchmarks/agent_query/broad_query_power_registration_v2.json create mode 100644 benchmarks/agent_query/broad_query_registration.json create mode 100644 benchmarks/agent_query/broad_query_registration_candidate_v3.json create mode 100644 benchmarks/agent_query/broad_query_registration_expanded.json create mode 100644 benchmarks/agent_query/broad_query_registration_expanded_review_binding.json create mode 100644 benchmarks/agent_query/broad_query_registration_reviewed.json create mode 100644 benchmarks/agent_query/broad_query_registration_v2.json create mode 100644 benchmarks/agent_query/broad_query_review.py create mode 100644 benchmarks/agent_query/broad_query_source_corrections.json create mode 100644 benchmarks/agent_query/broad_query_sources.json create mode 100644 benchmarks/agent_query/broad_query_workflow.md create mode 100644 benchmarks/agent_query/tests/test_broad_query_capture.py create mode 100644 benchmarks/agent_query/tests/test_broad_query_exposure.py create mode 100644 benchmarks/agent_query/tests/test_broad_query_panels.py create mode 100644 benchmarks/agent_query/tests/test_broad_query_review.py create mode 100644 benchmarks/performance/compass/go_oracle.py create mode 100644 benchmarks/performance/oracles/go-source-oracle.go create mode 100644 benchmarks/performance/tests/test_go_oracle.py diff --git a/benchmarks/agent_query/BROAD_QUERY_CAPTURE.md b/benchmarks/agent_query/BROAD_QUERY_CAPTURE.md new file mode 100644 index 000000000..b1d2b3555 --- /dev/null +++ b/benchmarks/agent_query/BROAD_QUERY_CAPTURE.md @@ -0,0 +1,167 @@ +# Frozen native query capture + +`broad_query_capture.py` is developer evaluation tooling. It reuses +`runner.run_bounded`, executes explicit preinstalled tools against explicit +prebuilt graphs, and never builds, installs, downloads, or changes source. +The collection adapter supports Darwin/Linux through the existing POSIX bounded +subprocess backend and rejects other hosts explicitly. Its self-test uses local +fake runners and small public-format graph fixtures: + +```bash +python3 -m benchmarks.agent_query.broad_query_capture --self-test +python3 -m benchmarks.agent_query.broad_query_capture \ + --capture /external/capture-input.json --output /external/new-capture-directory +python3 -m benchmarks.agent_query.broad_query_review \ + --check /external/new-capture-directory/report-provisional.json +``` + +No actual held-out captures belong in this checkout. The last command must +reject a provisional capture: completed collection is distinct from independent +source/output review, sufficient power, graph qualification, and a passing +comparative result. `COMPLETE.json` means the full matrix was captured and +provisional reporting succeeded; it does not mean the quality gates passed. +Failed runs leave an incomplete directory without that marker and preserve the +raw evidence. Output directories and files are never overwritten. + +Compact Compass `SEED` lines count as displayed declaration citations only when +they print a source location and sufficient owner identity that resolves uniquely +in that graph. Unlocated seeds and graph-only owners do not become candidates. +The seed's exact/approximate label makes no behavioral claim. A seed followed by +the same `NODE` citation counts once; both printed spans remain in the review +queue. Repeated citations of the identical resolved identity do not make a +relationship endpoint ambiguous; conflicting or unresolved citations still do. +This is display order, not an inferred retrieval rank. A changed normalizer needs +a new frozen experiment or an explicitly separate diagnostic analysis; retained +captures and primary reports are never silently replaced. + +## Freeze before execution + +The input has schema `compass.broad-query-capture-input/1`, `registration`, +`sourceRoots` (repository ID to absolute source root), and `plan`. External +input references are `{path: absolutePath, sha256: exactByteDigest}`. Every +registered source file, binary, dependency file, and graph is bounded and +verified before execution and rechecked afterward. Sources must have the +registered Git HEAD and a clean status before and after. Artifacts must live +outside both the product checkout and every source checkout. +Heldout registrations additionally require the frozen +[source/mechanism exposure lineage](BROAD_QUERY_REVIEW.md#source-and-mechanism-exposure-lineage) +and an exact `exposureEvidence` digest-to-external-reference mapping. The adapter +rejects missing history review, evidence drift and known confirmation reuse +before invoking any subprocess. Captured review inputs retain the verified +history bytes under contained references; original history inputs are rechecked +before completion. This verifies supplied records, not undisclosed exposure. +Executable symlink invocation paths are preserved, including Python virtual +environment prefixes. Hashing follows their current target, and post-capture +verification reopens the declared path so retargeting cannot evade the check. + +`plan` must equal `registration.capturePlan` exactly, and `plan.arms` must equal +`registration.arms`. Independent source review must already be confirmed and +source-first authoring attested, even when the panel is explicitly underpowered +and `qualificationEligible` is false. Source-only pending registrations cannot +accidentally launch either tool. + +Plan schema `compass.broad-query-capture-plan/1` contains: + +- `timeoutSeconds`: integer 1–300 per command. +- `normalizerSha256` and `assessorSha256`: exact hashes of the capture and + review Python modules, freezing parser and scoring code before collection. + The checker rejects captured commands/settings/binary or dependency hashes + that differ from this plan; reports from a changed assessor require a new + explicitly labeled analysis revision, never silent primary promotion. +- `tools`: exactly `compass` and `graphify`. Each has `version`, full `commit`, + `executable` reference, explicit `argvPrefix` array, `environment` reference, + `nativeOffline: true`, and recorded `settings` with `scopePolicy`, + `inferenceProfile`, `clustering`, and `queryInterface`. The environment is a + JSON object with `files` containing hash-bound dependency references and + optional version metadata. No environment variables or secrets are dumped. + The Graphify record additionally has a `releaseVerification` reference using + the review scorer's official-release contract. Python launchers may use an + explicit interpreter executable and module/script arguments; no runtime + installation or fallback is performed. +- `graphs`: one per repository/tool, each containing `repository`, `tool`, + `artifact`, `sourceCommit`, `sourceCleanBeforeAndAfter: true`, `sourceInventory`, + and `buildCapture`. Inventory JSON has `files` equal to the source registration + inventory. The existing bounded successful build capture is retained; this + adapter does not rebuild. Before/after source-clean build attestation is + required, and current source verification cannot retrospectively prove a + different build's cleanliness. Build JSON contains `argv`, `exit_code`, + `timed_out`, `output_limited`, `wall_ms`, `stdout_bytes`, `stderr_bytes`, + and hash-bound absolute `stdout`/`stderr` input references. The adapter copies + the original JSON and both raw streams, then publishes a rebased build record. + The assessor checks byte counts, the unchanged original fields/digests and + the graph/inventory hashes against the frozen inputs. Success flags alone + cannot stand in for a captured build. +- `arms`: exactly `{id: default}` and `{id: normalized, measure: utf8_bytes, + budgetBytes: INTEGER, overflow: fail}`. Default invokes native `query QUESTION + --graph GRAPH` with no renderer override. Normalized adds the native + `--text-budget` or `--budget` using floor(bytes/4), while actual UTF-8 output + bytes govern acceptance. Complete oversized output remains captured and the + row fails; output is never clipped into compliance. Opt-in Compass ranked + view is allowed only in an explicitly frozen normalized arm with + `compassView: ranked, configuredComparison: true`; resulting reports identify + the configured comparison, and cannot support a claim about native defaults. + Equal native token inputs do not imply identical estimates: Graphify uses + three Unicode characters per token. The separately frozen measured UTF-8 + ceiling applies to both tools without changing either native estimator. +- `environmentExecutable`: hash-bound system `env` executable. The runner uses + `env -i` with `childEnvironment.inherit: false`, frozen PATH `/usr/bin:/bin`, + and explicitly isolated `HOME`, all `XDG_*_HOME` paths, `APPDATA`, + `LOCALAPPDATA`, and `TMPDIR` contained under the fresh artifact directory. + `{artifactRoot}` is the sole path substitution. Unknown environment keys are + rejected. `GRAPHIFY_NO_AUTO_REFRESH=1` and `GRAPHIFY_QUERY_LOG_DISABLE=1` + explicitly keep query capture from refreshing the graph or writing query logs. + Optional Python bytecode/user-site/hash-seed and locale/timezone settings are + allowed only as frozen nonsecret values. No inherited model credentials or + user configuration are passed to either query process. +- `measurement`: an explicit hash-bound system `time` executable and + `style: darwin-l|gnu-v` for this host. Peak RSS is bytes (GNU kilobytes are + multiplied by 1024). Missing or ambiguous RSS remains unavailable and blocks + cost qualification. The wrapper is in every recorded argument vector. + +Collection order is task ID, default then normalized, tool ID. Every execution +is a fresh isolated CLI process with an unflushed OS cache. This is not a cold +cache claim. There are no adaptive retries, diagnostic query subprocesses, +free follow-ups, or task exclusions. Static graph inspection is separately +identified and supplies graph-stage diagnostics only. Wall time includes the +same measurement wrapper for both tools; stdout/stderr and limit/exit status +are retained separately. + +## Output normalization and review + +The normalizer has no registration/oracle input. It reads only the actual +first response and its tool's graph. Actual printed file/start/name anchors +must map to a unique graph declaration; collisions remain unresolved. A graph +neighbor absent from text is never credited. Printed qualified owners are +preserved. Missing receiver owners (for example `.method()`) require source +review rather than guessed ownership. Printed edge arrows are retained; +actual optional ` at=file:Lnn[-Lnn]` citations are separated from target labels +and retained as `printedSite` in the review queue. They contain no invented +source text. Initial connectivity claims keep occurrence arrays empty pending +independent source-aware citation review, even when source/graph metadata has +sites. Malformed citations never yield automatic connectivity claims. Identical +site-free connectivity claims are coalesced for both tools; every printed edge +line, duplicate, citation and exact byte span remains separately queued. Distinct +relations or directions are retained separately. This does not establish +column-level or complete call multiplicity. Display order is not retrieval rank. +Missing rank identities or rank gaps prevent automatic retrieval metrics. +Unexposed admission remains `null`. + +Every parsed declaration/relation and every unmatched text line has an exact +UTF-8 byte span/digest in `reviewQueue`. A declaration line asserts identity, +not an automatically inferred behavioral fact. No expected fact ID is assigned. +The initial typed claims are unjudged, and `normalizationComplete: false` +explicitly prevents qualification. An independent reviewer must inspect the +full raw response and pinned source, resolve every material ambiguity, preserve +unsupported claims, and check that no prose/context assertion has disappeared +from the precision denominator. AI review must be identified as AI review. +Tool labels can be blinded for review but distinctive output style can still +reveal the tool; disclose that limitation. + +Publish reviewed typed normalizations, a new run sidecar referencing them, +review judgments bound to those exact hashes, and a new review-input manifest +as new files. Preserve the original raw captures, original run and provisional +report. Set `normalizationComplete: true` only after that independent completeness +review. Then use the review scorer's `--score` to produce a new report and +`--check` to recompute eligibility. Programmatic checks verify integrity and +registered arithmetic; they cannot authenticate reviewer attestations or +establish source semantics automatically. diff --git a/benchmarks/agent_query/BROAD_QUERY_REVIEW.md b/benchmarks/agent_query/BROAD_QUERY_REVIEW.md new file mode 100644 index 000000000..947aad37c --- /dev/null +++ b/benchmarks/agent_query/BROAD_QUERY_REVIEW.md @@ -0,0 +1,518 @@ +# Source-fact broad-query review + +`broad_query_review.py` implements the developer-side scoring and integrity +checks for Plan 029. It does not run either product, install a competitor, infer +ground truth from grep, or qualify a source judgment by matching prose. Use the [frozen capture adapter](BROAD_QUERY_CAPTURE.md), which reuses +`runner.run_bounded`, to collect commands and retain their exact argument vectors, +exit/limit/timing fields and raw stdout/stderr in immutable artifact records. + +```bash +python3 -m benchmarks.agent_query.broad_query_review --self-test +python3 -m benchmarks.agent_query.broad_query_review \ + --validate-registration benchmarks/agent_query/broad_query_registration.json +python3 -m benchmarks.agent_query.broad_query_review \ + --score /external/evidence/input.json --output /external/evidence/report.json +python3 -m benchmarks.agent_query.broad_query_review --check /external/evidence/report.json +``` + +`--self-test` tests synthetic scorer contracts. Registration validation checks +shape, exact identities, source-file pins and internally coherent task oracles; +it accepts explicitly pending independent review. Neither command establishes +real-repository correctness. `--score` writes a new diagnostic report and exits +zero even when qualification is incomplete. `--check` recomputes the entire +assessment, rejects changed/fabricated/stale metrics, and exits zero only when +every registered qualification gate passes. Ineligible reports exit one; +malformed inputs or artifact-integrity failures exit two. Existing reports are +never overwritten. + +## Review and trust boundary + +### Source and mechanism exposure lineage + +A new heldout capture requires `registration.exposureLineage` with schema +`compass.broad-query-exposure-lineage/1`. A legacy registration without it stays +shape-valid and can produce an ineligible diagnostic report. It cannot launch a +new heldout capture or qualify through the current assessor. Explicit legacy +development captures retain their separate independent source-review gate. + +The inline lineage is part of the exact hashed source registration: + +- `frozenAt` and `historyCompleteThrough` equal the registration freeze; + `historyComplete` explicitly attests completeness, with an `author` identity. +- `independentReview` has `status: confirmed`, `reviewer`, `reviewerType` + (`human`/`ai`), `sourceAware: true`, `independentOfAuthor: true`, and UTC + `reviewedAt` no later than the freeze. The reviewer differs from both the + history and source authors. This audit is source-aware; it does not require + hiding historical tool identities. +- `repositoryAliases` contains explicit pairs of HTTPS repository URLs for + renames/mirrors. `mechanismLinks` contains pairs of stable mechanism IDs for + shared mechanisms, including mechanisms appearing in different repositories. + Both links are transitive. IDs use lowercase ASCII letters/digits and + `._:/-`; renaming an ID does not establish a new mechanism. +- `history` contains up to 8,192 events with unique `id`, UTC `observedAt` no + later than the independent review, `repositoryUrl`, full `commit`, + `mechanisms` (up to 32 IDs), `evidenceSha256`, and `kind`: one of + `source_development`, `graph_build`, `query_capture`, `answer_review`, or + `implementation_tuning`. Include all known prior exposure, not only exact + question repeats. Ordinary source-first oracle authoring for the new frozen + experiment is separate from prior development exposure. + +Every registered task declares `split: development|confirmation` and one to 32 +stable `mechanisms`. A heldout registration contains only confirmation tasks. +A confirmation task fails if any recorded history touches its repository +(across commits), the identical commit under another URL, or a linked mechanism. +Repository IDs and question wording do +not enter that decision. HTTPS origins normalize default ports, trailing dots, +`.git`/trailing slashes and GitHub host/path case; other renames require explicit +aliases. GitHub URLs must identify a repository root, not a tree/file page. +Escaped/ambiguous URLs fail. Aliased registered repositories or two URLs at the +same commit cannot count as two independent sources. No automatic semantic-equivalence claim is +made for mechanism IDs. + +Capture inputs supply `exposureEvidence`, mapping each distinct historical +digest to an absolute `{path, sha256}` reference. Exactly the registered digest +set is required. Before any subprocess, the adapter checks every evidence hash +and the lineage; each input is at most 64 MiB, with a 512 MiB aggregate history +limit. It copies these bytes into the external bundle and rechecks originals +before publishing completion. The review input carries the same mapping with +contained references; the assessor verifies it again. Original, lookup and union +panels retain identical lineage and mapped task split/mechanism assignments. + +The checker verifies the records supplied to it. It cannot discover omitted +experiments, authenticate reviewer identities, infer undeclared semantic reuse, +or turn an evidence digest into a true exposure claim. Independent history +completeness/source review is therefore a separate requirement, not a generated +pass label. Obtain independent split/history approval before materializing new +confirmation graphs. The final capture registration binds history through its +own freeze; preserve the earlier source-only review and build chronology +separately. The adapter operates on prebuilt graphs and cannot retrospectively +prove that ordering. A passing exposure check alone proves none of the +query-quality, statistical-power or graph-fidelity gates. + +### Required original and lookup panels + +`broad_query_panels.py` binds the linked declaration-lookup correction without +discarding its original named-behavior questions. Both canonical review reports +must pass their existing gates. A passing lookup report cannot substitute for a +missing or failing original report. The current source-author proposal contains +352 questions per panel, with 208 shared questions and 496 distinct questions. +Those drafts still require independent source/stratum review and prospective +power planning; these counts establish no quality result or statistical power. + +Before collecting, freeze a `compass.broad-query-panels/1` definition with: + +- `frozenAt`: a UTC timestamp after both source registrations and before capture; +- `assessorSha256`: `panel` and `canonical` SHA-256 values for + `broad_query_panels.py` and `broad_query_review.py`; +- `registrations`: `original` and `lookup` contained `{path, sha256}` references + to the complete, frozen source registrations; +- `lookupPairs`: explicit `{original, lookup}` task-ID pairs for every replacement. + +The binder preserves shared questions/facts, source inventories, source clusters, +cost arms, gate policy, graph scope and mapped metric cohorts. Lookup variants +must request their declaration identity. Source or rubric corrections require a +linked pre-output revision; the binder does not approve them automatically. + +Capture the union once in one isolated artifact root. Its source registration +must contain the exact union of both panels and the same capture plan, sources +and arms. After output review, project that union review input in place: + +```bash +python3 -m benchmarks.agent_query.broad_query_panels \ + --project /external/bundle/review-input.json \ + --definition /external/bundle/panels-definition.json +``` + +This writes `panels-original-input.json`, `panels-lookup-input.json` and their +run/review documents, then a `panels-projection.json` completion receipt. Existing +files are never overwritten. All projection documents remain in the union root +so its frozen home/cache/temp environment remains valid. Projection adds no +accepted source/output judgments and does not establish qualification. + +Score both projected inputs with the canonical `broad_query_review --score` +command. A `compass.broad-query-panel-review-input/1` document then binds a +`definition` reference and `reports.original` / `reports.lookup` references: + +```bash +python3 -m benchmarks.agent_query.broad_query_panels \ + --score /external/bundle/panel-input.json \ + --output /external/bundle/panel-report.json +python3 -m benchmarks.agent_query.broad_query_panels \ + --check /external/bundle/panel-report.json +``` + +Both reports are recomputed through the canonical checker. Tool, graph, arm and +environment captures must agree. Shared answers, normalization judgments and +costs must be identical; different task/tool/arm keys are charged separately even +when their stdout bytes happen to match. Query costs are deduplicated by that +key, and graph-build costs are listed once per repository/tool. The aggregate +report uses `compass.broad-query-panel-review/1`; `--check` exits 0 only when both +panels pass, 1 for an ineligible comparison and 2 for malformed or stale input. +As with the canonical checker, this verifies integrity and arithmetic, not the +authenticity of reviewer attestations or the truth of source judgments. + +Source meaning and answer meaning require independent review. A source author +freezes the questions, acceptable declarations, essential facts and expected +ambiguity/no-support outcomes before collection. A second independent reviewer +confirms source judgments and reviews anonymized output. This reviewer may be +human or AI, with its type and source-aware method disclosed. A reviewed typed +normalization maps actual output spans to graph-node identities and source +facts; it must distinguish retrieval rank from display/context order. The +normalizer is an evaluation adapter, not a tool-specific answer generator. + +The checker verifies pinned bytes, spans, unique graph identities, explicit +relations and sites, complete paired matrices, scorer arithmetic, costs and +statistical gates. It cannot authenticate a named reviewer, establish that an +attestation is truthful, or infer that a source line proves an arbitrary +semantic claim. A complete set of fabricated reviewer attestations is outside +its verification boundary. Reviewers must never label AI review as human +review. The report states that boundary explicitly. + +Development registrations, pending or non-independent reviews, unjudged assertions, +unreviewed candidates, post-output positive oracle additions, missing graph +qualification, and underpowered comparisons remain claim-ineligible. Correcting +an incomplete oracle requires preserving the original result and conducting a +new confirmation; a result cannot silently change its frozen gold labels. + +## Source registration contract + +Source witness text is one bounded UTF-8 line and may retain indentation tabs. +Newlines, carriage returns, NUL and escape controls remain invalid. Witness text +is preserved in the registration; other labels and identities still reject tabs. +Pinned file digests and source-line checks continue to bind the witness to its +source. Accepting indentation does not establish source meaning or independent +review. + +Schema: `compass.broad-query-registration/1`. The repository/source-preparation +workflow may add descriptive metadata. The core fields are: + +```json +{ + "schema": "compass.broad-query-registration/1", + "designation": "heldout", + "frozenAt": "2026-10-16T00:00:00Z", + "qualificationEligible": false, + "sourceReview": {"author": "source author", "blindToOutputs": true}, + "independentReview": { + "status": "pending", "reviewer": null, "toolLabelsBlinded": true + }, + "repositories": [{ + "id": "sample", "language": "python", "url": "https://example.invalid/sample", + "commit": "FULL_GIT_OBJECT_ID", + "files": [{"file": "src/store.py", "sha256": "HEX_SHA256"}] + }], + "tasks": [{ + "id": "save-task", "repository": "sample", "cluster": "save-mechanism", + "kind": "broad", "question": "where is persistent state saved", + "expected": "answer", + "relevant": [{"identity": { + "file": "src/store.py", "startLine": 10, "symbol": "save", "owner": "Store" + }, "grade": 3}], + "facts": [{ + "id": "save-fact", "essential": true, + "identity": {"file": "src/store.py", "startLine": 10, "symbol": "save", "owner": "Store"}, + "statement": "Save delegates persistence to the writer.", + "source": {"file": "src/store.py", "line": 12, "text": "return self.writer.flush()"} + }] + }] +} +``` + +The example uses placeholders and is not a valid qualification registration. +An identity has exactly `file`, `startLine`, `symbol`, and `owner`; the owner is +the reviewed lexical owner, or the empty string for a module-level declaration. +Paths are canonical repository-relative POSIX strings. Grades are 0–3; +grades 2–3 are relevant. Declaration file/start/name and owner must all agree +for answer scoring. Graph mapping uses the existing symmetric exact +file/start/terminal-name policy and requires a unique result. Source review +establishes lexical ownership; the normalizer cannot choose the graph node +whose name merely fits an expected answer. + +Task kinds are `broad`, `exact`, `directed`, `multihop`, `ambiguous`, and +`unsupported`. Expected outcomes are `answer`, `ambiguous`, and `abstain`. +Answerable tasks need positive identities and at least one essential fact. +Ambiguous tasks need at least two source-reviewed positive identities. +Unsupported tasks have empty `relevant`/`facts` arrays and a `rationale` scoped +to the registered source inventory. They mean “no supporting indexed evidence +in this scope”, not proof that a behavior cannot exist anywhere. + +A fact optionally includes a typed relationship: + +```json +{ + "relation": { + "source": {"file": "src/store.py", "startLine": 10, "symbol": "save", "owner": "Store"}, + "target": {"file": "src/writer.py", "startLine": 30, "symbol": "flush", "owner": "Writer"}, + "kind": "calls", "direction": "source_to_target", + "occurrences": [{"file": "src/store.py", "line": 12, "text": "return self.writer.flush()"}] + } +} +``` + +Both endpoint identities, relation kind, direction and the complete registered +occurrence set must agree. Repeating one occurrence cannot recover a different +missing one. Multihop tasks register each essential directed hop as a separate +fact; endpoints plus a list of relation names do not prove an intermediate +route. Native undirected artifacts explicitly report an unavailable direction +contract rather than an invented reverse-call error. + +## Evidence bundle contract + +All artifact references are `{ "path": "relative/path", "sha256": "64 lower-case hex digits" }`. +They resolve under the input manifest's directory; symlink escapes, `..`, +absolute artifact paths, digest drift, duplicate JSON keys and excessive +input sizes fail closed. Local `sourceRoots` are explicit external checkout +paths and are not product configuration. Keep raw captures and source content +outside the source repository under the source owner's access policy. + +Input schema `compass.broad-query-review-input/1` contains: + +```json +{ + "schema": "compass.broad-query-review-input/1", + "registration": {"path": "registration.json", "sha256": "HEX_SHA256"}, + "run": {"path": "run.json", "sha256": "HEX_SHA256"}, + "reviews": {"path": "reviews.json", "sha256": "HEX_SHA256"}, + "sourceRoots": {"sample": "/external/source/sample"}, + "graphQualification": [{ + "repository": "sample", + "manifest": {"path": "graph-audit-manifest.json", "sha256": "HEX_SHA256"} + }] +} +``` + +`graphQualification` supplies existing `compass.quality-audit/2` qualification +manifests. The checker calls the existing `run_audit` on the exact captured +Compass graph and source, preserving its 2,000-judgment, .995 precision, .99 +Wilson lower-bound and diversity/capability/source-coverage gates. It does not +trust an `eligible: true` summary from another file. Symmetric source-based +graph comparisons remain independent of this Compass production gate; neither +graph counts nor reproducing all competitor hypotheses replaces those audits. +For a multi-corpus production audit, supply one entry with `manifest`, relative +`graphRoot` and explicit external `sourceRoot` instead of `repository`. Its +corpus names, source roots, commits and graph hashes must match the captured +panel exactly. This preserves the existing aggregate 2,000-total and +400-per-corpus floors rather than requiring 2,000 judgments separately for +every repository. + +The run schema `compass.broad-query-capture/1` contains: + +- `registrationSha256` and UTC `startedAt`, after the source registration freeze. +- Exactly two `tools`, IDs `compass` and `graphify`, each with `version`, + `commit`, `nativeOffline: true`, recorded `settings`, and artifact references + `executable`, `environment`, and `normalizer`. Environment manifests must + retain the installed distribution/dependency/package hashes; a Python + launcher hash alone does not identify an installed competitor. + Settings explicitly name `scopePolicy`, `inferenceProfile`, `clustering`, and + `queryInterface`; missing settings prevent qualification. Record actual + differences rather than inventing equivalent internal feature flags. +- Graphify also has `releaseVerification`, referring to a record with + `version`, `commit`, UTC `checkedAt`, and + `source: "https://github.com/Graphify-Labs/graphify/releases"`. Pin the latest + release before collection, not divergent default-branch HEAD. +- One `graphs` entry per repository/tool: `repository`, `tool`, graph + `artifact`, `sourceCommit`, `sourceCleanBeforeAndAfter: true`, and + `sourceInventory`/`buildCapture` references. Both tools must receive the same + inventory hash for a repository. Retain parser omissions separately. + The successful bounded build record must explicitly have `exit_code: 0`, + `timed_out: false`, and `output_limited: false`. +- Exactly two `arms`, frozen identically in the registration: + `{"id":"default"}` and + `{"id":"normalized","measure":"utf8_bytes","budgetBytes":8000,"overflow":"fail"}`. + The byte budget is configurable before collection; it is a named proxy, + not a tokenizer. Normalized first-page scoring permits one capture and no + follow-ups. Overflow stays in the denominator as a failed execution; the + checker retains the complete raw response rather than clipping it. +- One `observations` entry for every task/tool/arm, containing `task`, `tool`, + `arm`, `normalized` reference, `captures` references (1–4), `condition`, + and `peakRssBytes`. Conditions are `fresh_cli`, `warm_cli`, `warm_mcp`, + `cold_cli`, or `unrecorded`. Missing measurements remain explicit and prevent + complete qualification. Interleave/randomize timing collection and retain + its order/setup metadata; this checker does not establish a latency win. + +Each capture is the existing bounded runner's field record: `argv`, +`exit_code`, `timed_out`, `output_limited`, `wall_ms`, `stdout_bytes`, +`stderr_bytes`, plus `stdout`/`stderr` artifact references instead of inline +text. Follow-up bytes and latency are charged in full. Nonzero exits, limits, +timeouts and over-budget normalized responses fail task success while remaining +in all cost and task denominators. +Broad and unsupported tasks must pass the exact frozen question as one argument +in each measured command. A tool-specific oracle-informed question rewrite is +rejected. Exact/directed controls may use the closest documented typed operation +with their preregistered source subjects; do not count known-ID preparation as +successful natural-language retrieval. + +## Typed response and judgments + +Each normalized response has schema `compass.broad-query-normalized/1`, matching +`task`, `tool`, and `arm`, plus: + +- `disposition`: `answered`, `ambiguous`, `abstain`, `error`, or `limit`. +- `rankingKind`: `retrieval`, `display`, or `unavailable`. A neighborhood's + arbitrary display order is not a ranked retrieval result. Retrieval metrics + are unavailable for display/context lists. +- `ranked`: candidate records `{identity,nodeId,firstPage,span?}`. A first-page + candidate must carry a raw first-capture `span`; it must uniquely resolve to + the recorded graph node at its exact declaration anchor. Ranks beyond the + first page may be retained from the documented typed ranked result/trace. +- `admission`: an instrumented array of admitted graph node IDs, or `null` if + the tool does not expose that stage. Missing instrumentation is unavailable, + never invented as a false admission failure. A candidate with an observed + rank must also be in an instrumented admitted set when one is supplied. +- `claims`: first-page fact assertions `{id,identity,relation?,span}`. Each span + is `{startByte,endByte,sha256}` over exact first-capture stdout bytes, with + `endByte` exclusive. Spans establish where the reviewed assertion appeared; + they do not establish its meaning. An independent reviewer must also confirm + complete normalization so omitted incorrect assertions cannot improve precision. + +Judgments schema `compass.broad-query-judgments/1` has `runSha256` and +`observations`. Each row repeats `task`, `tool`, and `arm`, binds +`normalizedSha256`, and has an `independentReview` object. Confirmed independent +source-aware review uses `status: "confirmed"`, `reviewerType: "human"` or +`"ai"`, a nonempty `reviewer`, `independentOfAuthor: true`, `sourceAware: true`, +and `toolLabelsBlinded: true`. The source author and independent source reviewer +must be distinct. Pending review is allowed as a diagnostic input and cannot +pass qualification. Reports disclose AI review and never call it human review. + +`claims` reviews contain `id`, `verdict: supported|unsupported|unjudged`, +`factId` (a frozen fact ID or null), and `reason`. A supported frozen fact +requires exact identity, relationship kind and direction agreement; a wrong owner or reversed +edge is unsupported even if the review labels it supported. A supported extra +assertion with null `factId` additionally needs a pinned `source` witness and +independent semantic review. Extra source files must be pinned before scoring. +Unknown fact IDs and duplicate semantic claims fail validation. + +`candidates` reviews contain `identity` and `grade` for displayed candidates +outside the frozen positive/negative identities. Unknown returned candidates +remain unjudged until reviewed. Newly discovered positive identities are +reported as post-output oracle corrections and prevent a held-out claim. +Changing a previously frozen grade is rejected. + +## Metrics and qualification + +An answerable task succeeds only when its first page identifies a relevant +declaration, answers every essential source fact, and contains no unsupported +or unjudged assertion/candidate. An ambiguity-only list cannot pass an +answerable task. A valid ambiguity answer explicitly retains at least two +source-reviewed possibilities. An unsupported task succeeds through explicit +abstention with no competing asserted candidates/facts. + +The report retains the source → graph → admission → rank → first-page ledger +for every registered fact. Graph absence and missing instrumentation remain +different outcomes. End-to-end success includes every answerable task; +graph-present conditional statistics cannot hide extraction misses. + +Success@5 and MRR@10 use frozen grade ≥2 relevant identities and documented +retrieval ranks. nDCG@10 uses gains `2^grade - 1` and `log2(rank + 1)` discounts. +Source-fact recall is recovered frozen facts divided by all frozen facts. +Claim precision uses supported/(supported+unsupported) reviewed claims; +unjudged assertions are reported separately and prevent qualification. Its +paired noninferiority score assigns an empty answer zero per-task precision +(`precisionWithEmptyZero`) to prevent improving precision by abstaining on +hard tasks. This score is distinct from pooled claim precision and is labeled +accordingly. False-empty rate is explicit abstention or no candidates/claims +on answerable tasks. All summaries retain repository, language and kind strata. + +Only the normalized arm determines the registered query claim. The scorer uses +a deterministic hierarchical paired bootstrap: average paraphrases within +their source-task cluster, resample repositories, then resample source-task +clusters within each selected repository. Repositories have equal weight. +Seed and 1,000–10,000 resamples are frozen in `gatePolicy`. The result reports +the paired estimate, percentile 95% interval, repository/cluster counts, and +method. The reported interval is widened by a conservative exact finite-task +paired-risk bound: lower confidence for definite task-cluster wins minus upper +confidence for any task-cluster loss, with Bonferroni-adjusted Clopper–Pearson +bounds. Paraphrases remain one cluster; partially positive clusters receive no +definite-win credit. This bound assumes independently sampled source tasks +conditional on the fixed repository panel and requires equal independent +cluster counts per repository. It does not establish population repository +inference. In particular, 32 all-tie controls retain a nonzero upper bound on +unseen losses and cannot establish .02 noninferiority. A bootstrap's zero +empirical variance is never treated as certainty. Purposive corpus selection +still limits generalization even if the interval passes. + +The proposed gates cannot be weakened through the manifest: + +- At least eight repositories, four languages with at least two repositories + each. A single panel follows 10 broad / 4 exact-or-directed / 2 multihop / + 2 ambiguous / 2 unsupported per repository. A heterogeneous source-first + expansion may instead freeze `metricCohorts` and `samplingDesign`, retaining + every original task and using balanced independent cluster counts and language + allocation within each primary metric cohort. +- At least 80 independent broad source-task clusters; paraphrases do not add + independent sample size. +- Broad first-page success improves by at least .10 observed, with paired + 95% lower bound above zero. This does not assert that the true gain is ≥.10. +- Exact/directed task success and answerable per-task claim-precision score + have paired lower bounds strictly greater than −.02. A repository-level + answerable-success regression worse than .10 requires independent investigation. +- Independent source-aware source/normalization review, complete provenance/judgments, + common frozen cost arms, and existing independent graph qualification. +- A frozen `powerAnalysis`, including `targetPower` ≥.8, + `expectedBroadDiscordance`, `expectedControlDiscordance`, `designEffect` ≥1, + and `developmentEvidence` artifact reference. The approximate paired-binary + planning floor uses `(z_.975 + z_power)^2 * (discordance - effect^2) / + effect^2 * designEffect` for broad superiority and + `(z_.975 + z_power)^2 * controlDiscordance / margin^2 * designEffect` for + control noninferiority. These are prospective planning checks, not post-hoc + “achieved power”. Independently review the assumptions and retained development + evidence; the checker cannot establish their representativeness. + +The initial 160-task registration is not a promise that these gates can pass. +In particular, a .02 noninferiority margin can require far more independent +controls. Plan any balanced expansion before looking at confirmation outputs, +retain failures, and report an inconclusive result when uncertainty or power +is inadequate. Do not reclassify development reruns as a fresh holdout. + +## Occurrences, prospective cohorts and power + +Relationship source witnesses always retain the frozen occurrences. The +answer rubric is separately predeclared with `fact.requireOccurrences`: true +requires exact printed line witnesses; false permits omitted sites for a +connectivity-only question while still requiring exact endpoints/kind/direction. +Any actually asserted site must be source-supported. No absent printed site is +backfilled from source or graph. The initial panel uses false only for questions +that ask identity/connectivity/path rather than sites/counts, accepted before +outputs by the source author and independent reviewers. File+line witnesses do +not establish column-distinct call multiplicity. Native graph qualification and +its occurrence/provenance gates remain separate. + +`metricCohorts` has exactly `broadSuccess`, `exactDirected`, and +`claimPrecisionEmptyZero`, each with `taskIds`, `rationale`, +`minimumRepositories` (at least eight), and `minimumIndependentClusters`. +Broad success and precision use the same frozen broad tasks; control success +uses exact/directed tasks. Every captured task remains in diagnostics, even +when a new source-only control cohort is the prospective NI population. +`samplingDesign` explains selection, source dependence and original-task +retention. Paraphrases, shared implementations and repeated endpoints do not +create independent clusters. + +The normal-planning floor above is diagnostic and cannot by itself establish +power of the actual acceptance rules. The scorer additionally computes an exact +finite-control pass probability from independent-cluster N and the predeclared +adverse probability, crediting no definite wins. At N=200, adverse probability +.005 gives only about .367 probability of passing that finite NI bound. An +assumed equality alternative of zero requires hash-bound pre-output evidence; +zero losses observed afterward cannot supply the assumption. + +`powerAnalysis.acceptanceRuleSimulation` must reference a frozen +`compass.broad-query-power-simulation/1` artifact with `frozenAt`, an independently +confirmed `independentReview`, and `metrics` for all three comparison names. +Each metric has `assumptions`, `method`, hash-bound simulation `code` and +`results`, `independentClusters`, and `powerLowerBound`. The planned lower bound +must reach registered target power and the actual panel must reach planned +clusters. Simulations must model the actual hierarchical and finite-risk gates, +including per-task precision's nonbinary distribution. The checker verifies +these records and reviewer attestations; it does not authenticate reviewers or +assert that their prospective model represents a population. Missing planning +records keep the result ineligible rather than converting observed significance +into post-hoc power. + +Graph-quality claim scope may be frozen in +`graphQualificationRepositories` (at least eight registered repositories, +including every broad comparison repository). Added strict-query control +repositories are explicitly query-only and receive no graph-quality superiority +claim. Reports enumerate both groups and audited graph hashes. Without an +explicit scope, every registered repository requires graph qualification. +The native audit retains its existing 2,000-total and 400-per-corpus floors; +eight audited broad corpora therefore need at least 3,200 accepted judgments. diff --git a/benchmarks/agent_query/broad_query_build_spec.json b/benchmarks/agent_query/broad_query_build_spec.json new file mode 100644 index 000000000..220fb2f41 --- /dev/null +++ b/benchmarks/agent_query/broad_query_build_spec.json @@ -0,0 +1,145 @@ +{ + "schema": "compass.broad-query-native-build-spec/1", + "status": "development_template_exposed_sources", + "scope": "These original8 sources and extra16 controls are exposed development inputs. Preserve their mandatory regression outcomes; this template does not define fresh confirmation data or establish graph/query superiority.", + "qualificationEligible": false, + "sourceRegistrationSha256": "9c5859dcb23b7aa29fdc0aa5e4610b48217da59c737ad81de746beb19f807d5c", + "graphQualificationRepositories": [ + "itsdangerous", + "marshmallow", + "httprouter", + "go-cache", + "log", + "humantime", + "p-queue", + "superjson" + ], + "sources": [ + { + "id": "itsdangerous", + "url": "https://github.com/pallets/itsdangerous", + "commit": "672971d66a2ef9f85151e53283113f33d642dabd", + "inventorySha256": "bba10a0daa9dce04d9ec598bb12fbeb4fa4f52878cd00c391035fd6065fc8d89" + }, + { + "id": "marshmallow", + "url": "https://github.com/marshmallow-code/marshmallow", + "commit": "d08471b17880c62b789971ada638707a11d3f8d4", + "inventorySha256": "65fea45e4d8294b46df43bfa8458594fc9442443cd70ed9c856876e3415286e0" + }, + { + "id": "httprouter", + "url": "https://github.com/julienschmidt/httprouter", + "commit": "484018016424d215c0b87c42f4c9b57d980fbd00", + "inventorySha256": "42a8e0fe1f185eccb4716816bf374797988d59ad1b84959ec6ac5a775681491f" + }, + { + "id": "go-cache", + "url": "https://github.com/patrickmn/go-cache", + "commit": "46f407853014144407b6c2ec7ccc76bf67958d93", + "inventorySha256": "7d69478248c3d2f5235a9ebfe26dd436b09bd5a815252918327584420456d67e" + }, + { + "id": "log", + "url": "https://github.com/rust-lang/log", + "commit": "27e3cf7a021dab43430a70cb2909c06ca8141f79", + "inventorySha256": "94b38f58a9cb84133faf2cb801e59279c25c15a0758fd05dd18ef2e639b2fc36" + }, + { + "id": "humantime", + "url": "https://github.com/tailhook/humantime", + "commit": "997ff566cc731e3ece5c792c8b83dd78e459f202", + "inventorySha256": "788d6afd828157348743ed7b49328d5a3e74fe4de3bfac8cb14279472e0d62b8" + }, + { + "id": "p-queue", + "url": "https://github.com/sindresorhus/p-queue", + "commit": "7f272ab5ff2254c0dc2537b7977e88ede1cd55e8", + "inventorySha256": "6bcc5040630c9bd7731e8595461e9ad6c8ba56c8230b97a55ed8621200ec3b99" + }, + { + "id": "superjson", + "url": "https://github.com/flightcontrolhq/superjson", + "commit": "aaa65e36e8e31da740265a56b27c965ca1c7b754", + "inventorySha256": "4870c799d1a94c9b29898bccd2feecdbdd674d88e5cf17098b543fd1e7e7664c" + } + ], + "requiresBeforeExecution": [ + "Root finishes native development/tests and selects the release-matching candidate binary", + "Verify exact binary SHA and version, installed Graphify0.9.92 package/interpreter/dependency hashes; recheck the official latest release at a new experiment freeze", + "Freeze merged source-reviewed task/cohort/power/settings manifest; no captures before independent review", + "Check all source roots clean and source inventory hashes unchanged; resolve templates as argv values, never shell interpolation" + ], + "limits": { + "timeoutSecondsPerBuild": 600, + "maxCapturedBytesPerStream": 16777216, + "maxWorkers": 1, + "outputLimitOutcome": "failure, never an empty successful graph" + }, + "childEnvironment": { + "inherit": false, + "PATH": "/usr/bin:/bin", + "HOME": "{isolatedHome}", + "XDG_CONFIG_HOME": "{isolatedHome}/config", + "XDG_DATA_HOME": "{isolatedHome}/data", + "XDG_CACHE_HOME": "{isolatedHome}/cache", + "XDG_STATE_HOME": "{isolatedHome}/state", + "APPDATA": "{isolatedHome}/appdata", + "LOCALAPPDATA": "{isolatedHome}/localappdata", + "TMPDIR": "{isolatedTmp}", + "GRAPHIFY_NO_AUTO_REFRESH": "1", + "GRAPHIFY_QUERY_LOG_DISABLE": "1" + }, + "builds": { + "compass": { + "argv": [ + "{compassBinary}", + "extract", + "{sourceRoot}", + "--code-only", + "--no-viz", + "--store", + "json", + "--max-workers", + "1", + "--out", + "{freshCompassOutput}" + ], + "graph": "{freshCompassOutput}/compass-out/snapshots/{publishedCurrentSnapshot}/graph.json", + "profile": "Native code-only typed JSON publication; optional provider/ProgramIR/network flags absent. JSON storage is declared for source graph auditing.", + "publishedSnapshotPointer": "{freshCompassOutput}/compass-out/current-snapshot", + "pointerValidation": "Read bounded current-snapshot; require one contained snapshot-* basename and coherent immutable graph/sidecars. Use existing runner._compass_graph validation; re-read publication identity before queries." + }, + "graphify": { + "argv": [ + "{graphifyPython}", + "-I", + "-B", + "-X", + "utf8", + "-m", + "graphify", + "extract", + "{sourceRoot}", + "--code-only", + "--max-workers", + "1", + "--out", + "{freshGraphifyOutput}" + ], + "graph": "{freshGraphifyOutput}/graphify-out/graph.json", + "release": "v0.9.92", + "sourceCommit": "86c57be8204673c2a1b9849a3292a7dd3ab373d8", + "profile": "Native local AST/code-only; optional semantic/provider/model-backed extraction excluded and labeled separately." + } + }, + "scopePolicy": "Registered tracked-source scope includes tests/examples. Record actual discovered file omissions at ledgerstage1; do not silently remove their tasks or mutate inputs. No-cluster/no-gitignore flags are not implicitly added; clustering/discovery retain declared native behavior.", + "outputsRequired": [ + "Bounded raw stdout/stderr plus argv/child nonsecret environment, exit/timeout/output-limit/walltime records", + "Graph and sidecar hashes and metadata; verify coherent publication before use", + "Source git status/commit/inventory before and after", + "Observe graph identity/multiplicity/provenance from source audits, not from another graph as oracle" + ], + "publication": "Fresh external per-repository output directories; fail if prior build artifacts exist. Generated graphs/caches must stay outside all source trees.", + "qualification": "No successful build or3200fact count is asserted. Source-sample feasibility and \u22652000accepted current graph judgments with required precision/Wilson bounds remain separate; insufficient small-corpus evidence stays unqualified." +} diff --git a/benchmarks/agent_query/broad_query_capture.py b/benchmarks/agent_query/broad_query_capture.py new file mode 100644 index 000000000..73c695c1e --- /dev/null +++ b/benchmarks/agent_query/broad_query_capture.py @@ -0,0 +1,589 @@ +"""Bounded, developer-only Plan 029 capture; never installs or builds a tool. + +A pre-output registration freezes the exact capturePlan. Queries use prebuilt +native graphs and explicit executables. Parsed spans are review suggestions, +not source facts: unresolved text and missing owners remain in a review queue. +Run ``python3 -m benchmarks.agent_query.broad_query_capture --help``. +""" +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +import json +from pathlib import Path +import platform +import re +import sys +from typing import Callable + +from benchmarks.agent_query.broad_query_oracle import publish_exclusive +from benchmarks.agent_query.broad_query_review import ( + ARMS, TOOLS, GraphIndex, Identity, MANIFEST_SCHEMA, MAX_GRAPH_BYTES, + MAX_JSON_BYTES, NORMALIZED_SCHEMA, REVIEW_SCHEMA, RUN_SCHEMA, + _array, _boolean, _choice, _digest, _integer, _json, _object, _relative, + _review_failures, _text, _timestamp, _verify_sources, main as review_main, + sha, validate_registration, verify_build_capture, exposure_failures, +) +from benchmarks.agent_query.community_tasks import read_bounded +from benchmarks.agent_query.runner import _terminal_symbol, run_bounded + +SCHEMA = "compass.broad-query-capture-plan/1" +INPUT_SCHEMA = "compass.broad-query-capture-input/1" +PENDING = dict(status="pending", reviewer=None, reviewerType=None, + sourceAware=False, independentOfAuthor=False, toolLabelsBlinded=False) +SAFE_KEY = re.compile(r"[a-zA-Z0-9][a-zA-Z0-9_.-]{0,127}\Z") +GRAPHIFY_NODE = re.compile(r"^NODE (?P