diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 752ae8d2..34d3c1b6 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -8,7 +8,7 @@ { "name": "bcquality", "source": "./", - "description": "Business Central AL quality knowledge base and review skills, packaged as an installable plugin. Exposes an AL review adapter while preserving BCQuality's internal Entry and action-skill protocols.", + "description": "Business Central quality knowledge and skills for AL code review and cited knowledge consultation.", "version": "0.2.0", "skills": [ "./skills/" diff --git a/.github/scripts/Test-SkillIndex.ps1 b/.github/scripts/Test-SkillIndex.ps1 index bc635f2b..21081cc7 100644 --- a/.github/scripts/Test-SkillIndex.ps1 +++ b/.github/scripts/Test-SkillIndex.ps1 @@ -99,6 +99,30 @@ try { 'microsoft/skills/review/al-finance-review.md' ) $review = @($skills | Where-Object id -eq 'al-code-review') + $knowledge = @($skills | Where-Object id -eq 'al-knowledge') + if ($knowledge.Count -ne 1 -or (@($knowledge[0].inputs) -join ',') -ne 'knowledge-query' -or + (@($knowledge[0].outputs) -join ',') -ne 'knowledge-response') { + throw 'Expected one knowledge-only al-knowledge action.' + } + $knowledgeCandidates = @($skills | Where-Object { 'knowledge-query' -in $_.inputs }) + if ($knowledgeCandidates.Count -ne 1 -or $knowledgeCandidates[0].id -ne 'al-knowledge') { + throw 'A knowledge-only request must exclude review actions.' + } + # Entry applies READ's non-empty technology intersection to indexed actions. + $alKnowledge = @($knowledgeCandidates | Where-Object { 'al' -in $_.filters.technologies }) + if ($alKnowledge.Count -ne 1 -or $alKnowledge[0].id -ne 'al-knowledge') { + throw 'Entry must admit knowledge-query with technologies: [al].' + } + foreach ($dimension in 'bc-version', 'countries', 'application-area') { + $expected = if ($dimension -eq 'countries') { 'w1' } else { 'all' } + if ($expected -notin $knowledge[0].filters.$dimension) { + throw "Missing knowledge action applicability: $dimension" + } + } + $reviewCandidates = @($skills | Where-Object { 'file-path' -in $_.inputs -and 'al' -in $_.filters.technologies }) + if ('al-code-review' -notin $reviewCandidates.id -or 'al-knowledge' -in $reviewCandidates.id) { + throw 'AL file input must admit code review and exclude knowledge consultation.' + } if ($review.Count -ne 1) { throw "Expected exactly one al-code-review record, found $($review.Count)." } diff --git a/.github/scripts/test_knowledge_response.py b/.github/scripts/test_knowledge_response.py new file mode 100644 index 00000000..cbe7d80b --- /dev/null +++ b/.github/scripts/test_knowledge_response.py @@ -0,0 +1,57 @@ +"""Validate knowledge-response examples and reject unsafe citation shapes.""" +import copy +import json +from pathlib import Path + +from jsonschema import Draft7Validator + +ROOT = Path(__file__).resolve().parents[2] +schema = json.loads((ROOT / 'schemas/knowledge-response.schema.json').read_text()) +Draft7Validator.check_schema(schema) +validator = Draft7Validator(schema) +base = { + 'skill': {'id': 'al-knowledge', 'version': 1}, + 'outcome': 'completed', 'question': 'When is SetLoadFields useful?', + 'answer': 'Use it for partial records under the cited conditions.', + 'references': [{ + 'path': 'microsoft/knowledge/performance/use-setloadfields-for-partial-records.md', + 'applicability': 'conditional', 'unknown': ['bc-version'], + }], 'suppressed': [], +} +assert (ROOT / base['references'][0]['path']).is_file() +cases = 0 + + +def check(value, valid): + global cases + errors = list(validator.iter_errors(value)) + assert bool(errors) != valid, [error.message for error in errors] + cases += 1 + + +check(base, True) +value = copy.deepcopy(base) +value['references'][0]['unknown'] = [] +check(value, False) +value['references'][0]['applicability'] = 'applicable' +check(value, True) +value['references'][0]['unknown'] = ['bc-version'] +check(value, False) +for outcome in ('failed', 'no-knowledge'): + value = copy.deepcopy(base) + value.update(outcome=outcome, **{'outcome-reason': 'No usable knowledge'}) + check(value, False) + value['references'] = [] + check(value, True) +value = copy.deepcopy(base) +value['references'] = [] +check(value, False) +value = copy.deepcopy(base) +value['outcome'] = 'partial' +check(value, False) +value['outcome-reason'] = 'Target version is unknown' +check(value, True) +value = copy.deepcopy(base) +value['references'][0]['path'] = 'invented/article.md' +check(value, False) +print(f'Knowledge-response schema: {cases} positive/negative cases passed.') diff --git a/.github/scripts/test_knowledge_response_semantics.py b/.github/scripts/test_knowledge_response_semantics.py new file mode 100644 index 00000000..a5a56793 --- /dev/null +++ b/.github/scripts/test_knowledge_response_semantics.py @@ -0,0 +1,172 @@ +"""Regress input fidelity and complete citation reads using consumer evidence.""" +import copy +import hashlib +import importlib.util +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +REPO = Path(__file__).resolve().parents[2] +TOOL = REPO / "tools/validate_knowledge_response.py" +spec = importlib.util.spec_from_file_location("knowledge_validator", TOOL) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + + +class KnowledgeSemantics(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + (self.root / "schemas").mkdir() + (self.root / "schemas/knowledge-response.schema.json").write_bytes((REPO / "schemas/knowledge-response.schema.json").read_bytes()) + self.path = "microsoft/knowledge/performance/example.md" + self.article = self.root / self.path + self.article.parent.mkdir(parents=True) + self.article.write_bytes(b"# Example\n\nComplete article content.\n") + self.question = " Question with accents: caf\u00e9?\r\n " + self.response = {"skill": {"id": "al-knowledge", "version": 1}, + "outcome": "completed", "question": self.question, "answer": "Supported answer.", + "references": [{"path": self.path, "applicability": "applicable", "unknown": []}], "suppressed": []} + content = self.article.read_bytes() + self.evidence = {"runId": "test-run", "questionSha256": hashlib.sha256(self.question.encode()).hexdigest(), + "reads": [{"path": self.path, "complete": True, "bytes": len(content), "sha256": hashlib.sha256(content).hexdigest()}]} + + def errors(self, evidence=True): + return module.validate_response(self.response, question=self.question, root=self.root, + evidence=self.evidence if evidence else None, run_id="test-run") + + def test_disabled_citation_layer(self): + errors = module.validate_response(self.response, question=self.question, root=self.root, + evidence=self.evidence, run_id="test-run", enabled_layers=["community"]) + self.assertTrue(any("disabled layer" in error for error in errors)) + + def test_configuration_suppression_disabled_layer(self): + self.response.update(outcome="no-knowledge", references=[], suppressed=[{"path": self.path, "reason": "configuration"}]) + self.assertEqual(module.validate_response(self.response, question=self.question, root=self.root, + run_id="test-run", enabled_layers=[]), []) + + def test_precedence_suppression_disabled_layer(self): + self.response.update(outcome="no-knowledge", references=[], suppressed=[{"path": self.path, "reason": "layer-precedence"}]) + self.assertTrue(module.validate_response(self.response, question=self.question, root=self.root, + run_id="test-run", enabled_layers=[])) + + def test_suppressed_path_traversal(self): + self.response["suppressed"] = [{"path": "microsoft/knowledge/../skills/x.md", "reason": "configuration"}] + self.assertTrue(self.errors()) + + def test_missing_suppressed_file(self): + self.response["suppressed"] = [{"path": "custom/knowledge/missing.md", "reason": "configuration"}] + self.assertTrue(self.errors()) + + def test_enabled_layers_cli(self): + response_file = self.root / "response.json" + question_file = self.root / "question.txt" + evidence_file = self.root / "reads.json" + response_file.write_text(json.dumps(self.response), encoding="utf-8") + question_file.write_bytes(self.question.encode("utf-8")) + evidence_file.write_text(json.dumps(self.evidence), encoding="utf-8") + result = subprocess.run([sys.executable, str(TOOL), str(response_file), "--root", str(self.root), + "--question-file", str(question_file), "--read-evidence", str(evidence_file), + "--run-id", "test-run", "--enabled-layers", "community"], capture_output=True, text=True) + self.assertEqual(result.returncode, 1) + self.assertIn("disabled layer", result.stdout) + + def test_unknown_enabled_layer(self): + self.assertTrue(module.validate_response(self.response, question=self.question, root=self.root, + evidence=self.evidence, run_id="test-run", enabled_layers=["unknown"])) + + def test_valid_and_unchanged(self): + original = copy.deepcopy(self.response) + self.assertEqual(self.errors(), []) + self.assertEqual(self.response, original) + + def test_rewritten_question(self): + self.response["question"] = "A paraphrase" + self.assertTrue(self.errors()) + + def test_question_whitespace(self): + self.response["question"] = self.question.strip() + self.assertTrue(self.errors()) + + def test_missing_article(self): + self.response["references"][0]["path"] = "microsoft/knowledge/performance/missing.md" + self.assertTrue(self.errors()) + + def test_missing_read_evidence(self): + self.assertTrue(self.errors(evidence=False)) + + def test_truncated_read(self): + self.evidence["reads"][0]["complete"] = False + self.assertTrue(self.errors()) + + def test_byte_count(self): + self.evidence["reads"][0]["bytes"] -= 1 + self.assertTrue(self.errors()) + + def test_changed_body(self): + self.article.write_bytes(b"Changed body\n") + self.assertTrue(self.errors()) + + def test_wrong_hash(self): + self.evidence["reads"][0]["sha256"] = "0" * 64 + self.assertTrue(self.errors()) + + def test_same_basename_other_layer(self): + self.evidence["reads"][0]["path"] = self.path.replace("microsoft/", "custom/") + self.assertTrue(self.errors()) + + def test_wrong_run(self): + self.evidence["runId"] = "previous-run" + self.assertTrue(self.errors()) + + def test_wrong_question_evidence(self): + self.evidence["questionSha256"] = "0" * 64 + self.assertTrue(self.errors()) + + def test_path_traversal(self): + self.response["references"][0]["path"] = "microsoft/knowledge/../skills/example.md" + self.assertTrue(self.errors()) + + def test_empty_reference_outcomes(self): + for outcome in ("failed", "no-knowledge"): + self.response.update(outcome=outcome, references=[], **{"outcome-reason": "No usable knowledge"}) + self.assertEqual(self.errors(evidence=False), []) + + def test_schema_violation(self): + self.response["references"][0].update(applicability="conditional", unknown=[]) + self.assertTrue(self.errors()) + + def test_duplicate_read_evidence(self): + self.evidence["reads"].append(copy.deepcopy(self.evidence["reads"][0])) + self.assertTrue(self.errors()) + + def test_cli_preserves_question_and_rejects_changes(self): + question = self.root / "question.txt" + question.write_bytes(self.question.encode()) + response = self.root / "response.json" + evidence = self.root / "evidence.json" + response.write_text(json.dumps(self.response), encoding="utf-8") + evidence.write_text(json.dumps(self.evidence), encoding="utf-8") + command = [sys.executable, str(TOOL), str(response), "--root", str(self.root), + "--question-file", str(question), "--read-evidence", str(evidence), "--run-id", "test-run"] + result = subprocess.run(command, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertTrue(json.loads(result.stdout)["valid"]) + self.response["question"] = self.question.strip() + response.write_text(json.dumps(self.response), encoding="utf-8") + result = subprocess.run(command, capture_output=True, text=True) + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + + def test_strict_json_duplicate_key(self): + file = self.root / "duplicate.json" + file.write_text('{"question":"one","question":"two"}') + with self.assertRaises(ValueError): + module.strict_json(file) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/scripts/validate_frontmatter.py b/.github/scripts/validate_frontmatter.py index 51ff546e..b68380f3 100644 --- a/.github/scripts/validate_frontmatter.py +++ b/.github/scripts/validate_frontmatter.py @@ -45,9 +45,9 @@ HOST_SKILL_REQUIRED_KEYS = {"name", "description"} STANDARD_INPUTS = { - "pr-diff", "object-list", "file-path", "folder-path", "repository", "telemetry-query", + "pr-diff", "object-list", "file-path", "folder-path", "repository", "telemetry-query", "knowledge-query", } -ALLOWED_OUTPUTS = {"findings-report"} +ALLOWED_OUTPUTS = {"findings-report", "knowledge-response"} VALID_SAMPLE_KINDS = {"good", "bad"} ACTION_SKILL_SECTIONS = ["Source", "Relevance", "Worklist", "Action", "Output"] @@ -342,7 +342,7 @@ def validate_action_skill(path: Path, parsed: Parsed, report: Report) -> None: else: bad = [x for x in out if x not in ALLOWED_OUTPUTS] if bad: - report.error(path, "R18", f"outputs contains non-allowed values {bad}; currently only {sorted(ALLOWED_OUTPUTS)} is defined", 1) + report.error(path, "R18", f"outputs contains non-allowed values {bad}; allowed values are {sorted(ALLOWED_OUTPUTS)}", 1) # R19 optional filter dimensions, if present if "bc-version" in fm: diff --git a/.github/workflows/validate-frontmatter.yml b/.github/workflows/validate-frontmatter.yml index dd7922d4..45ffde56 100644 --- a/.github/workflows/validate-frontmatter.yml +++ b/.github/workflows/validate-frontmatter.yml @@ -19,7 +19,13 @@ jobs: python-version: "3.12" - name: Install dependencies - run: pip install pyyaml + run: pip install pyyaml jsonschema - name: Run validator run: python .github/scripts/validate_frontmatter.py --root . + + - name: Validate knowledge response contract + run: python .github/scripts/test_knowledge_response.py + + - name: Validate knowledge response semantics + run: python .github/scripts/test_knowledge_response_semantics.py diff --git a/README.md b/README.md index 9aaea9a6..bd332205 100644 --- a/README.md +++ b/README.md @@ -26,8 +26,9 @@ copilot plugin install microsoft/BCQuality copilot plugin list ``` -The list should include `bcquality`. The plugin currently exposes the -[`al-code-review`](skills/al-code-review/SKILL.md) skill. Installation and skill +The list should include `bcquality`. Its public skill entry points are +[`al-code-review`](skills/al-code-review/SKILL.md) and +[`al-knowledge`](skills/al-knowledge/SKILL.md). Installation and skill discovery are the general pattern; reviewing an app is one example of using it. ### Example: Review a complete app folder @@ -64,6 +65,7 @@ the review can still discover knowledge by reading the folders. | --- | --- | | Choose direct reading, a supplied skill, or my own agent | [Ways to use BCQuality](docs/using-bcquality.md#choose-how-to-use-bcquality) | | Review a file, changes, a branch, or a particular concern | [Using BCQuality](docs/using-bcquality.md) | +| Support design and specification with cited knowledge | [Knowledge consultation](docs/knowledge-consultation.md) | | Resolve setup problems, incomplete reviews, or incorrect findings | [Troubleshooting and support](docs/troubleshooting.md) | | Browse the available guidance | [Knowledge by domain](docs/using-bcquality.md#knowledge-by-domain) | | Configure the plugin or use my organization's rules | [Customizing BCQuality](docs/customizing-bcquality.md) | diff --git a/docs/README.md b/docs/README.md index d9ca27b0..58523540 100644 --- a/docs/README.md +++ b/docs/README.md @@ -10,6 +10,7 @@ of BCQuality's internal protocol is needed. | --- | --- | | Choose direct reading, a supplied skill, or my own agent | [Ways to use BCQuality](using-bcquality.md#choose-how-to-use-bcquality) | | Review an app, file, changes, or branch | [Using BCQuality](using-bcquality.md) | +| Support design and specification with cited knowledge | [Knowledge consultation](knowledge-consultation.md) | | Understand a report and its limitations | [Reading your results](using-bcquality.md#reading-your-results) | | Find a particular rule or example | [Knowledge by domain](using-bcquality.md#knowledge-by-domain) | | Fix setup problems or report an incorrect finding | [Troubleshooting and support](troubleshooting.md) | @@ -27,6 +28,7 @@ prerequisites for using the plugin. | [How agents consume BCQuality](agent-consumption.md) | Architecture, repository structure, routing, and delivery of findings. | | [Standalone runner](standalone-runner.md) | Optional model selection, scheduling, retries, and telemetry. | | [Global skills](../skills/README.md) | Host adapters versus internal protocol files. | +| [Knowledge-response validation](knowledge-response-validation.md) | Optional consumer checks for question fidelity and complete citation reads. | | [Entry](../skills/entry.md) | Task context and skill dispatch. | | [READ](../skills/read.md) | Knowledge schema, applicability, and precedence. | | [DO](../skills/do.md) | Action-skill format and structured output contract. | diff --git a/docs/agent-consumption.md b/docs/agent-consumption.md index 06858c0a..cc5c3612 100644 --- a/docs/agent-consumption.md +++ b/docs/agent-consumption.md @@ -78,8 +78,9 @@ Add scheduling, retries, and rendering only when needed, using the - **Layer content** in `/microsoft/`, `/community/`, and `/custom/` — knowledge files and action skills grouped by authority. When BCQuality is installed as a standalone plugin, it additionally exposes -`skills/al-code-review/SKILL.md`. This is a host-format adapter, not another -action skill: it creates the task context and enters the same flow at Entry. +`skills/al-code-review/SKILL.md` and `skills/al-knowledge/SKILL.md`. These public +skill entry points tell the host agent how to create the task context and enter +the same flow at Entry. They are not additional action skills. ## Repository structure @@ -88,6 +89,7 @@ action skill: it creates the task context and enters the same flow at Entry. | `skills/entry.md` | Routes a task to action skills. | | `skills/read.md`, `skills/do.md`, `skills/write.md` | Stable knowledge, action-skill, and authoring contracts. | | `skills/al-code-review/SKILL.md` | Host-format plugin adapter. | +| `skills/al-knowledge/SKILL.md` | Public skill entry point for cited knowledge consultation. | | `/knowledge//` | Atomic articles and optional sibling samples. | | `/skills/` | Layer-owned action skills. | | `docs/` | Partner guides and integration references. | @@ -120,10 +122,11 @@ The orchestrator has a URL setting that points at BCQuality (default: `github.co The agent reads `/skills/entry.md` and runs it against the task context. Entry applies its Source → Relevance → Worklist → Action steps over the action skills under `*/skills/**/*.md` and returns a **dispatch record**: the set of action skills to invoke, plus a list of candidates it skipped (with reasons). Routing is a skill, not orchestrator logic. For a standalone plugin installation, the host activates the -`skills/al-code-review/SKILL.md` adapter first. That adapter preserves the -caller's actual goal, constructs the task context, and invokes Entry. It does -not select the internal `microsoft/skills/review/al-code-review.md` action skill -itself or duplicate Entry's preparation, routing, and failure semantics. +selected public skill entry point first. It preserves the caller's actual goal, +constructs the task context, and invokes Entry. It does not select an internal +action skill itself or duplicate Entry's preparation, routing, and failure +semantics. For knowledge consultation, it binds the exact question as +`knowledge-query`; see [knowledge consultation](knowledge-consultation.md). ### 3. Agent consumes the dispatch record The dispatch record names one or more action skills and the subset of inputs each should receive. If the outcome is `no-match` or `failed`, the agent returns the record to the orchestrator unchanged. @@ -165,7 +168,8 @@ security boundary. See [layer selection](customizing-bcquality.md#select-layers- The index changes only *how candidates are discovered*, never *which are selected*. The Worklist predicate is unchanged — `keywords` still drive selection — and the agent still opens each worklisted article **in full** to read its `## Best Practice` / `## Anti Pattern` rule bodies; the index is discovery metadata only and never substitutes for the article body. When no index is present, skills fall back to path-based discovery (collect by domain folder), so review still works. ### 6. Agent emits structured output -The output contract is defined in the DO meta-skill so that every action skill — today's and next year's — produces the same shape: +DO defines each declared output kind so consumers can validate its shared +contract. Review actions produce `findings-report`, whose structure includes: - **Outcome** — `completed`, `not-applicable`, `no-knowledge`, `partial`, or `failed`. An orchestrator can distinguish a clean run from a no-op from a failure without guessing. - **Findings** — what the skill observed (severity, message, optional location). @@ -174,11 +178,21 @@ The output contract is defined in the DO meta-skill so that every action skill - **Confidence** — per-finding evidence strength. - **Suppressed** — knowledge files that were discarded by layer precedence or configuration, so reviewers can see what was overridden. -The orchestrator parses this **without skill-specific logic**. This is the point of the contract: orchestrators and action skills evolve independently. +Knowledge-consultation actions produce `knowledge-response`, with the exact +question, supported answer and article references. See the +[DO knowledge-response contract](../skills/do.md#knowledge-response-contract) +and [consumer validation](knowledge-response-validation.md). + +The orchestrator validates the declared output kind without interpreting +private skill-specific fields. Action skills and consumers share the same +output contracts. ### 7. Orchestrator integrates The orchestrator turns findings into PR comments, build gates, or IDE diagnostics, and links the references back to the knowledge files so the PR author — human or agent — can read the guidance. +For knowledge consultation, it displays validated guidance with citations. +It does not convert that guidance into findings or use it as an approval gate. + ## Knowledge-backed and agent findings BCQuality is an **additive** knowledge layer. The agent surfaces two kinds of findings, both shaped to the same DO output contract: diff --git a/docs/assets/al-knowledge-agent-diagram-en.png b/docs/assets/al-knowledge-agent-diagram-en.png new file mode 100644 index 00000000..fc9702d2 Binary files /dev/null and b/docs/assets/al-knowledge-agent-diagram-en.png differ diff --git a/docs/assets/al-knowledge-diagram-en.mmd b/docs/assets/al-knowledge-diagram-en.mmd new file mode 100644 index 00000000..88ac4116 --- /dev/null +++ b/docs/assets/al-knowledge-diagram-en.mmd @@ -0,0 +1,14 @@ +flowchart TD + A["You or the Agent
Ask a question"] + B["1 · Skill entry point
Keeps your exact question"] + C["2 · Entry
Selects the knowledge action"] + D["3 · Find and filter
Topic, BC version, technology and country"] + E["4 · Read full articles
The index helps find articles; it is not evidence"] + F["5 · Answer with sources
knowledge-response"] + G["No useful knowledge
no-knowledge
No invented sources"] + A --> B --> C --> D --> E + E -->|Evidence found| F + E -->|No evidence| G + classDef node fill:#fce7f3,stroke:#ddd4da,color:#9e3868,rx:20,ry:20; + class A,B,C,D,E,F,G node; + linkStyle default stroke:#999,stroke-width:1.5px; diff --git a/docs/assets/al-knowledge-technical-en.mmd b/docs/assets/al-knowledge-technical-en.mmd new file mode 100644 index 00000000..d0639ad4 --- /dev/null +++ b/docs/assets/al-knowledge-technical-en.mmd @@ -0,0 +1,26 @@ +flowchart TD + Q["User or agent
Exact question + known target context"] + A["Public skill entry point
The agent reads SKILL.md
Preserve the question and resolve BCQuality root"] + E["Entry
Prepare the index when possible
Dispatch by goal and compatible inputs"] + D["Internal al-knowledge action"] + C[("Enabled knowledge corpus
custom / community / microsoft")] + R["READ rules
Find candidates, check applicability
and apply layer precedence"] + B["Read complete article bodies
Bounded helpers or native reads through EOF
Follow continuation pages"] + O["knowledge-response JSON
completed / no-knowledge / partial / failed"] + W["Consumer-collected read evidence
Run ID, exact paths, full-read status,
content hashes and byte counts"] + V{"Consumer validation
Schema + exact question
+ same-run full reads against live corpus"} + Y["Accept the unchanged response"] + N["Reject acceptance
Preserve raw response
Record validation errors separately"] + X["Entry no-match or failed
Return the dispatch record unchanged"] + Q --> A --> E + E -->|routed| D --> R --> B --> O --> V + E -->|no-match or failed| X + C --> R + B -. "successful tool results" .-> W + W --> V + Q -. "original bound question" .-> V + V -->|valid| Y + V -->|invalid| N + classDef process fill:#fce7f3,stroke:#ddd4da,color:#9e3868; + class Q,A,E,D,C,R,B,O,W,V,Y,N,X process; + linkStyle default stroke:#999,stroke-width:1.5px; diff --git a/docs/knowledge-consultation.md b/docs/knowledge-consultation.md new file mode 100644 index 00000000..6850e30d --- /dev/null +++ b/docs/knowledge-consultation.md @@ -0,0 +1,99 @@ +# Knowledge consultation for design and specification + +[Documentation](README.md) | [Public skill entry point](../skills/al-knowledge/SKILL.md) | [Response validation](knowledge-response-validation.md) + +`al-knowledge` helps AL coding agents design and specify Business Central +solutions using cited guidance from BCQuality. A user or agent can ask a focused +development question before source code exists. The skill follows BCQuality's +contracts and is independent of any particular consumer framework or agent role. +Available guidance is limited to the installed corpus. + +## Ask a question + +> Use al-knowledge to explain when SetLoadFields is useful in Business Central AL. +> Cite the BCQuality articles you read. Do not review my app. Return the +> knowledge-response JSON unchanged. + +Supply the BC version, technologies, countries and application area when known. +Do not invent target details to make an article applicable. Unknown dimensions +remain explicit on conditional references under [READ](../skills/read.md). + +## Request path + +The public skill entry point is the file the host agent reads to start. It +preserves the exact question as `knowledge-query`, resolves the installed +BCQuality root, and follows [Entry](../skills/entry.md). Entry prepares the +knowledge index when possible and selects an action with compatible input types. +A question-only request cannot dispatch review actions. + +The [internal knowledge action](../microsoft/skills/knowledge/al-knowledge.md) +finds relevant articles, checks applicability and layer precedence, reads their +full bodies, and returns cited guidance. Index entries support discovery but +are not evidence of complete reads. Bounded helpers require every continuation +page; when unavailable, native file reads must continue through EOF. + +```mermaid +flowchart TD + Q["User or agent
Exact question and known context"] + S["Public skill entry point
Prepare knowledge-query
Resolve the BCQuality root"] + E["Entry
Prepare index when possible
Select the knowledge action"] + A["Internal al-knowledge action
Apply READ rules
Read complete articles"] + R["knowledge-response
Answer, outcome and citations"] + V{"Consumer validation
Schema, exact question
and complete citation reads"} + OK["Accept the unchanged response"] + BAD["Preserve the raw response
Record validation errors separately"] + STOP["Return Entry dispatch record
no-match or failed"] + Q --> S --> E + E -->|routed| A --> R --> V + E -->|no-match or failed| STOP + V -->|valid| OK + V -->|invalid| BAD +``` + +The host supplies the executing agent. It can follow these instructions inline +when no skill-invocation tool exists. Reading the skill entry point and Entry is +only preparation; the agent must execute the dispatched action and read articles. + +![Knowledge consultation: the question, skill entry point, Entry, article reading and cited response or no-knowledge branch](assets/al-knowledge-agent-diagram-en.png) + +[Editable explanatory diagram](assets/al-knowledge-diagram-en.mmd) and +[technical diagram source](assets/al-knowledge-technical-en.mmd). + +## Response and acceptance + +The action returns `knowledge-response` under [DO](../skills/do.md#knowledge-response-contract) +and its [schema](../schemas/knowledge-response.schema.json). `completed` requires +a useful answer and a verified citation. `no-knowledge` explains a gap with no +citations. `partial` explains unfinished work and cites only articles read. +`failed` explains a failure with no citations. Entry's `no-match` and `failed` +dispatch records remain separate and return unchanged. + +The consumer validates the schema, exact bound question and complete citation +reads before acceptance. It preserves invalid output and records validation +errors separately. The optional [consumer validator](knowledge-response-validation.md) +checks question fidelity and full-read evidence against live files. Its evidence +comes from successful tool results in the same run, not the answering agent's +assertions. It does not independently prove that every claim follows a source. +READ still governs applicability and interpretation. + +`al-code-review` remains the skill for review findings. Knowledge consultation +provides guidance, not a review verdict or a design approval gate. + +## Prerequisites and optional tooling + +The host agent needs access to the public skill entry point, Entry, READ, DO, +the internal action, response schema and enabled knowledge articles. It must +be able to read complete files and retain complete tool results. Authentication +to the agent provider follows the chosen host. No app, compiler or Business +Central tenant is required for a question-only consultation. + +PowerShell 7.2 or later runs the existing bounded retrieval helpers. READ already +permits path discovery and native complete file reads when PowerShell, a helper, +or a prepared index is unavailable. Index write permission is needed only when +running Entry's generator. + +Python with `jsonschema` is needed only for the proposed consumer validator. +Another consumer can enforce the same contract with its own implementation. +The read-evidence manifest is one proposed implementation, not a requirement +that all hosts adopt that format. Complete citation reads and response validation +remain required regardless of the tools selected. diff --git a/docs/knowledge-response-validation.md b/docs/knowledge-response-validation.md new file mode 100644 index 00000000..a7fd3118 --- /dev/null +++ b/docs/knowledge-response-validation.md @@ -0,0 +1,117 @@ +# Validating a knowledge response + +`al-knowledge` returns guidance under the [DO knowledge-response contract](../skills/do.md#knowledge-response-contract). +The consumer checks its result before acceptance. This helper works independently +of consumer frameworks and hosts and does not execute the skill. + +## Optional tooling and existing contracts + +BCQuality already requires complete article reads and citation integrity under +READ and DO, and deterministic consumer validation for review reports. The new +knowledge-response contract applies these principles to knowledge consultation. + +This Python helper and its read-evidence manifest are proposed optional tooling, +not existing runtime requirements or a pre-existing BCQuality evidence standard. +Python with `jsonschema` is needed only to run this implementation. Consumers may +validate the same contract with another implementation and their native tools. +They still must verify the exact question and complete citation reads in the +current run; they do not have to adopt this script or its manifest format. + +The existing PowerShell retrieval helpers require PowerShell 7.2 or later. READ +already permits path discovery and native complete file reads when those helpers +or their prepared index are unavailable. The original repository's Python and +PyYAML frontmatter checks are development/CI tooling, not prerequisites for +question-only consultation. + + +In an inline host, the answering agent checks its own successful full-body tool +reads. This is self-attestation, not independent citation verification. A +consumer with a trusted read collector can independently check the evidence +using the optional validator. Neither mode proves that every claim correctly +interprets its sources. Report the mode used; never present inline checks as +independent verification. + +## Consumer inputs + +Preserve the raw response as one JSON object, without Markdown fences or extra +text. Save the exact bound question as UTF-8. Spaces and CRLF/LF line endings are +part of the question; the validator does not trim or normalize them. A UTF-8 BOM +is an encoding marker and not part of the bound question. + +The consumer collects read evidence from successful tool results in the same +run, independently of the action's answer. A claim in `answer`, an attempted +tool call, an index row, or a displayed filename is insufficient. Mark an +article complete only after receiving its full body through EOF, and compute +its SHA-256 and UTF-8 byte count from the exact complete file bytes received. +Preserve source encoding and line endings when collecting that evidence. + +The evidence file has this shape (values below are placeholders): + +```json +{ + "runId": "consumer-assigned-run-id", + "questionSha256": "sha256-of-the-exact-question-encoded-as-utf8", + "reads": [ + { + "path": "microsoft/knowledge/performance/use-setloadfields-for-partial-records.md", + "complete": true, + "bytes": 1234, + "sha256": "sha256-of-the-complete-article-file-bytes" + } + ] +} +``` + +Use a consumer-assigned identifier for the current run. Supply it separately +to the validator and reject evidence from another run or question. Do not ask +the answering agent to invent this manifest. The validator trusts the evidence +collector; hashes and a boolean alone cannot independently prove a tool read. + +## Run the checks + +Install `jsonschema` in the consumer's Python environment if it is unavailable. +From the BCQuality root: + +```powershell +python tools/validate_knowledge_response.py response.json ` + --root . --question-file question.txt ` + --read-evidence reads.json --run-id consumer-assigned-run-id +``` + +The tool checks the response schema, exact question, existing canonical citation +paths within the corpus, and complete-read evidence with matching current file +hashes and byte counts. A changed article, truncated read, missing evidence, +different-layer path with the same filename, or mismatched run is rejected. +The tool never repairs or rewrites the response. + +Exit codes: `0` means these checks passed, `1` means validation failed, and `2` +means invalid JSON or an input/resource error prevented validation. Failed +validation is not a successful knowledge response. Preserve the raw payload and +record the validation errors separately. Do not accept or display its answer as +validated BCQuality guidance. A response with no citations does not need a read +manifest, but still must satisfy the schema and preserve the question. + +Passing does not establish that every claim follows from its references, that +target applicability is correct, or that the corpus covers the question fully. +Those checks require the [READ rules](../skills/read.md) and actual execution +evidence. Catalog discovery and schema validity alone do not prove execution. + +## Enabled layers + +Supply the consumer configuration through `--enabled-layers microsoft community` +(or all three by default). An explicit empty list disables every layer. +Citations and layer-precedence suppressions must belong to enabled layers. +A `suppressed` entry with reason `configuration` may identify an existing +article in a disabled layer; it is an exclusion record, never supporting evidence. +All these paths must be canonical existing corpus files. + +## Regression tests + +```powershell +python .github/scripts/test_knowledge_response.py +python .github/scripts/test_knowledge_response_semantics.py +``` + +The first checks JSON Schema cases. The second uses fixture evidence to exercise +question fidelity, complete reads, current body hashes, path identity, and CLI +behavior. Fixture tests are not live host executions. diff --git a/docs/using-bcquality.md b/docs/using-bcquality.md index b65fcb09..83d65a87 100644 --- a/docs/using-bcquality.md +++ b/docs/using-bcquality.md @@ -15,6 +15,14 @@ and rendering; BCQuality does not install a BC extension or an agent. | Use a supplied skill | Follow the [plugin quick start](../README.md#quick-start). The currently exposed skill, `al-code-review`, performs reviews and returns findings. | | Use your own agent or workflow | Supply selected articles as context, as described below, or use the [integration bootstrap](agent-consumption.md#try-a-minimal-integration) to execute BCQuality action skills without the plugin. | +### Knowledge consultation + +Use `al-knowledge` to support design and specification with guidance from the +articles the agent actually reads. A question needs no app folder, file or diff. +The response contains the exact question, cited guidance and applicability +conditions. See [knowledge consultation](knowledge-consultation.md) for the +request path, prerequisites and validation. + ### Read and reuse an article Start with a concern, such as `SetLoadFields`, and search within diff --git a/microsoft/skills/knowledge/al-knowledge.md b/microsoft/skills/knowledge/al-knowledge.md new file mode 100644 index 00000000..4f32c727 --- /dev/null +++ b/microsoft/skills/knowledge/al-knowledge.md @@ -0,0 +1,78 @@ +--- +kind: action-skill +id: al-knowledge +version: 1 +title: AL knowledge consultation +description: Answer a Business Central development question with cited knowledge, without reviewing source code. +inputs: [knowledge-query] +outputs: [knowledge-response] +bc-version: [all] +technologies: [al] +countries: [w1] +application-area: [all] +--- + +# AL knowledge consultation + +`knowledge-query` is the caller's exact question. This skill accepts optional +target context from Entry. It does not require an app, a file, or a diff. + +## Source + +Search `knowledge-index.json` for matching domains, titles, descriptions, and +keywords across enabled layers. Resolve the index against BCQuality's root. +When the index could not be generated, discover candidate article paths in +`/knowledge/**/` instead. The index is discovery metadata, +never the article body or evidence of applicability. + +## Relevance + +Apply READ's frontmatter matching rules for any known BC version, technology, +country, and application area. Exclude definite mismatches. Keep conditional +matches with their exact unknown dimensions. Interpret competing articles and +layer precedence under READ; record any displaced article in `suppressed`. + +Frontmatter applicability is not question relevance. A universal country or +version filter does not make an article answer every domain question. Candidate +keyword overlap is discovery only. After reading, retain as supporting evidence +only articles that address the requested subject and its essential constraints. + +## Worklist + +Choose the smallest set of articles that answers the actual question. Match +the question's concepts to article descriptions, filenames, and keywords. +Open each chosen article in full, including relevant sibling samples if cited +by that article. If the question spans domains, select each necessary domain; +do not turn a narrow question into an exhaustive corpus summary. Follow READ's +[bounded retrieval workflow](../../../skills/read.md#bounded-retrieval-for-review-skills), +including all continuation pages and complete bodies. Never truncate helper +output or treat an unread body as evidence. Do not cite an article merely +because it appeared in the index. + +## Action + +Answer only what the read articles support. State conditions, exceptions, +unknown target dimensions, and any unresolved conflict. Cite exact article +paths in the answer and `references`; do not infer a path from a title. This +is guidance, not a code review: do not inspect source to produce defects, +invoke review action skills, emit severities, or generate a findings-report. +If no applicable article answers the question, return `no-knowledge` instead +of filling the gap from general model knowledge under BCQuality's name. + +Before choosing the outcome, check whether the supporting articles answer the +bound question rather than a broader or adjacent question. Do not replace an +unsupported domain or target with general coding or localization advice merely +because some keywords overlap. If none answers the requested subject, return +`no-knowledge` with `references: []`; the answer may explain the coverage gap, +but must not append adjacent guidance with citations. Candidate reads may be +recorded separately in execution diagnostics. Use `partial` only when articles +support a genuine part of the requested question, with the uncovered part +explicitly identified; adjacent-topic advice alone is not partial coverage. + +## Output + +Return one JSON value following `schemas/knowledge-response.schema.json` and +DO's knowledge-response rules. `question` preserves the caller's question. +`completed` requires an answer to the actual question and at least one fully +read, checked supporting reference. `partial` +requires a reason and only cites articles actually read; `failed` cites none. diff --git a/plugin.json b/plugin.json index c99785b0..735e83ae 100644 --- a/plugin.json +++ b/plugin.json @@ -1,6 +1,6 @@ { "name": "bcquality", - "description": "Quality skills and knowledge for Business Central development. Exposes a standalone AL review adapter backed by BCQuality's Entry protocol.", + "description": "Business Central quality knowledge and skills for AL code review and cited knowledge consultation.", "version": "0.2.0", "author": { "name": "microsoft/BCQuality", @@ -13,6 +13,7 @@ "al", "business-central", "code-review", + "knowledge", "quality" ], "skills": [ diff --git a/schemas/knowledge-response.schema.json b/schemas/knowledge-response.schema.json new file mode 100644 index 00000000..9fc45715 --- /dev/null +++ b/schemas/knowledge-response.schema.json @@ -0,0 +1,49 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/microsoft/BCQuality/schemas/knowledge-response.schema.json", + "title": "BCQuality knowledge consultation response", + "type": "object", + "additionalProperties": false, + "required": ["skill", "outcome", "question", "answer", "references", "suppressed"], + "properties": { + "skill": { + "type": "object", "additionalProperties": false, "required": ["id", "version"], + "properties": {"id": {"const": "al-knowledge"}, "version": {"type": "integer", "minimum": 1}} + }, + "outcome": {"enum": ["completed", "no-knowledge", "partial", "failed"]}, + "outcome-reason": {"type": "string", "minLength": 1}, + "question": {"type": "string", "minLength": 1}, + "answer": {"type": "string"}, + "references": { + "type": "array", "uniqueItems": true, + "items": { + "type": "object", "additionalProperties": false, + "required": ["path", "applicability", "unknown"], + "properties": { + "path": {"type": "string", "pattern": "^(microsoft|community|custom)/knowledge/.+\\.md$"}, + "sha": {"type": "string", "pattern": "^[a-f0-9]{40}$"}, + "applicability": {"enum": ["applicable", "conditional"]}, + "unknown": {"type": "array", "uniqueItems": true, "items": {"enum": ["bc-version", "technologies", "countries", "application-area"]}} + }, + "allOf": [ + {"if": {"properties": {"applicability": {"const": "applicable"}}}, "then": {"properties": {"unknown": {"maxItems": 0}}}}, + {"if": {"properties": {"applicability": {"const": "conditional"}}}, "then": {"properties": {"unknown": {"minItems": 1}}}} + ] + } + }, + "suppressed": { + "type": "array", "items": { + "type": "object", "additionalProperties": false, "required": ["path", "reason"], + "properties": { + "path": {"type": "string", "pattern": "^(microsoft|community|custom)/knowledge/.+\\.md$"}, + "reason": {"enum": ["layer-precedence", "configuration"]} + } + } + } + }, + "allOf": [ + {"if": {"properties": {"outcome": {"enum": ["partial", "failed"]}}}, "then": {"required": ["outcome-reason"]}}, + {"if": {"properties": {"outcome": {"enum": ["failed", "no-knowledge"]}}}, "then": {"properties": {"references": {"maxItems": 0}}}}, + {"if": {"properties": {"outcome": {"const": "completed"}}}, "then": {"properties": {"references": {"minItems": 1}, "answer": {"minLength": 1}}}} + ] +} diff --git a/schemas/skill-index.schema.json b/schemas/skill-index.schema.json index 01944bb9..6084ac26 100644 --- a/schemas/skill-index.schema.json +++ b/schemas/skill-index.schema.json @@ -48,7 +48,7 @@ "type": "array", "minItems": 1, "maxItems": 1, - "items": { "const": "findings-report" } + "items": { "enum": ["findings-report", "knowledge-response"] } }, "filters": { "type": "object", diff --git a/skills/README.md b/skills/README.md index 766131f5..e68992d4 100644 --- a/skills/README.md +++ b/skills/README.md @@ -1,7 +1,7 @@ # BCQuality global skills This folder contains BCQuality's layer-independent protocol files and the -host-native adapter used by standalone plugin installations. +public skill entry points used by standalone plugin installations. The protocol files have two kinds: @@ -26,17 +26,21 @@ Routing logic lives in Entry, not in the orchestrator. An agent that knows only READ and DO are read on demand — typically by the first action skill the agent executes after dispatch. They are not prerequisites for invoking Entry. WRITE is only used when scaffolding new content. -## Standalone plugin adapter +## Public skill entry points | Path | Role | |---|---| -| [`al-code-review/SKILL.md`](al-code-review/SKILL.md) | Exposes BCQuality through the standard `SKILL.md` format when this repository is installed as a plugin. | +| [`al-code-review/SKILL.md`](al-code-review/SKILL.md) | Starts a code review through the standard host `SKILL.md` format. | +| [`al-knowledge/SKILL.md`](al-knowledge/SKILL.md) | Starts a cited knowledge consultation for design, specification or a focused development question. | -The adapter is deliberately thin. It translates the caller's request into an -Entry task context, then follows Entry's dispatch without owning routing, -review, index, or output policy. It is not an action skill, is not considered -by Entry, and should not accumulate behavior already defined by `entry.md`, -`read.md`, `do.md`, or a layered action skill. +These files tell the host agent how to start the skill: resolve BCQuality's +root, prepare the caller's request, and follow Entry's dispatch. They do not +own routing, article selection, index preparation or output policy. They are +not action skills and are not candidates for Entry. Their behavior follows +`entry.md`, `read.md`, `do.md`, and the selected internal action skill. + +The same agent can execute the instructions inline when the host has no +separate skill-invocation tool. A file read alone does not complete execution. This gives the two skill formats distinct roles: diff --git a/skills/al-knowledge/SKILL.md b/skills/al-knowledge/SKILL.md new file mode 100644 index 00000000..5cdd6229 --- /dev/null +++ b/skills/al-knowledge/SKILL.md @@ -0,0 +1,51 @@ +--- +name: al-knowledge +description: Answer Business Central development questions from BCQuality's installed knowledge articles, with exact citations and no code review. Use for design, specification, or a focused knowledge question. +--- + +# AL knowledge consultation + +This is the public skill entry point: the file the host agent reads to start +the skill. It tells the agent how to prepare the question and follow +BCQuality's routing instructions. The same agent can execute all steps +inline; no separate program or agent is required. + +This file is distinct from `skills/entry.md`, which owns routing. Article +selection, precedence, and output policy belong to Entry, READ, DO, and the +dispatched internal action skill. + +## Execute + +1. Resolve `PLUGIN_ROOT` from the installed plugin's `plugin.json`; this file is + `PLUGIN_ROOT/skills/al-knowledge/SKILL.md`. Never assume the user's app root + is the plugin root. +2. Build Entry's `task-context`. Copy the caller's question verbatim into `goal`. + Set `inputs-available: [knowledge-query]` and bind `knowledge-query` to that + exact question. Pass `technologies`, `bc-version`, `countries`, and + `application-area` only when provided or reliably determined. Pass the + optional `BCQUALITY_ENABLED_LAYERS` and `BCQUALITY_DISABLED_SKILLS` settings + using the same parsing rules as `al-code-review`. +3. Read and execute `PLUGIN_ROOT/skills/entry.md`, including Preparation, + resolving all paths against `PLUGIN_ROOT`. If `pwsh` or index generation is + unavailable, use the documented path-based discovery fallback. +4. Execute only the dispatched action skills with their exact input subsets. + Read `PLUGIN_ROOT/skills/read.md` and `PLUGIN_ROOT/skills/do.md` on demand. + Do not call `al-code-review` for this knowledge question, synthesize a + dispatch, or turn an explanation into a finding. +5. Validate the action skill's `knowledge-response` under DO before accepting + it: check the schema, exact bound question, and current-run full reads of + every cited path. The consumer can use + `PLUGIN_ROOT/tools/validate_knowledge_response.py` with its own question + and read evidence. Preserve the raw result when validation fails; record + that failure separately instead of repairing or accepting the answer. + Inline checks by the answering agent are self-attestation, not independent + verification. Independent checks require a trusted consumer read collector. + Return a valid response unchanged, or Entry's `no-match` / `failed` + dispatch record unchanged. + +Where the host has no tool to invoke a skill and skills are instruction files the +agent reads (for example VS Code Copilot), invoking this skill means executing +steps 1–5 inline in the caller's own context. The `knowledge-response` the +dispatched action skill produces there is the response to return: producing it +that way is the protocol, not a fabrication. Lacking an invocation tool is never +a reason to skip this skill or to answer without its response. diff --git a/skills/do.md b/skills/do.md index a10e5b6c..983a4810 100644 --- a/skills/do.md +++ b/skills/do.md @@ -62,12 +62,16 @@ application-area: [all] `bc-version`, `technologies`, `countries`, `application-area` are optional filters that let an orchestrator pre-select applicable skills for a task. They follow the same semantics as in READ. `inputs` is a list of abstract input types the skill **accepts**. Standard values: -`pr-diff`, `object-list`, `file-path`, `folder-path`, `repository`, and -`telemetry-query`. Semantics are any-of: the orchestrator supplies whichever +`pr-diff`, `object-list`, `file-path`, `folder-path`, `repository`, +`telemetry-query`, and `knowledge-query` (a question without source-code scope). +Semantics are any-of: the orchestrator supplies whichever listed input types it has, and the skill is invoked with a non-empty subset of its declared `inputs`. A skill that cannot proceed with the supplied subset MUST return `outcome: "not-applicable"`. `outputs` is always a single-element -list naming the output kind; today only `findings-report` is defined. +list naming the output kind; `findings-report` covers reviews and +`knowledge-response` covers knowledge consultation. The two output kinds must +not be conflated. `knowledge-query` is a bound input value: its exact question +travels to a dispatched skill, while Entry's `goal` supports routing. `file-path` is one file. `folder-path` is a directory whose recursively contained files form the complete current-state input, such as a Business @@ -100,11 +104,13 @@ Every action skill MUST contain these five sections, in order: **Worklist.** Narrow the relevant candidates to the subset that applies to the current task. This is where the task-specific signal enters: the objects changed in the PR, the queries being audited, the skeleton being generated. Typical moves: match `keywords` against task vocabulary, match file topics against changed objects, deduplicate by concern. -**Action.** Execute the skill's work against the worklist. Evaluate each item in the worklist against the task input and emit findings. The action step is where skill behavior differs; the preceding three steps are uniform. +**Action.** Execute the skill's work against the worklist and produce its declared output kind. Review actions evaluate items against the task input and emit findings; knowledge-consultation actions answer the question using the articles read. The action step is where skill behavior differs; the preceding three steps are uniform. ## Output contract -Every action skill emits a single JSON document that conforms to this schema: +Every action skill emits a single JSON document conforming to its declared +output kind. `knowledge-response` is defined below; the following review +contract applies only to `outputs: [findings-report]`: The machine-readable structural schema is [`schemas/findings-report.schema.json`](../schemas/findings-report.schema.json). @@ -390,6 +396,48 @@ Severity taxonomy: - `minor` — quality concern; worth flagging but not a gate. - `info` — observation or context; not actionable on its own. +## Knowledge-response contract + +An action skill declaring `outputs: [knowledge-response]` emits one strict JSON +object conforming to [`schemas/knowledge-response.schema.json`](../schemas/knowledge-response.schema.json). +This output represents cited guidance, never a finding, severity, code location, +or code-generation artifact. The `question` is the exact bound +`knowledge-query`; `answer` is readable guidance and identifies the exact +article paths used. `references[]` contains only paths to articles whose full +bodies were opened in this run. An optional `sha` is an observed commit, not +an expected pin. A conditional reference lists each unknown applicability +dimension in `unknown[]`; fully applicable references use `applicable` with an +empty `unknown[]`. READ defines matching and conflict precedence. Record +displaced articles in `suppressed[]` without citing them as supporting guidance. + +`completed` requires a useful answer to the bound question and at least one +verified supporting reference. Applicability under READ does not establish +question relevance. Articles about an adjacent topic do not justify `completed` +or `partial` when none addresses the requested subject. In that case use +`no-knowledge` with no references and explain only the coverage gap. +`no-knowledge` means no applicable article answers the question; `answer` +may explain the gap but `references` stays empty. `partial` records the reason +and cites only articles actually read; `failed` records the reason with no +references. Never fabricate an article path, infer a finding from article +applicability, or report advice without an applicable article under the +BCQuality name. A consumer MUST validate the schema and confirm every cited +path exists in its live corpus and was read in full before accepting a result. +If citation integrity cannot be verified, return `failed` with no citations. + +For deterministic consumer checks, see +[knowledge-response validation](../docs/knowledge-response-validation.md). +The helper checks schema, exact question, and complete-read evidence against +the live corpus. It does not prove comprehension, source support for every +claim, or applicability; those checks still follow READ and this contract. + + +In an inline host, the answering agent checks its own successful full-body tool +reads. This is self-attestation, not independent citation verification. A +consumer with a trusted read collector can independently check the evidence +using the optional validator. Neither mode proves that every claim correctly +interprets its sources. Report the mode used; never present inline checks as +independent verification. + ## Composition (super-skills) A **super-skill** is an action skill whose frontmatter declares a non-empty `sub-skills: [...]`. A super-skill does not evaluate knowledge files directly; it invokes other action skills and composes their output. @@ -520,4 +568,9 @@ Conforms to the DO output contract. ## How orchestrators consume output -An orchestrator invokes an action skill with an input appropriate to the skill's declared `inputs`, receives the JSON output, and maps findings to its delivery surface (PR comments, build gates, IDE diagnostics). The orchestrator MUST NOT interpret skill-specific fields beyond the schema above. Skills that need richer semantics MUST encode them within the schema (for example, by adding structured `message` text) rather than extending the output shape. +An orchestrator invokes an action skill with an input appropriate to its declared +`inputs` and validates the JSON against its declared output schema. It can map +review findings to PR comments or diagnostics and display a knowledge response +as cited guidance. It MUST NOT treat guidance as findings or apply a review gate +to a knowledge response. Skills needing richer semantics must change a shared +contract deliberately instead of adding private fields to an output. diff --git a/skills/entry.md b/skills/entry.md index efa84a15..2df4674e 100644 --- a/skills/entry.md +++ b/skills/entry.md @@ -24,6 +24,7 @@ task-context: - pr-diff - file-path - folder-path + - knowledge-query # the caller's exact question without code scope technologies: [al] bc-version: 28 countries: [w1] @@ -175,7 +176,7 @@ Populated example (PR review on a repo where only `al-performance-review` is ena 1. Invoke Entry with the orchestrator-supplied task context. 2. Receive the dispatch record. -3. For each entry in `dispatch[]`, read the referenced action skill, execute its Source → Relevance → Worklist → Action steps per DO, and produce a findings-report. -4. Return the findings-reports to the orchestrator. When `outcome` is `no-match` or `failed`, return the dispatch record itself so the orchestrator can log the reason. +3. For each entry in `dispatch[]`, read the referenced action skill, execute its Source → Relevance → Worklist → Action steps per DO, and produce its declared output kind (`findings-report` or `knowledge-response`). A knowledge-only request with only `knowledge-query` cannot dispatch a review skill because its input types do not intersect. +4. Return each action skill's output unchanged to the orchestrator. When `outcome` is `no-match` or `failed`, return the dispatch record itself so the orchestrator can log the reason. READ and DO are the contracts that govern what the dispatched skills do. An agent that has not yet read READ and DO reads them when it executes the first dispatched skill — they are not prerequisites for invoking Entry. diff --git a/skills/read.md b/skills/read.md index dddd7dbb..dc5d03db 100644 --- a/skills/read.md +++ b/skills/read.md @@ -185,7 +185,20 @@ returned `body`, then request `remainingPaths` with `continuation.snapshot` as `-Snapshot` until `complete` is `true`, preserving the other request settings. Continuation is confined to that chunk. Bodies are original strict UTF-8 text with source byte counts and SHA-256 content hashes; -they are never summarized or truncated. Samples are not loaded unless +they are never summarized or truncated. Never clip helper output with +`Substring`, `Select-Object -First`, or a file-reader display limit. Parse one +bounded JSON page and expose its article bodies as multiline text for reading: + +```powershell +$page = & (Join-Path $root 'tools\Get-KnowledgeArticles.ps1') -Paths @($exactPath) | ConvertFrom-Json +$page.articles | ForEach-Object { $_.path; $_.body } +``` + +Inspect `complete`, `remainingPaths`, and `continuation.snapshot` on that page +before requesting the next page as described above. Printing a body is not +proof that the host received it through EOF: if the host truncates the display, +continue with bounded native reads through EOF or use a smaller batch. Never +cite an article whose complete body could not be read. Samples are not loaded unless requested explicitly with `-Samples` and exact sibling paths; their sibling article must match its prepared hash and contain the exact READ link. diff --git a/tools/Build-SkillIndex.ps1 b/tools/Build-SkillIndex.ps1 index d4028c22..47ac729e 100644 --- a/tools/Build-SkillIndex.ps1 +++ b/tools/Build-SkillIndex.ps1 @@ -186,10 +186,10 @@ foreach ($layer in 'microsoft', 'community', 'custom') { inputs = [string[]]@($frontmatter['inputs']) outputs = [string[]]@($frontmatter['outputs']) filters = [ordered]@{ - 'bc-version' = [object[]]$(if ($frontmatter.Contains('bc-version')) { $frontmatter['bc-version'] }) - technologies = [string[]]$(if ($frontmatter.Contains('technologies')) { $frontmatter['technologies'] }) - countries = [string[]]$(if ($frontmatter.Contains('countries')) { $frontmatter['countries'] }) - 'application-area' = [string[]]$(if ($frontmatter.Contains('application-area')) { $frontmatter['application-area'] }) + 'bc-version' = [object[]]@(if ($frontmatter.Contains('bc-version')) { $frontmatter['bc-version'] }) + technologies = [string[]]@(if ($frontmatter.Contains('technologies')) { $frontmatter['technologies'] }) + countries = [string[]]@(if ($frontmatter.Contains('countries')) { $frontmatter['countries'] }) + 'application-area' = [string[]]@(if ($frontmatter.Contains('application-area')) { $frontmatter['application-area'] }) } subSkills = [string[]]$subSkills sourceSha256 = $sourceSha256 diff --git a/tools/validate_knowledge_response.py b/tools/validate_knowledge_response.py new file mode 100644 index 00000000..50122459 --- /dev/null +++ b/tools/validate_knowledge_response.py @@ -0,0 +1,128 @@ +"""Validate a knowledge response against its question and trusted read evidence. + +Read evidence must come from the consumer's successful tool results in this run, +not from the answering agent's claims. This checks identity and complete-read +evidence; it does not prove comprehension or that every claim follows a source. +Requires jsonschema. Does not repair or rewrite the response. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path, PurePosixPath + +from jsonschema import Draft7Validator + + +def validate_response(response, *, question, root, evidence=None, run_id, enabled_layers=("microsoft", "community", "custom")): + root = Path(root).resolve() + schema = json.loads((root / "schemas/knowledge-response.schema.json").read_text(encoding="utf-8-sig")) + validator = Draft7Validator(schema) + errors = [f"Schema: {error.message}" for error in validator.iter_errors(response)] + if errors: + return errors + if response["question"] != question: + errors.append("question must equal the bound input exactly, including whitespace") + known_layers = {"microsoft", "community", "custom"} + enabled_layers = set(enabled_layers) + if not enabled_layers <= known_layers: + errors.append("Unknown enabled layers") + for entry in response["references"] + response["suppressed"]: + name = entry["path"] + parts = name.split("/") + file = (root / name).resolve() + if "\\" in name or any(part in ("", ".", "..") for part in parts) or not file.is_relative_to(root): + errors.append(f"Noncanonical corpus path: {name}") + elif not file.is_file(): + errors.append(f"Corpus path does not exist: {name}") + if parts[0] not in enabled_layers: + # Configuration exclusions may identify disabled layers, never citations. + if entry in response["references"] or entry.get("reason") != "configuration": + errors.append(f"Path belongs to a disabled layer: {name}") + reads = {} + if evidence is not None: + if not isinstance(evidence, dict): + errors.append("Read evidence must be an object") + else: + if evidence.get("runId") != run_id: + errors.append("Read evidence belongs to a different run") + expected_hash = hashlib.sha256(question.encode("utf-8")).hexdigest() + if evidence.get("questionSha256") != expected_hash: + errors.append("Read evidence belongs to a different question") + records = evidence.get("reads") + if not isinstance(records, list): + errors.append("Read evidence must contain a reads array") + else: + for record in records: + if not isinstance(record, dict) or not isinstance(record.get("path"), str): + errors.append("Invalid read evidence record") + continue + if record["path"] in reads: + errors.append(f"Duplicate read evidence: {record['path']}") + reads[record["path"]] = record + for reference in response["references"]: + name = reference["path"] + parts = name.split("/") + if "\\" in name or any(part in ("", ".", "..") for part in parts) or PurePosixPath(name).is_absolute(): + errors.append(f"Noncanonical citation path: {name}") + continue + file = (root / name).resolve() + if not file.is_relative_to(root) or not file.is_file(): + errors.append(f"Citation is missing or outside the corpus: {name}") + continue + record = reads.get(name) + if record is None or record.get("complete") is not True: + errors.append(f"No complete read evidence for citation: {name}") + continue + content = file.read_bytes() + try: + content.decode("utf-8") + except UnicodeDecodeError: + errors.append(f"Citation is not strict UTF-8: {name}") + continue + if type(record.get("bytes")) is not int or record["bytes"] != len(content): + errors.append(f"Read byte count does not match the full article: {name}") + if record.get("sha256") != hashlib.sha256(content).hexdigest(): + errors.append(f"Read content hash does not match the live article: {name}") + return errors + + +def strict_json(file): + def pairs(items): + result = {} + for key, value in items: + if key in result: + raise ValueError(f"Duplicate JSON key: {key}") + result[key] = value + return result + + def invalid_constant(value): + raise ValueError(f"Invalid JSON constant: {value}") + + return json.loads(Path(file).read_text(encoding="utf-8-sig"), object_pairs_hook=pairs, parse_constant=invalid_constant) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("response", type=Path) + parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[1]) + parser.add_argument("--question-file", type=Path, required=True, help="Exact bound input as UTF-8; no whitespace trimming") + parser.add_argument("--read-evidence", type=Path, help="Consumer-collected evidence; required for cited responses") + parser.add_argument("--run-id", required=True, help="Current run identifier supplied independently by the consumer") + parser.add_argument("--enabled-layers", nargs="*", choices=("microsoft", "community", "custom"), default=["microsoft", "community", "custom"], help="Consumer configuration; an empty list disables every layer") + args = parser.parse_args() + try: + errors = validate_response(strict_json(args.response), + question=args.question_file.read_bytes().decode("utf-8-sig"), root=args.root, + evidence=strict_json(args.read_evidence) if args.read_evidence else None, + run_id=args.run_id, enabled_layers=args.enabled_layers) + except (OSError, ValueError) as error: + print(json.dumps({"valid": False, "errors": [str(error)]})) + return 2 + print(json.dumps({"valid": not errors, "errors": errors})) + return int(bool(errors)) + + +if __name__ == "__main__": + raise SystemExit(main())