Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .claude-plugin/marketplace.json
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
{
"name": "bcquality",
"source": "./",
"description": "Business Central AL quality knowledge base and review skills, packaged as an installable plugin. Exposes an AL review adapter while preserving BCQuality's internal Entry and action-skill protocols.",
"description": "Business Central quality knowledge and skills for AL code review and cited knowledge consultation.",
"version": "0.2.0",
"skills": [
"./skills/"
Expand Down
24 changes: 24 additions & 0 deletions .github/scripts/Test-SkillIndex.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,30 @@ try {
'microsoft/skills/review/al-finance-review.md'
)
$review = @($skills | Where-Object id -eq 'al-code-review')
$knowledge = @($skills | Where-Object id -eq 'al-knowledge')
if ($knowledge.Count -ne 1 -or (@($knowledge[0].inputs) -join ',') -ne 'knowledge-query' -or
(@($knowledge[0].outputs) -join ',') -ne 'knowledge-response') {
throw 'Expected one knowledge-only al-knowledge action.'
}
$knowledgeCandidates = @($skills | Where-Object { 'knowledge-query' -in $_.inputs })
if ($knowledgeCandidates.Count -ne 1 -or $knowledgeCandidates[0].id -ne 'al-knowledge') {
throw 'A knowledge-only request must exclude review actions.'
}
# Entry applies READ's non-empty technology intersection to indexed actions.
$alKnowledge = @($knowledgeCandidates | Where-Object { 'al' -in $_.filters.technologies })
if ($alKnowledge.Count -ne 1 -or $alKnowledge[0].id -ne 'al-knowledge') {
throw 'Entry must admit knowledge-query with technologies: [al].'
}
foreach ($dimension in 'bc-version', 'countries', 'application-area') {
$expected = if ($dimension -eq 'countries') { 'w1' } else { 'all' }
if ($expected -notin $knowledge[0].filters.$dimension) {
throw "Missing knowledge action applicability: $dimension"
}
}
$reviewCandidates = @($skills | Where-Object { 'file-path' -in $_.inputs -and 'al' -in $_.filters.technologies })
if ('al-code-review' -notin $reviewCandidates.id -or 'al-knowledge' -in $reviewCandidates.id) {
throw 'AL file input must admit code review and exclude knowledge consultation.'
}
if ($review.Count -ne 1) {
throw "Expected exactly one al-code-review record, found $($review.Count)."
}
Expand Down
57 changes: 57 additions & 0 deletions .github/scripts/test_knowledge_response.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
"""Validate knowledge-response examples and reject unsafe citation shapes."""
import copy
import json
from pathlib import Path

from jsonschema import Draft7Validator

ROOT = Path(__file__).resolve().parents[2]
schema = json.loads((ROOT / 'schemas/knowledge-response.schema.json').read_text())
Draft7Validator.check_schema(schema)
validator = Draft7Validator(schema)
base = {
'skill': {'id': 'al-knowledge', 'version': 1},
'outcome': 'completed', 'question': 'When is SetLoadFields useful?',
'answer': 'Use it for partial records under the cited conditions.',
'references': [{
'path': 'microsoft/knowledge/performance/use-setloadfields-for-partial-records.md',
'applicability': 'conditional', 'unknown': ['bc-version'],
}], 'suppressed': [],
}
assert (ROOT / base['references'][0]['path']).is_file()
cases = 0


def check(value, valid):
global cases
errors = list(validator.iter_errors(value))
assert bool(errors) != valid, [error.message for error in errors]
cases += 1


check(base, True)
value = copy.deepcopy(base)
value['references'][0]['unknown'] = []
check(value, False)
value['references'][0]['applicability'] = 'applicable'
check(value, True)
value['references'][0]['unknown'] = ['bc-version']
check(value, False)
for outcome in ('failed', 'no-knowledge'):
value = copy.deepcopy(base)
value.update(outcome=outcome, **{'outcome-reason': 'No usable knowledge'})
check(value, False)
value['references'] = []
check(value, True)
value = copy.deepcopy(base)
value['references'] = []
check(value, False)
value = copy.deepcopy(base)
value['outcome'] = 'partial'
check(value, False)
value['outcome-reason'] = 'Target version is unknown'
check(value, True)
value = copy.deepcopy(base)
value['references'][0]['path'] = 'invented/article.md'
check(value, False)
print(f'Knowledge-response schema: {cases} positive/negative cases passed.')
172 changes: 172 additions & 0 deletions .github/scripts/test_knowledge_response_semantics.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,172 @@
"""Regress input fidelity and complete citation reads using consumer evidence."""
import copy
import hashlib
import importlib.util
import json
from pathlib import Path
import subprocess
import sys
import tempfile
import unittest

REPO = Path(__file__).resolve().parents[2]
TOOL = REPO / "tools/validate_knowledge_response.py"
spec = importlib.util.spec_from_file_location("knowledge_validator", TOOL)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)


class KnowledgeSemantics(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.addCleanup(self.temp.cleanup)
self.root = Path(self.temp.name)
(self.root / "schemas").mkdir()
(self.root / "schemas/knowledge-response.schema.json").write_bytes((REPO / "schemas/knowledge-response.schema.json").read_bytes())
self.path = "microsoft/knowledge/performance/example.md"
self.article = self.root / self.path
self.article.parent.mkdir(parents=True)
self.article.write_bytes(b"# Example\n\nComplete article content.\n")
self.question = " Question with accents: caf\u00e9?\r\n "
self.response = {"skill": {"id": "al-knowledge", "version": 1},
"outcome": "completed", "question": self.question, "answer": "Supported answer.",
"references": [{"path": self.path, "applicability": "applicable", "unknown": []}], "suppressed": []}
content = self.article.read_bytes()
self.evidence = {"runId": "test-run", "questionSha256": hashlib.sha256(self.question.encode()).hexdigest(),
"reads": [{"path": self.path, "complete": True, "bytes": len(content), "sha256": hashlib.sha256(content).hexdigest()}]}

def errors(self, evidence=True):
return module.validate_response(self.response, question=self.question, root=self.root,
evidence=self.evidence if evidence else None, run_id="test-run")

def test_disabled_citation_layer(self):
errors = module.validate_response(self.response, question=self.question, root=self.root,
evidence=self.evidence, run_id="test-run", enabled_layers=["community"])
self.assertTrue(any("disabled layer" in error for error in errors))

def test_configuration_suppression_disabled_layer(self):
self.response.update(outcome="no-knowledge", references=[], suppressed=[{"path": self.path, "reason": "configuration"}])
self.assertEqual(module.validate_response(self.response, question=self.question, root=self.root,
run_id="test-run", enabled_layers=[]), [])

def test_precedence_suppression_disabled_layer(self):
self.response.update(outcome="no-knowledge", references=[], suppressed=[{"path": self.path, "reason": "layer-precedence"}])
self.assertTrue(module.validate_response(self.response, question=self.question, root=self.root,
run_id="test-run", enabled_layers=[]))

def test_suppressed_path_traversal(self):
self.response["suppressed"] = [{"path": "microsoft/knowledge/../skills/x.md", "reason": "configuration"}]
self.assertTrue(self.errors())

def test_missing_suppressed_file(self):
self.response["suppressed"] = [{"path": "custom/knowledge/missing.md", "reason": "configuration"}]
self.assertTrue(self.errors())

def test_enabled_layers_cli(self):
response_file = self.root / "response.json"
question_file = self.root / "question.txt"
evidence_file = self.root / "reads.json"
response_file.write_text(json.dumps(self.response), encoding="utf-8")
question_file.write_bytes(self.question.encode("utf-8"))
evidence_file.write_text(json.dumps(self.evidence), encoding="utf-8")
result = subprocess.run([sys.executable, str(TOOL), str(response_file), "--root", str(self.root),
"--question-file", str(question_file), "--read-evidence", str(evidence_file),
"--run-id", "test-run", "--enabled-layers", "community"], capture_output=True, text=True)
self.assertEqual(result.returncode, 1)
self.assertIn("disabled layer", result.stdout)

def test_unknown_enabled_layer(self):
self.assertTrue(module.validate_response(self.response, question=self.question, root=self.root,
evidence=self.evidence, run_id="test-run", enabled_layers=["unknown"]))

def test_valid_and_unchanged(self):
original = copy.deepcopy(self.response)
self.assertEqual(self.errors(), [])
self.assertEqual(self.response, original)

def test_rewritten_question(self):
self.response["question"] = "A paraphrase"
self.assertTrue(self.errors())

def test_question_whitespace(self):
self.response["question"] = self.question.strip()
self.assertTrue(self.errors())

def test_missing_article(self):
self.response["references"][0]["path"] = "microsoft/knowledge/performance/missing.md"
self.assertTrue(self.errors())

def test_missing_read_evidence(self):
self.assertTrue(self.errors(evidence=False))

def test_truncated_read(self):
self.evidence["reads"][0]["complete"] = False
self.assertTrue(self.errors())

def test_byte_count(self):
self.evidence["reads"][0]["bytes"] -= 1
self.assertTrue(self.errors())

def test_changed_body(self):
self.article.write_bytes(b"Changed body\n")
self.assertTrue(self.errors())

def test_wrong_hash(self):
self.evidence["reads"][0]["sha256"] = "0" * 64
self.assertTrue(self.errors())

def test_same_basename_other_layer(self):
self.evidence["reads"][0]["path"] = self.path.replace("microsoft/", "custom/")
self.assertTrue(self.errors())

def test_wrong_run(self):
self.evidence["runId"] = "previous-run"
self.assertTrue(self.errors())

def test_wrong_question_evidence(self):
self.evidence["questionSha256"] = "0" * 64
self.assertTrue(self.errors())

def test_path_traversal(self):
self.response["references"][0]["path"] = "microsoft/knowledge/../skills/example.md"
self.assertTrue(self.errors())

def test_empty_reference_outcomes(self):
for outcome in ("failed", "no-knowledge"):
self.response.update(outcome=outcome, references=[], **{"outcome-reason": "No usable knowledge"})
self.assertEqual(self.errors(evidence=False), [])

def test_schema_violation(self):
self.response["references"][0].update(applicability="conditional", unknown=[])
self.assertTrue(self.errors())

def test_duplicate_read_evidence(self):
self.evidence["reads"].append(copy.deepcopy(self.evidence["reads"][0]))
self.assertTrue(self.errors())

def test_cli_preserves_question_and_rejects_changes(self):
question = self.root / "question.txt"
question.write_bytes(self.question.encode())
response = self.root / "response.json"
evidence = self.root / "evidence.json"
response.write_text(json.dumps(self.response), encoding="utf-8")
evidence.write_text(json.dumps(self.evidence), encoding="utf-8")
command = [sys.executable, str(TOOL), str(response), "--root", str(self.root),
"--question-file", str(question), "--read-evidence", str(evidence), "--run-id", "test-run"]
result = subprocess.run(command, capture_output=True, text=True)
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
self.assertTrue(json.loads(result.stdout)["valid"])
self.response["question"] = self.question.strip()
response.write_text(json.dumps(self.response), encoding="utf-8")
result = subprocess.run(command, capture_output=True, text=True)
self.assertEqual(result.returncode, 1, result.stdout + result.stderr)

def test_strict_json_duplicate_key(self):
file = self.root / "duplicate.json"
file.write_text('{"question":"one","question":"two"}')
with self.assertRaises(ValueError):
module.strict_json(file)


if __name__ == "__main__":
unittest.main()
6 changes: 3 additions & 3 deletions .github/scripts/validate_frontmatter.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,9 @@
HOST_SKILL_REQUIRED_KEYS = {"name", "description"}

STANDARD_INPUTS = {
"pr-diff", "object-list", "file-path", "folder-path", "repository", "telemetry-query",
"pr-diff", "object-list", "file-path", "folder-path", "repository", "telemetry-query", "knowledge-query",
}
ALLOWED_OUTPUTS = {"findings-report"}
ALLOWED_OUTPUTS = {"findings-report", "knowledge-response"}
VALID_SAMPLE_KINDS = {"good", "bad"}

ACTION_SKILL_SECTIONS = ["Source", "Relevance", "Worklist", "Action", "Output"]
Expand Down Expand Up @@ -342,7 +342,7 @@ def validate_action_skill(path: Path, parsed: Parsed, report: Report) -> None:
else:
bad = [x for x in out if x not in ALLOWED_OUTPUTS]
if bad:
report.error(path, "R18", f"outputs contains non-allowed values {bad}; currently only {sorted(ALLOWED_OUTPUTS)} is defined", 1)
report.error(path, "R18", f"outputs contains non-allowed values {bad}; allowed values are {sorted(ALLOWED_OUTPUTS)}", 1)

# R19 optional filter dimensions, if present
if "bc-version" in fm:
Expand Down
8 changes: 7 additions & 1 deletion .github/workflows/validate-frontmatter.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,13 @@ jobs:
python-version: "3.12"

- name: Install dependencies
run: pip install pyyaml
run: pip install pyyaml jsonschema

- name: Run validator
run: python .github/scripts/validate_frontmatter.py --root .

- name: Validate knowledge response contract
run: python .github/scripts/test_knowledge_response.py

- name: Validate knowledge response semantics
run: python .github/scripts/test_knowledge_response_semantics.py
6 changes: 4 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,9 @@ copilot plugin install microsoft/BCQuality
copilot plugin list
```

The list should include `bcquality`. The plugin currently exposes the
[`al-code-review`](skills/al-code-review/SKILL.md) skill. Installation and skill
The list should include `bcquality`. Its public skill entry points are
[`al-code-review`](skills/al-code-review/SKILL.md) and
[`al-knowledge`](skills/al-knowledge/SKILL.md). Installation and skill
discovery are the general pattern; reviewing an app is one example of using it.

### Example: Review a complete app folder
Expand Down Expand Up @@ -64,6 +65,7 @@ the review can still discover knowledge by reading the folders.
| --- | --- |
| Choose direct reading, a supplied skill, or my own agent | [Ways to use BCQuality](docs/using-bcquality.md#choose-how-to-use-bcquality) |
| Review a file, changes, a branch, or a particular concern | [Using BCQuality](docs/using-bcquality.md) |
| Support design and specification with cited knowledge | [Knowledge consultation](docs/knowledge-consultation.md) |
| Resolve setup problems, incomplete reviews, or incorrect findings | [Troubleshooting and support](docs/troubleshooting.md) |
| Browse the available guidance | [Knowledge by domain](docs/using-bcquality.md#knowledge-by-domain) |
| Configure the plugin or use my organization's rules | [Customizing BCQuality](docs/customizing-bcquality.md) |
Expand Down
2 changes: 2 additions & 0 deletions docs/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ of BCQuality's internal protocol is needed.
| --- | --- |
| Choose direct reading, a supplied skill, or my own agent | [Ways to use BCQuality](using-bcquality.md#choose-how-to-use-bcquality) |
| Review an app, file, changes, or branch | [Using BCQuality](using-bcquality.md) |
| Support design and specification with cited knowledge | [Knowledge consultation](knowledge-consultation.md) |
| Understand a report and its limitations | [Reading your results](using-bcquality.md#reading-your-results) |
| Find a particular rule or example | [Knowledge by domain](using-bcquality.md#knowledge-by-domain) |
| Fix setup problems or report an incorrect finding | [Troubleshooting and support](troubleshooting.md) |
Expand All @@ -27,6 +28,7 @@ prerequisites for using the plugin.
| [How agents consume BCQuality](agent-consumption.md) | Architecture, repository structure, routing, and delivery of findings. |
| [Standalone runner](standalone-runner.md) | Optional model selection, scheduling, retries, and telemetry. |
| [Global skills](../skills/README.md) | Host adapters versus internal protocol files. |
| [Knowledge-response validation](knowledge-response-validation.md) | Optional consumer checks for question fidelity and complete citation reads. |
| [Entry](../skills/entry.md) | Task context and skill dispatch. |
| [READ](../skills/read.md) | Knowledge schema, applicability, and precedence. |
| [DO](../skills/do.md) | Action-skill format and structured output contract. |
Expand Down
Loading
Loading