diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0cd2dfb..a87f3a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -259,3 +259,132 @@ jobs: run: | Write-Host "::error::CheckForUpdates outcomes: v1 updatesAvailable '$($env:UNCHANGED)' (expected false), v2 '$($env:MOVED)' (expected true), update without token '$($env:NOTOKEN)' (expected failure/token)" exit 1 + + # The scan against the real packages on nuget.org (the unit suites use stub packages). Every run is a dry run: no + # token, nothing pushed. nuget.org moves, so the checks are lower bounds and the log prints the numbers. + scan-action: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Prepare the fixtures + id: fixtures + shell: pwsh + run: | + . ./tests/Helpers/RepoFixture.ps1 + $scanOrg = New-FixtureRepo -Name 'scan-org' -Destination (Join-Path $env:RUNNER_TEMP 'scan-org') + $template = Join-Path $env:RUNNER_TEMP 'template-policy' + Copy-FixtureTree -Source template -Destination $template + Edit-FixtureJson -Path (Join-Path $template '.github' 'Rulebook-Settings.json') -Script { $_['quarantine'] = [ordered]@{ stages = @('default', 'ci'); prereleaseStages = @('default', 'ci') } } + # Without the seed titles of the compiler ids the scan must read them from CompilerDiagnosticsResources. + Edit-FixtureJson -Path (Join-Path $template 'catalog' 'diagnostics.json') -Script { foreach ($entry in @($_['diagnostics'] | Where-Object { $_['analyzer'] -eq 'Compiler' })) { $entry.Remove('title') } } + Add-Content -LiteralPath $env:GITHUB_OUTPUT -Value "scanOrg=$scanOrg", "template=$template" + + - name: Scan template/ without a policy (must fail) + id: nopolicy + continue-on-error: true + uses: ./actions/ScanDiagnostics + with: + repositoryRoot: template + dryRun: 'true' + + - name: Require the policy failure + if: steps.nopolicy.outcome != 'failure' || steps.nopolicy.outputs.failure != 'policy' + shell: pwsh + env: + FAILURE: ${{ steps.nopolicy.outputs.failure }} + run: | + Write-Host "::error::ScanDiagnostics did not stop on template/ without a policy (failure output: '$($env:FAILURE)')" + exit 1 + + - name: Dry run on the template with a policy + id: template + uses: ./actions/ScanDiagnostics + with: + repositoryRoot: ${{ steps.fixtures.outputs.template }} + dryRun: 'true' + + - name: Check the template scan + shell: pwsh + env: + CANDIDATE: ${{ steps.template.outputs.candidatePath }} + SCANNED: ${{ steps.template.outputs.scannedVersions }} + NEW_IDS: ${{ steps.template.outputs.newIds }} + FAILURE: ${{ steps.template.outputs.failure }} + run: | + Import-Module ./modules/Rulebook.Catalog.psd1, ./modules/Rulebook.Validate.psd1 + $failures = [System.Collections.Generic.List[string]]::new() + if ($env:FAILURE) { $failures.Add("failure '$($env:FAILURE)'") } + foreach ($package in 'microsoft.dynamics.businesscentral.development.tools@', 'alcops.analyzers@') { + if (-not ($env:SCANNED.Split(',') | Where-Object { $_.StartsWith($package) -and $_.EndsWith(':stable') })) { $failures.Add("scannedVersions '$($env:SCANNED)' has no stable $package") } + } + $state = Join-Path $env:CANDIDATE 'catalog' 'scan-state.json' + if (-not (Test-Json -Path $state -SchemaFile schemas/rulebook-scan-state.schema.json -ErrorAction SilentlyContinue)) { $failures.Add('catalog/scan-state.json does not match its schema') } + $catalog = Read-CatalogFile -Path (Join-Path $env:CANDIDATE 'catalog' 'diagnostics.json') + $withPackage = @($catalog.Entries.Values | Where-Object Package) + if ($withPackage.Count -lt 600) { $failures.Add("only $($withPackage.Count) catalog entries have a package") } + $untitled = @($catalog.Entries.Values | Where-Object { $_.Analyzer -eq 'Compiler' -and $_.Package -and [string]::IsNullOrWhiteSpace($_.Title) } | ForEach-Object Id) + if ($untitled.Count -gt 0) { $failures.Add("$($untitled.Count) compiler ids have no title after the scan (the CompilerDiagnosticsResources path): $(@($untitled | Select-Object -First 10) -join ', ')") } + $errors = @(Test-Rulebook -RepositoryRoot $env:CANDIDATE | Where-Object Severity -EQ 'error') + foreach ($finding in $errors) { $failures.Add("$($finding.Rule) $($finding.File) $($finding.Id): $($finding.Message)") } + $unseen = @($catalog.Entries.Values | Where-Object { -not $_.Package } | ForEach-Object Id) + $hidden = @($catalog.Entries.Values | Where-Object { -not $_.Advertised } | ForEach-Object Id) + $seed = Read-CatalogFile -Path template/catalog/diagnostics.json + $titles = @($catalog.Entries.Values | Where-Object { $seed.Entries.Contains($_.Id) -and $seed.Entries[$_.Id].Title -cne $_.Title }).Count + $docs = @($catalog.Entries.Values | Where-Object { $seed.Entries.Contains($_.Id) -and $seed.Entries[$_.Id].Docs -cne $_.Docs } | ForEach-Object Id) + Write-Host "Scanned: $($env:SCANNED)" + Write-Host "New ids: $($env:NEW_IDS); entries with a package: $($withPackage.Count) of $($catalog.Entries.Count)" + Write-Host "Refreshed from the seed: $titles titles, $($docs.Count) docs links ($($docs -join ', '))" + Write-Host "Not advertised: $($hidden -join ', ')" + Write-Host "Without a package: $($unseen -join ', ')" + foreach ($failure in $failures) { Write-Host "::error::$failure" } + if ($failures.Count -gt 0) { exit 1 } + + - name: Dry run on scan-org (30 catalog ids) + id: org + uses: ./actions/ScanDiagnostics + with: + repositoryRoot: ${{ steps.fixtures.outputs.scanOrg }} + dryRun: 'true' + + - name: Check the scan-org scan + shell: pwsh + env: + CANDIDATE: ${{ steps.org.outputs.candidatePath }} + ORG: ${{ steps.fixtures.outputs.scanOrg }} + NEW_IDS: ${{ steps.org.outputs.newIds }} + FAILURE: ${{ steps.org.outputs.failure }} + run: | + Import-Module ./modules/Rulebook.Quarantine.psd1 + $failures = [System.Collections.Generic.List[string]]::new() + if ($env:FAILURE) { $failures.Add("failure '$($env:FAILURE)'") } + if ([int]$env:NEW_IDS -lt 500) { $failures.Add("only $($env:NEW_IDS) new ids") } + foreach ($stage in 'default', 'ci') { + $count = (Read-QuarantineFile -Path (Join-Path $env:CANDIDATE "quarantine.$stage.json")).Rules.Count + Write-Host "quarantine.$stage.json: $count rules" + if ($count -lt 500) { $failures.Add("quarantine.$stage.json has $count rules") } + } + if ((Get-FileHash (Join-Path $env:CANDIDATE 'quarantine.vnext.json')).Hash -ne (Get-FileHash (Join-Path $env:ORG 'quarantine.vnext.json')).Hash) { $failures.Add('quarantine.vnext.json changed') } + foreach ($failure in $failures) { Write-Host "::error::$failure" } + if ($failures.Count -gt 0) { exit 1 } + + - name: Dry run on the scanned template (nothing new) + id: again + uses: ./actions/ScanDiagnostics + with: + repositoryRoot: ${{ steps.template.outputs.candidatePath }} + dryRun: 'true' + + - name: Require nothing-new within a minute + if: steps.again.outputs.result != 'nothing-new' || fromJSON(steps.again.outputs.elapsedSeconds || '999') >= 60 + shell: pwsh + env: + RESULT: ${{ steps.again.outputs.result }} + ELAPSED: ${{ steps.again.outputs.elapsedSeconds }} + run: | + Write-Host "::error::The second scan gave '$($env:RESULT)' in $($env:ELAPSED) s (expected nothing-new in under 60 s)" + exit 1 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a699850..3e3ac2d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -22,26 +22,28 @@ What exists today, and the work package that adds the rest. A folder is created | Path | Content | Added by | |---|---|---| -| `.github/workflows/ci.yml` | PSScriptAnalyzer, the matrix checks V1 to V14 and Pester on every pull request and every push to `main` and to a release branch (`v*`); the Validate action on two fixtures and on `template/`; the Publish action on `template/` without deploying, and the init script against the staged site served over HTTP; the CheckForUpdates action on `tests/fixtures/repos/update-org` against the local templates. | WP00, WP03, WP04, WP05, WP06, WP07 | +| `.github/workflows/ci.yml` | PSScriptAnalyzer, the matrix checks V1 to V14 and Pester on every pull request and every push to `main` and to a release branch (`v*`); the Validate action on two fixtures and on `template/`; the Publish action on `template/` without deploying, and the init script against the staged site served over HTTP; the CheckForUpdates action on `tests/fixtures/repos/update-org` against the local templates; the ScanDiagnostics action in dry runs against nuget.org. | WP00, WP03, WP04, WP05, WP06, WP07, WP08 | | `.github/workflows/` (deploy) | Copies `template/` into `ALCops/rulebook` and pins action references to `@v1`. | WP13 ([#15](https://github.com/ALCops/rulebook-engine/issues/15)) | | `.github/release.yml` | Maps pull request labels to release-note sections ([D38](docs/adr/0038-release-notes-are-generated-from-pull-request-labels.md)). | WP00 | | `.github/dependabot.yml` | Weekly, grouped updates of the GitHub Actions used by the workflows. | WP00 | | `.github/ISSUE_TEMPLATE/` | Issue forms: work package, task or spin-off. | written | | `PSScriptAnalyzerSettings.psd1` | Analyzer settings: errors and warnings, default rules, justified exclusions only. | WP00 | | `schemas/` | JSON schemas for every file in an organization rulebook repository, draft 2020-12, no `$id`; names and URLs in [docs/reference/naming.md](docs/reference/naming.md). | WP02 ([#4](https://github.com/ALCops/rulebook-engine/issues/4)) | -| `tests/` | `Smoke.Tests.ps1`, `Schemas.Tests.ps1`, `Rulebook.Generate.Tests.ps1`, `Rulebook.Validate.Tests.ps1`, `Validate.Action.Tests.ps1`, `Rulebook.Template.Tests.ps1`, `Rulebook.Publish.Tests.ps1`, `Publish.Action.Tests.ps1`, `Get-RulebookSkeletons.Tests.ps1`, `Rulebook.GitHub.Tests.ps1`, `Rulebook.Update.Tests.ps1` and `CheckForUpdates.Action.Tests.ps1` now; later one suite per module and action, with fixtures under `tests/fixtures/`. | WP00, WP02, WP03, WP04, WP05, WP06, WP07; WP12 ([#14](https://github.com/ALCops/rulebook-engine/issues/14)) and every module work package | +| `tests/` | `Smoke.Tests.ps1`, `Schemas.Tests.ps1`, `Rulebook.Generate.Tests.ps1`, `Rulebook.Validate.Tests.ps1`, `Validate.Action.Tests.ps1`, `Rulebook.Template.Tests.ps1`, `Rulebook.Publish.Tests.ps1`, `Publish.Action.Tests.ps1`, `Get-RulebookSkeletons.Tests.ps1`, `Rulebook.GitHub.Tests.ps1`, `Rulebook.Update.Tests.ps1`, `CheckForUpdates.Action.Tests.ps1`, `Rulebook.NuGet.Tests.ps1`, `Rulebook.Extract.Tests.ps1`, `Rulebook.Catalog.Tests.ps1`, `Rulebook.Quarantine.Tests.ps1`, `Rulebook.Scan.Tests.ps1` and `ScanDiagnostics.Action.Tests.ps1` now; later one suite per module and action, with fixtures under `tests/fixtures/`. | WP00, WP02, WP03, WP04, WP05, WP06, WP07, WP08; WP12 ([#14](https://github.com/ALCops/rulebook-engine/issues/14)) and every module work package | | `tests/fixtures/schemas/` | One file per case: `//.json`, each invalid file a one-change mutation of a valid one. | WP02 | | `tests/fixtures/repos/` | Organization rulebook repositories for the module suites. `valid-minimal` (30-id catalog, four levels, three stages), `stale-endpoints` and `update-org` (WP07) are complete on disk, their `rulesets/` written by `Update-RulebookEndpoints`; every other folder is an overlay holding only the files it changes, copied over `valid-minimal` by `New-FixtureRepo`. | WP03 | | `tests/fixtures/templates/` | `v1` and `v2`: two versions of a mini template for the update suites, with the exact difference list and the derivation of `repos/update-org` in its README. | WP07 | +| `tests/fixtures/stub-analyzers/` | C# stubs of the compiler and the twelve cop assemblies and `Build-StubPackage.ps1`, which compiles them with the Roslyn of pwsh into fixture nupkgs (variants tools 18.0.43.1464 and 30.0.42.60748-beta, ALCops 1.3.1, 1.4.0-beta.1 and 1.4.0) for the scan suites; `tests/Helpers/StubFeed.ps1` builds them once per source hash. | WP08 | | `tests/fixtures/matrix/` | `tiny`: a six-id level content in the layout of `docs/rulebook/` (inventory, matrix, resolved cells, levels, stages, twins) for the unit cases of the Template suite. | WP04 | -| `tests/Helpers/` | Helpers the suites dot-source in `BeforeAll`: `RepoFixture.ps1` copies a fixture into `TestDrive`, edits its JSON, creates git repositories, bare origins with a push-refusing hook and the synthetic performance rulebook. | WP03 | -| `modules/` | PowerShell modules shared by the actions, each a `.psm1` with a `.psd1` manifest: `Rulebook.Generate`, `Rulebook.Validate`, `Rulebook.Template`, `Rulebook.Publish`, `Rulebook.GitHub` and `Rulebook.Update` (written). | WP03 to WP09 | +| `tests/Helpers/` | Helpers the suites dot-source in `BeforeAll`: `RepoFixture.ps1` copies a fixture into `TestDrive`, edits its JSON, creates git repositories, bare origins with a push-refusing hook and the synthetic performance rulebook; `StubFeed.ps1` builds NuGet flat containers of the stub analyzer packages. | WP03, WP08 | +| `modules/` | PowerShell modules shared by the actions, each a `.psm1` with a `.psd1` manifest: `Rulebook.Generate`, `Rulebook.Validate`, `Rulebook.Template`, `Rulebook.Publish`, `Rulebook.GitHub`, `Rulebook.Update`, `Rulebook.NuGet`, `Rulebook.Extract`, `Rulebook.Catalog`, `Rulebook.Quarantine` and `Rulebook.Scan` (written). | WP03 to WP09 | | `actions/` | Composite actions, one folder each with `action.yaml` and its entry script. | WP03 to WP09 | | `actions/Validate/` | `action.yaml` and `Validate.ps1`: checks C1 to C16, annotations, the job summary with the effective diff ([ARCHITECTURE.md](docs/ARCHITECTURE.md) section 5.5). | WP03 | | `actions/Publish/` | `action.yaml` and `Publish.ps1`: stage the endpoints, the rendered skeletons and `index.html`, the Pages preflight, deploy to GitHub Pages, the reachability check ([ARCHITECTURE.md](docs/ARCHITECTURE.md) section 7.2, [docs/reference/publish-targets.md](docs/reference/publish-targets.md)). | WP05 | | `actions/CheckForUpdates/` | `action.yaml` and `CheckForUpdates.ps1`: the template update check (`update` other than `'Y'`) and the update pull request or direct commit (`update: 'Y'`), with the token from `GHTOKENWORKFLOW` ([docs/reference/update-mechanics.md](docs/reference/update-mechanics.md)). | WP07 | +| `actions/ScanDiagnostics/` | `action.yaml` and `ScanDiagnostics.ps1`: the diagnostic scan, its living pull request on `scan-diagnostics/` or a direct commit, and the dry run ([docs/reference/scan-mechanics.md](docs/reference/scan-mechanics.md)). | WP08 | | `scripts/` | User-facing scripts an AL project downloads and runs, self-contained (no engine module, PowerShell 7): `Get-RulebookSkeletons.ps1` downloads the published skeletons of a level into `.rulebook/`, one per stage ([ARCHITECTURE.md](docs/ARCHITECTURE.md) section 6.3). | WP06 ([#8](https://github.com/ALCops/rulebook-engine/issues/8)) | -| `template/` | Source of the template content deployed to `ALCops/rulebook`, 42 files: 10 hand-written files (settings, `Validate.yaml`, `Publish.yaml`, `UpdateRulebookSystemFiles.yaml`, `README.md`, `skeletons/README.md`, `overrides.json`, three quarantine files) and 32 files generated by `tools/rulebook/Build-Template.ps1` (`base/`, `stages/`, `catalog/`, `skeletons/`, `rulesets/`), see [docs/reference/template-content.md](docs/reference/template-content.md). | WP03, WP04, WP05, WP06, WP13 | +| `template/` | Source of the template content deployed to `ALCops/rulebook`, 43 files: 11 hand-written files (settings, `Validate.yaml`, `Publish.yaml`, `UpdateRulebookSystemFiles.yaml`, `ScanDiagnostics.yaml`, `README.md`, `skeletons/README.md`, `overrides.json`, three quarantine files) and 32 files generated by `tools/rulebook/Build-Template.ps1` (`base/`, `stages/`, `catalog/`, `skeletons/`, `rulesets/`), see [docs/reference/template-content.md](docs/reference/template-content.md). | WP03, WP04, WP05, WP06, WP07, WP08, WP13 | | `docs/`, `tools/rulebook/` | Architecture, decision records, level content and the scripts that build it and the template content (`Build-Template.ps1`, WP04). | written | ## 2. Conventions @@ -72,7 +74,7 @@ After any change to `docs/rulebook/`, `modules/Rulebook.Template.psm1` or `templ ## 4. CI -`.github/workflows/ci.yml` has four jobs on `ubuntu-latest` with a 15-minute timeout each, `test`, `validate-action`, `publish-action` and `update-action`. All four run on every pull request and on every push to `main` or a release branch (`v*`). `test` installs PSScriptAnalyzer and Pester 6, runs the analyzer over the whole repository and fails on any finding (after printing the findings table), runs the matrix checks V1 to V14 (`tools/rulebook/Test-Rulebook.ps1`, which exits 1 on a failed check), runs Pester from `tests/`, and uploads `testResults.xml` (NUnit) as the `testResults` artifact, also when a step failed. The workflow token is read-only, and a new push to a pull request cancels its running job; pushes to `main` and release branches always finish. +`.github/workflows/ci.yml` has five jobs on `ubuntu-latest`, `test`, `validate-action`, `publish-action` and `update-action` with a 15-minute timeout and `scan-action` with 20 minutes. All five run on every pull request and on every push to `main` or a release branch (`v*`). `test` installs PSScriptAnalyzer and Pester 6, runs the analyzer over the whole repository and fails on any finding (after printing the findings table), runs the matrix checks V1 to V14 (`tools/rulebook/Test-Rulebook.ps1`, which exits 1 on a failed check), runs Pester from `tests/`, and uploads `testResults.xml` (NUnit) as the `testResults` artifact, also when a step failed. The workflow token is read-only, and a new push to a pull request cancels its running job; pushes to `main` and release branches always finish. `validate-action` runs the composite action from the checkout (`uses: ./actions/Validate`) the way an organization workflow does: it must pass on `tests/fixtures/repos/valid-minimal`, pass on `template/` with `failOnWarning` (no warning either), and fail on `tests/fixtures/repos/stale-endpoints`, and the job fails otherwise. Its three steps pass `checkForUpdates: 'false'`, so the fixtures are not compared with `ALCops/rulebook`. On a pull request its `template/` step shows the effective diff of the template endpoints against the base branch. Its step on `stale-endpoints` prints one expected C12 error annotation. @@ -80,7 +82,9 @@ After any change to `docs/rulebook/`, `modules/Rulebook.Template.psm1` or `templ `update-action` runs `uses: ./actions/CheckForUpdates` in check mode on `tests/fixtures/repos/update-org` with the local templates of `tests/fixtures/templates/` (no download, no token): against `v1` it must report `updatesAvailable` `false`, against `v2` `true`, and in update mode without a token it must fail with the `failure` output `token`; a guard step fails the job on any other outcome (it prints one expected error annotation, the missing secret). -The ruleset requires all four checks (section 6). +`scan-action` runs `uses: ./actions/ScanDiagnostics` with `dryRun: 'true'` against the real packages on nuget.org (no token, nothing pushed; the unit suites use stub packages): on `template/` without a policy it must fail with the `failure` output `policy` (one expected error annotation); on a copy of `template/` with a policy it must scan both stable versions, give at least 600 catalog entries a package, write a valid `catalog/scan-state.json` and leave a candidate without validation errors, and it prints the new ids, the refreshed titles and docs links and the ids without a package; on `tests/fixtures/repos/scan-org` (30 catalog ids) it must quarantine at least 500 new ids in `default` and `ci` and leave `quarantine.vnext.json` alone; a second dry run on the scanned template must give `nothing-new` in under 60 s. nuget.org moves, so the checks are lower bounds. + +The ruleset requires the checks of section 6. The module versions are pinned in `ci.yml` (PSScriptAnalyzer 1.25.0, Pester 6.2.0) and bumped by hand, because Dependabot does not cover the PowerShell Gallery; it only updates the action tags. @@ -96,7 +100,7 @@ Applied on 2026-10-03 by the WP00 pull request ([#2](https://github.com/ALCops/r | Setting | Value | Command | |---|---|---| -| Ruleset `protect-main` on the default branch | Pull request required, 0 approvals, deletion and force-push blocked, checks `test`, `validate-action`, `publish-action` and `update-action` from GitHub Actions required, no bypass actors. `validate-action` was added on 2026-10-06 with the WP03 pull request ([#46](https://github.com/ALCops/rulebook-engine/pull/46)) through a PUT of the same JSON. `publish-action` was added the same way on 2026-10-06, after the first CI run of the WP05 pull request ([#59](https://github.com/ALCops/rulebook-engine/pull/59)) had reported the check; the PUT kept the other rules as the GET returned them. `update-action` is added the same way after the first CI run of the WP07 pull request ([#9](https://github.com/ALCops/rulebook-engine/issues/9)) has reported the check. | `gh api --method POST ... rulesets --input ruleset-engine.json`, later `gh api --method PUT ... rulesets/24420852 --input ruleset-engine.json` | +| Ruleset `protect-main` on the default branch | Pull request required, 0 approvals, deletion and force-push blocked, checks `test`, `validate-action`, `publish-action`, `update-action` and `scan-action` from GitHub Actions required, no bypass actors. `validate-action` was added on 2026-10-06 with the WP03 pull request ([#46](https://github.com/ALCops/rulebook-engine/pull/46)) through a PUT of the same JSON. `publish-action` was added the same way on 2026-10-06, after the first CI run of the WP05 pull request ([#59](https://github.com/ALCops/rulebook-engine/pull/59)) had reported the check; the PUT kept the other rules as the GET returned them. `update-action` is added the same way after the first CI run of the WP07 pull request ([#9](https://github.com/ALCops/rulebook-engine/issues/9)) has reported the check, and `scan-action` after the first CI run of the WP08 pull request ([#10](https://github.com/ALCops/rulebook-engine/issues/10)). | `gh api --method POST ... rulesets --input ruleset-engine.json`, later `gh api --method PUT ... rulesets/24420852 --input ruleset-engine.json` | | Workflow permissions | Read-only `GITHUB_TOKEN` (applied). "Actions may create and approve pull requests" is not applied yet: the repository-level PUT is refused (409) while the organization policy "Allow GitHub Actions to create and approve pull requests" is off. An org admin enables it under Org Settings > Actions > General; then the PUT below applies. | `gh api --method PUT ... actions/permissions/workflow` | | Labels | `dependencies` and `skip-changelog`, next to the defaults (`enhancement`, `bug`, `documentation`). | `gh label create` | @@ -125,7 +129,8 @@ The ruleset, saved as `ruleset-engine.json` outside the repository (`integration { "context": "test", "integration_id": 15368 }, { "context": "validate-action", "integration_id": 15368 }, { "context": "publish-action", "integration_id": 15368 }, - { "context": "update-action", "integration_id": 15368 } ] } } + { "context": "update-action", "integration_id": 15368 }, + { "context": "scan-action", "integration_id": 15368 } ] } } ] } ``` diff --git a/README.md b/README.md index 1d3a607..10772fe 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ The engine behind [ALCops/rulebook](https://github.com/ALCops/rulebook): the composite GitHub Actions, PowerShell modules, tests and contributor documentation that the workflows in an org rulebook repo call. Users never create a repository from this one; they create it from the template and their workflows reference `ALCops/rulebook-engine/actions/@v1`. -> **Status:** in development. The architecture and the decisions are written, the schemas exist (WP02), and so do the `Rulebook.Generate` and `Rulebook.Validate` modules, the `Validate` action and the template's `Validate` workflow (WP03), and the template content with its `Rulebook.Template` generators (WP04). The work is broken down into [work package issues](https://github.com/ALCops/rulebook-engine/issues?q=is%3Aissue+label%3Aworkpackage) (WP00 to WP15), ordered and tracked on the [Rulebook v1 project board](https://github.com/orgs/ALCops/projects/1); the parent issue [Rulebook v1 (#18)](https://github.com/ALCops/rulebook-engine/issues/18) holds the dependency graph and the suggested order. +> **Status:** in development. The architecture and the decisions are written, the schemas exist (WP02), and so do the generator and the validation (WP03), the template content (WP04), Publish (WP05), the AL project skeletons and the init script (WP06), the update from the template (WP07) and the diagnostic scan (WP08). The work is broken down into [work package issues](https://github.com/ALCops/rulebook-engine/issues?q=is%3Aissue+label%3Aworkpackage) (WP00 to WP15), ordered and tracked on the [Rulebook v1 project board](https://github.com/orgs/ALCops/projects/1); the parent issue [Rulebook v1 (#18)](https://github.com/ALCops/rulebook-engine/issues/18) holds the dependency graph and the suggested order. --- @@ -19,15 +19,15 @@ The engine behind [ALCops/rulebook](https://github.com/ALCops/rulebook): the com | Path | Content | Status | |---|---|---| -| `actions//action.yaml` | Composite actions: `Validate`, `Publish`, `CheckForUpdates`, `ScanDiagnostics`, `ChangeRule`. Each runs a PowerShell 7 script on `ubuntu-latest`. | `Validate` written (WP03); the others planned (WP05 to WP09) | -| `modules/Rulebook.*.psm1` | PowerShell modules shared by the actions, each with a `.psd1` manifest: `Rulebook.Generate` (level chain + stage deltas + twins setting + overrides + quarantine to sparse flat endpoints, effective diff), `Rulebook.Validate` (checks C1 to C16), `Rulebook.Template` (level, stage, twins, catalog and skeleton files of `template/` from `docs/rulebook/`), `Rulebook.Settings`, `Rulebook.NuGet`, `Rulebook.Git`. | `Rulebook.Generate` and `Rulebook.Validate` written (WP03), `Rulebook.Template` written (WP04); the others planned | -| `template/` | Source of the template content that a deploy workflow copies into `ALCops/rulebook`, pinning action references from `@main` to `@v1`. Its `base/`, `stages/` and `rulesets/` are generated from `docs/rulebook/`. | written (WP03, WP04): settings, `Validate.yaml`, README, empty overrides and quarantine, and the 32 generated files; the other workflows come with their work packages, the deploy with WP13 | -| `schemas/` | JSON schemas for every file in an organization rulebook repository (ruleset profiles, overrides, quarantine, twins, catalog, settings), served from the `v1` branch over raw URLs, which go live with WP13 ([#15](https://github.com/ALCops/rulebook-engine/issues/15)) and return 404 until then; the tests use the local files. See [docs/reference/naming.md](docs/reference/naming.md). | written (WP02) | -| `tests/` | Pester 6 suites, one per module and action, with fixtures under `tests/fixtures/`: the schema suite and its fixtures under `tests/fixtures/schemas/`; the Generate, Validate and Validate action suites with organization rulebook fixtures under `tests/fixtures/repos/` and helpers in `tests/Helpers/`. | smoke test (WP00), schema suite (WP02), Generate, Validate and action suites (WP03), Template suite (WP04); further suites per module | +| `actions//action.yaml` | Composite actions, each running a PowerShell 7 entry script on `ubuntu-latest`: `Validate` (checks, effective diff, update check), `Publish` (gate, stage, deploy, verify), `CheckForUpdates` (update from the template), `ScanDiagnostics` (the daily diagnostic scan). | `Validate` (WP03), `Publish` (WP05), `CheckForUpdates` (WP07) and `ScanDiagnostics` (WP08) written; `ChangeRule` planned (WP09) | +| `modules/Rulebook.*.psm1` | PowerShell modules shared by the actions, each with a `.psd1` manifest: `Rulebook.Generate` (level chain + stage deltas + twins setting + overrides + quarantine to sparse flat endpoints, effective diff), `Rulebook.Validate` (checks C1 to C16), `Rulebook.Template` (level, stage, twins, catalog and skeleton files of `template/` from `docs/rulebook/`), `Rulebook.Publish` (staging, Pages deploy, reachability), `Rulebook.Update` and `Rulebook.GitHub` (the update and the GitHub plumbing), and for the scan `Rulebook.NuGet`, `Rulebook.Extract`, `Rulebook.Catalog`, `Rulebook.Quarantine` and `Rulebook.Scan`. | written (WP03 to WP08) | +| `template/` | Source of the template content that a deploy workflow copies into `ALCops/rulebook`, pinning action references from `@main` to `@v1`. Its `base/`, `stages/` and `rulesets/` are generated from `docs/rulebook/`. | written (WP03 to WP08), 43 files: settings, the workflows `Validate.yaml`, `Publish.yaml`, `UpdateRulebookSystemFiles.yaml` and `ScanDiagnostics.yaml`, the README files, empty overrides and quarantine files, and the 32 generated files; the other workflows come with their work packages, the deploy with WP13 | +| `schemas/` | JSON schemas for every file in an organization rulebook repository (ruleset profiles, overrides, quarantine, twins, catalog, scan state, settings), served from the `v1` branch over raw URLs, which go live with WP13 ([#15](https://github.com/ALCops/rulebook-engine/issues/15)) and return 404 until then; the tests use the local files. See [docs/reference/naming.md](docs/reference/naming.md). | written (WP02) | +| `tests/` | Pester 6 suites, one per module and action, with fixtures under `tests/fixtures/`: the schema suite and its fixtures under `tests/fixtures/schemas/`; the Generate, Validate and Validate action suites with organization rulebook fixtures under `tests/fixtures/repos/` and helpers in `tests/Helpers/`. | a suite per module and action (WP00 to WP08); the scan suites build stub analyzer packages from `tests/fixtures/stub-analyzers/` at test time | | `docs/` | Architecture, decision records (`adr/`), references, and `docs/rulebook/` with the level content (inventory, matrix, composition spec). | written | | `tools/rulebook/` | PowerShell scripts that extract the inventory from the analyzer sources, build the matrix (ladders, stage columns, twin pairs, counts) and verify it (`Extract-Inventory.ps1`, `Build-Matrix.ps1`, `Test-Rulebook.ps1`). They import `Rulebook.Generate` for the diagnostic sort key and build portable paths; CI runs `Test-Rulebook.ps1`. `Build-Template.ps1` regenerates `template/` from `docs/rulebook/`. | written | | `scripts/` | User-facing scripts that an AL project downloads from the engine and runs, self-contained (PowerShell 7, no engine module): `Get-RulebookSkeletons.ps1` reads `/rulebook.json` and downloads the published skeletons of one level into `.rulebook/`, one file per stage. Linked from the index page of every published rulebook. | written (WP06) | -| `.github/workflows/` | `ci.yml` (PSScriptAnalyzer, the matrix checks V1 to V14, Pester, and the Validate action on two fixtures and on `template/`) and the deploy workflow. | CI written (WP00); deploy planned (WP13) | +| `.github/workflows/` | `ci.yml`: the job `test` (PSScriptAnalyzer, the matrix checks V1 to V14, Pester) and one job per action, `validate-action`, `publish-action`, `update-action` and `scan-action` (dry runs against nuget.org); the deploy workflow comes with WP13. | CI written (WP00); deploy planned (WP13) | | `CONTRIBUTING.md` | Conventions, running the checks locally, CI, branches, repository settings and pull request rules. | written | ## 2. Relation to the template diff --git a/actions/ScanDiagnostics/ScanDiagnostics.ps1 b/actions/ScanDiagnostics/ScanDiagnostics.ps1 new file mode 100644 index 0000000..6e1d327 --- /dev/null +++ b/actions/ScanDiagnostics/ScanDiagnostics.ps1 @@ -0,0 +1,277 @@ +#requires -Version 7.4 +<# +.SYNOPSIS +Entry script of the ScanDiagnostics action: scan the analyzer packages and publish the result as the living pull +request of an organization rulebook repository (or a direct commit). +.DESCRIPTION +1. Reads the settings and the secret name (ghTokenWorkflowSecretName). +2. Reads the quarantine policy (Get-QuarantinePolicy): no policy is one error annotation with the documented + message, failure policy, before any request. +3. Unless -DryRun: the token guard (The secret is needed to scan diagnostics), the exchange and the mask. +4. Get-RulebookScanPlan: a NuGet, extraction or other failure is one error annotation (failure nuget, extract, + error). +5. Mode nothing-new: one notice, result nothing-new, exit code 0. +6. A candidate that does not validate: one error annotation per finding, failure validation, nothing pushed. +7. -DryRun: the summary and the outputs (candidatePath included; the work folder is kept), result dry-run. +8. Publish-RulebookScan (or -PublishCommand): pull-request, pull-request-updated, direct-commit or no-changes. +9. The job summary (capped at -SummaryLimit) and GITHUB_OUTPUT: result, newIds, quarantined, changedDefaults, + released, scannedVersions (@:, comma separated), pullRequestUrl, candidatePath, + elapsedSeconds, failure (policy, token, nuget, extract, validation, push, pull-request, error; empty on success). +Returns { ExitCode, Result, Failure, Plan, Publish, Annotations, Summary, Outputs } and never calls exit, so tests run +it in-process; action.yaml exits with ExitCode. -PackageSource (a flat container folder), -RemoteUrl, -ApiUrl, +-WorkPath, -SummaryLimit, -Now and -PublishCommand are test seams; a -WorkPath the caller passes is left in place, +the temporary work folder the script names itself is removed at the end except after a dry run. +#> +[CmdletBinding()] +param( + [string]$RepositoryRoot = '.', + [AllowEmptyString()][string]$Token, + [bool]$IncludePrerelease = $true, + [switch]$DirectCommit, + [switch]$DryRun, + [AllowEmptyString()][string]$BaseBranch = $env:GITHUB_REF_NAME, + [AllowEmptyString()][string]$Actor = $env:GITHUB_ACTOR, + [AllowEmptyString()][string]$Repository = $env:GITHUB_REPOSITORY, + [AllowEmptyString()][string]$RemoteUrl, + [string]$ApiUrl = $(if ($env:GITHUB_API_URL) { $env:GITHUB_API_URL } else { 'https://api.github.com' }), + [AllowEmptyString()][string]$PackageSource, + [string]$WorkPath, + [string]$SummaryPath = $env:GITHUB_STEP_SUMMARY, + [string]$WorkspaceRoot = $env:GITHUB_WORKSPACE, + [int]$SummaryLimit = 900KB, + [System.DateTimeOffset]$Now = [System.DateTimeOffset]::UtcNow, + # Test seam: runs instead of Publish-RulebookScan with the same parameters (the script re-imports the modules, + # so a Pester mock of the function does not reach it). + [scriptblock]$PublishCommand +) + +Set-StrictMode -Version 3.0 +$watch = [System.Diagnostics.Stopwatch]::StartNew() +$modules = Join-Path $PSScriptRoot '..' '..' 'modules' +Import-Module (Join-Path $modules 'Rulebook.Generate.psd1') -Force +Import-Module (Join-Path $modules 'Rulebook.GitHub.psd1') -Force +Import-Module (Join-Path $modules 'Rulebook.Update.psd1') -Force +Import-Module (Join-Path $modules 'Rulebook.Quarantine.psd1') -Force +Import-Module (Join-Path $modules 'Rulebook.Scan.psd1') -Force + +$docsUrl = 'https://github.com/ALCops/rulebook/blob/main/docs/ghtokenworkflow.md' + +# The helpers below are copies of CheckForUpdates.ps1 (a shared action helper module is #58, WP12). +function Format-AnnotationText { + # Workflow command escaping: the message part escapes %, CR and LF; a property value also : and ,. + param([AllowNull()][string]$Text, [switch]$Property) + if ($null -eq $Text) { return '' } + $escaped = $Text.Replace('%', '%25').Replace("`r", '%0D').Replace("`n", '%0A') + if ($Property) { $escaped = $escaped.Replace(':', '%3A').Replace(',', '%2C') } + return $escaped +} + +function ConvertTo-SingleLine { + # A message on one Markdown line (list item or paragraph); no table escaping. + param([AllowNull()][string]$Text) + if ([string]::IsNullOrEmpty($Text)) { return '' } + return $Text.Replace("`r", ' ').Replace("`n", ' ') +} + +function Add-Annotation { + param([ValidateSet('error', 'warning', 'notice')][string]$Command = 'error', [string]$File, [string]$Title = 'ScanDiagnostics', [Parameter(Mandatory)][string]$Message) + $properties = "title=$(Format-AnnotationText $Title -Property)" + if ($File) { $properties = "file=$(Format-AnnotationText $File -Property),$properties" } + $line = "::$Command $properties::$(Format-AnnotationText $Message)" + $script:annotations.Add($line) + Write-Host $line +} + +function Write-Text { + param([AllowNull()][string]$Path, [Parameter(Mandatory)][AllowEmptyString()][string]$Text) + if ([string]::IsNullOrEmpty($Path)) { return } + [System.IO.File]::AppendAllText($Path, $Text.Replace("`r`n", "`n"), [System.Text.UTF8Encoding]::new($false)) +} + +# [System.IO.File] resolves a relative path against the process directory, not the PowerShell location. +$resolvePath = { param($Path) if ([string]::IsNullOrEmpty($Path)) { $Path } else { $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($Path) } } +$SummaryPath = & $resolvePath $SummaryPath +$ownWork = [string]::IsNullOrEmpty($WorkPath) +if ($ownWork) { + $tempRoot = if ($env:RUNNER_TEMP) { $env:RUNNER_TEMP } else { [System.IO.Path]::GetTempPath() } + $WorkPath = Join-Path $tempRoot ('rulebook-scan-' + [guid]::NewGuid().ToString('n').Substring(0, 8)) +} +$WorkPath = & $resolvePath $WorkPath +$script:annotations = [System.Collections.Generic.List[string]]::new() +$script:failure = $null +function Add-Failure { + param([Parameter(Mandatory)][string]$Kind) + if ($null -eq $script:failure) { $script:failure = $Kind } +} + +# Every token obtained by an exchange is masked before anything can print it. +$maskToken = { param([string]$Value) if (-not [string]::IsNullOrEmpty($Value)) { Write-Host "::add-mask::$Value" } } +$plan = $null +$publish = $null +$result = $null +$summaryMessage = $null +$failed = $false +$secretName = 'GHTOKENWORKFLOW' + +try { + # 1. Settings and the secret name. + $root = (Resolve-Path -LiteralPath $RepositoryRoot -ErrorAction Stop).ProviderPath + $workspace = if ([string]::IsNullOrEmpty($WorkspaceRoot)) { $root } else { (Resolve-Path -LiteralPath $WorkspaceRoot -ErrorAction Stop).ProviderPath } + $separator = [System.IO.Path]::DirectorySeparatorChar + $settingsFile = Join-Path $root '.github' 'Rulebook-Settings.json' + if (-not (Test-Path -LiteralPath $settingsFile -PathType Leaf)) { throw "Settings missing: .github/Rulebook-Settings.json in $root" } + try { + $settings = Get-Content -LiteralPath $settingsFile -Raw | ConvertFrom-Json -AsHashtable -ErrorAction Stop + } catch { + throw "Invalid JSON in .github/Rulebook-Settings.json: $($_.Exception.Message)" + } + $setting = { param([string[]]$Path) $value = $settings; foreach ($key in $Path) { if ($value -isnot [System.Collections.IDictionary] -or -not $value.Contains($key)) { return $null }; $value = $value[$key] }; return $value } + $name = [string](& $setting 'ghTokenWorkflowSecretName') + $nameValid = $name -cmatch '^[A-Za-z_][A-Za-z0-9_]*$' -and $name -inotmatch '^GITHUB_' + if (-not [string]::IsNullOrEmpty($name) -and $nameValid) { $secretName = $name } + + # 2. The quarantine policy, before any request (D14: no default). + try { + $null = Get-QuarantinePolicy -Settings $settings + } catch { + Add-Failure 'policy' + throw + } + + # 3. The write token, unless this is a dry run. + $access = $null + if (-not $DryRun) { + if (-not [string]::IsNullOrEmpty($name) -and -not $nameValid) { + Add-Failure 'token' + throw "ghTokenWorkflowSecretName '$name' in .github/Rulebook-Settings.json is not a valid secret name (letters, digits and underscores, not starting with a digit or GITHUB_). Read $docsUrl" + } + if ([string]::IsNullOrWhiteSpace($Token)) { + Add-Failure 'token' + throw "The $secretName secret is needed to scan diagnostics. Read $docsUrl" + } + if ([string]::IsNullOrEmpty($Repository)) { throw 'The scan needs the repository (GITHUB_REPOSITORY) as owner/name.' } + if ([string]::IsNullOrEmpty($BaseBranch)) { throw 'The scan needs baseBranch.' } + try { + $access = Get-GitHubAccessToken -Token $Token -Repository $Repository -ApiUrl $ApiUrl + } catch { + Add-Failure 'token' + throw "The $secretName secret could not be used: $($_.Exception.Message)" + } + if (-not [string]::IsNullOrEmpty($access.Token)) { & $maskToken $access.Token } + Write-Host "Write token: $($access.Kind)" + } + + # 4. The plan. + $plan = Get-RulebookScanPlan -RepositoryRoot $root -IncludePrerelease $IncludePrerelease -Source $PackageSource -WorkPath (Join-Path $WorkPath 'plan') -Now $Now + foreach ($channel in $plan.Channels) { Write-Host "NuGet: $($channel.PackageId) stable $($channel.Stable), prerelease $(if ($channel.Prerelease) { $channel.Prerelease } else { '-' })" } + foreach ($item in $plan.Scanned) { Write-Host "Scanned $($item.PackageId) $($item.Version) ($($item.Channel)): $($item.Records.Records.Count) ids, $(@($item.Diff.NewIds).Count) new, $($item.Extraction.ElapsedSeconds) s in the child process on .NET $($item.Extraction.Runtime)" } + foreach ($note in $plan.Notes) { Write-Host "Note: $note" } + if ($plan.Failure) { + Add-Failure $plan.Failure + $what = switch ($plan.Failure) { 'nuget' { 'The NuGet packages could not be read' } 'extract' { 'The diagnostics could not be extracted' } default { 'The scan failed' } } + throw "$what`: $($plan.FailureMessage)" + } + + if ($plan.Mode -eq 'nothing-new') { + # 5. Nothing new. + $versions = @($plan.Channels | ForEach-Object { "$($_.PackageId) $($_.Stable) stable" + $(if ($_.Prerelease) { ", $($_.Prerelease) prerelease" } else { '' }) }) + $summaryMessage = "No new package version ($($versions -join '; ')); nothing to do" + Add-Annotation -Command notice -Message $summaryMessage + $result = 'nothing-new' + } elseif (-not $plan.Valid) { + # 6. The candidate does not validate. + foreach ($finding in @($plan.Findings | Where-Object Severity -EQ 'error')) { + $message = if ($finding.Id) { "$($finding.Id): $($finding.Message)" } else { $finding.Message } + # C7 is a warning before the first scan and an error after it; name the remedy. + if ($finding.Rule -ceq 'C7' -and $finding.Id) { + $message += "; add $($finding.Id) to catalog/diagnostics.json or remove it from $($finding.File); the scan writes catalog/scan-state.json, which turns the C7 warning into an error" + } + $file = if ($finding.File) { [System.IO.Path]::GetRelativePath($workspace, (Join-Path $root $finding.File)).Replace($separator, '/') } else { $null } + Add-Annotation -File $file -Title $finding.Rule -Message "The scanned rulebook would not validate: $message" + } + Add-Failure 'validation' + $summaryMessage = 'The scanned rulebook does not validate; nothing was pushed. The findings come from the repository after the scan.' + $failed = $true + } elseif ($DryRun) { + # 7. Dry run. + $summaryMessage = "Dry run: $(Get-ScanTitle -Plan $plan); nothing was pushed" + Add-Annotation -Command notice -Message $summaryMessage + $result = 'dry-run' + } else { + # 8. Publish. + if (-not $plan.HeadSha) { Add-Annotation -Command warning -Message 'base-move guard inactive: the checkout HEAD could not be read' } + $labels = [string[]]@(& $setting 'commitOptions', 'pullRequestLabels' | Where-Object { $_ -is [string] -and $_ -ne '' }) + try { + $publishParameters = @{ + Plan = $plan; RepositoryRoot = $root; Repository = $Repository; RemoteUrl = $RemoteUrl; Token = $access.Token; BaseBranch = $BaseBranch + DirectCommit = [bool]$DirectCommit; Actor = $Actor; Labels = $labels; WorkPath = (Join-Path $WorkPath 'publish'); ApiUrl = $ApiUrl + } + $publish = if ($null -ne $PublishCommand) { & $PublishCommand @publishParameters } else { Publish-RulebookScan @publishParameters } + } catch { + $stage = [string]$_.Exception.Data['Stage'] + if ($stage -cnotin 'push', 'pull-request') { $stage = 'push' } + Add-Failure $stage + # The base branch moved between plan and publish: nothing is wrong with the token, the next run picks it up. + if ([string]$_.Exception.Data['Reason'] -ceq 'base-moved') { throw $_.Exception.Message } + $what = if ($stage -eq 'pull-request') { 'Failed to create or update the scan pull request' } else { 'Failed to push the scan' } + throw "$what. Make sure that the token in the secret $secretName is not expired and may write contents and pull requests of $Repository. Read $docsUrl (Error was: $($_.Exception.Message))" + } + $result = $publish.Result + switch ($publish.Result) { + 'no-changes' { $summaryMessage = 'No changes to commit' } + 'direct-commit' { $summaryMessage = "Scan committed to $($publish.Branch) ($(Get-ShortSha $publish.Sha))" } + 'pull-request-updated' { $summaryMessage = "Pull request updated: $($publish.PullRequestUrl)" } + default { $summaryMessage = "Pull request: $($publish.PullRequestUrl)" + $(if ($publish.Fallback) { ' (the direct commit was refused)' } else { '' }) } + } + Add-Annotation -Command notice -Message $summaryMessage + if ($publish.PSObject.Properties['ClosedPullRequestUrl'] -and $publish.ClosedPullRequestUrl) { Add-Annotation -Command notice -Message "Pull request closed: $($publish.ClosedPullRequestUrl)" } + } +} catch { + Add-Annotation -Message $_.Exception.Message + Add-Failure 'error' + $summaryMessage = $_.Exception.Message + $failed = $true +} finally { + if ($ownWork -and -not ($DryRun -and $result -eq 'dry-run') -and (Test-Path -LiteralPath $WorkPath)) { Remove-Item -LiteralPath $WorkPath -Recurse -Force -ErrorAction SilentlyContinue } +} + +# 9. Summary and outputs. +$summary = "## Diagnostic scan`n`n$(ConvertTo-SingleLine $summaryMessage)`n`n" +if ($null -ne $plan) { + try { + $summary = ConvertTo-ScanSummary -Plan $plan -Result $publish -Message $summaryMessage + } catch { + Write-Host "The summary could not be written in full: $($_.Exception.Message)" + } +} +$summary = $summary.Replace("`r`n", "`n") +# The runner caps a step summary at 1 MiB; stay well below it (-SummaryLimit, 900 KiB). +$where = if ($null -ne $publish -and $publish.PSObject.Properties['Body'] -and $publish.Body) { 'the pull request body' } else { 'the job log' } +$summary = Limit-SummaryText -Text $summary -MaxBytes $SummaryLimit -Footer "The summary was cut at $([math]::Floor($SummaryLimit / 1KB)) KiB; the full lists are in $where." +Write-Text -Path $SummaryPath -Text $summary +$counts = if ($null -ne $plan -and $null -ne $plan.Counts) { $plan.Counts } else { $null } +$outputs = [ordered]@{ + result = $(if ($failed) { '' } elseif ($result) { $result } else { '' }) + newIds = $(if ($counts) { $counts.NewIds } else { 0 }) + quarantined = $(if ($counts) { $counts.Quarantined } else { 0 }) + changedDefaults = $(if ($counts) { $counts.ChangedDefaults } else { 0 }) + released = $(if ($counts) { $counts.Released } else { 0 }) + scannedVersions = $(if ($null -ne $plan) { @($plan.Scanned | ForEach-Object { "$($_.PackageId)@$($_.Version):$($_.Channel)" }) -join ',' } else { '' }) + pullRequestUrl = $(if ($null -ne $publish -and $publish.PullRequestUrl) { $publish.PullRequestUrl } elseif ($null -ne $publish -and $publish.PSObject.Properties['ClosedPullRequestUrl'] -and $publish.ClosedPullRequestUrl) { $publish.ClosedPullRequestUrl } else { '' }) + candidatePath = $(if ($null -ne $plan -and $plan.CandidatePath -and $result -eq 'dry-run') { $plan.CandidatePath } else { '' }) + elapsedSeconds = [math]::Round($watch.Elapsed.TotalSeconds, 1).ToString([System.Globalization.CultureInfo]::InvariantCulture) + failure = $script:failure +} +if ($env:GITHUB_OUTPUT) { + Write-Text -Path $env:GITHUB_OUTPUT -Text ((@($outputs.GetEnumerator() | ForEach-Object { "$($_.Key)=$($_.Value)" }) -join "`n") + "`n") +} +[pscustomobject]@{ + ExitCode = $(if ($failed) { 1 } else { 0 }) + Result = $outputs.result + Failure = $script:failure + Plan = $plan + Publish = $publish + Annotations = $script:annotations.ToArray() + Summary = $summary + Outputs = $outputs +} diff --git a/actions/ScanDiagnostics/action.yaml b/actions/ScanDiagnostics/action.yaml new file mode 100644 index 0000000..5ec8783 --- /dev/null +++ b/actions/ScanDiagnostics/action.yaml @@ -0,0 +1,101 @@ +name: Rulebook ScanDiagnostics +description: Scans the newest Business Central compiler and ALCops packages for diagnostic ids, quarantines new ids by the organization's policy, records changed analyzer defaults and publishes the result as one living pull request (or a direct commit). +author: ALCops + +inputs: + token: + description: The GHTOKENWORKFLOW value (a personal access token or GitHub App JSON). Needed unless dryRun is 'true'. + required: false + default: '' + includePrerelease: + description: "'true' also scans the newest prerelease of each package when it is newer than the stable one." + required: false + default: 'true' + directCommit: + description: "'true' pushes to baseBranch instead of opening a pull request (falls back to the scan branch when the push is refused)." + required: false + default: 'false' + dryRun: + description: "'true' builds and validates the scan without a token and pushes nothing; the candidatePath output names the result." + required: false + default: 'false' + baseBranch: + description: The branch the scan starts from and the pull request targets. + required: false + default: ${{ github.ref_name }} + repositoryRoot: + description: Path of the rulebook repository, relative to the workspace. + required: false + default: '.' + actor: + description: The git author name of the scan commit. + required: false + default: ${{ github.actor }} + packageSource: + description: A NuGet flat container (URL or folder) instead of https://api.nuget.org/v3-flatcontainer (engine tests). + required: false + default: '' + +outputs: + result: + description: "pull-request, pull-request-updated, direct-commit, no-changes, nothing-new or dry-run; empty on failure." + value: ${{ steps.scan.outputs.result }} + newIds: + description: The number of new advertised diagnostic ids. + value: ${{ steps.scan.outputs.newIds }} + quarantined: + description: The number of new ids added to a quarantine file. + value: ${{ steps.scan.outputs.quarantined }} + changedDefaults: + description: The number of analyzer defaults a stable version changed. + value: ${{ steps.scan.outputs.changedDefaults }} + released: + description: The number of quarantine entries released because a level file mentions the id. + value: ${{ steps.scan.outputs.released }} + scannedVersions: + description: The scanned versions as @:, comma separated. + value: ${{ steps.scan.outputs.scannedVersions }} + pullRequestUrl: + description: The URL of the scan pull request (new, updated, or closed because the base holds its result). + value: ${{ steps.scan.outputs.pullRequestUrl }} + candidatePath: + description: After a dry run, the folder holding the scanned repository. + value: ${{ steps.scan.outputs.candidatePath }} + elapsedSeconds: + description: The run time of the step in seconds. + value: ${{ steps.scan.outputs.elapsedSeconds }} + failure: + description: "Why the step failed, empty on success: policy, token, nuget, extract, validation, push, pull-request or error." + value: ${{ steps.scan.outputs.failure }} + +runs: + using: composite + steps: + - name: Scan diagnostics + id: scan + shell: pwsh + # Inputs reach the script through the environment, never interpolated into the script (script injection). + env: + INPUT_TOKEN: ${{ inputs.token }} + INPUT_INCLUDEPRERELEASE: ${{ inputs.includePrerelease }} + INPUT_DIRECTCOMMIT: ${{ inputs.directCommit }} + INPUT_DRYRUN: ${{ inputs.dryRun }} + INPUT_BASEBRANCH: ${{ inputs.baseBranch }} + INPUT_REPOSITORYROOT: ${{ inputs.repositoryRoot }} + INPUT_ACTOR: ${{ inputs.actor }} + INPUT_PACKAGESOURCE: ${{ inputs.packageSource }} + run: | + $parameters = @{ + RepositoryRoot = $env:INPUT_REPOSITORYROOT + Token = $env:INPUT_TOKEN + IncludePrerelease = $env:INPUT_INCLUDEPRERELEASE -ne 'false' + DirectCommit = $env:INPUT_DIRECTCOMMIT -eq 'true' + DryRun = $env:INPUT_DRYRUN -eq 'true' + BaseBranch = $env:INPUT_BASEBRANCH + Actor = $env:INPUT_ACTOR + PackageSource = $env:INPUT_PACKAGESOURCE + } + # The result object is the last pipeline output; anything a module emits before it is ignored. + $result = @(& (Join-Path $env:GITHUB_ACTION_PATH 'ScanDiagnostics.ps1') @parameters)[-1] + if ($null -eq $result -or $null -eq $result.PSObject.Properties['ExitCode']) { Write-Host '::error::ScanDiagnostics.ps1 returned no result'; exit 1 } + exit $result.ExitCode diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 844271e..7306ebc 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -96,7 +96,7 @@ Everything an org repo contains after "Use this template". The **class** column | `.github/workflows/Validate.yaml` | On every pull request: schema, catalog coverage, regeneration check, effective diff report. Also runs the update check in check mode. | system | | `.github/workflows/Publish.yaml` | On push to the default branch and on demand: validate, refuse stale endpoints, deploy `rulesets/`, the rendered skeletons and `index.html` to the configured target, verify reachability. Never commits (D42). | system | | `.github/workflows/UpdateRulebookSystemFiles.yaml` | Manual, or scheduled when `update.schedule` is set (the update writes the `schedule:` trigger): pull the new template version into a PR (or a direct commit) with the regenerated endpoints and skeletons, after validating the result (section 7.3). | system | -| `.github/workflows/ScanDiagnostics.yaml` | Daily: NuGet scan, catalog diff, quarantine PR with regenerated endpoints. | system | +| `.github/workflows/ScanDiagnostics.yaml` | Daily (the `schedule:` comes from `scan.schedule`, shipped `17 4 * * *`) and manual: scan the newest stable and prerelease versions of the two analyzer packages on NuGet, record new ids and changed defaults in the catalog, quarantine new ids by the policy, release the ids a level file adopted, regenerate the endpoints and keep one living pull request on `scan-diagnostics/`, or push a direct commit (section 7.4). | system | | `.github/workflows/ChangeRule.yaml` | Manual form: one override entry, regenerate, open a PR. | system | | `.github/workflows/ApplyRulebookChange.yaml` | On an issue with the `rulebook-change` label: gate on collaborator association, apply the change set, regenerate, open a PR or commit (section 7.6). | system | | `.github/ISSUE_TEMPLATE/rulebook-change.yml`, `config.yml` | The issue form the dashboard prefills; blank issues stay enabled. | system | @@ -106,9 +106,9 @@ Everything an org repo contains after "Use this template". The **class** column | `stages/.json` (2 shipped files) | One delta per non-default stage, applied on top of every level's default result: `ci.json`, `vnext.json`. The `default` stage has no file. The shipped files carry the delta profile URL in `$schema`. Org-added stages are org-owned. | system | | `base/twins.json` | The PerTenantExtensionCop/AppSourceCop twin pairs the `twins` setting acts on (D23). | system | | `overrides.json` | The org's rule changes with scope selectors (D19). | org-owned | -| `quarantine..json` (one per stage, `quarantine.default.json` included) | Ids held back per stage, written by the scan. | org-owned | -| `catalog/diagnostics.json` | Every known diagnostic id with analyzer, package, default severity, enablement, first-seen version and channel (D24). The template ships a seed with `id`, `analyzer`, `defaultSeverity`, `enabledByDefault`, `title` and `docs` per inventory id; the first scan adds the package, versions and channel (section 5.6). | org-owned | -| `catalog/scan-state.json` | Reserved for the scan's state; name and schema come with WP08. | org-owned | +| `quarantine..json` (one per stage, `quarantine.default.json` included) | Ids held at `None` per stage while no level file mentions them, written by the scan from the policy; a level file that adopts an id releases it (housekeeping). A file whose `` is no stage of the settings is C16. | org-owned | +| `catalog/diagnostics.json` | Every known diagnostic id with analyzer, package, default severity, enablement, first-seen version and channel (D24). The template ships a seed with `id`, `analyzer`, `defaultSeverity`, `enabledByDefault`, `title` and `docs` per inventory id; the scan adds `package`, `firstSeenVersion`, `firstSeenChannel`, `firstStableVersion`, `lastSeenVersion` and the flags `advertised: false`, `deprecated: true` and `defaultChanges` (sections 5.4 and 7.4, D46). | org-owned | +| `catalog/scan-state.json` | The package version the scan recorded last per package and channel (`{ version, scannedAt }`), so a run without a new version stops after two index requests. Created by the first scan, never shipped; schema `rulebook-scan-state.schema.json`. | org-owned | | `rulesets/.ruleset.json` (default stage), `rulesets/..ruleset.json` (12 files in the shipped set) | The endpoints: generated from the level chain + stage delta + twins setting + overrides + quarantine, listing the ids whose effective action differs from the analyzer default, no includes. Committed. | generated (regenerated in their pull requests by Update, Scan, ChangeRule and Apply, or by hand with `Update-RulebookEndpoints`; checked by Validate (C12) and Publish, which never regenerate) | | `skeletons/..ruleset.json` (12 files, the stage suffix always written, `strict.default` included), `skeletons/README.md` | Copy-paste files for AL projects with `{BASEURL}`; one include of the endpoint. The README explains them next to the files; it is not published and is exempt from C11 (WP06). | generated from the settings (regenerated by every update with `New-RulebookSkeleton`); the README is system | | `docs/`, `README.md` | The org's own notes; the template ships a README that explains the layout. | never touched after creation | @@ -116,7 +116,7 @@ Everything an org repo contains after "Use this template". The **class** column The `rulesets/` folder is flat and every endpoint is self-contained, so the whole set is relocatable to any host without editing a file. -Every JSON file in this table except `catalog/scan-state.json` (WP08) and the files under `site/` has a schema in the engine under `schemas/`, served from the release branch as `https://raw.githubusercontent.com/ALCops/rulebook-engine/v1/schemas/.schema.json` (section 5.4). The `v1` URLs go live with WP13 ([#15](https://github.com/ALCops/rulebook-engine/issues/15)); until then they return 404 and the tests use the local files. The ruleset profile follows the folder: `base/` and `stages/` are delta, `rulesets/` is endpoint, `skeletons/` is skeleton. The generator never writes `$schema` into an endpoint or a skeleton; the compiler fetches them and they stay minimal. File names, slugs and the schema list are in [reference/naming.md](reference/naming.md). +Every JSON file in this table except the files under `site/` has a schema in the engine under `schemas/`, served from the release branch as `https://raw.githubusercontent.com/ALCops/rulebook-engine/v1/schemas/.schema.json` (section 5.4). The `v1` URLs go live with WP13 ([#15](https://github.com/ALCops/rulebook-engine/issues/15)); until then they return 404 and the tests use the local files. The ruleset profile follows the folder: `base/` and `stages/` are delta, `rulesets/` is endpoint, `skeletons/` is skeleton. The generator never writes `$schema` into an endpoint or a skeleton; the compiler fetches them and they stay minimal. File names, slugs and the schema list are in [reference/naming.md](reference/naming.md). ## 5. Generation model @@ -178,14 +178,14 @@ The checks are numbered `C1` to `C16` so that WP02 and WP03 can reference them; | C4 | Rule `action` is one of Error, Warning, Info, Hidden, None. Never `Default`. | error | `Default` fails deserialisation. | | C5 | Settings: `levels` and `stages` are non-empty ordered arrays; every name lowercases to `^[a-z0-9-]+$`; slugs are unique per array; `stages` contains `default`; every `basedOn` resolves to an existing `base/.ruleset.json` without a cycle; `twins` is `both`, `appsource` or `pte`; `quarantine.*` is `null` or a list of stage slugs; `baseUrl` has no trailing slash. | error | Every file name and URL is derived from these values. | | C6 | Every published level has `base/.ruleset.json`; every non-default stage has `stages/.json`; `stages/default.json` does not exist. | error | The default stage is the level result; a file for it would be a second truth. | -| C7 | Every id in level files, stage files, `base/twins.json`, `overrides.json` and the quarantine files exists in `catalog/diagnostics.json`. | warning while `catalog/scan-state.json` is absent, error once it exists (the first scan writes it, WP08) | Typos never reach an endpoint. | +| C7 | Every id in level files, stage files, `base/twins.json`, `overrides.json` and the quarantine files exists in `catalog/diagnostics.json`. | warning while `catalog/scan-state.json` is absent (the first scan creates it), error once it exists (WP08) | Typos never reach an endpoint. | | C8 | A stage entry whose id no published level enables. | warning | Dead entry; a stage never activates a rule. | | C9 | A delta entry equal to what the chain already gives; a file in `base/` or `stages/` that no settings entry references and that `unusedRulebookFiles` does not list by its repository-relative path (`stages/vnext.json`; a bare file name does not count, and the update matches entries the same way, [#49](https://github.com/ALCops/rulebook-engine/issues/49)). | warning | Dead weight, or a level the org forgot to publish or exclude. | | C10 | `overrides.json` selectors are lowercase level and stage slugs from the settings or `["*"]`; every entry has an action; a justification is optional (D37, D40). | error | Silent no-ops are the failure mode of a selector typo. | | C11 | No endpoint entry equals the catalog default of its id; exactly the `levels x stages` endpoints and skeletons exist, no others. Any file in `rulesets/` or `skeletons/` whose name is not an expected endpoint or skeleton name is C11, except `README.md` (exact name, ordinal), which the template ships in `skeletons/` and which Publish never stages; while C12 runs, a missing or stray `*.ruleset.json` in `rulesets/` is left to C12 ("would be created", "would be deleted"). The skeleton half applies only when `skeletons/` exists; the template ships them since WP04. | error | A listed default is dead weight; the index page and the AL projects rely on the names. | | C12 | Regeneration check: `rulesets/` equals the generator's output for the current inputs. Skipped with one warning while an input the generator needs has an error (section 5.5). | error | The committed endpoint is the published endpoint. | | C13 | A quarantine id that a level file now mentions. | warning | Housekeeping. | -| C14 | Every catalog entry has `defaultSeverity` and `enabledByDefault`; every pair in `base/twins.json` is one PTE id and one AS id, and `count` equals the number of pairs. | error | The sparse rule and the twins step depend on them. | +| C14 | Every catalog entry has `defaultSeverity` and `enabledByDefault`; every pair in `base/twins.json` is one PTE id and one AS id, and `count` equals the number of pairs; `catalog/scan-state.json`, when present, parses and matches its schema (not blocking: the generator does not read it). | error | The sparse rule and the twins step depend on them. | | C15 | A stage entry on an id the same stage's quarantine file lists and no file on the chain of any published level mentions. | warning | Dead while quarantined: quarantine wins over the stage entry until a level file adopts the id (D41). The stage file is a system file, so this is not an error. | | C16 | A `quarantine..json` whose `` is not a stage slug of the settings. | warning | A file nothing reads; its ids silently stop being quarantined. The generator reads `quarantine..json` for the settings stages only. The file still feeds C7 and C13, so a C13 "remove the quarantine entry" finding next to C16 on the same file is expected. `unusedRulebookFiles` cannot list a root-level file, so the only remedies are the three the message names: rename the file to a stage slug, remove it, or add the stage. | @@ -211,7 +211,9 @@ A rule is `id` (`^[A-Z]{2,3}[0-9]{4}i?$`), `action` (`Error`, `Warning`, `Info`, **Quarantine.** An entry is `id` and an optional `justification`, with no `action`: quarantine always means `None`, and only for ids no file on the level chain mentions. -**Twins, catalog, settings.** `base/twins.json` is `pairs` of one `PTE` and one `AS` id with an optional `title`, plus the generator's `generatedBy`, `setting`, `values` and `count`. The catalog is `version` 1 and `diagnostics[]`; an entry requires `id`, `defaultSeverity` and `enabledByDefault` and may carry more fields. The settings schema is closed (section 8). +**Twins, catalog, settings.** `base/twins.json` is `pairs` of one `PTE` and one `AS` id with an optional `title`, plus the generator's `generatedBy`, `setting`, `values` and `count`. The catalog is `version` 1 and `diagnostics[]`; an entry requires `id`, `defaultSeverity` and `enabledByDefault`, types the seed and scan fields (`title`, `docs`, `package`, the versions, `firstSeenChannel`, `advertised`, `deprecated`, `defaultChanges` as closed `{ version, field, from, to }` elements) and may carry more. The settings schema is closed (section 8). + +**Scan state.** `catalog/scan-state.json` is `version` 1 and `packages`, an object keyed by the lowercase package id whose `stable` and `prerelease` are `null` or `{ version, scannedAt }` (UTC, seconds); the schema is closed. The first scan creates it, the template never ships it, and its presence turns C7 into an error: from then on every id must be in the catalog. ### 5.5 Generate and Validate modules @@ -258,12 +260,24 @@ A rule is `id` (`^[A-Z]{2,3}[0-9]{4}i?$`), `action` (`Error`, `Warning`, `Info`, |---|---| | `Build-RulebookBase -RulebookDir -OutputPath` | One `.ruleset.json` per entry of `matrix/levels.json`: a root lists the ids whose default-stage cell in `resolved.json` differs from the analyzer default, every other level the ids whose cell differs from its `basedOn` level. Writes `base/twins.json` from `matrix/twins.json` with the twins schema URL, pairs sorted by the PTE side. | | `Build-RulebookStages -RulebookDir -OutputPath` | One `.json` per non-default entry of `matrix/stages.json`: the ids whose `matrix.json` column named after the stage is not `=`. | -| `Build-RulebookCatalog -RulebookDir -OutputPath` | The seed `catalog/diagnostics.json`: one entry per inventory id with `id`, `analyzer`, `defaultSeverity`, `enabledByDefault`, `title`, `docs`. | +| `Build-RulebookCatalog -RulebookDir -OutputPath` | The seed `catalog/diagnostics.json`: one entry per inventory id with `id`, `analyzer`, `defaultSeverity`, `enabledByDefault`, `title`, `docs`, written by `ConvertTo-CatalogJson` of `Rulebook.Catalog`, the writer the scan uses, so the seed and a scanned catalog share one layout. | | `New-RulebookSkeleton -SettingsPath -OutputPath` | One `..ruleset.json` per level and stage of the settings, including `{BASEURL}/rulesets/` with action `Default`. | Common to all four: entries in inventory order with the matrix row justification; the level and stage files carry the delta profile URL in `$schema`; UTF-8 without BOM, LF; a file is written only when its bytes differ, files of the folder that no input produces are deleted first; one `{ File, Path, Change }` object per change (`created`, `modified`, `deleted`); `-WhatIf` writes nothing and returns the same list. The functions throw, as engine tools, on matrix input they cannot use (rows out of inventory order, an id without resolved cells, a level that is not a slug, an unresolved `basedOn` or a cycle, no `default` stage, a stage without a matrix column, a value that is not an action). -`tools/rulebook/Build-Template.ps1 [-RulebookDir] [-TemplateDir] [-WhatIf]` runs the four functions and then `Update-RulebookEndpoints` on `template/`, prints one line per change or `template: current`, and returns the change objects. `tests/Rulebook.Template.Tests.ps1` regenerates `template/` from its 9 hand-written files and compares the bytes, so a matrix change without a template regeneration fails CI. +`tools/rulebook/Build-Template.ps1 [-RulebookDir] [-TemplateDir] [-WhatIf]` runs the four functions and then `Update-RulebookEndpoints` on `template/`, prints one line per change or `template: current`, and returns the change objects. `tests/Rulebook.Template.Tests.ps1` regenerates `template/` from its 11 hand-written files and compares the bytes, so a matrix change without a template regeneration fails CI. + +### 5.7 Scan modules + +The scan of section 7.4 (WP08) is five modules, each a `.psm1` with a `.psd1` manifest; the contract is [reference/scan-mechanics.md](reference/scan-mechanics.md). + +| Module | Functions | Role | +|---|---|---| +| `Rulebook.NuGet` | `Compare-NuGetVersion`, `Select-NuGetChannelVersion`, `Get-NuGetVersionIndex`, `Get-NuGetPackageUrl`, `Save-NuGetPackage` | NuGet semantic versioning, the newest stable and (newer) prerelease version, the flat-container index and the download of one version. No engine imports; a folder `-Source` is the offline seam. | +| `Rulebook.Extract` | `Resolve-AnalyzerFolder`, `Get-AnalyzerDescriptor`, `Invoke-DescriptorExtraction`, `ConvertTo-DiagnosticRecord`, `Get-ExpectedAssembly` | The analyzer folder for the running .NET, the descriptors by reflection in a child pwsh per package version (fails loudly on a load error or a missing expected assembly), one record per id of one package. | +| `Rulebook.Catalog` | `Read-CatalogFile`, `ConvertTo-CatalogJson`, `Write-CatalogFile`, `Get-CatalogDocsUrl`, `Update-CatalogFromScan`, `Read-ScanState`, `Write-ScanState`, `Get-NewPackageVersion` | The catalog with every scan field and unknown keys kept, the docs URL rule, one scanned version applied to the catalog (pure), the scan state. | +| `Rulebook.Quarantine` | `Get-QuarantinePolicy`, `Read-QuarantineFile`, `Write-QuarantineFile`, `New-QuarantineJustification`, `Add-QuarantineEntry`, `Invoke-QuarantineHousekeeping`, `Update-QuarantineFromScan` | The policy (no default, D14), the quarantine files in the template layout, new ids into the policy stages, housekeeping by the rule of C13. | +| `Rulebook.Scan` | `Get-RulebookScanPlan`, `Get-ScanTitle`, `ConvertTo-ScanPullRequestBody`, `ConvertTo-ScanSummary`, `Publish-RulebookScan` | The run of section 7.4 on a candidate tree, its title and body, and the living pull request (D45). | ## 6. Endpoints and skeletons @@ -407,25 +421,32 @@ The shipped level files in `base/`, the shipped stage files in `stages/` and `ba ### 7.4 Scan diagnostics (R9) -Trigger: daily schedule, `workflow_dispatch`. +Trigger: a daily `schedule:` (the update writes it from `scan.schedule`; the template ships `17 4 * * *`, `null` removes it) and `workflow_dispatch` with the inputs `includePrerelease` (default true) and `directCommit` (default false). A `Read the settings` step takes the secret name from `ghTokenWorkflowSecretName`; a scheduled run scans prereleases and takes `directCommit` = not `commitOptions.createPullRequest`. The action `actions/ScanDiagnostics` runs the modules of section 5.7; every detail is in [reference/scan-mechanics.md](reference/scan-mechanics.md). ```mermaid flowchart LR - idx[NuGet flat-container index
both packages, stable + prerelease] --> dl[download latest nupkg per package per channel
neutral tools package only, prerelease only if newer than stable] - dl --> ex[extract diagnostic ids, default severity, enablement
reflection in pwsh, WP01 spike b] - ex --> diff[diff with catalog/diagnostics.json] - diff -->|new ids| q[add to quarantine.stage.json
per policy in settings] - diff -->|changed defaults| dd[update catalog, list in PR] - diff -->|ids now in base| hk[remove from quarantine] - q & dd & hk --> gen[regenerate rulesets/] - gen --> pr[PR: catalog + quarantine + endpoints + summary table] + pol[policy set?
else fail before any request] --> idx[NuGet index of both packages
stable, and prerelease when newer] + idx --> st{new version
vs scan-state.json?} + st -->|no, nothing adopted| stop[nothing-new: stop] + st -->|no, an adopted id| hk[housekeeping only] + st -->|yes| ex[download, extract in a child pwsh
per version and channel] + ex --> cat[catalog: new ids, package fields,
stable default changes, text refresh, flags] + cat --> q[quarantine new ids by the policy] + q --> hk + hk --> gen[regenerate rulesets/, validate] + gen --> pr[rebuild scan-diagnostics/branch,
lease push, create or PATCH the pull request] ``` -Extraction loads the analyzer DLLs by reflection in pwsh: `Microsoft.Dynamics.Nav.CodeAnalysis.dll` and the four Microsoft cops from `tools//any/` of the tools package, the ALCops cops from `lib//` of `ALCops.Analyzers`, with `` the highest folder not newer than the pwsh runtime (`net10.0` on `ubuntu-latest` today) and never `netstandard2.1`, one pwsh process per package version and channel. Only the platform-neutral tools package is downloaded, and a prerelease counts only when it sorts after the stable version; see [spike (b)](reference/spikes/b-analyzer-dll-extraction.md). - -Policy from settings: `quarantine.stages` receives ids first seen in a stable package, `quarantine.prereleaseStages` receives ids first seen in a prerelease package. Both are mandatory (D14). Values are stage slugs from `settings.stages`. A typical choice: stable ids to `default` and `ci`, prerelease ids to `default` and `ci` as well, so `vnext` shows everything at default severity. A stage the org adds gets its own `quarantine..json` the first time the scan writes to it. Ids never leave the catalog; the catalog records `firstSeenVersion`, `firstSeenChannel` and `lastSeenVersion`, so a rule that disappears from a package is visible too. - -The catalog also records `defaultSeverity` and `enabledByDefault` per id (D24). When a package changes one, the scan updates the catalog, lists the change in the PR body ("LC0015: default Info -> Warning"), and regenerates the endpoints: an id whose level action now equals the new default drops out of the endpoint, an id whose default moved away from the level action is written. The reviewer sees the effective change per endpoint. +- **Fast path.** The policy is read first: both `quarantine` keys `null` (as shipped) or absent fail the run with the message that tells the organization what to set, before any request (D14). Then the two indexes are read and compared with `catalog/scan-state.json`; with no new version and no quarantined id a level file mentions, the run ends as `nothing-new` after two requests. With no new version but such an id, the run does housekeeping only. +- **Extraction.** Only the platform-neutral tools package and `alcops.analyzers` are downloaded. Each new version is extracted in its own pwsh process (an ALCops version is hosted by the tools version of the same channel), from `tools//any/` and `lib//` with `` the highest folder not newer than the pwsh runtime (`net10.0` on the `ubuntu-latest` image of spike (b), 20260927.320.1) and never `netstandard2.1` ([spike (b)](reference/spikes/b-analyzer-dll-extraction.md)). An assembly that does not load, an analyzer that cannot be instantiated or a missing expected cop fails the run; nothing is pushed. The extraction child gets no secret: the action inputs, every `*TOKEN`, `*_SECRET`, `*_PASSWORD`, `*_KEY` and `*_PAT` variable and the runner's command files (`GITHUB_ENV`, `GITHUB_PATH`, `GITHUB_OUTPUT`, `GITHUB_STATE`, `GITHUB_STEP_SUMMARY`) are removed from its environment, because it runs code from the downloaded packages; the workflow checks out with `persist-credentials: false`. +- **Catalog.** Stable before prerelease per package, tools before ALCops. A new id gets a full entry. The 628 seeded ids are known: the first scan fills their package fields and never quarantines them (D46). A stable version overwrites changed defaults and appends one `defaultChanges` element (D24), refreshes `title` and `docs` and sets the flags `advertised: false` (a descriptor no analyzer returns, cataloged, never quarantined) and `deprecated: true`; an id a stable version no longer carries keeps its entry and its `lastSeenVersion`. A prerelease never changes a default or a text; its default changes are listed in the pull request only. +- **Quarantine.** `quarantine.stages` receives ids new in a stable version and ids promoted from prerelease, `quarantine.prereleaseStages` ids new in a prerelease; both are mandatory, `[]` is valid. An id the catalog has as `advertised: false` that a stable version's analyzers return now goes live at its default and is quarantined in `quarantine.stages` like a new id, seeded or not. A stage the org added gets its `quarantine..json` the first time the scan writes to it; a slug that is not a stage never gets a file (C16). Every entry carries `New in (), quarantined . Review and adopt.` +- **Housekeeping.** A quarantined id that a file on a published level chain mentions is removed from every quarantine file (the rule of C13); the pull request lists it with the files that mention it. +- **Regenerate and validate.** The endpoints are regenerated on the candidate and `Test-Rulebook` runs on it; since the candidate carries `catalog/scan-state.json`, C7 is an error there. Any error fails the run and nothing is pushed. +- **One living pull request (D45).** Every run rebuilds the branch `scan-diagnostics/` from the base head as one commit with the full current result, pushes it with `--force-with-lease` (a push between the lease read and the push is rejected; a base branch that moved while the scan was planning fails the run before the push) and updates the title and body of the open pull request from that branch, or opens one with `commitOptions.pullRequestLabels`. A new package version without new ids or default changes still produces the pull request (a record run: `lastSeenVersion` and the scan state move). When the base already holds the result, or it went in as a direct commit, an open scan pull request is closed with a line saying why. +- **Direct commit.** With `directCommit` (or `commitOptions.createPullRequest: false` on a schedule) the commit goes to the branch itself; a refused push falls back to the scan branch and the pull request. +- **Title.** `Scan diagnostics: 3 new ids quarantined, 1 default changed (alcops 1.4.0)`: the counts first, then the newly scanned versions with the labels `alcops` and `tools`; `Scan diagnostics: alcops 1.3.2 recorded, no new diagnostics` for a record run, `Scan diagnostics: 1 quarantine entry released, no new package version` for housekeeping. +- **Dry run.** The action input `dryRun` builds and validates everything without a token and pushes nothing; the CI job `scan-action` runs it against nuget.org. ### 7.5 Change rule (R10) @@ -488,17 +509,18 @@ A change set is `{ version, note?, changes[] }` with `set` (write an override en "commitOptions": { "createPullRequest": true, "pullRequestLabels": ["rulebook"] }, "site": { "enabled": true, "includeJustifications": false, "updateMode": "skip" }, "update": { "schedule": null }, + "scan": { "schedule": "17 4 * * *" }, "unusedRulebookFiles": [] } ``` -`publish.target` selects one of `pages`, `dist-repo` (with `repository`, `branch`), `azure-blob` (with `storageAccount`, `container`, OIDC login) or `gist` (with `gistId`); v1 implements `pages` only, the other three are backlog (section 9). `baseUrl` ships empty in the template (`""`, the example above shows a set value); Publish fails with a proposal until the organization sets it (WP05). `quarantine` ships **without** values in the template; the scan fails until the org sets them. `twins` is `both` (default; the only valid choice when projects run just one of the two Microsoft cops), `appsource` or `pte` (D23). +`publish.target` selects one of `pages`, `dist-repo` (with `repository`, `branch`), `azure-blob` (with `storageAccount`, `container`, OIDC login) or `gist` (with `gistId`); v1 implements `pages` only, the other three are backlog (section 9). `baseUrl` ships empty in the template (`""`, the example above shows a set value); Publish fails with a proposal until the organization sets it (WP05). `quarantine` ships **without** values in the template; the scan fails until the org sets both keys (`[]` is valid and quarantines nowhere). `twins` is `both` (default; the only valid choice when projects run just one of the two Microsoft cops), `appsource` or `pte` (D23). `site` (D31, D35, D36): `enabled` builds and publishes the dashboard (default true; ignored with a warning for `dist-repo` and `gist`); `includeJustifications` publishes the organization's override and quarantine justification text (default false); `updateMode` is `skip` (keep locally changed site files on update) or `overwrite`. -`update` (WP07): `schedule` is a five-field cron string (for example `"0 6 * * 1"`) that the update writes into `UpdateRulebookSystemFiles.yaml` as its `schedule:` trigger, or `null` (shipped) for none. `unusedRulebookFiles` lists shipped files the update must not re-add and removes when present, by their repository-relative path with `/` (`stages/vnext.json`); the schema rejects a bare file name, a backslash and a leading slash. `templateSha` is written by every update; `templateUrl` is the template the next update pulls from. +`update` (WP07) and `scan` (WP08): `schedule` is a five-field cron string that the update writes into `UpdateRulebookSystemFiles.yaml` (`update.schedule`, shipped `null`: no schedule) or `ScanDiagnostics.yaml` (`scan.schedule`, shipped `"17 4 * * *"`: daily at 04:17 UTC) as its `schedule:` trigger, or `null` for none. An absent `update.schedule` removes the update schedule; an absent `scan` key or `scan.schedule` keeps the scan schedule the template ships (an organization from before WP08 has no `scan` key), and only an explicit `null` removes it. `unusedRulebookFiles` lists shipped files the update must not re-add and removes when present, by their repository-relative path with `/` (`stages/vnext.json`); the schema rejects a bare file name, a backslash and a leading slash. `templateSha` is written by every update; `templateUrl` is the template the next update pulls from. -The schema is closed: an unknown key at the top level or in `publish`, `quarantine`, `commitOptions`, `site` or `update` is an error, and a work package that needs a new key adds it to the schema in its own pull request. Required are `templateUrl`, `baseUrl` (empty, or `https://` with a DNS host name and an optional numeric port, no user info, and without a query, a fragment, `.` or `..` segments, quotes, backslashes, control characters or a trailing slash), `publish`, `quarantine` with both keys, `levels` and `stages`. `publish.target` requires its own fields: `dist-repo` needs `repository` (`owner/name`) and `branch`, `azure-blob` needs `storageAccount` and `container`, `gist` needs `gistId`; fields of another target are allowed and ignored. `templateSha` is empty or a 40-character commit sha. A level or stage entry has a `name` matching `^[A-Za-z0-9-]+$` and an optional `description`; a level may have `basedOn` with the same pattern. Beyond the schema, C5 checks that slugs are unique per array after lowercasing, that every `basedOn` resolves to an existing `base/.ruleset.json` without a cycle, and that quarantine values are stage slugs from the settings. +The schema is closed: an unknown key at the top level or in `publish`, `quarantine`, `commitOptions`, `site`, `update` or `scan` is an error, and a work package that needs a new key adds it to the schema in its own pull request. Required are `templateUrl`, `baseUrl` (empty, or `https://` with a DNS host name and an optional numeric port, no user info, and without a query, a fragment, `.` or `..` segments, quotes, backslashes, control characters or a trailing slash), `publish`, `quarantine` with both keys, `levels` and `stages`. `publish.target` requires its own fields: `dist-repo` needs `repository` (`owner/name`) and `branch`, `azure-blob` needs `storageAccount` and `container`, `gist` needs `gistId`; fields of another target are allowed and ignored. `templateSha` is empty or a 40-character commit sha. A level or stage entry has a `name` matching `^[A-Za-z0-9-]+$` and an optional `description`; a level may have `basedOn` with the same pattern. Beyond the schema, C5 checks that slugs are unique per array after lowercasing, that every `basedOn` resolves to an existing `base/.ruleset.json` without a cycle, and that quarantine values are stage slugs from the settings. `levels` and `stages` (D26, D28, D29): @@ -533,7 +555,10 @@ Every target ends with the same reachability check: `GET` each endpoint, skeleto | Endpoint committed but stale (inputs changed, not regenerated) | Consumers get yesterday's decision. | Regeneration check in Validate; Publish refuses to deploy stale endpoints and keeps the last good site (D42); every writing workflow regenerates in its pull request. | | Override selector typo | Silent no-op. | Selector validation; the ChangeRule PR body shows before and after per endpoint. | | Update PR overwrites an org edit in a system file | Edit lost. | File classes; org decisions live only in `overrides.json`; docs say which files are system files. | -| Scan adds an id the org wanted to see | Rule hidden until adopted. | Policy is explicit per org; the PR lists every new id with its default severity and docs link. | +| Scan adds an id the org wanted to see | Rule hidden until adopted. | Policy is explicit per org; the PR lists every new id with its default severity and docs link; adopting it in a level file releases it on the next run. | +| Someone pushes to the scan branch by hand | The push is replaced by the next run. | The branch is rebuilt from the base on every run with a lease push (D45): the lease covers the window between `ls-remote` and the push only (a push in that window is rejected, an earlier push is replaced), and a base branch that moved while the scan was planning fails the run before the push; the PR body says not to push to the branch. | +| A new package version does not load (a breaking analyzer build) | None: nothing is pushed. | Extraction fails loudly (`failure=extract`); the catalog and the endpoints stay at the last good scan until a fixed engine reads the package. | +| An id the catalog does not know after the first scan | The scan, an update and Validate fail (C7 is an error once `catalog/scan-state.json` exists). | Fix the typo or wait for the package that ships the id; C7 names the file and the id. | | VS Code does not re-fetch a changed remote file | Developers see stale rules until reload. | Documented in WP11: Developer: Reload Window (or saving a change to `app.json`) re-reads the ruleset once the CDN cache has expired (raw 5 minutes, measured; on Pages the WP05 live run saw a changed endpoint and a removed one within about 10 s of the deploy, measured from outside the runner, although Pages sends `max-age=600`, see [reference/publish-targets.md](reference/publish-targets.md)); see [spike e](reference/spikes/e-vscode-refetch.md) for every trigger. | | Token missing or expired | Update, scan and change-rule PRs fail. | Same message pattern as AL-Go; GitHub App recommended. | | A stranger opens a `rulebook-change` issue on a public repository | None at compile time; a workflow run. | Collaborator gate before parsing (D34); the issue is closed with a comment. | @@ -575,13 +600,15 @@ See the open decisions table in [adr/README.md](adr/README.md): O3 engine pinnin | AL project root | `.rulebook/.ruleset.json`, written by the init script or by hand | `.rulebook/ci.ruleset.json`, `.rulebook/default.ruleset.json` | | Published manifest | `/rulebook.json`: levels and stages since WP06 (D43), extended by WP14 | `https://contoso.github.io/rulebook/rulebook.json` | | Init script | `scripts/Get-RulebookSkeletons.ps1` in the engine, served from `raw.githubusercontent.com/ALCops/rulebook-engine/main/scripts/` (`v1` with WP13) | `./Get-RulebookSkeletons.ps1 -BaseUrl https://contoso.github.io/rulebook -Level strict` | -| Catalog | `catalog/diagnostics.json`; `catalog/scan-state.json` reserved for WP08 | | +| Catalog | `catalog/diagnostics.json`; `catalog/scan-state.json` (created by the first scan, schema `rulebook-scan-state.schema.json`) | | | Slug | lowercased `name`, `^[a-z0-9-]+$` | `vNext` becomes `vnext` | | Schema file | `schemas/.schema.json` in the engine, served as `https://raw.githubusercontent.com/ALCops/rulebook-engine/v1/schemas/.schema.json` (live with WP13, #15; 404 until then) | `schemas/rulebook-settings.schema.json` | | Ruleset schema profile | by folder: `base/`, `stages/` delta; `rulesets/` endpoint; `skeletons/` skeleton | `schemas/ruleset.delta.schema.json` | | Change set schema | `schemas/rulebook-changeset.schema.json`, name reserved for WP15 | | | Update branch | `update-rulebook-system-files//` | | | Update PR title | `[@] Update Rulebook System Files from ALCops/rulebook - ` | | +| Scan branch | `scan-diagnostics/`, rebuilt by every run (D45) | `scan-diagnostics/main` | +| Scan PR title | `Scan diagnostics: (