diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json new file mode 100644 index 0000000..c8c6db9 --- /dev/null +++ b/.claude-plugin/marketplace.json @@ -0,0 +1,18 @@ +{ + "name": "stacklet-agent-skills", + "owner": { + "name": "Stacklet" + }, + "metadata": { + "description": "Guardrails that keep coding agents from defining infrastructure that violates your Stacklet policies." + }, + "plugins": [ + { + "name": "stacklet", + "source": "./plugins/stacklet", + "description": "Steer coding agents away from defining infrastructure that violates the policies deployed on your Stacklet platform.", + "version": "0.0.1", + "author": { "name": "Stacklet" } + } + ] +} diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..6931f70 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @stacklet/engineering diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml new file mode 100644 index 0000000..39a2cc2 --- /dev/null +++ b/.github/dependabot.yaml @@ -0,0 +1,26 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "monthly" + cooldown: + default-days: 7 + groups: + github-actions: + patterns: + - "*" + commit-message: + prefix: chore + - package-ecosystem: "pre-commit" + directory: "/" + schedule: + interval: "monthly" + cooldown: + default-days: 7 + groups: + pre-commit: + patterns: + - "*" + commit-message: + prefix: chore diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..82f9489 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,27 @@ +name: "CI" +on: + push: + branches: + - main + pull_request: {} +permissions: {} +jobs: + Lint: + permissions: + contents: read + uses: stacklet/github-tooling/.github/workflows/run-script.yml@48cc95e34ca3ff105998f6b649ddee54bf6da4c2 # v1.1.0 + secrets: + STEP_SECURITY_API_KEY: ${{ secrets.STEP_SECURITY_API_KEY }} + with: + pre-commit-cache: true + run: | + just lint + Tests: + permissions: + contents: read + uses: stacklet/github-tooling/.github/workflows/run-script.yml@48cc95e34ca3ff105998f6b649ddee54bf6da4c2 # v1.1.0 + secrets: + STEP_SECURITY_API_KEY: ${{ secrets.STEP_SECURITY_API_KEY }} + with: + run: | + just test diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7b7c296 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +__pycache__/ +*.pyc +.venv/ +*.tmp diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..a0ee570 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,54 @@ +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: f12be1ebaa5351c1fc76472de98db2c3446c8253 # frozen: v0.16.10 + hooks: + - id: ruff-check + args: [--fix] + - id: ruff-format + - repo: https://github.com/asottile/pyupgrade + rev: 75992aaa40730136014f34227e0135f63fc951b4 # frozen: v3.21.2 + hooks: + - id: pyupgrade + args: [--py311-plus] + - repo: https://github.com/tox-dev/pyproject-fmt + rev: cc0b0f8d4107cfe80d6571644e639b05aee29128 # frozen: v2.28.1 + hooks: + - id: pyproject-fmt + - repo: https://github.com/abravalheri/validate-pyproject + rev: 17e12779811926f000bdfa1795f9e3735dd71ac3 # frozen: 0.26 + hooks: + - id: validate-pyproject + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: 3e8a8703264a2f4a69428a0aa4dcb512790b2c8c # frozen: v6.0.0 + hooks: + - id: check-json + - id: check-toml + - id: check-yaml + - id: trailing-whitespace + - id: end-of-file-fixer + - repo: https://github.com/rhysd/actionlint + rev: 914e7df21a07ef503a81201c76d2b11c789d3fca # frozen: v1.7.12 + hooks: + - id: actionlint + - repo: https://github.com/zizmorcore/zizmor-pre-commit + rev: fa412071e4f5d44d44f9e365f4676f9df92456a2 # frozen: v1.30.1 + hooks: + - id: zizmor + - repo: https://github.com/codespell-project/codespell + rev: 57b21406f092110c18776e39b0bda50d37c945c8 # frozen: v2.4.3 + hooks: + - id: codespell + additional_dependencies: [tomli] + - repo: https://github.com/google/yamlfmt + rev: 21ca5323a9c87ee37a434e0ca908efc0a89daa07 # frozen: v0.21.0 + hooks: + - id: yamlfmt + exclude: ^content/ + - repo: local + hooks: + - id: skills-ref + name: validate agent skill + language: unsupported + entry: uv run agentskills validate plugins/stacklet/skills/guardrails + files: ^plugins/stacklet/skills/guardrails/ + pass_filenames: false diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..6bdd734 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,99 @@ +# CLAUDE.md + +Guidance for Claude Code when working in this repository. + +## What this is + +Coding-agent guardrails derived from a Stacklet platform's policy posture: steer coding agents +away from defining infrastructure that violates the policies deployed by that platform. The +product thinking is in "Agent Skills User Stories.md" (not in this repo; ask the user for it). +Its key points: + +- Guardrails are **policies**; deployment scopes are **accounts**. A set of accounts is an + **account group**, a set of policies a **policy collection**; one of each linked is a + **binding**. The totality of bindings is the platform's **posture**. +- Implementation details in that doc are evidence of feasibility, not requirements. +- Perfection is not the goal: a skill that merely halves runtime violations is valuable. +- Never dogmatic enforcement: users may have good reasons to violate a guardrail. Present + guardrails as *facts*, not orders. + +Packaged as a Claude Code plugin marketplace: `.claude-plugin/marketplace.json` -> +`plugins/stacklet` (skill at `plugins/stacklet/skills/guardrails`). The plugin has no +`plugin.json`, deliberately: Claude Code ignores a marketplace entry's `mcpServers` when the plugin +ships one, and each deployment serves its own marketplace entry, which adds its MCP server. Still +prototype-quality code. `docs/design.md` says why it is built as it is; read it first. + +## Data comes from the Stacklet MCP server + +The scripts never talk to Stacklet: an agent runs the queries in +`plugins/stacklet/skills/guardrails/platform/` through the Stacklet MCP server's tools and saves the +results for `ingest.py`. Use `platform_graphql_get_types` when unsure of the API. Dataset exports +aren't used (their download URLs need credentials the MCP server doesn't give out). The flow and +its measured limits are in docs/design.md, "Data path". + +## Architecture decisions (settled with the user; don't re-litigate) + +Each is argued in docs/design.md, under the section named. + +- **Two caches** ("Two caches, deliberately"): `posture.db` is derived and rebuilt on each refresh; + `guardrails.db` is persistent (policy versions, distillations, account and membership + snapshots), never pruned, and started afresh rather than migrated when its layout changes. +- **A membership is checked only under the posture refresh it was fetched under**: the platform has + no change marker for a collection. +- **Deployments are named by their console URL's host**, each with its own cache directory, and + every script but `deployments.py` is told which ("Deployments"): tenant isolation is a + requirement. +- **Distillation is per (collection, resource type)**, keyed by the exact policy-version set, so + prompt or model changes never invalidate it; bundles group policies by what they select, and the + model never sees a description, an action's message or a `schedule` ("Distillation"). +- **Scope = a posture assessment plus recorded facts**, not counting rules: a model, isolated and + shared by every repo on the deployment, judges the bindings once; the committed scope file holds + facts with reasons; scope is a union, and excludes always win and are always surfaced ("Scope + model"). The `system` flag and group `priority` are deliberately ignored. +- **Policy names are unique** (`:`, enforced in-platform), so + `name@version` refs are trusted. +- Policy content is `source`, the policy as JSON (builtin `json`); models and agents read it as + JSON too. The scripts are stdlib-only (no PyYAML: the scope file is JSON). +- SKILL.md is organised around a goal (guardrails for a type in context before code for it is + written), with the scripts' notes ending in tags it maps to handling ("The skill's shape"). +- Accounts are found by the agent: which are in play for a change is a judgement, so the scripts + read neither the code nor the local environment. +- Server-side distillation is rejected for now (don't weld it onto the platform data model). + +## Platform quirks learned the hard way + +- A group with `dynamicFilter == ""` matches all *active* accounts in its provider (its + `member_count` is exact). `accountSummary` counts inactive accounts too, so it is never a + provider's total; `accounts(...)` is active-only unless filtered on `active`. Some providers lack + such a group: see `platform/active_accounts.graphql`. +- Bindings are never undeployed, so a null `lastDeployed` means never deployed: ingest leaves them + out. +- Nested `pageInfo { total }` (e.g. `accountMappings(first: 1) { pageInfo { total } }`) works and + is cheap. A connection's `first` defaults to 10 and isn't capped. +- Model calls (`scripts/_inference.py`) run `claude -p` from an empty temp dir with user settings + only: otherwise the nested call loads the *repo's* CLAUDE.md (verified: a planted instruction was + obeyed), and its output is cached for every repo. The binary is `$CLAUDE_CODE_EXECPATH` when set, + else `claude`, run as given with no `which` (on Windows that would search the repo first). Tests + unset `$CLAUDE_CODE_EXECPATH` so they only reach their stub. Overriding the command or model: + `_inference.py`'s docstring and `references/inference.md`. + +## Known gaps + +In docs/design.md ("Known gaps"); what the evals don't show is in tests/evals/README.md. What a +session most needs: a cache is only as fresh as the last session that synced it, and the skill's +evals see policy actions only in fixtures (real shapes are in the distillation's evals). + +## Working conventions + +- Run scripts with plain `python3` (stdlib only, Python 3.11+, also under a C locale; keep + it that way: no third-party deps, explicit `encoding="utf-8"`, no syntax newer than 3.11). +- Tests: `just test` (pytest via uv, args passed through; no model calls). Lint: `just lint` + (pre-commit via uv). Evals cost money, so only when asked (tests/evals/README.md): + `just evals OUT --yes` then `tests/evals/grade_skill.py OUT/iter`; `just assess-evals OUT --yes` + then `tests/evals/grade_assess.py OUT`; `just distill-evals OUT --yes` then + `tests/evals/grade_distill.py OUT`. What each prompt change measured is in its commit's message. + Postures and policies from the wild stay outside the repo; only ones with every name invented go + in. +- Don't commit unless asked. Don't delete the user's files without asking. +- Keep new script code in the style of the prototype: small scripts, stdlib + sqlite, comments that + explain *why*. diff --git a/README.md b/README.md index bd9a14f..5da61cc 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,21 @@ -# agent-skills -Agent skills for working with Stacklet +# Agent skills for working with Stacklet + +Coding-agent guardrails derived from a Stacklet platform's policy posture: steer coding agents +away from defining infrastructure that violates deployed policies. + +## Install (Claude Code) + +``` +/plugin marketplace add stacklet/agent-skills +/plugin install stacklet@stacklet-agent-skills +``` + +Requires the Stacklet MCP server and Python 3.11+ (the scripts use only the standard library). + +The skill runs its bundled scripts with `python3`, so Claude Code asks you to approve the first +call. (Don't expect the skill to pre-approve this itself: a skill that declares `allowed-tools` +needs an approval just to load, and is silently skipped in headless `claude -p` runs.) + +To stop the prompts, add a permission rule for the skill's scripts in your own settings. + +Prototype quality; see [`docs/design.md`](docs/design.md). diff --git a/docs/design.md b/docs/design.md new file mode 100644 index 0000000..1f589b7 --- /dev/null +++ b/docs/design.md @@ -0,0 +1,227 @@ +# Guardrails skill: design + +Why the skill is built the way it is, for people working on it +(`plugins/stacklet/skills/guardrails/`; paths below are relative to it). Not shipped with the +plugin. What each script does in detail is in its docstring; what the distillation is told is its +prompt (`distill.py`); what each measured change found is in its commit message. + +## Layout + +| file | role | +|---|---| +| `platform/*.graphql` | the queries an agent runs through the Stacklet MCP server | +| `deployments.py` | chooses the session's deployment (the only script without a DEPLOYMENT argument) | +| `ingest.py` | stores what the agent fetched: the posture (rebuilt and swapped), collections, accounts | +| `posture.py` | the posture digest the assessment reads, and what is due a fetch | +| `assess.py` | the posture assessment: which bound collections apply, and the questions settling the rest | +| `scope.py` | the repo's committed scope file (`.stacklet/guardrails/scope.json`): `decide`, `check`, `query` | +| `distill.py` | `index`, `show`, `source`: guardrails for a resource type, distilled and cached | +| `_cache.py`, `_inference.py` | one deployment's cache and how it is opened; model calls, isolated | +| `references/` | what SKILL.md's main path leaves out; `sync.md` is the sync, run by a background sub-agent | + +## Data path + +The scripts never talk to Stacklet: an agent fetches through the Stacklet MCP server and saves what +it gets for the scripts to ingest. Everything comes through GraphQL; dataset exports aren't used, +since their download URLs need credentials the MCP server doesn't give out. Two measured facts make +this work: Claude Code saves an MCP result over 25,000 tokens (about 50 policies) to a file, which +the agent copies, so content is never retyped; and a query naming whole collections is short to +write. + +- The **posture** (`posture.graphql`, one unpaginated call) gives each collection's size, not its + membership: that is large (one collection has had about 1,600 policies) and a repo needs little of + it. Bindings with a null `lastDeployed` are left out: bindings are never undeployed, so these + never were deployed. +- **Collections** are fetched lazily, membership and content together, what the repo's scope needs + first: its in-scope collections, and those an unanswered question names, so which resource types a + question bears on is known before anyone is asked. A small batch is filled out with other + collections due a fetch so its result comes back as a file too (warming the cache for other + repos). +- **Size has a ceiling:** 6 MB in one result worked, but at about 12 MB Claude Code dropped the MCP + connection, so `sourceYAML` (the same content again) isn't fetched; the largest collection seen is + about 3 MB. The platform doesn't cap `first:`, so `policyMappings(first: 10000)` returns a whole + collection in one page (verified in the platform's pager); ingest still refuses a truncated one. + Should a deployment outgrow that, `policyMappings` pages by keyset on policy name: sequential, so + not built until needed. +- **A refresh is best effort.** A failed one leaves the old cache, so guardrails survive the MCP + server being down; nothing refreshes a cache between sessions (that would need a direct GraphQL + client). + +## Deployments + +A user may have several Stacklet deployments, each with its own MCP server, and tenant isolation is +a requirement: every script but `deployments.py` takes the deployment as its first argument, and +nothing guesses it. A deployment is named by its console URL's host, which the platform takes from +its own configuration, never from the request, so a deployment has one name however it is reached; +it means the same on every machine, unlike an MCP server's name. + +`deployments.py` chooses once per session (SKILL.md step 1): + +0. None while the user has turned guardrails off for the repo (recorded in the scope file): said in + a line each session, never asked again, until they ask for guardrails. +1. The scope file's deployment when it records one, used through its server, else its cache; if + neither is available, none is chosen, never a different one. It is read even from an otherwise + invalid file: a broken file must not let a session drift to another tenant. +2. Else the only server; with no server, the only usable cache. +3. Else the user's answer, recorded in the scope file, so each repo is asked at most once. + +Guards against mixing tenants: + +- every response carries the console, and ingest refuses another deployment's; +- the posture db records its deployment, and `Cache.open` refuses one built for another; +- `decide` won't write facts against another deployment than the scope file's, and `check` and + `distill.py` ignore such a file. + +A renamed console orphans its cache, and its repos' scope files need editing by hand (no tool until +it happens). + +## Two caches, deliberately + +Both live in `~/.stacklet/guardrail-cache//`. + +- **`posture.db` is derived:** rebuilt from scratch and swapped in atomically on each refresh. +- **`guardrails.db` is persistent:** never touched by a refresh, never pruned. It holds what is + costly or slow to get again: immutable policy versions (ingest only adds; readers see the bound + versions, while an old `name@version` still resolves), distillations, and one snapshot per + looked-up account and per collection membership. No migrations while this is early: a db of an + older layout (`GUARDRAILS_LAYOUT`) is deleted and started afresh, since everything in it can be + fetched or made again; one of a newer layout is left alone and the scripts ask for a skill update, + or two versions would delete each other's db on every open. + +**Memberships and freshness.** A collection is *loaded* when its membership is fetched and every +version in it has content (or the posture says it is empty): missing data always reads as "not +loaded", never as "no guardrails". It is *checked* when it was fetched under the current posture +refresh: the platform has no change marker for a collection, and a count misses the commonest change +(a version bump), so every refresh makes every collection due again. + +The query carries the refresh it was made under (a `posture_` alias), so another session +refreshing between query and ingest changes nothing, and an older posture's fetch never replaces a +current one. A membership is the collection's current one, which can be ahead of what a binding that +doesn't auto-deploy has deployed. Disabled policies are deliberately not a concept: a membership is +what the platform returns. + +## Distillation + +`show` distils each loaded in-scope collection's policies of one resource type separately, and +prints their sections. Per collection, not per type: collections load lazily, so a per-type +distillation would be invalidated by every new one, and could carry an out-of-scope framework into +in-scope bullets. A policy in several collections repeats in each (no de-duplication). Always +scoped: an unscoped view would present collections nobody asked about as if they applied. + +- **Key:** collection, resource type, and a hash of the exact uuid+version set of that collection's + policies of that type. Any distillation of the same policies is good enough, whatever prompt or + model made it; `show --redistill` redoes one. Each is written in its own short transaction, never + held across a model call. +- **Bundles group policies by what they select** (filters, `query`, `conditions`), so the model sees + each condition once; actions, severity, compliance and mode stay per policy. +- **What the model never sees:** a policy's description (often stale or looser than its filters, and + the evals lost nothing without it), what an action's message says or where it goes + (`MESSAGE_FIELDS`), and a policy's `schedule` (Stacklet ignores it: a binding's executions decide + when a policy runs). None of these says what a resource must be, and free text written for people + is where stray instructions would be. +- **What a bullet says** is set by the prompt, each rule adopted only on a clear measured gain (its + commit says by how much). The distillation's evals (`tests/evals/distill/`) keep measuring them. + Faithfulness beyond that is not checked: validation (below) checks that bullets cite real + policies, not that their prose is right, and a second model checking the first was judged not + worth it while the evals show few misses. +- **Validation:** a `name@version` ref must name one of the bundle's policies (one at a version the + bundle lacks is made up, and removed); only each bullet's first line is kept, and only if it cites + a policy of the type; the only heading kept is the runtime-only one. Output with no bullet left is + asked for once more, then rejected, never cached. + +## Model calls + +Distillation (haiku) and the posture assessment (sonnet) run `claude -p` isolated from the repo: +from an empty temp dir, with no tools, no MCP servers and user settings only. Otherwise the nested +call would load the repo's CLAUDE.md (verified: a planted instruction was obeyed), and what it +writes is cached for every repo on the deployment. `$STACKLET_GUARDRAILS_INFERENCE_CMD` replaces the +command (it must isolate itself too), and `show` marks what a custom command made. + +## The skill's shape + +SKILL.md is written around one goal: before the agent writes code defining a resource type, the +guardrails for that type in the repo's scope are in its context (`distill.py show`), from a cache +complete for that scope and current. It was first a procedure (deployment, sync, scope, consult) +whose every condition was a step everyone walked through, and the steps tangled. Now the main path +is short, and the scripts say what keeps the goal from being met: every note `index` and `show` +print ends in a tag (`[sync: refresh]`, `[sync: fetch]`, `[question]`, `[tell the user]`) that +SKILL.md maps to its handling, with details in `references/`. + +- The agent waits only when guardrails may be *missing* for the type it is about to write + (collections not loaded, accounts not looked up, no assessment). Old data is used while the sync + checks it, and `show` runs again when the sync reports. +- Which questions bear on a type is the scripts' to say: `show` names those whose collections have + guardrails for it. +- The sync is mechanical, so it runs as a background sub-agent where one can start, ending with a + fixed-shape report. Only it makes the assessment, so nothing makes a second beside it. A sub-agent + can't answer permission prompts: one denied a tool says so, and the main agent syncs itself. +- A new posture with changed bindings has no assessment until a model call ends (minutes); the + newest earlier one stands in, its questions held back, since its judgements of the collections + still bound beat the all-accounts defaults. + +## Scope model + +A repo's scope file records **facts**, each with its reason: answers to the assessment's questions, +the accounts the code deploys to, and include/exclude decisions on collections. Scope is derived +from them against the live posture on every read, so nothing derived goes stale. Everything is +identified by uuid, question id or `PROVIDER:KEY`, with names alongside for people. + +**Scope is a union**, because the same code often deploys into many accounts and the goal is every +guardrail that applies anywhere it lands; nothing is ruled out by omission, and we never ask for a +complete list of accounts: + + in scope = assessment's applies + what yes-answers add - what they remove + + what looked-up accounts reach + includes - excludes + +**The posture assessment** (`assess.py`) replaces counting rules (coverage thresholds, group sizes): +postures in the wild roll a collection out over a dozen tag-batch groups, each small but together +everything minus an opt-out tag, so what a binding means depends on reading groups together, which a +model does well and rules don't. + +- Rules decide the certain cases: bound to an all-accounts group, it applies; no policies, ignored. +- A model reads the posture digest (no policy content) and judges every other bound collection: + `applies`, `ignore` (reached only through specific accounts, so only a lookup brings it in), or + `ask` (a part of the estate a developer can know their code by: an org, an OU, an environment + tag), with the fewest yes/no questions settling the `ask` ones. A question names groups, and what + a yes changes is derived from the bindings, never the model's list. +- It depends on the posture alone, so it is made once per set of bindings and shared by every repo + on the deployment, by the isolated model call, never the session agent: repo content must not + steer a judgement every repo reuses. It is validated before caching; a well-formed but wrong + judgement can't be caught that way. +- With no assessment at all, only all-accounts collections apply, and the scripts say so. +- The digest's shares use a group holding all of a provider's active accounts, never + `accountSummary`, which counts inactive ones too. + +**Account facts.** A looked-up account brings in every collection bound to a group it is in, even +one the assessment ignores or an answer removes; account discovery keeps a group per OU path, so +hierarchy is just group membership. An account not looked up adds nothing and is reported; a +snapshot over a day old still counts until a sync refreshes it. Which accounts are in play is the +agent's judgement about the change (a local profile or state file may be another project's), so +the scripts read neither. + +**Answers** count only while the assessment asks the same question in the same words. An include +says only that a collection applies, not where the code deploys, so nothing is inferred from it. + +**Excludes always win**, and they and what a yes-answer removes are reported on every `check`, +`index` and `show`. Nothing stops or reviews them: the scope file is committed, so it is visible and +reviewable in the repo, and the team's judgement is trusted (guardrails are facts, not orders). +Group priority and the `system` flag are deliberately ignored. + +## Known gaps + +- Nothing has run end to end on a live deployment with acting policies: policies' actions, `query` + and `conditions` as found in the wild are in the distillation's evals (24-38), and the assessment + was checked offline on 20 postures from the wild (8 such postures, names invented, are its evals), + but the skill's evals see actions only in fixtures. +- Policies' actions reach agents as names (`notify`, `tag`, `delete`, a `mark-for-op`'s deferred + op), not classified: agents read those better than a destructive/not flag. +- A cached distillation or assessment is model-written text served to every repo on the + deployment; isolation and validation limit what it can carry, not whether it is right. +- What the agent does when code knowingly keeps a violation (a comment in the code saying why, and + not raising it again) is undecided. +- Windows is untested: `python3` may not exist (SKILL.md offers `python` or `py -3`); encoding is + handled, paths aren't checked. +- Triggering is unmeasured in a session crowded with skills. In isolated runs the skill was the + first tool call on 36/36 should-trigger queries, and a CLAUDE.md snippet only added false + positives, so none is shipped. +- What the evals don't show is in `tests/evals/README.md`. diff --git a/justfile b/justfile new file mode 100644 index 0000000..b4588c4 --- /dev/null +++ b/justfile @@ -0,0 +1,19 @@ +# lint tree +lint: + uv run pre-commit run --all-files + +# run tests +test *args: + uv run pytest {{args}} + +# run the evals: agent sessions that cost money, so it prints the plan unless given --yes +evals out *args: + uv run python3 tests/evals/run_skill.py {{out}} {{args}} + +# run the posture assessment's evals: model calls that cost money, so it prints the plan unless given --yes +assess-evals out *args: + uv run python3 tests/evals/run_assess.py {{out}} {{args}} + +# run the distillation's evals: model calls that cost money, so it prints the plan unless given --yes +distill-evals out *args: + uv run python3 tests/evals/run_distill.py {{out}} {{args}} diff --git a/mise.lock b/mise.lock new file mode 100644 index 0000000..dac4a10 --- /dev/null +++ b/mise.lock @@ -0,0 +1,69 @@ +# @generated - this file is auto-generated by `mise lock` https://mise.en.dev/dev-tools/mise-lock.html + +[[tools.just]] +version = "1.59.0" +backend = "aqua:casey/just" + +[tools.just."platforms.linux-arm64"] +checksum = "sha256:23871b1702bfd550fb5f901778fd4290ecf2ee034a45a889fbe96d89b188086d" +url = "https://github.com/casey/just/releases/download/1.59.0/just-1.59.0-aarch64-unknown-linux-musl.tar.gz" + +[tools.just."platforms.linux-x64"] +checksum = "sha256:409044617b6f06517c1035e55ca38c8c856ff8b650e41bc2aa1ef82d5903a23e" +url = "https://github.com/casey/just/releases/download/1.59.0/just-1.59.0-x86_64-unknown-linux-musl.tar.gz" + +[tools.just."platforms.macos-arm64"] +checksum = "sha256:6574cea774ed4cae12e24b6aa11857de33e80dab4667cf628fd5840df96a2a9d" +url = "https://github.com/casey/just/releases/download/1.59.0/just-1.59.0-aarch64-apple-darwin.tar.gz" + +[tools.just."platforms.macos-x64"] +checksum = "sha256:036ead9df9bc31dbae9934ae3b4236ed124d0bafcffd33439460ae084d5c9cb7" +url = "https://github.com/casey/just/releases/download/1.59.0/just-1.59.0-x86_64-apple-darwin.tar.gz" + +[[tools.python]] +version = "3.14.10" +backend = "core:python" + +[tools.python."platforms.linux-arm64"] +checksum = "sha256:90f48337bc3dd4ae6eda81e001e4c981236d403b6b5a07778bbdf94ee423dc00" +url = "https://github.com/astral-sh/python-build-standalone/releases/download/20261026/cpython-3.14.10+20261026-aarch64-unknown-linux-gnu-install_only_stripped.tar.gz" +provenance = "github-attestations" + +[tools.python."platforms.linux-x64"] +checksum = "sha256:4d3c16c59ed0802a3d1d34d8bce504b15ceeaff39d62dadecc6a9fa803412aed" +url = "https://github.com/astral-sh/python-build-standalone/releases/download/20261026/cpython-3.14.10+20261026-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz" +provenance = "github-attestations" + +[tools.python."platforms.macos-arm64"] +checksum = "sha256:95eb8c87d16af37c54e9765e31d8c56c0d8ffa9968ea0f704f73a9b83f84e9f9" +url = "https://github.com/astral-sh/python-build-standalone/releases/download/20261026/cpython-3.14.10+20261026-aarch64-apple-darwin-install_only_stripped.tar.gz" +provenance = "github-attestations" + +[tools.python."platforms.macos-x64"] +checksum = "sha256:2fd8becd48cf0ff5f0a15df3c0db7cae7aa708dcaacf901bd30d6cdd6229370b" +url = "https://github.com/astral-sh/python-build-standalone/releases/download/20261026/cpython-3.14.10+20261026-x86_64-apple-darwin-install_only_stripped.tar.gz" +provenance = "github-attestations" + +[[tools.uv]] +version = "0.13.6" +backend = "aqua:astral-sh/uv" + +[tools.uv."platforms.linux-arm64"] +checksum = "sha256:819692e404d23e5aae6bc44c0e260f1f5492fd1910ab4693b2c650f926dbe800" +url = "https://github.com/astral-sh/uv/releases/download/0.13.6/uv-aarch64-unknown-linux-musl.tar.gz" +provenance = "github-attestations" + +[tools.uv."platforms.linux-x64"] +checksum = "sha256:5d61f26b6c7213779eda90edc6971ea6b0b796bfb8d49a574c8ad69b59838548" +url = "https://github.com/astral-sh/uv/releases/download/0.13.6/uv-x86_64-unknown-linux-musl.tar.gz" +provenance = "github-attestations" + +[tools.uv."platforms.macos-arm64"] +checksum = "sha256:60fbd5c8a1ed681845eea28d9d622fcde03c51e91a43465b64a1e17db1ed3472" +url = "https://github.com/astral-sh/uv/releases/download/0.13.6/uv-aarch64-apple-darwin.tar.gz" +provenance = "github-attestations" + +[tools.uv."platforms.macos-x64"] +checksum = "sha256:63ba17fdda55ad1a230d4b37fc6017ffa10c23688c83818bd3bfdf914c652a02" +url = "https://github.com/astral-sh/uv/releases/download/0.13.6/uv-x86_64-apple-darwin.tar.gz" +provenance = "github-attestations" diff --git a/mise.toml b/mise.toml new file mode 100644 index 0000000..0fa8056 --- /dev/null +++ b/mise.toml @@ -0,0 +1,8 @@ +[tools] +just = "1.59.0" +python = "3.14.10" +uv = "0.13.6" + +[settings] +lockfile = true +lockfile_platforms = ["linux-x64", "linux-arm64", "macos-x64", "macos-arm64"] diff --git a/plugins/stacklet/skills/guardrails/SKILL.md b/plugins/stacklet/skills/guardrails/SKILL.md new file mode 100644 index 0000000..bd213c4 --- /dev/null +++ b/plugins/stacklet/skills/guardrails/SKILL.md @@ -0,0 +1,260 @@ +--- +name: guardrails +description: Check the organisation's Stacklet cloud-governance policies before writing or changing code that defines or creates cloud resources. Use whenever you are about to create or modify cloud resources — in infrastructure-as-code (Terraform/OpenTofu aws_*/google_*/azurerm_*, CloudFormation/SAM, CDK, Pulumi, Bicep, Crossplane, Kubernetes manifests for cloud-backed resources) or at runtime through SDK/CLI calls (boto3 create_bucket/create_db_instance/run_instances, aws/gcloud/az CLI scripts, CDK custom resources) — for S3 buckets, databases, disks, security groups, IAM, queues, VMs and the like, even if the user never mentions Stacklet, policies, compliance or guardrails. Surfaces which deployed policies apply to those resource types so the code avoids violations (public buckets, unencrypted storage, open ingress, missing tags, etc.) before it ever reaches an account. Not for application code that merely uses existing cloud resources (uploading to a bucket, querying a table), or for questions about Stacklet itself. +--- + +# Stacklet guardrails + +The user's organisation deploys policies to its cloud accounts through Stacklet; resources that +violate them get flagged (or remediated) after deployment. Catching that while writing the +infrastructure code is far cheaper than finding out later, so before defining or changing a +cloud resource, look up which guardrails apply to that resource type and write code that +complies. + +Guardrails are **facts about the environment, not orders.** Tell the user what applies and what +you did about it. If they have a good reason to violate one, say what the consequence will be +and do what they ask. Never refuse or stall a task because of a guardrail, and never block +because this skill's data is unavailable: say so in a line and carry on. + +## The goal + +Before you write or change code that defines a cloud resource, the guardrails that apply to its +resource type in this repo are in your context: `distill.py show `, read from a +cache holding everything in this repo's scope as the platform has it now. Everything below serves +that. The cache is filled through the Stacklet MCP server, which is slow, so start filling it early +and in the background; whatever still keeps it short of complete and current is said by the scripts +and handled as below, never by stalling the task. + +## Running the scripts + +``` +python3 ${CLAUDE_SKILL_DIR}/scripts/