From 68ab843b2a17c2fcb22cefa7ca4d0f418f0b9414 Mon Sep 17 00:00:00 2001 From: Abhijit Date: Fri, 9 Oct 2026 12:16:27 +0530 Subject: [PATCH 1/9] Answer known questions locally; gate egress; approve every key use MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Local router: every prompt is classified on the operator's machine first. - an intent catalog (YAML, built-ins plus ~/.cometcli/intents) answers known questions — jailed validators, validator count, peers, sync, missed blocks, consensus, proposals, pending votes, rewards, balance, upgrade plan, slashing params — with read-only tools and a fixed template: no model call; repeats within 30s come from a cache - open-ended prompts, actions, long or multi-line prompts and anything the catalog doesn't clearly match go to the model; a failed lookup too - /incident triages locally; when one known case is clearly the root and has a playbook, cometcli runs it (changes and transactions still ask), records the incident and reports. A failed step hands over to the model with the triage and the steps done; a declined step stops - playbooks for node down, jailed for downtime, halt-height left set and container memory limit; kb show lists them - /llm forces the model, /route explains the routing, /cost counts local answers; agent.router: off disables it Egress gate, in front of every provider call: - key material (PEM/OpenSSH keys, consensus keys, BIP-39 mnemonics, keystores, file-keyring entries, private-key hex, API and bot tokens) withholds the whole message, in history too; a prompt with a mnemonic isn't sent at all - strict egress (default on validators): bash/read/grep/node.logs/fleet output reaches the model only as a local summary — identifiers masked, repeats folded, errors first; the UI and audit keep the raw output - keystore paths join the protected paths for the shell and file tools Key access: - every signature is approved on its own with the account, which key and where, the decoded messages, fee and the agent's stated reason; never auto-approved, audited as a key event; Telegram approvals of a key use expire in 5 minutes - the local keyring no longer signs during gas simulation — the key is untouched until the operator approves Co-Authored-By: Claude Opus 5.5 --- README.md | 16 + docs/COMMANDS.md | 40 +- docs/EXTENDING.md | 51 +++ internal/agent/commands.go | 34 +- internal/agent/compact.go | 1 + internal/agent/egress.go | 78 ++++ internal/agent/egress_test.go | 80 ++++ internal/agent/incident.go | 191 +++++++++ internal/agent/incident_test.go | 126 ++++++ internal/agent/loop.go | 35 +- internal/agent/loop_test.go | 11 + internal/agent/phase2_test.go | 1 + internal/agent/route.go | 166 ++++++++ internal/agent/route_test.go | 124 ++++++ internal/agent/session_test.go | 9 +- internal/audit/audit.go | 2 + internal/config/config.go | 8 + internal/egress/egress.go | 105 +++++ internal/egress/egress_test.go | 122 ++++++ internal/egress/summary.go | 183 +++++++++ internal/kb/cases/host.yaml | 6 + internal/kb/cases/sync.yaml | 7 + internal/kb/cases/upgrade.yaml | 6 + internal/kb/cases/validator.yaml | 6 + internal/kb/kb.go | 47 +++ internal/router/intents/chain.yaml | 84 ++++ internal/router/intents/node.yaml | 134 +++++++ internal/router/router.go | 369 ++++++++++++++++++ internal/router/router_test.go | 107 +++++ internal/testutil/fakenode/fakenode.go | 24 ++ internal/toolkit/context.go | 3 + internal/toolkit/keyuse.go | 75 ++++ internal/tools/common/common.go | 4 +- internal/tools/common/keyuse_test.go | 64 +++ internal/tools/fs/fs.go | 2 +- internal/tools/registry_test.go | 40 ++ internal/tools/shell/classify.go | 2 +- internal/tools/triage/tool.go | 13 +- internal/tui/chatui/chatui.go | 4 + .../tui/chatui/testdata/slash_menu.golden | 2 +- internal/tx/signer.go | 16 +- internal/watch/watch.go | 10 + internal/watch/watch_test.go | 20 + 43 files changed, 2412 insertions(+), 16 deletions(-) create mode 100644 internal/agent/egress.go create mode 100644 internal/agent/egress_test.go create mode 100644 internal/agent/incident.go create mode 100644 internal/agent/incident_test.go create mode 100644 internal/agent/route.go create mode 100644 internal/agent/route_test.go create mode 100644 internal/egress/egress.go create mode 100644 internal/egress/egress_test.go create mode 100644 internal/egress/summary.go create mode 100644 internal/router/intents/chain.yaml create mode 100644 internal/router/intents/node.yaml create mode 100644 internal/router/router.go create mode 100644 internal/router/router_test.go create mode 100644 internal/toolkit/keyuse.go create mode 100644 internal/tools/common/keyuse_test.go diff --git a/README.md b/README.md index 1d2f0a6..cc63e40 100644 --- a/README.md +++ b/README.md @@ -323,6 +323,21 @@ agent. operator-only. - **Redaction** — keys, mnemonics, JWTs, bearer tokens and URL credentials are scrubbed from everything entering and leaving the agent (tx hashes stay visible). +- **Egress gate** — the last check before any request reaches a model provider: + - Anything still carrying key material is withheld whole, never partly sent. This + covers PEM/OpenSSH keys, consensus keys, mnemonics, keystores, keyring entries and + API tokens. + - A message from you containing a mnemonic isn't sent at all. + - On validators (`agent.egress: strict`, the default) the model never sees raw shell, + file or log output. It gets a local summary instead: identifiers masked, repeats + folded. +- **Key access** — every signature is approved on its own. The approval shows the + account, which key and where it lives, the decoded messages, the fee and why. It is + never auto-approved, in any mode. Remote (Telegram) approvals of a key use expire in + 5 minutes. +- **Local answers** — known questions ("is anyone jailed?", "how many peers?") and + known incidents with a playbook are answered by cometcli itself with no model call. + `/llm` asks the model anyway; `/route` shows why a prompt went where it did. - **Audit** — every tool call, shell command, prompt and transaction lands in `~/.cometcli/audit/YYYY-MM-DD.jsonl` (`cometcli audit` to tail it). - **Bounded execution** — every tool runs under a deadline; long requests pause @@ -338,6 +353,7 @@ credentials # API keys saved by `config set-key` (0600) settings.json # permission rules, hooks, MCP servers (also per project: .cometcli/) COMET.md # memory the agent always reads (also per project) kb/ # your own knowledge-base cases +intents/ # your own known questions (answered without a model call) history/ # triage signal history per profile (30 days) incidents/ # incident records and postmortems per profile keys/ # ops keyring (file backend) diff --git a/docs/COMMANDS.md b/docs/COMMANDS.md index df4f32b..d60b06d 100644 --- a/docs/COMMANDS.md +++ b/docs/COMMANDS.md @@ -169,7 +169,17 @@ stop if tombstoned → fix the root cause → wait for sync → wait out the jai ### Signing transactions -Two signers; when both can sign, you pick one first, then approve the tx (the approval shows which signer will sign): +Every signature is a **key access** and is approved on its own. +- **What the approval shows**: the account, which key and where it lives, the decoded + messages, the fee, chain and sequence, and the agent's stated reason. +- **No shortcuts**: no rule, mode, autopilot or earlier approval covers it, and the next + signature asks again. +- **Gas**: estimated with an unsigned transaction, so the key isn't touched before you + approve. +- **Remote approvals**: in `watch`, a Telegram approval of a key use expires after + 5 minutes. + +Two signers; when both can sign, you pick one first, then approve the key use: - **cometcli's keyring** — `signer.key` (import with `cometcli keys add --recover`). - **the node container's own keyring** — the key never leaves the node: cometcli builds @@ -545,6 +555,32 @@ session started in this directory, `-r/--resume ` a specific one (an id pref `--model`, `--effort low|medium|high|xhigh|max`, `--max-tokens N`. Every conversation is saved after each turn to `~/.cometcli/sessions/` (0600, already redacted). +### Local answers and the model + +Every prompt is classified on your machine first. + +| What you ask | Who answers | +|---|---| +| A known question: "is anyone jailed?", "validator count", "how many peers", "is my node synced", "missed blocks", "open proposals", "do I need to vote", "rewards", "balance", "upgrade plan", "slashing params" | cometcli runs the read-only tools and answers from a fixed template. No model call, no tokens. Repeats within 30s come from a cache. | +| `/incident` where triage finds one clear root case that has a playbook | cometcli runs the playbook itself; changes and transactions still ask you. Today that's node down, jailed for downtime, halt-height left set, and container memory limit. A failed step hands over to the model with everything done so far. | +| Anything else: why/how questions, actions, unknown problems | The model. For `/incident` it starts from the triage cometcli already ran. | + +`/llm ` asks the model anyway, `/route` explains the last decision, and +`/cost` counts local answers next to tokens. To turn routing off, set +`agent.router: off`. Add your own questions in `~/.cometcli/intents/*.yaml` (see +[EXTENDING.md](EXTENDING.md#known-questions)). + +**What the model sees.** On validator profiles, `agent.egress: strict` is the default: +- Output of `bash`, `read`, `grep`, `node.logs` and `fleet.exec` reaches the model only + as a local summary. +- Addresses, IPs, hashes, hostnames and blobs are masked, and repeated lines are folded + with a count. +- Errors are kept first when output is long. +- Your screen and the audit log still show the raw output. + +RPC/sentry profiles and general mode default to `filtered` (raw output after redaction). +In every mode, anything carrying key material is withheld whole. + ### Agent session commands (TUI, REPL, web) | Command | Effect | @@ -615,6 +651,8 @@ agent: autopilot: [local-change] # optional; on-chain is rejected redact_hosts: [val.internal] # masked before any text reaches the LLM redact_endpoints: true # also mask this profile's endpoint/SSH hosts + egress: "" # strict | filtered — default strict on validators (raw output only as local summaries) + router: on # answer known questions and known incidents locally (off = always the model) no_stream: false # set true for endpoints that mishandle streaming effort: medium # reasoning depth; mapped per provider (see below) max_tokens: 0 # output cap per round; 0 = provider default diff --git a/docs/EXTENDING.md b/docs/EXTENDING.md index 38422e3..59bec1e 100644 --- a/docs/EXTENDING.md +++ b/docs/EXTENDING.md @@ -218,6 +218,28 @@ list of them: signals: {node.peers: 0, config.pex: false, logs.peer_net: 3} # must match ``` +A case can also carry a **playbook**: the fix as steps cometcli runs itself on +`/incident` when the case is the clear root cause (the only root, or the most severe), with +no model call: + +```yaml + playbook: + - {do: node.logs, args: {lines: 40}, note: "the last lines before it stopped"} + - {do: node.service, args: {action: start}} + - {do: wait.until, args: {condition: synced, timeout: 1800}} + - {do: val.unjail, when: "val.jailed == true", note: "asks you"} + - {do: bash, args: {command: "docker update --memory 4g {unit}"}} +``` + +- `do` is any tool. +- `when` is a condition on the triage signals; the step is skipped if it doesn't hold. +- String arguments may use `{unit}`, `{home}`, `{binary}` and `{container_home}` from the + profile. +- Every step goes through the normal gates: shell commands are classified, changes ask, + and transactions ask for key access. +- A declined step stops the playbook. A failed step hands the incident to the model with + what was done. + Conditions are ` ` with `== != > >= < <= contains exists missing`; values are numbers, `true`/`false`, or strings. A signal triage couldn't collect never satisfies a condition (except `missing`). `cometcli node triage` prints every signal name @@ -225,3 +247,32 @@ and value — write conditions against those. Fix steps must start with `[read]` or `[tx]`; the agent still goes through the normal approvals for each. Ranking: more matched conditions first, then severity. The built-in suite checks every case's `test` fixture ranks it in the top 3 and that a healthy node matches nothing. + +## Known questions + +Questions cometcli answers without a model call live in an intent catalog. Built-ins ship +in the binary (`internal/router/intents/`). Yours go in `~/.cometcli/intents/*.yaml`; the +same `id` overrides a built-in. + +```yaml +- id: sentry-peers + title: peers on the sentries + scope: node # node: one node | chain: any node of the chain + patterns: ['^sentry peers$'] # regexes over the lowercased prompt; a hit routes here + examples: # phrasings; a close enough prompt routes here + - how many peers do the sentries have + steps: # read-only tools only (observe/diagnose) + - {as: p, tool: node.peers} + answer: | # Go text/template; each step is . with .Text .Data .Err + **{{.node}}** has {{get .p.Data "count"}} peers. + {{trim .p.Text}} + ttl: 30s # reuse the answer this long +``` + +Some prompts always go to the model, whatever matches: +- open-ended ones: why, how do I, explain, should I, … +- actions: unjail, restart, vote yes, … +- multi-line prompts and long prompts + +If a step fails, the model answers instead. Template functions: `get`, `trim`, `lines`, +`filter`, `count`. diff --git a/internal/agent/commands.go b/internal/agent/commands.go index 103d98a..bfc7531 100644 --- a/internal/agent/commands.go +++ b/internal/agent/commands.go @@ -44,7 +44,9 @@ const CommandHelp = ` /mode [ops|accept-edits|readonly|bypass] show or set the /reset, /clear clear conversation, start a new audit session /status current setup: model, mode, scope, memory, MCP (+ live node snapshot in node mode) /compact [focus] summarize the conversation to free context - /cost token usage for this session + /cost token usage for this session, and prompts answered locally + /llm ask the model even when cometcli knows the answer + /route why the last prompt was answered locally or sent to the model /effort [low|medium|high|xhigh|max|default] show or set reasoning depth /permissions show permission rules /allow|/ask|/deny add a rule for this session, e.g. /allow bash(docker logs:*) @@ -274,12 +276,31 @@ func RunCommand(a *Agent, c *toolkit.Context, reg *toolkit.Registry, line string return "compacted — the summary is attached to your next message", nil }}, nil + case "/llm": + if err := needAgent(); err != nil { + return CmdResult{}, err + } + q := strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(line), f[0])) + if q == "" { + return CmdResult{}, fmt.Errorf("usage: /llm — ask the model even when cometcli knows the answer") + } + return CmdResult{Text: "/llm", Prompt: forceModel + q}, nil + + case "/route": + if err := needAgent(); err != nil { + return CmdResult{}, err + } + return CmdResult{Text: a.RouteWhy()}, nil + case "/cost", "/usage": if err := needAgent(); err != nil { return CmdResult{}, err } u, last := a.Usage() txt := fmt.Sprintf("input %s (cached %s) · output %s", humanTok(u.Input), humanTok(u.CacheRead), humanTok(u.Output)) + if r := a.Routes(); r.Local+r.Model > 0 { + txt += fmt.Sprintf("\nprompts: %d answered locally (no tokens%s), %d sent to the model", r.Local+r.Cached, cachedNote(r.Cached), r.Model) + } if last > 0 { txt += fmt.Sprintf("\ncontext %s of %s — auto-compacts at %s", humanTok(last), humanTok(a.contextWindow()), humanTok(a.compactThreshold())) } @@ -484,6 +505,10 @@ func RunCommand(a *Agent, c *toolkit.Context, reg *toolkit.Registry, line string prompt = "This is about one of the operator's nodes: call use_node with the right profile first.\n\n" + prompt } } + if name == "incident" && a.Node() && a.routerOn() { + // triage and known cases run locally first (runIncident) + prompt = incidentMarker + args + } return CmdResult{Text: text, Prompt: prompt}, nil } return CmdResult{}, ErrUnknownCommand @@ -548,3 +573,10 @@ func humanTok(n int) string { } return fmt.Sprint(n) } + +func cachedNote(n int) string { + if n == 0 { + return "" + } + return fmt.Sprintf(", %d from cache", n) +} diff --git a/internal/agent/compact.go b/internal/agent/compact.go index 69a5cb8..df31f15 100644 --- a/internal/agent/compact.go +++ b/internal/agent/compact.go @@ -112,6 +112,7 @@ func (a *Agent) compactNow(ctx context.Context, instructions string, midTurn boo Model: a.Model, System: a.system(), Messages: hist, Tools: a.toolDefs(), MaxTok: 8192, Effort: a.Effort, } + a.guard(req) resp, err := a.Provider.Chat(ctx, req) if err != nil { return err diff --git a/internal/agent/egress.go b/internal/agent/egress.go new file mode 100644 index 0000000..bdc4702 --- /dev/null +++ b/internal/agent/egress.go @@ -0,0 +1,78 @@ +package agent + +import ( + "strings" + + "github.com/abhijitkrm/cometcli/internal/audit" + "github.com/abhijitkrm/cometcli/internal/egress" +) + +// egressMode is how much raw output the model may see in the current +// scope (it changes when the session moves onto a node). +func (a *Agent) egressMode() egress.Mode { + p := a.Ctx.Profile + if p == nil { + return egress.ModeFor(a.conf.Egress, "", false) + } + setting := a.conf.Egress + if p.Agent.Egress != "" { + setting = p.Agent.Egress + } + return egress.ModeFor(setting, p.Role, true) +} + +// guard is the last check before a request leaves the machine: any part +// of it that still carries key material is replaced whole — in the +// session history too, so it's never offered again. +func (a *Agent) guard(req *Request) { + if f := egress.Scan(req.System); len(f) > 0 { + a.withheld("system prompt", f) + req.System = egress.Withheld(f) + a.sys = req.System + } + for i := range req.Messages { + m := &req.Messages[i] + if f := egress.Scan(m.Text); len(f) > 0 { + a.withheld(m.Role+" message", f) + m.Text = egress.Withheld(f) + } + if len(m.RawSteps) > 0 { + if f := egress.Scan(string(m.RawSteps)); len(f) > 0 { + a.withheld("raw model output", f) + m.RawSteps, m.RawProvider = nil, "" + } + } + for j := range m.Calls { + if f := egress.Scan(string(m.Calls[j].Args)); len(f) > 0 { + a.withheld("tool call arguments", f) + m.Calls[j].Args = []byte(`{}`) + } + } + if f := egress.Scan(string(m.JSONArgs)); len(f) > 0 { + m.JSONArgs = nil + } + } + // req.Messages aliases a.history: the withheld forms are now the history +} + +func (a *Agent) withheld(what string, f []egress.Finding) { + a.emit(Event{Kind: EvNotice, Text: "withheld " + what + " from the model: it contained key material (" + egress.Kinds(f) + ")"}) + if lg := a.Audit(); lg != nil { + _ = lg.Log(audit.KindEgress, a.profileName(), map[string]any{"what": what, "kinds": egress.Kinds(f)}) + } +} + +// purpose is the agent's latest stated reason (its text before the tool +// call), shown when a key is about to be used. +func (a *Agent) purpose() string { + for i := len(a.history) - 1; i >= 0; i-- { + m := a.history[i] + if m.Role == "user" { + return "" + } + if m.Role == "assistant" && strings.TrimSpace(m.Text) != "" { + return clip(strings.Join(strings.Fields(a.Redact.Text(m.Text)), " "), 300) + } + } + return "" +} diff --git a/internal/agent/egress_test.go b/internal/agent/egress_test.go new file mode 100644 index 0000000..c2886d3 --- /dev/null +++ b/internal/agent/egress_test.go @@ -0,0 +1,80 @@ +package agent + +import ( + "context" + "encoding/json" + "strings" + "testing" + + "github.com/cosmos/go-bip39" + + "github.com/abhijitkrm/cometcli/internal/toolkit" +) + +// providerSaw is everything sent to the provider, as one string. +func providerSaw(p *mockProvider) string { return strings.Join(p.sent, "\n") } + +func TestStrictProfilesSendSummariesNotRawOutput(t *testing.T) { + tool := stubTool{name: "node.logs", tier: toolkit.TierDiagnose, run: func(*toolkit.Context, toolkit.Args) (*toolkit.Result, error) { + return &toolkit.Result{Text: "ERR dial tcp 10.20.30.40:26656: i/o timeout\nERR dial tcp 10.20.30.41:26656: i/o timeout\n"}, nil + }} + prov := &mockProvider{responses: []*Response{ + {Calls: []Call{{ID: "c1", Name: "node__logs", Args: json.RawMessage(`{}`)}}}, + {Text: "ok", Done: true}, + }} + a := newTestAgent(t, prov, tool) // a validator profile: strict by default + var shown string + a.OnEvent = func(e Event) { + if e.Kind == EvToolResult { + shown = e.Output + } + } + a.Run(context.Background(), "logs") + sent := providerSaw(prov) + if strings.Contains(sent, "10.20.30.4") || !strings.Contains(sent, "local summary") || !strings.Contains(sent, "(×2)") { + t.Fatalf("model should see only the local summary:\n%s", sent) + } + if !strings.Contains(shown, "10.20.30.40") { + t.Errorf("the operator's UI keeps the raw output, got %q", shown) + } +} + +func TestKeyMaterialNeverReachesTheProvider(t *testing.T) { + ent, _ := bip39.NewEntropy(256) + mn, _ := bip39.NewMnemonic(ent) + tool := stubTool{name: "node.config", tier: toolkit.TierDiagnose, run: func(*toolkit.Context, toolkit.Args) (*toolkit.Result, error) { + // redaction doesn't know this shape; the gate must stop it + return &toolkit.Result{Text: "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQ\n-----END OPENSSH PRIVATE KEY-----"}, nil + }} + prov := &mockProvider{responses: []*Response{ + {Calls: []Call{{ID: "c1", Name: "node__config", Args: json.RawMessage(`{}`)}}}, + {Text: "ok", Done: true}, + }} + a := newTestAgent(t, prov, tool) + a.conf.Egress = "filtered" // the gate applies in every mode + var notices []string + a.OnEvent = func(e Event) { + if e.Kind == EvNotice { + notices = append(notices, e.Text) + } + } + if _, err := a.Run(context.Background(), "show the deploy key"); err != nil { + t.Fatal(err) + } + sent := providerSaw(prov) + if strings.Contains(sent, "b3BlbnNzaC1rZXktdjEAAAAABG5vbmU") || !strings.Contains(sent, "withheld by cometcli") { + t.Fatalf("private key reached the provider:\n%s", sent) + } + if len(notices) == 0 { + t.Error("the operator should be told something was withheld") + } + + // a mnemonic typed by the operator isn't sent at all + before := len(prov.sent) + if _, err := a.Run(context.Background(), "restore my key: "+mn); err == nil || !strings.Contains(err.Error(), "key material") { + t.Fatalf("want a refusal, got %v", err) + } + if len(prov.sent) != before || strings.Contains(providerSaw(prov), strings.Fields(mn)[5]+" "+strings.Fields(mn)[6]) { + t.Fatal("a message with a mnemonic must not be sent") + } +} diff --git a/internal/agent/incident.go b/internal/agent/incident.go new file mode 100644 index 0000000..bb8a0f6 --- /dev/null +++ b/internal/agent/incident.go @@ -0,0 +1,191 @@ +package agent + +import ( + "context" + "fmt" + "strings" + + "github.com/abhijitkrm/cometcli/internal/audit" + "github.com/abhijitkrm/cometcli/internal/config" + "github.com/abhijitkrm/cometcli/internal/egress" + "github.com/abhijitkrm/cometcli/internal/kb" + "github.com/abhijitkrm/cometcli/internal/toolkit" + "github.com/abhijitkrm/cometcli/internal/tools/triage" +) + +// incidentMarker prefixes an /incident that cometcli works itself first. +const incidentMarker = "\x1fincident\x1f" + +// runIncident works /incident without the model when it can: triage runs +// locally, and when one known case is clearly the root cause and has a +// playbook, cometcli runs it — every change and transaction still asks +// the operator. Anything else goes to the model with the triage attached, +// so it starts from the evidence instead of collecting it again. +func (a *Agent) runIncident(ctx context.Context, what string) (string, error) { + procedure := builtinCommands["incident"].Expand(what, nil, nil) + handoff := func(context string) (string, error) { + prompt := procedure + if context != "" { + prompt += "\n\n" + context + } + return a.run(ctx, forceModel+prompt) + } + t, ok := a.Reg.Get("node.triage") + if !ok { + return handoff("") + } + res, err := a.runTool(ctx, t, toolkit.Args{}) + if err != nil { + return handoff("") + } + triageNote := "[cometcli already ran node.triage for this incident — start from it; run it again only after something changed]\n" + + egress.ForModel(a.egressMode(), "node.triage", res.Text) + sig, _ := res.Data["signals"].(kb.Signals) + roots, _ := res.Data["roots"].([]string) + base := triage.LoadKB(a.Ctx) + if len(roots) == 0 { + return handoff(triageNote) + } + c := base.Cases[roots[0]] + switch { + case c == nil || len(c.Playbook) == 0: + return handoff(triageNote) + case len(roots) > 1 && base.Cases[roots[1]] != nil && base.Cases[roots[1]].Severity == c.Severity: + // two equally serious root causes: that's a judgement call + return handoff(triageNote) + } + + a.emit(Event{Kind: EvText, Text: fmt.Sprintf("Known case **%s** — %s.\nRunning its playbook locally, no model call; changes and transactions still ask you.", c.ID, c.Title)}) + var done []string + for i, st := range c.Playbook { + label := st.Do + if st.Note != "" { + label += " — " + st.Note + } + if !st.Applies(sig) { + done = append(done, fmt.Sprintf("skipped %s (%s doesn't hold)", st.Do, st.When)) + continue + } + tool, ok := a.Reg.Get(st.Do) + if !ok { + return handoff(triageNote + "\n\n" + playbookNote(c, done, fmt.Sprintf("step %d: no tool %s", i+1, st.Do))) + } + r, err := a.runTool(ctx, tool, playArgs(st.Args, a.Ctx.Profile)) + if err != nil { + if strings.Contains(err.Error(), "denied") || ctx.Err() != nil { + // the operator said no: stop here, don't let a model push on + text := fmt.Sprintf("Stopped at step %d (%s): %v\n\nDone so far:\n%s", i+1, label, err, bullets(done)) + return a.localIncidentDone(what, c, text), nil + } + return handoff(triageNote + "\n\n" + playbookNote(c, done, fmt.Sprintf("step %d (%s) failed: %v", i+1, label, err))) + } + done = append(done, fmt.Sprintf("%s: %s", label, firstLine(a.Redact.Text(r.Text)))) + } + + _, matched := c.Matches(sig) + outcome := "resolved — " + c.ID + " playbook completed" + recorded := "" + if rt, ok := a.Reg.Get("incident.record"); ok { + rr, err := a.runTool(ctx, rt, toolkit.Args{ + "title": c.Title, "case": c.ID, "root_cause": c.Title + " (matched " + strings.Join(matched, ", ") + ")", + "evidence": strings.Join(matched, "\n"), "actions": strings.Join(done, "\n"), "outcome": outcome, + }) + if err == nil { + recorded = "\n\n" + firstLine(rr.Text) + } + } + text := fmt.Sprintf("**Resolved: %s** (known case `%s`)\n\nEvidence: %s\n\nWhat was done:\n%s%s", + c.Title, c.ID, strings.Join(matched, ", "), bullets(done), recorded) + return a.localIncidentDone(what, c, text), nil +} + +func (a *Agent) localIncidentDone(what string, c *kb.Case, text string) string { + text += "\n\n_worked locally from the knowledge base · no model call_" + a.emit(Event{Kind: EvText, Text: text}) + kept := text + if a.egressMode() == egress.Strict { + kept = egress.Mask(text) + } + a.history = append(a.history, + Msg{Role: "user", Text: a.turnPrefix() + "/incident " + what}, + Msg{Role: "assistant", Text: "[worked locally by cometcli from case " + c.ID + "]\n" + kept}) + a.routes.Local++ + if lg := a.Audit(); lg != nil { + _ = lg.Log(audit.KindPrompt, a.profileName(), map[string]any{"text": "/incident " + what, "route": "local", "case": c.ID}) + } + return text +} + +// playbookNote tells the model what the playbook already did. +func playbookNote(c *kb.Case, done []string, failure string) string { + return fmt.Sprintf("[cometcli matched case %s and ran its playbook until a step failed — continue from here, don't repeat the steps that worked]\ndone:\n%s\n%s", + c.ID, bullets(done), failure) +} + +func bullets(items []string) string { + if len(items) == 0 { + return "- (nothing yet)" + } + return "- " + strings.Join(items, "\n- ") +} + +// playArgs fills {unit}, {home}, {binary} and {container_home} in string +// arguments from the profile. +func playArgs(args map[string]any, p *config.Profile) toolkit.Args { + r := strings.NewReplacer("{unit}", shellWord(p.Service.Unit), "{home}", shellWord(p.Home), + "{binary}", shellWord(p.Binary), "{container_home}", shellWord(p.Signer.ContainerHome)) + out := toolkit.Args{} + for k, v := range args { + if s, ok := v.(string); ok { + v = r.Replace(s) + } + out[k] = v + } + return out +} + +// shellWord keeps a profile value usable in a command: plain paths and +// names pass through (so ~ still expands), anything else is quoted. +func shellWord(s string) string { + plain := func(r rune) bool { + return r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || strings.ContainsRune("/._-~+:@", r) + } + for _, r := range s { + if !plain(r) { + return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" + } + } + return s +} + +// runTool runs one registry tool for cometcli's own procedures, through +// the same gates as the model's calls: permission rules, approvals for +// changes and key use, the audit log and the operator's UI. +func (a *Agent) runTool(ctx context.Context, t toolkit.Tool, args toolkit.Args) (*toolkit.Result, error) { + name := t.Name() + shown := a.Redact.Args(args) + a.emit(Event{Kind: EvToolStart, Tool: name, Tier: t.Tier().String(), Args: shown}) + runCtx, cancel := a.toolCtx(ctx, t, args) + defer cancel() + defer runCtx.Close() + err := a.ruleGate(runCtx, t, shown) + var res *toolkit.Result + if err == nil { + res, err = t.Run(runCtx, args) + } + if lg := a.Audit(); lg != nil { + var data map[string]any + if res != nil { + data = a.Redact.Args(res.Data) + } + lg.ToolSeen(a.profileName(), name, t.Tier().String(), shown, data, err, "") + } + if err != nil { + a.emit(Event{Kind: EvToolResult, Tool: name, Tier: t.Tier().String(), Text: "error: " + a.Redact.Text(firstLine(err.Error()))}) + return nil, err + } + text := a.Redact.Text(ansiRe.ReplaceAllString(res.Text, "")) + res.Text = text + a.emit(Event{Kind: EvToolResult, Tool: name, Tier: t.Tier().String(), Text: firstLine(text), Output: preview(text)}) + return res, nil +} diff --git a/internal/agent/incident_test.go b/internal/agent/incident_test.go new file mode 100644 index 0000000..8d29301 --- /dev/null +++ b/internal/agent/incident_test.go @@ -0,0 +1,126 @@ +package agent + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/abhijitkrm/cometcli/internal/kb" + "github.com/abhijitkrm/cometcli/internal/toolkit" +) + +// incidentAgent has stub node tools that record what ran. +func incidentAgent(t *testing.T, prov *mockProvider, roots []string, sig kb.Signals, fail map[string]error) (*Agent, *[]string) { + t.Helper() + var ran []string + stub := func(name string, tier toolkit.Tier, data map[string]any) stubTool { + return stubTool{name: name, tier: tier, run: func(_ *toolkit.Context, a toolkit.Args) (*toolkit.Result, error) { + ran = append(ran, strings.TrimSpace(name+" "+a.String("action", "")+a.String("condition", ""))) + if err := fail[name]; err != nil { + return nil, err + } + return &toolkit.Result{Text: name + " ok", Data: data}, nil + }} + } + a := routedAgent(t, prov, + stub("node.triage", toolkit.TierDiagnose, map[string]any{"roots": roots, "signals": sig}), + stub("node.logs", toolkit.TierObserve, nil), + stub("node.service", toolkit.TierLocalChange, nil), + stub("wait.until", toolkit.TierObserve, nil), + stub("val.unjail", toolkit.TierOnChain, nil), + stub("incident.record", toolkit.TierDiagnose, nil), + ) + return a, &ran +} + +func runIncidentCmd(t *testing.T, a *Agent, what string) string { + t.Helper() + cmd, err := RunCommand(a, a.Ctx, a.Reg, "/incident "+what) + if err != nil { + t.Fatal(err) + } + out, err := a.Run(context.Background(), cmd.Prompt) + if err != nil { + t.Fatal(err) + } + return out +} + +func TestKnownIncidentRunsItsPlaybookLocally(t *testing.T) { + prov := &mockProvider{} + a, ran := incidentAgent(t, prov, []string{"node-process-down"}, kb.Signals{"proc.running": false, "val.jailed": true}, nil) + out := runIncidentCmd(t, a, "val1 stopped signing") + if prov.calls != 0 { + t.Fatalf("a known case with a playbook needs no model call (calls=%d)", prov.calls) + } + want := []string{"node.triage", "node.logs", "node.service start", "wait.until synced", "wait.until unjailable", "val.unjail", "wait.until in-consensus", "incident.record"} + if strings.Join(*ran, "|") != strings.Join(want, "|") { + t.Fatalf("ran %v\nwant %v", *ran, want) + } + if !strings.Contains(out, "Resolved: Node process/container not running") || !strings.Contains(out, "no model call") { + t.Fatalf("report: %s", out) + } + if a.Routes().Local != 1 { + t.Errorf("routes = %+v", a.Routes()) + } +} + +func TestPlaybookSkipsStepsWhoseConditionFails(t *testing.T) { + prov := &mockProvider{} + a, ran := incidentAgent(t, prov, []string{"node-process-down"}, kb.Signals{"proc.running": false, "val.jailed": false}, nil) + out := runIncidentCmd(t, a, "") + for _, r := range *ran { + if r == "val.unjail" { + t.Fatal("a validator that isn't jailed must not be unjailed") + } + } + if !strings.Contains(out, "skipped val.unjail") { + t.Errorf("report should say what was skipped:\n%s", out) + } +} + +func TestFailedPlaybookStepHandsOffWithContext(t *testing.T) { + prov := &mockProvider{} + a, _ := incidentAgent(t, prov, []string{"node-process-down"}, kb.Signals{"proc.running": false}, + map[string]error{"node.service": fmt.Errorf("container exited again: exit 1")}) + runIncidentCmd(t, a, "") + if prov.calls == 0 { + t.Fatal("a failed step goes to the model") + } + sent := prov.sent[0] + for _, want := range []string{"already ran node.triage", "ran its playbook until a step failed", "container exited again", "node.logs"} { + if !strings.Contains(sent, want) { + t.Errorf("handoff lacks %q", want) + } + } +} + +func TestDeclinedStepStopsWithoutTheModel(t *testing.T) { + prov := &mockProvider{} + a, ran := incidentAgent(t, prov, []string{"node-process-down"}, kb.Signals{"proc.running": false}, + map[string]error{"node.service": fmt.Errorf("denied: start the node")}) + out := runIncidentCmd(t, a, "") + if prov.calls != 0 || !strings.Contains(out, "Stopped at step 2") { + t.Fatalf("calls=%d out=%s", prov.calls, out) + } + if (*ran)[len(*ran)-1] != "node.service start" { + t.Fatalf("nothing may run after a declined step: %v", *ran) + } +} + +func TestUnknownIncidentGoesToTheModelWithTriage(t *testing.T) { + prov := &mockProvider{} + a, ran := incidentAgent(t, prov, nil, kb.Signals{}, nil) + runIncidentCmd(t, a, "something odd") + if prov.calls == 0 || len(*ran) != 1 || !strings.Contains(prov.sent[0], "already ran node.triage") || !strings.Contains(prov.sent[0], "something odd") { + t.Fatalf("calls=%d ran=%v", prov.calls, *ran) + } + // a case without a playbook goes the same way + prov2 := &mockProvider{} + a2, _ := incidentAgent(t, prov2, []string{"cfg-parse-error"}, kb.Signals{}, nil) + runIncidentCmd(t, a2, "") + if prov2.calls == 0 { + t.Fatal("a case without a playbook needs the model") + } +} diff --git a/internal/agent/loop.go b/internal/agent/loop.go index b687e95..ab50f0c 100644 --- a/internal/agent/loop.go +++ b/internal/agent/loop.go @@ -16,8 +16,10 @@ import ( "github.com/abhijitkrm/cometcli/internal/audit" "github.com/abhijitkrm/cometcli/internal/client/host" "github.com/abhijitkrm/cometcli/internal/config" + "github.com/abhijitkrm/cometcli/internal/egress" "github.com/abhijitkrm/cometcli/internal/hooks" "github.com/abhijitkrm/cometcli/internal/redact" + "github.com/abhijitkrm/cometcli/internal/router" "github.com/abhijitkrm/cometcli/internal/settings" "github.com/abhijitkrm/cometcli/internal/toolkit" ) @@ -92,7 +94,11 @@ type Agent struct { rounds int // model rounds in the latest Run ownCtx bool // a.Ctx was built by SwitchScope ext *ext // settings, hooks, commands, MCP servers - parent *Agent // set on subagents + catalog *router.Catalog // known questions answered without a model call + answers router.Cache + routes RouteStats + lastRoute router.Match + parent *Agent // set on subagents def *settings.AgentDef } @@ -157,6 +163,11 @@ func New(c *toolkit.Context, reg *toolkit.Registry) (*Agent, error) { } a.WorkRoot = cwd a.Tools = toolkit.NewSession("") + if d, err := config.Path("intents"); err == nil { + if a.catalog, err = router.Load(d); err != nil { + e.notes = append(e.notes, "intents: "+err.Error()+" — local answers are off") + } + } a.ext = e if a.Model == "" { a.Model = modelOf(prov) @@ -345,7 +356,22 @@ func (a *Agent) run(ctx context.Context, input string) (string, error) { } } defer a.grantTurnRules()() + if f := egress.Scan(input); len(f) > 0 { // before redaction: refuse, don't half-send + a.withheld("your message", f) + return "", fmt.Errorf("not sent: your message contained key material (%s). cometcli never needs a key in chat — keys are used only through its signing tools, on this machine", egress.Kinds(f)) + } input = a.Redact.Text(input) + forced := strings.HasPrefix(input, forceModel) + input = strings.TrimPrefix(input, forceModel) + if strings.HasPrefix(input, incidentMarker) { + return a.runIncident(ctx, strings.TrimPrefix(input, incidentMarker)) + } + if !forced && a.routerOn() { + if text, ok := a.routeLocal(ctx, input); ok { + return text, nil + } + } + a.routes.Model++ if lg := a.Audit(); lg != nil { _ = lg.Log(audit.KindPrompt, a.profileName(), map[string]any{"text": input}) } @@ -497,6 +523,7 @@ func (a *Agent) chat(ctx context.Context, req *Request) (*Response, error) { ctx = WithRetryNotice(ctx, func(attempt int, wait time.Duration, reason string) { a.emit(Event{Kind: EvNotice, Text: fmt.Sprintf("%s: %s — retrying in %s (attempt %d)", a.Provider.Name(), reason, wait.Round(100*time.Millisecond), attempt+1)}) }) + a.guard(req) if s, ok := a.Provider.(Streamer); ok && a.Stream { return s.Stream(ctx, req, func(chunk string) { a.emit(Event{Kind: EvDelta, Text: chunk}) @@ -570,6 +597,7 @@ func (a *Agent) execCall(ctx context.Context, call Call) Msg { } runCtx, cancel := a.toolCtx(ctx, t, args) + runCtx.Purpose = a.purpose() var res *toolkit.Result in := a.hookInput(hooks.PreToolUse) in.ToolName, in.ToolInput = name, args @@ -620,6 +648,9 @@ func (a *Agent) execCall(ctx context.Context, call Call) Msg { text = clip(text, limit) // bound context growth text = a.Redact.Text(text) } + // what the model sees: on strict profiles raw output is replaced by a + // local summary (the operator's UI and the audit log keep the raw text) + llmText := egress.ForModel(a.egressMode(), name, text) if lg := a.Audit(); lg != nil { var data map[string]any if res != nil { @@ -635,7 +666,7 @@ func (a *Agent) execCall(ctx context.Context, call Call) Msg { } text += postNote a.emit(Event{Kind: EvToolResult, Tool: name, Tier: t.Tier().String(), Text: firstLine(text), Output: preview(text)}) - return Msg{Role: "tool", CallID: call.ID, ToolName: name, Text: text} + return Msg{Role: "tool", CallID: call.ID, ToolName: name, Text: llmText + postNote} } // toolCtx derives the per-call tool context: the turn's cancellation, a diff --git a/internal/agent/loop_test.go b/internal/agent/loop_test.go index 7bae35f..7a1d274 100644 --- a/internal/agent/loop_test.go +++ b/internal/agent/loop_test.go @@ -19,6 +19,7 @@ type mockProvider struct { calls int lastReq *Request reqs []*Request + sent []string // each request as sent (history is mutated later) err error } @@ -27,6 +28,16 @@ func (m *mockProvider) Chat(ctx context.Context, r *Request) (*Response, error) m.calls++ m.lastReq = r m.reqs = append(m.reqs, r) + var b strings.Builder + b.WriteString(r.System) + for _, msg := range r.Messages { + b.WriteString(msg.Text) + b.Write(msg.RawSteps) + for _, c := range msg.Calls { + b.Write(c.Args) + } + } + m.sent = append(m.sent, b.String()) if m.err != nil { return nil, m.err } diff --git a/internal/agent/phase2_test.go b/internal/agent/phase2_test.go index d9968c3..84116e4 100644 --- a/internal/agent/phase2_test.go +++ b/internal/agent/phase2_test.go @@ -132,6 +132,7 @@ func TestBashOutputUsesToolLimit(t *testing.T) { {Text: "ok", Done: true}, }} a, _ := newShellAgent(t, prov) + a.conf.Egress = "filtered" a.Run(context.Background(), "x") if n := len(lastToolText(a)); n < 19000 || n > 30500 { t.Fatalf("bash output length %d — should keep up to 30k", n) diff --git a/internal/agent/route.go b/internal/agent/route.go new file mode 100644 index 0000000..04a5c8b --- /dev/null +++ b/internal/agent/route.go @@ -0,0 +1,166 @@ +package agent + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/abhijitkrm/cometcli/internal/audit" + "github.com/abhijitkrm/cometcli/internal/config" + "github.com/abhijitkrm/cometcli/internal/egress" + "github.com/abhijitkrm/cometcli/internal/router" + "github.com/abhijitkrm/cometcli/internal/toolkit" +) + +// forceModel prefixes a prompt that must reach the model (/llm). +const forceModel = "\x1fmodel\x1f" + +// RouteStats counts how prompts were answered this session. +type RouteStats struct { + Local int // answered from the intent catalog, no model call + Cached int // of those, served from the answer cache + Model int // sent to the model +} + +// Routes reports how this session's prompts were answered. +func (a *Agent) Routes() RouteStats { return a.routes } + +func (a *Agent) routerOn() bool { + return a.parent == nil && a.catalog != nil && !strings.EqualFold(a.conf.Router, "off") +} + +// routeLocal answers a known question from the intent catalog without a +// model call. It reports false when the model should handle the prompt: +// not a known question, no node to ask, or a tool failed. +func (a *Agent) routeLocal(ctx context.Context, input string) (string, bool) { + m := a.catalog.Classify(input) + a.lastRoute = m + if m.Intent == nil { + return "", false + } + in := m.Intent + p := a.routeProfile(input, in) + if p == nil { + a.lastRoute.Why += "; no single node to ask" + return "", false + } + key := in.ID + "|" + p.Name + now := time.Now() + if text, ok := a.answers.Get(key, now); ok { + a.routes.Cached++ + return a.answeredLocally(input, in, text+"\n\n_answered locally from a recent check · /llm to ask the model_"), true + } + // read-only steps: nothing here may ask for, or get, an approval + base := &toolkit.Context{Context: ctx, Profile: p, Cfg: a.Ctx.Cfg, Audit: a.Audit(), Approver: toolkit.DenyApprover, ReadOnly: true} + results := map[string]router.Result{} + for _, st := range in.Steps { + t, ok := a.Reg.Get(st.Tool) + if !ok || t.Tier() > toolkit.TierDiagnose { + // the catalog only reads: anything else is the model's call + a.lastRoute.Why += "; " + st.Tool + " isn't a read-only tool" + return "", false + } + args := map[string]any{} + for k, v := range st.Args { + args[k] = v + } + a.emit(Event{Kind: EvToolStart, Tool: st.Tool, Tier: t.Tier().String(), Args: a.Redact.Args(args)}) + sub, cancel := toolkit.WithDeadline(base, toolkit.CallTimeout(t, args)) + sub.ToolName = st.Tool + res, err := t.Run(sub, args) + sub.Close() + cancel() + r := router.Result{} + if err != nil { + r.Err = err.Error() + a.emit(Event{Kind: EvToolResult, Tool: st.Tool, Text: "error: " + a.Redact.Text(firstLine(r.Err))}) + if !st.Optional { + // the model explains failures better than a template + a.lastRoute.Why += "; " + st.Tool + " failed" + return "", false + } + } else { + r.Text, r.Data = ansiRe.ReplaceAllString(res.Text, ""), res.Data + a.emit(Event{Kind: EvToolResult, Tool: st.Tool, Text: firstLine(a.Redact.Text(r.Text)), Output: preview(a.Redact.Text(r.Text))}) + } + if lg := a.Audit(); lg != nil { + var data map[string]any + if res != nil { + data = a.Redact.Args(res.Data) + } + lg.ToolSeen(p.Name, st.Tool, t.Tier().String(), args, data, err, "") + } + results[st.As] = r + } + text, err := in.Render(p.Name, results) + if err != nil { + a.lastRoute.Why += "; answer template: " + err.Error() + return "", false + } + text = a.Redact.Text(text) + a.answers.Put(key, text, in.TTL, now) + a.routes.Local++ + return a.answeredLocally(input, in, text+"\n\n_answered locally · no model call · /llm to ask the model_"), true +} + +// answeredLocally records a local answer: shown to the operator, kept in +// the conversation (so a later model turn knows it), and audited. +func (a *Agent) answeredLocally(input string, in *router.Intent, text string) string { + a.emit(Event{Kind: EvText, Text: text}) + kept := text + if a.egressMode() == egress.Strict { + kept = egress.Mask(text) + } + a.history = append(a.history, + Msg{Role: "user", Text: a.turnPrefix() + input}, + Msg{Role: "assistant", Text: "[answered locally by cometcli: " + in.Title + "]\n" + kept}) + if lg := a.Audit(); lg != nil { + _ = lg.Log(audit.KindPrompt, a.profileName(), map[string]any{"text": input, "route": "local", "intent": in.ID}) + } + return text +} + +// routeProfile picks the node a known question is about: the session's +// node, else the one the prompt names, else the active profile, else the +// only one. Chain-wide questions can use any of these. +func (a *Agent) routeProfile(input string, in *router.Intent) *config.Profile { + if a.Node() { + if p, _ := a.nodeFor(input); p != nil && p.Name != a.Ctx.Profile.Name && in.Scope == "node" { + return p + } + return a.Ctx.Profile + } + if p, _ := a.nodeFor(input); p != nil { + return p + } + cfg := a.Ctx.Cfg + if cfg == nil || len(cfg.Profiles) == 0 { + return nil + } + if cfg.Active != "" { + if p, ok := cfg.Profiles[cfg.Active]; ok { + p.Name = cfg.Active + return p + } + } + if len(cfg.Profiles) == 1 { + for n, p := range cfg.Profiles { + p.Name = n + return p + } + } + return nil +} + +// RouteWhy explains where the latest prompt went (for /route). +func (a *Agent) RouteWhy() string { + m := a.lastRoute + if m.Intent != nil { + return fmt.Sprintf("answered locally: %s (%s)", m.Intent.ID, m.Why) + } + if m.Why == "" { + return "no prompt classified yet" + } + return "sent to the model: " + m.Why +} diff --git a/internal/agent/route_test.go b/internal/agent/route_test.go new file mode 100644 index 0000000..d76735c --- /dev/null +++ b/internal/agent/route_test.go @@ -0,0 +1,124 @@ +package agent + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/abhijitkrm/cometcli/internal/router" + "github.com/abhijitkrm/cometcli/internal/toolkit" +) + +func routedAgent(t *testing.T, prov *mockProvider, tools ...toolkit.Tool) *Agent { + t.Helper() + a := newTestAgent(t, prov, tools...) + cat, err := router.Load() + if err != nil { + t.Fatal(err) + } + a.catalog = cat + return a +} + +func jailedTool(runs *int, jailed int, fail bool) stubTool { + return stubTool{name: "chain.validators", tier: toolkit.TierObserve, run: func(_ *toolkit.Context, a toolkit.Args) (*toolkit.Result, error) { + *runs++ + if fail { + return nil, fmt.Errorf("dial 127.0.0.1:9090: connection refused") + } + if a.String("status", "") != "jailed" { + return nil, fmt.Errorf("want status=jailed, got %q", a.String("status", "")) + } + return &toolkit.Result{Text: fmt.Sprintf("val3 JAILED\ntotal: %d\n", jailed), Data: map[string]any{"count": jailed}}, nil + }} +} + +func TestKnownQuestionsAreAnsweredWithoutTheModel(t *testing.T) { + prov := &mockProvider{} + runs := 0 + a := routedAgent(t, prov, jailedTool(&runs, 1, false)) + out, err := a.Run(context.Background(), "is anyone jailed?") + if err != nil { + t.Fatal(err) + } + if prov.calls != 0 || runs != 1 { + t.Fatalf("model calls=%d tool runs=%d", prov.calls, runs) + } + if !strings.Contains(out, "1 validator(s) jailed") || !strings.Contains(out, "answered locally") { + t.Fatalf("answer: %q", out) + } + // asked again within the TTL: served from the cache, nothing re-run + if _, err := a.Run(context.Background(), "any validators jailed"); err != nil { + t.Fatal(err) + } + if r := a.Routes(); r.Local != 1 || r.Cached != 1 || r.Model != 0 || runs != 1 { + t.Fatalf("routes=%+v runs=%d", r, runs) + } + // the conversation keeps the local answers for later model turns + if len(a.history) != 4 || !strings.Contains(a.history[1].Text, "answered locally by cometcli") { + t.Fatalf("history: %+v", a.history) + } +} + +func TestOpenQuestionsAndForcedPromptsGoToTheModel(t *testing.T) { + prov := &mockProvider{} + runs := 0 + a := routedAgent(t, prov, jailedTool(&runs, 1, false)) + for _, q := range []string{"why is val3 jailed?", forceModel + "is anyone jailed?"} { + if _, err := a.Run(context.Background(), q); err != nil { + t.Fatal(err) + } + } + if prov.calls != 2 || runs != 0 || a.Routes().Model != 2 { + t.Fatalf("model calls=%d runs=%d routes=%+v", prov.calls, runs, a.Routes()) + } + if strings.Contains(prov.sent[1], forceModel) { + t.Error("the /llm marker must not reach the model") + } + if !strings.Contains(a.RouteWhy(), "sent to the model") { + t.Errorf("RouteWhy = %q", a.RouteWhy()) + } +} + +func TestFailedLookupFallsBackToTheModel(t *testing.T) { + prov := &mockProvider{} + runs := 0 + a := routedAgent(t, prov, jailedTool(&runs, 0, true)) + if _, err := a.Run(context.Background(), "is anyone jailed?"); err != nil { + t.Fatal(err) + } + if runs != 1 || prov.calls != 1 || !strings.Contains(a.RouteWhy(), "failed") { + t.Fatalf("runs=%d calls=%d why=%q", runs, prov.calls, a.RouteWhy()) + } +} + +func TestRouterOnlyRunsReadOnlyTools(t *testing.T) { + prov := &mockProvider{} + ran := false + // a catalog entry pointing at a tool that changes things is refused + sneaky := stubTool{name: "node.peers", tier: toolkit.TierLocalChange, run: func(*toolkit.Context, toolkit.Args) (*toolkit.Result, error) { + ran = true + return &toolkit.Result{}, nil + }} + a := routedAgent(t, prov, sneaky) + if _, err := a.Run(context.Background(), "how many peers"); err != nil { + t.Fatal(err) + } + if ran || prov.calls != 1 { + t.Fatalf("ran=%v model calls=%d", ran, prov.calls) + } +} + +func TestRouterOff(t *testing.T) { + prov := &mockProvider{} + runs := 0 + a := routedAgent(t, prov, jailedTool(&runs, 1, false)) + a.conf.Router = "off" + if _, err := a.Run(context.Background(), "is anyone jailed?"); err != nil { + t.Fatal(err) + } + if prov.calls != 1 || runs != 0 { + t.Fatalf("router off: calls=%d runs=%d", prov.calls, runs) + } +} diff --git a/internal/agent/session_test.go b/internal/agent/session_test.go index dea917e..378546f 100644 --- a/internal/agent/session_test.go +++ b/internal/agent/session_test.go @@ -16,7 +16,6 @@ import ( "github.com/abhijitkrm/cometcli/internal/toolkit" ) -const testMnemonic = "legal winner thank year wave sausage worth useful legal winner thank yellow" // streamProvider wraps mockProvider with a Streamer that emits each // response's text in two chunks. @@ -38,14 +37,15 @@ func TestUserInputRedactedBeforeLLM(t *testing.T) { prov := &mockProvider{responses: []*Response{{Text: "ok", Done: true}}} a := newTestAgent(t, prov) a.Redact = redact.NewRedactor("val.internal") - if _, err := a.Run(context.Background(), "my seed is "+testMnemonic+" and host val.internal is down"); err != nil { + // (a mnemonic refuses the whole message: TestKeyMaterialNeverReachesTheProvider) + if _, err := a.Run(context.Background(), "host val.internal is down, token=abcd1234secret"); err != nil { t.Fatal(err) } sent := prov.lastReq.Messages[0].Text - if strings.Contains(sent, "sausage") || strings.Contains(sent, "val.internal") { + if strings.Contains(sent, "abcd1234secret") || strings.Contains(sent, "val.internal") { t.Fatalf("secret reached the provider: %q", sent) } - if !strings.Contains(sent, "[REDACTED_MNEMONIC]") || !strings.Contains(sent, "[REDACTED_HOST]") { + if !strings.Contains(sent, "[REDACTED_HOST]") { t.Fatalf("missing redaction markers: %q", sent) } } @@ -385,6 +385,7 @@ func TestToolOutputANSIStripped(t *testing.T) { {Text: "ok", Done: true}, }} a := newTestAgent(t, prov, tool) + a.conf.Egress = "filtered" a.Run(context.Background(), "logs") if got := a.history[2].Text; got != "10:15AM INF committed height=15171" { t.Fatalf("tool text = %q", got) diff --git a/internal/audit/audit.go b/internal/audit/audit.go index 4358e46..3cbca31 100644 --- a/internal/audit/audit.go +++ b/internal/audit/audit.go @@ -27,6 +27,8 @@ const ( KindPrompt Kind = "prompt" // an agent prompt (redacted) KindLLM Kind = "llm" // an LLM response digest KindAlert Kind = "alert" // a monitor alert firing + KindEgress Kind = "egress" // content withheld from the model (key material) + KindKey Kind = "key" // a key used to sign (which, where, for what, approved by) ) // Event is one audit record. diff --git a/internal/config/config.go b/internal/config/config.go index e50995f..381f23d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -223,6 +223,14 @@ type AgentConf struct { RedactHosts []string `yaml:"redact_hosts,omitempty"` // RedactEndpoints also masks the profile's own endpoint/SSH hosts. RedactEndpoints bool `yaml:"redact_endpoints,omitempty"` + // Egress is how much raw output reaches the model: strict (shell, + // file and log output only as local summaries) or filtered (raw after + // redaction). Default: strict for validators, filtered otherwise. Key + // material is withheld in every mode. + Egress string `yaml:"egress,omitempty"` + // Router answers known questions (~/.cometcli/intents and built-ins) + // without a model call: on (default) | off. + Router string `yaml:"router,omitempty"` // NoStream disables token streaming for endpoints that mishandle it. NoStream bool `yaml:"no_stream,omitempty"` // Effort is the reasoning depth: low | medium | high | xhigh | max diff --git a/internal/egress/egress.go b/internal/egress/egress.go new file mode 100644 index 0000000..c6eeb41 --- /dev/null +++ b/internal/egress/egress.go @@ -0,0 +1,105 @@ +// Package egress is the last check on everything cometcli sends to a model +// provider. Redaction (package redact) masks what it recognizes field by +// field; egress is the gate behind it: a message that still carries key +// material is withheld whole, never partially sent. +// +// It also makes the local summaries that stand in for raw command output +// on strict profiles (validators by default), so a node's shell and log +// output never leaves the machine verbatim. +package egress + +import ( + "fmt" + "regexp" + "sort" + "strings" + + "github.com/cosmos/go-bip39" +) + +// Finding is one kind of key material found in a text. +type Finding struct { + Kind string // short name: "mnemonic", "pem-private-key", … +} + +type detector struct { + kind string + re *regexp.Regexp +} + +// detectors recognize key material by shape. Each is specific enough that +// a match is worth withholding a whole message for. +var detectors = []detector{ + {"pem-private-key", regexp.MustCompile(`-----BEGIN [A-Z0-9 ]*PRIVATE KEY( BLOCK)?-----`)}, + {"consensus-private-key", regexp.MustCompile(`(?s)"priv_key"\s*:\s*\{\s*"type"\s*:\s*"[^"]*PrivKey[^"]*"\s*,\s*"value"\s*:\s*"[A-Za-z0-9+/=]{40,}"`)}, + {"consensus-private-key", regexp.MustCompile(`"type"\s*:\s*"tendermint/PrivKey[A-Za-z0-9]*"\s*,\s*"value"\s*:\s*"[A-Za-z0-9+/=]{40,}"`)}, + {"keystore", regexp.MustCompile(`(?s)"crypto"\s*:\s*\{.{0,400}"ciphertext"\s*:\s*"[0-9a-fA-F]{32,}"`)}, + {"private-key-hex", regexp.MustCompile(`(?i)\b(priv(ate)?[_ -]?key|secret[_ -]?key|privkey|sk)\b["'\s:=]{1,6}(0x)?[0-9a-fA-F]{64}\b`)}, + {"keyring-entry", regexp.MustCompile(`\beyJhbGciOiJQQkVTMi[A-Za-z0-9_-]{20,}`)}, // cosmos file keyring (PBES2 JWE) + {"api-key", regexp.MustCompile(`\bsk-(ant|or-v1|proj)-[A-Za-z0-9_-]{20,}`)}, + {"api-key", regexp.MustCompile(`\b(ghp|gho|ghs|ghu)_[A-Za-z0-9]{30,}\b|\bgithub_pat_[A-Za-z0-9_]{40,}`)}, + {"api-key", regexp.MustCompile(`\bAKIA[0-9A-Z]{16}\b`)}, + {"api-key", regexp.MustCompile(`\bxox[abprs]-[A-Za-z0-9-]{20,}`)}, + {"api-key", regexp.MustCompile(`\bAIza[0-9A-Za-z_-]{35}\b`)}, + {"bot-token", regexp.MustCompile(`\b[0-9]{8,10}:AA[A-Za-z0-9_-]{33}\b`)}, +} + +var wordRe = regexp.MustCompile(`[a-z]+`) + +// mnemonicRun reports whether text holds 12 or more consecutive BIP-39 +// words. Common English function words (the, a, is, of, to, and) aren't +// in the list, so prose essentially never makes such a run. +func mnemonicRun(s string) bool { + run := 0 + for _, w := range wordRe.FindAllString(strings.ToLower(s), -1) { + if _, ok := bip39.ReverseWordMap[w]; ok { + run++ + if run >= 12 { + return true + } + continue + } + run = 0 + } + return false +} + +// Scan reports the kinds of key material in s (nil when clean). +func Scan(s string) []Finding { + if s == "" { + return nil + } + seen := map[string]bool{} + var out []Finding + add := func(k string) { + if !seen[k] { + seen[k] = true + out = append(out, Finding{Kind: k}) + } + } + for _, d := range detectors { + if d.re.MatchString(s) { + add(d.kind) + } + } + if mnemonicRun(s) { + add("mnemonic") + } + sort.Slice(out, func(i, j int) bool { return out[i].Kind < out[j].Kind }) + return out +} + +// Kinds joins findings for messages. +func Kinds(f []Finding) string { + var k []string + for _, x := range f { + k = append(k, x.Kind) + } + return strings.Join(k, ", ") +} + +// Withheld is what replaces a message that carried key material. +func Withheld(f []Finding) string { + return fmt.Sprintf("[withheld by cometcli: this content contained key material (%s) and never leaves this machine. "+ + "Don't ask for it again; keys are only used through cometcli's signing tools.]", Kinds(f)) +} diff --git a/internal/egress/egress_test.go b/internal/egress/egress_test.go new file mode 100644 index 0000000..b897087 --- /dev/null +++ b/internal/egress/egress_test.go @@ -0,0 +1,122 @@ +package egress + +import ( + "strings" + "testing" + + "github.com/cosmos/go-bip39" +) + +func TestScanFindsKeyMaterial(t *testing.T) { + ent, _ := bip39.NewEntropy(256) + mn, _ := bip39.NewMnemonic(ent) + for name, in := range map[string]string{ + "mnemonic": "recover with: " + mn, + "mnemonic upper": strings.ToUpper(mn), + "mnemonic in json": `{"mnemonic":"` + mn + `"}`, + "priv_validator_key": `{"address":"AB","pub_key":{"type":"tendermint/PubKeyEd25519","value":"x"},"priv_key":{"type":"tendermint/PrivKeyEd25519","value":"q2Ej1vT0Yd8kbm0GhX9wJ0Yv6p7c9W3qkH1sXy8bVwQ0c2Vt9tO3y7mJx3qT8f3h4Lr2Vb6ZpN9w0Qn1aYk5sA=="}}`, + "openssh": "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAA\n-----END OPENSSH PRIVATE KEY-----", + "armored cosmos": "-----BEGIN TENDERMINT PRIVATE KEY-----\nkdf: bcrypt\n", + "eth private key": "private key: 0x4c0883a69102937d6231471b5dbb6204fe5129617082792ae468d01a3f362318", + "keystore": `{"crypto":{"cipher":"aes-128-ctr","ciphertext":"5318b4d5bcd28de64ee5559e671353e16f075ecae9f99c7a79a38af5f869aa46","kdf":"scrypt"}}`, + "file keyring": "eyJhbGciOiJQQkVTMi1IUzI1NitBMTI4S1ciLCJjcmVhdGVkIjoiMjAyNi0x", + "anthropic key": "ANTHROPIC_API_KEY=sk-ant-api03-AbCdEfGhIjKlMnOpQrStUvWxYz0123456789", + "openrouter key": "sk-or-v1-0123456789abcdef0123456789abcdef0123456789abcdef", + "github token": "ghp_ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789", + "telegram token": "123456789:AAEhBP0av28pWx0nPnRk3wd0Cf6bS7m1aZQ", + } { + if len(Scan(in)) == 0 { + t.Errorf("%s: not detected", name) + } + } +} + +func TestScanLeavesOrdinaryOutputAlone(t *testing.T) { + for _, in := range []string{ + "The validator is jailed because the node was down for longer than the signed blocks window allows.", + "txhash: 9F86D081884C7D659A2FEAA0C55AD015A3BF4F1B2B0B822CD15D6C15B0F00A08", + `{"pub_key":{"type":"tendermint/PubKeyEd25519","value":"q2Ej1vT0Yd8kbm0GhX9wJ0Yv6p7c9W3qkH1sXy8bVwQ="}}`, + "block hash 0x4c0883a69102937d6231471b5dbb6204fe5129617082792ae468d01a3f362318 at height 1200", + "cosmosvaloper1gk5aka9mc8rw2uxm3xmzrtne526vj9t2ldw5c3 jailed until 2026-10-09T10:00:00Z", + "open sesame: abandon the plan, the river is wide", // a few BIP-39 words, not a run + "word salad: ability able about above absent absorb", // 6-word run + "-----BEGIN PUBLIC KEY-----", + } { + if f := Scan(in); len(f) != 0 { + t.Errorf("false positive %v in %q", f, in) + } + } +} + +func TestModeFor(t *testing.T) { + cases := []struct { + setting, role string + node bool + want Mode + }{ + {"", "validator", true, Strict}, + {"", "", true, Strict}, + {"", "rpc", true, Filtered}, + {"", "validator", false, Filtered}, + {"filtered", "validator", true, Filtered}, + {"strict", "rpc", true, Strict}, + } + for _, c := range cases { + if got := ModeFor(c.setting, c.role, c.node); got != c.want { + t.Errorf("ModeFor(%q,%q,%v) = %s, want %s", c.setting, c.role, c.node, got, c.want) + } + } +} + +func TestSummarizeMasksAndFolds(t *testing.T) { + var in strings.Builder + for h := 100; h < 140; h++ { + in.WriteString("2026-10-09T10:00:00Z INF committed state height=" + itoa(h) + " module=state\n") + } + in.WriteString("ERR dial tcp 10.0.3.7:26656: connection refused peer=4b2e7f0c9a1d3e5f7a9b1c3d5e7f9a1b3c5d7e9f\n") + in.WriteString("validator cosmosvaloper1gk5aka9mc8rw2uxm3xmzrtne526vj9t2ldw5c3 rpc http://val1.internal.example:26657 laddr tcp://0.0.0.0:26657\n") + in.WriteString("[exit code 1]\n") + out := Summarize(in.String()) + for _, leak := range []string{"10.0.3.7", "4b2e7f0c9a1d", "gk5aka9mc8rw", "val1.internal"} { + if strings.Contains(out, leak) { + t.Errorf("summary leaks %q:\n%s", leak, out) + } + } + for _, keep := range []string{"(×40)", "height=139", "connection refused", "cosmosvaloper1‹addr›", "tcp://0.0.0.0:26657", "[exit code 1]", "strict egress"} { + if !strings.Contains(out, keep) { + t.Errorf("summary lost %q:\n%s", keep, out) + } + } +} + +func TestSummarizeKeepsErrorsWhenLong(t *testing.T) { + var in strings.Builder + in.WriteString("panic: runtime error: invalid memory address\n") + for i := 0; i < 500; i++ { + in.WriteString("line kind " + strings.Repeat("x", i%7+1) + " " + string(rune('a'+i%26)) + string(rune('a'+i/26%26)) + "\n") + } + out := Summarize(in.String()) + if !strings.Contains(out, "panic: runtime error") || !strings.Contains(out, "other distinct lines not shown") { + t.Errorf("long output should keep the panic and say what was dropped:\n%s", out[:400]) + } + if n := strings.Count(out, "\n"); n > maxGroups+3 { + t.Errorf("summary has %d lines, cap is %d", n, maxGroups) + } +} + +func TestForModel(t *testing.T) { + raw := "dial 10.1.2.3 failed\n" + if ForModel(Filtered, "bash", raw) != raw { + t.Error("filtered mode passes output through") + } + if strings.Contains(ForModel(Strict, "bash", raw), "10.1.2.3") { + t.Error("strict bash output must be summarized") + } + if ForModel(Strict, "val.status", raw) != raw { + t.Error("structured tools aren't summarized") + } +} + +func itoa(i int) string { + return strings.TrimLeft(strings.Repeat("0", 3)+string(rune('0'+i/100))+string(rune('0'+i/10%10))+string(rune('0'+i%10)), "0") +} diff --git a/internal/egress/summary.go b/internal/egress/summary.go new file mode 100644 index 0000000..eb471d1 --- /dev/null +++ b/internal/egress/summary.go @@ -0,0 +1,183 @@ +package egress + +import ( + "fmt" + "regexp" + "strings" +) + +// Mode is how much raw output may reach the model. +type Mode string + +const ( + // Strict: shell, file and log output reach the model only as local + // summaries (masked, deduplicated lines). Default on validators. + Strict Mode = "strict" + // Filtered: raw output after redaction and the key-material gate. + Filtered Mode = "filtered" +) + +// ModeFor picks the egress mode: an explicit setting wins; otherwise +// validators are strict and everything else filtered. +func ModeFor(setting, role string, nodeMode bool) Mode { + switch Mode(strings.ToLower(setting)) { + case Strict: + return Strict + case Filtered: + return Filtered + } + if nodeMode && (role == "" || role == "validator") { + return Strict + } + return Filtered +} + +// rawTools return raw shell, file or log output; on strict profiles the +// model sees their local summary instead. +var rawTools = map[string]bool{ + "bash": true, "read": true, "grep": true, + "node.logs": true, "fleet.exec": true, "fleet.shell": true, +} + +// maskedTools return structured results with embedded log lines; those +// lines are masked but the structure is kept. +var maskedTools = map[string]bool{ + "node.triage": true, "fleet.triage": true, "val.jail-check": true, "node.config": true, +} + +// ForModel turns a tool's output into what the model may see in mode. +func ForModel(mode Mode, tool, text string) string { + if mode != Strict { + return text + } + switch { + case rawTools[tool]: + return Summarize(text) + case maskedTools[tool]: + return Mask(text) + } + return text +} + +var ( + blobRe = regexp.MustCompile(`[A-Za-z0-9+/_=-]{24,}`) + hexRe = regexp.MustCompile(`\b(0x)?[0-9a-fA-F]{20,}\b`) + ipv4Re = regexp.MustCompile(`\b(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})\b`) + ipv6Re = regexp.MustCompile(`\b([0-9a-fA-F]{1,4}:){3,7}[0-9a-fA-F]{1,4}\b`) + bech32Re = regexp.MustCompile(`\b([a-z]{2,20}1)[02-9ac-hj-np-z]{38,90}\b`) + emailRe = regexp.MustCompile(`\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b`) + urlRe = regexp.MustCompile(`\b([a-z][a-z0-9+.-]*://)([^/\s:@]+)`) + digitsRe = regexp.MustCompile(`\d+`) + alertRe = regexp.MustCompile(`(?i)\b(err(or)?|eror|panic|fatal|fail(ed|ure)?|warn(ing)?|wrn|denied|refused|killed|oom|timeout|timed out|cannot|can't|unable|invalid|corrupt|no space)\b`) +) + +// Mask hides identifiers and blobs in a line while keeping its shape: +// addresses, keys, hashes, IPs and hostnames become placeholders; words, +// numbers, paths and log levels stay. +func Mask(s string) string { + s = emailRe.ReplaceAllString(s, "‹email›") + s = urlRe.ReplaceAllStringFunc(s, func(m string) string { + sub := urlRe.FindStringSubmatch(m) + if sub[2] == "localhost" || sub[2] == "127.0.0.1" || sub[2] == "0.0.0.0" { + return m + } + return sub[1] + "‹host›" + }) + s = bech32Re.ReplaceAllString(s, "${1}‹addr›") + s = hexRe.ReplaceAllString(s, "‹hex›") + s = ipv4Re.ReplaceAllStringFunc(s, func(ip string) string { + if strings.HasPrefix(ip, "127.") || ip == "0.0.0.0" { + return ip + } + return "‹ip›" + }) + s = ipv6Re.ReplaceAllString(s, "‹ip›") + s = blobRe.ReplaceAllStringFunc(s, func(b string) string { + // keep long plain words and paths (snake_case, kebab, dotted names) + if !strings.ContainsAny(b, "0123456789") || strings.Count(b, "/") >= 2 { + return b + } + return "‹blob›" + }) + return s +} + +// maxGroups bounds a summary's distinct lines. +const maxGroups = 60 + +// Summarize makes the local stand-in for raw output: masked lines, with +// repeats (same line up to numbers) folded into one with a count, and +// error-like lines kept first when there are too many to show. +func Summarize(text string) string { + lines := strings.Split(strings.TrimRight(text, "\n"), "\n") + type group struct { + line string // latest masked instance + count int + alert bool + first int + } + idx := map[string]*group{} + var order []*group + for i, l := range lines { + l = strings.TrimRight(l, " \t\r") + if l == "" { + continue + } + m := Mask(l) + key := digitsRe.ReplaceAllString(m, "#") + g := idx[key] + if g == nil { + g = &group{first: i, alert: alertRe.MatchString(m)} + idx[key] = g + order = append(order, g) + } + g.line = m + g.count++ + } + shown := order + dropped := 0 + if len(order) > maxGroups { + var alerts, rest []*group + for _, g := range order { + if g.alert { + alerts = append(alerts, g) + } else { + rest = append(rest, g) + } + } + if len(alerts) > maxGroups { + alerts = alerts[len(alerts)-maxGroups:] // the most recent + } + room := maxGroups - len(alerts) + if room > len(rest) { + room = len(rest) + } + keep := map[*group]bool{} + for _, g := range alerts { + keep[g] = true + } + for _, g := range rest[len(rest)-room:] { // the latest context lines + keep[g] = true + } + shown = nil + for _, g := range order { + if keep[g] { + shown = append(shown, g) + } + } + dropped = len(order) - len(shown) + } + var b strings.Builder + fmt.Fprintf(&b, "[local summary: %d lines, %d distinct; identifiers masked, the raw output stays on this machine (strict egress)]\n", len(lines), len(order)) + for _, g := range shown { + b.WriteString(g.line) + if g.count > 1 { + fmt.Fprintf(&b, " (×%d)", g.count) + } + b.WriteByte('\n') + } + if dropped > 0 { + fmt.Fprintf(&b, "[%d other distinct lines not shown — narrow the command (grep, tail) to see them]\n", dropped) + } + return b.String() +} diff --git a/internal/kb/cases/host.yaml b/internal/kb/cases/host.yaml index 67a98a5..e4279e9 100644 --- a/internal/kb/cases/host.yaml +++ b/internal/kb/cases/host.yaml @@ -149,5 +149,11 @@ - "[change] restart the container if it's not recovering on its own" verify: - "proc.restarts stops increasing; proc.mem_used_pct_of_limit well below 90; wait.until synced" + playbook: + - {do: bash, args: {command: "docker update --memory 4g --memory-swap 4g {unit}", description: "raise the container memory limit to 4 GB"}, note: "raise the memory limit (also fix it in compose so it survives recreation)"} + - {do: node.service, args: {action: restart}, note: "restart with the new limit"} + - {do: wait.until, args: {condition: synced, timeout: 1800}, note: "wait until it has caught up"} + - {do: wait.until, args: {condition: unjailable, timeout: 1800}, when: "val.jailed == true", note: "wait out the jail period"} + - {do: val.unjail, when: "val.jailed == true", note: "unjail (asks you)"} test: signals: {proc.mem_limit_mb: 120, proc.oom_kills: 14, proc.restarts: 7} diff --git a/internal/kb/cases/sync.yaml b/internal/kb/cases/sync.yaml index d35524c..236b415 100644 --- a/internal/kb/cases/sync.yaml +++ b/internal/kb/cases/sync.yaml @@ -36,6 +36,13 @@ - "proc.running true and stays up >5 min; wait.until synced" warnings: - an upgrade halt exits on purpose — restarting the old binary re-halts; follow upgrade-halt-binary-missing + playbook: + - {do: node.logs, args: {lines: 40}, note: "the last lines before it stopped"} + - {do: node.service, args: {action: start}, note: "start the node"} + - {do: wait.until, args: {condition: synced, timeout: 1800}, note: "wait until it has caught up"} + - {do: wait.until, args: {condition: unjailable, timeout: 1800}, when: "val.jailed == true", note: "wait out the jail period"} + - {do: val.unjail, when: "val.jailed == true", note: "unjail (signed by the operator key — asks you)"} + - {do: wait.until, args: {condition: in-consensus, timeout: 600}, when: "val.jailed == true", note: "check it signs again"} test: signals: {proc.running: false, proc.state: exited} diff --git a/internal/kb/cases/upgrade.yaml b/internal/kb/cases/upgrade.yaml index deb89a3..9c40f91 100644 --- a/internal/kb/cases/upgrade.yaml +++ b/internal/kb/cases/upgrade.yaml @@ -60,6 +60,12 @@ - left over from a manual coordinated upgrade fix: - "[change] set halt-height = 0 after the coordinated halt is done; restart" + playbook: + - {do: bash, args: {command: "f={home}/config/app.toml && sed 's/^halt-height = .*/halt-height = 0/' \"$f\" > \"$f.cometcli-tmp\" && cat \"$f.cometcli-tmp\" > \"$f\" && rm \"$f.cometcli-tmp\" && grep '^halt-height' \"$f\"", description: "set halt-height = 0 in app.toml"}, note: "clear the leftover halt-height"} + - {do: node.service, args: {action: restart}, note: "restart so the node picks it up"} + - {do: wait.until, args: {condition: synced, timeout: 1800}, note: "wait until it has caught up"} + - {do: wait.until, args: {condition: unjailable, timeout: 1800}, when: "val.jailed == true", note: "wait out the jail period"} + - {do: val.unjail, when: "val.jailed == true", note: "unjail (asks you)"} test: signals: {app.blocks_to_halt_height: 200, app.halt_height: 1200} diff --git a/internal/kb/cases/validator.yaml b/internal/kb/cases/validator.yaml index e2fc879..e779287 100644 --- a/internal/kb/cases/validator.yaml +++ b/internal/kb/cases/validator.yaml @@ -25,6 +25,12 @@ warnings: - never unjail while catching up — it misses blocks again immediately - never copy priv_validator_key to a second running node to "get back fast" (double sign → tombstone) + playbook: + - {do: val.jail-check, note: "why it was jailed and what blocks the unjail"} + - {do: wait.until, args: {condition: synced, timeout: 1800}, note: "never unjail a node that isn't caught up"} + - {do: wait.until, args: {condition: unjailable, timeout: 1800}, note: "wait out the jail period"} + - {do: val.unjail, note: "unjail (signed by the operator key — asks you)"} + - {do: wait.until, args: {condition: in-consensus, timeout: 600}, note: "check it signs again"} test: signals: {val.jailed: true, val.tombstoned: false} diff --git a/internal/kb/kb.go b/internal/kb/kb.go index 772b268..5cae021 100644 --- a/internal/kb/kb.go +++ b/internal/kb/kb.go @@ -46,6 +46,10 @@ type Case struct { Verify []string `yaml:"verify,omitempty"` Warnings []string `yaml:"warnings,omitempty"` Refs []string `yaml:"refs,omitempty"` + // Playbook is the fix as steps cometcli runs itself, with no model, + // when this case is the clear root cause. Changes and transactions in + // it still ask the operator. + Playbook []PlayStep `yaml:"playbook,omitempty"` Test struct { Signals Signals `yaml:"signals"` } `yaml:"test,omitempty"` @@ -57,6 +61,23 @@ type Case struct { var severityRank = map[string]int{"critical": 4, "high": 3, "medium": 2, "low": 1} +// PlayStep is one playbook step: a registry tool and its arguments. +// String arguments may use {unit}, {home}, {binary} and {container_home} +// from the profile. +type PlayStep struct { + Do string `yaml:"do"` + Args map[string]any `yaml:"args,omitempty"` + // When is a signal condition evaluated on the triage signals; the step + // is skipped when it doesn't hold. + When string `yaml:"when,omitempty"` + Note string `yaml:"note,omitempty"` // what the step is for, shown to the operator + + when *Cond +} + +// Applies reports whether the step runs for these signals. +func (s *PlayStep) Applies(sig Signals) bool { return s.when == nil || s.when.Eval(sig) } + // compile parses and checks a case. func (c *Case) compile() error { if c.ID == "" || c.Title == "" { @@ -90,6 +111,20 @@ func (c *Case) compile() error { } c.any = append(c.any, cond) } + for i := range c.Playbook { + st := &c.Playbook[i] + if st.Do == "" { + return fmt.Errorf("case %s: playbook step %d has no tool (do)", c.ID, i+1) + } + st.when = nil + if st.When != "" { + cond, err := ParseCond(st.When) + if err != nil { + return fmt.Errorf("case %s: playbook step %d: %w", c.ID, i+1, err) + } + st.when = &cond + } + } for _, x := range c.Explains { if x == c.ID { return fmt.Errorf("case %s explains itself", c.ID) @@ -324,6 +359,18 @@ func (c *Case) Render() string { section("fix", c.Fix) section("verify", c.Verify) section("NEVER", c.Warnings) + var play []string + for _, st := range c.Playbook { + line := st.Do + if st.Note != "" { + line += " — " + st.Note + } + if st.When != "" { + line += " (when " + st.When + ")" + } + play = append(play, line) + } + section("playbook (run by cometcli itself on /incident when this is the clear root cause)", play) if len(c.Refs) > 0 { fmt.Fprintf(&b, "refs: %s\n", strings.Join(c.Refs, " ")) } diff --git a/internal/router/intents/chain.yaml b/internal/router/intents/chain.yaml new file mode 100644 index 0000000..bb928c6 --- /dev/null +++ b/internal/router/intents/chain.yaml @@ -0,0 +1,84 @@ +- id: jailed-validators + title: jailed validators on the chain + scope: chain + patterns: + - '^(is )?any(one|body)? (validator )?jailed$' + - '^(list )?jailed validators?$' + - '^who is jailed$' + examples: + - is anyone jailed + - are any validators jailed + - which validators are jailed + - list jailed validators + steps: + - {as: j, tool: chain.validators, args: {status: jailed}} + answer: | + {{$n := get .j.Data "count"}}{{if $n}}**{{$n}} validator(s) jailed:**{{else}}**No validator is jailed.**{{end}} + {{trim .j.Text}} + +- id: validator-count + title: active validator count + scope: chain + patterns: + - '^(what is the )?validators? count$' + - '^how many (active |bonded )?validators?( are there)?$' + examples: + - what is the validator count + - how many validators are in the active set + - how many bonded validators + - active set size + steps: + - {as: v, tool: chain.validators, args: {status: bonded}} + answer: | + **{{get .v.Data "count"}} bonded validators** on the chain. + {{trim .v.Text}} + +- id: proposals + title: proposals in voting + scope: chain + patterns: + - '^(open|active|current) (governance )?proposals?$' + - '^proposals?$' + examples: + - are there any open proposals + - active governance proposals + - what proposals are in voting + - list proposals + steps: + - {as: g, tool: chain.gov, args: {status: voting}} + answer: | + {{$n := count (get .g.Data "proposals")}}{{if $n}}**{{$n}} proposal(s) in voting:**{{else}}**No proposals in voting.**{{end}} + {{trim .g.Text}} + +- id: upgrade-plan + title: scheduled upgrade + scope: chain + patterns: + - '^(is there an? |any )?upgrade (plan|scheduled|planned)$' + - '^upgrade plan$' + examples: + - is there an upgrade scheduled + - any upgrade planned + - what is the next upgrade + - upgrade plan + steps: + - {as: u, tool: chain.upgrade-plan} + answer: | + {{if get .u.Data "plan"}}**Upgrade scheduled:**{{else}}**No upgrade scheduled.**{{end}} + {{trim .u.Text}} + +- id: slashing-params + title: slashing parameters + scope: chain + patterns: + - '^(slashing|chain) param(eter)?s?$' + - '^signed blocks window$' + examples: + - what are the slashing params + - what is the signed blocks window + - downtime jail duration + - chain params + steps: + - {as: p, tool: chain.params} + answer: | + {{trim .p.Text}} diff --git a/internal/router/intents/node.yaml b/internal/router/intents/node.yaml new file mode 100644 index 0000000..ed898b0 --- /dev/null +++ b/internal/router/intents/node.yaml @@ -0,0 +1,134 @@ +# Known questions answered without a model call. Each runs read-only +# tools and renders a fixed answer. Add your own in ~/.cometcli/intents/. +# +# patterns: regexes over the normalized prompt (lowercase, no punctuation) +# examples: phrasings; a prompt close enough to one routes here +# scope: node (one specific node) | chain (any node of the chain) +# answer: Go text/template; each step's result is . with .Text, +# .Data and .Err; .node is the profile name + +- id: node-status + title: node height and sync + patterns: + - '^(node )?status$' + - '^(is (it|the node|my node) )?(synced|in sync|catching up)$' + - '^(what is the |current )?(block )?height$' + examples: + - is my node synced + - is the node catching up + - what height is the node at + - node sync status + - latest block height + steps: + - {as: st, tool: node.status} + answer: | + **{{.node}}** — {{if get .st.Data "catching_up"}}catching up, at height {{get .st.Data "height"}}{{else}}in sync at height {{get .st.Data "height"}}{{end}} + {{trim .st.Text}} + +- id: node-peers + title: peer count + patterns: + - '^(how many )?peers$' + - '^peer count$' + examples: + - how many peers do I have + - how many peers is the node connected to + - peer count + - list peers + steps: + - {as: p, tool: node.peers} + answer: | + **{{.node}}** has {{get .p.Data "count"}} peers{{if not (get .p.Data "listening")}} and is **not listening** for inbound connections{{end}}. + {{trim .p.Text}} + +- id: val-me + title: my validator's status + patterns: + - '^(is )?my validator (ok|okay|bonded|jailed|active|status)$' + - '^am i jailed$' + - '^validator status$' + examples: + - is my validator jailed + - is my validator bonded + - my validator status + - am I jailed + - is the validator active + steps: + - {as: v, tool: val.status} + - {as: s, tool: val.signing, optional: true} + answer: | + {{$j := get .v.Data "jailed"}}**{{get .v.Data "moniker"}}** ({{.node}}) is {{if $j}}**jailed**{{else}}{{get .v.Data "status"}}{{end}}. + {{trim .v.Text}} + {{if not .s.Err}}{{trim .s.Text}}{{end}} + {{if $j}}Run `/incident` to find why and recover it.{{end}} + +- id: val-signing + title: missed blocks and uptime + patterns: + - '^(missed blocks|uptime|signing info)$' + examples: + - how many blocks have I missed + - what is my uptime + - missed blocks count + - signing info + steps: + - {as: s, tool: val.signing} + answer: | + **{{.node}}** — {{get .s.Data "missed"}} missed in the {{get .s.Data "window"}}-block window ({{get .s.Data "uptime_pct"}}% uptime){{if get .s.Data "tombstoned"}}, **TOMBSTONED**{{end}}. + {{trim .s.Text}} + +- id: val-consensus + title: in consensus and signing + patterns: + - '^am i (signing|in consensus)$' + - '^(is my validator )?(signing|in consensus)$' + examples: + - am I signing blocks + - is my validator in consensus + - am I in the active set and signing + steps: + - {as: c, tool: val.consensus} + answer: | + **{{.node}}** {{if get .c.Data "in_consensus"}}is in consensus — signed {{get .c.Data "signed"}} of the last {{get .c.Data "checked"}} blocks{{else}}is **not** in consensus{{end}}. + {{trim .c.Text}} + +- id: val-rewards + title: rewards and commission + patterns: + - '^(my )?(rewards|commission)$' + examples: + - how much rewards do I have + - outstanding rewards and commission + - my commission + steps: + - {as: r, tool: val.rewards} + answer: | + **{{.node}}** rewards: + {{trim .r.Text}} + +- id: val-pending-votes + title: proposals I haven't voted on + patterns: + - '^(do i (need|have) to vote|pending votes|have i voted)$' + examples: + - do I need to vote on anything + - have I voted on the open proposals + - pending votes + steps: + - {as: v, tool: val.votes} + answer: | + {{$n := count (get .v.Data "pending_votes")}}{{if $n}}**{{$n}} proposal(s) still need your vote.**{{else}}No votes pending.{{end}} + {{trim .v.Text}} + +- id: balance + title: signer balance + patterns: + - '^(my |account )?balance$' + examples: + - what is my balance + - how much do I have in my account + - account balance + steps: + - {as: b, tool: chain.balance} + answer: | + {{trim .b.Text}} diff --git a/internal/router/router.go b/internal/router/router.go new file mode 100644 index 0000000..84ccbd2 --- /dev/null +++ b/internal/router/router.go @@ -0,0 +1,369 @@ +// Package router answers known questions without a model call. Every +// prompt is classified locally first: a question the intent catalog +// covers ("is anyone jailed?", "how many peers?") runs its read-only tools +// and renders a fixed answer; anything open-ended, any action, and +// anything the catalog doesn't clearly cover goes to the model. +// +// Intents are data (YAML), like knowledge-base cases: built-ins are +// embedded, and ~/.cometcli/intents/*.yaml adds or overrides by id. +package router + +import ( + "bytes" + "embed" + "fmt" + "io/fs" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "sync" + "text/template" + "time" + + "gopkg.in/yaml.v3" +) + +//go:embed intents/*.yaml +var builtin embed.FS + +// Intent is one known question and how to answer it locally. +type Intent struct { + ID string `yaml:"id"` + Title string `yaml:"title"` + Patterns []string `yaml:"patterns"` // regexes over the normalized prompt; a hit routes locally + Examples []string `yaml:"examples"` // phrasings; a close enough prompt routes locally + // Scope: "node" needs one node (the session's, the one named, or the + // active profile); "chain" can be answered from any node of the chain. + Scope string `yaml:"scope"` + Steps []Step `yaml:"steps"` + Answer string `yaml:"answer"` // text/template over the steps' results + TTL time.Duration `yaml:"ttl"` // how long an answer is reused (default 30s) + + res []*regexp.Regexp + tpl *template.Template + ex [][]string // examples as token sets +} + +// Step is a read-only tool call. +type Step struct { + As string `yaml:"as"` + Tool string `yaml:"tool"` + Args map[string]any `yaml:"args"` + Optional bool `yaml:"optional"` +} + +// Result is a step's outcome, as the answer template sees it. +type Result struct { + Text string + Data map[string]any + Err string +} + +// Catalog is the set of intents. +type Catalog struct { + Intents []*Intent +} + +// Load reads the built-in intents and those in dirs (later ids override). +func Load(dirs ...string) (*Catalog, error) { + byID := map[string]*Intent{} + var order []string + add := func(name string, raw []byte) error { + var list []*Intent + if err := yaml.Unmarshal(raw, &list); err != nil { + return fmt.Errorf("%s: %w", name, err) + } + for _, in := range list { + if err := in.compile(); err != nil { + return fmt.Errorf("%s: intent %s: %w", name, in.ID, err) + } + if _, ok := byID[in.ID]; !ok { + order = append(order, in.ID) + } + byID[in.ID] = in + } + return nil + } + files, _ := fs.Glob(builtin, "intents/*.yaml") + sort.Strings(files) + for _, f := range files { + raw, _ := builtin.ReadFile(f) + if err := add(f, raw); err != nil { + return nil, err + } + } + for _, d := range dirs { + user, _ := filepath.Glob(filepath.Join(d, "*.yaml")) + sort.Strings(user) + for _, f := range user { + raw, err := os.ReadFile(f) + if err != nil { + return nil, err + } + if err := add(f, raw); err != nil { + return nil, err + } + } + } + c := &Catalog{} + for _, id := range order { + c.Intents = append(c.Intents, byID[id]) + } + return c, nil +} + +func (in *Intent) compile() error { + if in.ID == "" || len(in.Steps) == 0 || in.Answer == "" { + return fmt.Errorf("needs id, steps and answer") + } + if in.Scope == "" { + in.Scope = "node" + } + if in.TTL == 0 { + in.TTL = 30 * time.Second + } + for _, p := range in.Patterns { + re, err := regexp.Compile(p) + if err != nil { + return err + } + in.res = append(in.res, re) + } + for _, e := range in.Examples { + in.ex = append(in.ex, tokens(Normalize(e))) + } + for i := range in.Steps { + if in.Steps[i].As == "" { + in.Steps[i].As = fmt.Sprintf("s%d", i) + } + } + tpl, err := template.New(in.ID).Funcs(funcs).Option("missingkey=zero").Parse(in.Answer) + if err != nil { + return err + } + in.tpl = tpl + return nil +} + +// --- classification ---------------------------------------------------------- + +// Match is the classifier's verdict. +type Match struct { + Intent *Intent // nil: the model handles it + Score float64 + Why string // why it went where it went (for /why and tests) +} + +// Threshold is the example similarity needed to answer locally. +const Threshold = 0.8 + +var ( + punctRe = regexp.MustCompile(`[^a-z0-9\s.\-_]+`) + spaceRe = regexp.MustCompile(`\s+`) + // open-ended or acting prompts always go to the model + openRe = regexp.MustCompile(`\b(why|how (do|can|should|to|would)|explain|should (i|we)|what if|help|fix|investigate|troubleshoot|diagnose|recommend|compare|write|create|vote (yes|no|abstain|on|for)|unjail|restart|stop|start|delegate|undelegate|send|withdraw|edit|change|update|upgrade to|install|delete|remove|prune|reset|migrate|but|because|then|after|before)\b`) +) + +// Normalize lowercases a prompt and strips punctuation and extra space. +func Normalize(s string) string { + s = strings.ToLower(strings.TrimSpace(s)) + s = punctRe.ReplaceAllString(s, " ") + return strings.TrimSpace(spaceRe.ReplaceAllString(s, " ")) +} + +var stop = map[string]bool{ + "a": true, "an": true, "the": true, "is": true, "are": true, "am": true, "do": true, "does": true, + "my": true, "our": true, "me": true, "i": true, "we": true, "of": true, "on": true, "in": true, + "for": true, "to": true, "it": true, "this": true, "that": true, "there": true, "what": true, + "whats": true, "please": true, "can": true, "you": true, "tell": true, "show": true, "check": true, + "right": true, "now": true, "currently": true, "current": true, "any": true, "anything": true, + "node": true, "s": true, "with": true, "at": true, "be": true, "have": true, "has": true, +} + +var synonyms = map[string]string{ + "val": "validator", "vals": "validator", "validators": "validator", "jail": "jailed", + "jails": "jailed", "proposals": "proposal", "props": "proposal", "gov": "proposal", "governance": "proposal", + "peer": "peers", "connections": "peers", "connected": "peers", "blockheight": "height", "block": "height", + "blocks": "height", "synced": "sync", "syncing": "sync", "catching": "sync", "caught": "sync", + "missed": "miss", "missing": "miss", "misses": "miss", "votes": "vote", "voted": "vote", "voting": "vote", + "rewards": "reward", "commissions": "commission", "balances": "balance", "funds": "balance", + "upgrades": "upgrade", "scheduled": "upgrade", "planned": "upgrade", "count": "many", "number": "many", + "total": "many", "signing": "sign", "signed": "sign", "signs": "sign", "params": "param", "parameters": "param", +} + +func tokens(s string) []string { + seen := map[string]bool{} + var out []string + for _, w := range strings.Fields(s) { + w = strings.Trim(w, ".-_") + if v, ok := synonyms[w]; ok { + w = v + } + if stop[w] || w == "" || seen[w] { + continue + } + seen[w] = true + out = append(out, w) + } + return out +} + +// dice is the overlap of two token sets: 2|A∩B| / (|A|+|B|). +func dice(a, b []string) float64 { + if len(a) == 0 || len(b) == 0 { + return 0 + } + set := map[string]bool{} + for _, w := range a { + set[w] = true + } + n := 0 + for _, w := range b { + if set[w] { + n++ + } + } + return 2 * float64(n) / float64(len(a)+len(b)) +} + +// Classify decides whether a prompt can be answered locally. +func (c *Catalog) Classify(prompt string) Match { + if strings.Contains(strings.TrimSpace(prompt), "\n") { + return Match{Why: "multi-line prompt"} + } + n := Normalize(prompt) + if n == "" { + return Match{Why: "empty"} + } + if len(strings.Fields(n)) > 14 { + return Match{Why: "long prompt"} + } + if m := openRe.FindString(n); m != "" { + return Match{Why: "open-ended or an action (" + m + ")"} + } + for _, in := range c.Intents { + for _, re := range in.res { + if re.MatchString(n) { + return Match{Intent: in, Score: 1, Why: "pattern " + re.String()} + } + } + } + tk := tokens(n) + var best *Intent + bestScore := 0.0 + for _, in := range c.Intents { + for _, ex := range in.ex { + if s := dice(tk, ex); s > bestScore { + best, bestScore = in, s + } + } + } + if best != nil && bestScore >= Threshold { + return Match{Intent: best, Score: bestScore, Why: fmt.Sprintf("similar to %s examples (%.2f)", best.ID, bestScore)} + } + why := "no known question matches" + if best != nil { + why = fmt.Sprintf("closest is %s at %.2f, below %.2f", best.ID, bestScore, Threshold) + } + return Match{Score: bestScore, Why: why} +} + +// --- answers ------------------------------------------------------------------- + +var funcs = template.FuncMap{ + "trim": strings.TrimSpace, + "lines": func(s string) []string { + return strings.Split(strings.TrimSpace(s), "\n") + }, + "get": func(m map[string]any, k string) any { + if m == nil { + return nil + } + return m[k] + }, + // filter keeps the maps in a list whose key equals v + "filter": func(list any, k string, v any) []map[string]any { + var out []map[string]any + switch l := list.(type) { + case []map[string]any: + for _, m := range l { + if fmt.Sprint(m[k]) == fmt.Sprint(v) { + out = append(out, m) + } + } + case []any: + for _, x := range l { + if m, ok := x.(map[string]any); ok && fmt.Sprint(m[k]) == fmt.Sprint(v) { + out = append(out, m) + } + } + } + return out + }, + "count": func(x any) int { + switch v := x.(type) { + case []map[string]any: + return len(v) + case []any: + return len(v) + case string: + return len(v) + case map[string]any: + return len(v) + } + return 0 + }, +} + +// Render fills the intent's answer from its steps' results. +func (in *Intent) Render(node string, results map[string]Result) (string, error) { + var b bytes.Buffer + data := map[string]any{"node": node} + for k, v := range results { + data[k] = v + } + if err := in.tpl.Execute(&b, data); err != nil { + return "", err + } + out := strings.TrimSpace(b.String()) + if out == "" { + return "", fmt.Errorf("empty answer") + } + return out, nil +} + +// --- cache --------------------------------------------------------------------- + +// Cache keeps recent local answers. +type Cache struct { + mu sync.Mutex + m map[string]cached +} + +type cached struct { + text string + exp time.Time +} + +// Get returns a fresh answer for key. +func (c *Cache) Get(key string, now time.Time) (string, bool) { + c.mu.Lock() + defer c.mu.Unlock() + e, ok := c.m[key] + if !ok || now.After(e.exp) { + return "", false + } + return e.text, true +} + +// Put stores an answer for ttl. +func (c *Cache) Put(key, text string, ttl time.Duration, now time.Time) { + c.mu.Lock() + defer c.mu.Unlock() + if c.m == nil { + c.m = map[string]cached{} + } + c.m[key] = cached{text: text, exp: now.Add(ttl)} +} diff --git a/internal/router/router_test.go b/internal/router/router_test.go new file mode 100644 index 0000000..94a5afb --- /dev/null +++ b/internal/router/router_test.go @@ -0,0 +1,107 @@ +package router + +import ( + "os" + "strings" + "testing" + "time" +) + +func TestClassify(t *testing.T) { + c, err := Load() + if err != nil { + t.Fatal(err) + } + local := map[string]string{ + "is anyone jailed?": "jailed-validators", + "Any validators jailed": "jailed-validators", + "what is the validator count": "validator-count", + "how many validators?": "validator-count", + "How many peers do I have?": "node-peers", + "peers": "node-peers", + "is my node synced?": "node-status", + "status": "node-status", + "is my validator jailed?": "val-me", + "am I jailed": "val-me", + "open proposals": "proposals", + "are there any open proposals?": "proposals", + "is there an upgrade scheduled?": "upgrade-plan", + "what is my uptime": "val-signing", + "do I need to vote?": "val-pending-votes", + "what's the signed blocks window": "slashing-params", + "am I signing": "val-consensus", + "my balance": "balance", + } + for q, want := range local { + m := c.Classify(q) + if m.Intent == nil || m.Intent.ID != want { + got := "model" + if m.Intent != nil { + got = m.Intent.ID + } + t.Errorf("%q → %s (%s), want %s", q, got, m.Why, want) + } + } + for _, q := range []string{ + "why is my validator jailed?", + "unjail my validator", + "restart the node", + "how do I add a peer", + "vote yes on proposal 3", + "is my validator jailed and why did it happen", + "explain the slashing params", + "the node keeps restarting every few seconds, nothing in the logs", + "what's the weather like", + "check disk usage on the host", + "compare peers with yesterday", + "is anyone jailed\nalso check peers", + } { + if m := c.Classify(q); m.Intent != nil { + t.Errorf("%q should go to the model, routed to %s (%s)", q, m.Intent.ID, m.Why) + } + } +} + +func TestRender(t *testing.T) { + c, _ := Load() + var in *Intent + for _, x := range c.Intents { + if x.ID == "jailed-validators" { + in = x + } + } + out, err := in.Render("val1", map[string]Result{"j": {Text: "no validator is jailed\ntotal: 0\n", Data: map[string]any{"count": 0}}}) + if err != nil || !strings.HasPrefix(out, "**No validator is jailed.**") { + t.Fatalf("%q %v", out, err) + } + out, _ = in.Render("val1", map[string]Result{"j": {Text: "val3 JAILED\ntotal: 1\n", Data: map[string]any{"count": 1}}}) + if !strings.HasPrefix(out, "**1 validator(s) jailed:**") { + t.Fatalf("%q", out) + } +} + +func TestCacheExpires(t *testing.T) { + var c Cache + now := time.Now() + c.Put("k", "v", 30*time.Second, now) + if v, ok := c.Get("k", now.Add(10*time.Second)); !ok || v != "v" { + t.Fatal("fresh entry missing") + } + if _, ok := c.Get("k", now.Add(31*time.Second)); ok { + t.Fatal("stale entry served") + } +} + +func TestUserIntentsOverride(t *testing.T) { + d := t.TempDir() + _ = writeFile(d+"/mine.yaml", "- id: node-peers\n examples: [peer roster]\n steps: [{tool: node.peers}]\n answer: custom\n") + c, err := Load(d) + if err != nil { + t.Fatal(err) + } + if m := c.Classify("peer roster"); m.Intent == nil || m.Intent.Answer != "custom" { + t.Fatalf("user intent should override the built-in: %+v", m) + } +} + +func writeFile(p, s string) error { return os.WriteFile(p, []byte(s), 0o644) } diff --git a/internal/testutil/fakenode/fakenode.go b/internal/testutil/fakenode/fakenode.go index c33bcd6..192914e 100644 --- a/internal/testutil/fakenode/fakenode.go +++ b/internal/testutil/fakenode/fakenode.go @@ -99,6 +99,7 @@ type Node struct { Proposals []*govv1.Proposal Simulations int + simSigs [][]byte // signatures of the latest simulated tx Broadcasts []Tx // accepted to the mempool, in order Rejected []string // raw logs of rejected broadcasts committed map[string]*abciv1beta1.TxResponse @@ -265,6 +266,10 @@ func (s *txSrv) Simulate(_ context.Context, r *txv1beta1.SimulateRequest) (*txv1 n.mu.Lock() defer n.mu.Unlock() n.Simulations++ + var tr txv1beta1.TxRaw + if proto.Unmarshal(r.TxBytes, &tr) == nil { + n.simSigs = tr.Signatures + } d, err := n.verifyTx(r.TxBytes, n.AccNum, true) if err != nil { return nil, status.Error(codes.Unknown, err.Error()) @@ -468,3 +473,22 @@ func (n *Node) Profile(t testing.TB, algo string) *config.Profile { } return p } + +// LastSimSignatures is how many signatures the latest simulated tx had. +func (n *Node) LastSimSignatures() int { + n.mu.Lock() + defer n.mu.Unlock() + return len(n.simSigs) +} + +// LastSimSigned reports whether the latest simulated tx was really signed. +func (n *Node) LastSimSigned() bool { + n.mu.Lock() + defer n.mu.Unlock() + for _, s := range n.simSigs { + if len(s) > 0 { + return true + } + } + return false +} diff --git a/internal/toolkit/context.go b/internal/toolkit/context.go index 71e5d6a..350d50b 100644 --- a/internal/toolkit/context.go +++ b/internal/toolkit/context.go @@ -53,6 +53,9 @@ type Context struct { // HookDecision is a PreToolUse hook's verdict for this call: // "allow" skips the prompt, "ask" forces one ("" = no opinion). HookDecision string + // Purpose is why the current action is being taken (the agent's + // stated reason), shown when a key is about to be used. + Purpose string mu sync.Mutex comet *comet.Client diff --git a/internal/toolkit/keyuse.go b/internal/toolkit/keyuse.go new file mode 100644 index 0000000..5165606 --- /dev/null +++ b/internal/toolkit/keyuse.go @@ -0,0 +1,75 @@ +package toolkit + +import ( + "fmt" + "strings" + + "github.com/abhijitkrm/cometcli/internal/audit" +) + +// KeyUseMark marks an approval request as a key use (front-ends may treat +// it specially: remote approvals of key use expire sooner). +const KeyUseMark = "_key_use" + +// KeyUse is one use of a signing key, shown to the operator before the key +// is touched. +type KeyUse struct { + Address string // the account the key signs as + Where string // which key and where it lives (signer.Describe) + Op string // what it signs, e.g. "transaction" + Doc string // the decoded document: messages, fee, chain, sequence + Purpose string // why, in the agent's or operator's words +} + +func (k KeyUse) String() string { + var b strings.Builder + fmt.Fprintf(&b, "KEY ACCESS — sign a %s and broadcast it\n", k.Op) + fmt.Fprintf(&b, " account: %s\n", k.Address) + fmt.Fprintf(&b, " key: %s — the key stays there; cometcli only gets the signature\n", k.Where) + if k.Purpose != "" { + fmt.Fprintf(&b, " why: %s\n", k.Purpose) + } + b.WriteString(" scope: this one signature — the next use asks again\n") + b.WriteString(k.Doc) + return b.String() +} + +// ApproveKeyUse asks the operator, every time, before a key signs. No +// permission rule, mode, autopilot or earlier approval covers a key use, +// and the decision is audited as its own event. +func ApproveKeyUse(c *Context, k KeyUse, detail map[string]any) error { + if k.Purpose == "" { + k.Purpose = c.Purpose + } + if detail == nil { + detail = map[string]any{} + } + detail[KeyUseMark] = true + delete(detail, "doc") // the prompt carries it; front-ends would show it twice + prompt := k.String() + name := "" + if c.Profile != nil { + name = c.Profile.Name + } + var ok bool + var err error + if c.Approver == nil { + err = fmt.Errorf("signing needs the operator's approval and there's no one to ask — run it interactively") + } else { + ok, err = c.Approver(c, prompt, TierOnChain, detail) + } + if c.Audit != nil { + _ = c.Audit.Log(audit.KindKey, name, map[string]any{ + "account": k.Address, "key": k.Where, "op": k.Op, "purpose": k.Purpose, + "tool": c.ToolName, "approved": ok && err == nil, + }) + c.Audit.Approval(name, strings.SplitN(prompt, "\n", 2)[0], ok && err == nil) + } + if err != nil { + return err + } + if !ok { + return fmt.Errorf("denied: the operator didn't approve using the key for %s", k.Address) + } + return nil +} diff --git a/internal/tools/common/common.go b/internal/tools/common/common.go index 3cfd71d..4830e15 100644 --- a/internal/tools/common/common.go +++ b/internal/tools/common/common.go @@ -234,7 +234,9 @@ func BroadcastMsgs(c *toolkit.Context, msgs tx.Msgs, memo string, meta map[strin for k, v := range meta { detail[k] = v } - if err := c.Approve("broadcast transaction\n"+built.Doc.String(), toolkit.TierOnChain, detail); err != nil { + // signing is key access: asked every time, never auto-approved + use := toolkit.KeyUse{Address: tb.Address(), Where: tb.Signer().Describe(), Op: "transaction", Doc: built.Doc.String(), Purpose: meta["purpose"]} + if err := toolkit.ApproveKeyUse(c, use, detail); err != nil { return nil, err } // sign only after approval: a container keyring asks for its diff --git a/internal/tools/common/keyuse_test.go b/internal/tools/common/keyuse_test.go new file mode 100644 index 0000000..d071f24 --- /dev/null +++ b/internal/tools/common/keyuse_test.go @@ -0,0 +1,64 @@ +package common + +import ( + "strings" + "testing" + + "github.com/abhijitkrm/cometcli/internal/toolkit" + "github.com/abhijitkrm/cometcli/internal/tx" +) + +// Signing is key access: asked every time with what, where and why — +// whatever auto-approval the session has. +func TestKeyUseIsAlwaysAskedWithDetail(t *testing.T) { + n, c, asked := setup(t, "eth_secp256k1") + c.AutoApproveBelow = toolkit.TierOnChain + 1 // as permissive as a context can be + c.Purpose = "unjail after the node was restarted and synced" + var details []map[string]any + approve := c.Approver + c.Approver = func(cc *toolkit.Context, prompt string, tier toolkit.Tier, d map[string]any) (bool, error) { + details = append(details, d) + return approve(cc, prompt, tier, d) + } + for i := 0; i < 2; i++ { + if _, err := BroadcastMsgs(c, send("cosmos1dest"), "", nil, tx.Options{}); err != nil { + t.Fatal(err) + } + } + if len(*asked) != 2 { + t.Fatalf("each signature must be approved: asked %d times for 2", len(*asked)) + } + p := (*asked)[0] + tb, _ := c.TxSigner() + if tb.Address() == "" { + t.Fatal("no signer address") + } + for _, want := range []string{"KEY ACCESS", "account: " + tb.Address(), "cometcli keyring", "why: unjail after", "this one signature", "/cosmos.bank.v1beta1.MsgSend"} { + if !strings.Contains(p, want) { + t.Errorf("approval lacks %q:\n%s", want, p) + } + } + if details[0][toolkit.KeyUseMark] != true { + t.Error("key use isn't marked for front-ends") + } + if bs, _, _ := n.Snapshot(); len(bs) != 2 { + t.Fatalf("broadcasts = %d", len(bs)) + } +} + +func TestDeclinedKeyUseNeverSigns(t *testing.T) { + n, c, _ := setup(t, "secp256k1") + c.Approver = func(*toolkit.Context, string, toolkit.Tier, map[string]any) (bool, error) { return false, nil } + _, err := BroadcastMsgs(c, send("cosmos1dest"), "", nil, tx.Options{}) + if err == nil || !strings.Contains(err.Error(), "didn't approve using the key") { + t.Fatalf("want a key-use denial, got %v", err) + } + // gas was simulated with an unsigned tx: the key wasn't touched + bs, _, sims := n.Snapshot() + if len(bs) != 0 || sims != 1 { + t.Fatalf("broadcasts=%d sims=%d", len(bs), sims) + } + if s := n.LastSimSignatures(); s != 1 || n.LastSimSigned() { + t.Fatalf("simulation tx should carry one empty signature (got %d, signed=%v)", s, n.LastSimSigned()) + } +} diff --git a/internal/tools/fs/fs.go b/internal/tools/fs/fs.go index 84c5949..914eae6 100644 --- a/internal/tools/fs/fs.go +++ b/internal/tools/fs/fs.go @@ -82,7 +82,7 @@ func (t *target) inRoot(c *toolkit.Context) bool { return err == nil && rel != ".." && !strings.HasPrefix(rel, "../") } -var protectedRe = regexp.MustCompile(`priv_validator_key\.json$|node_key\.json$|(^|/)\.?mnemonics?(\.txt)?(/|$)|mnemonic|(^|/)keyring-(file|test|os)(/|$)|/\.ssh/id_|/\.cometcli/keys(/|$)|/\.gnupg/`) +var protectedRe = regexp.MustCompile(`priv_validator_key\.json$|node_key\.json$|(^|/)\.?mnemonics?(\.txt)?(/|$)|mnemonic|(^|/)keyring-(file|test|os)(/|$)|/\.ssh/id_|/\.cometcli/keys(/|$)|/\.gnupg/|(^|/)keystore(/|$)|(^|/)utc--[0-9]`) // forbidden explains why a path may never pass through the agent. func forbidden(p string, write bool) string { diff --git a/internal/tools/registry_test.go b/internal/tools/registry_test.go index 9b04bef..f96fb81 100644 --- a/internal/tools/registry_test.go +++ b/internal/tools/registry_test.go @@ -5,6 +5,8 @@ import ( "strings" "testing" + "github.com/abhijitkrm/cometcli/internal/kb" + "github.com/abhijitkrm/cometcli/internal/router" "github.com/abhijitkrm/cometcli/internal/toolkit" ) @@ -32,3 +34,41 @@ func TestToolSchemasHaveNoNulls(t *testing.T) { } } } + +// Playbooks and local answers name tools by string: every name must exist, +// and local answers may only read. +func TestPlaybooksAndIntentsUseRealTools(t *testing.T) { + reg := toolkit.NewRegistry() + RegisterAll(reg) + base := kb.Load("", "") + if len(base.Errs) > 0 { + t.Fatalf("kb: %v", base.Errs) + } + n := 0 + for id, c := range base.Cases { + for _, st := range c.Playbook { + n++ + if _, ok := reg.Get(st.Do); !ok { + t.Errorf("case %s: playbook step %q is not a tool", id, st.Do) + } + } + } + if n == 0 { + t.Fatal("no playbooks found") + } + cat, err := router.Load() + if err != nil { + t.Fatal(err) + } + for _, in := range cat.Intents { + for _, st := range in.Steps { + tl, ok := reg.Get(st.Tool) + switch { + case !ok: + t.Errorf("intent %s: %q is not a tool", in.ID, st.Tool) + case tl.Tier() > toolkit.TierDiagnose: + t.Errorf("intent %s: %s is a %s tool; local answers only read", in.ID, st.Tool, tl.Tier()) + } + } + } +} diff --git a/internal/tools/shell/classify.go b/internal/tools/shell/classify.go index dc429f0..9d52a25 100644 --- a/internal/tools/shell/classify.go +++ b/internal/tools/shell/classify.go @@ -379,7 +379,7 @@ func isDigits(s string) bool { // --- sensitive material -------------------------------------------------- -var protectedRe = regexp.MustCompile(`priv_validator_key\.json|node_key\.json|(^|/)\.?mnemonics?(\.txt)?(/|$)|mnemonic|keyring-(file|test|os)|/\.ssh/id_|\.cometcli/keys|\.gnupg/`) +var protectedRe = regexp.MustCompile(`priv_validator_key\.json|node_key\.json|(^|/)\.?mnemonics?(\.txt)?(/|$)|mnemonic|keyring-(file|test|os)|/\.ssh/id_|\.cometcli/keys|\.gnupg/|(^|/)keystore(/|$)|(^|/)utc--[0-9]`) func protectedPath(word string) (bool, string) { if m := protectedRe.FindString(strings.ToLower(word)); m != "" { diff --git a/internal/tools/triage/tool.go b/internal/tools/triage/tool.go index be4439f..954de33 100644 --- a/internal/tools/triage/tool.go +++ b/internal/tools/triage/tool.go @@ -68,10 +68,21 @@ func (Triage) Run(c *toolkit.Context, a toolkit.Args) (*toolkit.Result, error) { base := LoadKB(c) hits := base.Match(r.Signals, r.Chain) return &toolkit.Result{Text: Render(r, hits, a.String("signals", "")), Data: map[string]any{ - "chain": r.Chain, "signals": r.Signals, "sources": r.Sources, "cases": hitIDs(hits), + "chain": r.Chain, "signals": r.Signals, "sources": r.Sources, "cases": hitIDs(hits), "roots": rootIDs(hits), }}, nil } +// rootIDs are the incident cases nothing else explains, best first. +func rootIDs(hits []kb.Hit) []string { + var ids []string + for _, h := range hits { + if h.SymptomOf == "" && h.Case.Kind != "advisory" { + ids = append(ids, h.Case.ID) + } + } + return ids +} + func hitIDs(hits []kb.Hit) []string { var ids []string for _, h := range hits { diff --git a/internal/tui/chatui/chatui.go b/internal/tui/chatui/chatui.go index 0085ccb..5cf4397 100644 --- a/internal/tui/chatui/chatui.go +++ b/internal/tui/chatui/chatui.go @@ -1147,6 +1147,10 @@ func describe(p *pendingApproval) (string, string) { } title := "Approval needed" switch { + case d[toolkit.KeyUseMark] == true: + title = "Key access · sign and broadcast" + lines = append(lines, p.prompt) + shown[toolkit.KeyUseMark] = true case p.tier == toolkit.TierOnChain: title = "Transaction" lines = append(lines, p.prompt) diff --git a/internal/tui/chatui/testdata/slash_menu.golden b/internal/tui/chatui/testdata/slash_menu.golden index 4cdcbd8..6e742aa 100644 --- a/internal/tui/chatui/testdata/slash_menu.golden +++ b/internal/tui/chatui/testdata/slash_menu.golden @@ -4,5 +4,5 @@ ╰──────────────────────────────────────────────────────────────────────────────╯ ❯ /clear clear conversation, start a new audit session /compact [focus] summarize the conversation to free context - /cost token usage for this session + /cost token usage for this session, and prompts answered locally ? for shortcuts general · fake/m \ No newline at end of file diff --git a/internal/tx/signer.go b/internal/tx/signer.go index fc1d66e..cf4c555 100644 --- a/internal/tx/signer.go +++ b/internal/tx/signer.go @@ -39,9 +39,19 @@ type LocalSigner struct { func (s *LocalSigner) Address() string { return s.Addr } func (s *LocalSigner) Describe() string { return "cometcli keyring (" + s.Key.Name + ")" } -// SimBytes signs for real — simulation of a signed tx is the most faithful. -func (s *LocalSigner) SimBytes(ctx context.Context, b *Builder, p *Prepared) ([]byte, error) { - return s.Sign(ctx, b, p) +// SimBytes builds an unsigned tx carrying the public key: the SDK skips +// signature checks when simulating, and the key isn't touched before the +// operator approves the signature. +func (s *LocalSigner) SimBytes(_ context.Context, b *Builder, p *Prepared) ([]byte, error) { + pk, err := s.pubKeyAny() + if err != nil { + return nil, err + } + authInfo, err := b.authInfo(pk, p.Opt, p.Seq) + if err != nil { + return nil, err + } + return proto.Marshal(&txv1beta1.TxRaw{BodyBytes: p.Body, AuthInfoBytes: authInfo, Signatures: [][]byte{{}}}) } func (s *LocalSigner) Sign(_ context.Context, b *Builder, p *Prepared) ([]byte, error) { diff --git a/internal/watch/watch.go b/internal/watch/watch.go index 5791691..402489d 100644 --- a/internal/watch/watch.go +++ b/internal/watch/watch.go @@ -249,6 +249,9 @@ func (w *Watcher) Loop(ctx context.Context, interval time.Duration, once bool) { } } +// KeyUseTimeout bounds a remote approval of a key use. +var KeyUseTimeout = 5 * time.Minute + // RemoteApprover adapts an asker (Telegram) into a toolkit.Approver: // transactions are refused unless allowTx, and the question carries the // tool's detail (command, diff, tx document). @@ -277,6 +280,13 @@ func RemoteApprover(ask func(ctx context.Context, text string) (bool, string, er if c != nil { ctx = c } + if detail[toolkit.KeyUseMark] == true { + // a key use is approved for this moment only + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, KeyUseTimeout) + defer cancel() + b.WriteString("\n\n⏱ expires in " + KeyUseTimeout.String()) + } ok, _, err := ask(ctx, b.String()) if err != nil { return false, fmt.Errorf("remote approval failed: %w", err) diff --git a/internal/watch/watch_test.go b/internal/watch/watch_test.go index a777ef8..27b5ca9 100644 --- a/internal/watch/watch_test.go +++ b/internal/watch/watch_test.go @@ -179,3 +179,23 @@ func TestTelegramApproval(t *testing.T) { t.Fatalf("timeout: ok=%v edited=%v", ok, f2.edited) } } + +func TestRemoteKeyUseApprovalExpires(t *testing.T) { + var text string + var deadline time.Time + ap := RemoteApprover(func(ctx context.Context, s string) (bool, string, error) { + text = s + deadline, _ = ctx.Deadline() + return true, "", nil + }, true) + ok, err := ap(nil, "KEY ACCESS — sign a transaction", toolkit.TierOnChain, map[string]any{toolkit.KeyUseMark: true}) + if err != nil || !ok { + t.Fatal(ok, err) + } + if !strings.Contains(text, "expires in 5m") || strings.Contains(text, toolkit.KeyUseMark) { + t.Errorf("message: %q", text) + } + if time.Until(deadline) > KeyUseTimeout || time.Until(deadline) < KeyUseTimeout-time.Minute { + t.Errorf("key use approval should expire in %s, deadline in %s", KeyUseTimeout, time.Until(deadline)) + } +} From ddd5d7cf4b9ac8510ac86b1f30f6c5ceb5f0b41e Mon Sep 17 00:00:00 2001 From: Abhijit Date: Fri, 9 Oct 2026 13:05:53 +0530 Subject: [PATCH 2/9] Fixes from the end-to-end run on a drill network MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - shell guard: refuse deleting a node's data dir or stores (state reset, takes priv_validator_state.json with it), copying or dd onto priv_validator_state.json, interpreter scripts that touch it, and d start / init / rollback — a model with every approval granted ran all of these on a drill node - key_seed.json (evmd testnet's operator mnemonic) is protected key material - strict egress masks peer topology (node.peers, sec.exposure, fleet.status) - playbook conditions are checked against fresh signals: a halted node is jailed while the playbook runs - container-memory-limit and node-crash-loop explain val-in-set-while-syncing, so the playbook isn't handed to the model - the key use of a playbook step says which case and step it's for; raw-output steps are summarized by line count in the report - config set agent.router / agent.egress; profile values merge - init offers only persistent keyring backends (test is in-memory) Co-Authored-By: Claude Opus 5.5 --- internal/agent/incident.go | 50 ++++++++++++++++++--- internal/agent/incident_test.go | 5 ++- internal/agent/session_test.go | 1 - internal/cli/configcmd.go | 14 ++++++ internal/cli/init.go | 20 +++++++-- internal/config/config.go | 2 + internal/egress/egress_test.go | 7 +++ internal/egress/summary.go | 2 + internal/kb/cases/host.yaml | 2 +- internal/kb/cases/sync.yaml | 2 +- internal/tools/fs/fs.go | 2 +- internal/tools/shell/classify.go | 65 ++++++++++++++++++++++++--- internal/tools/shell/classify_test.go | 51 ++++++++++++++++++++- 13 files changed, 203 insertions(+), 20 deletions(-) diff --git a/internal/agent/incident.go b/internal/agent/incident.go index bb8a0f6..aef6e2e 100644 --- a/internal/agent/incident.go +++ b/internal/agent/incident.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "strings" + "time" "github.com/abhijitkrm/cometcli/internal/audit" "github.com/abhijitkrm/cometcli/internal/config" @@ -34,7 +35,7 @@ func (a *Agent) runIncident(ctx context.Context, what string) (string, error) { if !ok { return handoff("") } - res, err := a.runTool(ctx, t, toolkit.Args{}) + res, err := a.runTool(ctx, t, toolkit.Args{}, "/incident: triage") if err != nil { return handoff("") } @@ -62,6 +63,11 @@ func (a *Agent) runIncident(ctx context.Context, what string) (string, error) { if st.Note != "" { label += " — " + st.Note } + if st.When != "" { + // conditions are about now, not about the triage minutes ago: + // a halted node gets jailed while the playbook runs + a.refreshSignals(ctx, sig, st.When) + } if !st.Applies(sig) { done = append(done, fmt.Sprintf("skipped %s (%s doesn't hold)", st.Do, st.When)) continue @@ -70,7 +76,8 @@ func (a *Agent) runIncident(ctx context.Context, what string) (string, error) { if !ok { return handoff(triageNote + "\n\n" + playbookNote(c, done, fmt.Sprintf("step %d: no tool %s", i+1, st.Do))) } - r, err := a.runTool(ctx, tool, playArgs(st.Args, a.Ctx.Profile)) + why := fmt.Sprintf("/incident %s — known case %s (%s), playbook step %d: %s", what, c.ID, c.Title, i+1, st.Note) + r, err := a.runTool(ctx, tool, playArgs(st.Args, a.Ctx.Profile), why) if err != nil { if strings.Contains(err.Error(), "denied") || ctx.Err() != nil { // the operator said no: stop here, don't let a model push on @@ -79,7 +86,7 @@ func (a *Agent) runIncident(ctx context.Context, what string) (string, error) { } return handoff(triageNote + "\n\n" + playbookNote(c, done, fmt.Sprintf("step %d (%s) failed: %v", i+1, label, err))) } - done = append(done, fmt.Sprintf("%s: %s", label, firstLine(a.Redact.Text(r.Text)))) + done = append(done, fmt.Sprintf("%s: %s", label, stepOutcome(st.Do, r.Text))) } _, matched := c.Matches(sig) @@ -89,7 +96,7 @@ func (a *Agent) runIncident(ctx context.Context, what string) (string, error) { rr, err := a.runTool(ctx, rt, toolkit.Args{ "title": c.Title, "case": c.ID, "root_cause": c.Title + " (matched " + strings.Join(matched, ", ") + ")", "evidence": strings.Join(matched, "\n"), "actions": strings.Join(done, "\n"), "outcome": outcome, - }) + }, "/incident: record") if err == nil { recorded = "\n\n" + firstLine(rr.Text) } @@ -161,13 +168,14 @@ func shellWord(s string) string { // runTool runs one registry tool for cometcli's own procedures, through // the same gates as the model's calls: permission rules, approvals for // changes and key use, the audit log and the operator's UI. -func (a *Agent) runTool(ctx context.Context, t toolkit.Tool, args toolkit.Args) (*toolkit.Result, error) { +func (a *Agent) runTool(ctx context.Context, t toolkit.Tool, args toolkit.Args, purpose string) (*toolkit.Result, error) { name := t.Name() shown := a.Redact.Args(args) a.emit(Event{Kind: EvToolStart, Tool: name, Tier: t.Tier().String(), Args: shown}) runCtx, cancel := a.toolCtx(ctx, t, args) defer cancel() defer runCtx.Close() + runCtx.Purpose = purpose err := a.ruleGate(runCtx, t, shown) var res *toolkit.Result if err == nil { @@ -189,3 +197,35 @@ func (a *Agent) runTool(ctx context.Context, t toolkit.Tool, args toolkit.Args) a.emit(Event{Kind: EvToolResult, Tool: name, Tier: t.Tier().String(), Text: firstLine(text), Output: preview(text)}) return res, nil } + +// stepOutcome is a step's result in one line for the report: the first +// line for most tools, a line count for raw output (a log's first line +// says nothing about the incident). +func stepOutcome(tool, text string) string { + text = strings.TrimSpace(text) + switch tool { + case "node.logs", "bash", "read", "grep": + return fmt.Sprintf("%d lines of output", len(strings.Split(text, "\n"))) + } + return firstLine(text) +} + +// refreshSignals re-collects the signal families a condition names +// ("val.jailed == true" → val.*) into sig. Collection failures keep the +// old values. +func (a *Agent) refreshSignals(ctx context.Context, sig kb.Signals, cond string) { + if sig == nil || a.Ctx.Profile == nil { + return + } + name, _, _ := strings.Cut(strings.TrimSpace(cond), " ") + family, _, ok := strings.Cut(name, ".") + if !ok { + return + } + c, cancel := toolkit.WithDeadline(a.Ctx.Derive(ctx), time.Minute) + defer cancel() + defer c.Close() + for k, v := range triage.CollectFor(c, 5*time.Minute, []string{family + "."}).Signals { + sig[k] = v + } +} diff --git a/internal/agent/incident_test.go b/internal/agent/incident_test.go index 8d29301..f00d828 100644 --- a/internal/agent/incident_test.go +++ b/internal/agent/incident_test.go @@ -15,8 +15,11 @@ func incidentAgent(t *testing.T, prov *mockProvider, roots []string, sig kb.Sign t.Helper() var ran []string stub := func(name string, tier toolkit.Tier, data map[string]any) stubTool { - return stubTool{name: name, tier: tier, run: func(_ *toolkit.Context, a toolkit.Args) (*toolkit.Result, error) { + return stubTool{name: name, tier: tier, run: func(c *toolkit.Context, a toolkit.Args) (*toolkit.Result, error) { ran = append(ran, strings.TrimSpace(name+" "+a.String("action", "")+a.String("condition", ""))) + if name == "val.unjail" && !strings.Contains(c.Purpose, "known case node-process-down") { + t.Errorf("the key use should say why: purpose %q", c.Purpose) + } if err := fail[name]; err != nil { return nil, err } diff --git a/internal/agent/session_test.go b/internal/agent/session_test.go index 378546f..b4e329e 100644 --- a/internal/agent/session_test.go +++ b/internal/agent/session_test.go @@ -16,7 +16,6 @@ import ( "github.com/abhijitkrm/cometcli/internal/toolkit" ) - // streamProvider wraps mockProvider with a Streamer that emits each // response's text in two chunks. type streamProvider struct{ *mockProvider } diff --git a/internal/cli/configcmd.go b/internal/cli/configcmd.go index 4ac3df7..fd2ad19 100644 --- a/internal/cli/configcmd.go +++ b/internal/cli/configcmd.go @@ -43,6 +43,20 @@ var agentKeys = map[string]func(*config.AgentConf, string) error{ a.Tools = v return nil }, + "router": func(a *config.AgentConf, v string) error { + if v != "" && v != "on" && v != "off" { + return fmt.Errorf("router: want on or off") + } + a.Router = v + return nil + }, + "egress": func(a *config.AgentConf, v string) error { + if v != "" && v != "strict" && v != "filtered" { + return fmt.Errorf("egress: want strict or filtered") + } + a.Egress = v + return nil + }, "max_tokens": intKey(func(a *config.AgentConf) *int { return &a.MaxTokens }), "max_turns": intKey(func(a *config.AgentConf) *int { return &a.MaxTurns }), "context_window": intKey(func(a *config.AgentConf) *int { return &a.ContextWindow }), diff --git a/internal/cli/init.go b/internal/cli/init.go index 56b115a..da4505a 100644 --- a/internal/cli/init.go +++ b/internal/cli/init.go @@ -672,8 +672,7 @@ func setupSigner(tc *toolkit.Context, d discovered, r *bufio.Reader, out io.Writ say(" · skipped — import later: cometcli keys add --name operator --recover, then cometcli profile add %s --signer operator", p.Name) return } - note("file = password-encrypted file · os = OS keychain · test = plaintext, testnets only") - p.Signer.Backend = ask(r, out, "keyring backend (file|os|test)", "file") + p.Signer.Backend = askBackend(r, out, note) mnemonic, err := readHidden(r, out, "operator mnemonic: ") if err != nil || strings.TrimSpace(mnemonic) == "" { say(" ! no mnemonic read — skipped") @@ -702,8 +701,7 @@ func setupSigner(tc *toolkit.Context, d discovered, r *bufio.Reader, out io.Writ // newOpsKey generates a fresh key (non-validator profiles only). func newOpsKey(p *config.Profile, r *bufio.Reader, out io.Writer, say func(string, ...any), note func(string)) { keyName := ask(r, out, "key name", "ops") - note("file = password-encrypted file · os = OS keychain · test = plaintext, testnets only") - p.Signer.Backend = ask(r, out, "keyring backend (file|os|test)", "file") + p.Signer.Backend = askBackend(r, out, note) ring, err := keys.Open(p) if err != nil { say(" ! keyring: %v", err) @@ -736,3 +734,17 @@ func readHidden(r *bufio.Reader, out io.Writer, prompt string) (string, error) { } func shq(s string) string { return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" } + +// askBackend asks where cometcli keeps a key. Only persistent backends +// are offered: "test" is in-memory and would lose the key on exit. +func askBackend(r *bufio.Reader, out io.Writer, note func(string)) string { + note("file = password-encrypted file (COMETCLI_KEYRING_PASSWORD) · os = the OS keychain") + for { + switch b := ask(r, out, "keyring backend (file|os)", "file"); b { + case "file", "os": + return b + default: + note("want file or os") + } + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 381f23d..003f5f8 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -114,6 +114,8 @@ func MergeAgent(base, o AgentConf) AgentConf { str(&r.BaseURL, o.BaseURL) str(&r.APIKeyEnv, o.APIKeyEnv) str(&r.Mode, o.Mode) + str(&r.Router, o.Router) + str(&r.Egress, o.Egress) str(&r.Effort, o.Effort) str(&r.Tools, o.Tools) num(&r.MaxTokens, o.MaxTokens) diff --git a/internal/egress/egress_test.go b/internal/egress/egress_test.go index b897087..10f8648 100644 --- a/internal/egress/egress_test.go +++ b/internal/egress/egress_test.go @@ -120,3 +120,10 @@ func TestForModel(t *testing.T) { func itoa(i int) string { return strings.TrimLeft(strings.Repeat("0", 3)+string(rune('0'+i/100))+string(rune('0'+i/10%10))+string(rune('0'+i%10)), "0") } + +func TestStrictMasksPeerTopology(t *testing.T) { + out := ForModel(Strict, "node.peers", "peers: 1\n ef87b12f751d59d232d51b3230f9398b71a9066e@172.30.9.3 node1\n") + if strings.Contains(out, "172.30.9.3") || strings.Contains(out, "ef87b12f751d") || !strings.Contains(out, "peers: 1") { + t.Errorf("peers in strict mode: %q", out) + } +} diff --git a/internal/egress/summary.go b/internal/egress/summary.go index eb471d1..10824d1 100644 --- a/internal/egress/summary.go +++ b/internal/egress/summary.go @@ -43,6 +43,8 @@ var rawTools = map[string]bool{ // lines are masked but the structure is kept. var maskedTools = map[string]bool{ "node.triage": true, "fleet.triage": true, "val.jail-check": true, "node.config": true, + // peer topology: node ids and IPs of sentries and peers + "node.peers": true, "sec.exposure": true, "fleet.status": true, } // ForModel turns a tool's output into what the model may see in mode. diff --git a/internal/kb/cases/host.yaml b/internal/kb/cases/host.yaml index e4279e9..7262530 100644 --- a/internal/kb/cases/host.yaml +++ b/internal/kb/cases/host.yaml @@ -133,7 +133,7 @@ - id: container-memory-limit title: Container memory limit too low for the node (OOM kills inside the container) - explains: [node-crash-loop, node-process-down, node-rpc-down, node-restarted-recently, node-killed-sigkill, host-oom-killed, val-jailed-downtime, val-jailed-waiting-period, val-not-signing, val-jail-imminent, node-catching-up, evm-rpc-down] + explains: [node-crash-loop, node-process-down, node-rpc-down, node-restarted-recently, node-killed-sigkill, host-oom-killed, val-jailed-downtime, val-jailed-waiting-period, val-not-signing, val-jail-imminent, node-catching-up, evm-rpc-down, val-in-set-while-syncing] severity: critical symptoms: restarts with no error in the logs; docker OOMKilled may be false because the node runs under a wrapper shell match: diff --git a/internal/kb/cases/sync.yaml b/internal/kb/cases/sync.yaml index 236b415..55ce2b3 100644 --- a/internal/kb/cases/sync.yaml +++ b/internal/kb/cases/sync.yaml @@ -48,7 +48,7 @@ - id: node-crash-loop title: Node crash-looping (restarting repeatedly) - explains: [node-process-down, node-rpc-down, evm-rpc-down, node-restarted-recently, node-killed-sigkill, node-catching-up, val-jailed-downtime, val-jailed-waiting-period, val-not-signing, val-jail-imminent, val-signing-degraded] + explains: [node-process-down, node-rpc-down, evm-rpc-down, node-restarted-recently, node-killed-sigkill, node-catching-up, val-jailed-downtime, val-jailed-waiting-period, val-not-signing, val-jail-imminent, val-signing-degraded, val-in-set-while-syncing] severity: critical match: all: ["proc.restarts >= 3", "proc.uptime_s < 600"] diff --git a/internal/tools/fs/fs.go b/internal/tools/fs/fs.go index 914eae6..d56a51b 100644 --- a/internal/tools/fs/fs.go +++ b/internal/tools/fs/fs.go @@ -82,7 +82,7 @@ func (t *target) inRoot(c *toolkit.Context) bool { return err == nil && rel != ".." && !strings.HasPrefix(rel, "../") } -var protectedRe = regexp.MustCompile(`priv_validator_key\.json$|node_key\.json$|(^|/)\.?mnemonics?(\.txt)?(/|$)|mnemonic|(^|/)keyring-(file|test|os)(/|$)|/\.ssh/id_|/\.cometcli/keys(/|$)|/\.gnupg/|(^|/)keystore(/|$)|(^|/)utc--[0-9]`) +var protectedRe = regexp.MustCompile(`priv_validator_key\.json$|node_key\.json$|(^|/)\.?mnemonics?(\.txt)?(/|$)|mnemonic|(^|/)keyring-(file|test|os)(/|$)|/\.ssh/id_|/\.cometcli/keys(/|$)|/\.gnupg/|(^|/)keystore(/|$)|(^|/)utc--[0-9]|(^|/)key_seed\.json$`) // forbidden explains why a path may never pass through the agent. func forbidden(p string, write bool) string { diff --git a/internal/tools/shell/classify.go b/internal/tools/shell/classify.go index 9d52a25..b6071ce 100644 --- a/internal/tools/shell/classify.go +++ b/internal/tools/shell/classify.go @@ -31,11 +31,19 @@ type Opts struct { } type acc struct { - opts Opts - v Verdict - depth int + opts Opts + v Verdict + depth int + cmdText string // the whole command line, heredocs included } +var ( + copiers = map[string]bool{"cp": true, "install": true, "ln": true, "rsync": true, "scp": true} + interpreters = map[string]bool{"python": true, "python3": true, "python2": true, "perl": true, "node": true, "ruby": true, "php": true, "jq": false} + // a node's data dir (or what's in it), by name or by the stores it holds + nodeDataRe = regexp.MustCompile(`(^|/)data/?(\*)?$|(^|/)data/(\*|application\.db|blockstore\.db|state\.db|tx_index\.db|evidence\.db|cs\.wal|snapshots)(/|$)|(application|blockstore|state|evidence)\.db/?$|(^|/)cs\.wal(/|$)`) +) + func (a *acc) raise(t toolkit.Tier, reason string) { if t > a.v.Tier || a.v.Reason == "" && t == a.v.Tier && t >= toolkit.TierLocalChange { a.v.Tier, a.v.Reason = t, reason @@ -62,7 +70,7 @@ func (a *acc) note(n string) { // wrappers (sudo, env, xargs, docker exec, bash -c) and invoked scripts // all taken into account. Unknown commands are local-change (ask). func Classify(cmd string, o Opts) Verdict { - a := &acc{opts: o, v: Verdict{Tier: toolkit.TierDiagnose}} + a := &acc{opts: o, v: Verdict{Tier: toolkit.TierDiagnose}, cmdText: cmd} a.line(cmd) return a.v } @@ -379,7 +387,7 @@ func isDigits(s string) bool { // --- sensitive material -------------------------------------------------- -var protectedRe = regexp.MustCompile(`priv_validator_key\.json|node_key\.json|(^|/)\.?mnemonics?(\.txt)?(/|$)|mnemonic|keyring-(file|test|os)|/\.ssh/id_|\.cometcli/keys|\.gnupg/|(^|/)keystore(/|$)|(^|/)utc--[0-9]`) +var protectedRe = regexp.MustCompile(`priv_validator_key\.json|node_key\.json|(^|/)\.?mnemonics?(\.txt)?(/|$)|mnemonic|keyring-(file|test|os)|/\.ssh/id_|\.cometcli/keys|\.gnupg/|(^|/)keystore(/|$)|(^|/)utc--[0-9]|key_seed\.json`) func protectedPath(word string) (bool, string) { if m := protectedRe.FindString(strings.ToLower(word)); m != "" { @@ -464,6 +472,28 @@ func (a *acc) simple(words []string) { if (name == "rm" || name == "mv" || name == "truncate" || name == "shred") && containsWord(args, "priv_validator_state.json") { a.forbid("removing priv_validator_state.json risks double-signing") } + // copying over the double-sign guard: cp/install/ln/rsync to it, dd of= + if copiers[name] { + if dst := lastNonFlag(args); strings.Contains(dst, "priv_validator_state.json") || (strings.HasSuffix(dst, "/data") || strings.HasSuffix(dst, "/data/")) && containsWord(args, "priv_validator_state") { + a.forbid("overwriting priv_validator_state.json risks double-signing") + } + } + if name == "dd" && containsWord(args, "of=") && containsWord(args, "priv_validator_state.json") { + a.forbid("overwriting priv_validator_state.json risks double-signing") + } + // deleting a node's data directory is a state reset, and takes the + // double-sign guard (priv_validator_state.json) with it + if name == "rm" || name == "shred" || (name == "find" && containsWord(args, "-delete")) { + for _, x := range nonFlags(args) { + if nodeDataRe.MatchString(x) { + a.forbid("deleting " + x + " wipes the node's chain data and priv_validator_state.json (a state reset, double-sign risk) — the operator must do it") + break + } + } + } + if interpreters[name] && a.cmdText != "" && strings.Contains(a.cmdText, "priv_validator_state") { + a.forbid("a script that touches priv_validator_state.json risks double-signing — the operator handles that file") + } switch name { case "sudo", "doas": @@ -610,6 +640,21 @@ func (a *acc) chainCmd(name string, args []string) { case "unsafe-reset-all": a.forbid(name + " unsafe-reset-all wipes chain state — the operator must run it") return + case "start": + if hasFlag(args, "--help", "-h") { + return + } + a.forbid("starting " + name + " by hand runs a second signer next to the node's service (double-sign risk) — start it with node.service") + return + case "init": + if hasFlag(args, "--help", "-h") { + return + } + a.forbid(name + " init writes new node and validator keys — the operator does it") + return + case "rollback": + a.forbid(name + " rollback rewrites chain state — the operator must run it") + return case "config": if len(args) > 1 && (args[1] == "get" || args[1] == "view") { return @@ -1066,3 +1111,13 @@ func stripComments(b []byte) []byte { } return []byte(strings.Join(lines, "\n")) } + +// lastNonFlag is the last argument that isn't a flag (a copy's target). +func lastNonFlag(a []string) string { + for i := len(a) - 1; i >= 0; i-- { + if !strings.HasPrefix(a[i], "-") { + return a[i] + } + } + return "" +} diff --git a/internal/tools/shell/classify_test.go b/internal/tools/shell/classify_test.go index 9f3c70b..bbdaf26 100644 --- a/internal/tools/shell/classify_test.go +++ b/internal/tools/shell/classify_test.go @@ -76,7 +76,7 @@ func TestClassify(t *testing.T) { {"find . -name '*.log' -delete", local}, {"cat > config.toml < Date: Fri, 9 Oct 2026 13:54:45 +0530 Subject: [PATCH 3/9] Fixes from the live testnet run over SSH - fee denom and gas price come from the node's own app.toml (host home, or inside the container) when the profile doesn't set them: a profile made with profile add couldn't unjail ('no fee denom') - profile add: --signer-mode, --container-home, --container-keyring, so container signing works without init - val-pvs-ahead no longer fires on healthy validators of a fast chain: the chain height is read with the state file, and the case needs evidence the node isn't signing - triage collectors get 60s on SSH nodes (20s timed out across the WAN) - gov propose defaults to a text proposal - failed steps of local answers and playbooks render as errors Co-Authored-By: Claude Opus 5.5 --- internal/agent/incident.go | 2 +- internal/agent/route.go | 2 +- internal/cli/init.go | 16 +------ internal/cli/profile.go | 14 +++++- internal/kb/cases/validator.yaml | 5 +- internal/toolkit/fees.go | 82 ++++++++++++++++++++++++++++++++ internal/toolkit/fees_test.go | 29 +++++++++++ internal/tools/common/common.go | 1 + internal/tools/govtool/gov.go | 6 ++- internal/tools/triage/collect.go | 22 +++++++-- 10 files changed, 155 insertions(+), 24 deletions(-) create mode 100644 internal/toolkit/fees.go create mode 100644 internal/toolkit/fees_test.go diff --git a/internal/agent/incident.go b/internal/agent/incident.go index aef6e2e..cc62750 100644 --- a/internal/agent/incident.go +++ b/internal/agent/incident.go @@ -189,7 +189,7 @@ func (a *Agent) runTool(ctx context.Context, t toolkit.Tool, args toolkit.Args, lg.ToolSeen(a.profileName(), name, t.Tier().String(), shown, data, err, "") } if err != nil { - a.emit(Event{Kind: EvToolResult, Tool: name, Tier: t.Tier().String(), Text: "error: " + a.Redact.Text(firstLine(err.Error()))}) + a.emit(Event{Kind: EvToolResult, Tool: name, Tier: t.Tier().String(), Err: a.Redact.Text(firstLine(err.Error()))}) return nil, err } text := a.Redact.Text(ansiRe.ReplaceAllString(res.Text, "")) diff --git a/internal/agent/route.go b/internal/agent/route.go index 04a5c8b..267bfec 100644 --- a/internal/agent/route.go +++ b/internal/agent/route.go @@ -74,7 +74,7 @@ func (a *Agent) routeLocal(ctx context.Context, input string) (string, bool) { r := router.Result{} if err != nil { r.Err = err.Error() - a.emit(Event{Kind: EvToolResult, Tool: st.Tool, Text: "error: " + a.Redact.Text(firstLine(r.Err))}) + a.emit(Event{Kind: EvToolResult, Tool: st.Tool, Err: a.Redact.Text(firstLine(r.Err))}) if !st.Optional { // the model explains failures better than a template a.lastRoute.Why += "; " + st.Tool + " failed" diff --git a/internal/cli/init.go b/internal/cli/init.go index da4505a..b774d84 100644 --- a/internal/cli/init.go +++ b/internal/cli/init.go @@ -559,21 +559,7 @@ func minGasPriceDenom(ctx context.Context, h host.Host, home string) (string, st if err != nil { return "", "" } - for _, line := range strings.Split(string(b), "\n") { - line = strings.TrimSpace(line) - if !strings.HasPrefix(line, "minimum-gas-prices") { - continue - } - _, v, _ := strings.Cut(line, "=") - v = strings.Trim(strings.TrimSpace(v), `",`) - v, _, _ = strings.Cut(v, ",") // first denom when several are accepted - i := 0 - for i < len(v) && (v[i] >= '0' && v[i] <= '9' || v[i] == '.') { - i++ - } - return v[:i], strings.TrimSpace(v[i:]) - } - return "", "" + return toolkit.ParseMinGasPrices(string(b)) } // homeFromArgs extracts `--home /path` or `--home=/path` from a cmdline. diff --git a/internal/cli/profile.go b/internal/cli/profile.go index 676f76b..f31e545 100644 --- a/internal/cli/profile.go +++ b/internal/cli/profile.go @@ -154,6 +154,15 @@ func profileAddCmd() *cobra.Command { if fl.Changed("signer-backend") { np.Signer.Backend = p.Signer.Backend } + if fl.Changed("signer-mode") { + np.Signer.Mode = p.Signer.Mode + } + if fl.Changed("container-home") { + np.Signer.ContainerHome = p.Signer.ContainerHome + } + if fl.Changed("container-keyring") { + np.Signer.ContainerKeyring = p.Signer.ContainerKeyring + } if fl.Changed("agent-provider") { np.Agent.Provider = p.Agent.Provider } @@ -217,7 +226,10 @@ func profileAddCmd() *cobra.Command { f.StringVar(&p.Service.Unit, "unit", "", "service unit/container name") f.StringSliceVar(&p.Sentries, "sentries", nil, "profiles of the sentry nodes this validator peers through (comma-separated)") f.StringVar(&signerKey, "signer", "", "ops key name in keyring") - f.StringVar(&p.Signer.Backend, "signer-backend", "", "os | file | test") + f.StringVar(&p.Signer.Backend, "signer-backend", "", "os | file") + f.StringVar(&p.Signer.Mode, "signer-mode", "", "local (cometcli keyring) | container (the node's own keyring); empty asks when both work") + f.StringVar(&p.Signer.ContainerHome, "container-home", "", "node home inside the container, for signing there (e.g. /data/node0/evmd)") + f.StringVar(&p.Signer.ContainerKeyring, "container-keyring", "", "keyring backend inside the container: test | file (detected when empty)") f.StringVar(&feeDenom, "fee-denom", "", "fee denom (e.g. atest)") f.StringVar(&valoper, "valoper", "", "validator operator address (read-only signing/jail views without a signer)") f.StringVar(&p.Agent.Provider, "agent-provider", "", "anthropic | openai | groq | openai-compat | off") diff --git a/internal/kb/cases/validator.yaml b/internal/kb/cases/validator.yaml index e779287..1ac05d7 100644 --- a/internal/kb/cases/validator.yaml +++ b/internal/kb/cases/validator.yaml @@ -254,8 +254,9 @@ severity: high symptoms: node synced but won't sign; logs show height regression / conflicting data match: - all: ["pvs.ahead_by > 1"] - any: ["node.catching_up == false", "logs.height_regression > 0"] + all: ["pvs.ahead_by > 1", "node.catching_up == false"] + # a validator that is signing can't be refusing to sign + any: ["val.signed_recent_pct < 50", "logs.height_regression > 0"] confirm: - "sec.doublesign shows priv_validator_state height vs node height" causes: diff --git a/internal/toolkit/fees.go b/internal/toolkit/fees.go new file mode 100644 index 0000000..bf95314 --- /dev/null +++ b/internal/toolkit/fees.go @@ -0,0 +1,82 @@ +package toolkit + +import ( + "path" + "strings" + "time" + + "github.com/abhijitkrm/cometcli/internal/client/host" +) + +// ParseMinGasPrices reads app.toml's minimum-gas-prices +// ("1000000000atest") as amount and denom — the first one when several +// are accepted. +func ParseMinGasPrices(appToml string) (amount, denom string) { + for _, line := range strings.Split(appToml, "\n") { + line = strings.TrimSpace(line) + if !strings.HasPrefix(line, "minimum-gas-prices") { + continue + } + _, v, _ := strings.Cut(line, "=") + v = strings.Trim(strings.TrimSpace(v), `",`) + v, _, _ = strings.Cut(v, ",") + i := 0 + for i < len(v) && (v[i] >= '0' && v[i] <= '9' || v[i] == '.') { + i++ + } + return v[:i], strings.TrimSpace(v[i:]) + } + return "", "" +} + +// FeeDefaults fills metadata.fee_denom and gas_price from the node's own +// app.toml when the profile doesn't set them: fees are paid in what the +// node's mempool accepts. Read from the host home, or inside the +// container for docker nodes. Remembered for the process, not saved. +func (c *Context) FeeDefaults() { + p := c.Profile + if p == nil || (p.Metadata["fee_denom"] != "" && p.Metadata["gas_price"] != "") { + return + } + h, err := c.Host() + if err != nil { + return + } + var raw string + if p.Home != "" { + if b, err := h.ReadFile(c, path.Join(p.Home, "config", "app.toml")); err == nil { + raw = string(b) + } + } + if raw == "" && p.Service.Type == "docker" && p.Service.Unit != "" { + homes := []string{p.Signer.ContainerHome} + if bin := p.Binary; bin != "" { + homes = append(homes, "/root/."+bin, "/home/"+bin+"/."+bin) + } + for _, home := range homes { + if home == "" { + continue + } + sub, cancel := WithDeadline(c, 15*time.Second) + res, err := host.Exec(sub, h, "docker exec "+shq(p.Service.Unit)+" cat "+shq(path.Join(home, "config", "app.toml")), 1<<20) + cancel() + if err == nil && res.Code == 0 { + raw = res.Output + break + } + } + } + amount, denom := ParseMinGasPrices(raw) + if denom == "" { + return + } + if p.Metadata == nil { + p.Metadata = map[string]string{} + } + if p.Metadata["fee_denom"] == "" { + p.Metadata["fee_denom"] = denom + } + if p.Metadata["gas_price"] == "" && amount != "" { + p.Metadata["gas_price"] = amount + } +} diff --git a/internal/toolkit/fees_test.go b/internal/toolkit/fees_test.go new file mode 100644 index 0000000..d665089 --- /dev/null +++ b/internal/toolkit/fees_test.go @@ -0,0 +1,29 @@ +package toolkit + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/abhijitkrm/cometcli/internal/client/host" + "github.com/abhijitkrm/cometcli/internal/config" +) + +func TestFeeDefaultsFromTheNodesAppToml(t *testing.T) { + home := t.TempDir() + _ = os.MkdirAll(filepath.Join(home, "config"), 0o755) + _ = os.WriteFile(filepath.Join(home, "config", "app.toml"), []byte("minimum-gas-prices = \"10000000000adex\"\n"), 0o644) + c := &Context{Context: context.Background(), Profile: &config.Profile{Name: "v", Home: home}} + c.SetHost(&host.Local{}) + c.FeeDefaults() + if c.Profile.Metadata["fee_denom"] != "adex" || c.Profile.Metadata["gas_price"] != "10000000000" { + t.Fatalf("metadata = %v", c.Profile.Metadata) + } + // set values are kept + c.Profile.Metadata = map[string]string{"fee_denom": "uatom", "gas_price": "0.1"} + c.FeeDefaults() + if c.Profile.Metadata["fee_denom"] != "uatom" { + t.Fatal("an explicit fee denom was overwritten") + } +} diff --git a/internal/tools/common/common.go b/internal/tools/common/common.go index 4830e15..4704901 100644 --- a/internal/tools/common/common.go +++ b/internal/tools/common/common.go @@ -59,6 +59,7 @@ func identify(c *toolkit.Context) { if c.Profile != nil && c.Profile.Metadata["valoper"] == "" { _, _ = ValoperFromNode(c) } + c.FeeDefaults() } // ValoperFromNode finds the validator this node signs for: the node's diff --git a/internal/tools/govtool/gov.go b/internal/tools/govtool/gov.go index a9e1801..59c1389 100644 --- a/internal/tools/govtool/gov.go +++ b/internal/tools/govtool/gov.go @@ -142,7 +142,11 @@ func Build(c *toolkit.Context, a toolkit.Args, proposer string) (*Plan, error) { } p.Detail["voting_period"] = votingPeriod.String() - switch kind := a.String("kind", ""); kind { + kind := a.String("kind", "") + if kind == "" && a.String("height", "") == "" && a.String("in_blocks", "") == "" && a.String("messages", "") == "" { + kind = "text" // a title and summary alone are a text proposal + } + switch kind { case "text": if msg.Metadata == "" { // the SDK rejects a proposal with neither messages nor metadata diff --git a/internal/tools/triage/collect.go b/internal/tools/triage/collect.go index 5a620b9..b44940a 100644 --- a/internal/tools/triage/collect.go +++ b/internal/tools/triage/collect.go @@ -34,6 +34,9 @@ import ( // Timeout bounds each collector (tests lower it). var Timeout = 20 * time.Second +// RemoteTimeout bounds each collector on a node reached over SSH. +var RemoteTimeout = 60 * time.Second + // Report is one sweep's outcome. type Report struct { Chain string // cosmos-sdk | cosmos-evm @@ -131,7 +134,11 @@ func CollectFor(c *toolkit.Context, since time.Duration, names []string) *Report wg.Add(1) go func(col collector) { defer wg.Done() - ctx, cancel := context.WithTimeout(c, Timeout) + limit := Timeout + if c.Profile != nil && c.Profile.Transport.Type == "ssh" { + limit = RemoteTimeout // every probe crosses the WAN + } + ctx, cancel := context.WithTimeout(c, limit) defer cancel() cc := c.Derive(ctx) local := kb.Signals{} @@ -143,7 +150,7 @@ func CollectFor(c *toolkit.Context, since time.Duration, names []string) *Report select { case err = <-done: case <-ctx.Done(): - err = fmt.Errorf("timed out after %s", Timeout) + err = fmt.Errorf("timed out after %s", limit) } mu2.Lock() defer mu2.Unlock() @@ -760,6 +767,13 @@ func collectConfig(c *toolkit.Context, r *Report, set func(string, any)) error { } if json.Unmarshal(raw, &pvs) == nil { set("pvs.height", num(pvs.Height)) + // compare with the height right now: on a fast chain the + // height read seconds earlier makes a healthy signer look ahead + if cc, err := c.Comet(); err == nil { + if st, err := cc.Status(c); err == nil { + set("pvs.ahead_by", num(pvs.Height)-float64(st.SyncInfo.LatestBlockHeight)) + } + } } } if len(errs) == 2 { @@ -856,7 +870,9 @@ func derive(r *Report) { s["upgrade.blocks_to_halt"] = uh - h } if ph, ok := f("pvs.height"); ok { - s["pvs.ahead_by"] = ph - h + if _, measured := s["pvs.ahead_by"]; !measured { + s["pvs.ahead_by"] = ph - h + } } if hh, ok := f("app.halt_height"); ok && hh > 0 { s["app.blocks_to_halt_height"] = hh - h From 042defefa92a7bd42daae5eb963631bf38cf271e Mon Sep 17 00:00:00 2001 From: Abhijit Date: Fri, 9 Oct 2026 14:46:13 +0530 Subject: [PATCH 4/9] More fixes from the live testnet run - node.service stops docker nodes with a 60s grace period: docker's default 10s SIGKILLed evmd on every stop (exit 137) - node-killed-sigkill (a forced or timed-out stop) gets a playbook, so /incident revives such a node locally: the live run went to the model - wait.until fails when the condition isn't reached (non-zero exit; a playbook no longer carries on as if it held), and signal waits honor --validator instead of watching the profile's own validator - -p explains that it never approves transactions or key use and names the exact cometcli command to run in a terminal (the model invented one) - a playbook stops, rather than handing off to the model, when -p can't approve a step - gov propose: integer flags no longer block the text default Co-Authored-By: Claude Opus 5.5 --- internal/agent/incident.go | 2 +- internal/cli/chat.go | 15 +++++++++++++-- internal/kb/cases/more.yaml | 9 +++++++++ internal/tools/govtool/gov.go | 2 +- internal/tools/node/node.go | 8 +++++++- internal/tools/node/service_test.go | 2 +- internal/tools/waittool/wait.go | 17 ++++++++++++++++- internal/tools/waittool/wait_test.go | 13 +++++++------ 8 files changed, 55 insertions(+), 13 deletions(-) diff --git a/internal/agent/incident.go b/internal/agent/incident.go index cc62750..850abb8 100644 --- a/internal/agent/incident.go +++ b/internal/agent/incident.go @@ -79,7 +79,7 @@ func (a *Agent) runIncident(ctx context.Context, what string) (string, error) { why := fmt.Sprintf("/incident %s — known case %s (%s), playbook step %d: %s", what, c.ID, c.Title, i+1, st.Note) r, err := a.runTool(ctx, tool, playArgs(st.Args, a.Ctx.Profile), why) if err != nil { - if strings.Contains(err.Error(), "denied") || ctx.Err() != nil { + if strings.Contains(err.Error(), "denied") || strings.Contains(err.Error(), "not approved") || ctx.Err() != nil { // the operator said no: stop here, don't let a model push on text := fmt.Sprintf("Stopped at step %d (%s): %v\n\nDone so far:\n%s", i+1, label, err, bullets(done)) return a.localIncidentDone(what, c, text), nil diff --git a/internal/cli/chat.go b/internal/cli/chat.go index 91500a9..b89a0a3 100644 --- a/internal/cli/chat.go +++ b/internal/cli/chat.go @@ -175,8 +175,19 @@ func chatCtx(cmd *cobra.Command, p *config.Profile, headless bool) (*toolkit.Con return c, nil } -func headlessApprover(_ *toolkit.Context, prompt string, tier toolkit.Tier, _ map[string]any) (bool, error) { - return false, fmt.Errorf("not approved: %s needs approval and -p has no one to ask — allow it with --allowedTools or --permission-mode, or run interactively", tier) +func headlessApprover(c *toolkit.Context, prompt string, tier toolkit.Tier, _ map[string]any) (bool, error) { + run := "run it interactively" + if c != nil && c.ToolName != "" { + run = "run `cometcli" + if c.Profile != nil { + run += " --profile " + c.Profile.Name + } + run += " " + strings.ReplaceAll(c.ToolName, ".", " ") + "` in a terminal — it shows the request and asks" + } + if tier >= toolkit.TierOnChain { + return false, fmt.Errorf("not approved: -p never approves %s actions (transactions and key use need a person) — %s", tier, run) + } + return false, fmt.Errorf("not approved: %s needs approval and -p has no one to ask — allow it with --allowedTools or --permission-mode, or %s", tier, run) } // configureChatAgent applies the agent flags plus the chat-only ones. diff --git a/internal/kb/cases/more.yaml b/internal/kb/cases/more.yaml index 2bd4945..3f9512d 100644 --- a/internal/kb/cases/more.yaml +++ b/internal/kb/cases/more.yaml @@ -53,6 +53,15 @@ - kernel OOM killer; a stop that exceeded the stop timeout (container stop -t / TimeoutStopSec) fix: - "[change] OOM: follow host-oom-killed; stop timeout: raise it so the node flushes state on shutdown" + # an OOM kill is matched by host-oom-killed / container-memory-limit, + # which explain this case — left here, it was a forced or timed-out stop + playbook: + - {do: node.logs, args: {lines: 40}, note: "the last lines before it was killed"} + - {do: node.service, args: {action: start}, note: "start the node"} + - {do: wait.until, args: {condition: synced, timeout: 1800}, note: "wait until it has caught up"} + - {do: wait.until, args: {condition: unjailable, timeout: 1800}, when: "val.jailed == true", note: "wait out the jail period"} + - {do: val.unjail, when: "val.jailed == true", note: "unjail (signed by the operator key — asks you)"} + - {do: wait.until, args: {condition: in-consensus, timeout: 600}, when: "val.jailed == true", note: "check it signs again"} test: signals: {proc.running: false, proc.exit_code: 137} diff --git a/internal/tools/govtool/gov.go b/internal/tools/govtool/gov.go index 59c1389..8d3559c 100644 --- a/internal/tools/govtool/gov.go +++ b/internal/tools/govtool/gov.go @@ -143,7 +143,7 @@ func Build(c *toolkit.Context, a toolkit.Args, proposer string) (*Plan, error) { p.Detail["voting_period"] = votingPeriod.String() kind := a.String("kind", "") - if kind == "" && a.String("height", "") == "" && a.String("in_blocks", "") == "" && a.String("messages", "") == "" { + if kind == "" && a.Int("height", 0) == 0 && a.Int("in_blocks", 0) == 0 && a.String("messages", "") == "" { kind = "text" // a title and summary alone are a text proposal } switch kind { diff --git a/internal/tools/node/node.go b/internal/tools/node/node.go index 1d7d13b..63edeaa 100644 --- a/internal/tools/node/node.go +++ b/internal/tools/node/node.go @@ -232,7 +232,13 @@ func svcRun(c *toolkit.Context, h host.Host, action string) (string, int, error) if action == "status" { cmd = "docker inspect -f '{{.State.Status}}|{{.State.Pid}}|{{.State.StartedAt}}|{{.Config.Image}}|{{.State.ExitCode}}|{{.State.Health.Status}}' " + common.ShellQ(unit) + " 2>/dev/null || docker ps -a --filter name=" + common.ShellQ(unit) } else { - cmd = "docker " + action + " " + common.ShellQ(unit) + // docker's default 10s stop timeout SIGKILLs a node still + // flushing state (exit 137); give it a minute to shut down + grace := "" + if action == "stop" || action == "restart" { + grace = "-t 60 " + } + cmd = "docker " + action + " " + grace + common.ShellQ(unit) } case "launchd": if action == "status" { diff --git a/internal/tools/node/service_test.go b/internal/tools/node/service_test.go index 58008be..132aeda 100644 --- a/internal/tools/node/service_test.go +++ b/internal/tools/node/service_test.go @@ -48,7 +48,7 @@ func TestServiceCommands(t *testing.T) { {"systemd", "status", "systemctl status 'evmd.service'"}, {"systemd", "restart", "systemctl restart 'evmd.service'"}, {"docker", "status", "docker ps -a --filter name='evmd'"}, - {"docker", "stop", "docker stop 'evmd'"}, + {"docker", "stop", "docker stop -t 60 'evmd'"}, {"launchd", "status", "launchctl list | grep -i 'evmd'"}, {"launchd", "restart", "launchctl kickstart -k gui/$(id -u)/'evmd'"}, } diff --git a/internal/tools/waittool/wait.go b/internal/tools/waittool/wait.go index 5dcb2ca..4b58296 100644 --- a/internal/tools/waittool/wait.go +++ b/internal/tools/waittool/wait.go @@ -73,6 +73,19 @@ func (Until) Run(c *toolkit.Context, a toolkit.Args) (*toolkit.Result, error) { if err != nil { return nil, err } + if v := a.String("validator", ""); v != "" && cond == "signal" && c.Profile != nil { + // val.* signals are about the profile's validator: point them at + // the one asked for + p := *c.Profile + p.Metadata = map[string]string{} + for k, x := range c.Profile.Metadata { + p.Metadata[k] = x + } + p.Metadata["valoper"] = v + sub := c.Derive(c) + sub.Profile = &p + c = sub + } iv := max(time.Duration(a.Int("interval", 10))*time.Second, MinInterval) if v, ok := a["interval"].(float64); ok && v > 0 && v < 1 { iv = max(time.Duration(v*float64(time.Second)), MinInterval) @@ -111,7 +124,9 @@ func (Until) Run(c *toolkit.Context, a toolkit.Args) (*toolkit.Result, error) { for k, v := range last.data { data[k] = v } - return &toolkit.Result{Text: fmt.Sprintf("✗ %s not reached within %s — last: %s", cond, timeout(a), last.status), Data: data}, nil + // an error, not a result: scripts get a failing exit code, and a + // playbook mustn't carry on as if the condition held + return nil, fmt.Errorf("%s not reached within %s — last: %s", cond, timeout(a), last.status) } func progress(c *toolkit.Context, s string) { diff --git a/internal/tools/waittool/wait_test.go b/internal/tools/waittool/wait_test.go index b221fdd..9ea966c 100644 --- a/internal/tools/waittool/wait_test.go +++ b/internal/tools/waittool/wait_test.go @@ -116,9 +116,10 @@ func TestWaitTimesOutWithLastState(t *testing.T) { _, chain, c, _ := setup(t) chain.Set(func(c *fakenode.Comet) { c.Signs = func(int64) bool { return false } }) start := time.Now() - res, err := (Until{}).Run(c, toolkit.Args{"condition": "signing", "window": float64(10), "timeout": float64(1), "interval": 0.05}) - if err != nil || res.Data["done"] != false || !strings.Contains(res.Text, "signed 0/10") || time.Since(start) > 3*time.Second { - t.Fatalf("timeout: %v %v (%s)", res, err, time.Since(start)) + _, err := (Until{}).Run(c, toolkit.Args{"condition": "signing", "window": float64(10), "timeout": float64(1), "interval": 0.05}) + // a wait that isn't reached is an error carrying the last state + if err == nil || !strings.Contains(err.Error(), "not reached") || !strings.Contains(err.Error(), "signed 0/10") || time.Since(start) > 3*time.Second { + t.Fatalf("timeout: %v (%s)", err, time.Since(start)) } } @@ -158,8 +159,8 @@ func TestWaitSignalExpressions(t *testing.T) { t.Fatal("bad expression accepted") } // an uncollectable signal never holds and says so - res, err = Until{}.Run(c, toolkit.Args{"condition": "signal", "value": "evm.height > 0", "interval": 0.02, "timeout": 1}) - if err != nil || res.Data["done"] != false || !strings.Contains(res.Text, "unavailable") { - t.Fatalf("%v %v", res, err) + _, err = Until{}.Run(c, toolkit.Args{"condition": "signal", "value": "evm.height > 0", "interval": 0.02, "timeout": 1}) + if err == nil || !strings.Contains(err.Error(), "unavailable") { + t.Fatalf("%v", err) } } From 678743da146f15cff69fe7bf33988fbc075fd8f0 Mon Sep 17 00:00:00 2001 From: Abhijit Date: Fri, 9 Oct 2026 16:20:57 +0530 Subject: [PATCH 5/9] Governed upgrades for docker nodes: releases, build, ready, switch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - upgrade.releases: releases newer than the running image (version and source repo from its labels), the upgrade handler names each one registers, and a choice when there are several - upgrade.build: build the target on the node with the Dockerfile that built the running image (beside the compose project, else found under the home dir); optionally adds a no-op handler (RunMigrations) when the release registers none for the upgrade; checks version and handler in the built binary - upgrade.ready: the new image registers the plan name, the running binary doesn't, the deployed compose file is known - gov.propose --kind upgrade on docker nodes needs --image and refuses unless upgrade.ready passes: a name the new binary doesn't register halts the chain for good, one the old binary has switches nothing - upgrade.switch: only after the 'UPGRADE NEEDED' halt, and only to an image that registers the halted plan — sed the image in the deployed compose file (backup kept), compose down, up -d, same data dir Found on primium-1: every v0.6.x release registers only v0.5.0-to-v0.6.0, which v0.6.0 has too, so v0.6.1 can't be reached by a governed upgrade without a handler of its own. Co-Authored-By: Claude Opus 5.5 --- internal/tools/govtool/gov.go | 19 + internal/tools/upgradetool/flow.go | 535 ++++++++++++++++++++++++ internal/tools/upgradetool/flow_test.go | 45 ++ internal/tools/upgradetool/upgrade.go | 1 + 4 files changed, 600 insertions(+) create mode 100644 internal/tools/upgradetool/flow.go create mode 100644 internal/tools/upgradetool/flow_test.go diff --git a/internal/tools/govtool/gov.go b/internal/tools/govtool/gov.go index 8d3559c..335c06b 100644 --- a/internal/tools/govtool/gov.go +++ b/internal/tools/govtool/gov.go @@ -22,6 +22,7 @@ import ( "github.com/abhijitkrm/cometcli/internal/toolkit" "github.com/abhijitkrm/cometcli/internal/tools/common" + "github.com/abhijitkrm/cometcli/internal/tools/upgradetool" "github.com/abhijitkrm/cometcli/internal/tx" ) @@ -70,6 +71,7 @@ func (proposeTool) Schema() map[string]any { "height": toolkit.Int("upgrade: absolute halt height"), "in_blocks": toolkit.Int("upgrade: halt this many blocks from now (instead of height)"), "info": toolkit.Str("upgrade: plan info (binaries JSON / release URL)"), + "image": toolkit.Str("upgrade, docker nodes: the image to switch to — its binary must register the plan name (checked)"), "messages": toolkit.Str(`messages: JSON array of proposal msgs, e.g. [{"@type":"/cosmos.bank.v1beta1.MsgSend",…}]`), "memo": toolkit.Str("tx memo"), }), "kind", "title", "summary") @@ -157,6 +159,23 @@ func Build(c *toolkit.Context, a toolkit.Args, proposer string) (*Plan, error) { if !planNameRe.MatchString(name) { return nil, fmt.Errorf("upgrade name %q: use the new binary's upgrade handler name (letters, digits, . _ -)", name) } + // a name the new binary doesn't register halts the chain for good; + // one the running binary registers switches nothing — check both + // against the binaries before anything goes on chain + if c.Profile != nil && c.Profile.Service.Type == "docker" { + image := a.String("image", "") + if image == "" { + return nil, fmt.Errorf("pass image (the one you'll switch to): cometcli checks the plan name %q against its binary and the running one first — a wrong name halts the chain permanently", name) + } + ok, report, _, err := upgradetool.Ready(c, name, image) + if err != nil { + return nil, err + } + if !ok { + return nil, fmt.Errorf("not proposing — this node isn't ready for plan %q:\n%s", name, report) + } + p.Notes = append(p.Notes, "checked on "+c.Profile.Name+": "+strings.ReplaceAll(strings.TrimSpace(report), "\n", "; ")+" — run upgrade.ready on every other validator too") + } cur, blockTime, err := chainClock(c) if err != nil { return nil, err diff --git a/internal/tools/upgradetool/flow.go b/internal/tools/upgradetool/flow.go new file mode 100644 index 0000000..d62dba9 --- /dev/null +++ b/internal/tools/upgradetool/flow.go @@ -0,0 +1,535 @@ +package upgradetool + +// The governed-upgrade flow for docker-run nodes, in the order an operator +// runs it: +// +// upgrade.releases which releases are newer than what's running +// upgrade.build build the target image on the node, from source +// upgrade.ready the new image registers the plan's handler and the +// running binary doesn't — gov.propose checks this too +// upgrade.switch after the halt: swap the image in the deployed +// compose file and recreate the container +// +// The plan name is the dangerous part. A name the new binary doesn't +// register halts the chain for good; a name the running binary already +// registers is applied by the old binary itself and nothing switches. So +// both are checked against the binaries themselves, not the source. + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "path" + "regexp" + "sort" + "strconv" + "strings" + "time" + + "github.com/abhijitkrm/cometcli/internal/client/host" + "github.com/abhijitkrm/cometcli/internal/toolkit" + "github.com/abhijitkrm/cometcli/internal/tools/common" +) + +// RegisterFlow adds the docker upgrade-flow tools. +func RegisterFlow(r *toolkit.Registry) { + r.Register(releasesTool{}) + r.Register(buildTool{}) + r.Register(readyTool{}) + r.Register(switchTool{}) +} + +// --- the running container --------------------------------------------------- + +// running is what the node's container runs and how it was deployed. +type running struct { + Image, Version, Repo string // image ref; image version label; source repo (owner/name) + ComposeFile string // the deployed compose file (from compose labels) + ComposeDir string + Binary string // binary path inside the image +} + +func dockerOnly(c *toolkit.Context) error { + if c.Profile == nil || c.Profile.Service.Type != "docker" || c.Profile.Service.Unit == "" { + return fmt.Errorf("this upgrade flow is for docker nodes (service.type docker, service.unit set)") + } + return nil +} + +func sh(c *toolkit.Context, script string, limit time.Duration) (string, int, error) { + h, err := c.Host() + if err != nil { + return "", -1, err + } + sub, cancel := toolkit.WithDeadline(c, limit) + defer cancel() + res, err := host.Exec(sub, h, script, 4<<20) + c.LogShell(script, res.Code) + return res.Output, res.Code, err +} + +func inspectRunning(c *toolkit.Context) (*running, error) { + if err := dockerOnly(c); err != nil { + return nil, err + } + unit := common.ShellQ(c.Profile.Service.Unit) + out, code, err := sh(c, "docker inspect -f '{{.Config.Image}}|{{json .Config.Labels}}' "+unit, time.Minute) + if err != nil || code != 0 { + return nil, fmt.Errorf("docker inspect %s: %v %s", c.Profile.Service.Unit, err, strings.TrimSpace(out)) + } + img, labelsJSON, _ := strings.Cut(strings.TrimSpace(out), "|") + var labels map[string]string + _ = json.Unmarshal([]byte(labelsJSON), &labels) + r := &running{Image: img, Version: labels["org.opencontainers.image.version"], Binary: "/evmd/evmd"} + if src := labels["org.opencontainers.image.source"]; strings.Contains(src, "github.com/") { + _, r.Repo, _ = strings.Cut(src, "github.com/") + r.Repo = strings.TrimSuffix(strings.TrimSuffix(r.Repo, "/"), ".git") + } + if f := labels["com.docker.compose.project.config_files"]; f != "" { + r.ComposeFile = strings.Split(f, ",")[0] + r.ComposeDir = labels["com.docker.compose.project.working_dir"] + } + if bin := c.Profile.Binary; bin != "" { + r.Binary = "/evmd/" + bin + } + if r.Version == "" { + if _, tag, ok := strings.Cut(img, ":"); ok { + r.Version = tag + } + } + return r, nil +} + +// --- upgrade.releases --------------------------------------------------------- + +type releasesTool struct{} + +func (releasesTool) Name() string { return "upgrade.releases" } +func (releasesTool) Desc() string { + return "Releases newer than what the node runs (version and source repo read from the running image), oldest first, " + + "with the upgrade handler names each one registers in evmd/upgrades.go. Pick the target before upgrade.build." +} +func (releasesTool) Schema() map[string]any { + return toolkit.ObjSchema(map[string]any{ + "repo": toolkit.Str("github owner/repo (default: the image's source label, or metadata.repo)"), + }) +} +func (releasesTool) Tier() toolkit.Tier { return toolkit.TierObserve } + +func (releasesTool) Run(c *toolkit.Context, a toolkit.Args) (*toolkit.Result, error) { + cur := "" + repo := a.String("repo", "") + if r, err := inspectRunning(c); err == nil { + cur = r.Version + if repo == "" { + repo = r.Repo + } + } + if repo == "" && c.Profile != nil { + repo = c.Profile.Metadata["repo"] + } + if repo == "" { + return nil, fmt.Errorf("no source repo: the image has no org.opencontainers.image.source label — pass repo") + } + tags, err := listReleases(c, repo) + if err != nil { + return nil, err + } + var newer []string + for _, t := range tags { + if cur == "" || semverLess(cur, t) { + newer = append(newer, t) + } + } + sort.Slice(newer, func(i, j int) bool { return semverLess(newer[i], newer[j]) }) + var b strings.Builder + fmt.Fprintf(&b, "running %s (%s)\n", orNone(cur), repo) + if len(newer) == 0 { + b.WriteString("no newer release\n") + return &toolkit.Result{Text: b.String(), Data: map[string]any{"current": cur, "repo": repo, "releases": newer}}, nil + } + handlers := map[string][]string{} + for _, t := range newer { + handlers[t] = upgradeNames(c, repo, t) + fmt.Fprintf(&b, " %-10s registers %s\n", t, orNone(strings.Join(handlers[t], ", "))) + } + curHandlers := upgradeNames(c, repo, cur) + if len(curHandlers) > 0 { + fmt.Fprintf(&b, "running %s registers %s — a plan with that name would be applied by the old binary itself\n", cur, strings.Join(curHandlers, ", ")) + } + data := map[string]any{"current": cur, "repo": repo, "releases": newer, "handlers": handlers, "current_handlers": curHandlers} + if len(newer) > 1 && c.Chooser != nil { + i, err := c.Chooser(c, fmt.Sprintf("%d releases are newer than %s — which one is the target?", len(newer), orNone(cur)), newer) + if err != nil { + return nil, err + } + data["chosen"] = newer[i] + fmt.Fprintf(&b, "target: %s\n", newer[i]) + } else if len(newer) == 1 { + data["chosen"] = newer[0] + } else { + b.WriteString("several candidates: confirm the target with the operator before building\n") + } + return &toolkit.Result{Text: b.String(), Data: data}, nil +} + +func orNone(s string) string { + if s == "" { + return "(none)" + } + return s +} + +func listReleases(c *toolkit.Context, repo string) ([]string, error) { + req, err := http.NewRequestWithContext(c, "GET", "https://api.github.com/repos/"+repo+"/releases?per_page=100", nil) + if err != nil { + return nil, err + } + req.Header.Set("Accept", "application/vnd.github+json") + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + return nil, fmt.Errorf("github releases for %s: %s", repo, resp.Status) + } + var rels []struct { + Tag string `json:"tag_name"` + Draft bool `json:"draft"` + Prerelease bool `json:"prerelease"` + } + if err := json.NewDecoder(resp.Body).Decode(&rels); err != nil { + return nil, err + } + var out []string + for _, r := range rels { + if !r.Draft && !r.Prerelease && semverRe.MatchString(r.Tag) { + out = append(out, r.Tag) + } + } + return out, nil +} + +var ( + semverRe = regexp.MustCompile(`^v?(\d+)\.(\d+)\.(\d+)$`) + upgradeNameRe = regexp.MustCompile(`(?m)^\s*(?:const\s+)?\w*UpgradeName\w*\s*=\s*"([^"]+)"`) + setHandlerRe = regexp.MustCompile(`SetUpgradeHandler\(\s*"([^"]+)"`) +) + +func semverLess(a, b string) bool { + pa, pb := semverRe.FindStringSubmatch(a), semverRe.FindStringSubmatch(b) + if pa == nil || pb == nil { + return a < b + } + for i := 1; i <= 3; i++ { + x, _ := strconv.Atoi(pa[i]) + y, _ := strconv.Atoi(pb[i]) + if x != y { + return x < y + } + } + return false +} + +// upgradeNames reads the handler names a tag registers in evmd/upgrades.go. +func upgradeNames(c *toolkit.Context, repo, tag string) []string { + if tag == "" { + return nil + } + req, err := http.NewRequestWithContext(c, "GET", "https://raw.githubusercontent.com/"+repo+"/"+tag+"/evmd/upgrades.go", nil) + if err != nil { + return nil + } + resp, err := http.DefaultClient.Do(req) + if err != nil || resp.StatusCode != 200 { + if resp != nil { + resp.Body.Close() + } + return nil + } + defer resp.Body.Close() + raw, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + seen := map[string]bool{} + var out []string + for _, re := range []*regexp.Regexp{upgradeNameRe, setHandlerRe} { + for _, m := range re.FindAllStringSubmatch(string(raw), -1) { + if !seen[m[1]] { + seen[m[1]] = true + out = append(out, m[1]) + } + } + } + return out +} + +// --- upgrade.build ------------------------------------------------------------ + +type buildTool struct{} + +func (buildTool) Name() string { return "upgrade.build" } +func (buildTool) Desc() string { + return "Build the target release's image on the node from source, with the node's own Dockerfile (the one that built the " + + "running image). Optionally adds a no-op upgrade handler (RunMigrations only) when the release registers none for this " + + "upgrade; checks the built binary's version and handler. Run it on every node before proposing." +} +func (buildTool) Schema() map[string]any { + return toolkit.ObjSchema(map[string]any{ + "tag": toolkit.Str("release tag to build, e.g. v0.6.1"), + "image": toolkit.Str("image to produce (default: the running image's name with the new tag)"), + "dockerfile": toolkit.Str("Dockerfile on the node (default: Dockerfile.primium beside the compose project, or one level up)"), + "handler": toolkit.Str("add a no-op upgrade handler with this plan name if the release lacks it, e.g. v0.6.0-to-v0.6.1"), + "repo": toolkit.Str("github owner/repo (default: the running image's source)"), + }, "tag") +} +func (buildTool) Tier() toolkit.Tier { return toolkit.TierLocalChange } +func (buildTool) Timeout(toolkit.Args) time.Duration { return 50 * time.Minute } + +var tagRe = regexp.MustCompile(`^[A-Za-z0-9._-]+$`) + +func (buildTool) Run(c *toolkit.Context, a toolkit.Args) (*toolkit.Result, error) { + r, err := inspectRunning(c) + if err != nil { + return nil, err + } + tag := a.String("tag", "") + handler := a.String("handler", "") + repo := a.String("repo", r.Repo) + if !tagRe.MatchString(tag) || (handler != "" && !tagRe.MatchString(handler)) || repo == "" { + return nil, fmt.Errorf("tag, handler and repo must be plain names (letters, digits, . _ -)") + } + name, _, _ := strings.Cut(r.Image, ":") + image := a.String("image", name+":"+tag) + if image == r.Image { + return nil, fmt.Errorf("%s is the image running now — build a different tag", image) + } + dockerfile := a.String("dockerfile", "") + if dockerfile == "" && r.ComposeDir != "" { + for _, d := range []string{r.ComposeDir, path.Dir(r.ComposeDir)} { + p := path.Join(d, "Dockerfile.primium") + if _, code, _ := sh(c, "test -f "+common.ShellQ(p), time.Minute); code == 0 { + dockerfile = p + break + } + } + } + if dockerfile == "" { + // deployments differ per host: look under the user's home + out, _, _ := sh(c, "find \"$HOME\" -maxdepth 7 -name Dockerfile.primium -not -path '*/.cometcli-build/*' 2>/dev/null | head -10", 2*time.Minute) + var found []string + for _, l := range strings.Split(strings.TrimSpace(out), "\n") { + if l = strings.TrimSpace(l); l != "" { + found = append(found, l) + } + } + switch len(found) { + case 1: + dockerfile = found[0] + case 0: + return nil, fmt.Errorf("no Dockerfile.primium beside the compose project or under the home dir — pass dockerfile (the one that built %s)", r.Image) + default: + return nil, fmt.Errorf("several Dockerfile.primium on this host — pass dockerfile with the one that built %s:\n %s", r.Image, strings.Join(found, "\n ")) + } + } + src := "$HOME/.cometcli-build/" + path.Base(repo) + "-" + tag + var script strings.Builder + fmt.Fprintf(&script, "set -e\nsrc=%s\nrm -rf \"$src\" && mkdir -p \"$(dirname \"$src\")\"\n", src) + fmt.Fprintf(&script, "git clone -q --depth 1 --branch %s https://github.com/%s.git \"$src\"\ncd \"$src\"\n", common.ShellQ(tag), repo) + fmt.Fprintf(&script, "git fetch -q --depth 1 origin 'refs/tags/*:refs/tags/*' || true\n") + if handler != "" { + // a no-op handler next to the release's own: the plan only + // switches binaries, migrations run for whatever modules changed + fmt.Fprintf(&script, `f=evmd/upgrades.go +grep -q 'func (app EVMD) RegisterUpgradeHandlers() {' "$f" || { echo "evmd/upgrades.go has no RegisterUpgradeHandlers to extend"; exit 3; } +if ! grep -q '"%[1]s"' "$f"; then + sed -i 's|func (app EVMD) RegisterUpgradeHandlers() {|func (app EVMD) RegisterUpgradeHandlers() {\n\tapp.UpgradeKeeper.SetUpgradeHandler("%[1]s", func(ctx context.Context, _ upgradetypes.Plan, fromVM module.VersionMap) (module.VersionMap, error) {\n\t\treturn app.ModuleManager.RunMigrations(ctx, app.Configurator(), fromVM)\n\t})|' "$f" + echo "added upgrade handler %[1]s" +fi +`, handler) + } + fmt.Fprintf(&script, "docker build -q -f %s -t %s --label name=primium --label org.opencontainers.image.version=%s --label org.opencontainers.image.source=https://github.com/%s . >/dev/null\n", + common.ShellQ(dockerfile), common.ShellQ(image), common.ShellQ(tag), repo) + fmt.Fprintf(&script, "echo \"version: $(docker run --rm --entrypoint %s %s version 2>&1 | head -1)\"\n", r.Binary, common.ShellQ(image)) + detail := map[string]any{"image": image, "tag": tag, "repo": repo, "dockerfile": dockerfile, "source": src} + if handler != "" { + detail["adds handler"] = handler + } + if err := c.Approve(fmt.Sprintf("build %s on %s from %s@%s (takes several minutes; the node keeps running)", image, c.Profile.Name, repo, tag), toolkit.TierLocalChange, detail); err != nil { + return nil, err + } + if c.Progress != nil { + c.Progress("building " + image + " — several minutes") + } + out, code, err := sh(c, script.String(), 45*time.Minute) + if err != nil || code != 0 { + return nil, fmt.Errorf("build failed (exit %d): %v\n%s", code, err, tail(out, 30)) + } + var b strings.Builder + fmt.Fprintf(&b, "built %s from %s@%s with %s\n%s", image, repo, tag, dockerfile, strings.TrimSpace(out)) + data := map[string]any{"image": image, "tag": tag} + if handler != "" { + ok, err := imageHas(c, image, r.Binary, handler) + if err != nil || !ok { + return nil, fmt.Errorf("built %s but its binary doesn't contain the handler %q — don't propose with it", image, handler) + } + fmt.Fprintf(&b, "\nhandler %s: in the new binary ✓", handler) + data["handler"] = handler + } + return &toolkit.Result{Text: b.String(), Data: data}, nil +} + +func tail(s string, n int) string { + l := strings.Split(strings.TrimSpace(s), "\n") + if len(l) > n { + l = l[len(l)-n:] + } + return strings.Join(l, "\n") +} + +// imageHas reports whether an image's binary contains the plan name. +func imageHas(c *toolkit.Context, image, binary, name string) (bool, error) { + out, _, err := sh(c, fmt.Sprintf("docker run --rm --entrypoint sh %s -c 'grep -c -a -F %s %s || true'", common.ShellQ(image), common.ShellQ(name), binary), 2*time.Minute) + if err != nil { + return false, err + } + n, _ := strconv.Atoi(strings.TrimSpace(lastLine(out))) + return n > 0, nil +} + +// runningHas reports whether the running container's binary contains it. +func runningHas(c *toolkit.Context, binary, name string) (bool, error) { + out, _, err := sh(c, fmt.Sprintf("docker exec %s sh -c 'grep -c -a -F %s %s || true'", common.ShellQ(c.Profile.Service.Unit), common.ShellQ(name), binary), time.Minute) + if err != nil { + return false, err + } + n, _ := strconv.Atoi(strings.TrimSpace(lastLine(out))) + return n > 0, nil +} + +func lastLine(s string) string { + l := strings.Split(strings.TrimSpace(s), "\n") + return l[len(l)-1] +} + +// --- upgrade.ready ------------------------------------------------------------ + +type readyTool struct{} + +func (readyTool) Name() string { return "upgrade.ready" } +func (readyTool) Desc() string { + return "Is this node ready for a governed upgrade to `image` under plan `name`? The new image must exist and its binary " + + "register the name; the running binary must NOT (it would apply the plan itself and nothing switches); the deployed " + + "compose file must be found. Run on every validator before proposing." +} +func (readyTool) Schema() map[string]any { + return toolkit.ObjSchema(map[string]any{ + "name": toolkit.Str("upgrade plan name"), + "image": toolkit.Str("the image to switch to after the halt"), + }, "name", "image") +} +func (readyTool) Tier() toolkit.Tier { return toolkit.TierObserve } + +func (readyTool) Run(c *toolkit.Context, a toolkit.Args) (*toolkit.Result, error) { + ok, text, data, err := Ready(c, a.String("name", ""), a.String("image", "")) + if err != nil { + return nil, err + } + data["ready"] = ok + return &toolkit.Result{Text: text, Data: data}, nil +} + +// Ready checks one node for a governed upgrade (see upgrade.ready). +func Ready(c *toolkit.Context, name, image string) (bool, string, map[string]any, error) { + r, err := inspectRunning(c) + if err != nil { + return false, "", nil, err + } + if !tagRe.MatchString(name) || image == "" { + return false, "", nil, fmt.Errorf("name and image are required") + } + var b strings.Builder + ok := true + check := func(pass bool, line string) { + mark := "✓" + if !pass { + mark, ok = "✗", false + } + fmt.Fprintf(&b, "%s %s\n", mark, line) + } + _, code, _ := sh(c, "docker image inspect "+common.ShellQ(image)+" >/dev/null 2>&1", time.Minute) + check(code == 0, "image "+image+" is on the node") + if code == 0 { + has, err := imageHas(c, image, r.Binary, name) + check(err == nil && has, fmt.Sprintf("the new binary registers %q", name)) + } + old, err := runningHas(c, r.Binary, name) + check(err == nil && !old, fmt.Sprintf("the running binary (%s) does not register %q", r.Image, name)) + check(r.ComposeFile != "", "deployed compose file: "+orNone(r.ComposeFile)) + if ok { + b.WriteString("ready — the old binary will halt at the plan height and this image can resume it\n") + } + return ok, b.String(), map[string]any{"running_image": r.Image, "compose_file": r.ComposeFile}, nil +} + +// --- upgrade.switch ----------------------------------------------------------- + +type switchTool struct{} + +func (switchTool) Name() string { return "upgrade.switch" } +func (switchTool) Desc() string { + return "After the old binary halted at the upgrade height: set the new image in the node's deployed docker-compose file " + + "and recreate the container (compose down, up -d) on the same data dir. Refuses before the halt." +} +func (switchTool) Schema() map[string]any { + return toolkit.ObjSchema(map[string]any{ + "image": toolkit.Str("the image to run now, e.g. primium-evm:v0.6.1"), + }, "image") +} +func (switchTool) Tier() toolkit.Tier { return toolkit.TierLocalChange } +func (switchTool) Timeout(toolkit.Args) time.Duration { return 10 * time.Minute } + +var upgradeNeededRe = regexp.MustCompile(`UPGRADE "([^"]+)" NEEDED at height: (\d+)`) + +func (switchTool) Run(c *toolkit.Context, a toolkit.Args) (*toolkit.Result, error) { + r, err := inspectRunning(c) + if err != nil { + return nil, err + } + image := a.String("image", "") + if image == "" || strings.ContainsAny(image, "'\"|\\ ") { + return nil, fmt.Errorf("image is required (e.g. primium-evm:v0.6.1)") + } + if r.ComposeFile == "" { + return nil, fmt.Errorf("%s isn't managed by docker compose — no deployed compose file to update", c.Profile.Service.Unit) + } + if r.Image == image { + return nil, fmt.Errorf("%s already runs %s", c.Profile.Service.Unit, image) + } + // only after the halt: the old binary must stop itself at the height + logs, _, _ := sh(c, "docker logs --tail 300 "+common.ShellQ(c.Profile.Service.Unit)+" 2>&1 | grep -a 'UPGRADE \"' | tail -1", time.Minute) + m := upgradeNeededRe.FindStringSubmatch(logs) + if m == nil { + return nil, fmt.Errorf("%s hasn't halted for an upgrade (no `UPGRADE \"…\" NEEDED` in its log) — switching early runs the new binary on pre-upgrade state", c.Profile.Service.Unit) + } + if ok, err := imageHas(c, image, r.Binary, m[1]); err != nil || !ok { + return nil, fmt.Errorf("%s doesn't register the halted plan %q — it can't resume this chain", image, m[1]) + } + dir := r.ComposeDir + if dir == "" { + dir = path.Dir(r.ComposeFile) + } + script := fmt.Sprintf("set -e\ncp %[1]s %[1]s.pre-%[2]s\nsed -i 's|image: *%[3]s *$|image: %[4]s|' %[1]s\ngrep -q 'image: %[4]s' %[1]s\ncd %[5]s\ndocker compose -f %[1]s down\ndocker compose -f %[1]s up -d\n", + r.ComposeFile, strings.ReplaceAll(path.Base(image), ":", "-"), regexp.QuoteMeta(r.Image), image, common.ShellQ(dir)) + detail := map[string]any{"compose": r.ComposeFile, "from": r.Image, "to": image, "halted": m[1] + " at " + m[2]} + if err := c.Approve(fmt.Sprintf("switch %s from %s to %s (halted for %s at %s): edit %s, compose down + up -d", c.Profile.Service.Unit, r.Image, image, m[1], m[2], r.ComposeFile), toolkit.TierLocalChange, detail); err != nil { + return nil, err + } + out, code, err := sh(c, script, 8*time.Minute) + if err != nil || code != 0 { + return nil, fmt.Errorf("switch failed (exit %d): %v\n%s\nthe compose file was backed up beside it (.pre-*)", code, err, tail(out, 20)) + } + return &toolkit.Result{Text: fmt.Sprintf("%s now runs %s (compose file %s; backup kept as .pre-%s)\nnext: wait.until synced, then in-consensus", + c.Profile.Service.Unit, image, r.ComposeFile, strings.ReplaceAll(path.Base(image), ":", "-")), + Data: map[string]any{"image": image, "plan": m[1], "height": m[2]}}, nil +} diff --git a/internal/tools/upgradetool/flow_test.go b/internal/tools/upgradetool/flow_test.go new file mode 100644 index 0000000..eb9d289 --- /dev/null +++ b/internal/tools/upgradetool/flow_test.go @@ -0,0 +1,45 @@ +package upgradetool + +import ( + "sort" + "testing" +) + +func TestSemverOrder(t *testing.T) { + tags := []string{"v0.7.3", "v0.6.10", "v0.6.2", "v0.7.0", "v0.6.1"} + sort.Slice(tags, func(i, j int) bool { return semverLess(tags[i], tags[j]) }) + want := []string{"v0.6.1", "v0.6.2", "v0.6.10", "v0.7.0", "v0.7.3"} + for i := range want { + if tags[i] != want[i] { + t.Fatalf("got %v", tags) + } + } + if !semverLess("v0.6.0", "v0.6.1") || semverLess("v0.6.1", "v0.6.0") || semverLess("v0.6.1", "v0.6.1") { + t.Fatal("pairwise order wrong") + } +} + +func TestHandlerNamesFromSource(t *testing.T) { + src := `const UpgradeName = "v0.6.0-to-v0.7.0" +func (app EVMD) RegisterUpgradeHandlers() { + app.UpgradeKeeper.SetUpgradeHandler("v0.6.0-to-v0.6.1", func() {}) + app.UpgradeKeeper.SetUpgradeHandler( + UpgradeName,` + var got []string + for _, m := range upgradeNameRe.FindAllStringSubmatch(src, -1) { + got = append(got, m[1]) + } + for _, m := range setHandlerRe.FindAllStringSubmatch(src, -1) { + got = append(got, m[1]) + } + if len(got) != 2 || got[0] != "v0.6.0-to-v0.7.0" || got[1] != "v0.6.0-to-v0.6.1" { + t.Fatalf("names = %v", got) + } +} + +func TestUpgradeNeededLine(t *testing.T) { + m := upgradeNeededRe.FindStringSubmatch(`ERR UPGRADE "v0.6.0-to-v0.6.1" NEEDED at height: 18150000: module=x/upgrade`) + if m == nil || m[1] != "v0.6.0-to-v0.6.1" || m[2] != "18150000" { + t.Fatalf("%v", m) + } +} diff --git a/internal/tools/upgradetool/upgrade.go b/internal/tools/upgradetool/upgrade.go index 23db9a7..3f671d8 100644 --- a/internal/tools/upgradetool/upgrade.go +++ b/internal/tools/upgradetool/upgrade.go @@ -19,6 +19,7 @@ func Register(r *toolkit.Registry) { r.Register(checkTool{}) r.Register(prepareTool{}) r.Register(watchTool{}) + RegisterFlow(r) } type checkTool struct{} From 9ef961eef6bb9798465d60d1a04c212fd56f169b Mon Sep 17 00:00:00 2001 From: Abhijit Date: Fri, 9 Oct 2026 16:52:42 +0530 Subject: [PATCH 6/9] upgrade.releases lists the candidates when no one can choose Co-Authored-By: Claude Opus 5.5 --- internal/tools/upgradetool/flow.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/internal/tools/upgradetool/flow.go b/internal/tools/upgradetool/flow.go index d62dba9..5be004e 100644 --- a/internal/tools/upgradetool/flow.go +++ b/internal/tools/upgradetool/flow.go @@ -160,12 +160,13 @@ func (releasesTool) Run(c *toolkit.Context, a toolkit.Args) (*toolkit.Result, er } data := map[string]any{"current": cur, "repo": repo, "releases": newer, "handlers": handlers, "current_handlers": curHandlers} if len(newer) > 1 && c.Chooser != nil { - i, err := c.Chooser(c, fmt.Sprintf("%d releases are newer than %s — which one is the target?", len(newer), orNone(cur)), newer) - if err != nil { - return nil, err + if i, err := c.Chooser(c, fmt.Sprintf("%d releases are newer than %s — which one is the target?", len(newer), orNone(cur)), newer); err == nil { + data["chosen"] = newer[i] + fmt.Fprintf(&b, "target: %s\n", newer[i]) + } else { + // no one to ask: list them and leave the choice to the operator + b.WriteString("several candidates: confirm the target with the operator before building\n") } - data["chosen"] = newer[i] - fmt.Fprintf(&b, "target: %s\n", newer[i]) } else if len(newer) == 1 { data["chosen"] = newer[0] } else { From f61d2295f524aa64aaea8982d9a0ef44557b907d Mon Sep 17 00:00:00 2001 From: Abhijit Date: Fri, 9 Oct 2026 17:34:53 +0530 Subject: [PATCH 7/9] Name images built from official releases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit upgrade.build takes an image template — {tag} is the release tag, {version} the tag without v — e.g. primium-{tag} → primium-v0.7.3. profile add --image-name saves it as the profile's naming scheme. The build is the official tag as-is; the upgrade-handler patch is for forks only and stays opt-in. Co-Authored-By: Claude Opus 5.5 --- internal/cli/profile.go | 15 ++++++++++++++- internal/tools/upgradetool/flow.go | 21 +++++++++++++++++---- internal/tools/upgradetool/flow_test.go | 18 ++++++++++++++++++ 3 files changed, 49 insertions(+), 5 deletions(-) diff --git a/internal/cli/profile.go b/internal/cli/profile.go index f31e545..4a5edfc 100644 --- a/internal/cli/profile.go +++ b/internal/cli/profile.go @@ -78,7 +78,7 @@ func profileCmd() *cobra.Command { func profileAddCmd() *cobra.Command { var p config.Profile - var signerKey, feeDenom, valoper string + var signerKey, feeDenom, valoper, imageName string cmd := &cobra.Command{ Use: "add ", Aliases: []string{"edit", "set"}, @@ -160,6 +160,12 @@ func profileAddCmd() *cobra.Command { if fl.Changed("container-home") { np.Signer.ContainerHome = p.Signer.ContainerHome } + if fl.Changed("image-name") { + if np.Metadata == nil { + np.Metadata = map[string]string{} + } + np.Metadata["image"] = imageName + } if fl.Changed("container-keyring") { np.Signer.ContainerKeyring = p.Signer.ContainerKeyring } @@ -202,6 +208,12 @@ func profileAddCmd() *cobra.Command { } p.Metadata["valoper"] = valoper } + if imageName != "" { + if p.Metadata == nil { + p.Metadata = map[string]string{} + } + p.Metadata["image"] = imageName + } return cfg.UpsertProfile(&p) }, } @@ -229,6 +241,7 @@ func profileAddCmd() *cobra.Command { f.StringVar(&p.Signer.Backend, "signer-backend", "", "os | file") f.StringVar(&p.Signer.Mode, "signer-mode", "", "local (cometcli keyring) | container (the node's own keyring); empty asks when both work") f.StringVar(&p.Signer.ContainerHome, "container-home", "", "node home inside the container, for signing there (e.g. /data/node0/evmd)") + f.StringVar(&imageName, "image-name", "", "how images built from releases are named; {tag} = release tag, e.g. primium-{tag} → primium-v0.7.2") f.StringVar(&p.Signer.ContainerKeyring, "container-keyring", "", "keyring backend inside the container: test | file (detected when empty)") f.StringVar(&feeDenom, "fee-denom", "", "fee denom (e.g. atest)") f.StringVar(&valoper, "valoper", "", "validator operator address (read-only signing/jail views without a signer)") diff --git a/internal/tools/upgradetool/flow.go b/internal/tools/upgradetool/flow.go index 5be004e..cae3913 100644 --- a/internal/tools/upgradetool/flow.go +++ b/internal/tools/upgradetool/flow.go @@ -278,9 +278,9 @@ func (buildTool) Desc() string { func (buildTool) Schema() map[string]any { return toolkit.ObjSchema(map[string]any{ "tag": toolkit.Str("release tag to build, e.g. v0.6.1"), - "image": toolkit.Str("image to produce (default: the running image's name with the new tag)"), + "image": toolkit.Str("image to produce; {tag} is the release tag, {version} the tag without v — e.g. primium-{tag} → primium-v0.7.2 (default: the profile's metadata.image, else the running image's name with the new tag)"), "dockerfile": toolkit.Str("Dockerfile on the node (default: Dockerfile.primium beside the compose project, or one level up)"), - "handler": toolkit.Str("add a no-op upgrade handler with this plan name if the release lacks it, e.g. v0.6.0-to-v0.6.1"), + "handler": toolkit.Str("forks only: add a no-op upgrade handler with this plan name (official releases ship their own — leave empty)"), "repo": toolkit.Str("github owner/repo (default: the running image's source)"), }, "tag") } @@ -300,8 +300,7 @@ func (buildTool) Run(c *toolkit.Context, a toolkit.Args) (*toolkit.Result, error if !tagRe.MatchString(tag) || (handler != "" && !tagRe.MatchString(handler)) || repo == "" { return nil, fmt.Errorf("tag, handler and repo must be plain names (letters, digits, . _ -)") } - name, _, _ := strings.Cut(r.Image, ":") - image := a.String("image", name+":"+tag) + image := ImageFor(c, a.String("image", ""), r.Image, tag) if image == r.Image { return nil, fmt.Errorf("%s is the image running now — build a different tag", image) } @@ -534,3 +533,17 @@ func (switchTool) Run(c *toolkit.Context, a toolkit.Args) (*toolkit.Result, erro c.Profile.Service.Unit, image, r.ComposeFile, strings.ReplaceAll(path.Base(image), ":", "-")), Data: map[string]any{"image": image, "plan": m[1], "height": m[2]}}, nil } + +// ImageFor names the image built from a release tag: an explicit template, +// else the profile's metadata.image, else the running image's name with +// the new tag. {tag} is the tag (v0.7.2), {version} the tag without v. +func ImageFor(c *toolkit.Context, template, runningImage, tag string) string { + if template == "" && c.Profile != nil { + template = c.Profile.Metadata["image"] + } + if template == "" { + name, _, _ := strings.Cut(runningImage, ":") + return name + ":" + tag + } + return strings.NewReplacer("{tag}", tag, "{version}", strings.TrimPrefix(tag, "v")).Replace(template) +} diff --git a/internal/tools/upgradetool/flow_test.go b/internal/tools/upgradetool/flow_test.go index eb9d289..08fcd07 100644 --- a/internal/tools/upgradetool/flow_test.go +++ b/internal/tools/upgradetool/flow_test.go @@ -1,6 +1,8 @@ package upgradetool import ( + "github.com/abhijitkrm/cometcli/internal/config" + "github.com/abhijitkrm/cometcli/internal/toolkit" "sort" "testing" ) @@ -43,3 +45,19 @@ func TestUpgradeNeededLine(t *testing.T) { t.Fatalf("%v", m) } } + +func TestImageNames(t *testing.T) { + c := &toolkit.Context{Profile: &config.Profile{Metadata: map[string]string{"image": "primium-{tag}"}}} + cases := []struct{ tmpl, running, want string }{ + {"", "primium-evm:v0.6.1", "primium-v0.7.2"}, // the profile's naming scheme + {"primium:{version}", "primium-evm:v0.6.1", "primium:0.7.2"}, // explicit wins + } + for _, x := range cases { + if got := ImageFor(c, x.tmpl, x.running, "v0.7.2"); got != x.want { + t.Errorf("ImageFor(%q) = %q, want %q", x.tmpl, got, x.want) + } + } + if got := ImageFor(&toolkit.Context{Profile: &config.Profile{}}, "", "primium-evm:v0.6.1", "v0.7.2"); got != "primium-evm:v0.7.2" { + t.Errorf("default = %q", got) + } +} From 7d88b8e987e0c13503774188da82d76645e5759e Mon Sep 17 00:00:00 2001 From: Abhijit Date: Fri, 9 Oct 2026 17:37:07 +0530 Subject: [PATCH 8/9] upgrade.build checks the release's Go version against the Dockerfile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A node's Dockerfile was written for the release it runs; a newer release may need a newer Go (cosmos/evm v0.7.x needs 1.25.9, the deployed Dockerfile pins 1.25.8). The build reads go.mod's go/toolchain line and, when the builder image is older, builds from a copy with the Go image raised — the original Dockerfile is untouched — and says so in a line. Co-Authored-By: Claude Opus 5.5 --- internal/tools/upgradetool/flow.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/internal/tools/upgradetool/flow.go b/internal/tools/upgradetool/flow.go index cae3913..3adc760 100644 --- a/internal/tools/upgradetool/flow.go +++ b/internal/tools/upgradetool/flow.go @@ -348,8 +348,20 @@ if ! grep -q '"%[1]s"' "$f"; then fi `, handler) } - fmt.Fprintf(&script, "docker build -q -f %s -t %s --label name=primium --label org.opencontainers.image.version=%s --label org.opencontainers.image.source=https://github.com/%s . >/dev/null\n", - common.ShellQ(dockerfile), common.ShellQ(image), common.ShellQ(tag), repo) + // preflight: the node's Dockerfile was written for the running release; + // a newer release may need a newer Go than its builder image has — + // build from a copy with the Go image raised (the original stays as is) + fmt.Fprintf(&script, `df=%s +need=$(awk '/^toolchain go/{sub("go","",$2); print $2; exit} /^go [0-9]/{print $2; exit}' go.mod) +have=$(sed -n 's/^FROM golang:\([0-9][0-9.]*\).*/\1/p' "$df" | head -1) +if [ -n "$need" ] && [ -n "$have" ] && [ "$(printf '%%s\n%%s\n' "$have" "$need" | sort -V | head -1)" != "$need" ]; then + sed "s/^FROM golang:$have/FROM golang:$need/" "$df" > .cometcli.Dockerfile + df=.cometcli.Dockerfile + echo "note: $(basename %s) builds with Go $have; $(git describe --tags 2>/dev/null) needs Go $need — built with golang:$need (your Dockerfile is unchanged)" +fi +`, common.ShellQ(dockerfile), common.ShellQ(dockerfile)) + fmt.Fprintf(&script, "docker build -q -f \"$df\" -t %s --label name=primium --label org.opencontainers.image.version=%s --label org.opencontainers.image.source=https://github.com/%s . >/dev/null\n", + common.ShellQ(image), common.ShellQ(tag), repo) fmt.Fprintf(&script, "echo \"version: $(docker run --rm --entrypoint %s %s version 2>&1 | head -1)\"\n", r.Binary, common.ShellQ(image)) detail := map[string]any{"image": image, "tag": tag, "repo": repo, "dockerfile": dockerfile, "source": src} if handler != "" { From c2f7849212f09101111247ac0517435cadb23d0c Mon Sep 17 00:00:00 2001 From: Abhijit Date: Fri, 9 Oct 2026 17:54:50 +0530 Subject: [PATCH 9/9] Ask before building a consensus-breaking release upgrade.build reads the release notes: a state- or consensus-breaking release always asks (RequireApproval, whatever --yes or autopilot say) and the prompt says every validator must switch together at an upgrade height. The Go builder bump is decided before the prompt and shown in it. upgrade.releases marks breaking releases. Co-Authored-By: Claude Opus 5.5 --- internal/tools/upgradetool/flow.go | 84 ++++++++++++++++++++++++- internal/tools/upgradetool/flow_test.go | 20 ++++++ 2 files changed, 102 insertions(+), 2 deletions(-) diff --git a/internal/tools/upgradetool/flow.go b/internal/tools/upgradetool/flow.go index 3adc760..0b43750 100644 --- a/internal/tools/upgradetool/flow.go +++ b/internal/tools/upgradetool/flow.go @@ -152,7 +152,11 @@ func (releasesTool) Run(c *toolkit.Context, a toolkit.Args) (*toolkit.Result, er handlers := map[string][]string{} for _, t := range newer { handlers[t] = upgradeNames(c, repo, t) - fmt.Fprintf(&b, " %-10s registers %s\n", t, orNone(strings.Join(handlers[t], ", "))) + mark := "" + if why := breaking(c, repo, t); why != "" { + mark = " [" + why + "]" + } + fmt.Fprintf(&b, " %-10s registers %s%s\n", t, orNone(strings.Join(handlers[t], ", ")), mark) } curHandlers := upgradeNames(c, repo, cur) if len(curHandlers) > 0 { @@ -367,7 +371,18 @@ fi if handler != "" { detail["adds handler"] = handler } - if err := c.Approve(fmt.Sprintf("build %s on %s from %s@%s (takes several minutes; the node keeps running)", image, c.Profile.Name, repo, tag), toolkit.TierLocalChange, detail); err != nil { + // what the operator should know before saying yes + if need, have := goNeeded(c, repo, tag), dockerfileGo(c, dockerfile); need != "" && have != "" && semverLess("v"+have, "v"+need) { + detail["builder"] = fmt.Sprintf("Go %s → %s (%s needs it; the build uses a copy, your Dockerfile is unchanged)", have, need, tag) + } + prompt := fmt.Sprintf("build %s on %s from %s@%s (takes several minutes; the node keeps running)", image, c.Profile.Name, repo, tag) + if why := breaking(c, repo, tag); why != "" { + // consensus-breaking: always asked, whatever the auto-approval + detail["consensus"] = why + " — every validator must switch together at an upgrade height (gov proposal or halt-height); never restart a node onto it early" + if err := toolkit.RequireApproval(c, "CONSENSUS-BREAKING release — "+prompt, toolkit.TierLocalChange, detail); err != nil { + return nil, err + } + } else if err := c.Approve(prompt, toolkit.TierLocalChange, detail); err != nil { return nil, err } if c.Progress != nil { @@ -559,3 +574,68 @@ func ImageFor(c *toolkit.Context, template, runningImage, tag string) string { } return strings.NewReplacer("{tag}", tag, "{version}", strings.TrimPrefix(tag, "v")).Replace(template) } + +var ( + breakingRe = regexp.MustCompile(`(?i)\b(state|consensus)[ -]breaking\b`) + notBreakingRe = regexp.MustCompile(`(?i)\bnon[ -](state|consensus)[ -]breaking\b|\bnot (state|consensus)[ -]breaking\b`) + goLineRe = regexp.MustCompile(`(?m)^toolchain go([0-9.]+)|^go ([0-9][0-9.]*)`) + fromGoRe = regexp.MustCompile(`(?m)^FROM golang:([0-9][0-9.]*)`) +) + +// breaking says whether a release's notes call it state- or +// consensus-breaking ("" when they don't, or can't be read). +func breaking(c *toolkit.Context, repo, tag string) string { + body := fetch(c, "https://api.github.com/repos/"+repo+"/releases/tags/"+tag) + var rel struct { + Body string `json:"body"` + } + if json.Unmarshal([]byte(body), &rel) != nil || notBreakingRe.MatchString(rel.Body) { + return "" + } + if m := breakingRe.FindString(rel.Body); m != "" { + return strings.ToLower(m) + " per the release notes" + } + return "" +} + +// goNeeded is the Go version a release's go.mod asks for. +func goNeeded(c *toolkit.Context, repo, tag string) string { + m := goLineRe.FindStringSubmatch(fetch(c, "https://raw.githubusercontent.com/"+repo+"/"+tag+"/go.mod")) + switch { + case m == nil: + return "" + case m[1] != "": + return m[1] + } + return m[2] +} + +// dockerfileGo is the Go version of the node Dockerfile's builder image. +func dockerfileGo(c *toolkit.Context, dockerfile string) string { + out, _, err := sh(c, "cat "+common.ShellQ(dockerfile), time.Minute) + if err != nil { + return "" + } + if m := fromGoRe.FindStringSubmatch(out); m != nil { + return m[1] + } + return "" +} + +func fetch(c *toolkit.Context, url string) string { + req, err := http.NewRequestWithContext(c, "GET", url, nil) + if err != nil { + return "" + } + req.Header.Set("Accept", "application/vnd.github+json") + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "" + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + return "" + } + raw, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + return string(raw) +} diff --git a/internal/tools/upgradetool/flow_test.go b/internal/tools/upgradetool/flow_test.go index 08fcd07..2e08df2 100644 --- a/internal/tools/upgradetool/flow_test.go +++ b/internal/tools/upgradetool/flow_test.go @@ -61,3 +61,23 @@ func TestImageNames(t *testing.T) { t.Errorf("default = %q", got) } } + +func TestBreakingNotes(t *testing.T) { + for note, want := range map[string]bool{ + "__This release is state breaking.__": true, + "Consensus-breaking change to the fee market": true, + "This is a non-state-breaking patch release": false, + "Bug fixes only.": false, + } { + got := breakingRe.MatchString(note) && !notBreakingRe.MatchString(note) + if got != want { + t.Errorf("%q: breaking=%v, want %v", note, got, want) + } + } + if m := goLineRe.FindStringSubmatch("module x\n\ngo 1.25.9\n"); m == nil || m[2] != "1.25.9" { + t.Fatalf("go line: %v", m) + } + if m := fromGoRe.FindStringSubmatch("ARG X\nFROM golang:1.25.8-alpine AS b\n"); m == nil || m[1] != "1.25.8" { + t.Fatalf("from line: %v", m) + } +}