From e2b64f8cbb538890fba16aba99f47d171f414376 Mon Sep 17 00:00:00 2001 From: Eli Sherman Date: Thu, 29 Oct 2026 15:47:00 -0400 Subject: [PATCH 1/4] Read the session and message steps' trace attributes off the wire The exporter already starts a span for any mapped method in methodsEvaluated, but it had no sources for the attributes the mapping lists on the session and message spans, so those spans went out without them. This adds sources for acs.session.reason, acs.content.types, and the session span's optional acs.tenant_id, acs.user.id and acs.policy_mode. acs.content.types holds one entry per content item, in wire order, and is not deduplicated. The mapping does not say whether it is a set; this keeps the counts, and is a one-line change if the Spec leads prefer a set. This Guardian does not evaluate the four methods yet, and PR #229 keeps them out of methods_evaluated, so the spans do not appear in a live run. The tests drive them through the exporter's methodsEvaluated option, as #322 describes. Provenance on message content items is left for a later slice. Part of #322 Co-Authored-By: Claude Opus 5.5 Signed-off-by: Eli Sherman --- .../agt/packages/guardian/src/otel-mapping.ts | 19 ++- .../guardian/test/otel-trace-exporter.test.ts | 135 +++++++++++++++++- 2 files changed, 151 insertions(+), 3 deletions(-) diff --git a/reference-implementations/agt/packages/guardian/src/otel-mapping.ts b/reference-implementations/agt/packages/guardian/src/otel-mapping.ts index 05d9443b..1c2db768 100644 --- a/reference-implementations/agt/packages/guardian/src/otel-mapping.ts +++ b/reference-implementations/agt/packages/guardian/src/otel-mapping.ts @@ -194,6 +194,15 @@ function policyColumn(references: unknown, field: string): string[] | undefined return references.map((reference) => asString(asObject(reference)?.[field]) ?? ""); } +/** `content[].type` of a message payload, one entry per item in the wire's + * order and not deduplicated, so a reader can count the items of each type. + * An item without a string `type` adds nothing. Absent when no item has one. */ +function contentTypes(content: unknown): string[] | undefined { + if (!Array.isArray(content)) return undefined; + const types = content.flatMap((item) => asString(asObject(item)?.type) ?? []); + return types.length === 0 ? undefined : types; +} + /** The parts of a request envelope the source tables read. Read off the * parsed JSON rather than the validated envelope, because the span starts * before validation runs and an envelope that fails the schema is still a @@ -226,12 +235,18 @@ export type Source = (parts: Parts) => AttributeValue | undefined; * attribute `step_to_span` can name on a span this Guardian emits. The * mapping decides WHICH of these a span carries; this table only says HOW * each one is read. Paths follow the hook payload schemas - * (`hooks/tool-call-request.json`, `hooks/tool-call-result.json`) and - * `request-envelope.json`'s `AcsParams.metadata`. + * (`hooks/tool-call-request.json`, `hooks/tool-call-result.json`, + * `hooks/session-start.json`, `hooks/session-end.json`, + * `hooks/user-message.json`) and `request-envelope.json`'s `AcsParams`. */ export const STEP_ATTRIBUTE_SOURCES: Readonly>> = { "acs.session.id": (r) => asString(r.metadata.session_id), "acs.agent.id": (r) => asString(r.metadata.agent_id), + "acs.tenant_id": (r) => asString(r.params.tenant_id), + "acs.user.id": (r) => asString(asObject(r.payload.user_identity)?.user_id), + "acs.policy_mode": (r) => asString(r.payload.policy_mode), + "acs.session.reason": (r) => asString(r.payload.reason), + "acs.content.types": (r) => contentTypes(r.payload.content), "gen_ai.tool.name": (r) => asString(r.tool.name), "acs.tool.provider": (r) => asString(r.tool.provider), "acs.tool.version": (r) => asString(r.tool.version), diff --git a/reference-implementations/agt/packages/guardian/test/otel-trace-exporter.test.ts b/reference-implementations/agt/packages/guardian/test/otel-trace-exporter.test.ts index 0d69ca9d..43e5b40c 100644 --- a/reference-implementations/agt/packages/guardian/test/otel-trace-exporter.test.ts +++ b/reference-implementations/agt/packages/guardian/test/otel-trace-exporter.test.ts @@ -33,6 +33,9 @@ const MAPPING = loadOtelMapping(); * the same source server.ts builds the exporter from. */ const METHODS_EVALUATED = buildServerHello().methods_evaluated; const TOOL_CALL_SPAN = MAPPING.stepToSpan["steps/toolCallRequest"]!; +/** The session and message steps (#322 slice 1). This Guardian does not + * evaluate them yet, so the tests pass them in methodsEvaluated themselves. */ +const SESSION_AND_MESSAGE_METHODS = ["steps/sessionStart", "steps/sessionEnd", "steps/userMessage", "steps/agentResponse"]; /** A Provenance object as provenance.json defines it: `provenance_id` and * `origin` required, everything else optional. */ @@ -589,7 +592,7 @@ describe("createOtelTraceExporter -- what one exchange becomes", () => { // or that attribute quietly stops appearing when the mapping changes -- // which is what #227's conditional_attributes would have done. it("has a source for every attribute the real mapping lists on an evaluated method, and for every decision attribute", () => { - for (const method of METHODS_EVALUATED) { + for (const method of [...METHODS_EVALUATED, ...SESSION_AND_MESSAGE_METHODS]) { const entry = MAPPING.stepToSpan[method]; if (entry === undefined) { continue; @@ -620,6 +623,136 @@ describe("createOtelTraceExporter -- what one exchange becomes", () => { }); }); +describe("createOtelTraceExporter -- the session and message steps (#322)", () => { + function sessionAndMessageExporter() { + return memoryExporter({ methodsEvaluated: [...METHODS_EVALUATED, ...SESSION_AND_MESSAGE_METHODS] }); + } + + /** `tenant_id` sits on AcsParams, beside `metadata`, so makeEnvelope has + * no option for it. */ + function withTenant(envelope: Record, tenantId: string): Record { + return { ...envelope, params: { ...(envelope.params as Record), tenant_id: tenantId } }; + } + + const cases: { method: string; envelope: (sessionId: string) => Record; span: string; attributes: Record }[] = [ + { + method: "steps/sessionStart", + envelope: (sessionId) => + withTenant( + makeEnvelope("steps/sessionStart", { user_identity: { user_id: "user-1" }, policy_mode: "strict" }, { sessionId }), + "tenant-1", + ), + span: "acs.session", + attributes: { "acs.tenant_id": "tenant-1", "acs.user.id": "user-1", "acs.policy_mode": "strict" }, + }, + { + method: "steps/sessionEnd", + envelope: (sessionId) => makeEnvelope("steps/sessionEnd", { reason: "completed" }, { sessionId }), + span: "acs.session.end", + attributes: { "acs.session.reason": "completed" }, + }, + { + method: "steps/userMessage", + envelope: (sessionId) => + makeEnvelope( + "steps/userMessage", + { content: [{ type: "text", value: "hi" }, { type: "image", value: "aGk=" }, { type: "text", value: "again" }] }, + { sessionId }, + ), + span: "acs.message.user", + attributes: { "acs.content.types": ["text", "image", "text"] }, + }, + { + method: "steps/agentResponse", + envelope: (sessionId) => makeEnvelope("steps/agentResponse", { content: [{ type: "text", value: "done" }] }, { sessionId }), + span: "acs.message.agent", + attributes: { "acs.agent.id": "agent-1" }, + }, + ]; + + for (const { method, envelope, span: spanName, attributes } of cases) { + it(`${method} becomes one ${spanName} span carrying every attribute the mapping lists and the wire supplied`, async () => { + const { exporter, spans } = sessionAndMessageExporter(); + const sessionId = crypto.randomUUID(); + exporter.end(exporter.start(envelope(sessionId), method), decisionResponse(1, { decision: "allow" })); + await exporter.flush(); + + expect(spans()).toHaveLength(1); + const [span] = spans(); + expect(span?.name).toBe(spanName); + expect(span?.attributes["acs.session.id"]).toBe(sessionId); + expect(typeof span?.attributes["acs.request_id"]).toBe("string"); + for (const [name, value] of Object.entries(attributes)) { + expect({ name, value: span?.attributes[name] }).toEqual({ name, value }); + } + for (const name of MAPPING.stepToSpan[method]!.required_attributes) { + expect({ name, present: span?.attributes[name] !== undefined }).toEqual({ name, present: true }); + } + }); + } + + it("leaves off the session span's optional attributes when the wire does not carry them", async () => { + const { exporter, spans } = sessionAndMessageExporter(); + exporter.end(exporter.start(makeEnvelope("steps/sessionStart", {}), "steps/sessionStart"), decisionResponse(1, { decision: "allow" })); + await exporter.flush(); + + const attributes = spans()[0]?.attributes ?? {}; + for (const name of ["acs.tenant_id", "acs.user.id", "acs.policy_mode"]) { + expect({ name, present: name in attributes }).toEqual({ name, present: false }); + } + }); + + // One entry per content item, in wire order, so a reader can count the + // images in a message. An item without a string type adds nothing, and a + // message that leaves no type at all gets no attribute. + const contentCases: { label: string; content: unknown; want: string[] | undefined }[] = [ + { label: "no content field", content: undefined, want: undefined }, + { label: "an empty content array", content: [], want: undefined }, + { label: "content that is not an array", content: "hi", want: undefined }, + { label: "only items without a string type", content: [{ value: "hi" }, { type: 7, value: "x" }], want: undefined }, + { label: "a mix of typed and untyped items", content: [{ type: "file", value: "a" }, { value: "b" }, { type: "audio", value: "c" }], want: ["file", "audio"] }, + ]; + + for (const { label, content, want } of contentCases) { + it(`sets acs.content.types for ${label} to ${JSON.stringify(want)}`, async () => { + const { exporter, spans } = sessionAndMessageExporter(); + const payload = content === undefined ? {} : { content }; + exporter.end(exporter.start(makeEnvelope("steps/userMessage", payload), "steps/userMessage"), decisionResponse(1, { decision: "allow" })); + await exporter.flush(); + + expect(spans()[0]?.attributes["acs.content.types"]).toEqual(want); + }); + } + + it("records a decision on a session span the same way it does on a tool call span", async () => { + const { exporter, spans } = sessionAndMessageExporter(); + exporter.end( + exporter.start(makeEnvelope("steps/sessionEnd", { reason: "completed" }), "steps/sessionEnd"), + decisionResponse(1, { decision: "allow", reason_codes: ["session_closed"] }), + ); + await exporter.flush(); + + const [event] = spans()[0]?.events ?? []; + expect(event?.name).toBe("acs.decision"); + expect(event?.attributes?.["acs.decision"]).toBe("allow"); + expect(event?.attributes?.["acs.reason_codes"]).toEqual(["session_closed"]); + }); + + it("records a guardian_error failure on a message span answered with a JSON-RPC error", async () => { + const { exporter, spans } = sessionAndMessageExporter(); + exporter.end( + exporter.start(makeEnvelope("steps/userMessage", { content: [{ type: "text", value: "hi" }] }, { id: 4 }), "steps/userMessage"), + { jsonrpc: "2.0", id: 4, error: { code: -32020, message: "evaluation failed" } }, + ); + await exporter.flush(); + + const [span] = spans(); + expect(span?.status.code).toBe(SpanStatusCode.ERROR); + expect(span?.events.map((event) => event.name)).toEqual(["acs.decision.failure"]); + expect(span?.events[0]?.attributes?.["acs.failure.class"]).toBe("guardian_error"); + }); +}); + describe("Guardian trace export wiring", () => { it("emits one span per decided step, carrying the session and the decision event", async () => { const { exporter, spans } = memoryExporter(); From 92114f91f0405ce0e61be4d310e5073cc66b37ff Mon Sep 17 00:00:00 2001 From: Eli Sherman Date: Thu, 29 Oct 2026 16:15:50 -0400 Subject: [PATCH 2/4] Carry message and Intent provenance onto the session and message spans The mapping's provenance rule applies to every hook payload that attaches Provenance. The message hooks attach it to each content item, and sessionStart to its Intent as parser_provenance, but the exporter read it only from tool arguments and outputs, so acs.message.user, acs.message.agent and acs.session went out without it. provenanceObjects now reads those carriers too, with the same columns, order, padding and limit. The session and message tests now check that exactly one span with the mapped name exists before asserting that an attribute is absent, so a dropped span can no longer pass them. They also compare acs.request_id and acs.agent.id against values the test chose, and run the decision and guardian_error checks on all four methods. Part of #322 Co-Authored-By: Claude Opus 5.5 Signed-off-by: Eli Sherman --- .../agt/packages/guardian/src/otel-mapping.ts | 10 +- .../guardian/test/otel-trace-exporter.test.ts | 184 ++++++++++++------ 2 files changed, 131 insertions(+), 63 deletions(-) diff --git a/reference-implementations/agt/packages/guardian/src/otel-mapping.ts b/reference-implementations/agt/packages/guardian/src/otel-mapping.ts index 1c2db768..93f37f55 100644 --- a/reference-implementations/agt/packages/guardian/src/otel-mapping.ts +++ b/reference-implementations/agt/packages/guardian/src/otel-mapping.ts @@ -341,13 +341,17 @@ export const PROVENANCE_ATTRIBUTE_SOURCES: Record { }); }); -describe("createOtelTraceExporter -- the session and message steps (#322)", () => { +// #322 slice 1: the session and message steps. This Guardian does not +// evaluate them yet, so these tests hand them to the exporter through +// methodsEvaluated, as the issue describes. +describe("createOtelTraceExporter -- the session and message steps", () => { function sessionAndMessageExporter() { return memoryExporter({ methodsEvaluated: [...METHODS_EVALUATED, ...SESSION_AND_MESSAGE_METHODS] }); } - /** `tenant_id` sits on AcsParams, beside `metadata`, so makeEnvelope has - * no option for it. */ - function withTenant(envelope: Record, tenantId: string): Record { - return { ...envelope, params: { ...(envelope.params as Record), tenant_id: tenantId } }; + /** makeEnvelope fixes `agent_id` and has no option for `tenant_id`, which + * sits on AcsParams beside `metadata`. */ + function stepEnvelope( + method: string, + payload: Record, + { sessionId, requestId, agentId, tenantId }: { sessionId?: string; requestId?: string; agentId?: string; tenantId?: string } = {}, + ): Record { + const envelope = makeEnvelope(method, payload, { sessionId, requestId }); + const params = envelope.params as Record; + return { + ...envelope, + params: { + ...params, + ...(tenantId === undefined ? {} : { tenant_id: tenantId }), + metadata: { ...(params.metadata as Record), ...(agentId === undefined ? {} : { agent_id: agentId }) }, + }, + }; } - const cases: { method: string; envelope: (sessionId: string) => Record; span: string; attributes: Record }[] = [ + /** The one span a single exchange produced, after checking that it is the + * only one and has the name the method maps to, so a test that looks for + * a missing attribute cannot pass because the span itself is missing. */ + async function onlySpan(method: string, envelope: Record, response: unknown = decisionResponse(1, { decision: "allow" })) { + const { exporter, spans } = sessionAndMessageExporter(); + exporter.end(exporter.start(envelope, method), response); + await exporter.flush(); + expect(spans()).toHaveLength(1); + const [span] = spans(); + expect(span?.name).toBe(MAPPING.stepToSpan[method]!.span_name); + return span!; + } + + const cases: { method: string; span: string; envelope: (ids: { sessionId: string; requestId: string }) => Record; attributes: Record }[] = [ { method: "steps/sessionStart", - envelope: (sessionId) => - withTenant( - makeEnvelope("steps/sessionStart", { user_identity: { user_id: "user-1" }, policy_mode: "strict" }, { sessionId }), - "tenant-1", - ), span: "acs.session", + envelope: (ids) => + stepEnvelope("steps/sessionStart", { user_identity: { user_id: "user-1" }, policy_mode: "strict" }, { ...ids, tenantId: "tenant-1" }), attributes: { "acs.tenant_id": "tenant-1", "acs.user.id": "user-1", "acs.policy_mode": "strict" }, }, { method: "steps/sessionEnd", - envelope: (sessionId) => makeEnvelope("steps/sessionEnd", { reason: "completed" }, { sessionId }), span: "acs.session.end", + envelope: (ids) => stepEnvelope("steps/sessionEnd", { reason: "completed" }, ids), attributes: { "acs.session.reason": "completed" }, }, { method: "steps/userMessage", - envelope: (sessionId) => - makeEnvelope( + span: "acs.message.user", + envelope: (ids) => + stepEnvelope( "steps/userMessage", { content: [{ type: "text", value: "hi" }, { type: "image", value: "aGk=" }, { type: "text", value: "again" }] }, - { sessionId }, + ids, ), - span: "acs.message.user", attributes: { "acs.content.types": ["text", "image", "text"] }, }, { method: "steps/agentResponse", - envelope: (sessionId) => makeEnvelope("steps/agentResponse", { content: [{ type: "text", value: "done" }] }, { sessionId }), span: "acs.message.agent", - attributes: { "acs.agent.id": "agent-1" }, + envelope: (ids) => stepEnvelope("steps/agentResponse", { content: [{ type: "text", value: "done" }] }, { ...ids, agentId: "responder-7" }), + attributes: { "acs.agent.id": "responder-7" }, }, ]; - for (const { method, envelope, span: spanName, attributes } of cases) { + for (const { method, span: spanName, envelope, attributes } of cases) { it(`${method} becomes one ${spanName} span carrying every attribute the mapping lists and the wire supplied`, async () => { - const { exporter, spans } = sessionAndMessageExporter(); const sessionId = crypto.randomUUID(); - exporter.end(exporter.start(envelope(sessionId), method), decisionResponse(1, { decision: "allow" })); - await exporter.flush(); + const requestId = crypto.randomUUID(); + const span = await onlySpan(method, envelope({ sessionId, requestId })); - expect(spans()).toHaveLength(1); - const [span] = spans(); - expect(span?.name).toBe(spanName); - expect(span?.attributes["acs.session.id"]).toBe(sessionId); - expect(typeof span?.attributes["acs.request_id"]).toBe("string"); + expect(span.name).toBe(spanName); + expect(span.attributes["acs.session.id"]).toBe(sessionId); + expect(span.attributes["acs.request_id"]).toBe(requestId); for (const [name, value] of Object.entries(attributes)) { - expect({ name, value: span?.attributes[name] }).toEqual({ name, value }); + expect({ name, value: span.attributes[name] }).toEqual({ name, value }); } for (const name of MAPPING.stepToSpan[method]!.required_attributes) { - expect({ name, present: span?.attributes[name] !== undefined }).toEqual({ name, present: true }); + expect({ name, present: span.attributes[name] !== undefined }).toEqual({ name, present: true }); } }); } it("leaves off the session span's optional attributes when the wire does not carry them", async () => { - const { exporter, spans } = sessionAndMessageExporter(); - exporter.end(exporter.start(makeEnvelope("steps/sessionStart", {}), "steps/sessionStart"), decisionResponse(1, { decision: "allow" })); - await exporter.flush(); + const span = await onlySpan("steps/sessionStart", stepEnvelope("steps/sessionStart", {})); - const attributes = spans()[0]?.attributes ?? {}; for (const name of ["acs.tenant_id", "acs.user.id", "acs.policy_mode"]) { - expect({ name, present: name in attributes }).toEqual({ name, present: false }); + expect({ name, present: name in span.attributes }).toEqual({ name, present: false }); } }); @@ -715,42 +735,86 @@ describe("createOtelTraceExporter -- the session and message steps (#322)", () = for (const { label, content, want } of contentCases) { it(`sets acs.content.types for ${label} to ${JSON.stringify(want)}`, async () => { - const { exporter, spans } = sessionAndMessageExporter(); - const payload = content === undefined ? {} : { content }; - exporter.end(exporter.start(makeEnvelope("steps/userMessage", payload), "steps/userMessage"), decisionResponse(1, { decision: "allow" })); - await exporter.flush(); + const span = await onlySpan("steps/userMessage", stepEnvelope("steps/userMessage", content === undefined ? {} : { content })); - expect(spans()[0]?.attributes["acs.content.types"]).toEqual(want); + expect(span.attributes["acs.content.types"]).toEqual(want); }); } - it("records a decision on a session span the same way it does on a tool call span", async () => { - const { exporter, spans } = sessionAndMessageExporter(); - exporter.end( - exporter.start(makeEnvelope("steps/sessionEnd", { reason: "completed" }), "steps/sessionEnd"), - decisionResponse(1, { decision: "allow", reason_codes: ["session_closed"] }), - ); - await exporter.flush(); + // The provenance rule is the tool gates' (one column per attribute, one + // element per Provenance object, source_id padded with ""), read from + // where these payloads attach Provenance: each content item, and the + // session Intent's parser_provenance. + it("carries the provenance of each message content item that has one, in content order", async () => { + for (const method of ["steps/userMessage", "steps/agentResponse"]) { + const span = await onlySpan( + method, + stepEnvelope(method, { + content: [ + { type: "text", value: "a", provenance: provenance("user_input", { source_id: "user-1" }) }, + // No provenance: contributes no element to any column. + { type: "text", value: "b" }, + { type: "file", value: "c", provenance: provenance("retrieval") }, + ], + }), + ); - const [event] = spans()[0]?.events ?? []; - expect(event?.name).toBe("acs.decision"); - expect(event?.attributes?.["acs.decision"]).toBe("allow"); - expect(event?.attributes?.["acs.reason_codes"]).toEqual(["session_closed"]); + expect({ method, origin: span.attributes["acs.provenance.origin"] }).toEqual({ method, origin: ["user_input", "retrieval"] }); + expect({ method, sourceId: span.attributes["acs.provenance.source_id"] }).toEqual({ method, sourceId: ["user-1", ""] }); + } }); - it("records a guardian_error failure on a message span answered with a JSON-RPC error", async () => { - const { exporter, spans } = sessionAndMessageExporter(); - exporter.end( - exporter.start(makeEnvelope("steps/userMessage", { content: [{ type: "text", value: "hi" }] }, { id: 4 }), "steps/userMessage"), - { jsonrpc: "2.0", id: 4, error: { code: -32020, message: "evaluation failed" } }, + it("carries the session Intent's parser_provenance on the session span", async () => { + const span = await onlySpan( + "steps/sessionStart", + stepEnvelope("steps/sessionStart", { + intent: { raw: "summarize the report", parser_provenance: provenance("system", { source_id: "intent-parser" }) }, + }), ); - await exporter.flush(); - const [span] = spans(); - expect(span?.status.code).toBe(SpanStatusCode.ERROR); - expect(span?.events.map((event) => event.name)).toEqual(["acs.decision.failure"]); - expect(span?.events[0]?.attributes?.["acs.failure.class"]).toBe("guardian_error"); + expect(span.attributes["acs.provenance.origin"]).toEqual(["system"]); + expect(span.attributes["acs.provenance.source_id"]).toEqual(["intent-parser"]); + }); + + it("carries no provenance attribute on a session or message span whose payload attaches none", async () => { + for (const [method, payload] of [ + ["steps/sessionStart", { intent: { raw: "summarize the report" } }], + ["steps/userMessage", { content: [{ type: "text", value: "hi" }] }], + ] as const) { + const span = await onlySpan(method, stepEnvelope(method, payload)); + + expect({ method, keys: Object.keys(span.attributes).filter((key) => key.startsWith("acs.provenance.")) }).toEqual({ method, keys: [] }); + } }); + + for (const method of SESSION_AND_MESSAGE_METHODS) { + it(`records the decision on a ${method} span the same way it does on a tool call span`, async () => { + const span = await onlySpan( + method, + stepEnvelope(method, { reason: "completed", content: [{ type: "text", value: "hi" }] }), + decisionResponse(1, { decision: "allow", reason_codes: ["session_closed"] }), + ); + + expect(span.events.map((event) => event.name)).toEqual(["acs.decision"]); + expect(span.events[0]?.attributes?.["acs.decision"]).toBe("allow"); + expect(span.events[0]?.attributes?.["acs.reason_codes"]).toEqual(["session_closed"]); + }); + + it(`records a guardian_error failure on a ${method} span answered with a JSON-RPC error`, async () => { + const span = await onlySpan( + method, + makeEnvelope(method, { reason: "completed", content: [{ type: "text", value: "hi" }] }, { id: 4 }), + { jsonrpc: "2.0", id: 4, error: { code: -32020, message: "evaluation failed" } }, + ); + + expect(span.status.code).toBe(SpanStatusCode.ERROR); + expect(span.events.map((event) => event.name)).toEqual(["acs.decision.failure"]); + const failure = span.events[0]?.attributes; + expect(failure?.["acs.failure.class"]).toBe("guardian_error"); + expect(failure?.["acs.failure.reported_by"]).toBe("guardian"); + expect(failure?.["acs.failure.posture"]).toBe("fail_open"); + }); + } }); describe("Guardian trace export wiring", () => { From 30105e48f0c163a77be819663c91f654e2ec42f0 Mon Sep 17 00:00:00 2001 From: Eli Sherman Date: Thu, 29 Oct 2026 16:19:15 -0400 Subject: [PATCH 3/4] Show the session and message spans in the sample trace and the READMEs #322 asks for the sample trace and the trace walkthrough to show the new spans. The sample's exporter now also spans steps/sessionStart, steps/userMessage, steps/agentResponse and steps/sessionEnd, through methodsEvaluated as the tests do, and the generator sends one request for each. This Guardian does not dispatch those steps yet, so each span carries a guardian_error failure rather than a decision; its name and attributes, message provenance included, are real exporter output. A sample test pins the four spans and the attributes their requests carried. The tree README says why "Produce a trace" shows no session or message span in a live run and points to the sample, and the Guardian README and the OpenTelemetry page count the sample's spans again. Part of #322 Co-Authored-By: Claude Opus 5.5 Signed-off-by: Eli Sherman --- docs/spec/trace/extend_opentelemetry.md | 2 +- reference-implementations/agt/README.md | 4 +- .../agt/packages/guardian/README.md | 4 +- .../agt/samples/trace/acs-trace-sample.jsonl | 4 +- .../agt/scripts/make-sample-trace.ts | 48 +++++++++++++++++-- .../agt/test/sample-trace.test.ts | 26 ++++++++++ 6 files changed, 78 insertions(+), 10 deletions(-) diff --git a/docs/spec/trace/extend_opentelemetry.md b/docs/spec/trace/extend_opentelemetry.md index dfb27c04..8e9cb0d6 100644 --- a/docs/spec/trace/extend_opentelemetry.md +++ b/docs/spec/trace/extend_opentelemetry.md @@ -238,7 +238,7 @@ The collector path, set with `ACS_OTEL_ENDPOINT`, sends the same spans over OTLP The two Guardian outputs fail independently. The first failed write or send disables that output for the rest of the process and prints one line on stderr. The other output keeps running, and the decision goes out unchanged, as the [failure isolation](#failure-isolation) rule below requires. The [Guardian README](https://github.com/GenAI-Security-Project/agent-control-standard/blob/main/reference-implementations/agt/packages/guardian/README.md#troubleshooting) lists the messages you may see. -The committed [sample trace](https://github.com/GenAI-Security-Project/agent-control-standard/blob/main/reference-implementations/agt/samples/trace/acs-trace-sample.jsonl) holds both streams in one file: five spans from the Guardian and one from the converter. Every value in it is synthetic. +The committed [sample trace](https://github.com/GenAI-Security-Project/agent-control-standard/blob/main/reference-implementations/agt/samples/trace/acs-trace-sample.jsonl) holds both streams in one file: nine spans from the Guardian and one from the converter. Every value in it is synthetic. ### What the labels don't prove diff --git a/reference-implementations/agt/README.md b/reference-implementations/agt/README.md index c50cd580..41acceed 100644 --- a/reference-implementations/agt/README.md +++ b/reference-implementations/agt/README.md @@ -262,6 +262,8 @@ To read the collector's file in step 7, run the same command with `.acs/otel-col You should see two `gen_ai.tool.call` spans, one `allow` and one `deny`, and one `gen_ai.tool.result` span with `allow`. The spans carry the attributes [`trace/otel-mapping.json`](../../specification/v0.1.0/trace/otel-mapping.json) lists, and the [Guardian README](packages/guardian/README.md#export-traces) says which ones appear when. +You won't see the session and message spans (`acs.session`, `acs.message.user`, `acs.message.agent`, `acs.session.end`) in this run. The exporter can write them, but this Guardian doesn't evaluate those steps yet, so it doesn't span them, and the Claude Code shim doesn't send them. The [committed sample](samples/trace/acs-trace-sample.jsonl) shows all four. + **6. Capture the failures only the host sees.** With the Guardian stopped, send one more step. The shim can't reach the Guardian, so it proceeds under the posture the Guardian declared when the session started, which is `proceed` for the stock Guardian, and records that in `.acs/audit.jsonl`. The converter turns that record into a span in the same format. ```bash @@ -312,7 +314,7 @@ Verified on macOS (arm64): natively with otelcol-contrib 0.162.0, and with the D **Run it again.** Every output here except the converter's appends, so a second run on the same `.acs` doubles every count, and the converter refuses to run at all. Stop the Guardian and the collector first, with Ctrl-C or `docker stop acs-otelcol`. A process that is still running keeps writing into the deleted file, and its spans are lost without an error. Then delete it: `rm -rf .acs && mkdir -p .acs && chmod 700 .acs`. -**What this is, and isn't.** The exporter covers the two hook methods this Guardian evaluates, `steps/toolCallRequest` and `steps/toolCallResult`. It isn't an ACS-Trace conformance claim. Traces driven through the shim also lack `acs.capability`, because the shim doesn't send it yet ([issue #274](https://github.com/GenAI-Security-Project/agent-control-standard/issues/274)). They carry `acs.request_id_ref` on a result when the host supplied its call id. The committed sample drives the Guardian directly, so it carries `acs.capability`, and `acs.request_id_ref` on its results. Spans also lack `acs.evaluator` until the Guardian sends `metadata.evaluator` (PR #229). +**What this is, and isn't.** In a live run, the exporter covers the two hook methods this Guardian evaluates, `steps/toolCallRequest` and `steps/toolCallResult`. It can also write spans for `steps/sessionStart`, `steps/userMessage`, `steps/agentResponse` and `steps/sessionEnd`, which the committed sample shows, but a live run has none until the Guardian handles those steps ([issue #322](https://github.com/GenAI-Security-Project/agent-control-standard/issues/322)). It isn't an ACS-Trace conformance claim. Traces driven through the shim also lack `acs.capability`, because the shim doesn't send it yet ([issue #274](https://github.com/GenAI-Security-Project/agent-control-standard/issues/274)). They carry `acs.request_id_ref` on a result when the host supplied its call id. The committed sample drives the Guardian directly, so it carries `acs.capability`, and `acs.request_id_ref` on its results. Spans also lack `acs.evaluator` until the Guardian sends `metadata.evaluator` (PR #229). The trace files, the audit log and the collector's port can be written by any process running as you, the observed agent included, and in a container by other containers on the same Docker network, so the output isn't tamper-evident. `service.name` (`acs-reference-guardian`, `acs-reference-audit-converter`) is a label each writer sets for itself, not authentication, and loopback isn't authentication either. These labels belong to the reference implementation, not to the ACS specification. diff --git a/reference-implementations/agt/packages/guardian/README.md b/reference-implementations/agt/packages/guardian/README.md index 7ae1cd96..80c2ebed 100644 --- a/reference-implementations/agt/packages/guardian/README.md +++ b/reference-implementations/agt/packages/guardian/README.md @@ -118,13 +118,13 @@ It prints how many spans reached the file and exits non-zero when that is fewer ### Sample trace -[`samples/trace/acs-trace-sample.jsonl`](../../samples/trace/acs-trace-sample.jsonl) is what this export looks like, so you can read it without running the Guardian. It holds six steps: an allowed tool call and its result, linked by `acs.request_id_ref`, a denied call carrying reasoning, reason codes, policy references and provenance, a call the Guardian answered with an error, a decision that finished after the timeout, and a step the host could not deliver, converted from its audit log. The decisions come from the stock policy, and the one late decision is delayed on purpose. Every value in it is synthetic. +[`samples/trace/acs-trace-sample.jsonl`](../../samples/trace/acs-trace-sample.jsonl) is what this export looks like, so you can read it without running the Guardian. It holds six tool steps: an allowed tool call and its result, linked by `acs.request_id_ref`, a denied call carrying reasoning, reason codes, policy references and provenance, a call the Guardian answered with an error, a decision that finished after the timeout, and a step the host could not deliver, converted from its audit log. The decisions come from the stock policy, and the one late decision is delayed on purpose. It also holds one span for each session and message step: `acs.session`, `acs.message.user` (with content types and provenance), `acs.message.agent` and `acs.session.end`. This Guardian does not dispatch those steps yet, so the script hands them to its exporter through `methodsEvaluated`, and each one carries a `guardian_error` failure rather than a decision. Every value in it is synthetic. ```bash bun run sample:trace ``` -That command regenerates it. `test/sample-trace.test.ts` checks the sample against `otel-mapping.json` and fails when the exporter's output no longer matches it, so a change to the exporter or the mapping comes with a regenerated sample. The converted span has no `gen_ai.tool.name`, because the audit log does not record the tool (issue #274). Like the exporter, the sample covers two hook methods and is not an ACS-Trace conformance claim. +That command regenerates it. `test/sample-trace.test.ts` checks the sample against `otel-mapping.json` and fails when the exporter's output no longer matches it, so a change to the exporter or the mapping comes with a regenerated sample. The converted span has no `gen_ai.tool.name`, because the audit log does not record the tool (issue #274). The sample is not an ACS-Trace conformance claim. ### Troubleshooting diff --git a/reference-implementations/agt/samples/trace/acs-trace-sample.jsonl b/reference-implementations/agt/samples/trace/acs-trace-sample.jsonl index 437a8f22..1fc2a211 100644 --- a/reference-implementations/agt/samples/trace/acs-trace-sample.jsonl +++ b/reference-implementations/agt/samples/trace/acs-trace-sample.jsonl @@ -1,2 +1,2 @@ -{"resourceSpans":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"acs-reference-guardian"}}],"droppedAttributesCount":0},"scopeSpans":[{"scope":{"name":"acs-reference-guardian"},"spans":[{"traceId":"9785c34288d27dbd92fcf12e53ac3315","spanId":"31258fb4b343b292","name":"gen_ai.tool.call","kind":1,"startTimeUnixNano":"1792065946937000000","endTimeUnixNano":"1792065947036723875","attributes":[{"key":"acs.session.id","value":{"stringValue":"5a3c1e7e-0b1d-4c6a-9e2f-7d4b8c0a1f01"}},{"key":"acs.request_id","value":{"stringValue":"1b6f0c2a-3d4e-4f5a-8b6c-7d8e9f0a1b01"}},{"key":"gen_ai.tool.name","value":{"stringValue":"Bash"}},{"key":"acs.capability","value":{"stringValue":"process.exec"}}],"droppedAttributesCount":0,"events":[{"attributes":[{"key":"acs.decision","value":{"stringValue":"allow"}}],"name":"acs.decision","timeUnixNano":"1792065947036676541","droppedAttributesCount":0}],"droppedEventsCount":0,"status":{"code":0},"links":[],"droppedLinksCount":0,"flags":257},{"traceId":"112037a108c3893225309fa193536b77","spanId":"46883c95863bb8fb","name":"gen_ai.tool.result","kind":1,"startTimeUnixNano":"1792065947038000000","endTimeUnixNano":"1792065947088187667","attributes":[{"key":"acs.session.id","value":{"stringValue":"5a3c1e7e-0b1d-4c6a-9e2f-7d4b8c0a1f01"}},{"key":"acs.request_id","value":{"stringValue":"1b6f0c2a-3d4e-4f5a-8b6c-7d8e9f0a1b02"}},{"key":"gen_ai.tool.name","value":{"stringValue":"Bash"}},{"key":"acs.exit_status","value":{"stringValue":"success"}},{"key":"acs.request_id_ref","value":{"stringValue":"1b6f0c2a-3d4e-4f5a-8b6c-7d8e9f0a1b01"}}],"droppedAttributesCount":0,"events":[{"attributes":[{"key":"acs.decision","value":{"stringValue":"allow"}}],"name":"acs.decision","timeUnixNano":"1792065947088182292","droppedAttributesCount":0}],"droppedEventsCount":0,"status":{"code":0},"links":[],"droppedLinksCount":0,"flags":257},{"traceId":"981a929f7cea4c999642a38dd3d822b1","spanId":"3b199edb6364b94d","name":"gen_ai.tool.call","kind":1,"startTimeUnixNano":"1792065947089000000","endTimeUnixNano":"1792065947141424750","attributes":[{"key":"acs.session.id","value":{"stringValue":"5a3c1e7e-0b1d-4c6a-9e2f-7d4b8c0a1f01"}},{"key":"acs.request_id","value":{"stringValue":"1b6f0c2a-3d4e-4f5a-8b6c-7d8e9f0a1b03"}},{"key":"gen_ai.tool.name","value":{"stringValue":"Bash"}},{"key":"acs.provenance.origin","value":{"arrayValue":{"values":[{"stringValue":"user_input"},{"stringValue":"agent_generated"}]}}},{"key":"acs.provenance.source_id","value":{"arrayValue":{"values":[{"stringValue":"sample-user-1"},{"stringValue":""}]}}}],"droppedAttributesCount":0,"events":[{"attributes":[{"key":"acs.decision","value":{"stringValue":"deny"}},{"key":"acs.reasoning","value":{"stringValue":"This command was blocked because it matches a destructive-shell-command pattern. Policy: destructive_shell_command_blocked, from AGT's stock bundle (agt_stock). Matched at offset 0."}},{"key":"acs.reason_codes","value":{"arrayValue":{"values":[{"stringValue":"destructive_shell_command_blocked"}]}}},{"key":"acs.policy.ids","value":{"arrayValue":{"values":[{"stringValue":"agt_stock"}]}}},{"key":"acs.policy.versions","value":{"arrayValue":{"values":[{"stringValue":""}]}}},{"key":"acs.policy.rule_ids","value":{"arrayValue":{"values":[{"stringValue":"destructive_shell_command_blocked"}]}}}],"name":"acs.decision","timeUnixNano":"1792065947141419958","droppedAttributesCount":0}],"droppedEventsCount":0,"status":{"code":0},"links":[],"droppedLinksCount":0,"flags":257},{"traceId":"95c92494c004b23662900ae8c17ea04c","spanId":"02d46f81d8206a0f","name":"gen_ai.tool.call","kind":1,"startTimeUnixNano":"1792065947141000000","endTimeUnixNano":"1792065947141270000","attributes":[{"key":"acs.session.id","value":{"stringValue":"5a3c1e7e-0b1d-4c6a-9e2f-7d4b8c0a1f01"}},{"key":"acs.request_id","value":{"stringValue":"1b6f0c2a-3d4e-4f5a-8b6c-7d8e9f0a1b04"}},{"key":"gen_ai.tool.name","value":{"stringValue":"Bash"}}],"droppedAttributesCount":0,"events":[{"attributes":[{"key":"acs.failure.class","value":{"stringValue":"guardian_error"}},{"key":"acs.failure.posture","value":{"stringValue":"fail_open"}},{"key":"acs.failure.reported_by","value":{"stringValue":"guardian"}},{"key":"acs.failure.error_code","value":{"intValue":-32010}},{"key":"acs.failure.elapsed_ms","value":{"intValue":0}},{"key":"acs.failure.payload_bytes","value":{"intValue":340}}],"name":"acs.decision.failure","timeUnixNano":"1792065947141268125","droppedAttributesCount":0}],"droppedEventsCount":0,"status":{"code":2,"message":"The Guardian returned an error instead of a decision."},"links":[],"droppedLinksCount":0,"flags":257},{"traceId":"e5703a6e87d75324de28fd7cc555724b","spanId":"e813670f52f6a811","name":"gen_ai.tool.call","kind":1,"startTimeUnixNano":"1792065947142000000","endTimeUnixNano":"1792065949203024291","attributes":[{"key":"acs.session.id","value":{"stringValue":"5a3c1e7e-0b1d-4c6a-9e2f-7d4b8c0a1f01"}},{"key":"acs.request_id","value":{"stringValue":"1b6f0c2a-3d4e-4f5a-8b6c-7d8e9f0a1b05"}},{"key":"gen_ai.tool.name","value":{"stringValue":"Bash"}}],"droppedAttributesCount":0,"events":[{"attributes":[{"key":"acs.decision","value":{"stringValue":"allow"}}],"name":"acs.decision","timeUnixNano":"1792065949202988875","droppedAttributesCount":0},{"attributes":[{"key":"acs.failure.class","value":{"stringValue":"timeout"}},{"key":"acs.failure.posture","value":{"stringValue":"fail_open"}},{"key":"acs.failure.reported_by","value":{"stringValue":"guardian"}},{"key":"acs.failure.timeout_ms","value":{"intValue":1500}},{"key":"acs.failure.elapsed_ms","value":{"intValue":2061}},{"key":"acs.failure.payload_bytes","value":{"intValue":367}}],"name":"acs.decision.failure","timeUnixNano":"1792065949203022666","droppedAttributesCount":0}],"droppedEventsCount":0,"status":{"code":0},"links":[],"droppedLinksCount":0,"flags":257}]}]}]} -{"resourceSpans":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"acs-reference-audit-converter"}}],"droppedAttributesCount":0},"scopeSpans":[{"scope":{"name":"acs-reference-audit-converter"},"spans":[{"traceId":"9b082079ac3695104b2da1df27340e16","spanId":"99e8d283763d16d3","name":"gen_ai.tool.call","kind":1,"startTimeUnixNano":"1792065949205000000","endTimeUnixNano":"1792065949205000000","attributes":[{"key":"acs.session.id","value":{"stringValue":"5a3c1e7e-0b1d-4c6a-9e2f-7d4b8c0a1f01"}},{"key":"acs.request_id","value":{"stringValue":"1b6f0c2a-3d4e-4f5a-8b6c-7d8e9f0a1b06"}}],"droppedAttributesCount":0,"events":[{"attributes":[{"key":"acs.failure.class","value":{"stringValue":"unreachable"}},{"key":"acs.failure.posture","value":{"stringValue":"fail_open"}},{"key":"acs.failure.reported_by","value":{"stringValue":"observed_agent"}},{"key":"acs.failure.detail","value":{"stringValue":"The host could not reach the Guardian."}}],"name":"acs.decision.failure","timeUnixNano":"1792065949205000000","droppedAttributesCount":0}],"droppedEventsCount":0,"status":{"code":2,"message":"The host could not reach the Guardian."},"links":[],"droppedLinksCount":0,"flags":257}]}]}]} +{"resourceSpans":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"acs-reference-guardian"}}],"droppedAttributesCount":0},"scopeSpans":[{"scope":{"name":"acs-reference-guardian"},"spans":[{"traceId":"5ef9fd86c6c8dd57296b5ee582ed4a67","spanId":"7093eb8a16bdfb04","name":"acs.session","kind":1,"startTimeUnixNano":"1793305019002000000","endTimeUnixNano":"1793305019047317166","attributes":[{"key":"acs.session.id","value":{"stringValue":"5a3c1e7e-0b1d-4c6a-9e2f-7d4b8c0a1f01"}},{"key":"acs.request_id","value":{"stringValue":"1b6f0c2a-3d4e-4f5a-8b6c-7d8e9f0a1b07"}},{"key":"acs.tenant_id","value":{"stringValue":"sample-tenant"}},{"key":"acs.user.id","value":{"stringValue":"sample-user-1"}},{"key":"acs.policy_mode","value":{"stringValue":"moderate"}}],"droppedAttributesCount":0,"events":[{"attributes":[{"key":"acs.failure.class","value":{"stringValue":"guardian_error"}},{"key":"acs.failure.posture","value":{"stringValue":"fail_open"}},{"key":"acs.failure.reported_by","value":{"stringValue":"guardian"}},{"key":"acs.failure.error_code","value":{"intValue":-32011}},{"key":"acs.failure.elapsed_ms","value":{"intValue":45}},{"key":"acs.failure.payload_bytes","value":{"intValue":378}}],"name":"acs.decision.failure","timeUnixNano":"1793305019047291291","droppedAttributesCount":0}],"droppedEventsCount":0,"status":{"code":2,"message":"The Guardian returned an error instead of a decision."},"links":[],"droppedLinksCount":0,"flags":257},{"traceId":"714011a5cf60d26b2cc56ca5a915f226","spanId":"ef966490196e9315","name":"acs.message.user","kind":1,"startTimeUnixNano":"1793305019050000000","endTimeUnixNano":"1793305019050082291","attributes":[{"key":"acs.session.id","value":{"stringValue":"5a3c1e7e-0b1d-4c6a-9e2f-7d4b8c0a1f01"}},{"key":"acs.request_id","value":{"stringValue":"1b6f0c2a-3d4e-4f5a-8b6c-7d8e9f0a1b08"}},{"key":"acs.content.types","value":{"arrayValue":{"values":[{"stringValue":"text"},{"stringValue":"file"}]}}},{"key":"acs.provenance.origin","value":{"arrayValue":{"values":[{"stringValue":"user_input"},{"stringValue":"user_input"}]}}},{"key":"acs.provenance.source_id","value":{"arrayValue":{"values":[{"stringValue":"sample-user-1"},{"stringValue":""}]}}}],"droppedAttributesCount":0,"events":[{"attributes":[{"key":"acs.failure.class","value":{"stringValue":"guardian_error"}},{"key":"acs.failure.posture","value":{"stringValue":"fail_open"}},{"key":"acs.failure.reported_by","value":{"stringValue":"guardian"}},{"key":"acs.failure.error_code","value":{"intValue":-32011}},{"key":"acs.failure.elapsed_ms","value":{"intValue":0}},{"key":"acs.failure.payload_bytes","value":{"intValue":544}}],"name":"acs.decision.failure","timeUnixNano":"1793305019050080500","droppedAttributesCount":0}],"droppedEventsCount":0,"status":{"code":2,"message":"The Guardian returned an error instead of a decision."},"links":[],"droppedLinksCount":0,"flags":257},{"traceId":"805ab4b2fab8a1be55842b5ef8c70a63","spanId":"ab5ed5c7f2ecbf24","name":"gen_ai.tool.call","kind":1,"startTimeUnixNano":"1793305019050000000","endTimeUnixNano":"1793305019184660459","attributes":[{"key":"acs.session.id","value":{"stringValue":"5a3c1e7e-0b1d-4c6a-9e2f-7d4b8c0a1f01"}},{"key":"acs.request_id","value":{"stringValue":"1b6f0c2a-3d4e-4f5a-8b6c-7d8e9f0a1b01"}},{"key":"gen_ai.tool.name","value":{"stringValue":"Bash"}},{"key":"acs.capability","value":{"stringValue":"process.exec"}}],"droppedAttributesCount":0,"events":[{"attributes":[{"key":"acs.decision","value":{"stringValue":"allow"}}],"name":"acs.decision","timeUnixNano":"1793305019184656459","droppedAttributesCount":0}],"droppedEventsCount":0,"status":{"code":0},"links":[],"droppedLinksCount":0,"flags":257},{"traceId":"17a4515998d775f4f1cadde3637c0e9b","spanId":"f3a2be84cd9c239e","name":"gen_ai.tool.result","kind":1,"startTimeUnixNano":"1793305019185000000","endTimeUnixNano":"1793305019224180500","attributes":[{"key":"acs.session.id","value":{"stringValue":"5a3c1e7e-0b1d-4c6a-9e2f-7d4b8c0a1f01"}},{"key":"acs.request_id","value":{"stringValue":"1b6f0c2a-3d4e-4f5a-8b6c-7d8e9f0a1b02"}},{"key":"gen_ai.tool.name","value":{"stringValue":"Bash"}},{"key":"acs.exit_status","value":{"stringValue":"success"}},{"key":"acs.request_id_ref","value":{"stringValue":"1b6f0c2a-3d4e-4f5a-8b6c-7d8e9f0a1b01"}}],"droppedAttributesCount":0,"events":[{"attributes":[{"key":"acs.decision","value":{"stringValue":"allow"}}],"name":"acs.decision","timeUnixNano":"1793305019224174542","droppedAttributesCount":0}],"droppedEventsCount":0,"status":{"code":0},"links":[],"droppedLinksCount":0,"flags":257},{"traceId":"23ca78ebab2d5978ba341358cc4e4cbb","spanId":"8149ae1ad14576d2","name":"gen_ai.tool.call","kind":1,"startTimeUnixNano":"1793305019225000000","endTimeUnixNano":"1793305019269762917","attributes":[{"key":"acs.session.id","value":{"stringValue":"5a3c1e7e-0b1d-4c6a-9e2f-7d4b8c0a1f01"}},{"key":"acs.request_id","value":{"stringValue":"1b6f0c2a-3d4e-4f5a-8b6c-7d8e9f0a1b03"}},{"key":"gen_ai.tool.name","value":{"stringValue":"Bash"}},{"key":"acs.provenance.origin","value":{"arrayValue":{"values":[{"stringValue":"user_input"},{"stringValue":"agent_generated"}]}}},{"key":"acs.provenance.source_id","value":{"arrayValue":{"values":[{"stringValue":"sample-user-1"},{"stringValue":""}]}}}],"droppedAttributesCount":0,"events":[{"attributes":[{"key":"acs.decision","value":{"stringValue":"deny"}},{"key":"acs.reasoning","value":{"stringValue":"This command was blocked because it matches a destructive-shell-command pattern. Policy: destructive_shell_command_blocked, from AGT's stock bundle (agt_stock). Matched at offset 0."}},{"key":"acs.reason_codes","value":{"arrayValue":{"values":[{"stringValue":"destructive_shell_command_blocked"}]}}},{"key":"acs.policy.ids","value":{"arrayValue":{"values":[{"stringValue":"agt_stock"}]}}},{"key":"acs.policy.versions","value":{"arrayValue":{"values":[{"stringValue":""}]}}},{"key":"acs.policy.rule_ids","value":{"arrayValue":{"values":[{"stringValue":"destructive_shell_command_blocked"}]}}}],"name":"acs.decision","timeUnixNano":"1793305019269756292","droppedAttributesCount":0}],"droppedEventsCount":0,"status":{"code":0},"links":[],"droppedLinksCount":0,"flags":257},{"traceId":"6927cd5bce85611c153ea86ed33c22b1","spanId":"c20ec42c1318e6cc","name":"gen_ai.tool.call","kind":1,"startTimeUnixNano":"1793305019270000000","endTimeUnixNano":"1793305019270503291","attributes":[{"key":"acs.session.id","value":{"stringValue":"5a3c1e7e-0b1d-4c6a-9e2f-7d4b8c0a1f01"}},{"key":"acs.request_id","value":{"stringValue":"1b6f0c2a-3d4e-4f5a-8b6c-7d8e9f0a1b04"}},{"key":"gen_ai.tool.name","value":{"stringValue":"Bash"}}],"droppedAttributesCount":0,"events":[{"attributes":[{"key":"acs.failure.class","value":{"stringValue":"guardian_error"}},{"key":"acs.failure.posture","value":{"stringValue":"fail_open"}},{"key":"acs.failure.reported_by","value":{"stringValue":"guardian"}},{"key":"acs.failure.error_code","value":{"intValue":-32010}},{"key":"acs.failure.elapsed_ms","value":{"intValue":0}},{"key":"acs.failure.payload_bytes","value":{"intValue":340}}],"name":"acs.decision.failure","timeUnixNano":"1793305019270501541","droppedAttributesCount":0}],"droppedEventsCount":0,"status":{"code":2,"message":"The Guardian returned an error instead of a decision."},"links":[],"droppedLinksCount":0,"flags":257},{"traceId":"fae5d60b305e654a2bc4f2a16141bcdb","spanId":"0758b853ccd279ff","name":"gen_ai.tool.call","kind":1,"startTimeUnixNano":"1793305019270000000","endTimeUnixNano":"1793305021325893334","attributes":[{"key":"acs.session.id","value":{"stringValue":"5a3c1e7e-0b1d-4c6a-9e2f-7d4b8c0a1f01"}},{"key":"acs.request_id","value":{"stringValue":"1b6f0c2a-3d4e-4f5a-8b6c-7d8e9f0a1b05"}},{"key":"gen_ai.tool.name","value":{"stringValue":"Bash"}}],"droppedAttributesCount":0,"events":[{"attributes":[{"key":"acs.decision","value":{"stringValue":"allow"}}],"name":"acs.decision","timeUnixNano":"1793305021325861000","droppedAttributesCount":0},{"attributes":[{"key":"acs.failure.class","value":{"stringValue":"timeout"}},{"key":"acs.failure.posture","value":{"stringValue":"fail_open"}},{"key":"acs.failure.reported_by","value":{"stringValue":"guardian"}},{"key":"acs.failure.timeout_ms","value":{"intValue":1500}},{"key":"acs.failure.elapsed_ms","value":{"intValue":2056}},{"key":"acs.failure.payload_bytes","value":{"intValue":367}}],"name":"acs.decision.failure","timeUnixNano":"1793305021325891709","droppedAttributesCount":0}],"droppedEventsCount":0,"status":{"code":0},"links":[],"droppedLinksCount":0,"flags":257},{"traceId":"d4198e273ccb1d935b8586d3c433271d","spanId":"e59613b7d791257d","name":"acs.message.agent","kind":1,"startTimeUnixNano":"1793305021327000000","endTimeUnixNano":"1793305021327054167","attributes":[{"key":"acs.session.id","value":{"stringValue":"5a3c1e7e-0b1d-4c6a-9e2f-7d4b8c0a1f01"}},{"key":"acs.request_id","value":{"stringValue":"1b6f0c2a-3d4e-4f5a-8b6c-7d8e9f0a1b09"}},{"key":"acs.agent.id","value":{"stringValue":"sample-agent"}}],"droppedAttributesCount":0,"events":[{"attributes":[{"key":"acs.failure.class","value":{"stringValue":"guardian_error"}},{"key":"acs.failure.posture","value":{"stringValue":"fail_open"}},{"key":"acs.failure.reported_by","value":{"stringValue":"guardian"}},{"key":"acs.failure.error_code","value":{"intValue":-32011}},{"key":"acs.failure.elapsed_ms","value":{"intValue":0}},{"key":"acs.failure.payload_bytes","value":{"intValue":361}}],"name":"acs.decision.failure","timeUnixNano":"1793305021327052625","droppedAttributesCount":0}],"droppedEventsCount":0,"status":{"code":2,"message":"The Guardian returned an error instead of a decision."},"links":[],"droppedLinksCount":0,"flags":257},{"traceId":"643c2bd5895dcc0438069e18b009fb65","spanId":"d03ec28676892938","name":"acs.session.end","kind":1,"startTimeUnixNano":"1793305021327000000","endTimeUnixNano":"1793305021327027708","attributes":[{"key":"acs.session.id","value":{"stringValue":"5a3c1e7e-0b1d-4c6a-9e2f-7d4b8c0a1f01"}},{"key":"acs.request_id","value":{"stringValue":"1b6f0c2a-3d4e-4f5a-8b6c-7d8e9f0a1b0a"}},{"key":"acs.session.reason","value":{"stringValue":"completed"}}],"droppedAttributesCount":0,"events":[{"attributes":[{"key":"acs.failure.class","value":{"stringValue":"guardian_error"}},{"key":"acs.failure.posture","value":{"stringValue":"fail_open"}},{"key":"acs.failure.reported_by","value":{"stringValue":"guardian"}},{"key":"acs.failure.error_code","value":{"intValue":-32011}},{"key":"acs.failure.elapsed_ms","value":{"intValue":0}},{"key":"acs.failure.payload_bytes","value":{"intValue":300}}],"name":"acs.decision.failure","timeUnixNano":"1793305021327026417","droppedAttributesCount":0}],"droppedEventsCount":0,"status":{"code":2,"message":"The Guardian returned an error instead of a decision."},"links":[],"droppedLinksCount":0,"flags":257}]}]}]} +{"resourceSpans":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"acs-reference-audit-converter"}}],"droppedAttributesCount":0},"scopeSpans":[{"scope":{"name":"acs-reference-audit-converter"},"spans":[{"traceId":"a736a7218a6fc9b77dfacc7911d837e5","spanId":"cce88178cd6d755c","name":"gen_ai.tool.call","kind":1,"startTimeUnixNano":"1793305021329000000","endTimeUnixNano":"1793305021329000000","attributes":[{"key":"acs.session.id","value":{"stringValue":"5a3c1e7e-0b1d-4c6a-9e2f-7d4b8c0a1f01"}},{"key":"acs.request_id","value":{"stringValue":"1b6f0c2a-3d4e-4f5a-8b6c-7d8e9f0a1b06"}}],"droppedAttributesCount":0,"events":[{"attributes":[{"key":"acs.failure.class","value":{"stringValue":"unreachable"}},{"key":"acs.failure.posture","value":{"stringValue":"fail_open"}},{"key":"acs.failure.reported_by","value":{"stringValue":"observed_agent"}},{"key":"acs.failure.detail","value":{"stringValue":"The host could not reach the Guardian."}}],"name":"acs.decision.failure","timeUnixNano":"1793305021329000000","droppedAttributesCount":0}],"droppedEventsCount":0,"status":{"code":2,"message":"The host could not reach the Guardian."},"links":[],"droppedLinksCount":0,"flags":257}]}]}]} diff --git a/reference-implementations/agt/scripts/make-sample-trace.ts b/reference-implementations/agt/scripts/make-sample-trace.ts index de4d5e4a..b6a1e52d 100644 --- a/reference-implementations/agt/scripts/make-sample-trace.ts +++ b/reference-implementations/agt/scripts/make-sample-trace.ts @@ -1,8 +1,11 @@ /** * Writes the committed sample trace (issue #266): what the reference * Guardian's OTLP export actually looks like, from a fixed set of synthetic - * requests through the two live hook methods, plus one host-side failure run - * through the audit converter. + * requests through the two live hook methods and the four session and + * message steps (#322), plus one host-side failure run through the audit + * converter. The Guardian does not dispatch the session and message steps + * yet, so this script hands them to its exporter through methodsEvaluated, + * and each of their spans carries a guardian_error failure, not a decision. * * bun run sample:trace * @@ -55,13 +58,29 @@ export const SCENARIOS: readonly Scenario[] = [ { name: "error", requestId: "1b6f0c2a-3d4e-4f5a-8b6c-7d8e9f0a1b04" }, { name: "late", requestId: "1b6f0c2a-3d4e-4f5a-8b6c-7d8e9f0a1b05", decision: "allow" }, { name: "unreachable", requestId: "1b6f0c2a-3d4e-4f5a-8b6c-7d8e9f0a1b06" }, + // The session and message steps (#322 slice 1). This Guardian does not + // dispatch them yet and answers each with an error, so none has a decision. + { name: "session-start", requestId: "1b6f0c2a-3d4e-4f5a-8b6c-7d8e9f0a1b07" }, + { name: "user-message", requestId: "1b6f0c2a-3d4e-4f5a-8b6c-7d8e9f0a1b08" }, + { name: "agent-response", requestId: "1b6f0c2a-3d4e-4f5a-8b6c-7d8e9f0a1b09" }, + { name: "session-end", requestId: "1b6f0c2a-3d4e-4f5a-8b6c-7d8e9f0a1b0a" }, ]; +/** The session and message steps, which the sample's exporter spans although + * this Guardian does not evaluate them yet, so the sample shows their spans. */ +const SESSION_AND_MESSAGE_METHODS = ["steps/sessionStart", "steps/userMessage", "steps/agentResponse", "steps/sessionEnd"]; + function requestIdOf(name: string): string { return SCENARIOS.find((scenario) => scenario.name === name)!.requestId; } -function envelope(method: string, requestId: string, payload: Record, id: number | null): Record { +function envelope( + method: string, + requestId: string, + payload: Record, + id: number | null, + extraParams: Record = {}, +): Record { return { jsonrpc: "2.0", method, @@ -72,6 +91,7 @@ function envelope(method: string, requestId: string, payload: Record = {}): Record[] { return [ + envelope( + "steps/sessionStart", + requestIdOf("session-start"), + { user_identity: { user_id: "sample-user-1" }, policy_mode: "moderate" }, + 6, + { tenant_id: "sample-tenant" }, + ), + envelope( + "steps/userMessage", + requestIdOf("user-message"), + { + content: [ + { type: "text", value: "Tidy up this directory.", provenance: { provenance_id: "prov-sample-3", origin: "user_input", source_id: "sample-user-1" } }, + { type: "file", value: "notes.txt", provenance: { provenance_id: "prov-sample-4", origin: "user_input" } }, + ], + }, + 7, + ), envelope("steps/toolCallRequest", requestIdOf("allowed"), bash("ls -la", { capability: "process.exec" }), 1), envelope( "steps/toolCallResult", @@ -117,6 +155,8 @@ function requests(): Record[] { // No JSON-RPC id: the Guardian answers with an error, not a decision. envelope("steps/toolCallRequest", requestIdOf("error"), bash("pwd"), null), envelope("steps/toolCallRequest", requestIdOf("late"), bash(`echo ${SLOW_MARKER}`), 5), + envelope("steps/agentResponse", requestIdOf("agent-response"), { content: [{ type: "text", value: "Listed the directory; did not delete it." }] }, 8), + envelope("steps/sessionEnd", requestIdOf("session-end"), { reason: "completed" }, 9), ]; } @@ -157,7 +197,7 @@ export async function generateSampleTrace(outPath: string = SAMPLE_PATH): Promis }; const traceExporter = createOtelTraceExporter({ filePath: guardianTrace, - methodsEvaluated: buildServerHello().methods_evaluated, + methodsEvaluated: [...buildServerHello().methods_evaluated, ...SESSION_AND_MESSAGE_METHODS], timeoutConfig: { default_ms: TIMEOUT_MS }, posture: "fail_open", }); diff --git a/reference-implementations/agt/test/sample-trace.test.ts b/reference-implementations/agt/test/sample-trace.test.ts index dbc86ab5..19ea5657 100644 --- a/reference-implementations/agt/test/sample-trace.test.ts +++ b/reference-implementations/agt/test/sample-trace.test.ts @@ -162,6 +162,32 @@ describe("the committed sample trace matches the mapping", () => { } }); + // The session and message steps (#322 slice 1). This Guardian does not + // dispatch them yet, so each carries a guardian_error failure rather than + // a decision, but its span and the attributes its request carried are real. + it("shows the session and message spans with the attributes their requests carried", () => { + const spanOf = (name: string) => { + const span = spanFor(SCENARIOS.find((scenario) => scenario.name === name)!.requestId); + return { name: span?.name, attributes: attributes(span) }; + }; + + const start = spanOf("session-start"); + expect(start.name).toBe("acs.session"); + expect(start.attributes).toMatchObject({ "acs.tenant_id": "sample-tenant", "acs.user.id": "sample-user-1", "acs.policy_mode": "moderate" }); + + const user = spanOf("user-message"); + expect(user.name).toBe("acs.message.user"); + expect(user.attributes).toMatchObject({ "acs.content.types": ["text", "file"], "acs.provenance.origin": ["user_input", "user_input"] }); + + const agent = spanOf("agent-response"); + expect(agent.name).toBe("acs.message.agent"); + expect(agent.attributes).toMatchObject({ "acs.agent.id": "sample-agent" }); + + const end = spanOf("session-end"); + expect(end.name).toBe("acs.session.end"); + expect(end.attributes).toMatchObject({ "acs.session.reason": "completed" }); + }); + it("links the result to its request through acs.request_id_ref", () => { const result = SCENARIOS.find((scenario) => scenario.requestIdRef !== undefined)!; expect(attributes(spanFor(result.requestId))["acs.request_id_ref"]).toBe(result.requestIdRef); From b744d4d2ec7153f701ef01ccd4a616e19123ab5d Mon Sep 17 00:00:00 2001 From: Eli Sherman Date: Wed, 4 Nov 2026 11:58:03 -0500 Subject: [PATCH 4/4] Use a provenance origin the schema allows in the message provenance test The test gave one content item `origin: "retrieval"`, which provenance.json does not allow (it is `retrieved`), so both message envelopes in that test failed their hook schemas. Found by validating every slice-1 test envelope against request-envelope.json and its hook schema, as the review asked. The other envelopes are valid, apart from the acs.content.types edge cases, which are invalid on purpose. Part of #322 Co-Authored-By: Claude Opus 5.5 Signed-off-by: Eli Sherman --- .../agt/packages/guardian/test/otel-trace-exporter.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reference-implementations/agt/packages/guardian/test/otel-trace-exporter.test.ts b/reference-implementations/agt/packages/guardian/test/otel-trace-exporter.test.ts index cb3828de..9f85ce33 100644 --- a/reference-implementations/agt/packages/guardian/test/otel-trace-exporter.test.ts +++ b/reference-implementations/agt/packages/guardian/test/otel-trace-exporter.test.ts @@ -763,12 +763,12 @@ describe("createOtelTraceExporter -- the session and message steps", () => { { type: "text", value: "a", provenance: provenance("user_input", { source_id: "user-1" }) }, // No provenance: contributes no element to any column. { type: "text", value: "b" }, - { type: "file", value: "c", provenance: provenance("retrieval") }, + { type: "file", value: "c", provenance: provenance("retrieved") }, ], }), ); - expect({ method, origin: span.attributes["acs.provenance.origin"] }).toEqual({ method, origin: ["user_input", "retrieval"] }); + expect({ method, origin: span.attributes["acs.provenance.origin"] }).toEqual({ method, origin: ["user_input", "retrieved"] }); expect({ method, sourceId: span.attributes["acs.provenance.source_id"] }).toEqual({ method, sourceId: ["user-1", ""] }); } });