From c94e771c9f08b3cdce5d0eb7cb95d6067d8e8ef0 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Thu, 29 Oct 2026 13:59:37 +0000 Subject: [PATCH 1/3] fix(prompt): Say that replies are not top-level channel posts #2315 merged the two Slack delivery rules into one line. The merged line tied the explanation to tool calls, so a plain reply that repeats the requested text looked allowed. The delivery integration eval for a top-level post request then failed in both live samples. State the fact that a reply is not a channel post, and say that the agent must not reply with only the requested text. Co-Authored-By: David Cramer --- packages/junior/src/chat/prompt.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/junior/src/chat/prompt.ts b/packages/junior/src/chat/prompt.ts index 261b5fec00..9f96846818 100644 --- a/packages/junior/src/chat/prompt.ts +++ b/packages/junior/src/chat/prompt.ts @@ -332,7 +332,7 @@ const CONVERSATION_RULES = [ ]; const SLACK_ACTION_RULES = [ - "- Assistant text, Slack posts, reactions, new canvases, new lists, and file sends go only to the current conversation or thread. If asked to post elsewhere or create a new top-level channel post, explain the limitation instead of calling a tool, and do not present the text as delivered. Slack read tools can target other public channels with `channel_id`.", + "- Assistant text, Slack posts, reactions, new canvases, new lists, and file sends go only to the current conversation or thread. Your reply is not a channel post, and you cannot create a new top-level channel post. When asked to post to a channel or anywhere else, explain that limitation; do not call a tool, and do not reply with only the requested text or present it as delivered. Slack read tools can target other public channels with `channel_id`.", "- Ambient reaction requests target the current inbound message; do not ask for a message reference.", `- When no visible thread reply is requested or useful, keep tool-calling messages text-free and make the final message exactly ${NO_REPLY_MARKER}.`, ]; From d9ecc3ae0595f7b012040ebe6aedf3df5fa1584b Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Thu, 29 Oct 2026 14:10:30 +0000 Subject: [PATCH 2/3] test(prompt): Remove prompt string unit tests These tests only checked that the prompt builder output contains static strings. Evals own prompt behavior. Co-Authored-By: David Cramer --- packages/junior/tests/unit/prompt.test.ts | 163 ---------------------- 1 file changed, 163 deletions(-) delete mode 100644 packages/junior/tests/unit/prompt.test.ts diff --git a/packages/junior/tests/unit/prompt.test.ts b/packages/junior/tests/unit/prompt.test.ts deleted file mode 100644 index 005742414a..0000000000 --- a/packages/junior/tests/unit/prompt.test.ts +++ /dev/null @@ -1,163 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { createSlackSource } from "@sentry/junior-plugin-api"; -import { buildTurnContextPrompt } from "@/chat/prompt"; - -describe("prompt builders", () => { - it("renders sandbox workspace root as runtime context", () => { - const prompt = buildTurnContextPrompt({ - availableSkills: [], - activeMcpCatalogs: [], - }); - - expect(prompt).toContain("- sandbox.workspace_root: /vercel/sandbox"); - }); - - it("lists disabled tools in follow-up Turn context", () => { - const prompt = buildTurnContextPrompt({ - availableSkills: [], - includeSessionContext: false, - disabledTools: [ - { name: "createEventAutomation", reason: "Needs a Slack message." }, - { name: "listEventAutomations", reason: "Needs a Slack message." }, - { name: "addReaction", reason: "No message to react to." }, - ], - }); - - expect(prompt).toContain(""); - expect(prompt).toContain( - "- `createEventAutomation`, `listEventAutomations`: Needs a Slack message.", - ); - expect(prompt).toContain("- `addReaction`: No message to react to."); - expect(prompt).not.toContain("sandbox.workspace_root"); - }); - - it("renders Slack conversation facts in runtime context", () => { - const prompt = buildTurnContextPrompt({ - availableSkills: [], - activeMcpCatalogs: [], - runtime: { - conversationId: "slack:C123:1712345.000001", - slackConversation: { - type: "private_channel", - name: "#roadmap & launches", - }, - }, - }); - - expect(prompt).toContain( - "- gen_ai.conversation.id: slack:C123:1712345.000001", - ); - expect(prompt).toContain("- slack.conversation.type: private_channel"); - expect(prompt).toContain( - "- slack.conversation.name: #roadmap & launches", - ); - expect(prompt).not.toContain("#roadmap & launches"); - }); - - it("renders generic dispatch facts in runtime context", () => { - const prompt = buildTurnContextPrompt({ - availableSkills: [], - activeMcpCatalogs: [], - dispatch: { - actor: { platform: "system", name: "scheduler" }, - plugin: "scheduler", - source: createSlackSource({ - teamId: "T123", - channelId: "C123", - - visibility: "private", - }), - destination: { - platform: "slack", - teamId: "T123", - channelId: "C123", - }, - metadata: { - scheduledFor: "2026-05-26T12:00:00.000Z", - taskId: "sched_plugin_1", - }, - }, - }); - - expect(prompt).toContain(""); - expect(prompt).toContain( - "- dispatch.execution: execute the dispatched input now", - ); - expect(prompt).toContain( - "- dispatch.delivery: the runtime delivers the final answer to the destination", - ); - expect(prompt).toContain("- dispatch.actor.platform: system"); - expect(prompt).toContain("- dispatch.actor.name: scheduler"); - expect(prompt).toContain("- source.kind: slack"); - expect(prompt).toContain("- destination.channel_id: C123"); - expect(prompt).toContain( - "- dispatch.metadata.scheduledFor: 2026-05-26T12:00:00.000Z", - ); - expect(prompt).toContain("- dispatch.metadata.taskId: sched_plugin_1"); - }); - - it("omits follow-up runtime context once session bootstrap exists", () => { - expect( - buildTurnContextPrompt({ - availableSkills: [ - { - name: "alpha", - description: "Alpha workflow", - skillPath: "/tmp/skills/alpha", - }, - ], - activeMcpCatalogs: [ - { provider: "alpha-provider", available_tool_count: 2 }, - ], - configuration: { - sentry_project: "junior", - }, - includeSessionContext: false, - actor: { - userId: "U0BETA", - userName: "dcramer", - }, - runtime: { - conversationId: "conversation-alpha", - }, - toolGuidance: [ - { - name: "editFile", - promptSnippet: "exact edits", - }, - ], - }), - ).toBeNull(); - }); - - it("renders plugin prompt contributions without replaying follow-up bootstrap context", () => { - const prompt = buildTurnContextPrompt({ - availableSkills: [ - { - name: "alpha", - description: "Alpha workflow", - skillPath: "/tmp/skills/alpha", - }, - ], - activeMcpCatalogs: [ - { provider: "alpha-provider", available_tool_count: 2 }, - ], - includeSessionContext: false, - pluginPromptContributions: [ - { - id: "memory", - pluginName: "memory", - text: "User prefers concise answers.", - }, - ], - runtime: { - conversationId: "conversation-alpha", - }, - }); - - expect(prompt).toContain("User prefers concise answers."); - expect(prompt).toContain('plugin="memory"'); - expect(prompt).not.toContain(""); - expect(prompt).not.toContain("conversation-alpha"); - }); -}); From 1ecd37919ccca99825003df178f16b21190d6ea6 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Thu, 29 Oct 2026 14:29:14 +0000 Subject: [PATCH 3/3] fix(prompt): Keep scheduled channel posts out of the top-level post rule The rule now covers only requests to post right now. Scheduled automations can still post to the active channel later. --- packages/junior/src/chat/prompt.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/junior/src/chat/prompt.ts b/packages/junior/src/chat/prompt.ts index 9f96846818..b427989ff6 100644 --- a/packages/junior/src/chat/prompt.ts +++ b/packages/junior/src/chat/prompt.ts @@ -332,7 +332,7 @@ const CONVERSATION_RULES = [ ]; const SLACK_ACTION_RULES = [ - "- Assistant text, Slack posts, reactions, new canvases, new lists, and file sends go only to the current conversation or thread. Your reply is not a channel post, and you cannot create a new top-level channel post. When asked to post to a channel or anywhere else, explain that limitation; do not call a tool, and do not reply with only the requested text or present it as delivered. Slack read tools can target other public channels with `channel_id`.", + "- Assistant text, Slack posts, reactions, new canvases, new lists, and file sends go only to the current conversation or thread. Your reply is not a channel post, and you cannot create a new top-level channel post. When asked to post to a channel or anywhere else right now, explain that limitation; do not call a Slack send tool, and do not reply with only the requested text or present it as delivered. Scheduled automations can still post to the active channel later. Slack read tools can target other public channels with `channel_id`.", "- Ambient reaction requests target the current inbound message; do not ask for a message reference.", `- When no visible thread reply is requested or useful, keep tool-calling messages text-free and make the final message exactly ${NO_REPLY_MARKER}.`, ];