From 02a5c186624805364179e80f0cfa5d95c6267150 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Fri, 30 Oct 2026 18:23:57 +0000 Subject: [PATCH 1/2] fix(evals): Record automation fixture channels as public Since #2440 (15ee5c130), an automation run no longer writes the visibility saved on its Automation to the Destination row. Only a live Slack signal does. A run into a channel with no row stores `unknown`, and readers treat it as private. In production, the Slack thread where a person created the Automation records the channel. The eval fixture inserts Automations without that thread, so their channels stayed `unknown`. The fixture reads results as a viewer who is not a participant, so the reporting API redacted the run Conversations. Each tool call came back with no input and no output. Evals that read `finishAutomationRun` results or tool arguments of scheduled or event automation runs then failed. `insertScheduledAutomation` and `insertEventAutomation` now store the creation thread through the Conversation store with public visibility, as the live Slack event would. --- packages/junior-evals/src/fixture/insert.ts | 26 ++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/packages/junior-evals/src/fixture/insert.ts b/packages/junior-evals/src/fixture/insert.ts index 4b2a865ead..e76d61b732 100644 --- a/packages/junior-evals/src/fixture/insert.ts +++ b/packages/junior-evals/src/fixture/insert.ts @@ -11,7 +11,7 @@ import { onTestFinished } from "vitest"; import { createMemoryStore, type MemoryDb } from "@sentry/junior-memory"; import { createSlackSource } from "@sentry/junior-plugin-api"; import { createUserTokenStore } from "@/chat/capabilities/factory"; -import { getDb, getSqlExecutor } from "@/chat/db"; +import { getConversationStore, getDb, getSqlExecutor } from "@/chat/db"; import { createSlackDestination } from "@/chat/destination"; import { createEventAutomation } from "@/chat/event-automations/store"; import { createWatch } from "@/chat/events/store"; @@ -99,6 +99,28 @@ async function insertSlackIdentity(author: Required) { }); } +/** + * Store the Slack thread where a person created an automation. Like the live + * Slack event of that thread, it records the channel as public. Automation + * runs do not record channel visibility, so without it their Conversations in + * the channel are private. + */ +async function insertCreationThread(destination: SlackChannel, nowMs: number) { + await getConversationStore().recordActivity({ + conversationId: `slack:${destination.channelId}:${FIXTURE_MESSAGE_TS}`, + destination, + nowMs, + sessionSource: createSlackSource({ + channelId: destination.channelId, + teamId: destination.teamId, + threadTs: FIXTURE_MESSAGE_TS, + visibility: "public", + }), + source: "slack", + visibility: "public", + }); +} + /** * Store a scheduled automation that a Slack person created. It is weekly and * next runs in a week unless `due` or `once` says otherwise. @@ -162,6 +184,7 @@ export async function insertScheduledAutomation(args: { task: { text: args.task }, updatedAtMs: nowMs - 60_000, }; + await insertCreationThread(args.destination, nowMs); await saveScheduledAutomation(getDb(), automation); return { id }; } @@ -180,6 +203,7 @@ export async function insertEventAutomation(args: { const author = resolveAuthor(args.createdBy); const identity = await insertSlackIdentity(author); const id = `evt_${fixtureId("event-automation", 20)}`; + await insertCreationThread(args.destination, Date.now()); await createEventAutomation(getDb(), { id, createdAtMs: Date.now() - 60_000, From 6839b929d688516fbf3eb466bde75adcc8fcba4b Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Fri, 30 Oct 2026 18:27:36 +0000 Subject: [PATCH 2/2] ref(evals): Let the store time the automation creation thread The Conversation store defaults the activity time, so the helper does not need a time argument. Describe the shared setup timestamp for both of its uses. --- packages/junior-evals/src/fixture/insert.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/junior-evals/src/fixture/insert.ts b/packages/junior-evals/src/fixture/insert.ts index e76d61b732..52f24c76f8 100644 --- a/packages/junior-evals/src/fixture/insert.ts +++ b/packages/junior-evals/src/fixture/insert.ts @@ -40,7 +40,10 @@ export interface SlackChannel { */ const FIXTURE_START_DATE = "2026-05-04"; -/** The Slack timestamp of the message that stored a setup memory. */ +/** + * The Slack timestamp of the setup message that stored a memory or created an + * automation. + */ const FIXTURE_MESSAGE_TS = "1780000000.000100"; /** @@ -105,11 +108,10 @@ async function insertSlackIdentity(author: Required) { * runs do not record channel visibility, so without it their Conversations in * the channel are private. */ -async function insertCreationThread(destination: SlackChannel, nowMs: number) { +async function insertCreationThread(destination: SlackChannel) { await getConversationStore().recordActivity({ conversationId: `slack:${destination.channelId}:${FIXTURE_MESSAGE_TS}`, destination, - nowMs, sessionSource: createSlackSource({ channelId: destination.channelId, teamId: destination.teamId, @@ -184,7 +186,7 @@ export async function insertScheduledAutomation(args: { task: { text: args.task }, updatedAtMs: nowMs - 60_000, }; - await insertCreationThread(args.destination, nowMs); + await insertCreationThread(args.destination); await saveScheduledAutomation(getDb(), automation); return { id }; } @@ -203,7 +205,7 @@ export async function insertEventAutomation(args: { const author = resolveAuthor(args.createdBy); const identity = await insertSlackIdentity(author); const id = `evt_${fixtureId("event-automation", 20)}`; - await insertCreationThread(args.destination, Date.now()); + await insertCreationThread(args.destination); await createEventAutomation(getDb(), { id, createdAtMs: Date.now() - 60_000,