From d6ec115348d0581fc2e6729298db7f31c776d1d6 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Tue, 7 Apr 2026 16:11:31 -0700 Subject: [PATCH 1/7] v0.6.29: login improvements, posthog telemetry (#4026) * feat(posthog): Add tracking on mothership abort (#4023) Co-authored-by: Theodore Li * fix(login): fix captcha headers for manual login (#4025) * fix(signup): fix turnstile key loading * fix(login): fix captcha header passing * Catch user already exists, remove login form captcha --- apps/sim/app/(auth)/signup/signup-form.tsx | 11 +++-------- .../app/workspace/[workspaceId]/home/home.tsx | 12 ++++++++++-- .../w/[workflowId]/components/panel/panel.tsx | 19 ++++++++++++++++++- apps/sim/lib/posthog/events.ts | 5 +++++ 4 files changed, 36 insertions(+), 11 deletions(-) diff --git a/apps/sim/app/(auth)/signup/signup-form.tsx b/apps/sim/app/(auth)/signup/signup-form.tsx index 55a0508ec1b..afb27cd729a 100644 --- a/apps/sim/app/(auth)/signup/signup-form.tsx +++ b/apps/sim/app/(auth)/signup/signup-form.tsx @@ -270,10 +270,8 @@ function SignupFormContent({ name: sanitizedName, }, { - fetchOptions: { - headers: { - ...(token ? { 'x-captcha-response': token } : {}), - }, + headers: { + ...(token ? { 'x-captcha-response': token } : {}), }, onError: (ctx) => { logger.error('Signup error:', ctx.error) @@ -282,10 +280,7 @@ function SignupFormContent({ let errorCode = 'unknown' if (ctx.error.code?.includes('USER_ALREADY_EXISTS')) { errorCode = 'user_already_exists' - errorMessage.push( - 'An account with this email already exists. Please sign in instead.' - ) - setEmailError(errorMessage[0]) + setEmailError('An account with this email already exists. Please sign in instead.') } else if ( ctx.error.code?.includes('BAD_REQUEST') || ctx.error.message?.includes('Email and password sign up is not enabled') diff --git a/apps/sim/app/workspace/[workspaceId]/home/home.tsx b/apps/sim/app/workspace/[workspaceId]/home/home.tsx index d76f17ff454..38367339197 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/home.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/home.tsx @@ -223,6 +223,14 @@ export function Home({ chatId }: HomeProps = {}) { posthogRef.current = posthog }, [posthog]) + const handleStopGeneration = useCallback(() => { + captureEvent(posthogRef.current, 'task_generation_aborted', { + workspace_id: workspaceId, + view: 'mothership', + }) + stopGeneration() + }, [stopGeneration, workspaceId]) + const handleSubmit = useCallback( (text: string, fileAttachments?: FileAttachmentForApi[], contexts?: ChatContext[]) => { const trimmed = text.trim() @@ -334,7 +342,7 @@ export function Home({ chatId }: HomeProps = {}) { defaultValue={initialPrompt} onSubmit={handleSubmit} isSending={isSending} - onStopGeneration={stopGeneration} + onStopGeneration={handleStopGeneration} userId={session?.user?.id} onContextAdd={handleContextAdd} /> @@ -359,7 +367,7 @@ export function Home({ chatId }: HomeProps = {}) { isSending={isSending} isReconnecting={isReconnecting} onSubmit={handleSubmit} - onStopGeneration={stopGeneration} + onStopGeneration={handleStopGeneration} messageQueue={messageQueue} onRemoveQueuedMessage={removeFromQueue} onSendQueuedMessage={sendNow} diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/panel.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/panel.tsx index 4d485c763ce..da51910789b 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/panel.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/panel.tsx @@ -4,6 +4,7 @@ import { memo, useCallback, useEffect, useRef, useState } from 'react' import { createLogger } from '@sim/logger' import { History, Plus, Square } from 'lucide-react' import { useParams, useRouter } from 'next/navigation' +import { usePostHog } from 'posthog-js/react' import { useShallow } from 'zustand/react/shallow' import { BubbleChatClose, @@ -33,6 +34,7 @@ import { import { Lock, Unlock, Upload } from '@/components/emcn/icons' import { VariableIcon } from '@/components/icons' import { useSession } from '@/lib/auth/auth-client' +import { captureEvent } from '@/lib/posthog/client' import { generateWorkflowJson } from '@/lib/workflows/operations/import-export' import { ConversationListItem } from '@/app/workspace/[workspaceId]/components' import { MothershipChat } from '@/app/workspace/[workspaceId]/home/components' @@ -101,6 +103,9 @@ export const Panel = memo(function Panel({ workspaceId: propWorkspaceId }: Panel const params = useParams() const workspaceId = propWorkspaceId ?? (params.workspaceId as string) + const posthog = usePostHog() + const posthogRef = useRef(posthog) + const panelRef = useRef(null) const fileInputRef = useRef(null) const { activeTab, setActiveTab, panelWidth, _hasHydrated, setHasHydrated } = usePanelStore( @@ -264,6 +269,10 @@ export const Panel = memo(function Panel({ workspaceId: propWorkspaceId }: Panel loadCopilotChats() }, [loadCopilotChats]) + useEffect(() => { + posthogRef.current = posthog + }, [posthog]) + const handleCopilotSelectChat = useCallback((chat: { id: string; title: string | null }) => { setCopilotChatId(chat.id) setCopilotChatTitle(chat.title) @@ -394,6 +403,14 @@ export const Panel = memo(function Panel({ workspaceId: propWorkspaceId }: Panel [copilotEditQueuedMessage] ) + const handleCopilotStopGeneration = useCallback(() => { + captureEvent(posthogRef.current, 'task_generation_aborted', { + workspace_id: workspaceId, + view: 'copilot', + }) + copilotStopGeneration() + }, [copilotStopGeneration, workspaceId]) + const handleCopilotSubmit = useCallback( (text: string, fileAttachments?: FileAttachmentForApi[], contexts?: ChatContext[]) => { const trimmed = text.trim() @@ -833,7 +850,7 @@ export const Panel = memo(function Panel({ workspaceId: propWorkspaceId }: Panel isSending={copilotIsSending} isReconnecting={copilotIsReconnecting} onSubmit={handleCopilotSubmit} - onStopGeneration={copilotStopGeneration} + onStopGeneration={handleCopilotStopGeneration} messageQueue={copilotMessageQueue} onRemoveQueuedMessage={copilotRemoveFromQueue} onSendQueuedMessage={copilotSendNow} diff --git a/apps/sim/lib/posthog/events.ts b/apps/sim/lib/posthog/events.ts index 537a9864282..faf9895bf62 100644 --- a/apps/sim/lib/posthog/events.ts +++ b/apps/sim/lib/posthog/events.ts @@ -378,6 +378,11 @@ export interface PostHogEventMap { workspace_id: string } + task_generation_aborted: { + workspace_id: string + view: 'mothership' | 'copilot' + } + task_message_sent: { workspace_id: string has_attachments: boolean From 6f07856dd4ec2cba4ef58c5af292eada9b62dcf7 Mon Sep 17 00:00:00 2001 From: waleed Date: Wed, 1 Jul 2026 23:59:39 -0700 Subject: [PATCH 2/7] feat(langsmith): add run update, get run, and feedback tools - add langsmith_update_run (PATCH /runs/{id}) to complete the create-then-patch tracing lifecycle - add langsmith_get_run (GET /runs/{id}) to read a run back - add langsmith_create_feedback (POST /feedback) to attach scores/corrections to runs - wire all three into the LangSmith block, reusing shared subBlocks across operations - fix feedback-capture template to use real feedback API instead of faking it via tagged runs - switch manual Object.fromEntries filtering to filterUndefined per repo convention --- apps/sim/blocks/blocks/langsmith.ts | 184 ++++++++++++++++++-- apps/sim/tools/langsmith/create_feedback.ts | 128 ++++++++++++++ apps/sim/tools/langsmith/create_run.ts | 5 +- apps/sim/tools/langsmith/get_run.ts | 82 +++++++++ apps/sim/tools/langsmith/index.ts | 3 + apps/sim/tools/langsmith/types.ts | 77 +++++++- apps/sim/tools/langsmith/update_run.ts | 114 ++++++++++++ apps/sim/tools/registry.ts | 11 +- 8 files changed, 587 insertions(+), 17 deletions(-) create mode 100644 apps/sim/tools/langsmith/create_feedback.ts create mode 100644 apps/sim/tools/langsmith/get_run.ts create mode 100644 apps/sim/tools/langsmith/update_run.ts diff --git a/apps/sim/blocks/blocks/langsmith.ts b/apps/sim/blocks/blocks/langsmith.ts index cb8d20dff6b..f4d1743e559 100644 --- a/apps/sim/blocks/blocks/langsmith.ts +++ b/apps/sim/blocks/blocks/langsmith.ts @@ -23,6 +23,9 @@ export const LangsmithBlock: BlockConfig = { options: [ { label: 'Create Run', id: 'langsmith_create_run' }, { label: 'Create Runs Batch', id: 'langsmith_create_runs_batch' }, + { label: 'Update Run', id: 'langsmith_update_run' }, + { label: 'Get Run', id: 'langsmith_get_run' }, + { label: 'Create Feedback', id: 'langsmith_create_feedback' }, ], value: () => 'langsmith_create_run', }, @@ -41,13 +44,27 @@ export const LangsmithBlock: BlockConfig = { placeholder: 'Auto-generated if blank', condition: { field: 'operation', value: 'langsmith_create_run' }, }, + { + id: 'runId', + title: 'Run ID', + type: 'short-input', + placeholder: 'ID of the run to update, retrieve, or attach feedback to', + required: { + field: 'operation', + value: ['langsmith_update_run', 'langsmith_get_run', 'langsmith_create_feedback'], + }, + condition: { + field: 'operation', + value: ['langsmith_update_run', 'langsmith_get_run', 'langsmith_create_feedback'], + }, + }, { id: 'name', title: 'Name', type: 'short-input', placeholder: 'Run name', required: { field: 'operation', value: 'langsmith_create_run' }, - condition: { field: 'operation', value: 'langsmith_create_run' }, + condition: { field: 'operation', value: ['langsmith_create_run', 'langsmith_update_run'] }, }, { id: 'run_type', @@ -78,7 +95,7 @@ export const LangsmithBlock: BlockConfig = { title: 'End Time', type: 'short-input', placeholder: '2025-01-01T12:00:30Z', - condition: { field: 'operation', value: 'langsmith_create_run' }, + condition: { field: 'operation', value: ['langsmith_create_run', 'langsmith_update_run'] }, mode: 'advanced', }, { @@ -94,7 +111,7 @@ export const LangsmithBlock: BlockConfig = { title: 'Outputs', type: 'code', placeholder: '{"output":"value"}', - condition: { field: 'operation', value: 'langsmith_create_run' }, + condition: { field: 'operation', value: ['langsmith_create_run', 'langsmith_update_run'] }, mode: 'advanced', }, { @@ -102,7 +119,7 @@ export const LangsmithBlock: BlockConfig = { title: 'Metadata', type: 'code', placeholder: '{"ls_model":"gpt-4"}', - condition: { field: 'operation', value: 'langsmith_create_run' }, + condition: { field: 'operation', value: ['langsmith_create_run', 'langsmith_update_run'] }, mode: 'advanced', }, { @@ -110,7 +127,7 @@ export const LangsmithBlock: BlockConfig = { title: 'Tags', type: 'code', placeholder: '["production","workflow"]', - condition: { field: 'operation', value: 'langsmith_create_run' }, + condition: { field: 'operation', value: ['langsmith_create_run', 'langsmith_update_run'] }, mode: 'advanced', }, { @@ -150,7 +167,7 @@ export const LangsmithBlock: BlockConfig = { title: 'Status', type: 'short-input', placeholder: 'success', - condition: { field: 'operation', value: 'langsmith_create_run' }, + condition: { field: 'operation', value: ['langsmith_create_run', 'langsmith_update_run'] }, mode: 'advanced', }, { @@ -158,7 +175,7 @@ export const LangsmithBlock: BlockConfig = { title: 'Error', type: 'long-input', placeholder: 'Error message', - condition: { field: 'operation', value: 'langsmith_create_run' }, + condition: { field: 'operation', value: ['langsmith_create_run', 'langsmith_update_run'] }, mode: 'advanced', }, { @@ -174,7 +191,7 @@ export const LangsmithBlock: BlockConfig = { title: 'Events', type: 'code', placeholder: '[{"event":"token","value":1}]', - condition: { field: 'operation', value: 'langsmith_create_run' }, + condition: { field: 'operation', value: ['langsmith_create_run', 'langsmith_update_run'] }, mode: 'advanced', }, { @@ -207,9 +224,66 @@ Required: id (existing run UUID), name, run_type ("tool"|"chain"|"llm"|"retrieve Common patch fields: outputs, end_time, status, error`, }, }, + { + id: 'key', + title: 'Feedback Key', + type: 'short-input', + placeholder: 'e.g. correctness, user_score', + required: { field: 'operation', value: 'langsmith_create_feedback' }, + condition: { field: 'operation', value: 'langsmith_create_feedback' }, + }, + { + id: 'score', + title: 'Score', + type: 'short-input', + placeholder: 'e.g. 1, 0.5, 0', + condition: { field: 'operation', value: 'langsmith_create_feedback' }, + }, + { + id: 'value', + title: 'Value', + type: 'short-input', + placeholder: 'e.g. good, bad', + condition: { field: 'operation', value: 'langsmith_create_feedback' }, + mode: 'advanced', + }, + { + id: 'comment', + title: 'Comment', + type: 'long-input', + placeholder: 'Explanation for the feedback', + condition: { field: 'operation', value: 'langsmith_create_feedback' }, + mode: 'advanced', + }, + { + id: 'correction', + title: 'Correction', + type: 'code', + placeholder: '{"output":"the corrected value"}', + condition: { field: 'operation', value: 'langsmith_create_feedback' }, + mode: 'advanced', + }, + { + id: 'feedbackSourceType', + title: 'Feedback Source', + type: 'dropdown', + options: [ + { label: 'API', id: 'api' }, + { label: 'App', id: 'app' }, + { label: 'Model', id: 'model' }, + ], + condition: { field: 'operation', value: 'langsmith_create_feedback' }, + mode: 'advanced', + }, ], tools: { - access: ['langsmith_create_run', 'langsmith_create_runs_batch'], + access: [ + 'langsmith_create_run', + 'langsmith_create_runs_batch', + 'langsmith_update_run', + 'langsmith_get_run', + 'langsmith_create_feedback', + ], config: { tool: (params) => params.operation, params: (params) => { @@ -242,6 +316,42 @@ Common patch fields: outputs, end_time, status, error`, } } + if (params.operation === 'langsmith_update_run') { + return { + apiKey: params.apiKey, + runId: params.runId, + name: params.name, + end_time: params.end_time, + outputs: parseJsonValue(params.outputs, 'outputs'), + extra: parseJsonValue(params.extra, 'metadata'), + tags: parseJsonValue(params.tags, 'tags'), + status: params.status, + error: params.error, + events: parseJsonValue(params.events, 'events'), + } + } + + if (params.operation === 'langsmith_get_run') { + return { + apiKey: params.apiKey, + runId: params.runId, + } + } + + if (params.operation === 'langsmith_create_feedback') { + return { + apiKey: params.apiKey, + runId: params.runId, + key: params.key, + score: + params.score === '' || params.score === undefined ? undefined : Number(params.score), + value: params.value, + comment: params.comment, + correction: parseJsonValue(params.correction, 'correction'), + feedbackSourceType: params.feedbackSourceType || undefined, + } + } + return { apiKey: params.apiKey, id: params.id, @@ -269,6 +379,10 @@ Common patch fields: outputs, end_time, status, error`, operation: { type: 'string', description: 'Operation to perform' }, apiKey: { type: 'string', description: 'LangSmith API key' }, id: { type: 'string', description: 'Run identifier' }, + runId: { + type: 'string', + description: 'ID of the run to update, retrieve, or attach feedback to', + }, name: { type: 'string', description: 'Run name' }, run_type: { type: 'string', description: 'Run type' }, start_time: { type: 'string', description: 'Run start time (ISO)' }, @@ -287,13 +401,43 @@ Common patch fields: outputs, end_time, status, error`, events: { type: 'json', description: 'Events array' }, post: { type: 'json', description: 'Runs to ingest in batch' }, patch: { type: 'json', description: 'Runs to update in batch' }, + key: { type: 'string', description: 'Feedback metric name' }, + score: { type: 'string', description: 'Numeric score for the feedback metric' }, + value: { type: 'string', description: 'Categorical value for the feedback metric' }, + comment: { type: 'string', description: 'Comment explaining the feedback' }, + correction: { type: 'json', description: 'Corrected output for the run' }, + feedbackSourceType: { + type: 'string', + description: 'Origin of the feedback (api, app, or model)', + }, }, outputs: { - accepted: { type: 'boolean', description: 'Whether ingestion was accepted' }, - runId: { type: 'string', description: 'Run ID for single run' }, + accepted: { type: 'boolean', description: 'Whether ingestion or the update was accepted' }, + runId: { type: 'string', description: 'Run ID for single-run operations' }, runIds: { type: 'array', description: 'Run IDs for batch ingest' }, message: { type: 'string', description: 'LangSmith response message' }, messages: { type: 'array', description: 'Per-run response messages' }, + id: { type: 'string', description: 'Run ID (get run) or feedback ID (create feedback)' }, + name: { type: 'string', description: 'Run name (get run)' }, + runType: { type: 'string', description: 'Run type (get run)' }, + status: { type: 'string', description: 'Run status (get run)' }, + startTime: { type: 'string', description: 'Run start time (get run)' }, + endTime: { type: 'string', description: 'Run end time (get run)' }, + error: { type: 'string', description: 'Error details (get run)' }, + tags: { type: 'array', description: 'Tags attached to the run (get run)' }, + sessionId: { type: 'string', description: 'Project (session) ID the run belongs to (get run)' }, + traceId: { type: 'string', description: 'Trace ID (get run)' }, + parentRunId: { type: 'string', description: 'Parent run ID (get run)' }, + totalTokens: { type: 'number', description: 'Total tokens consumed by the run (get run)' }, + totalCost: { type: 'string', description: 'Total cost of the run (get run)' }, + key: { type: 'string', description: 'Feedback metric name (create feedback)' }, + score: { type: 'number', description: 'Score recorded for the feedback (create feedback)' }, + value: { + type: 'string', + description: 'Categorical value recorded for the feedback (create feedback)', + }, + comment: { type: 'string', description: 'Comment recorded for the feedback (create feedback)' }, + createdAt: { type: 'string', description: 'When the feedback was created (create feedback)' }, }, } @@ -324,11 +468,20 @@ export const LangsmithBlockMeta = { icon: LangsmithIcon, title: 'LangSmith feedback capture', prompt: - 'Build a workflow that collects user-reported agent failures from a table and forwards each as a tagged LangSmith run with the inputs and expected output for later review.', + 'Build a workflow that collects user-reported agent failures from a table and attaches each as scored LangSmith feedback on the originating run for later review.', modules: ['tables', 'agent', 'workflows'], category: 'engineering', tags: ['engineering', 'automation'], }, + { + icon: LangsmithIcon, + title: 'LangSmith run completion', + prompt: + 'Build a workflow that creates a LangSmith run when an agent step starts, then updates it with outputs, status, and end time once the step finishes so traces always show the full lifecycle.', + modules: ['agent', 'workflows'], + category: 'engineering', + tags: ['engineering', 'monitoring'], + }, { icon: LangsmithIcon, title: 'LangSmith batch run shipper', @@ -381,5 +534,12 @@ export const LangsmithBlockMeta = { content: '# Batch Export Runs\n\nShip multiple completed runs to LangSmith at once instead of one by one.\n\n## Steps\n1. Collect the runs to export, each with name, type, inputs, outputs, and timing.\n2. Assign a shared project so the runs land together.\n3. Submit them as a single batch.\n\n## Output\nReturn how many runs were exported, the project they landed in, and any runs that failed validation.', }, + { + name: 'attach-feedback-to-run', + description: + 'Attach a score, categorical value, or correction to an existing LangSmith run for evaluation.', + content: + '# Attach Feedback to a Run\n\nRecord a human or automated judgment on a run that already exists in LangSmith.\n\n## Steps\n1. Identify the run ID the feedback applies to.\n2. Choose a feedback key (e.g. "correctness", "user_score") and a score, value, or comment.\n3. Include a correction if the expected output is known.\n4. Submit the feedback.\n\n## Output\nConfirm the feedback ID and the run it was attached to.', + }, ], } as const satisfies BlockMeta diff --git a/apps/sim/tools/langsmith/create_feedback.ts b/apps/sim/tools/langsmith/create_feedback.ts new file mode 100644 index 00000000000..1da987d6b59 --- /dev/null +++ b/apps/sim/tools/langsmith/create_feedback.ts @@ -0,0 +1,128 @@ +import { generateId } from '@sim/utils/id' +import { filterUndefined } from '@sim/utils/object' +import type { + LangsmithCreateFeedbackParams, + LangsmithCreateFeedbackResponse, +} from '@/tools/langsmith/types' +import type { ToolConfig } from '@/tools/types' + +export const langsmithCreateFeedbackTool: ToolConfig< + LangsmithCreateFeedbackParams, + LangsmithCreateFeedbackResponse +> = { + id: 'langsmith_create_feedback', + name: 'LangSmith Create Feedback', + description: 'Attach a score, correction, or comment to a LangSmith run.', + version: '1.0.0', + params: { + apiKey: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'LangSmith API key', + }, + runId: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'ID of the run to attach feedback to', + }, + key: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Feedback metric name (e.g. "correctness", "user_score")', + }, + score: { + type: 'number', + required: false, + visibility: 'user-or-llm', + description: 'Numeric or boolean score for the feedback metric', + }, + value: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Categorical value for the feedback metric', + }, + comment: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Free-text comment explaining the feedback', + }, + correction: { + type: 'json', + required: false, + visibility: 'user-or-llm', + description: 'Corrected output for the run', + }, + feedbackSourceType: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Origin of the feedback (api, app, or model)', + }, + }, + request: { + url: () => 'https://api.smith.langchain.com/feedback', + method: 'POST', + headers: (params) => ({ + 'X-Api-Key': params.apiKey, + 'Content-Type': 'application/json', + }), + body: (params) => { + const payload: Record = { + id: generateId(), + run_id: params.runId.trim(), + key: params.key, + score: params.score, + value: params.value, + comment: params.comment, + correction: params.correction, + feedback_source: params.feedbackSourceType + ? { type: params.feedbackSourceType } + : undefined, + } + + return filterUndefined(payload) + }, + }, + transformResponse: async (response) => { + const data = (await response.json()) as Record + + return { + success: true, + output: { + id: data.id as string, + key: data.key as string, + runId: (data.run_id as string) ?? null, + score: (data.score as number | boolean) ?? null, + value: (data.value as string | number | boolean) ?? null, + comment: (data.comment as string) ?? null, + createdAt: (data.created_at as string) ?? null, + }, + } + }, + outputs: { + id: { type: 'string', description: 'Feedback ID' }, + key: { type: 'string', description: 'Feedback metric name' }, + runId: { + type: 'string', + description: 'ID of the run the feedback was attached to', + optional: true, + }, + score: { type: 'number', description: 'Score recorded for the feedback', optional: true }, + value: { + type: 'string', + description: 'Categorical value recorded for the feedback', + optional: true, + }, + comment: { type: 'string', description: 'Comment recorded for the feedback', optional: true }, + createdAt: { + type: 'string', + description: 'When the feedback was created (ISO)', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/langsmith/create_run.ts b/apps/sim/tools/langsmith/create_run.ts index 510757ac23c..0132dfdca2f 100644 --- a/apps/sim/tools/langsmith/create_run.ts +++ b/apps/sim/tools/langsmith/create_run.ts @@ -1,3 +1,4 @@ +import { filterUndefined } from '@sim/utils/object' import type { LangsmithCreateRunParams, LangsmithCreateRunResponse } from '@/tools/langsmith/types' import { normalizeLangsmithRunPayload } from '@/tools/langsmith/utils' import type { ToolConfig } from '@/tools/types' @@ -141,9 +142,7 @@ export const langsmithCreateRunTool: ToolConfig< events: params.events, } - return Object.fromEntries( - Object.entries(normalizedPayload).filter(([, value]) => value !== undefined) - ) + return filterUndefined(normalizedPayload) }, }, transformResponse: async (response, params) => { diff --git a/apps/sim/tools/langsmith/get_run.ts b/apps/sim/tools/langsmith/get_run.ts new file mode 100644 index 00000000000..bd13700a3bb --- /dev/null +++ b/apps/sim/tools/langsmith/get_run.ts @@ -0,0 +1,82 @@ +import type { LangsmithGetRunParams, LangsmithGetRunResponse } from '@/tools/langsmith/types' +import type { ToolConfig } from '@/tools/types' + +export const langsmithGetRunTool: ToolConfig = { + id: 'langsmith_get_run', + name: 'LangSmith Get Run', + description: 'Retrieve a single LangSmith run by ID.', + version: '1.0.0', + params: { + apiKey: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'LangSmith API key', + }, + runId: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'ID of the run to retrieve', + }, + }, + request: { + url: (params) => `https://api.smith.langchain.com/runs/${params.runId.trim()}`, + method: 'GET', + headers: (params) => ({ + 'X-Api-Key': params.apiKey, + }), + }, + transformResponse: async (response) => { + const data = (await response.json()) as Record + + return { + success: true, + output: { + id: data.id as string, + name: data.name as string, + runType: data.run_type as string, + status: (data.status as string) ?? null, + startTime: (data.start_time as string) ?? null, + endTime: (data.end_time as string) ?? null, + inputs: (data.inputs as Record) ?? null, + outputs: (data.outputs as Record) ?? null, + error: (data.error as string) ?? null, + tags: (data.tags as string[]) ?? [], + sessionId: (data.session_id as string) ?? null, + traceId: (data.trace_id as string) ?? null, + parentRunId: (data.parent_run_id as string) ?? null, + totalTokens: (data.total_tokens as number) ?? null, + totalCost: (data.total_cost as string) ?? null, + }, + } + }, + outputs: { + id: { type: 'string', description: 'Run ID' }, + name: { type: 'string', description: 'Run name' }, + runType: { + type: 'string', + description: 'Run type (tool, chain, llm, retriever, embedding, prompt, parser)', + }, + status: { type: 'string', description: 'Run status', optional: true }, + startTime: { type: 'string', description: 'Run start time (ISO)', optional: true }, + endTime: { type: 'string', description: 'Run end time (ISO)', optional: true }, + inputs: { type: 'json', description: 'Run inputs payload', optional: true }, + outputs: { type: 'json', description: 'Run outputs payload', optional: true }, + error: { type: 'string', description: 'Error details, if the run failed', optional: true }, + tags: { type: 'array', description: 'Tags attached to the run', items: { type: 'string' } }, + sessionId: { + type: 'string', + description: 'Project (session) ID the run belongs to', + optional: true, + }, + traceId: { type: 'string', description: 'Trace ID', optional: true }, + parentRunId: { type: 'string', description: 'Parent run ID', optional: true }, + totalTokens: { + type: 'number', + description: 'Total tokens consumed by the run', + optional: true, + }, + totalCost: { type: 'string', description: 'Total cost of the run', optional: true }, + }, +} diff --git a/apps/sim/tools/langsmith/index.ts b/apps/sim/tools/langsmith/index.ts index f173b10a217..5f7e6019a8c 100644 --- a/apps/sim/tools/langsmith/index.ts +++ b/apps/sim/tools/langsmith/index.ts @@ -1,2 +1,5 @@ +export { langsmithCreateFeedbackTool } from '@/tools/langsmith/create_feedback' export { langsmithCreateRunTool } from '@/tools/langsmith/create_run' export { langsmithCreateRunsBatchTool } from '@/tools/langsmith/create_runs_batch' +export { langsmithGetRunTool } from '@/tools/langsmith/get_run' +export { langsmithUpdateRunTool } from '@/tools/langsmith/update_run' diff --git a/apps/sim/tools/langsmith/types.ts b/apps/sim/tools/langsmith/types.ts index 860a72e303f..19a57169bbe 100644 --- a/apps/sim/tools/langsmith/types.ts +++ b/apps/sim/tools/langsmith/types.ts @@ -57,4 +57,79 @@ export interface LangsmithCreateRunsBatchResponse extends ToolResponse { } } -export type LangsmithResponse = LangsmithCreateRunResponse | LangsmithCreateRunsBatchResponse +export interface LangsmithUpdateRunParams { + apiKey: string + runId: string + name?: string + end_time?: string + outputs?: Record + extra?: Record + tags?: string[] + status?: string + error?: string + events?: Record[] +} + +export interface LangsmithUpdateRunResponse extends ToolResponse { + output: { + accepted: boolean + runId: string + } +} + +export interface LangsmithGetRunParams { + apiKey: string + runId: string +} + +export interface LangsmithGetRunResponse extends ToolResponse { + output: { + id: string + name: string + runType: string + status: string | null + startTime: string | null + endTime: string | null + inputs: Record | null + outputs: Record | null + error: string | null + tags: string[] + sessionId: string | null + traceId: string | null + parentRunId: string | null + totalTokens: number | null + totalCost: string | null + } +} + +export type LangsmithFeedbackSourceType = 'api' | 'app' | 'model' + +export interface LangsmithCreateFeedbackParams { + apiKey: string + runId: string + key: string + score?: number | boolean + value?: string + comment?: string + correction?: Record + feedbackSourceType?: LangsmithFeedbackSourceType +} + +export interface LangsmithCreateFeedbackResponse extends ToolResponse { + output: { + id: string + key: string + runId: string | null + score: number | boolean | null + value: string | number | boolean | null + comment: string | null + createdAt: string | null + } +} + +export type LangsmithResponse = + | LangsmithCreateRunResponse + | LangsmithCreateRunsBatchResponse + | LangsmithUpdateRunResponse + | LangsmithGetRunResponse + | LangsmithCreateFeedbackResponse diff --git a/apps/sim/tools/langsmith/update_run.ts b/apps/sim/tools/langsmith/update_run.ts new file mode 100644 index 00000000000..3d77486dcd8 --- /dev/null +++ b/apps/sim/tools/langsmith/update_run.ts @@ -0,0 +1,114 @@ +import { filterUndefined } from '@sim/utils/object' +import type { LangsmithUpdateRunParams, LangsmithUpdateRunResponse } from '@/tools/langsmith/types' +import type { ToolConfig } from '@/tools/types' + +export const langsmithUpdateRunTool: ToolConfig< + LangsmithUpdateRunParams, + LangsmithUpdateRunResponse +> = { + id: 'langsmith_update_run', + name: 'LangSmith Update Run', + description: 'Patch an existing LangSmith run with outputs, status, or timing once it completes.', + version: '1.0.0', + params: { + apiKey: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'LangSmith API key', + }, + runId: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'ID of the run to update', + }, + name: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Corrected run name', + }, + end_time: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Run end time in ISO-8601 format', + }, + outputs: { + type: 'json', + required: false, + visibility: 'user-or-llm', + description: 'Outputs payload', + }, + extra: { + type: 'json', + required: false, + visibility: 'user-or-llm', + description: 'Additional metadata (extra)', + }, + tags: { + type: 'json', + required: false, + visibility: 'user-or-llm', + description: 'Array of tag strings', + }, + status: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Run status', + }, + error: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Error details', + }, + events: { + type: 'json', + required: false, + visibility: 'user-or-llm', + description: 'Structured events array', + }, + }, + request: { + url: (params) => `https://api.smith.langchain.com/runs/${params.runId.trim()}`, + method: 'PATCH', + headers: (params) => ({ + 'X-Api-Key': params.apiKey, + 'Content-Type': 'application/json', + }), + body: (params) => { + const payload: Record = { + name: params.name, + end_time: params.end_time, + outputs: params.outputs, + extra: params.extra, + tags: params.tags, + status: params.status, + error: params.error, + events: params.events, + } + + return filterUndefined(payload) + }, + }, + transformResponse: async (_response, params) => ({ + success: true, + output: { + accepted: true, + runId: params?.runId.trim() ?? '', + }, + }), + outputs: { + accepted: { + type: 'boolean', + description: 'Whether the run update was accepted', + }, + runId: { + type: 'string', + description: 'ID of the run that was updated', + }, + }, +} diff --git a/apps/sim/tools/registry.ts b/apps/sim/tools/registry.ts index 2bb910c4eef..c9121640267 100644 --- a/apps/sim/tools/registry.ts +++ b/apps/sim/tools/registry.ts @@ -1905,7 +1905,13 @@ import { knowledgeUploadChunkTool, knowledgeUpsertDocumentTool, } from '@/tools/knowledge' -import { langsmithCreateRunsBatchTool, langsmithCreateRunTool } from '@/tools/langsmith' +import { + langsmithCreateFeedbackTool, + langsmithCreateRunsBatchTool, + langsmithCreateRunTool, + langsmithGetRunTool, + langsmithUpdateRunTool, +} from '@/tools/langsmith' import { latexCompileTool, latexGetPackageTool, @@ -7222,6 +7228,9 @@ export const tools: Record = { linear_list_project_statuses: linearListProjectStatusesTool, langsmith_create_run: langsmithCreateRunTool, langsmith_create_runs_batch: langsmithCreateRunsBatchTool, + langsmith_update_run: langsmithUpdateRunTool, + langsmith_get_run: langsmithGetRunTool, + langsmith_create_feedback: langsmithCreateFeedbackTool, latex_compile: latexCompileTool, latex_get_package: latexGetPackageTool, latex_list_fonts: latexListFontsTool, From 62e907eed0291fa943c57f1d2752a2836db70bb0 Mon Sep 17 00:00:00 2001 From: waleed Date: Thu, 2 Jul 2026 00:03:43 -0700 Subject: [PATCH 3/7] fix(langsmith): reject non-numeric feedback score instead of silently sending null Number() on an invalid score string produces NaN, which serializes to JSON null and would still be sent to LangSmith. Throw instead, matching the existing parseJsonValue error pattern. --- apps/sim/blocks/blocks/langsmith.ts | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/apps/sim/blocks/blocks/langsmith.ts b/apps/sim/blocks/blocks/langsmith.ts index f4d1743e559..7a755a0bd9c 100644 --- a/apps/sim/blocks/blocks/langsmith.ts +++ b/apps/sim/blocks/blocks/langsmith.ts @@ -339,12 +339,22 @@ Common patch fields: outputs, end_time, status, error`, } if (params.operation === 'langsmith_create_feedback') { + const parseScore = (value: unknown) => { + if (value === undefined || value === null || value === '') { + return undefined + } + const parsed = Number(value) + if (Number.isNaN(parsed)) { + throw new Error(`Invalid score: "${value}" is not a number`) + } + return parsed + } + return { apiKey: params.apiKey, runId: params.runId, key: params.key, - score: - params.score === '' || params.score === undefined ? undefined : Number(params.score), + score: parseScore(params.score), value: params.value, comment: params.comment, correction: parseJsonValue(params.correction, 'correction'), From 8c915c0b163411971250f01859ab463ee89635f1 Mon Sep 17 00:00:00 2001 From: waleed Date: Thu, 2 Jul 2026 00:08:50 -0700 Subject: [PATCH 4/7] fix(langsmith): harden error handling and validation on new run tools - update_run, get_run, create_feedback now check response.ok before parsing/returning, matching the cloudwatch/zoom convention instead of silently returning success on a 4xx/404 - block outputs schema now exposes inputs/outputs for Get Run - update_run requires at least one field to patch, matching the batch-ingest guard for post/patch --- apps/sim/blocks/blocks/langsmith.ts | 35 ++++++++++++++++----- apps/sim/tools/langsmith/create_feedback.ts | 5 +++ apps/sim/tools/langsmith/get_run.ts | 5 +++ apps/sim/tools/langsmith/update_run.ts | 21 ++++++++----- 4 files changed, 51 insertions(+), 15 deletions(-) diff --git a/apps/sim/blocks/blocks/langsmith.ts b/apps/sim/blocks/blocks/langsmith.ts index 7a755a0bd9c..3b0e423fd40 100644 --- a/apps/sim/blocks/blocks/langsmith.ts +++ b/apps/sim/blocks/blocks/langsmith.ts @@ -317,17 +317,34 @@ Common patch fields: outputs, end_time, status, error`, } if (params.operation === 'langsmith_update_run') { + const name = params.name + const end_time = params.end_time + const outputs = parseJsonValue(params.outputs, 'outputs') + const extra = parseJsonValue(params.extra, 'metadata') + const tags = parseJsonValue(params.tags, 'tags') + const status = params.status + const error = params.error + const events = parseJsonValue(params.events, 'events') + + if ( + [name, end_time, outputs, extra, tags, status, error, events].every( + (value) => value === undefined + ) + ) { + throw new Error('Provide at least one field to update') + } + return { apiKey: params.apiKey, runId: params.runId, - name: params.name, - end_time: params.end_time, - outputs: parseJsonValue(params.outputs, 'outputs'), - extra: parseJsonValue(params.extra, 'metadata'), - tags: parseJsonValue(params.tags, 'tags'), - status: params.status, - error: params.error, - events: parseJsonValue(params.events, 'events'), + name, + end_time, + outputs, + extra, + tags, + status, + error, + events, } } @@ -433,6 +450,8 @@ Common patch fields: outputs, end_time, status, error`, status: { type: 'string', description: 'Run status (get run)' }, startTime: { type: 'string', description: 'Run start time (get run)' }, endTime: { type: 'string', description: 'Run end time (get run)' }, + inputs: { type: 'json', description: 'Run inputs payload (get run)' }, + outputs: { type: 'json', description: 'Run outputs payload (get run)' }, error: { type: 'string', description: 'Error details (get run)' }, tags: { type: 'array', description: 'Tags attached to the run (get run)' }, sessionId: { type: 'string', description: 'Project (session) ID the run belongs to (get run)' }, diff --git a/apps/sim/tools/langsmith/create_feedback.ts b/apps/sim/tools/langsmith/create_feedback.ts index 1da987d6b59..0f7e7b9843c 100644 --- a/apps/sim/tools/langsmith/create_feedback.ts +++ b/apps/sim/tools/langsmith/create_feedback.ts @@ -89,6 +89,11 @@ export const langsmithCreateFeedbackTool: ToolConfig< }, }, transformResponse: async (response) => { + if (!response.ok) { + const errorText = await response.text() + throw new Error(`LangSmith create feedback failed (${response.status}): ${errorText}`) + } + const data = (await response.json()) as Record return { diff --git a/apps/sim/tools/langsmith/get_run.ts b/apps/sim/tools/langsmith/get_run.ts index bd13700a3bb..aed049797e8 100644 --- a/apps/sim/tools/langsmith/get_run.ts +++ b/apps/sim/tools/langsmith/get_run.ts @@ -28,6 +28,11 @@ export const langsmithGetRunTool: ToolConfig { + if (!response.ok) { + const errorText = await response.text() + throw new Error(`LangSmith get run failed (${response.status}): ${errorText}`) + } + const data = (await response.json()) as Record return { diff --git a/apps/sim/tools/langsmith/update_run.ts b/apps/sim/tools/langsmith/update_run.ts index 3d77486dcd8..a3f21146cb0 100644 --- a/apps/sim/tools/langsmith/update_run.ts +++ b/apps/sim/tools/langsmith/update_run.ts @@ -94,13 +94,20 @@ export const langsmithUpdateRunTool: ToolConfig< return filterUndefined(payload) }, }, - transformResponse: async (_response, params) => ({ - success: true, - output: { - accepted: true, - runId: params?.runId.trim() ?? '', - }, - }), + transformResponse: async (response, params) => { + if (!response.ok) { + const errorText = await response.text() + throw new Error(`LangSmith update run failed (${response.status}): ${errorText}`) + } + + return { + success: true, + output: { + accepted: true, + runId: params?.runId.trim() ?? '', + }, + } + }, outputs: { accepted: { type: 'boolean', From b191a540d61628928410387555cd415b7dbc80d0 Mon Sep 17 00:00:00 2001 From: waleed Date: Thu, 2 Jul 2026 08:40:38 -0700 Subject: [PATCH 5/7] fix(langsmith): align get_run/update_run outputs and narrow feedback score type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - get_run now also outputs runId (alias of id) so workflows can read the run identifier consistently across all operations on the block - update_run now parses and surfaces the response message instead of discarding the body entirely, matching create_run's pattern - narrow LangsmithCreateFeedbackParams.score to number — the tool param is declared as JSON-schema 'number' and the block's parseScore never produces a boolean, so the wider type was dead and misleading --- apps/sim/tools/langsmith/create_feedback.ts | 4 ++-- apps/sim/tools/langsmith/get_run.ts | 5 +++++ apps/sim/tools/langsmith/types.ts | 6 ++++-- apps/sim/tools/langsmith/update_run.ts | 17 +++++++++++++++++ 4 files changed, 28 insertions(+), 4 deletions(-) diff --git a/apps/sim/tools/langsmith/create_feedback.ts b/apps/sim/tools/langsmith/create_feedback.ts index 0f7e7b9843c..c1585bc0f48 100644 --- a/apps/sim/tools/langsmith/create_feedback.ts +++ b/apps/sim/tools/langsmith/create_feedback.ts @@ -37,7 +37,7 @@ export const langsmithCreateFeedbackTool: ToolConfig< type: 'number', required: false, visibility: 'user-or-llm', - description: 'Numeric or boolean score for the feedback metric', + description: 'Numeric score for the feedback metric', }, value: { type: 'string', @@ -102,7 +102,7 @@ export const langsmithCreateFeedbackTool: ToolConfig< id: data.id as string, key: data.key as string, runId: (data.run_id as string) ?? null, - score: (data.score as number | boolean) ?? null, + score: (data.score as number) ?? null, value: (data.value as string | number | boolean) ?? null, comment: (data.comment as string) ?? null, createdAt: (data.created_at as string) ?? null, diff --git a/apps/sim/tools/langsmith/get_run.ts b/apps/sim/tools/langsmith/get_run.ts index aed049797e8..22582859ffb 100644 --- a/apps/sim/tools/langsmith/get_run.ts +++ b/apps/sim/tools/langsmith/get_run.ts @@ -39,6 +39,7 @@ export const langsmithGetRunTool: ToolConfig @@ -120,7 +122,7 @@ export interface LangsmithCreateFeedbackResponse extends ToolResponse { id: string key: string runId: string | null - score: number | boolean | null + score: number | null value: string | number | boolean | null comment: string | null createdAt: string | null diff --git a/apps/sim/tools/langsmith/update_run.ts b/apps/sim/tools/langsmith/update_run.ts index a3f21146cb0..503d4b0acf9 100644 --- a/apps/sim/tools/langsmith/update_run.ts +++ b/apps/sim/tools/langsmith/update_run.ts @@ -100,11 +100,23 @@ export const langsmithUpdateRunTool: ToolConfig< throw new Error(`LangSmith update run failed (${response.status}): ${errorText}`) } + const responseText = await response.text() + let message: string | null = null + if (responseText) { + try { + const data = JSON.parse(responseText) as Record + message = typeof data.message === 'string' ? data.message : null + } catch { + // Response body isn't JSON (e.g. empty object or plain text) — no message to surface + } + } + return { success: true, output: { accepted: true, runId: params?.runId.trim() ?? '', + message, }, } }, @@ -117,5 +129,10 @@ export const langsmithUpdateRunTool: ToolConfig< type: 'string', description: 'ID of the run that was updated', }, + message: { + type: 'string', + description: 'Response message from LangSmith, if provided', + optional: true, + }, }, } From a41a8e26cbeb43911f846a4c0a4eb05f01e4bdc6 Mon Sep 17 00:00:00 2001 From: waleed Date: Thu, 2 Jul 2026 08:45:13 -0700 Subject: [PATCH 6/7] fix(langsmith): reject empty-string patch fields and empty PATCH bodies - block mapper now normalizes blank name/end_time/status/error inputs to undefined instead of forwarding empty strings, which would clear those fields on the LangSmith run - update_run tool now throws if the filtered PATCH body is empty, guarding direct/programmatic tool calls that bypass the block's own "at least one field" check --- apps/sim/blocks/blocks/langsmith.ts | 10 ++++++---- apps/sim/tools/langsmith/update_run.ts | 7 ++++++- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/apps/sim/blocks/blocks/langsmith.ts b/apps/sim/blocks/blocks/langsmith.ts index 3b0e423fd40..028fe76c32c 100644 --- a/apps/sim/blocks/blocks/langsmith.ts +++ b/apps/sim/blocks/blocks/langsmith.ts @@ -301,6 +301,8 @@ Common patch fields: outputs, end_time, status, error`, return value } + const emptyToUndefined = (value: unknown) => (value === '' ? undefined : value) + if (params.operation === 'langsmith_create_runs_batch') { const post = parseJsonValue(params.post, 'post runs') const patch = parseJsonValue(params.patch, 'patch runs') @@ -317,13 +319,13 @@ Common patch fields: outputs, end_time, status, error`, } if (params.operation === 'langsmith_update_run') { - const name = params.name - const end_time = params.end_time + const name = emptyToUndefined(params.name) + const end_time = emptyToUndefined(params.end_time) const outputs = parseJsonValue(params.outputs, 'outputs') const extra = parseJsonValue(params.extra, 'metadata') const tags = parseJsonValue(params.tags, 'tags') - const status = params.status - const error = params.error + const status = emptyToUndefined(params.status) + const error = emptyToUndefined(params.error) const events = parseJsonValue(params.events, 'events') if ( diff --git a/apps/sim/tools/langsmith/update_run.ts b/apps/sim/tools/langsmith/update_run.ts index 503d4b0acf9..7242ce0f3bf 100644 --- a/apps/sim/tools/langsmith/update_run.ts +++ b/apps/sim/tools/langsmith/update_run.ts @@ -91,7 +91,12 @@ export const langsmithUpdateRunTool: ToolConfig< events: params.events, } - return filterUndefined(payload) + const filtered = filterUndefined(payload) + if (Object.keys(filtered).length === 0) { + throw new Error('Provide at least one field to update') + } + + return filtered }, }, transformResponse: async (response, params) => { From 51322c8751b93244dcb0873174867502827d0e8f Mon Sep 17 00:00:00 2001 From: waleed Date: Thu, 2 Jul 2026 08:48:36 -0700 Subject: [PATCH 7/7] fix(langsmith): normalize empty-string patch fields at the tool layer too Direct/agent tool calls bypass the block's own emptyToUndefined guard, so update_run now normalizes blank name/end_time/status/error itself before filtering, matching the block-level fix. --- apps/sim/tools/langsmith/update_run.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/apps/sim/tools/langsmith/update_run.ts b/apps/sim/tools/langsmith/update_run.ts index 7242ce0f3bf..28fa4028601 100644 --- a/apps/sim/tools/langsmith/update_run.ts +++ b/apps/sim/tools/langsmith/update_run.ts @@ -80,14 +80,16 @@ export const langsmithUpdateRunTool: ToolConfig< 'Content-Type': 'application/json', }), body: (params) => { + const emptyToUndefined = (value?: string) => (value === '' ? undefined : value) + const payload: Record = { - name: params.name, - end_time: params.end_time, + name: emptyToUndefined(params.name), + end_time: emptyToUndefined(params.end_time), outputs: params.outputs, extra: params.extra, tags: params.tags, - status: params.status, - error: params.error, + status: emptyToUndefined(params.status), + error: emptyToUndefined(params.error), events: params.events, }