diff --git a/apps/sim/blocks/blocks/langsmith.ts b/apps/sim/blocks/blocks/langsmith.ts index cb8d20dff6b..028fe76c32c 100644 --- a/apps/sim/blocks/blocks/langsmith.ts +++ b/apps/sim/blocks/blocks/langsmith.ts @@ -23,6 +23,9 @@ export const LangsmithBlock: BlockConfig = { options: [ { label: 'Create Run', id: 'langsmith_create_run' }, { label: 'Create Runs Batch', id: 'langsmith_create_runs_batch' }, + { label: 'Update Run', id: 'langsmith_update_run' }, + { label: 'Get Run', id: 'langsmith_get_run' }, + { label: 'Create Feedback', id: 'langsmith_create_feedback' }, ], value: () => 'langsmith_create_run', }, @@ -41,13 +44,27 @@ export const LangsmithBlock: BlockConfig = { placeholder: 'Auto-generated if blank', condition: { field: 'operation', value: 'langsmith_create_run' }, }, + { + id: 'runId', + title: 'Run ID', + type: 'short-input', + placeholder: 'ID of the run to update, retrieve, or attach feedback to', + required: { + field: 'operation', + value: ['langsmith_update_run', 'langsmith_get_run', 'langsmith_create_feedback'], + }, + condition: { + field: 'operation', + value: ['langsmith_update_run', 'langsmith_get_run', 'langsmith_create_feedback'], + }, + }, { id: 'name', title: 'Name', type: 'short-input', placeholder: 'Run name', required: { field: 'operation', value: 'langsmith_create_run' }, - condition: { field: 'operation', value: 'langsmith_create_run' }, + condition: { field: 'operation', value: ['langsmith_create_run', 'langsmith_update_run'] }, }, { id: 'run_type', @@ -78,7 +95,7 @@ export const LangsmithBlock: BlockConfig = { title: 'End Time', type: 'short-input', placeholder: '2025-01-01T12:00:30Z', - condition: { field: 'operation', value: 'langsmith_create_run' }, + condition: { field: 'operation', value: ['langsmith_create_run', 'langsmith_update_run'] }, mode: 'advanced', }, { @@ -94,7 +111,7 @@ export const LangsmithBlock: BlockConfig = { title: 'Outputs', type: 'code', placeholder: '{"output":"value"}', - condition: { field: 'operation', value: 'langsmith_create_run' }, + condition: { field: 'operation', value: ['langsmith_create_run', 'langsmith_update_run'] }, mode: 'advanced', }, { @@ -102,7 +119,7 @@ export const LangsmithBlock: BlockConfig = { title: 'Metadata', type: 'code', placeholder: '{"ls_model":"gpt-4"}', - condition: { field: 'operation', value: 'langsmith_create_run' }, + condition: { field: 'operation', value: ['langsmith_create_run', 'langsmith_update_run'] }, mode: 'advanced', }, { @@ -110,7 +127,7 @@ export const LangsmithBlock: BlockConfig = { title: 'Tags', type: 'code', placeholder: '["production","workflow"]', - condition: { field: 'operation', value: 'langsmith_create_run' }, + condition: { field: 'operation', value: ['langsmith_create_run', 'langsmith_update_run'] }, mode: 'advanced', }, { @@ -150,7 +167,7 @@ export const LangsmithBlock: BlockConfig = { title: 'Status', type: 'short-input', placeholder: 'success', - condition: { field: 'operation', value: 'langsmith_create_run' }, + condition: { field: 'operation', value: ['langsmith_create_run', 'langsmith_update_run'] }, mode: 'advanced', }, { @@ -158,7 +175,7 @@ export const LangsmithBlock: BlockConfig = { title: 'Error', type: 'long-input', placeholder: 'Error message', - condition: { field: 'operation', value: 'langsmith_create_run' }, + condition: { field: 'operation', value: ['langsmith_create_run', 'langsmith_update_run'] }, mode: 'advanced', }, { @@ -174,7 +191,7 @@ export const LangsmithBlock: BlockConfig = { title: 'Events', type: 'code', placeholder: '[{"event":"token","value":1}]', - condition: { field: 'operation', value: 'langsmith_create_run' }, + condition: { field: 'operation', value: ['langsmith_create_run', 'langsmith_update_run'] }, mode: 'advanced', }, { @@ -207,9 +224,66 @@ Required: id (existing run UUID), name, run_type ("tool"|"chain"|"llm"|"retrieve Common patch fields: outputs, end_time, status, error`, }, }, + { + id: 'key', + title: 'Feedback Key', + type: 'short-input', + placeholder: 'e.g. correctness, user_score', + required: { field: 'operation', value: 'langsmith_create_feedback' }, + condition: { field: 'operation', value: 'langsmith_create_feedback' }, + }, + { + id: 'score', + title: 'Score', + type: 'short-input', + placeholder: 'e.g. 1, 0.5, 0', + condition: { field: 'operation', value: 'langsmith_create_feedback' }, + }, + { + id: 'value', + title: 'Value', + type: 'short-input', + placeholder: 'e.g. good, bad', + condition: { field: 'operation', value: 'langsmith_create_feedback' }, + mode: 'advanced', + }, + { + id: 'comment', + title: 'Comment', + type: 'long-input', + placeholder: 'Explanation for the feedback', + condition: { field: 'operation', value: 'langsmith_create_feedback' }, + mode: 'advanced', + }, + { + id: 'correction', + title: 'Correction', + type: 'code', + placeholder: '{"output":"the corrected value"}', + condition: { field: 'operation', value: 'langsmith_create_feedback' }, + mode: 'advanced', + }, + { + id: 'feedbackSourceType', + title: 'Feedback Source', + type: 'dropdown', + options: [ + { label: 'API', id: 'api' }, + { label: 'App', id: 'app' }, + { label: 'Model', id: 'model' }, + ], + condition: { field: 'operation', value: 'langsmith_create_feedback' }, + mode: 'advanced', + }, ], tools: { - access: ['langsmith_create_run', 'langsmith_create_runs_batch'], + access: [ + 'langsmith_create_run', + 'langsmith_create_runs_batch', + 'langsmith_update_run', + 'langsmith_get_run', + 'langsmith_create_feedback', + ], config: { tool: (params) => params.operation, params: (params) => { @@ -227,6 +301,8 @@ Common patch fields: outputs, end_time, status, error`, return value } + const emptyToUndefined = (value: unknown) => (value === '' ? undefined : value) + if (params.operation === 'langsmith_create_runs_batch') { const post = parseJsonValue(params.post, 'post runs') const patch = parseJsonValue(params.patch, 'patch runs') @@ -242,6 +318,69 @@ Common patch fields: outputs, end_time, status, error`, } } + if (params.operation === 'langsmith_update_run') { + const name = emptyToUndefined(params.name) + const end_time = emptyToUndefined(params.end_time) + const outputs = parseJsonValue(params.outputs, 'outputs') + const extra = parseJsonValue(params.extra, 'metadata') + const tags = parseJsonValue(params.tags, 'tags') + const status = emptyToUndefined(params.status) + const error = emptyToUndefined(params.error) + const events = parseJsonValue(params.events, 'events') + + if ( + [name, end_time, outputs, extra, tags, status, error, events].every( + (value) => value === undefined + ) + ) { + throw new Error('Provide at least one field to update') + } + + return { + apiKey: params.apiKey, + runId: params.runId, + name, + end_time, + outputs, + extra, + tags, + status, + error, + events, + } + } + + if (params.operation === 'langsmith_get_run') { + return { + apiKey: params.apiKey, + runId: params.runId, + } + } + + if (params.operation === 'langsmith_create_feedback') { + const parseScore = (value: unknown) => { + if (value === undefined || value === null || value === '') { + return undefined + } + const parsed = Number(value) + if (Number.isNaN(parsed)) { + throw new Error(`Invalid score: "${value}" is not a number`) + } + return parsed + } + + return { + apiKey: params.apiKey, + runId: params.runId, + key: params.key, + score: parseScore(params.score), + value: params.value, + comment: params.comment, + correction: parseJsonValue(params.correction, 'correction'), + feedbackSourceType: params.feedbackSourceType || undefined, + } + } + return { apiKey: params.apiKey, id: params.id, @@ -269,6 +408,10 @@ Common patch fields: outputs, end_time, status, error`, operation: { type: 'string', description: 'Operation to perform' }, apiKey: { type: 'string', description: 'LangSmith API key' }, id: { type: 'string', description: 'Run identifier' }, + runId: { + type: 'string', + description: 'ID of the run to update, retrieve, or attach feedback to', + }, name: { type: 'string', description: 'Run name' }, run_type: { type: 'string', description: 'Run type' }, start_time: { type: 'string', description: 'Run start time (ISO)' }, @@ -287,13 +430,45 @@ Common patch fields: outputs, end_time, status, error`, events: { type: 'json', description: 'Events array' }, post: { type: 'json', description: 'Runs to ingest in batch' }, patch: { type: 'json', description: 'Runs to update in batch' }, + key: { type: 'string', description: 'Feedback metric name' }, + score: { type: 'string', description: 'Numeric score for the feedback metric' }, + value: { type: 'string', description: 'Categorical value for the feedback metric' }, + comment: { type: 'string', description: 'Comment explaining the feedback' }, + correction: { type: 'json', description: 'Corrected output for the run' }, + feedbackSourceType: { + type: 'string', + description: 'Origin of the feedback (api, app, or model)', + }, }, outputs: { - accepted: { type: 'boolean', description: 'Whether ingestion was accepted' }, - runId: { type: 'string', description: 'Run ID for single run' }, + accepted: { type: 'boolean', description: 'Whether ingestion or the update was accepted' }, + runId: { type: 'string', description: 'Run ID for single-run operations' }, runIds: { type: 'array', description: 'Run IDs for batch ingest' }, message: { type: 'string', description: 'LangSmith response message' }, messages: { type: 'array', description: 'Per-run response messages' }, + id: { type: 'string', description: 'Run ID (get run) or feedback ID (create feedback)' }, + name: { type: 'string', description: 'Run name (get run)' }, + runType: { type: 'string', description: 'Run type (get run)' }, + status: { type: 'string', description: 'Run status (get run)' }, + startTime: { type: 'string', description: 'Run start time (get run)' }, + endTime: { type: 'string', description: 'Run end time (get run)' }, + inputs: { type: 'json', description: 'Run inputs payload (get run)' }, + outputs: { type: 'json', description: 'Run outputs payload (get run)' }, + error: { type: 'string', description: 'Error details (get run)' }, + tags: { type: 'array', description: 'Tags attached to the run (get run)' }, + sessionId: { type: 'string', description: 'Project (session) ID the run belongs to (get run)' }, + traceId: { type: 'string', description: 'Trace ID (get run)' }, + parentRunId: { type: 'string', description: 'Parent run ID (get run)' }, + totalTokens: { type: 'number', description: 'Total tokens consumed by the run (get run)' }, + totalCost: { type: 'string', description: 'Total cost of the run (get run)' }, + key: { type: 'string', description: 'Feedback metric name (create feedback)' }, + score: { type: 'number', description: 'Score recorded for the feedback (create feedback)' }, + value: { + type: 'string', + description: 'Categorical value recorded for the feedback (create feedback)', + }, + comment: { type: 'string', description: 'Comment recorded for the feedback (create feedback)' }, + createdAt: { type: 'string', description: 'When the feedback was created (create feedback)' }, }, } @@ -324,11 +499,20 @@ export const LangsmithBlockMeta = { icon: LangsmithIcon, title: 'LangSmith feedback capture', prompt: - 'Build a workflow that collects user-reported agent failures from a table and forwards each as a tagged LangSmith run with the inputs and expected output for later review.', + 'Build a workflow that collects user-reported agent failures from a table and attaches each as scored LangSmith feedback on the originating run for later review.', modules: ['tables', 'agent', 'workflows'], category: 'engineering', tags: ['engineering', 'automation'], }, + { + icon: LangsmithIcon, + title: 'LangSmith run completion', + prompt: + 'Build a workflow that creates a LangSmith run when an agent step starts, then updates it with outputs, status, and end time once the step finishes so traces always show the full lifecycle.', + modules: ['agent', 'workflows'], + category: 'engineering', + tags: ['engineering', 'monitoring'], + }, { icon: LangsmithIcon, title: 'LangSmith batch run shipper', @@ -381,5 +565,12 @@ export const LangsmithBlockMeta = { content: '# Batch Export Runs\n\nShip multiple completed runs to LangSmith at once instead of one by one.\n\n## Steps\n1. Collect the runs to export, each with name, type, inputs, outputs, and timing.\n2. Assign a shared project so the runs land together.\n3. Submit them as a single batch.\n\n## Output\nReturn how many runs were exported, the project they landed in, and any runs that failed validation.', }, + { + name: 'attach-feedback-to-run', + description: + 'Attach a score, categorical value, or correction to an existing LangSmith run for evaluation.', + content: + '# Attach Feedback to a Run\n\nRecord a human or automated judgment on a run that already exists in LangSmith.\n\n## Steps\n1. Identify the run ID the feedback applies to.\n2. Choose a feedback key (e.g. "correctness", "user_score") and a score, value, or comment.\n3. Include a correction if the expected output is known.\n4. Submit the feedback.\n\n## Output\nConfirm the feedback ID and the run it was attached to.', + }, ], } as const satisfies BlockMeta diff --git a/apps/sim/tools/langsmith/create_feedback.ts b/apps/sim/tools/langsmith/create_feedback.ts new file mode 100644 index 00000000000..c1585bc0f48 --- /dev/null +++ b/apps/sim/tools/langsmith/create_feedback.ts @@ -0,0 +1,133 @@ +import { generateId } from '@sim/utils/id' +import { filterUndefined } from '@sim/utils/object' +import type { + LangsmithCreateFeedbackParams, + LangsmithCreateFeedbackResponse, +} from '@/tools/langsmith/types' +import type { ToolConfig } from '@/tools/types' + +export const langsmithCreateFeedbackTool: ToolConfig< + LangsmithCreateFeedbackParams, + LangsmithCreateFeedbackResponse +> = { + id: 'langsmith_create_feedback', + name: 'LangSmith Create Feedback', + description: 'Attach a score, correction, or comment to a LangSmith run.', + version: '1.0.0', + params: { + apiKey: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'LangSmith API key', + }, + runId: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'ID of the run to attach feedback to', + }, + key: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Feedback metric name (e.g. "correctness", "user_score")', + }, + score: { + type: 'number', + required: false, + visibility: 'user-or-llm', + description: 'Numeric score for the feedback metric', + }, + value: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Categorical value for the feedback metric', + }, + comment: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Free-text comment explaining the feedback', + }, + correction: { + type: 'json', + required: false, + visibility: 'user-or-llm', + description: 'Corrected output for the run', + }, + feedbackSourceType: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Origin of the feedback (api, app, or model)', + }, + }, + request: { + url: () => 'https://api.smith.langchain.com/feedback', + method: 'POST', + headers: (params) => ({ + 'X-Api-Key': params.apiKey, + 'Content-Type': 'application/json', + }), + body: (params) => { + const payload: Record = { + id: generateId(), + run_id: params.runId.trim(), + key: params.key, + score: params.score, + value: params.value, + comment: params.comment, + correction: params.correction, + feedback_source: params.feedbackSourceType + ? { type: params.feedbackSourceType } + : undefined, + } + + return filterUndefined(payload) + }, + }, + transformResponse: async (response) => { + if (!response.ok) { + const errorText = await response.text() + throw new Error(`LangSmith create feedback failed (${response.status}): ${errorText}`) + } + + const data = (await response.json()) as Record + + return { + success: true, + output: { + id: data.id as string, + key: data.key as string, + runId: (data.run_id as string) ?? null, + score: (data.score as number) ?? null, + value: (data.value as string | number | boolean) ?? null, + comment: (data.comment as string) ?? null, + createdAt: (data.created_at as string) ?? null, + }, + } + }, + outputs: { + id: { type: 'string', description: 'Feedback ID' }, + key: { type: 'string', description: 'Feedback metric name' }, + runId: { + type: 'string', + description: 'ID of the run the feedback was attached to', + optional: true, + }, + score: { type: 'number', description: 'Score recorded for the feedback', optional: true }, + value: { + type: 'string', + description: 'Categorical value recorded for the feedback', + optional: true, + }, + comment: { type: 'string', description: 'Comment recorded for the feedback', optional: true }, + createdAt: { + type: 'string', + description: 'When the feedback was created (ISO)', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/langsmith/create_run.ts b/apps/sim/tools/langsmith/create_run.ts index 510757ac23c..0132dfdca2f 100644 --- a/apps/sim/tools/langsmith/create_run.ts +++ b/apps/sim/tools/langsmith/create_run.ts @@ -1,3 +1,4 @@ +import { filterUndefined } from '@sim/utils/object' import type { LangsmithCreateRunParams, LangsmithCreateRunResponse } from '@/tools/langsmith/types' import { normalizeLangsmithRunPayload } from '@/tools/langsmith/utils' import type { ToolConfig } from '@/tools/types' @@ -141,9 +142,7 @@ export const langsmithCreateRunTool: ToolConfig< events: params.events, } - return Object.fromEntries( - Object.entries(normalizedPayload).filter(([, value]) => value !== undefined) - ) + return filterUndefined(normalizedPayload) }, }, transformResponse: async (response, params) => { diff --git a/apps/sim/tools/langsmith/get_run.ts b/apps/sim/tools/langsmith/get_run.ts new file mode 100644 index 00000000000..22582859ffb --- /dev/null +++ b/apps/sim/tools/langsmith/get_run.ts @@ -0,0 +1,92 @@ +import type { LangsmithGetRunParams, LangsmithGetRunResponse } from '@/tools/langsmith/types' +import type { ToolConfig } from '@/tools/types' + +export const langsmithGetRunTool: ToolConfig = { + id: 'langsmith_get_run', + name: 'LangSmith Get Run', + description: 'Retrieve a single LangSmith run by ID.', + version: '1.0.0', + params: { + apiKey: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'LangSmith API key', + }, + runId: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'ID of the run to retrieve', + }, + }, + request: { + url: (params) => `https://api.smith.langchain.com/runs/${params.runId.trim()}`, + method: 'GET', + headers: (params) => ({ + 'X-Api-Key': params.apiKey, + }), + }, + transformResponse: async (response) => { + if (!response.ok) { + const errorText = await response.text() + throw new Error(`LangSmith get run failed (${response.status}): ${errorText}`) + } + + const data = (await response.json()) as Record + + return { + success: true, + output: { + id: data.id as string, + runId: data.id as string, + name: data.name as string, + runType: data.run_type as string, + status: (data.status as string) ?? null, + startTime: (data.start_time as string) ?? null, + endTime: (data.end_time as string) ?? null, + inputs: (data.inputs as Record) ?? null, + outputs: (data.outputs as Record) ?? null, + error: (data.error as string) ?? null, + tags: (data.tags as string[]) ?? [], + sessionId: (data.session_id as string) ?? null, + traceId: (data.trace_id as string) ?? null, + parentRunId: (data.parent_run_id as string) ?? null, + totalTokens: (data.total_tokens as number) ?? null, + totalCost: (data.total_cost as string) ?? null, + }, + } + }, + outputs: { + id: { type: 'string', description: 'Run ID' }, + runId: { + type: 'string', + description: 'Run ID (alias of id, for consistency with other operations)', + }, + name: { type: 'string', description: 'Run name' }, + runType: { + type: 'string', + description: 'Run type (tool, chain, llm, retriever, embedding, prompt, parser)', + }, + status: { type: 'string', description: 'Run status', optional: true }, + startTime: { type: 'string', description: 'Run start time (ISO)', optional: true }, + endTime: { type: 'string', description: 'Run end time (ISO)', optional: true }, + inputs: { type: 'json', description: 'Run inputs payload', optional: true }, + outputs: { type: 'json', description: 'Run outputs payload', optional: true }, + error: { type: 'string', description: 'Error details, if the run failed', optional: true }, + tags: { type: 'array', description: 'Tags attached to the run', items: { type: 'string' } }, + sessionId: { + type: 'string', + description: 'Project (session) ID the run belongs to', + optional: true, + }, + traceId: { type: 'string', description: 'Trace ID', optional: true }, + parentRunId: { type: 'string', description: 'Parent run ID', optional: true }, + totalTokens: { + type: 'number', + description: 'Total tokens consumed by the run', + optional: true, + }, + totalCost: { type: 'string', description: 'Total cost of the run', optional: true }, + }, +} diff --git a/apps/sim/tools/langsmith/index.ts b/apps/sim/tools/langsmith/index.ts index f173b10a217..5f7e6019a8c 100644 --- a/apps/sim/tools/langsmith/index.ts +++ b/apps/sim/tools/langsmith/index.ts @@ -1,2 +1,5 @@ +export { langsmithCreateFeedbackTool } from '@/tools/langsmith/create_feedback' export { langsmithCreateRunTool } from '@/tools/langsmith/create_run' export { langsmithCreateRunsBatchTool } from '@/tools/langsmith/create_runs_batch' +export { langsmithGetRunTool } from '@/tools/langsmith/get_run' +export { langsmithUpdateRunTool } from '@/tools/langsmith/update_run' diff --git a/apps/sim/tools/langsmith/types.ts b/apps/sim/tools/langsmith/types.ts index 860a72e303f..ff289163391 100644 --- a/apps/sim/tools/langsmith/types.ts +++ b/apps/sim/tools/langsmith/types.ts @@ -57,4 +57,81 @@ export interface LangsmithCreateRunsBatchResponse extends ToolResponse { } } -export type LangsmithResponse = LangsmithCreateRunResponse | LangsmithCreateRunsBatchResponse +export interface LangsmithUpdateRunParams { + apiKey: string + runId: string + name?: string + end_time?: string + outputs?: Record + extra?: Record + tags?: string[] + status?: string + error?: string + events?: Record[] +} + +export interface LangsmithUpdateRunResponse extends ToolResponse { + output: { + accepted: boolean + runId: string + message: string | null + } +} + +export interface LangsmithGetRunParams { + apiKey: string + runId: string +} + +export interface LangsmithGetRunResponse extends ToolResponse { + output: { + id: string + runId: string + name: string + runType: string + status: string | null + startTime: string | null + endTime: string | null + inputs: Record | null + outputs: Record | null + error: string | null + tags: string[] + sessionId: string | null + traceId: string | null + parentRunId: string | null + totalTokens: number | null + totalCost: string | null + } +} + +export type LangsmithFeedbackSourceType = 'api' | 'app' | 'model' + +export interface LangsmithCreateFeedbackParams { + apiKey: string + runId: string + key: string + score?: number + value?: string + comment?: string + correction?: Record + feedbackSourceType?: LangsmithFeedbackSourceType +} + +export interface LangsmithCreateFeedbackResponse extends ToolResponse { + output: { + id: string + key: string + runId: string | null + score: number | null + value: string | number | boolean | null + comment: string | null + createdAt: string | null + } +} + +export type LangsmithResponse = + | LangsmithCreateRunResponse + | LangsmithCreateRunsBatchResponse + | LangsmithUpdateRunResponse + | LangsmithGetRunResponse + | LangsmithCreateFeedbackResponse diff --git a/apps/sim/tools/langsmith/update_run.ts b/apps/sim/tools/langsmith/update_run.ts new file mode 100644 index 00000000000..28fa4028601 --- /dev/null +++ b/apps/sim/tools/langsmith/update_run.ts @@ -0,0 +1,145 @@ +import { filterUndefined } from '@sim/utils/object' +import type { LangsmithUpdateRunParams, LangsmithUpdateRunResponse } from '@/tools/langsmith/types' +import type { ToolConfig } from '@/tools/types' + +export const langsmithUpdateRunTool: ToolConfig< + LangsmithUpdateRunParams, + LangsmithUpdateRunResponse +> = { + id: 'langsmith_update_run', + name: 'LangSmith Update Run', + description: 'Patch an existing LangSmith run with outputs, status, or timing once it completes.', + version: '1.0.0', + params: { + apiKey: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'LangSmith API key', + }, + runId: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'ID of the run to update', + }, + name: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Corrected run name', + }, + end_time: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Run end time in ISO-8601 format', + }, + outputs: { + type: 'json', + required: false, + visibility: 'user-or-llm', + description: 'Outputs payload', + }, + extra: { + type: 'json', + required: false, + visibility: 'user-or-llm', + description: 'Additional metadata (extra)', + }, + tags: { + type: 'json', + required: false, + visibility: 'user-or-llm', + description: 'Array of tag strings', + }, + status: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Run status', + }, + error: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Error details', + }, + events: { + type: 'json', + required: false, + visibility: 'user-or-llm', + description: 'Structured events array', + }, + }, + request: { + url: (params) => `https://api.smith.langchain.com/runs/${params.runId.trim()}`, + method: 'PATCH', + headers: (params) => ({ + 'X-Api-Key': params.apiKey, + 'Content-Type': 'application/json', + }), + body: (params) => { + const emptyToUndefined = (value?: string) => (value === '' ? undefined : value) + + const payload: Record = { + name: emptyToUndefined(params.name), + end_time: emptyToUndefined(params.end_time), + outputs: params.outputs, + extra: params.extra, + tags: params.tags, + status: emptyToUndefined(params.status), + error: emptyToUndefined(params.error), + events: params.events, + } + + const filtered = filterUndefined(payload) + if (Object.keys(filtered).length === 0) { + throw new Error('Provide at least one field to update') + } + + return filtered + }, + }, + transformResponse: async (response, params) => { + if (!response.ok) { + const errorText = await response.text() + throw new Error(`LangSmith update run failed (${response.status}): ${errorText}`) + } + + const responseText = await response.text() + let message: string | null = null + if (responseText) { + try { + const data = JSON.parse(responseText) as Record + message = typeof data.message === 'string' ? data.message : null + } catch { + // Response body isn't JSON (e.g. empty object or plain text) — no message to surface + } + } + + return { + success: true, + output: { + accepted: true, + runId: params?.runId.trim() ?? '', + message, + }, + } + }, + outputs: { + accepted: { + type: 'boolean', + description: 'Whether the run update was accepted', + }, + runId: { + type: 'string', + description: 'ID of the run that was updated', + }, + message: { + type: 'string', + description: 'Response message from LangSmith, if provided', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/registry.ts b/apps/sim/tools/registry.ts index 2bb910c4eef..c9121640267 100644 --- a/apps/sim/tools/registry.ts +++ b/apps/sim/tools/registry.ts @@ -1905,7 +1905,13 @@ import { knowledgeUploadChunkTool, knowledgeUpsertDocumentTool, } from '@/tools/knowledge' -import { langsmithCreateRunsBatchTool, langsmithCreateRunTool } from '@/tools/langsmith' +import { + langsmithCreateFeedbackTool, + langsmithCreateRunsBatchTool, + langsmithCreateRunTool, + langsmithGetRunTool, + langsmithUpdateRunTool, +} from '@/tools/langsmith' import { latexCompileTool, latexGetPackageTool, @@ -7222,6 +7228,9 @@ export const tools: Record = { linear_list_project_statuses: linearListProjectStatusesTool, langsmith_create_run: langsmithCreateRunTool, langsmith_create_runs_batch: langsmithCreateRunsBatchTool, + langsmith_update_run: langsmithUpdateRunTool, + langsmith_get_run: langsmithGetRunTool, + langsmith_create_feedback: langsmithCreateFeedbackTool, latex_compile: latexCompileTool, latex_get_package: latexGetPackageTool, latex_list_fonts: latexListFontsTool,