diff --git a/.gitignore b/.gitignore index a90e1c82d..930b3e1e6 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,5 @@ service-account*.json # Local Cursor scratch (committed rules under .cursor/rules/ stay tracked) .cursor/notes/ .cursor/plans/ + +.vscode diff --git a/packages/trueforge-core/tests/orchestration/helpers/helpers.ts b/packages/trueforge-core/tests/orchestration/helpers/helpers.ts new file mode 100644 index 000000000..e4d40d1af --- /dev/null +++ b/packages/trueforge-core/tests/orchestration/helpers/helpers.ts @@ -0,0 +1,212 @@ +import type { ILLM } from '../../../src/core/llm/ILLM'; +import type { ExtendedChatCompletionChunk, RawAssistantMessageWithUsage } from '../../../src/core/llm/LLMTypes'; +import { getEmptyUsage } from '../../../src/core/llm/LLMTypes'; +import type { IToolSet, ToolSource } from '../../../src/core/mcp/IMCPServer'; +import { toolResultResponse } from '../../../src/core/mcp/IMCPServer'; +import { ToolSet } from '../../../src/core/mcp/ToolSet'; +import type { + AgentThreadExecutionEvent, + AgentThreadExecutionResult, + AgentThreadSendBatch, +} from '../../../src/core/runtime/AgentThread.types'; +import type { AgentThreadOrchestrator } from '../../../src/core/runtime/AgentThreadOrchestrator'; + +export const WRITE_NOTE_TOOL_NAME = 'write_note'; +export const WRITE_NOTE_CALL_ID = 'call-write'; +export const WRITE_NOTE_ARGUMENTS = JSON.stringify({ text: 'hello' }); +export const WRITE_NOTE_RESULT = 'note written'; + +/** One streamed chunk plus a stop completion. Used when the test needs a text reply and no tool calls. */ +// eslint-disable-next-line @typescript-eslint/require-await -- async generator fixture, not awaiting I/O +export async function* textReplyStream( + text: string, +): AsyncGenerator { + yield { + id: 'chunk-text', + object: 'chat.completion.chunk', + created: 0, + model: 'test-model', + choices: [{ index: 0, delta: { role: 'assistant', content: text }, finish_reason: 'stop' }], + }; + return { + output: { role: 'assistant', content: text }, + usage: getEmptyUsage(), + finish_reason: 'stop', + }; +} + +// eslint-disable-next-line @typescript-eslint/require-await -- async generator fixture, not awaiting I/O +export async function* createSubAgentStream() { + yield { + id: 'chunk-tool', + object: 'chat.completion.chunk', + created: 0, + model: 'test-model', + choices: [ + { + index: 0, + delta: { + role: 'assistant', + tool_calls: [ + { + index: 0, + id: 'call-sub', + type: 'function', + function: { + name: 'create_sub_agent', + arguments: JSON.stringify({ name: 'worker', input: 'do the delegated task' }), + }, + }, + ], + }, + finish_reason: 'tool_calls', + }, + ], + }; + + return { + output: { + role: 'assistant', + content: null, + tool_calls: [ + { + id: 'call-sub', + type: 'function', + function: { + name: 'create_sub_agent', + arguments: JSON.stringify({ name: 'worker', input: 'do the delegated task' }), + }, + }, + ], + }, + usage: getEmptyUsage(), + finish_reason: 'tool_calls', + }; +} + +// eslint-disable-next-line @typescript-eslint/require-await -- async generator fixture, not awaiting I/O +export async function* writeNoteToolCallStream() { + yield { + id: 'chunk-write-note', + object: 'chat.completion.chunk', + created: 0, + model: 'test-model', + choices: [ + { + index: 0, + delta: { + role: 'assistant', + tool_calls: [ + { + index: 0, + id: WRITE_NOTE_CALL_ID, + type: 'function', + function: { + name: WRITE_NOTE_TOOL_NAME, + arguments: WRITE_NOTE_ARGUMENTS, + }, + }, + ], + }, + finish_reason: 'tool_calls', + }, + ], + }; + + return { + output: { + role: 'assistant', + content: null, + tool_calls: [ + { + id: WRITE_NOTE_CALL_ID, + type: 'function', + function: { + name: WRITE_NOTE_TOOL_NAME, + arguments: WRITE_NOTE_ARGUMENTS, + }, + }, + ], + }, + usage: getEmptyUsage(), + finish_reason: 'tool_calls', + }; +} + +function makeWriteNoteSource(callTool: ToolSource['callTool']): ToolSource { + return { + name: 'notes', + id: 'notes', + listTools: () => + Promise.resolve({ + result: { + tools: [ + { + name: WRITE_NOTE_TOOL_NAME, + description: 'Write a note', + inputSchema: { + type: 'object', + properties: { text: { type: 'string' } }, + }, + preload: true, + }, + ], + }, + wasInitialized: undefined, + }), + callTool, + toolCallInfo: () => + Promise.resolve({ + type: 'mcp', + mcp_server_id: 'notes', + mcp_server_name: 'notes', + original_tool_name: WRITE_NOTE_TOOL_NAME, + }), + }; +} + +/** Approval-gated write_note tool set; `callTool` spy proves allow runs the source and deny does not. */ +export function makeApprovalGatedWriteNoteToolSet(): { + toolSet: IToolSet; + callTool: jest.Mock; +} { + const callTool = jest.fn(() => Promise.resolve(toolResultResponse({ text: WRITE_NOTE_RESULT }))); + return { + toolSet: new ToolSet({ + source: makeWriteNoteSource(callTool), + selectors: { + enableTools: ['@all'], + disableTools: [], + preloadTools: [], + requireApprovalForTools: [WRITE_NOTE_TOOL_NAME], + }, + preload: true, + }), + callTool, + }; +} + +/** Consume send() then execute(); return raw events and the generator result. */ +export async function runTurn(input: { + orchestrator: AgentThreadOrchestrator; + sendBatch: AgentThreadSendBatch; + signal?: AbortSignal | undefined; +}): Promise<{ events: AgentThreadExecutionEvent[]; result: AgentThreadExecutionResult }> { + for await (const _event of input.orchestrator.send(input.sendBatch)) { + void _event; + } + const events: AgentThreadExecutionEvent[] = []; + const iterator = input.orchestrator.execute({ + signal: input.signal ?? new AbortController().signal, + }); + let step = await iterator.next(); + while (!step.done) { + events.push(step.value); + step = await iterator.next(); + } + return { events, result: step.value }; +} + +export function llmCreateInputs(llm: ILLM): unknown[] { + return jest.mocked(llm).create.mock.calls.map(call => call[0]); +} diff --git a/packages/trueforge-core/tests/orchestration/orchestration.test.ts b/packages/trueforge-core/tests/orchestration/orchestration.test.ts new file mode 100644 index 000000000..3dceffddb --- /dev/null +++ b/packages/trueforge-core/tests/orchestration/orchestration.test.ts @@ -0,0 +1,88 @@ +/** One root thread, no tools: user message in, text reply out. */ +import { EventType } from '../../src/core/events/schema'; +import { AgentThread } from '../../src/core/runtime/AgentThread'; +import { InternalEventType } from '../../src/core/runtime/AgentThread.types'; +import { AgentThreadOrchestrator } from '../../src/core/runtime/AgentThreadOrchestrator'; +import { NOOP_AGENT_TRACING } from '../../src/core/tracing/NoopAgentTracing'; +import { makeSilentLogger } from '../core/harnessMocks'; +import { llmCreateInputs, runTurn, textReplyStream } from './helpers/helpers'; + +const THREAD_ID = 'main'; +const REPLY = 'hello from the mocked model'; +const INSTRUCTION = 'You are running in a test setup.'; + +const EXPECTED_EVENTS = [ + { type: EventType.MODEL_MESSAGE, thread_id: THREAD_ID }, + { type: EventType.MODEL_MESSAGE_DELTA, thread_id: THREAD_ID, content: REPLY }, + { + type: InternalEventType.AGENT_CONTEXT_APPEND, + thread_id: THREAD_ID, + context: [{ role: 'assistant', content: REPLY }], + }, + { type: InternalEventType.AGENT_DONE, thread_id: THREAD_ID, status: 'done' }, +]; + +const OUTPUT = { + output: { thread_id: THREAD_ID, content: REPLY }, + required_actions: [], +}; + +const EXPECTED_LLM_INPUT = [ + { + stream: true, + messages: [ + { role: 'system', content: expect.stringContaining(INSTRUCTION) }, + { role: 'user', content: 'hello' }, + ], + }, +]; + +describe('orchestration: mocked LLM and no tools', () => { + it('sends a user message and finishes the thread with a text reply', async () => { + const thread = new AgentThread({ + definition: { + modelClient: { + create: jest.fn().mockImplementation(() => textReplyStream(REPLY)), + createNonStream: jest.fn().mockImplementation(() => textReplyStream(REPLY)), + }, + instruction: INSTRUCTION, + messages: undefined, + modelParams: undefined, + responseFormat: undefined, + iterationLimit: undefined, + toolSets: undefined, + }, + threadId: THREAD_ID, + title: 'orchestration', + parent: undefined, + agentInfo: undefined, + context: undefined, + currentContextUsage: undefined, + preComputedCompletion: undefined, + sandbox: undefined, + capabilities: undefined, + capabilityState: undefined, + tracing: NOOP_AGENT_TRACING, + logger: makeSilentLogger(), + }); + + // Orchestrator owns the thread map and fans send/execute across live threads. + // This case has only the root thread, so sub-agent creation must never run. + const orchestrator = new AgentThreadOrchestrator({ + agentThreads: new Map([[thread.threadId, thread]]), + createDynamicSubAgentThread: () => Promise.reject(new Error('unexpected sub-agent in no-tool test')), + tracing: NOOP_AGENT_TRACING, + logger: makeSilentLogger(), + }); + + const { events, result } = await runTurn({ + orchestrator, + sendBatch: [{ type: EventType.USER_MESSAGE, content: 'hello' }], + }); + + expect(events).toMatchObject(EXPECTED_EVENTS); + expect(result).toMatchObject(OUTPUT); + expect(result.root_agent_error).toBeUndefined(); + expect(llmCreateInputs(thread.definition.modelClient)).toMatchObject(EXPECTED_LLM_INPUT); + }); +}); diff --git a/packages/trueforge-core/tests/orchestration/orchestrationApproval.test.ts b/packages/trueforge-core/tests/orchestration/orchestrationApproval.test.ts new file mode 100644 index 000000000..9260b2524 --- /dev/null +++ b/packages/trueforge-core/tests/orchestration/orchestrationApproval.test.ts @@ -0,0 +1,293 @@ +/** Pause on write_note approval, then resume after allow or deny. */ +import { EventType } from '../../src/core/events/schema'; +import { AgentThread } from '../../src/core/runtime/AgentThread'; +import { InternalEventType, type AgentThreadConstructorInput } from '../../src/core/runtime/AgentThread.types'; +import { AgentThreadOrchestrator } from '../../src/core/runtime/AgentThreadOrchestrator'; +import { NOOP_AGENT_TRACING } from '../../src/core/tracing/NoopAgentTracing'; +import { makeSilentLogger } from '../core/harnessMocks'; +import { + llmCreateInputs, + makeApprovalGatedWriteNoteToolSet, + runTurn, + textReplyStream, + WRITE_NOTE_ARGUMENTS, + WRITE_NOTE_CALL_ID, + WRITE_NOTE_RESULT, + WRITE_NOTE_TOOL_NAME, + writeNoteToolCallStream, +} from './helpers/helpers'; + +const ROOT_ID = 'thread_root'; +const DENY_REASON = 'not allowed in this test'; +/** ToolSet deny → isError path wraps the text payload again for context. */ +const DENY_TOOL_CONTENT = JSON.stringify({ + error: [{ type: 'text', text: JSON.stringify({ error: `User denied tool call: ${DENY_REASON}` }) }], +}); +const INSTRUCTION = 'You are running in a test setup.'; + +const WRITE_NOTE_TOOLS = [{ function: { name: WRITE_NOTE_TOOL_NAME } }]; + +const EXPECTED_TURN_1_EVENTS = [ + { type: EventType.MODEL_MESSAGE, thread_id: ROOT_ID }, + { type: EventType.MODEL_MESSAGE_DELTA, thread_id: ROOT_ID }, + { + type: InternalEventType.AGENT_CONTEXT_APPEND, + thread_id: ROOT_ID, + context: [ + { + role: 'assistant', + content: null, + tool_calls: [ + { + id: WRITE_NOTE_CALL_ID, + type: 'function', + function: { name: WRITE_NOTE_TOOL_NAME, arguments: WRITE_NOTE_ARGUMENTS }, + }, + ], + }, + ], + }, + { + type: EventType.TOOL_APPROVAL_REQUIRED, + thread_id: ROOT_ID, + tool_calls: [{ id: WRITE_NOTE_CALL_ID }], + }, +]; + +const TURN_1_OUTPUT = { + output: null, + required_actions: [ + { + type: EventType.TOOL_APPROVAL_REQUIRED, + thread_id: ROOT_ID, + tool_calls: [{ id: WRITE_NOTE_CALL_ID }], + }, + ], +}; + +const EXPECTED_TURN_1_INPUT = [ + { + tools: WRITE_NOTE_TOOLS, + messages: [ + { role: 'system', content: expect.stringContaining(INSTRUCTION) }, + { role: 'user', content: 'hello' }, + ], + }, +]; + +describe('orchestration: pause then resume on tool approval', () => { + describe('allow', () => { + const ROOT_FINAL = 'note saved'; + + const EXPECTED_TURN_2_EVENTS = [ + { type: EventType.TOOL_RESPONSE, thread_id: ROOT_ID, tool_call_id: WRITE_NOTE_CALL_ID }, + { + type: InternalEventType.AGENT_CONTEXT_APPEND, + thread_id: ROOT_ID, + context: [{ role: 'tool', tool_call_id: WRITE_NOTE_CALL_ID, content: WRITE_NOTE_RESULT }], + }, + { type: EventType.MODEL_MESSAGE, thread_id: ROOT_ID }, + { type: EventType.MODEL_MESSAGE_DELTA, thread_id: ROOT_ID, content: ROOT_FINAL }, + { + type: InternalEventType.AGENT_CONTEXT_APPEND, + thread_id: ROOT_ID, + context: [{ role: 'assistant', content: ROOT_FINAL }], + }, + { type: InternalEventType.AGENT_DONE, thread_id: ROOT_ID, status: 'done' }, + ]; + + const TURN_2_OUTPUT = { + output: { thread_id: ROOT_ID, content: ROOT_FINAL }, + required_actions: [], + }; + + const EXPECTED_TURN_2_INPUT = { + tools: WRITE_NOTE_TOOLS, + messages: [ + { role: 'system', content: expect.stringContaining(INSTRUCTION) }, + { role: 'user', content: 'hello' }, + { + role: 'assistant', + content: null, + tool_calls: [ + { + id: WRITE_NOTE_CALL_ID, + type: 'function', + function: { name: WRITE_NOTE_TOOL_NAME, arguments: WRITE_NOTE_ARGUMENTS }, + }, + ], + }, + { role: 'tool', tool_call_id: WRITE_NOTE_CALL_ID, content: WRITE_NOTE_RESULT }, + ], + }; + + it('pauses for write_note approval, then finishes after allow', async () => { + const { orchestrator, thread, callTool } = makeApprovalHarness(ROOT_FINAL); + + const paused = await runTurn({ + orchestrator, + sendBatch: [{ type: EventType.USER_MESSAGE, content: 'hello' }], + }); + expect(paused.events).toMatchObject(EXPECTED_TURN_1_EVENTS); + expect(paused.result).toMatchObject(TURN_1_OUTPUT); + expect(paused.result.root_agent_error).toBeUndefined(); + expect(llmCreateInputs(thread.definition.modelClient)).toMatchObject(EXPECTED_TURN_1_INPUT); + expect(callTool).not.toHaveBeenCalled(); + + const resumed = await runTurn({ + orchestrator, + sendBatch: [ + { + type: EventType.USER_TOOL_APPROVAL, + thread_id: ROOT_ID, + tool_call_id: WRITE_NOTE_CALL_ID, + approval: { status: 'allow' }, + }, + ], + }); + expect(resumed.events).toMatchObject(EXPECTED_TURN_2_EVENTS); + expect(resumed.result).toMatchObject(TURN_2_OUTPUT); + expect(resumed.result.root_agent_error).toBeUndefined(); + expect(callTool).toHaveBeenCalledTimes(1); + expect(llmCreateInputs(thread.definition.modelClient)).toMatchObject([ + ...EXPECTED_TURN_1_INPUT, + EXPECTED_TURN_2_INPUT, + ]); + }); + }); + + describe('deny', () => { + const ROOT_FINAL = 'ok, I will not write the note'; + + const EXPECTED_TURN_2_EVENTS = [ + { type: EventType.TOOL_RESPONSE, thread_id: ROOT_ID, tool_call_id: WRITE_NOTE_CALL_ID }, + { + type: InternalEventType.AGENT_CONTEXT_APPEND, + thread_id: ROOT_ID, + context: [{ role: 'tool', tool_call_id: WRITE_NOTE_CALL_ID, content: DENY_TOOL_CONTENT }], + }, + { type: EventType.MODEL_MESSAGE, thread_id: ROOT_ID }, + { type: EventType.MODEL_MESSAGE_DELTA, thread_id: ROOT_ID, content: ROOT_FINAL }, + { + type: InternalEventType.AGENT_CONTEXT_APPEND, + thread_id: ROOT_ID, + context: [{ role: 'assistant', content: ROOT_FINAL }], + }, + { type: InternalEventType.AGENT_DONE, thread_id: ROOT_ID, status: 'done' }, + ]; + + const TURN_2_OUTPUT = { + output: { thread_id: ROOT_ID, content: ROOT_FINAL }, + required_actions: [], + }; + + const EXPECTED_TURN_2_INPUT = { + tools: WRITE_NOTE_TOOLS, + messages: [ + { role: 'system', content: expect.stringContaining(INSTRUCTION) }, + { role: 'user', content: 'hello' }, + { + role: 'assistant', + content: null, + tool_calls: [ + { + id: WRITE_NOTE_CALL_ID, + type: 'function', + function: { name: WRITE_NOTE_TOOL_NAME, arguments: WRITE_NOTE_ARGUMENTS }, + }, + ], + }, + { role: 'tool', tool_call_id: WRITE_NOTE_CALL_ID, content: DENY_TOOL_CONTENT }, + ], + }; + + it('pauses for write_note approval, then finishes after deny without running the tool', async () => { + const { orchestrator, thread, callTool } = makeApprovalHarness(ROOT_FINAL); + + const paused = await runTurn({ + orchestrator, + sendBatch: [{ type: EventType.USER_MESSAGE, content: 'hello' }], + }); + expect(paused.events).toMatchObject(EXPECTED_TURN_1_EVENTS); + expect(paused.result).toMatchObject(TURN_1_OUTPUT); + expect(callTool).not.toHaveBeenCalled(); + + // Deny is a new turn (send + execute), same as allow — turn 1 already stopped at approval. + const resumed = await runTurn({ + orchestrator, + sendBatch: [ + { + type: EventType.USER_TOOL_APPROVAL, + thread_id: ROOT_ID, + tool_call_id: WRITE_NOTE_CALL_ID, + approval: { status: 'deny', reason: DENY_REASON }, + }, + ], + }); + expect(resumed.events).toMatchObject(EXPECTED_TURN_2_EVENTS); + expect(resumed.result).toMatchObject(TURN_2_OUTPUT); + expect(resumed.result.root_agent_error).toBeUndefined(); + expect(callTool).not.toHaveBeenCalled(); + expect(llmCreateInputs(thread.definition.modelClient)).toMatchObject([ + ...EXPECTED_TURN_1_INPUT, + EXPECTED_TURN_2_INPUT, + ]); + }); + }); +}); + +function makeApprovalHarness(finalReply: string): { + orchestrator: AgentThreadOrchestrator; + thread: AgentThread; + callTool: jest.Mock; +} { + const { toolSet, callTool } = makeApprovalGatedWriteNoteToolSet(); + const agentThreadInput: AgentThreadConstructorInput = { + definition: { + modelClient: { + create: jest + .fn() + .mockImplementationOnce(() => writeNoteToolCallStream()) + .mockImplementation(() => textReplyStream(finalReply)), + createNonStream: jest.fn(), + }, + instruction: INSTRUCTION, + messages: undefined, + modelParams: undefined, + responseFormat: undefined, + iterationLimit: undefined, + toolSets: undefined, + }, + threadId: ROOT_ID, + title: 'orchestration-approval', + parent: undefined, + agentInfo: undefined, + context: undefined, + currentContextUsage: undefined, + preComputedCompletion: undefined, + sandbox: undefined, + capabilities: [ + { + systemToolSets: [toolSet], + preSendProcessors: undefined, + preLLMProcessors: undefined, + preLLMEphemeralProcessors: undefined, + postToolCallProcessors: undefined, + toolResponseProcessors: undefined, + instructionBuilders: undefined, + }, + ], + capabilityState: undefined, + tracing: NOOP_AGENT_TRACING, + logger: makeSilentLogger(), + }; + + const thread = new AgentThread(agentThreadInput); + const orchestrator = new AgentThreadOrchestrator({ + agentThreads: new Map([[thread.threadId, thread]]), + createDynamicSubAgentThread: () => Promise.reject(new Error('unexpected sub-agent in approval test')), + tracing: NOOP_AGENT_TRACING, + logger: makeSilentLogger(), + }); + return { orchestrator, thread, callTool }; +} diff --git a/packages/trueforge-core/tests/orchestration/orchestrationWithTools.test.ts b/packages/trueforge-core/tests/orchestration/orchestrationWithTools.test.ts new file mode 100644 index 000000000..6db326c38 --- /dev/null +++ b/packages/trueforge-core/tests/orchestration/orchestrationWithTools.test.ts @@ -0,0 +1,210 @@ +import type { AgentDefinition, CreateDynamicSubAgentThread } from '../../src/core'; +import { dynamicSubAgents } from '../../src/core/capabilities/builtins/DynamicSubAgents'; +import { EventType } from '../../src/core/events/schema'; +import type { ILLM } from '../../src/core/llm/ILLM'; +import { AgentThread } from '../../src/core/runtime/AgentThread'; +import { InternalEventType, type AgentThreadConstructorInput } from '../../src/core/runtime/AgentThread.types'; +import { + AgentThreadOrchestrator, + type AgentThreadOrchestratorInput, +} from '../../src/core/runtime/AgentThreadOrchestrator'; +import { NOOP_AGENT_TRACING } from '../../src/core/tracing/NoopAgentTracing'; +import { makeSilentLogger } from '../core/harnessMocks'; +import { createSubAgentStream, llmCreateInputs, runTurn, textReplyStream } from './helpers/helpers'; + +const ROOT_ID = 'thread_root'; +const TOOL_CALL_ID = 'call-sub'; +const CHILD_REPLY = 'hello from the child'; +const ROOT_FINAL = 'How are you?'; +const INSTRUCTION = 'You are running in a test setup.'; +const CHILD_TASK = 'do the delegated task'; + +const CREATE_SUB_AGENT_ARGS = JSON.stringify({ name: 'worker', input: CHILD_TASK }); + +/** Root delegates via create_sub_agent; child result returns to parent; root finishes. */ +const EXPECTED_EVENTS = [ + { type: EventType.MODEL_MESSAGE, thread_id: ROOT_ID }, + { type: EventType.MODEL_MESSAGE_DELTA, thread_id: ROOT_ID }, + { + type: InternalEventType.AGENT_CONTEXT_APPEND, + thread_id: ROOT_ID, + context: [ + { + role: 'assistant', + content: null, + tool_calls: [ + { + id: TOOL_CALL_ID, + type: 'function', + function: { name: 'create_sub_agent', arguments: CREATE_SUB_AGENT_ARGS }, + }, + ], + }, + ], + }, + // create_sub_agent tool path yields an empty append before THREAD_CREATED. + { type: InternalEventType.AGENT_CONTEXT_APPEND, thread_id: ROOT_ID, context: [] }, + { + type: EventType.THREAD_CREATED, + title: 'worker', + parent: { thread_id: ROOT_ID, tool_call_id: TOOL_CALL_ID }, + }, + { type: EventType.MODEL_MESSAGE, thread_id: expect.any(String) }, + { type: EventType.MODEL_MESSAGE_DELTA, thread_id: expect.any(String), content: CHILD_REPLY }, + { + type: InternalEventType.AGENT_CONTEXT_APPEND, + thread_id: expect.any(String), + context: [{ role: 'assistant', content: CHILD_REPLY }], + }, + { type: EventType.TOOL_RESPONSE, thread_id: ROOT_ID, tool_call_id: TOOL_CALL_ID }, + { + type: InternalEventType.AGENT_CONTEXT_APPEND, + thread_id: ROOT_ID, + context: [{ role: 'tool', tool_call_id: TOOL_CALL_ID, content: CHILD_REPLY }], + }, + { type: InternalEventType.AGENT_DONE, thread_id: expect.any(String), status: 'done' }, + { type: EventType.MODEL_MESSAGE, thread_id: ROOT_ID }, + { type: EventType.MODEL_MESSAGE_DELTA, thread_id: ROOT_ID, content: ROOT_FINAL }, + { + type: InternalEventType.AGENT_CONTEXT_APPEND, + thread_id: ROOT_ID, + context: [{ role: 'assistant', content: ROOT_FINAL }], + }, + { type: InternalEventType.AGENT_DONE, thread_id: ROOT_ID, status: 'done' }, +]; + +const OUTPUT = { + output: { thread_id: ROOT_ID, content: ROOT_FINAL }, + required_actions: [], +}; + +const EXPECTED_ROOT_LLM_INPUT = [ + { + messages: [ + { role: 'system', content: expect.stringContaining(INSTRUCTION) }, + { role: 'user', content: 'hello' }, + ], + }, + { + messages: [ + { role: 'system', content: expect.stringContaining(INSTRUCTION) }, + { role: 'user', content: 'hello' }, + { + role: 'assistant', + content: null, + tool_calls: [ + { + id: TOOL_CALL_ID, + type: 'function', + function: { + name: 'create_sub_agent', + arguments: CREATE_SUB_AGENT_ARGS, + }, + }, + ], + }, + { role: 'tool', tool_call_id: TOOL_CALL_ID, content: CHILD_REPLY }, + ], + }, +]; + +describe('orchestration: dynamic sub-agent', () => { + it('delegates via create_sub_agent, routes child result to parent, then finishes', async () => { + let agentThreadInput: AgentThreadConstructorInput = { + // AgentDefinition + definition: { + // This is an instance if ILLM + modelClient: { + create: jest + .fn() + .mockImplementationOnce(() => createSubAgentStream()) + .mockImplementation(() => textReplyStream(ROOT_FINAL)), + createNonStream: jest.fn(), + }, + instruction: INSTRUCTION, + // Undefined + messages: undefined, + modelParams: undefined, + responseFormat: undefined, + iterationLimit: undefined, + toolSets: undefined, + }, + threadId: ROOT_ID, + title: 'orchestration-with-tools', + // Undefined + parent: undefined, + agentInfo: undefined, + context: undefined, + currentContextUsage: undefined, + preComputedCompletion: undefined, + sandbox: undefined, + capabilities: [dynamicSubAgents({ sandboxAvailable: false, tracing: NOOP_AGENT_TRACING })], + capabilityState: undefined, + // Default + tracing: NOOP_AGENT_TRACING, + logger: makeSilentLogger(), + }; + + let thread_1 = new AgentThread(agentThreadInput); + let childLLM: ILLM | undefined; + + const createSubAgentThread: CreateDynamicSubAgentThread = async ({ + parentDefinition, + request, + threadId, + parent, + }) => { + childLLM = { + create: jest.fn().mockImplementation(() => textReplyStream(CHILD_REPLY)), + createNonStream: jest.fn().mockImplementation(() => textReplyStream(CHILD_REPLY)), + }; + + const agentDefinition: AgentDefinition = { + modelClient: childLLM, + instruction: undefined, + messages: [{ role: 'user', content: request.input }], + modelParams: parentDefinition.modelParams, + responseFormat: undefined, + iterationLimit: parentDefinition.iterationLimit, + toolSets: undefined, + }; + return new AgentThread({ + definition: agentDefinition, + threadId, + title: request.name, + parent, + agentInfo: request, + context: undefined, + currentContextUsage: undefined, + preComputedCompletion: undefined, + sandbox: undefined, + capabilities: undefined, + capabilityState: undefined, + tracing: NOOP_AGENT_TRACING, + logger: makeSilentLogger(), + }); + }; + + let orchestratorInput: AgentThreadOrchestratorInput = { + agentThreads: new Map([[thread_1.threadId, thread_1]]), + createDynamicSubAgentThread: createSubAgentThread, + tracing: NOOP_AGENT_TRACING, + logger: makeSilentLogger(), + }; + + const orchestrator = new AgentThreadOrchestrator(orchestratorInput); + + const { events, result } = await runTurn({ + orchestrator, + sendBatch: [{ type: EventType.USER_MESSAGE, content: 'hello' }], + }); + + expect(events).toMatchObject(EXPECTED_EVENTS); + expect(result).toMatchObject(OUTPUT); + expect(result.root_agent_error).toBeUndefined(); + expect(llmCreateInputs(thread_1.definition.modelClient)).toMatchObject(EXPECTED_ROOT_LLM_INPUT); + if (childLLM === undefined) { + throw new Error('expected child LLM to be created'); + } + }); +});