From 275f8b3785ef858b3e4fac347c7b1e1f3f360834 Mon Sep 17 00:00:00 2001 From: Raman Tehlan Date: Tue, 25 Aug 2026 17:02:05 +0530 Subject: [PATCH 01/19] add sample e2e test for trueforge-core --- package.json | 1 + packages/trueforge-core/jest.config.cjs | 3 +- packages/trueforge-core/jest.e2e.config.cjs | 35 ++++++++ packages/trueforge-core/package.json | 3 +- packages/trueforge-core/tests/e2e/helpers.ts | 30 +++++++ .../tests/e2e/orchestration.test.ts | 80 +++++++++++++++++++ packages/trueforge-core/tsconfig.json | 2 +- 7 files changed, 151 insertions(+), 3 deletions(-) create mode 100644 packages/trueforge-core/jest.e2e.config.cjs create mode 100644 packages/trueforge-core/tests/e2e/helpers.ts create mode 100644 packages/trueforge-core/tests/e2e/orchestration.test.ts diff --git a/package.json b/package.json index 3be2de1e0..36643b451 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "test:frontend": "pnpm --filter frontend test", "test:chart-version": "bash tests/scripts/resolve-chart-version.test.sh", "test:trueforge-core": "pnpm --filter @truefoundry/trueforge-core test", + "test:trueforge-core:e2e": "pnpm --filter @truefoundry/trueforge-core test:e2e", "test:trueforge": "pnpm --filter @truefoundry/trueforge test", "test:local-sandbox:contract": "pnpm --filter @truefoundry/trueforge test:local-sandbox:contract", "smoke:local-sandbox": "pnpm --filter @truefoundry/trueforge smoke:local-sandbox", diff --git a/packages/trueforge-core/jest.config.cjs b/packages/trueforge-core/jest.config.cjs index 935b21b7e..93d553ff8 100644 --- a/packages/trueforge-core/jest.config.cjs +++ b/packages/trueforge-core/jest.config.cjs @@ -37,5 +37,6 @@ module.exports = { roots: ['/tests'], testMatch: ['**/tests/**/*.test.ts'], // Compile-time suites are enforced by `tsc --noEmit`, not the Jest runner. - testPathIgnorePatterns: ['\\.compile\\.test\\.ts$'], + // E2E lives under tests/e2e and is run via jest.e2e.config.cjs. + testPathIgnorePatterns: ['\\.compile\\.test\\.ts$', '/tests/e2e/'], }; diff --git a/packages/trueforge-core/jest.e2e.config.cjs b/packages/trueforge-core/jest.e2e.config.cjs new file mode 100644 index 000000000..d15ca7019 --- /dev/null +++ b/packages/trueforge-core/jest.e2e.config.cjs @@ -0,0 +1,35 @@ +/** @type {import('jest').Config} */ +module.exports = { + testEnvironment: 'node', + transform: { + '^.+\\.tsx?$': [ + '@swc/jest', + { + jsc: { + parser: { syntax: 'typescript', decorators: true }, + target: 'es2022', + }, + module: { type: 'commonjs' }, + }, + ], + '^.+\\.js$': [ + '@swc/jest', + { + jsc: { + parser: { syntax: 'ecmascript' }, + target: 'es2022', + }, + module: { type: 'commonjs' }, + }, + ], + }, + transformIgnorePatterns: [], + moduleNameMapper: { + '^(\\.{1,2}/.*)\\.js$': '$1', + }, + setupFilesAfterEnv: ['/tests/setup.ts'], + testTimeout: 60_000, + maxWorkers: 1, + roots: ['/tests/e2e'], + testMatch: ['/tests/e2e/**/*.test.ts'], +}; diff --git a/packages/trueforge-core/package.json b/packages/trueforge-core/package.json index b1f2a1c1a..691e33f11 100644 --- a/packages/trueforge-core/package.json +++ b/packages/trueforge-core/package.json @@ -95,7 +95,8 @@ "build:pkg": "node scripts/write-dist-package-json.mjs", "build:check": "node scripts/check-dist.mjs", "typecheck": "pnpm run build:gen && tsc --noEmit", - "test": "pnpm run build:gen && jest --config jest.config.cjs", + "test": "pnpm run build:gen && jest --config jest.config.cjs && jest --config jest.e2e.config.cjs", + "test:e2e": "pnpm run build:gen && jest --config jest.e2e.config.cjs", "pack:dry": "pnpm pack --dry-run" }, "dependencies": { diff --git a/packages/trueforge-core/tests/e2e/helpers.ts b/packages/trueforge-core/tests/e2e/helpers.ts new file mode 100644 index 000000000..49d99c933 --- /dev/null +++ b/packages/trueforge-core/tests/e2e/helpers.ts @@ -0,0 +1,30 @@ +import type { ILLM } from '../../src/core/llm/ILLM'; +import type { ExtendedChatCompletionChunk, RawAssistantMessageWithUsage } from '../../src/core/llm/LLMTypes'; +import { getEmptyUsage } from '../../src/core/llm/LLMTypes'; + +/** One streamed chunk plus a stop completion. Used when the test needs a text reply and no tool calls. */ +// eslint-disable-next-line @typescript-eslint/require-await -- async generator fixture, not awaiting I/O +export async function* textReplyStream( + text: string, +): AsyncGenerator { + yield { + id: 'chunk-text', + object: 'chat.completion.chunk', + created: 0, + model: 'test-model', + choices: [{ index: 0, delta: { role: 'assistant', content: text }, finish_reason: 'stop' }], + }; + return { + output: { role: 'assistant', content: text }, + usage: getEmptyUsage(), + finish_reason: 'stop', + }; +} + +/** ILLM that always streams `text` and then stops. */ +export function makeTextLlm(text: string): ILLM { + return { + create: jest.fn().mockImplementation(() => textReplyStream(text)), + createNonStream: jest.fn().mockImplementation(() => textReplyStream(text)), + }; +} diff --git a/packages/trueforge-core/tests/e2e/orchestration.test.ts b/packages/trueforge-core/tests/e2e/orchestration.test.ts new file mode 100644 index 000000000..b2a06b8c2 --- /dev/null +++ b/packages/trueforge-core/tests/e2e/orchestration.test.ts @@ -0,0 +1,80 @@ +import { EventType } from '../../src/core/events/schema'; +import { AgentThread } from '../../src/core/runtime/AgentThread'; +import { InternalEventType } from '../../src/core/runtime/AgentThread.types'; +import { AgentThreadOrchestrator } from '../../src/core/runtime/AgentThreadOrchestrator'; +import { NOOP_AGENT_TRACING } from '../../src/core/tracing/NoopAgentTracing'; +import { makeSilentLogger } from '../agent-session/testHelpers'; +import { makeTextLlm } from './helpers'; + +const THREAD_ID = 'main'; +const REPLY = 'hello from the mocked model'; + +/** Root thread with a one-shot text LLM and no tool sets. */ +function makeTextLlmThread(): AgentThread { + return new AgentThread({ + threadId: THREAD_ID, + title: 'e2e-orchestration', + tracing: NOOP_AGENT_TRACING, + logger: makeSilentLogger(), + definition: { + modelClient: makeTextLlm(REPLY), + instruction: 'You are running in a test setup.', + }, + }); +} + +describe('core E2E: orchestrator with mocked LLM and no tools', () => { + it('sends a user message and finishes the thread with a text reply', async () => { + const thread = makeTextLlmThread(); + // Orchestrator owns the thread map and fans send/execute across live threads. + // This case has only the root thread, so sub-agent creation must never run. + const orchestrator = new AgentThreadOrchestrator({ + agentThreads: new Map([[thread.threadId, thread]]), + createDynamicSubAgentThread: () => Promise.reject(new Error('unexpected sub-agent in no-tool test')), + tracing: NOOP_AGENT_TRACING, + logger: makeSilentLogger(), + }); + + // send() commits user input into thread context; it does not call the LLM. + const sendTypes: string[] = []; + for await (const event of orchestrator.send([{ type: EventType.USER_MESSAGE, content: 'hello' }])) { + sendTypes.push(event.type); + } + expect(sendTypes).toEqual([InternalEventType.AGENT_CONTEXT_APPEND]); + + // execute() runs the LLM loop. Manual next() is required so we can read the + // generator's return value (AgentThreadExecutionResult) after the last yield. + const types: string[] = []; + const iterator = orchestrator.execute({ signal: new AbortController().signal }); + let step = await iterator.next(); + while (!step.done) { + types.push(step.value.type); + step = await iterator.next(); + } + + // Happy path: stream the reply, then a terminal AGENT_DONE. No tools or child threads. + expect(types).toContain(EventType.MODEL_MESSAGE_DELTA); + expect(types).toContain(EventType.MODEL_MESSAGE); + expect(types[types.length - 1]).toBe(InternalEventType.AGENT_DONE); + expect(types).not.toContain(EventType.TOOL_RESPONSE); + expect(types).not.toContain(EventType.THREAD_CREATED); + + // Result is the orchestrator return, not an event: final assistant output, no pause/error. + expect(step.value.required_actions).toEqual([]); + expect(step.value.root_agent_error).toBeUndefined(); + expect(step.value.output).toMatchObject({ + type: EventType.MODEL_MESSAGE, + thread_id: THREAD_ID, + content: REPLY, + }); + + // Durable thread context after send + execute: user turn plus the assistant reply. + const snapshot = thread.toSnapshot(); + expect(snapshot.context).toEqual( + expect.arrayContaining([ + expect.objectContaining({ role: 'user', content: 'hello' }), + expect.objectContaining({ role: 'assistant', content: REPLY }), + ]), + ); + }); +}); diff --git a/packages/trueforge-core/tsconfig.json b/packages/trueforge-core/tsconfig.json index 85ca45efe..3883bd9f1 100644 --- a/packages/trueforge-core/tsconfig.json +++ b/packages/trueforge-core/tsconfig.json @@ -12,6 +12,6 @@ "openai/resources/chat": ["./node_modules/openai/resources/chat/index.d.ts"] } }, - "include": ["src/**/*", "tests/**/*", "tsup.config.ts", "jest.config.cjs"], + "include": ["src/**/*", "tests/**/*", "tsup.config.ts", "jest.config.cjs", "jest.e2e.config.cjs"], "exclude": ["node_modules", "dist"] } From b0eecef8e1096d66c6d07fb051fb0c06e3723bc8 Mon Sep 17 00:00:00 2001 From: Raman Tehlan Date: Wed, 26 Aug 2026 20:04:53 +0530 Subject: [PATCH 02/19] Add orchestration e2e with sub-agents --- packages/trueforge-core/tests/e2e/README.md | 412 ++++++++++++++++++ packages/trueforge-core/tests/e2e/helpers.ts | 80 +++- .../tests/e2e/orchestration.test.ts | 4 +- .../tests/e2e/orchestrationWithTools.test.ts | 117 +++++ 4 files changed, 610 insertions(+), 3 deletions(-) create mode 100644 packages/trueforge-core/tests/e2e/README.md create mode 100644 packages/trueforge-core/tests/e2e/orchestrationWithTools.test.ts diff --git a/packages/trueforge-core/tests/e2e/README.md b/packages/trueforge-core/tests/e2e/README.md new file mode 100644 index 000000000..0ae1ad3c0 --- /dev/null +++ b/packages/trueforge-core/tests/e2e/README.md @@ -0,0 +1,412 @@ +# Core runtime E2E tests + +End-to-end tests for `AgentThreadOrchestrator` and `AgentThread` in `@truefoundry/trueforge-core`. + +These tests wire the real orchestration loop with **mocked LLMs** and **no database**. They exist to learn and verify how a turn flows through the harness before adding persistence (`SessionHandle`), HTTP, or real model providers. + +## What we are testing + +| Layer | In scope | Out of scope | +| --------------------------------- | ------------------------------------------------------------------------- | ------------------------------------------- | +| `AgentThreadOrchestrator.send` | Route input to threads, validate, append context | Postgres / Redis store writes | +| `AgentThreadOrchestrator.execute` | Run leaf threads, merge streams, spawn sub-agents, return terminal result | `TurnHandle.stream` persistence | +| `AgentThread` | LLM loop, tool execution, context mutations | Real OpenAI / Vercel AI calls | +| Sub-agent lifecycle | `create_sub_agent` tool → child thread → result back to parent | Full `SessionHandle` resolver / spec wiring | + +**Goal:** prove the orchestrator correctly coordinates one root thread (Program 1) and a root + dynamic child thread (Program 2). + +## Why this design + +Production creates the orchestrator inside `SessionHandle.createTurn`: + +```text +resolve definitions → build AgentThread map → new AgentThreadOrchestrator → send → persist → execute (via TurnHandle) +``` + +These E2E tests **skip the store and session layer** and talk to the orchestrator directly. That keeps the surface area small while still exercising the same `send` / `execute` contract production uses. + +```mermaid +flowchart LR + subgraph production["Production path"] + SH[SessionHandle] + Store[(ISessionStore)] + OrchP[AgentThreadOrchestrator] + SH --> Store + SH --> OrchP + end + + subgraph e2e["E2E tests"] + Test[Jest test] + OrchE[AgentThreadOrchestrator] + MockLLM[Mock ILLM] + Test --> OrchE + OrchE --> MockLLM + end + + OrchP -. same class .- OrchE +``` + +## Files + +| File | Role | +| -------------------------------- | -------------------------------------------------------------------------------------------------- | +| `orchestration.test.ts` | **Program 1** - single root thread, text-only reply, full assertions | +| `orchestrationWithTools.test.ts` | **Program 2** - root delegates to sub-agent via `create_sub_agent`, logging only (assertions TODO) | +| `helpers.ts` | Mock LLM streams, logger factory | +| `jest.e2e.config.cjs` | Jest config scoped to this folder | + +## Core components under test + +### `AgentThread` + +One conversation thread. Holds: + +- **`definition`** - `modelClient` (`ILLM`), optional `instruction`, `toolSets`, etc. +- **`context`** - LLM message history (user, assistant, tool messages) +- **`send(messages)`** - append user input, approvals, or tool responses to context (no LLM call) +- **`execute({ signal })`** - run the state machine: LLM → tools → pause or done + +### `AgentThreadOrchestrator` + +Owns a `Map` and coordinates a turn: + +- **`send(batch)`** - fan out messages to the right threads, validate, delegate to each thread's `send` +- **`execute({ signal })`** - run **leaf** threads in parallel (up to 5), merge event streams, handle sub-agent creation/completion +- **`createDynamicSubAgentThread`** - factory callback invoked when the root calls `create_sub_agent`; must return a new `AgentThread` (not called at construction time) + +### `CreateDynamicSubAgentThread` + +```ts +(input: { + parentDefinition: AgentDefinition; + request: AgentInfo; // { type: 'dynamic', name, input, model? } + threadId: string; // orchestrator already minted this + parent: AgentParent; // { thread_id, tool_call_id } + signal: AbortSignal; +}) => Promise; +``` + +Pass the **function reference** to the orchestrator. Do not call it yourself. + +## Turn lifecycle: `send` then `execute` + +These are separate steps on purpose (same as production: send before commit, then execute). + +```mermaid +sequenceDiagram + participant Test + participant Orch as AgentThreadOrchestrator + participant Thread as AgentThread + participant LLM as Mock ILLM + + Test->>Orch: send([USER_MESSAGE]) + Orch->>Thread: send(messages) + Thread-->>Orch: AGENT_CONTEXT_APPEND + Orch-->>Test: yield append events + + Note over Test,LLM: send does NOT call the model + + Test->>Orch: execute({ signal }) + loop until AGENT_DONE or pause + Orch->>Thread: execute({ signal }) + Thread->>LLM: create(streaming) + LLM-->>Thread: chunks / tool_calls + Thread-->>Orch: model.message.delta, model.message, ... + Orch-->>Test: yield execution events + end + Orch-->>Test: return AgentThreadExecutionResult +``` + +**Important:** `send` returns an async generator. You must consume it with `for await`; otherwise the user message never lands in context. + +**Important:** `execute` also returns an async generator. The **return value** (final assistant output, required pauses, errors) is only available after the last `next()` when `done === true`. + +## Mock LLM helpers (`helpers.ts`) + +| Helper | Behavior | +| ------------------------- | ---------------------------------------------------------------------------- | +| `textReplyStream(text)` | One streaming chunk + stop completion with fixed text | +| `makeTextLLM(text)` | `ILLM` that always replies with `text` (used for child threads) | +| `createSubAgentStream()` | First root call: stream a `create_sub_agent` tool call | +| `makeRootLLM(finalReply)` | First `create()` → sub-agent tool call; every later call → `finalReply` text | +| `makeDummyLogger()` | Winston logger with colorized console output for debugging | + +Root and child threads use **different** `ILLM` instances so each can follow its own scripted sequence. + +--- + +## Program 1: text-only happy path + +**File:** `orchestration.test.ts` + +### Setup + +| Piece | Value | +| ----------------------------- | -------------------------------------------- | +| Root thread id | `"main"` | +| LLM | `makeTextLLM("hello from the mocked model")` | +| Tool sets | none | +| `createDynamicSubAgentThread` | rejects if ever called | +| Tracing | `NOOP_AGENT_TRACING` | +| Logger | silent (`makeSilentLogger`) | + +### Data flow + +```mermaid +flowchart TD + A["send: USER_MESSAGE 'hello'"] --> B["context: user message appended"] + B --> C["execute: llm-call-required"] + C --> D["Mock LLM streams text reply"] + D --> E["context: assistant message appended"] + E --> F["AGENT_DONE on root"] + F --> G["execute returns output + empty required_actions"] +``` + +### Expected event types + +**After `send`:** + +```text +internal.agent.context.append +``` + +**During `execute` (order may include duplicates / internal appends):** + +```text +model.message.delta +model.message +internal.agent.done ← last yielded event +``` + +**Must NOT appear:** + +```text +thread.created +tool.response +``` + +### Passing expectations (assertions) + +- `step.value.output.content` === `"hello from the mocked model"` +- `step.value.required_actions` === `[]` +- `step.value.root_agent_error` is undefined +- Root snapshot context contains user `"hello"` and assistant reply + +--- + +## Program 2: sub-agent delegation + +**File:** `orchestrationWithTools.test.ts` + +### Setup + +| Piece | Root thread | Child thread | +| ---------------- | ----------------------------- | ----------------------------------------------- | +| Thread id | `"thread_1"` (fixed) | minted by orchestrator at runtime | +| LLM | `makeRootLLM("How are you?")` | `makeTextLLM("hello from the child")` | +| Tool sets | `[new DynamicSubAgents(...)]` | `undefined` (no nested sub-agents) | +| Instruction | test setup string | `undefined` (harness adds `SUB_AGENT_IDENTITY`) | +| Initial messages | none | `[{ role: 'user', content: request.input }]` | +| Parent link | none | `{ thread_id, tool_call_id }` from orchestrator | + +`createSubAgentThread` is a top-level `CreateDynamicSubAgentThread` implementation (mirrors a simplified `SessionHandle.makeCreateDynamicSubAgentThread`). + +### Scripted LLM behavior + +1. **Root call 1** - model returns `create_sub_agent` with `{ name: 'worker', input: '...' }` +2. **Child call 1** - model returns `"hello from the child"` +3. **Root call 2** - model returns `"How are you?"` + +### Thread tree over time + +```mermaid +flowchart TD + subgraph phase1["After root LLM call 1"] + R1["thread_1 (root)
open create_sub_agent tool call"] + end + + subgraph phase2["After sub-agent created"] + R2["thread_1 (root)
waiting on tool call"] + C["child thread (leaf)
runs execute"] + R2 --- C + end + + subgraph phase3["After child AGENT_DONE"] + R3["thread_1 (root, leaf again)
tool result appended"] + end + + phase1 --> phase2 --> phase3 +``` + +Only **leaf** threads run. While the child exists, the root is paused (not a leaf). When the child finishes, the orchestrator: + +1. Yields `tool.response` on the parent +2. `send()`s the child's result into the parent as a tool message +3. Removes the child from the thread map +4. Resumes the root for LLM call 2 + +### Data flow + +```mermaid +sequenceDiagram + participant Test + participant Orch as Orchestrator + participant Root as thread_1 + participant Child as sub-agent + participant RootLLM as makeRootLLM + participant ChildLLM as makeTextLLM + + Test->>Orch: send(USER_MESSAGE) + Test->>Orch: execute() + + Root->>RootLLM: create() #1 + RootLLM-->>Root: create_sub_agent tool call + Root-->>Orch: internal.agent.create_subagent + Orch->>Orch: createSubAgentThread(...) + Orch-->>Test: thread.created + + Child->>ChildLLM: create() + ChildLLM-->>Child: "hello from the child" + Child-->>Orch: model.message, AGENT_DONE (child) + + Orch-->>Test: tool.response (parent) + Orch->>Root: send(tool message with child result) + + Root->>RootLLM: create() #2 + RootLLM-->>Root: "How are you?" + Root-->>Orch: model.message, AGENT_DONE (root) + Orch-->>Test: return { output: "How are you?", ... } +``` + +### Expected event types (from logging) + +Typical `execute` event sequence: + +```text +model.message / model.message.delta ← root tool call +internal.agent.context.append ← (internal, may repeat) +thread.created ← child registered +model.message / model.message.delta ← child reply +tool.response ← child result routed to parent +internal.agent.done ← child finished (thread_id = child) +model.message / model.message.delta ← root final reply +internal.agent.done ← root finished (last event) +``` + +`internal.agent.create_subagent` is handled inside the orchestrator and is **not** yielded to the test consumer. + +### Expected final state + +**`execute` return value:** + +| Field | Expected | +| ------------------ | --------------------------------------------------- | +| `output.content` | `"How are you?"` (root final reply, not child text) | +| `required_actions` | `[]` | +| `root_agent_error` | undefined | + +**Root thread context (after send + execute):** + +```text +1. user: "hello" +2. assistant: tool_call create_sub_agent (id: call-sub) +3. tool: "hello from the child" +4. assistant: "How are you?" +``` + +### Current test status + +Program 2 currently **logs** events and the final result via `makeDummyLogger()`. It does **not** yet assert on event types or final state. Add the same style of expectations as Program 1 when ready. + +Suggested assertions to add: + +```ts +expect(types).toContain(EventType.THREAD_CREATED); +expect(types).toContain(EventType.TOOL_RESPONSE); +expect(types.at(-1)).toBe(InternalEventType.AGENT_DONE); +expect(step.value.output?.content).toBe('How are you?'); +``` + +--- + +## Running tests + +From `packages/trueforge-core`: + +```bash +pnpm test:e2e +``` + +Single file: + +```bash +pnpm test:e2e -- orchestration.test.ts +pnpm test:e2e -- orchestrationWithTools.test.ts +``` + +From repo root: + +```bash +pnpm test:trueforge-core:e2e +``` + +E2E tests use `jest.e2e.config.cjs` (`maxWorkers: 1`, 60s timeout). Unit tests under `tests/` (excluding `tests/e2e/`) run separately via `jest.config.cjs`. + +## Logging during tests + +- `tests/setup.ts` mocks `console.log` / `console.warn` / `console.error` for all Jest runs, including E2E. +- `makeDummyLogger()` uses Winston's `Console` transport and **does** print to the terminal. +- Program 2 logs: + - `send complete` with event types + - each `execute event` with `type` and `thread_id` + - `execute result` with full type list and terminal output + +The orchestrator itself does not log on the happy path. Test-side logging is intentional for learning. + +To debug with less noise, run a single file (see above). + +## Relationship to production + +| E2E test | Production equivalent | +| -------------------------------------- | ------------------------------------------------------ | +| `new AgentThread({ definition, ... })` | `SessionHandle.buildThreads` + resolver | +| `createSubAgentThread` callback | `SessionHandle.makeCreateDynamicSubAgentThread` | +| `orchestrator.send` + `execute` | `SessionHandle.createTurn` + `TurnHandle.stream` | +| In-memory `thread.toSnapshot()` | `ISessionStore.createTurn` / persisted context appends | +| `NOOP_AGENT_TRACING` | `resolver.createTracing()` | + +Production adds: store persistence, turn records, event folding for SSE, sandbox resolution, full builtin capabilities from `AgentSpec`, and MCP servers beyond `DynamicSubAgents`. + +## Planned coverage (not yet implemented) + +| Program | Scenario | +| ------- | -------------------------------------------------------------------------------------------------- | +| **3** | Pause on `tool.approval.required` or `tool.response.required`, then resume with `send` + `execute` | +| **4** | Reject user message while sub-agent is live (`InvalidAgentSendInputError`) | +| **5** | MCP auth required (`internal.mcp.auth_required` merge across parallel sub-agents) | + +## Quick reference: orchestrator inputs + +```ts +new AgentThreadOrchestrator({ + agentThreads: new Map([[rootThreadId, rootThread]]), + createDynamicSubAgentThread, // function reference, not a call + tracing: NOOP_AGENT_TRACING, + logger, +}); +``` + +Every turn: + +```ts +for await (const _ of orchestrator.send(input)) { + /* collect appends */ +} +const it = orchestrator.execute({ signal }); +let step = await it.next(); +while (!step.done) { + // step.value is a streamed execution event + step = await it.next(); +} +// step.value is AgentThreadExecutionResult +``` diff --git a/packages/trueforge-core/tests/e2e/helpers.ts b/packages/trueforge-core/tests/e2e/helpers.ts index 49d99c933..b6ff59904 100644 --- a/packages/trueforge-core/tests/e2e/helpers.ts +++ b/packages/trueforge-core/tests/e2e/helpers.ts @@ -1,3 +1,5 @@ +import type { Logger } from 'winston'; +import winston from 'winston'; import type { ILLM } from '../../src/core/llm/ILLM'; import type { ExtendedChatCompletionChunk, RawAssistantMessageWithUsage } from '../../src/core/llm/LLMTypes'; import { getEmptyUsage } from '../../src/core/llm/LLMTypes'; @@ -21,10 +23,86 @@ export async function* textReplyStream( }; } +export async function* createSubAgentStream() { + yield { + id: 'chunk-tool', + object: 'chat.completion.chunk', + created: 0, + model: 'test-model', + // Choices + choices: [ + { + index: 0, + delta: { + role: 'assistant', + // Tool Calls + tool_calls: [ + { + index: 0, + id: 'call-sub', + type: 'function', + function: { + name: 'create_sub_agent', + arguments: JSON.stringify({ name: 'worker', input: 'do the delegated task' }), + }, + }, + ], // tool calls end + }, // Delta end + finish_reason: 'tool_calls', + }, + ], // Choices end + }; // yeild end + + return { + output: { + role: 'assistant', + content: null, + tool_calls: [ + { + id: 'call-sub', + type: 'function', + function: { + name: 'create_sub_agent', + arguments: JSON.stringify({ name: 'worker', input: 'do the delegated task [output]' }), + }, // Function end + }, + ], // Tool calls end + }, // Output end + usage: getEmptyUsage(), + finish_reason: 'tool_calls', + }; +} // function end + /** ILLM that always streams `text` and then stops. */ -export function makeTextLlm(text: string): ILLM { +export function makeTextLLM(text: string): ILLM { return { create: jest.fn().mockImplementation(() => textReplyStream(text)), createNonStream: jest.fn().mockImplementation(() => textReplyStream(text)), }; } + +export function makeRootLLM(finalReply: string): ILLM { + return { + create: jest + .fn() + .mockImplementationOnce(() => createSubAgentStream()) + .mockImplementation(() => textReplyStream(finalReply)), + createNonStream: jest.fn(), + }; +} + +export function makeDummyLogger(): Logger { + const logger = winston.createLogger({ + level: 'debug', + format: winston.format.combine( + winston.format.colorize(), + winston.format.printf(({ level, message, ...meta }) => { + const details = Object.keys(meta).length > 0 ? ` ${JSON.stringify(meta)}` : ''; + return `${level}: ${String(message)}${details}`; + }), + ), + transports: [new winston.transports.Console()], + }); + logger.child = () => logger; + return logger; +} diff --git a/packages/trueforge-core/tests/e2e/orchestration.test.ts b/packages/trueforge-core/tests/e2e/orchestration.test.ts index b2a06b8c2..bf9a54e4b 100644 --- a/packages/trueforge-core/tests/e2e/orchestration.test.ts +++ b/packages/trueforge-core/tests/e2e/orchestration.test.ts @@ -4,7 +4,7 @@ import { InternalEventType } from '../../src/core/runtime/AgentThread.types'; import { AgentThreadOrchestrator } from '../../src/core/runtime/AgentThreadOrchestrator'; import { NOOP_AGENT_TRACING } from '../../src/core/tracing/NoopAgentTracing'; import { makeSilentLogger } from '../agent-session/testHelpers'; -import { makeTextLlm } from './helpers'; +import { makeTextLLM } from './helpers'; const THREAD_ID = 'main'; const REPLY = 'hello from the mocked model'; @@ -17,7 +17,7 @@ function makeTextLlmThread(): AgentThread { tracing: NOOP_AGENT_TRACING, logger: makeSilentLogger(), definition: { - modelClient: makeTextLlm(REPLY), + modelClient: makeTextLLM(REPLY), instruction: 'You are running in a test setup.', }, }); diff --git a/packages/trueforge-core/tests/e2e/orchestrationWithTools.test.ts b/packages/trueforge-core/tests/e2e/orchestrationWithTools.test.ts new file mode 100644 index 000000000..aba6dbc4b --- /dev/null +++ b/packages/trueforge-core/tests/e2e/orchestrationWithTools.test.ts @@ -0,0 +1,117 @@ +import type { AgentDefinition, CreateDynamicSubAgentThread } from '../../src/core'; +import { DynamicSubAgents } from '../../src/core/capabilities/builtins/DynamicSubAgents'; +import { EventType } from '../../src/core/events/schema'; +import { AgentThread } from '../../src/core/runtime/AgentThread'; +import { type AgentThreadConstructorInput } from '../../src/core/runtime/AgentThread.types'; +import { + AgentThreadOrchestrator, + type AgentThreadOrchestratorInput, +} from '../../src/core/runtime/AgentThreadOrchestrator'; +import { NOOP_AGENT_TRACING } from '../../src/core/tracing/NoopAgentTracing'; +import { makeDummyLogger, makeRootLLM, makeTextLLM } from './helpers'; + +function makeMainLLMThread(threadId: string, reply: string, title: string): AgentThread { + let agentDefinition: AgentDefinition = { + // This is an instance if ILLM + modelClient: makeRootLLM(reply), + instruction: 'You are running in a test setup.', + // Undefined + messages: undefined, + modelParams: undefined, + responseFormat: undefined, + iterationLimit: undefined, + toolSets: [new DynamicSubAgents({ tracing: NOOP_AGENT_TRACING })], + }; + + let agentThreadInput: AgentThreadConstructorInput = { + definition: agentDefinition, + threadId: threadId, + title: title, + // Undefined + parent: undefined, + agentInfo: undefined, + context: undefined, + currentContextUsage: undefined, + preComputedCompletion: undefined, + sandbox: undefined, + capabilities: undefined, + capabilityState: undefined, + // Default + tracing: NOOP_AGENT_TRACING, + logger: makeDummyLogger(), + }; + + let agentThread = new AgentThread(agentThreadInput); + + return agentThread; +} + +const createSubAgentThread: CreateDynamicSubAgentThread = async ({ parentDefinition, request, threadId, parent }) => { + const agentDefinition: AgentDefinition = { + modelClient: makeTextLLM('hello from the child'), // Child has text only LLM, + // Not sure if this should be taken from the parent, or left alone + instruction: undefined, + messages: [{ role: 'user', content: request.input }], + modelParams: parentDefinition.modelParams, + responseFormat: undefined, + iterationLimit: parentDefinition.iterationLimit, + toolSets: undefined, // No parents tools sent to the child + }; + return new AgentThread({ + definition: agentDefinition, + threadId, + title: request.name, + parent, + agentInfo: request, + context: undefined, + currentContextUsage: undefined, + preComputedCompletion: undefined, + sandbox: undefined, + capabilities: undefined, + capabilityState: undefined, + tracing: NOOP_AGENT_TRACING, + logger: makeDummyLogger(), + }); +}; + +describe('core E2E: orchestrator with mocked LLM and no tools', () => { + it('sends a user message and finishes the thread with a text reply', async () => { + const logger = makeDummyLogger(); + const thread_1 = makeMainLLMThread('thread_1', 'How are you?', 'e2e-orchestration-with-tools'); + + let orchestratorInput: AgentThreadOrchestratorInput = { + agentThreads: new Map([[thread_1.threadId, thread_1]]), + createDynamicSubAgentThread: createSubAgentThread, + tracing: NOOP_AGENT_TRACING, + logger, + }; + + const orchestrator = new AgentThreadOrchestrator(orchestratorInput); + + const sendTypes: string[] = []; + for await (const event of orchestrator.send([{ type: EventType.USER_MESSAGE, content: 'hello' }])) { + sendTypes.push(event.type); + } + logger.info('send complete', { sendTypes }); + + const types: string[] = []; + const iterator = orchestrator.execute({ signal: new AbortController().signal }); + let step = await iterator.next(); + while (!step.done) { + const event = step.value; + logger.info('execute event', { + type: event.type, + thread_id: 'thread_id' in event ? event.thread_id : null, + }); + types.push(event.type); + step = await iterator.next(); + } + + logger.info('execute result', { + types, + output: step.value.output?.content ?? null, + required_actions: step.value.required_actions.map(action => action.type), + root_agent_error: step.value.root_agent_error ?? null, + }); + }); +}); From f0a084817bf030f0dd09f6181daab68f738438bf Mon Sep 17 00:00:00 2001 From: Raman Tehlan Date: Mon, 31 Aug 2026 11:10:41 +0530 Subject: [PATCH 03/19] add e2e test, expectation vs reality --- .../tests/e2e/{ => helpers}/helpers.ts | 6 +- .../tests/e2e/helpers/turnExpectations.ts | 232 ++++++++++++++++++ .../tests/e2e/helpers/turnFlowLogger.ts | 180 ++++++++++++++ .../tests/e2e/orchestration.test.ts | 2 +- .../tests/e2e/orchestrationWithTools.test.ts | 153 ++++++++---- 5 files changed, 524 insertions(+), 49 deletions(-) rename packages/trueforge-core/tests/e2e/{ => helpers}/helpers.ts (94%) create mode 100644 packages/trueforge-core/tests/e2e/helpers/turnExpectations.ts create mode 100644 packages/trueforge-core/tests/e2e/helpers/turnFlowLogger.ts diff --git a/packages/trueforge-core/tests/e2e/helpers.ts b/packages/trueforge-core/tests/e2e/helpers/helpers.ts similarity index 94% rename from packages/trueforge-core/tests/e2e/helpers.ts rename to packages/trueforge-core/tests/e2e/helpers/helpers.ts index b6ff59904..dbee1100e 100644 --- a/packages/trueforge-core/tests/e2e/helpers.ts +++ b/packages/trueforge-core/tests/e2e/helpers/helpers.ts @@ -1,8 +1,8 @@ import type { Logger } from 'winston'; import winston from 'winston'; -import type { ILLM } from '../../src/core/llm/ILLM'; -import type { ExtendedChatCompletionChunk, RawAssistantMessageWithUsage } from '../../src/core/llm/LLMTypes'; -import { getEmptyUsage } from '../../src/core/llm/LLMTypes'; +import type { ILLM } from '../../../src/core/llm/ILLM'; +import type { ExtendedChatCompletionChunk, RawAssistantMessageWithUsage } from '../../../src/core/llm/LLMTypes'; +import { getEmptyUsage } from '../../../src/core/llm/LLMTypes'; /** One streamed chunk plus a stop completion. Used when the test needs a text reply and no tool calls. */ // eslint-disable-next-line @typescript-eslint/require-await -- async generator fixture, not awaiting I/O diff --git a/packages/trueforge-core/tests/e2e/helpers/turnExpectations.ts b/packages/trueforge-core/tests/e2e/helpers/turnExpectations.ts new file mode 100644 index 000000000..e1630d4f1 --- /dev/null +++ b/packages/trueforge-core/tests/e2e/helpers/turnExpectations.ts @@ -0,0 +1,232 @@ +/** + * Collect and normalize an orchestrator turn so tests can compare + * one `expected` object to one `actual` object. + * + * Layout of a TurnActual / TurnExpected: + * sendTypes - event types from send() + * executeTrace - compact projected events from execute() + * result - generator return value (stripped) + * context - root thread snapshot context (stripped) + */ +import type { Logger } from 'winston'; +import { EventType } from '../../../src/core/events/schema'; +import type { AgentThread } from '../../../src/core/runtime/AgentThread'; +import type { + AgentThreadExecutionEvent, + AgentThreadExecutionResult, + AgentThreadSendBatch, + ContextMessage, +} from '../../../src/core/runtime/AgentThread.types'; +import { InternalEventType } from '../../../src/core/runtime/AgentThread.types'; +import type { AgentThreadOrchestrator } from '../../../src/core/runtime/AgentThreadOrchestrator'; +import { isLLMContextMessage } from '../../../src/core/runtime/contextUtils'; +import { logExecuteEvent, logSendEvent, logTurnPhase, logTurnResult } from './turnFlowLogger'; + +/** Placeholder substituted for the runtime-minted child thread id. */ +export const CHILD_THREAD_PLACEHOLDER = ''; + +export type ExecuteTraceRow = { + type: string; + thread_id: string | null; + tool_call_id?: string; + content?: string | null; + title?: string; + parent?: { thread_id: string; tool_call_id: string }; +}; + +export type ContextRow = { + role: string; + content?: string | null; + tool_call_id?: string; + tool_calls?: Array<{ + id: string; + function: { name: string; arguments: string }; + }>; +}; + +export type TurnResultRow = { + output: { thread_id: string; content: string | null } | null; + required_actions: string[]; + root_agent_error: { error: string } | null; +}; + +export type TurnActual = { + sendTypes: string[]; + executeTrace: ExecuteTraceRow[]; + result: TurnResultRow; + context: ContextRow[]; +}; + +export type TurnExpected = TurnActual; + +/** Normalize OpenAI-style content to a plain string or null for comparisons. */ +function contentToString(content: unknown): string | null { + if (typeof content === 'string') { + return content; + } + if (content === null || content === undefined) { + return null; + } + return JSON.stringify(content); +} + +function projectExecuteEvent(event: AgentThreadExecutionEvent): ExecuteTraceRow { + const threadId = 'thread_id' in event ? (event.thread_id ?? null) : null; + const base: ExecuteTraceRow = { type: event.type, thread_id: threadId }; + + switch (event.type) { + case EventType.MODEL_MESSAGE: + return { ...base, content: contentToString(event.content) }; + case EventType.TOOL_RESPONSE: + return { ...base, tool_call_id: event.tool_call_id }; + case EventType.THREAD_CREATED: + return { + ...base, + title: event.title, + parent: { + thread_id: event.parent.thread_id, + tool_call_id: event.parent.tool_call_id, + }, + }; + default: + return base; + } +} + +function projectContext(context: ContextMessage[]): ContextRow[] { + return context.map((msg): ContextRow => { + if (!isLLMContextMessage(msg)) { + return { role: 'approval_decision' }; + } + if (msg.role === 'user') { + return { + role: 'user', + content: contentToString(msg.content), + }; + } + if (msg.role === 'assistant') { + const row: ContextRow = { + role: 'assistant', + content: contentToString(msg.content), + }; + if (msg.tool_calls) { + row.tool_calls = msg.tool_calls.map(tc => ({ + id: tc.id, + function: { + name: tc.function.name, + arguments: tc.function.arguments, + }, + })); + } + return row; + } + return { + role: 'tool', + tool_call_id: msg.tool_call_id, + content: msg.content, + }; + }); +} + +function projectResult(result: AgentThreadExecutionResult): TurnResultRow { + return { + output: result.output + ? { + thread_id: result.output.thread_id, + content: contentToString(result.output.content), + } + : null, + required_actions: result.required_actions.map(action => action.type), + root_agent_error: result.root_agent_error ? { error: result.root_agent_error.error } : null, + }; +} + +/** Drop noisy append / delta events so the trace reads as the turn story. */ +export function compactExecuteTrace(trace: ExecuteTraceRow[]): ExecuteTraceRow[] { + return trace.filter( + row => row.type !== InternalEventType.AGENT_CONTEXT_APPEND && row.type !== EventType.MODEL_MESSAGE_DELTA, + ); +} + +function normalizeChildThreadId(trace: ExecuteTraceRow[], childThreadId: string | null): ExecuteTraceRow[] { + if (childThreadId === null) { + return trace; + } + return trace.map(row => { + if (row.thread_id !== childThreadId) { + return row; + } + return { ...row, thread_id: CHILD_THREAD_PLACEHOLDER }; + }); +} + +/** + * Run send() then execute() and return a normalized turn snapshot + * suitable for expected-vs-actual comparison. + * + * Pass `logger` to print every event with source → destination labels + * (see turnFlowLogger.ts). + */ +export async function runOrchestratorTurn(input: { + orchestrator: AgentThreadOrchestrator; + rootThread: AgentThread; + sendBatch: AgentThreadSendBatch; + signal?: AbortSignal | undefined; + logger?: Logger | undefined; +}): Promise { + const { logger } = input; + + if (logger) { + logTurnPhase(logger, 'send', 'append user/tool input into thread context (no LLM call)'); + } + + const sendTypes: string[] = []; + for await (const event of input.orchestrator.send(input.sendBatch)) { + sendTypes.push(event.type); + if (logger) { + logSendEvent(logger, event); + } + } + + if (logger) { + logTurnPhase(logger, 'execute', 'run leaf threads; orchestrator merges streams and routes sub-agents'); + } + + const rawExecuteTrace: ExecuteTraceRow[] = []; + const iterator = input.orchestrator.execute({ + signal: input.signal ?? new AbortController().signal, + }); + + let step = await iterator.next(); + let executeIndex = 0; + while (!step.done) { + if (logger) { + logExecuteEvent(logger, step.value, executeIndex); + executeIndex += 1; + } + rawExecuteTrace.push(projectExecuteEvent(step.value)); + step = await iterator.next(); + } + + if (logger) { + logTurnPhase(logger, 'result', 'generator return value (not a streamed event)'); + logTurnResult(logger, step.value); + } + + const childThreadId = rawExecuteTrace.find(row => row.type === EventType.THREAD_CREATED)?.thread_id ?? null; + + return { + sendTypes, + executeTrace: normalizeChildThreadId(compactExecuteTrace(rawExecuteTrace), childThreadId), + result: projectResult(step.value), + context: projectContext(input.rootThread.toSnapshot().context), + }; +} + +/** Layered compare so a failure names sendTypes / executeTrace / result / context. */ +export function expectTurn(actual: TurnActual, expected: TurnExpected): void { + expect(actual.sendTypes).toEqual(expected.sendTypes); + expect(actual.executeTrace).toEqual(expected.executeTrace); + expect(actual.result).toEqual(expected.result); + expect(actual.context).toEqual(expected.context); +} diff --git a/packages/trueforge-core/tests/e2e/helpers/turnFlowLogger.ts b/packages/trueforge-core/tests/e2e/helpers/turnFlowLogger.ts new file mode 100644 index 000000000..7bf38c62b --- /dev/null +++ b/packages/trueforge-core/tests/e2e/helpers/turnFlowLogger.ts @@ -0,0 +1,180 @@ +/** + * Human-readable turn flow logging for e2e learning. + * Labels which component emitted each event and where it goes next. + */ +import type { Logger } from 'winston'; +import { EventType } from '../../../src/core/events/schema'; +import type { + AgentThreadAppendContext, + AgentThreadExecutionEvent, + AgentThreadExecutionResult, +} from '../../../src/core/runtime/AgentThread.types'; +import { InternalEventType } from '../../../src/core/runtime/AgentThread.types'; + +type FlowPhase = 'send' | 'execute' | 'result'; + +function threadLabel(threadId: string | null | undefined): string { + if (threadId === null || threadId === undefined) { + return '(no thread)'; + } + return `AgentThread(${threadId})`; +} + +function contentPreview(content: unknown): string | undefined { + if (typeof content === 'string') { + return content.length > 80 ? `${content.slice(0, 80)}…` : content; + } + if (content === null || content === undefined) { + return undefined; + } + return JSON.stringify(content).slice(0, 80); +} + +function describeSendEvent(event: AgentThreadAppendContext): { + flow: string; + detail: Record; +} { + return { + flow: `Test → Orchestrator.send → ${threadLabel(event.thread_id)}.send`, + detail: { + type: event.type, + thread_id: event.thread_id, + appended_roles: event.context.map(m => ('role' in m ? m.role : 'approval')), + output_types: event.output.map(o => o.type), + }, + }; +} + +function describeExecuteEvent(event: AgentThreadExecutionEvent): { + flow: string; + detail: Record; +} { + const threadId = 'thread_id' in event ? event.thread_id : null; + + switch (event.type) { + case EventType.MODEL_MESSAGE: + return { + flow: `${threadLabel(threadId)} → Orchestrator → Test`, + detail: { + type: event.type, + thread_id: threadId, + note: 'Empty shell starts the stream; text arrives later via deltas / context.append', + content: contentPreview(event.content) ?? null, + }, + }; + case EventType.MODEL_MESSAGE_DELTA: + return { + flow: `${threadLabel(threadId)} → Orchestrator → Test`, + detail: { + type: event.type, + thread_id: threadId, + content: contentPreview(event.content), + tool_call_names: event.tool_calls?.map(tc => tc.function?.name).filter(Boolean), + }, + }; + case InternalEventType.AGENT_CONTEXT_APPEND: + return { + flow: `${threadLabel(event.thread_id)} → Orchestrator → Test (durable append)`, + detail: { + type: event.type, + thread_id: event.thread_id, + appended_roles: event.context.map(m => ('role' in m ? m.role : 'approval')), + output_types: event.output.map(o => o.type), + has_completion: Boolean(event.completion), + }, + }; + case EventType.THREAD_CREATED: + return { + flow: `Orchestrator.createDynamicSubAgentThread → new ${threadLabel(event.thread_id)} (registered in map)`, + detail: { + type: event.type, + thread_id: event.thread_id, + title: event.title, + parent: event.parent, + agent_info: event.agent_info, + note: 'Internal AGENT_CREATE_SUBAGENT was swallowed; this is what the consumer sees', + }, + }; + case EventType.TOOL_RESPONSE: + return { + flow: `Orchestrator → ${threadLabel(threadId)}.send (child result routed to parent tool call)`, + detail: { + type: event.type, + thread_id: threadId, + tool_call_id: event.tool_call_id, + content: contentPreview(event.content), + }, + }; + case InternalEventType.AGENT_DONE: { + const isChild = Boolean(event.parent); + return { + flow: isChild + ? `${threadLabel(threadId)} → Orchestrator (child done; parent will resume)` + : `${threadLabel(threadId)} → Orchestrator → Test (root done; execute stops)`, + detail: { + type: event.type, + thread_id: threadId, + status: event.status, + parent: event.parent ?? null, + output_preview: contentPreview(event.output?.content), + send_to_parent: event.send_to_parent + ? { + tool_call_id: event.send_to_parent.tool_call_id, + content: contentPreview(event.send_to_parent.content), + } + : null, + }, + }; + } + case EventType.TOOL_APPROVAL_REQUIRED: + case EventType.TOOL_RESPONSE_REQUIRED: + return { + flow: `${threadLabel(threadId)} → Orchestrator → Test (pause; wait for user send)`, + detail: { type: event.type, thread_id: threadId }, + }; + case InternalEventType.MCP_AUTH_REQUIRED: + return { + flow: `Orchestrator → Test (auth pause)`, + detail: { type: event.type }, + }; + default: + return { + flow: `${threadLabel(threadId)} → Orchestrator → Test`, + detail: { type: event.type, thread_id: threadId }, + }; + } +} + +function describeResult(result: AgentThreadExecutionResult): { + flow: string; + detail: Record; +} { + return { + flow: 'Orchestrator.execute return → Test', + detail: { + output_thread_id: result.output?.thread_id ?? null, + output_content: contentPreview(result.output?.content) ?? null, + required_actions: result.required_actions.map(a => a.type), + root_agent_error: result.root_agent_error?.error ?? null, + }, + }; +} + +export function logTurnPhase(logger: Logger, phase: FlowPhase, message: string): void { + logger.info(`──────── ${phase.toUpperCase()} ──────── ${message}`); +} + +export function logSendEvent(logger: Logger, event: AgentThreadAppendContext): void { + const { flow, detail } = describeSendEvent(event); + logger.info(`[send] ${flow}`, detail); +} + +export function logExecuteEvent(logger: Logger, event: AgentThreadExecutionEvent, index: number): void { + const { flow, detail } = describeExecuteEvent(event); + logger.info(`[execute #${String(index)}] ${flow}`, detail); +} + +export function logTurnResult(logger: Logger, result: AgentThreadExecutionResult): void { + const { flow, detail } = describeResult(result); + logger.info(`[result] ${flow}`, detail); +} diff --git a/packages/trueforge-core/tests/e2e/orchestration.test.ts b/packages/trueforge-core/tests/e2e/orchestration.test.ts index bf9a54e4b..c862cf82f 100644 --- a/packages/trueforge-core/tests/e2e/orchestration.test.ts +++ b/packages/trueforge-core/tests/e2e/orchestration.test.ts @@ -4,7 +4,7 @@ import { InternalEventType } from '../../src/core/runtime/AgentThread.types'; import { AgentThreadOrchestrator } from '../../src/core/runtime/AgentThreadOrchestrator'; import { NOOP_AGENT_TRACING } from '../../src/core/tracing/NoopAgentTracing'; import { makeSilentLogger } from '../agent-session/testHelpers'; -import { makeTextLLM } from './helpers'; +import { makeTextLLM } from './helpers/helpers'; const THREAD_ID = 'main'; const REPLY = 'hello from the mocked model'; diff --git a/packages/trueforge-core/tests/e2e/orchestrationWithTools.test.ts b/packages/trueforge-core/tests/e2e/orchestrationWithTools.test.ts index aba6dbc4b..925594009 100644 --- a/packages/trueforge-core/tests/e2e/orchestrationWithTools.test.ts +++ b/packages/trueforge-core/tests/e2e/orchestrationWithTools.test.ts @@ -2,13 +2,118 @@ import type { AgentDefinition, CreateDynamicSubAgentThread } from '../../src/cor import { DynamicSubAgents } from '../../src/core/capabilities/builtins/DynamicSubAgents'; import { EventType } from '../../src/core/events/schema'; import { AgentThread } from '../../src/core/runtime/AgentThread'; -import { type AgentThreadConstructorInput } from '../../src/core/runtime/AgentThread.types'; +import { InternalEventType, type AgentThreadConstructorInput } from '../../src/core/runtime/AgentThread.types'; import { AgentThreadOrchestrator, type AgentThreadOrchestratorInput, } from '../../src/core/runtime/AgentThreadOrchestrator'; import { NOOP_AGENT_TRACING } from '../../src/core/tracing/NoopAgentTracing'; -import { makeDummyLogger, makeRootLLM, makeTextLLM } from './helpers'; +import { makeDummyLogger, makeRootLLM, makeTextLLM } from './helpers/helpers'; +import { expectTurn, runOrchestratorTurn } from './helpers/turnExpectations'; + +const ROOT_ID = 'thread_root'; +const TOOL_CALL_ID = 'call-sub'; +const CHILD_REPLY = 'hello from the child'; +const ROOT_FINAL = 'How are you?'; + +const EXPECTED = { + sendTypes: [InternalEventType.AGENT_CONTEXT_APPEND], + executeTrace: [ + // model.message is an empty stream shell; text lands in deltas / context / result. + { + type: EventType.MODEL_MESSAGE, + thread_id: ROOT_ID, + content: null, + }, + { + type: EventType.THREAD_CREATED, + thread_id: '', + title: 'worker', + parent: { thread_id: ROOT_ID, tool_call_id: TOOL_CALL_ID }, + }, + { + type: EventType.MODEL_MESSAGE, + thread_id: '', + content: null, + }, + { + type: EventType.TOOL_RESPONSE, + thread_id: ROOT_ID, + tool_call_id: TOOL_CALL_ID, + }, + { + type: InternalEventType.AGENT_DONE, + thread_id: '', + }, + { + type: EventType.MODEL_MESSAGE, + thread_id: ROOT_ID, + content: null, + }, + { + type: InternalEventType.AGENT_DONE, + thread_id: ROOT_ID, + }, + ], + result: { + output: { thread_id: ROOT_ID, content: ROOT_FINAL }, + required_actions: [], + root_agent_error: null, + }, + context: [ + { role: 'user', content: 'hello' }, + { + role: 'assistant', + content: null, + tool_calls: [ + { + id: TOOL_CALL_ID, + function: { + name: 'create_sub_agent', + arguments: JSON.stringify({ + name: 'worker', + input: 'do the delegated task [output]', + }), + }, + }, + ], + }, + { + role: 'tool', + tool_call_id: TOOL_CALL_ID, + content: CHILD_REPLY, + }, + { + role: 'assistant', + content: ROOT_FINAL, + }, + ], +}; + +describe('core E2E: orchestrator with dynamic sub-agent', () => { + it('delegates via create_sub_agent, routes child result to parent, then finishes', async () => { + const logger = makeDummyLogger(); + const thread_1 = makeMainLLMThread(ROOT_ID, ROOT_FINAL, 'e2e-orchestration-with-tools'); + + let orchestratorInput: AgentThreadOrchestratorInput = { + agentThreads: new Map([[thread_1.threadId, thread_1]]), + createDynamicSubAgentThread: createSubAgentThread, + tracing: NOOP_AGENT_TRACING, + logger, + }; + + const orchestrator = new AgentThreadOrchestrator(orchestratorInput); + + const actual = await runOrchestratorTurn({ + orchestrator, + rootThread: thread_1, + sendBatch: [{ type: EventType.USER_MESSAGE, content: 'hello' }], + logger, + }); + + expectTurn(actual, EXPECTED); + }); +}); function makeMainLLMThread(threadId: string, reply: string, title: string): AgentThread { let agentDefinition: AgentDefinition = { @@ -48,7 +153,7 @@ function makeMainLLMThread(threadId: string, reply: string, title: string): Agen const createSubAgentThread: CreateDynamicSubAgentThread = async ({ parentDefinition, request, threadId, parent }) => { const agentDefinition: AgentDefinition = { - modelClient: makeTextLLM('hello from the child'), // Child has text only LLM, + modelClient: makeTextLLM(CHILD_REPLY), // Child has text only LLM, // Not sure if this should be taken from the parent, or left alone instruction: undefined, messages: [{ role: 'user', content: request.input }], @@ -73,45 +178,3 @@ const createSubAgentThread: CreateDynamicSubAgentThread = async ({ parentDefinit logger: makeDummyLogger(), }); }; - -describe('core E2E: orchestrator with mocked LLM and no tools', () => { - it('sends a user message and finishes the thread with a text reply', async () => { - const logger = makeDummyLogger(); - const thread_1 = makeMainLLMThread('thread_1', 'How are you?', 'e2e-orchestration-with-tools'); - - let orchestratorInput: AgentThreadOrchestratorInput = { - agentThreads: new Map([[thread_1.threadId, thread_1]]), - createDynamicSubAgentThread: createSubAgentThread, - tracing: NOOP_AGENT_TRACING, - logger, - }; - - const orchestrator = new AgentThreadOrchestrator(orchestratorInput); - - const sendTypes: string[] = []; - for await (const event of orchestrator.send([{ type: EventType.USER_MESSAGE, content: 'hello' }])) { - sendTypes.push(event.type); - } - logger.info('send complete', { sendTypes }); - - const types: string[] = []; - const iterator = orchestrator.execute({ signal: new AbortController().signal }); - let step = await iterator.next(); - while (!step.done) { - const event = step.value; - logger.info('execute event', { - type: event.type, - thread_id: 'thread_id' in event ? event.thread_id : null, - }); - types.push(event.type); - step = await iterator.next(); - } - - logger.info('execute result', { - types, - output: step.value.output?.content ?? null, - required_actions: step.value.required_actions.map(action => action.type), - root_agent_error: step.value.root_agent_error ?? null, - }); - }); -}); From 9144a3935b1e58e7e6482b939c751a29d7777023 Mon Sep 17 00:00:00 2001 From: Raman Tehlan Date: Tue, 1 Sep 2026 11:01:56 +0530 Subject: [PATCH 04/19] e2e: add orchestration with approval --- .../tests/e2e/helpers/helpers.ts | 113 +++++++++++ .../tests/e2e/helpers/turnExpectations.ts | 5 +- .../tests/e2e/orchestration.test.ts | 79 ++++---- .../tests/e2e/orchestrationApproval.test.ts | 176 ++++++++++++++++++ 4 files changed, 329 insertions(+), 44 deletions(-) create mode 100644 packages/trueforge-core/tests/e2e/orchestrationApproval.test.ts diff --git a/packages/trueforge-core/tests/e2e/helpers/helpers.ts b/packages/trueforge-core/tests/e2e/helpers/helpers.ts index dbee1100e..3d6405fbd 100644 --- a/packages/trueforge-core/tests/e2e/helpers/helpers.ts +++ b/packages/trueforge-core/tests/e2e/helpers/helpers.ts @@ -3,6 +3,14 @@ import winston from 'winston'; import type { ILLM } from '../../../src/core/llm/ILLM'; import type { ExtendedChatCompletionChunk, RawAssistantMessageWithUsage } from '../../../src/core/llm/LLMTypes'; import { getEmptyUsage } from '../../../src/core/llm/LLMTypes'; +import type { IToolSet, ToolSource } from '../../../src/core/mcp/IMCPServer'; +import { toolResultResponse } from '../../../src/core/mcp/IMCPServer'; +import { ToolSet } from '../../../src/core/mcp/ToolSet'; + +export const WRITE_NOTE_TOOL_NAME = 'write_note'; +export const WRITE_NOTE_CALL_ID = 'call-write'; +export const WRITE_NOTE_ARGUMENTS = JSON.stringify({ text: 'hello' }); +export const WRITE_NOTE_RESULT = 'note written'; /** One streamed chunk plus a stop completion. Used when the test needs a text reply and no tool calls. */ // eslint-disable-next-line @typescript-eslint/require-await -- async generator fixture, not awaiting I/O @@ -91,6 +99,111 @@ export function makeRootLLM(finalReply: string): ILLM { }; } +export async function* writeNoteToolCallStream() { + yield { + id: 'chunk-write-note', + object: 'chat.completion.chunk', + created: 0, + model: 'test-model', + choices: [ + { + index: 0, + delta: { + role: 'assistant', + tool_calls: [ + { + index: 0, + id: WRITE_NOTE_CALL_ID, + type: 'function', + function: { + name: WRITE_NOTE_TOOL_NAME, + arguments: WRITE_NOTE_ARGUMENTS, + }, + }, + ], + }, + finish_reason: 'tool_calls', + }, + ], + }; + + return { + output: { + role: 'assistant', + content: null, + tool_calls: [ + { + id: WRITE_NOTE_CALL_ID, + type: 'function', + function: { + name: WRITE_NOTE_TOOL_NAME, + arguments: WRITE_NOTE_ARGUMENTS, + }, + }, + ], + }, + usage: getEmptyUsage(), + finish_reason: 'tool_calls', + }; +} + +/** First create() requests write_note; later calls stream `finalReply`. */ +export function makeApprovalThenTextLLM(finalReply: string): ILLM { + return { + create: jest + .fn() + .mockImplementationOnce(() => writeNoteToolCallStream()) + .mockImplementation(() => textReplyStream(finalReply)), + createNonStream: jest.fn(), + }; +} + +function makeWriteNoteSource(): ToolSource { + return { + name: 'notes', + id: 'notes', + listTools: () => + Promise.resolve({ + result: { + tools: [ + { + name: WRITE_NOTE_TOOL_NAME, + description: 'Write a note', + inputSchema: { + type: 'object', + properties: { text: { type: 'string' } }, + }, + preload: true, + }, + ], + }, + wasInitialized: undefined, + }), + callTool: () => Promise.resolve(toolResultResponse({ text: WRITE_NOTE_RESULT })), + toolCallInfo: () => + Promise.resolve({ + type: 'mcp', + mcp_server_id: 'notes', + mcp_server_name: 'notes', + original_tool_name: WRITE_NOTE_TOOL_NAME, + }), + }; +} + +/** User tool set that pauses until write_note is approved. */ +export function makeApprovalGatedWriteNoteToolSet(): IToolSet { + return new ToolSet({ + source: makeWriteNoteSource(), + selectors: { + enableTools: ['@all'], + disableTools: [], + preloadTools: [], + requireApprovalForTools: [WRITE_NOTE_TOOL_NAME], + }, + preload: true, + }); +} + export function makeDummyLogger(): Logger { const logger = winston.createLogger({ level: 'debug', diff --git a/packages/trueforge-core/tests/e2e/helpers/turnExpectations.ts b/packages/trueforge-core/tests/e2e/helpers/turnExpectations.ts index e1630d4f1..043ec836d 100644 --- a/packages/trueforge-core/tests/e2e/helpers/turnExpectations.ts +++ b/packages/trueforge-core/tests/e2e/helpers/turnExpectations.ts @@ -79,6 +79,9 @@ function projectExecuteEvent(event: AgentThreadExecutionEvent): ExecuteTraceRow return { ...base, content: contentToString(event.content) }; case EventType.TOOL_RESPONSE: return { ...base, tool_call_id: event.tool_call_id }; + case EventType.TOOL_APPROVAL_REQUIRED: + case EventType.TOOL_RESPONSE_REQUIRED: + return { ...base, tool_call_id: event.tool_calls[0]?.id }; case EventType.THREAD_CREATED: return { ...base, @@ -96,7 +99,7 @@ function projectExecuteEvent(event: AgentThreadExecutionEvent): ExecuteTraceRow function projectContext(context: ContextMessage[]): ContextRow[] { return context.map((msg): ContextRow => { if (!isLLMContextMessage(msg)) { - return { role: 'approval_decision' }; + return { role: 'approval_decision', tool_call_id: msg.tool_call_id }; } if (msg.role === 'user') { return { diff --git a/packages/trueforge-core/tests/e2e/orchestration.test.ts b/packages/trueforge-core/tests/e2e/orchestration.test.ts index c862cf82f..4525f54da 100644 --- a/packages/trueforge-core/tests/e2e/orchestration.test.ts +++ b/packages/trueforge-core/tests/e2e/orchestration.test.ts @@ -3,19 +3,44 @@ import { AgentThread } from '../../src/core/runtime/AgentThread'; import { InternalEventType } from '../../src/core/runtime/AgentThread.types'; import { AgentThreadOrchestrator } from '../../src/core/runtime/AgentThreadOrchestrator'; import { NOOP_AGENT_TRACING } from '../../src/core/tracing/NoopAgentTracing'; -import { makeSilentLogger } from '../agent-session/testHelpers'; -import { makeTextLLM } from './helpers/helpers'; +import { makeDummyLogger, makeTextLLM } from './helpers/helpers'; +import { expectTurn, runOrchestratorTurn } from './helpers/turnExpectations'; const THREAD_ID = 'main'; const REPLY = 'hello from the mocked model'; +const EXPECTED = { + sendTypes: [InternalEventType.AGENT_CONTEXT_APPEND], + executeTrace: [ + // model.message is an empty stream shell; text lands in deltas / context / result. + { + type: EventType.MODEL_MESSAGE, + thread_id: THREAD_ID, + content: null, + }, + { + type: InternalEventType.AGENT_DONE, + thread_id: THREAD_ID, + }, + ], + result: { + output: { thread_id: THREAD_ID, content: REPLY }, + required_actions: [], + root_agent_error: null, + }, + context: [ + { role: 'user', content: 'hello' }, + { role: 'assistant', content: REPLY }, + ], +}; + /** Root thread with a one-shot text LLM and no tool sets. */ function makeTextLlmThread(): AgentThread { return new AgentThread({ threadId: THREAD_ID, title: 'e2e-orchestration', tracing: NOOP_AGENT_TRACING, - logger: makeSilentLogger(), + logger: makeDummyLogger(), definition: { modelClient: makeTextLLM(REPLY), instruction: 'You are running in a test setup.', @@ -25,6 +50,7 @@ function makeTextLlmThread(): AgentThread { describe('core E2E: orchestrator with mocked LLM and no tools', () => { it('sends a user message and finishes the thread with a text reply', async () => { + const logger = makeDummyLogger(); const thread = makeTextLlmThread(); // Orchestrator owns the thread map and fans send/execute across live threads. // This case has only the root thread, so sub-agent creation must never run. @@ -32,49 +58,16 @@ describe('core E2E: orchestrator with mocked LLM and no tools', () => { agentThreads: new Map([[thread.threadId, thread]]), createDynamicSubAgentThread: () => Promise.reject(new Error('unexpected sub-agent in no-tool test')), tracing: NOOP_AGENT_TRACING, - logger: makeSilentLogger(), + logger, }); - // send() commits user input into thread context; it does not call the LLM. - const sendTypes: string[] = []; - for await (const event of orchestrator.send([{ type: EventType.USER_MESSAGE, content: 'hello' }])) { - sendTypes.push(event.type); - } - expect(sendTypes).toEqual([InternalEventType.AGENT_CONTEXT_APPEND]); - - // execute() runs the LLM loop. Manual next() is required so we can read the - // generator's return value (AgentThreadExecutionResult) after the last yield. - const types: string[] = []; - const iterator = orchestrator.execute({ signal: new AbortController().signal }); - let step = await iterator.next(); - while (!step.done) { - types.push(step.value.type); - step = await iterator.next(); - } - - // Happy path: stream the reply, then a terminal AGENT_DONE. No tools or child threads. - expect(types).toContain(EventType.MODEL_MESSAGE_DELTA); - expect(types).toContain(EventType.MODEL_MESSAGE); - expect(types[types.length - 1]).toBe(InternalEventType.AGENT_DONE); - expect(types).not.toContain(EventType.TOOL_RESPONSE); - expect(types).not.toContain(EventType.THREAD_CREATED); - - // Result is the orchestrator return, not an event: final assistant output, no pause/error. - expect(step.value.required_actions).toEqual([]); - expect(step.value.root_agent_error).toBeUndefined(); - expect(step.value.output).toMatchObject({ - type: EventType.MODEL_MESSAGE, - thread_id: THREAD_ID, - content: REPLY, + const actual = await runOrchestratorTurn({ + orchestrator, + rootThread: thread, + sendBatch: [{ type: EventType.USER_MESSAGE, content: 'hello' }], + logger, }); - // Durable thread context after send + execute: user turn plus the assistant reply. - const snapshot = thread.toSnapshot(); - expect(snapshot.context).toEqual( - expect.arrayContaining([ - expect.objectContaining({ role: 'user', content: 'hello' }), - expect.objectContaining({ role: 'assistant', content: REPLY }), - ]), - ); + expectTurn(actual, EXPECTED); }); }); diff --git a/packages/trueforge-core/tests/e2e/orchestrationApproval.test.ts b/packages/trueforge-core/tests/e2e/orchestrationApproval.test.ts new file mode 100644 index 000000000..acfd4c58d --- /dev/null +++ b/packages/trueforge-core/tests/e2e/orchestrationApproval.test.ts @@ -0,0 +1,176 @@ +import type { AgentDefinition } from '../../src/core'; +import { EventType } from '../../src/core/events/schema'; +import { AgentThread } from '../../src/core/runtime/AgentThread'; +import { InternalEventType, type AgentThreadConstructorInput } from '../../src/core/runtime/AgentThread.types'; +import { AgentThreadOrchestrator } from '../../src/core/runtime/AgentThreadOrchestrator'; +import { NOOP_AGENT_TRACING } from '../../src/core/tracing/NoopAgentTracing'; +import { + makeApprovalGatedWriteNoteToolSet, + makeApprovalThenTextLLM, + makeDummyLogger, + WRITE_NOTE_ARGUMENTS, + WRITE_NOTE_CALL_ID, + WRITE_NOTE_RESULT, + WRITE_NOTE_TOOL_NAME, +} from './helpers/helpers'; +import { expectTurn, runOrchestratorTurn } from './helpers/turnExpectations'; + +const ROOT_ID = 'thread_root'; +const ROOT_FINAL = 'note saved'; + +const EXPECTED_PAUSE = { + sendTypes: [InternalEventType.AGENT_CONTEXT_APPEND], + executeTrace: [ + { + type: EventType.MODEL_MESSAGE, + thread_id: ROOT_ID, + content: null, + }, + { + type: EventType.TOOL_APPROVAL_REQUIRED, + thread_id: ROOT_ID, + tool_call_id: WRITE_NOTE_CALL_ID, + }, + ], + result: { + output: null, + required_actions: [EventType.TOOL_APPROVAL_REQUIRED], + root_agent_error: null, + }, + context: [ + { role: 'user', content: 'hello' }, + { + role: 'assistant', + content: null, + tool_calls: [ + { + id: WRITE_NOTE_CALL_ID, + function: { + name: WRITE_NOTE_TOOL_NAME, + arguments: WRITE_NOTE_ARGUMENTS, + }, + }, + ], + }, + ], +}; + +const EXPECTED_RESUME = { + sendTypes: [InternalEventType.AGENT_CONTEXT_APPEND], + executeTrace: [ + { + type: EventType.TOOL_RESPONSE, + thread_id: ROOT_ID, + tool_call_id: WRITE_NOTE_CALL_ID, + }, + { + type: EventType.MODEL_MESSAGE, + thread_id: ROOT_ID, + content: null, + }, + { + type: InternalEventType.AGENT_DONE, + thread_id: ROOT_ID, + }, + ], + result: { + output: { thread_id: ROOT_ID, content: ROOT_FINAL }, + required_actions: [], + root_agent_error: null, + }, + context: [ + { role: 'user', content: 'hello' }, + { + role: 'assistant', + content: null, + tool_calls: [ + { + id: WRITE_NOTE_CALL_ID, + function: { + name: WRITE_NOTE_TOOL_NAME, + arguments: WRITE_NOTE_ARGUMENTS, + }, + }, + ], + }, + { + role: 'approval_decision', + tool_call_id: WRITE_NOTE_CALL_ID, + }, + { + role: 'tool', + tool_call_id: WRITE_NOTE_CALL_ID, + content: WRITE_NOTE_RESULT, + }, + { + role: 'assistant', + content: ROOT_FINAL, + }, + ], +}; + +describe('core E2E: orchestrator pause then resume on tool approval', () => { + it('pauses for write_note approval, then finishes after allow', async () => { + const logger = makeDummyLogger(); + const thread = makeApprovalThread(); + const orchestrator = new AgentThreadOrchestrator({ + agentThreads: new Map([[thread.threadId, thread]]), + createDynamicSubAgentThread: () => Promise.reject(new Error('unexpected sub-agent in approval test')), + tracing: NOOP_AGENT_TRACING, + logger, + }); + + const paused = await runOrchestratorTurn({ + orchestrator, + rootThread: thread, + sendBatch: [{ type: EventType.USER_MESSAGE, content: 'hello' }], + logger, + }); + expectTurn(paused, EXPECTED_PAUSE); + + const resumed = await runOrchestratorTurn({ + orchestrator, + rootThread: thread, + sendBatch: [ + { + type: EventType.USER_TOOL_APPROVAL, + thread_id: ROOT_ID, + tool_call_id: WRITE_NOTE_CALL_ID, + approval: { status: 'allow' }, + }, + ], + logger, + }); + expectTurn(resumed, EXPECTED_RESUME); + }); +}); + +function makeApprovalThread(): AgentThread { + const agentDefinition: AgentDefinition = { + modelClient: makeApprovalThenTextLLM(ROOT_FINAL), + instruction: 'You are running in a test setup.', + messages: undefined, + modelParams: undefined, + responseFormat: undefined, + iterationLimit: undefined, + toolSets: [makeApprovalGatedWriteNoteToolSet()], + }; + + const agentThreadInput: AgentThreadConstructorInput = { + definition: agentDefinition, + threadId: ROOT_ID, + title: 'e2e-orchestration-approval', + parent: undefined, + agentInfo: undefined, + context: undefined, + currentContextUsage: undefined, + preComputedCompletion: undefined, + sandbox: undefined, + capabilities: undefined, + capabilityState: undefined, + tracing: NOOP_AGENT_TRACING, + logger: makeDummyLogger(), + }; + + return new AgentThread(agentThreadInput); +} From 9b8835d95a1e866b32297ade09564a2d9cc5340c Mon Sep 17 00:00:00 2001 From: Raman Tehlan Date: Tue, 1 Sep 2026 13:47:24 +0530 Subject: [PATCH 05/19] e2e: lint fix --- .../tests/e2e/helpers/helpers.ts | 22 +++++++------- .../tests/e2e/helpers/turnExpectations.ts | 29 +++++++++++-------- .../tests/e2e/orchestrationWithTools.test.ts | 2 +- 3 files changed, 29 insertions(+), 24 deletions(-) diff --git a/packages/trueforge-core/tests/e2e/helpers/helpers.ts b/packages/trueforge-core/tests/e2e/helpers/helpers.ts index 3d6405fbd..a9582b42c 100644 --- a/packages/trueforge-core/tests/e2e/helpers/helpers.ts +++ b/packages/trueforge-core/tests/e2e/helpers/helpers.ts @@ -31,19 +31,18 @@ export async function* textReplyStream( }; } +// eslint-disable-next-line @typescript-eslint/require-await -- async generator fixture, not awaiting I/O export async function* createSubAgentStream() { yield { id: 'chunk-tool', object: 'chat.completion.chunk', created: 0, model: 'test-model', - // Choices choices: [ { index: 0, delta: { role: 'assistant', - // Tool Calls tool_calls: [ { index: 0, @@ -54,12 +53,12 @@ export async function* createSubAgentStream() { arguments: JSON.stringify({ name: 'worker', input: 'do the delegated task' }), }, }, - ], // tool calls end - }, // Delta end + ], + }, finish_reason: 'tool_calls', }, - ], // Choices end - }; // yeild end + ], + }; return { output: { @@ -71,15 +70,15 @@ export async function* createSubAgentStream() { type: 'function', function: { name: 'create_sub_agent', - arguments: JSON.stringify({ name: 'worker', input: 'do the delegated task [output]' }), - }, // Function end + arguments: JSON.stringify({ name: 'worker', input: 'do the delegated task' }), + }, }, - ], // Tool calls end - }, // Output end + ], + }, usage: getEmptyUsage(), finish_reason: 'tool_calls', }; -} // function end +} /** ILLM that always streams `text` and then stops. */ export function makeTextLLM(text: string): ILLM { @@ -99,6 +98,7 @@ export function makeRootLLM(finalReply: string): ILLM { }; } +// eslint-disable-next-line @typescript-eslint/require-await -- async generator fixture, not awaiting I/O export async function* writeNoteToolCallStream() { yield { id: 'chunk-write-note', diff --git a/packages/trueforge-core/tests/e2e/helpers/turnExpectations.ts b/packages/trueforge-core/tests/e2e/helpers/turnExpectations.ts index 043ec836d..a226b2c7e 100644 --- a/packages/trueforge-core/tests/e2e/helpers/turnExpectations.ts +++ b/packages/trueforge-core/tests/e2e/helpers/turnExpectations.ts @@ -25,37 +25,37 @@ import { logExecuteEvent, logSendEvent, logTurnPhase, logTurnResult } from './tu /** Placeholder substituted for the runtime-minted child thread id. */ export const CHILD_THREAD_PLACEHOLDER = ''; -export type ExecuteTraceRow = { +export interface ExecuteTraceRow { type: string; thread_id: string | null; tool_call_id?: string; content?: string | null; title?: string; parent?: { thread_id: string; tool_call_id: string }; -}; +} -export type ContextRow = { +export interface ContextRow { role: string; content?: string | null; tool_call_id?: string; - tool_calls?: Array<{ + tool_calls?: { id: string; function: { name: string; arguments: string }; - }>; -}; + }[]; +} -export type TurnResultRow = { +export interface TurnResultRow { output: { thread_id: string; content: string | null } | null; required_actions: string[]; root_agent_error: { error: string } | null; -}; +} -export type TurnActual = { +export interface TurnActual { sendTypes: string[]; executeTrace: ExecuteTraceRow[]; result: TurnResultRow; context: ContextRow[]; -}; +} export type TurnExpected = TurnActual; @@ -80,8 +80,13 @@ function projectExecuteEvent(event: AgentThreadExecutionEvent): ExecuteTraceRow case EventType.TOOL_RESPONSE: return { ...base, tool_call_id: event.tool_call_id }; case EventType.TOOL_APPROVAL_REQUIRED: - case EventType.TOOL_RESPONSE_REQUIRED: - return { ...base, tool_call_id: event.tool_calls[0]?.id }; + case EventType.TOOL_RESPONSE_REQUIRED: { + const toolCallId = event.tool_calls[0]?.id; + if (toolCallId === undefined) { + return base; + } + return { ...base, tool_call_id: toolCallId }; + } case EventType.THREAD_CREATED: return { ...base, diff --git a/packages/trueforge-core/tests/e2e/orchestrationWithTools.test.ts b/packages/trueforge-core/tests/e2e/orchestrationWithTools.test.ts index 925594009..4942f3242 100644 --- a/packages/trueforge-core/tests/e2e/orchestrationWithTools.test.ts +++ b/packages/trueforge-core/tests/e2e/orchestrationWithTools.test.ts @@ -72,7 +72,7 @@ const EXPECTED = { name: 'create_sub_agent', arguments: JSON.stringify({ name: 'worker', - input: 'do the delegated task [output]', + input: 'do the delegated task', }), }, }, From 5c66bbc222a4601c7b009dc7c6918442bbaf4fb5 Mon Sep 17 00:00:00 2001 From: Raman Tehlan Date: Tue, 1 Sep 2026 16:29:34 +0530 Subject: [PATCH 06/19] orchestration test: update name | add debug run | remove context match | remove logging & normalization --- .vscode/launch.json | 18 ++ package.json | 3 +- packages/trueforge-core/jest.config.cjs | 4 +- ...nfig.cjs => jest.orchestration.config.cjs} | 6 +- packages/trueforge-core/package.json | 5 +- .../tests/e2e/helpers/turnExpectations.ts | 240 ------------------ .../tests/e2e/helpers/turnFlowLogger.ts | 180 ------------- .../tests/e2e/orchestrationApproval.test.ts | 176 ------------- .../tests/{e2e => orchestration}/README.md | 57 ++--- .../{e2e => orchestration}/helpers/helpers.ts | 39 +-- .../orchestration.test.ts | 54 ++-- .../orchestrationApproval.test.ts | 117 +++++++++ .../orchestrationWithTools.test.ts | 118 +++------ packages/trueforge-core/tsconfig.json | 2 +- 14 files changed, 242 insertions(+), 777 deletions(-) create mode 100644 .vscode/launch.json rename packages/trueforge-core/{jest.e2e.config.cjs => jest.orchestration.config.cjs} (81%) delete mode 100644 packages/trueforge-core/tests/e2e/helpers/turnExpectations.ts delete mode 100644 packages/trueforge-core/tests/e2e/helpers/turnFlowLogger.ts delete mode 100644 packages/trueforge-core/tests/e2e/orchestrationApproval.test.ts rename packages/trueforge-core/tests/{e2e => orchestration}/README.md (85%) rename packages/trueforge-core/tests/{e2e => orchestration}/helpers/helpers.ts (85%) rename packages/trueforge-core/tests/{e2e => orchestration}/orchestration.test.ts (57%) create mode 100644 packages/trueforge-core/tests/orchestration/orchestrationApproval.test.ts rename packages/trueforge-core/tests/{e2e => orchestration}/orchestrationWithTools.test.ts (59%) diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 000000000..4037fdd27 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,18 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "type": "pwa-node", + "request": "launch", + "name": "Debug trueforge-core orchestration (current file)", + "cwd": "${workspaceFolder}/packages/trueforge-core", + "runtimeExecutable": "pnpm", + "runtimeArgs": ["test:orchestration", "--", "--runInBand", "--testTimeout", "0", "${file}"], + "console": "integratedTerminal", + "autoAttachChildProcesses": true, + "skipFiles": ["/**", "**/node_modules/**"], + "sourceMaps": true, + "resolveSourceMapLocations": ["${workspaceFolder}/packages/trueforge-core/**", "!**/node_modules/**"] + } + ] +} diff --git a/package.json b/package.json index 38734630d..f1e5ea6b0 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,8 @@ "test:frontend": "pnpm --filter frontend test", "test:chart-version": "bash tests/scripts/resolve-chart-version.test.sh", "test:trueforge-core": "pnpm --filter @truefoundry/trueforge-core test", - "test:trueforge-core:e2e": "pnpm --filter @truefoundry/trueforge-core test:e2e", + "test:trueforge-core:orchestration": "pnpm --filter @truefoundry/trueforge-core test:orchestration", + "test:trueforge-core:orchestration:debug": "pnpm --filter @truefoundry/trueforge-core test:orchestration:debug", "test:trueforge": "pnpm --filter @truefoundry/trueforge test", "test:local-sandbox:contract": "pnpm --filter @truefoundry/trueforge test:local-sandbox:contract", "smoke:local-sandbox": "pnpm --filter @truefoundry/trueforge smoke:local-sandbox", diff --git a/packages/trueforge-core/jest.config.cjs b/packages/trueforge-core/jest.config.cjs index 93d553ff8..31df920f6 100644 --- a/packages/trueforge-core/jest.config.cjs +++ b/packages/trueforge-core/jest.config.cjs @@ -37,6 +37,6 @@ module.exports = { roots: ['/tests'], testMatch: ['**/tests/**/*.test.ts'], // Compile-time suites are enforced by `tsc --noEmit`, not the Jest runner. - // E2E lives under tests/e2e and is run via jest.e2e.config.cjs. - testPathIgnorePatterns: ['\\.compile\\.test\\.ts$', '/tests/e2e/'], + // Orchestration tests live under tests/orchestration and are run via jest.orchestration.config.cjs. + testPathIgnorePatterns: ['\\.compile\\.test\\.ts$', '/tests/orchestration/'], }; diff --git a/packages/trueforge-core/jest.e2e.config.cjs b/packages/trueforge-core/jest.orchestration.config.cjs similarity index 81% rename from packages/trueforge-core/jest.e2e.config.cjs rename to packages/trueforge-core/jest.orchestration.config.cjs index d15ca7019..d042e3bbf 100644 --- a/packages/trueforge-core/jest.e2e.config.cjs +++ b/packages/trueforge-core/jest.orchestration.config.cjs @@ -10,6 +10,7 @@ module.exports = { target: 'es2022', }, module: { type: 'commonjs' }, + sourceMaps: 'inline', }, ], '^.+\\.js$': [ @@ -20,6 +21,7 @@ module.exports = { target: 'es2022', }, module: { type: 'commonjs' }, + sourceMaps: 'inline', }, ], }, @@ -30,6 +32,6 @@ module.exports = { setupFilesAfterEnv: ['/tests/setup.ts'], testTimeout: 60_000, maxWorkers: 1, - roots: ['/tests/e2e'], - testMatch: ['/tests/e2e/**/*.test.ts'], + roots: ['/tests/orchestration'], + testMatch: ['/tests/orchestration/**/*.test.ts'], }; diff --git a/packages/trueforge-core/package.json b/packages/trueforge-core/package.json index 691e33f11..ac0ba17b8 100644 --- a/packages/trueforge-core/package.json +++ b/packages/trueforge-core/package.json @@ -95,8 +95,9 @@ "build:pkg": "node scripts/write-dist-package-json.mjs", "build:check": "node scripts/check-dist.mjs", "typecheck": "pnpm run build:gen && tsc --noEmit", - "test": "pnpm run build:gen && jest --config jest.config.cjs && jest --config jest.e2e.config.cjs", - "test:e2e": "pnpm run build:gen && jest --config jest.e2e.config.cjs", + "test": "pnpm run build:gen && jest --config jest.config.cjs && jest --config jest.orchestration.config.cjs", + "test:orchestration": "pnpm run build:gen && jest --config jest.orchestration.config.cjs", + "test:orchestration:debug": "pnpm run build:gen && node --inspect-brk ./node_modules/jest/bin/jest.js --config jest.orchestration.config.cjs --runInBand --testTimeout 0", "pack:dry": "pnpm pack --dry-run" }, "dependencies": { diff --git a/packages/trueforge-core/tests/e2e/helpers/turnExpectations.ts b/packages/trueforge-core/tests/e2e/helpers/turnExpectations.ts deleted file mode 100644 index a226b2c7e..000000000 --- a/packages/trueforge-core/tests/e2e/helpers/turnExpectations.ts +++ /dev/null @@ -1,240 +0,0 @@ -/** - * Collect and normalize an orchestrator turn so tests can compare - * one `expected` object to one `actual` object. - * - * Layout of a TurnActual / TurnExpected: - * sendTypes - event types from send() - * executeTrace - compact projected events from execute() - * result - generator return value (stripped) - * context - root thread snapshot context (stripped) - */ -import type { Logger } from 'winston'; -import { EventType } from '../../../src/core/events/schema'; -import type { AgentThread } from '../../../src/core/runtime/AgentThread'; -import type { - AgentThreadExecutionEvent, - AgentThreadExecutionResult, - AgentThreadSendBatch, - ContextMessage, -} from '../../../src/core/runtime/AgentThread.types'; -import { InternalEventType } from '../../../src/core/runtime/AgentThread.types'; -import type { AgentThreadOrchestrator } from '../../../src/core/runtime/AgentThreadOrchestrator'; -import { isLLMContextMessage } from '../../../src/core/runtime/contextUtils'; -import { logExecuteEvent, logSendEvent, logTurnPhase, logTurnResult } from './turnFlowLogger'; - -/** Placeholder substituted for the runtime-minted child thread id. */ -export const CHILD_THREAD_PLACEHOLDER = ''; - -export interface ExecuteTraceRow { - type: string; - thread_id: string | null; - tool_call_id?: string; - content?: string | null; - title?: string; - parent?: { thread_id: string; tool_call_id: string }; -} - -export interface ContextRow { - role: string; - content?: string | null; - tool_call_id?: string; - tool_calls?: { - id: string; - function: { name: string; arguments: string }; - }[]; -} - -export interface TurnResultRow { - output: { thread_id: string; content: string | null } | null; - required_actions: string[]; - root_agent_error: { error: string } | null; -} - -export interface TurnActual { - sendTypes: string[]; - executeTrace: ExecuteTraceRow[]; - result: TurnResultRow; - context: ContextRow[]; -} - -export type TurnExpected = TurnActual; - -/** Normalize OpenAI-style content to a plain string or null for comparisons. */ -function contentToString(content: unknown): string | null { - if (typeof content === 'string') { - return content; - } - if (content === null || content === undefined) { - return null; - } - return JSON.stringify(content); -} - -function projectExecuteEvent(event: AgentThreadExecutionEvent): ExecuteTraceRow { - const threadId = 'thread_id' in event ? (event.thread_id ?? null) : null; - const base: ExecuteTraceRow = { type: event.type, thread_id: threadId }; - - switch (event.type) { - case EventType.MODEL_MESSAGE: - return { ...base, content: contentToString(event.content) }; - case EventType.TOOL_RESPONSE: - return { ...base, tool_call_id: event.tool_call_id }; - case EventType.TOOL_APPROVAL_REQUIRED: - case EventType.TOOL_RESPONSE_REQUIRED: { - const toolCallId = event.tool_calls[0]?.id; - if (toolCallId === undefined) { - return base; - } - return { ...base, tool_call_id: toolCallId }; - } - case EventType.THREAD_CREATED: - return { - ...base, - title: event.title, - parent: { - thread_id: event.parent.thread_id, - tool_call_id: event.parent.tool_call_id, - }, - }; - default: - return base; - } -} - -function projectContext(context: ContextMessage[]): ContextRow[] { - return context.map((msg): ContextRow => { - if (!isLLMContextMessage(msg)) { - return { role: 'approval_decision', tool_call_id: msg.tool_call_id }; - } - if (msg.role === 'user') { - return { - role: 'user', - content: contentToString(msg.content), - }; - } - if (msg.role === 'assistant') { - const row: ContextRow = { - role: 'assistant', - content: contentToString(msg.content), - }; - if (msg.tool_calls) { - row.tool_calls = msg.tool_calls.map(tc => ({ - id: tc.id, - function: { - name: tc.function.name, - arguments: tc.function.arguments, - }, - })); - } - return row; - } - return { - role: 'tool', - tool_call_id: msg.tool_call_id, - content: msg.content, - }; - }); -} - -function projectResult(result: AgentThreadExecutionResult): TurnResultRow { - return { - output: result.output - ? { - thread_id: result.output.thread_id, - content: contentToString(result.output.content), - } - : null, - required_actions: result.required_actions.map(action => action.type), - root_agent_error: result.root_agent_error ? { error: result.root_agent_error.error } : null, - }; -} - -/** Drop noisy append / delta events so the trace reads as the turn story. */ -export function compactExecuteTrace(trace: ExecuteTraceRow[]): ExecuteTraceRow[] { - return trace.filter( - row => row.type !== InternalEventType.AGENT_CONTEXT_APPEND && row.type !== EventType.MODEL_MESSAGE_DELTA, - ); -} - -function normalizeChildThreadId(trace: ExecuteTraceRow[], childThreadId: string | null): ExecuteTraceRow[] { - if (childThreadId === null) { - return trace; - } - return trace.map(row => { - if (row.thread_id !== childThreadId) { - return row; - } - return { ...row, thread_id: CHILD_THREAD_PLACEHOLDER }; - }); -} - -/** - * Run send() then execute() and return a normalized turn snapshot - * suitable for expected-vs-actual comparison. - * - * Pass `logger` to print every event with source → destination labels - * (see turnFlowLogger.ts). - */ -export async function runOrchestratorTurn(input: { - orchestrator: AgentThreadOrchestrator; - rootThread: AgentThread; - sendBatch: AgentThreadSendBatch; - signal?: AbortSignal | undefined; - logger?: Logger | undefined; -}): Promise { - const { logger } = input; - - if (logger) { - logTurnPhase(logger, 'send', 'append user/tool input into thread context (no LLM call)'); - } - - const sendTypes: string[] = []; - for await (const event of input.orchestrator.send(input.sendBatch)) { - sendTypes.push(event.type); - if (logger) { - logSendEvent(logger, event); - } - } - - if (logger) { - logTurnPhase(logger, 'execute', 'run leaf threads; orchestrator merges streams and routes sub-agents'); - } - - const rawExecuteTrace: ExecuteTraceRow[] = []; - const iterator = input.orchestrator.execute({ - signal: input.signal ?? new AbortController().signal, - }); - - let step = await iterator.next(); - let executeIndex = 0; - while (!step.done) { - if (logger) { - logExecuteEvent(logger, step.value, executeIndex); - executeIndex += 1; - } - rawExecuteTrace.push(projectExecuteEvent(step.value)); - step = await iterator.next(); - } - - if (logger) { - logTurnPhase(logger, 'result', 'generator return value (not a streamed event)'); - logTurnResult(logger, step.value); - } - - const childThreadId = rawExecuteTrace.find(row => row.type === EventType.THREAD_CREATED)?.thread_id ?? null; - - return { - sendTypes, - executeTrace: normalizeChildThreadId(compactExecuteTrace(rawExecuteTrace), childThreadId), - result: projectResult(step.value), - context: projectContext(input.rootThread.toSnapshot().context), - }; -} - -/** Layered compare so a failure names sendTypes / executeTrace / result / context. */ -export function expectTurn(actual: TurnActual, expected: TurnExpected): void { - expect(actual.sendTypes).toEqual(expected.sendTypes); - expect(actual.executeTrace).toEqual(expected.executeTrace); - expect(actual.result).toEqual(expected.result); - expect(actual.context).toEqual(expected.context); -} diff --git a/packages/trueforge-core/tests/e2e/helpers/turnFlowLogger.ts b/packages/trueforge-core/tests/e2e/helpers/turnFlowLogger.ts deleted file mode 100644 index 7bf38c62b..000000000 --- a/packages/trueforge-core/tests/e2e/helpers/turnFlowLogger.ts +++ /dev/null @@ -1,180 +0,0 @@ -/** - * Human-readable turn flow logging for e2e learning. - * Labels which component emitted each event and where it goes next. - */ -import type { Logger } from 'winston'; -import { EventType } from '../../../src/core/events/schema'; -import type { - AgentThreadAppendContext, - AgentThreadExecutionEvent, - AgentThreadExecutionResult, -} from '../../../src/core/runtime/AgentThread.types'; -import { InternalEventType } from '../../../src/core/runtime/AgentThread.types'; - -type FlowPhase = 'send' | 'execute' | 'result'; - -function threadLabel(threadId: string | null | undefined): string { - if (threadId === null || threadId === undefined) { - return '(no thread)'; - } - return `AgentThread(${threadId})`; -} - -function contentPreview(content: unknown): string | undefined { - if (typeof content === 'string') { - return content.length > 80 ? `${content.slice(0, 80)}…` : content; - } - if (content === null || content === undefined) { - return undefined; - } - return JSON.stringify(content).slice(0, 80); -} - -function describeSendEvent(event: AgentThreadAppendContext): { - flow: string; - detail: Record; -} { - return { - flow: `Test → Orchestrator.send → ${threadLabel(event.thread_id)}.send`, - detail: { - type: event.type, - thread_id: event.thread_id, - appended_roles: event.context.map(m => ('role' in m ? m.role : 'approval')), - output_types: event.output.map(o => o.type), - }, - }; -} - -function describeExecuteEvent(event: AgentThreadExecutionEvent): { - flow: string; - detail: Record; -} { - const threadId = 'thread_id' in event ? event.thread_id : null; - - switch (event.type) { - case EventType.MODEL_MESSAGE: - return { - flow: `${threadLabel(threadId)} → Orchestrator → Test`, - detail: { - type: event.type, - thread_id: threadId, - note: 'Empty shell starts the stream; text arrives later via deltas / context.append', - content: contentPreview(event.content) ?? null, - }, - }; - case EventType.MODEL_MESSAGE_DELTA: - return { - flow: `${threadLabel(threadId)} → Orchestrator → Test`, - detail: { - type: event.type, - thread_id: threadId, - content: contentPreview(event.content), - tool_call_names: event.tool_calls?.map(tc => tc.function?.name).filter(Boolean), - }, - }; - case InternalEventType.AGENT_CONTEXT_APPEND: - return { - flow: `${threadLabel(event.thread_id)} → Orchestrator → Test (durable append)`, - detail: { - type: event.type, - thread_id: event.thread_id, - appended_roles: event.context.map(m => ('role' in m ? m.role : 'approval')), - output_types: event.output.map(o => o.type), - has_completion: Boolean(event.completion), - }, - }; - case EventType.THREAD_CREATED: - return { - flow: `Orchestrator.createDynamicSubAgentThread → new ${threadLabel(event.thread_id)} (registered in map)`, - detail: { - type: event.type, - thread_id: event.thread_id, - title: event.title, - parent: event.parent, - agent_info: event.agent_info, - note: 'Internal AGENT_CREATE_SUBAGENT was swallowed; this is what the consumer sees', - }, - }; - case EventType.TOOL_RESPONSE: - return { - flow: `Orchestrator → ${threadLabel(threadId)}.send (child result routed to parent tool call)`, - detail: { - type: event.type, - thread_id: threadId, - tool_call_id: event.tool_call_id, - content: contentPreview(event.content), - }, - }; - case InternalEventType.AGENT_DONE: { - const isChild = Boolean(event.parent); - return { - flow: isChild - ? `${threadLabel(threadId)} → Orchestrator (child done; parent will resume)` - : `${threadLabel(threadId)} → Orchestrator → Test (root done; execute stops)`, - detail: { - type: event.type, - thread_id: threadId, - status: event.status, - parent: event.parent ?? null, - output_preview: contentPreview(event.output?.content), - send_to_parent: event.send_to_parent - ? { - tool_call_id: event.send_to_parent.tool_call_id, - content: contentPreview(event.send_to_parent.content), - } - : null, - }, - }; - } - case EventType.TOOL_APPROVAL_REQUIRED: - case EventType.TOOL_RESPONSE_REQUIRED: - return { - flow: `${threadLabel(threadId)} → Orchestrator → Test (pause; wait for user send)`, - detail: { type: event.type, thread_id: threadId }, - }; - case InternalEventType.MCP_AUTH_REQUIRED: - return { - flow: `Orchestrator → Test (auth pause)`, - detail: { type: event.type }, - }; - default: - return { - flow: `${threadLabel(threadId)} → Orchestrator → Test`, - detail: { type: event.type, thread_id: threadId }, - }; - } -} - -function describeResult(result: AgentThreadExecutionResult): { - flow: string; - detail: Record; -} { - return { - flow: 'Orchestrator.execute return → Test', - detail: { - output_thread_id: result.output?.thread_id ?? null, - output_content: contentPreview(result.output?.content) ?? null, - required_actions: result.required_actions.map(a => a.type), - root_agent_error: result.root_agent_error?.error ?? null, - }, - }; -} - -export function logTurnPhase(logger: Logger, phase: FlowPhase, message: string): void { - logger.info(`──────── ${phase.toUpperCase()} ──────── ${message}`); -} - -export function logSendEvent(logger: Logger, event: AgentThreadAppendContext): void { - const { flow, detail } = describeSendEvent(event); - logger.info(`[send] ${flow}`, detail); -} - -export function logExecuteEvent(logger: Logger, event: AgentThreadExecutionEvent, index: number): void { - const { flow, detail } = describeExecuteEvent(event); - logger.info(`[execute #${String(index)}] ${flow}`, detail); -} - -export function logTurnResult(logger: Logger, result: AgentThreadExecutionResult): void { - const { flow, detail } = describeResult(result); - logger.info(`[result] ${flow}`, detail); -} diff --git a/packages/trueforge-core/tests/e2e/orchestrationApproval.test.ts b/packages/trueforge-core/tests/e2e/orchestrationApproval.test.ts deleted file mode 100644 index acfd4c58d..000000000 --- a/packages/trueforge-core/tests/e2e/orchestrationApproval.test.ts +++ /dev/null @@ -1,176 +0,0 @@ -import type { AgentDefinition } from '../../src/core'; -import { EventType } from '../../src/core/events/schema'; -import { AgentThread } from '../../src/core/runtime/AgentThread'; -import { InternalEventType, type AgentThreadConstructorInput } from '../../src/core/runtime/AgentThread.types'; -import { AgentThreadOrchestrator } from '../../src/core/runtime/AgentThreadOrchestrator'; -import { NOOP_AGENT_TRACING } from '../../src/core/tracing/NoopAgentTracing'; -import { - makeApprovalGatedWriteNoteToolSet, - makeApprovalThenTextLLM, - makeDummyLogger, - WRITE_NOTE_ARGUMENTS, - WRITE_NOTE_CALL_ID, - WRITE_NOTE_RESULT, - WRITE_NOTE_TOOL_NAME, -} from './helpers/helpers'; -import { expectTurn, runOrchestratorTurn } from './helpers/turnExpectations'; - -const ROOT_ID = 'thread_root'; -const ROOT_FINAL = 'note saved'; - -const EXPECTED_PAUSE = { - sendTypes: [InternalEventType.AGENT_CONTEXT_APPEND], - executeTrace: [ - { - type: EventType.MODEL_MESSAGE, - thread_id: ROOT_ID, - content: null, - }, - { - type: EventType.TOOL_APPROVAL_REQUIRED, - thread_id: ROOT_ID, - tool_call_id: WRITE_NOTE_CALL_ID, - }, - ], - result: { - output: null, - required_actions: [EventType.TOOL_APPROVAL_REQUIRED], - root_agent_error: null, - }, - context: [ - { role: 'user', content: 'hello' }, - { - role: 'assistant', - content: null, - tool_calls: [ - { - id: WRITE_NOTE_CALL_ID, - function: { - name: WRITE_NOTE_TOOL_NAME, - arguments: WRITE_NOTE_ARGUMENTS, - }, - }, - ], - }, - ], -}; - -const EXPECTED_RESUME = { - sendTypes: [InternalEventType.AGENT_CONTEXT_APPEND], - executeTrace: [ - { - type: EventType.TOOL_RESPONSE, - thread_id: ROOT_ID, - tool_call_id: WRITE_NOTE_CALL_ID, - }, - { - type: EventType.MODEL_MESSAGE, - thread_id: ROOT_ID, - content: null, - }, - { - type: InternalEventType.AGENT_DONE, - thread_id: ROOT_ID, - }, - ], - result: { - output: { thread_id: ROOT_ID, content: ROOT_FINAL }, - required_actions: [], - root_agent_error: null, - }, - context: [ - { role: 'user', content: 'hello' }, - { - role: 'assistant', - content: null, - tool_calls: [ - { - id: WRITE_NOTE_CALL_ID, - function: { - name: WRITE_NOTE_TOOL_NAME, - arguments: WRITE_NOTE_ARGUMENTS, - }, - }, - ], - }, - { - role: 'approval_decision', - tool_call_id: WRITE_NOTE_CALL_ID, - }, - { - role: 'tool', - tool_call_id: WRITE_NOTE_CALL_ID, - content: WRITE_NOTE_RESULT, - }, - { - role: 'assistant', - content: ROOT_FINAL, - }, - ], -}; - -describe('core E2E: orchestrator pause then resume on tool approval', () => { - it('pauses for write_note approval, then finishes after allow', async () => { - const logger = makeDummyLogger(); - const thread = makeApprovalThread(); - const orchestrator = new AgentThreadOrchestrator({ - agentThreads: new Map([[thread.threadId, thread]]), - createDynamicSubAgentThread: () => Promise.reject(new Error('unexpected sub-agent in approval test')), - tracing: NOOP_AGENT_TRACING, - logger, - }); - - const paused = await runOrchestratorTurn({ - orchestrator, - rootThread: thread, - sendBatch: [{ type: EventType.USER_MESSAGE, content: 'hello' }], - logger, - }); - expectTurn(paused, EXPECTED_PAUSE); - - const resumed = await runOrchestratorTurn({ - orchestrator, - rootThread: thread, - sendBatch: [ - { - type: EventType.USER_TOOL_APPROVAL, - thread_id: ROOT_ID, - tool_call_id: WRITE_NOTE_CALL_ID, - approval: { status: 'allow' }, - }, - ], - logger, - }); - expectTurn(resumed, EXPECTED_RESUME); - }); -}); - -function makeApprovalThread(): AgentThread { - const agentDefinition: AgentDefinition = { - modelClient: makeApprovalThenTextLLM(ROOT_FINAL), - instruction: 'You are running in a test setup.', - messages: undefined, - modelParams: undefined, - responseFormat: undefined, - iterationLimit: undefined, - toolSets: [makeApprovalGatedWriteNoteToolSet()], - }; - - const agentThreadInput: AgentThreadConstructorInput = { - definition: agentDefinition, - threadId: ROOT_ID, - title: 'e2e-orchestration-approval', - parent: undefined, - agentInfo: undefined, - context: undefined, - currentContextUsage: undefined, - preComputedCompletion: undefined, - sandbox: undefined, - capabilities: undefined, - capabilityState: undefined, - tracing: NOOP_AGENT_TRACING, - logger: makeDummyLogger(), - }; - - return new AgentThread(agentThreadInput); -} diff --git a/packages/trueforge-core/tests/e2e/README.md b/packages/trueforge-core/tests/orchestration/README.md similarity index 85% rename from packages/trueforge-core/tests/e2e/README.md rename to packages/trueforge-core/tests/orchestration/README.md index 0ae1ad3c0..86cb0d025 100644 --- a/packages/trueforge-core/tests/e2e/README.md +++ b/packages/trueforge-core/tests/orchestration/README.md @@ -1,4 +1,4 @@ -# Core runtime E2E tests +# Orchestration tests End-to-end tests for `AgentThreadOrchestrator` and `AgentThread` in `@truefoundry/trueforge-core`. @@ -23,7 +23,7 @@ Production creates the orchestrator inside `SessionHandle.createTurn`: resolve definitions → build AgentThread map → new AgentThreadOrchestrator → send → persist → execute (via TurnHandle) ``` -These E2E tests **skip the store and session layer** and talk to the orchestrator directly. That keeps the surface area small while still exercising the same `send` / `execute` contract production uses. +These orchestration tests **skip the store and session layer** and talk to the orchestrator directly. That keeps the surface area small while still exercising the same `send` / `execute` contract production uses. ```mermaid flowchart LR @@ -35,7 +35,7 @@ flowchart LR SH --> OrchP end - subgraph e2e["E2E tests"] + subgraph orch["Orchestration tests"] Test[Jest test] OrchE[AgentThreadOrchestrator] MockLLM[Mock ILLM] @@ -48,12 +48,12 @@ flowchart LR ## Files -| File | Role | -| -------------------------------- | -------------------------------------------------------------------------------------------------- | -| `orchestration.test.ts` | **Program 1** - single root thread, text-only reply, full assertions | -| `orchestrationWithTools.test.ts` | **Program 2** - root delegates to sub-agent via `create_sub_agent`, logging only (assertions TODO) | -| `helpers.ts` | Mock LLM streams, logger factory | -| `jest.e2e.config.cjs` | Jest config scoped to this folder | +| File | Role | +| -------------------------------- | -------------------------------------------------------------------- | +| `orchestration.test.ts` | **Program 1** - single root thread, text-only reply, full assertions | +| `orchestrationWithTools.test.ts` | **Program 2** - root delegates to sub-agent via `create_sub_agent` | +| `helpers.ts` | Mock LLM streams and approval-gated tools | +| `jest.orchestration.config.cjs` | Jest config scoped to this folder | ## Core components under test @@ -129,7 +129,6 @@ sequenceDiagram | `makeTextLLM(text)` | `ILLM` that always replies with `text` (used for child threads) | | `createSubAgentStream()` | First root call: stream a `create_sub_agent` tool call | | `makeRootLLM(finalReply)` | First `create()` → sub-agent tool call; every later call → `finalReply` text | -| `makeDummyLogger()` | Winston logger with colorized console output for debugging | Root and child threads use **different** `ILLM` instances so each can follow its own scripted sequence. @@ -314,19 +313,6 @@ internal.agent.done ← root finished (last event) 4. assistant: "How are you?" ``` -### Current test status - -Program 2 currently **logs** events and the final result via `makeDummyLogger()`. It does **not** yet assert on event types or final state. Add the same style of expectations as Program 1 when ready. - -Suggested assertions to add: - -```ts -expect(types).toContain(EventType.THREAD_CREATED); -expect(types).toContain(EventType.TOOL_RESPONSE); -expect(types.at(-1)).toBe(InternalEventType.AGENT_DONE); -expect(step.value.output?.content).toBe('How are you?'); -``` - --- ## Running tests @@ -334,40 +320,29 @@ expect(step.value.output?.content).toBe('How are you?'); From `packages/trueforge-core`: ```bash -pnpm test:e2e +pnpm test:orchestration ``` Single file: ```bash -pnpm test:e2e -- orchestration.test.ts -pnpm test:e2e -- orchestrationWithTools.test.ts +pnpm test:orchestration -- orchestration.test.ts +pnpm test:orchestration -- orchestrationWithTools.test.ts ``` From repo root: ```bash -pnpm test:trueforge-core:e2e +pnpm test:trueforge-core:orchestration ``` -E2E tests use `jest.e2e.config.cjs` (`maxWorkers: 1`, 60s timeout). Unit tests under `tests/` (excluding `tests/e2e/`) run separately via `jest.config.cjs`. - -## Logging during tests - -- `tests/setup.ts` mocks `console.log` / `console.warn` / `console.error` for all Jest runs, including E2E. -- `makeDummyLogger()` uses Winston's `Console` transport and **does** print to the terminal. -- Program 2 logs: - - `send complete` with event types - - each `execute event` with `type` and `thread_id` - - `execute result` with full type list and terminal output - -The orchestrator itself does not log on the happy path. Test-side logging is intentional for learning. +Orchestration tests use `jest.orchestration.config.cjs` (`maxWorkers: 1`, 60s timeout). Unit tests under `tests/` (excluding `tests/orchestration/`) run separately via `jest.config.cjs`. -To debug with less noise, run a single file (see above). +Threads and the orchestrator still take a Winston logger (required by the runtime). These tests use `makeSilentLogger()` from `tests/core/harnessMocks.ts`, so the suite does not print turn flow. ## Relationship to production -| E2E test | Production equivalent | +| Orchestration test | Production equivalent | | -------------------------------------- | ------------------------------------------------------ | | `new AgentThread({ definition, ... })` | `SessionHandle.buildThreads` + resolver | | `createSubAgentThread` callback | `SessionHandle.makeCreateDynamicSubAgentThread` | diff --git a/packages/trueforge-core/tests/e2e/helpers/helpers.ts b/packages/trueforge-core/tests/orchestration/helpers/helpers.ts similarity index 85% rename from packages/trueforge-core/tests/e2e/helpers/helpers.ts rename to packages/trueforge-core/tests/orchestration/helpers/helpers.ts index a9582b42c..b94c73615 100644 --- a/packages/trueforge-core/tests/e2e/helpers/helpers.ts +++ b/packages/trueforge-core/tests/orchestration/helpers/helpers.ts @@ -1,11 +1,15 @@ -import type { Logger } from 'winston'; -import winston from 'winston'; import type { ILLM } from '../../../src/core/llm/ILLM'; import type { ExtendedChatCompletionChunk, RawAssistantMessageWithUsage } from '../../../src/core/llm/LLMTypes'; import { getEmptyUsage } from '../../../src/core/llm/LLMTypes'; import type { IToolSet, ToolSource } from '../../../src/core/mcp/IMCPServer'; import { toolResultResponse } from '../../../src/core/mcp/IMCPServer'; import { ToolSet } from '../../../src/core/mcp/ToolSet'; +import type { + AgentThreadExecutionEvent, + AgentThreadExecutionResult, + AgentThreadSendBatch, +} from '../../../src/core/runtime/AgentThread.types'; +import type { AgentThreadOrchestrator } from '../../../src/core/runtime/AgentThreadOrchestrator'; export const WRITE_NOTE_TOOL_NAME = 'write_note'; export const WRITE_NOTE_CALL_ID = 'call-write'; @@ -204,18 +208,23 @@ export function makeApprovalGatedWriteNoteToolSet(): IToolSet { }); } -export function makeDummyLogger(): Logger { - const logger = winston.createLogger({ - level: 'debug', - format: winston.format.combine( - winston.format.colorize(), - winston.format.printf(({ level, message, ...meta }) => { - const details = Object.keys(meta).length > 0 ? ` ${JSON.stringify(meta)}` : ''; - return `${level}: ${String(message)}${details}`; - }), - ), - transports: [new winston.transports.Console()], +/** Consume send() then execute(); return raw events and the generator result. */ +export async function runTurn(input: { + orchestrator: AgentThreadOrchestrator; + sendBatch: AgentThreadSendBatch; + signal?: AbortSignal | undefined; +}): Promise<{ events: AgentThreadExecutionEvent[]; result: AgentThreadExecutionResult }> { + for await (const _event of input.orchestrator.send(input.sendBatch)) { + void _event; + } + const events: AgentThreadExecutionEvent[] = []; + const iterator = input.orchestrator.execute({ + signal: input.signal ?? new AbortController().signal, }); - logger.child = () => logger; - return logger; + let step = await iterator.next(); + while (!step.done) { + events.push(step.value); + step = await iterator.next(); + } + return { events, result: step.value }; } diff --git a/packages/trueforge-core/tests/e2e/orchestration.test.ts b/packages/trueforge-core/tests/orchestration/orchestration.test.ts similarity index 57% rename from packages/trueforge-core/tests/e2e/orchestration.test.ts rename to packages/trueforge-core/tests/orchestration/orchestration.test.ts index 4525f54da..4d3175d8b 100644 --- a/packages/trueforge-core/tests/e2e/orchestration.test.ts +++ b/packages/trueforge-core/tests/orchestration/orchestration.test.ts @@ -3,44 +3,31 @@ import { AgentThread } from '../../src/core/runtime/AgentThread'; import { InternalEventType } from '../../src/core/runtime/AgentThread.types'; import { AgentThreadOrchestrator } from '../../src/core/runtime/AgentThreadOrchestrator'; import { NOOP_AGENT_TRACING } from '../../src/core/tracing/NoopAgentTracing'; -import { makeDummyLogger, makeTextLLM } from './helpers/helpers'; -import { expectTurn, runOrchestratorTurn } from './helpers/turnExpectations'; +import { makeSilentLogger } from '../core/harnessMocks'; +import { makeTextLLM, runTurn } from './helpers/helpers'; const THREAD_ID = 'main'; const REPLY = 'hello from the mocked model'; -const EXPECTED = { - sendTypes: [InternalEventType.AGENT_CONTEXT_APPEND], - executeTrace: [ - // model.message is an empty stream shell; text lands in deltas / context / result. - { - type: EventType.MODEL_MESSAGE, - thread_id: THREAD_ID, - content: null, - }, - { - type: InternalEventType.AGENT_DONE, - thread_id: THREAD_ID, - }, - ], - result: { - output: { thread_id: THREAD_ID, content: REPLY }, - required_actions: [], - root_agent_error: null, - }, - context: [ - { role: 'user', content: 'hello' }, - { role: 'assistant', content: REPLY }, - ], +const EXPECTED_EVENTS = [ + { type: EventType.MODEL_MESSAGE, thread_id: THREAD_ID }, + { type: EventType.MODEL_MESSAGE_DELTA, thread_id: THREAD_ID, content: REPLY }, + { type: InternalEventType.AGENT_CONTEXT_APPEND, thread_id: THREAD_ID }, + { type: InternalEventType.AGENT_DONE, thread_id: THREAD_ID, status: 'done' }, +]; + +const OUTPUT = { + output: { thread_id: THREAD_ID, content: REPLY }, + required_actions: [], }; /** Root thread with a one-shot text LLM and no tool sets. */ function makeTextLlmThread(): AgentThread { return new AgentThread({ threadId: THREAD_ID, - title: 'e2e-orchestration', + title: 'orchestration', tracing: NOOP_AGENT_TRACING, - logger: makeDummyLogger(), + logger: makeSilentLogger(), definition: { modelClient: makeTextLLM(REPLY), instruction: 'You are running in a test setup.', @@ -48,9 +35,8 @@ function makeTextLlmThread(): AgentThread { }); } -describe('core E2E: orchestrator with mocked LLM and no tools', () => { +describe('orchestration: mocked LLM and no tools', () => { it('sends a user message and finishes the thread with a text reply', async () => { - const logger = makeDummyLogger(); const thread = makeTextLlmThread(); // Orchestrator owns the thread map and fans send/execute across live threads. // This case has only the root thread, so sub-agent creation must never run. @@ -58,16 +44,16 @@ describe('core E2E: orchestrator with mocked LLM and no tools', () => { agentThreads: new Map([[thread.threadId, thread]]), createDynamicSubAgentThread: () => Promise.reject(new Error('unexpected sub-agent in no-tool test')), tracing: NOOP_AGENT_TRACING, - logger, + logger: makeSilentLogger(), }); - const actual = await runOrchestratorTurn({ + const { events, result } = await runTurn({ orchestrator, - rootThread: thread, sendBatch: [{ type: EventType.USER_MESSAGE, content: 'hello' }], - logger, }); - expectTurn(actual, EXPECTED); + expect(events).toMatchObject(EXPECTED_EVENTS); + expect(result).toMatchObject(OUTPUT); + expect(result.root_agent_error).toBeUndefined(); }); }); diff --git a/packages/trueforge-core/tests/orchestration/orchestrationApproval.test.ts b/packages/trueforge-core/tests/orchestration/orchestrationApproval.test.ts new file mode 100644 index 000000000..be06249ad --- /dev/null +++ b/packages/trueforge-core/tests/orchestration/orchestrationApproval.test.ts @@ -0,0 +1,117 @@ +import type { AgentDefinition } from '../../src/core'; +import { EventType } from '../../src/core/events/schema'; +import { AgentThread } from '../../src/core/runtime/AgentThread'; +import { InternalEventType, type AgentThreadConstructorInput } from '../../src/core/runtime/AgentThread.types'; +import { AgentThreadOrchestrator } from '../../src/core/runtime/AgentThreadOrchestrator'; +import { NOOP_AGENT_TRACING } from '../../src/core/tracing/NoopAgentTracing'; +import { makeSilentLogger } from '../core/harnessMocks'; +import { + makeApprovalGatedWriteNoteToolSet, + makeApprovalThenTextLLM, + runTurn, + WRITE_NOTE_CALL_ID, +} from './helpers/helpers'; + +const ROOT_ID = 'thread_root'; +const ROOT_FINAL = 'note saved'; + +const EXPECTED_PAUSE_EVENTS = [ + { type: EventType.MODEL_MESSAGE, thread_id: ROOT_ID }, + { type: EventType.MODEL_MESSAGE_DELTA, thread_id: ROOT_ID }, + { type: InternalEventType.AGENT_CONTEXT_APPEND, thread_id: ROOT_ID }, + { + type: EventType.TOOL_APPROVAL_REQUIRED, + thread_id: ROOT_ID, + tool_calls: [{ id: WRITE_NOTE_CALL_ID }], + }, +]; + +const PAUSE_OUTPUT = { + output: null, + required_actions: [ + { + type: EventType.TOOL_APPROVAL_REQUIRED, + thread_id: ROOT_ID, + tool_calls: [{ id: WRITE_NOTE_CALL_ID }], + }, + ], +}; + +const EXPECTED_RESUME_EVENTS = [ + { type: EventType.TOOL_RESPONSE, thread_id: ROOT_ID, tool_call_id: WRITE_NOTE_CALL_ID }, + { type: InternalEventType.AGENT_CONTEXT_APPEND, thread_id: ROOT_ID }, + { type: EventType.MODEL_MESSAGE, thread_id: ROOT_ID }, + { type: EventType.MODEL_MESSAGE_DELTA, thread_id: ROOT_ID, content: ROOT_FINAL }, + { type: InternalEventType.AGENT_CONTEXT_APPEND, thread_id: ROOT_ID }, + { type: InternalEventType.AGENT_DONE, thread_id: ROOT_ID, status: 'done' }, +]; + +const RESUME_OUTPUT = { + output: { thread_id: ROOT_ID, content: ROOT_FINAL }, + required_actions: [], +}; + +describe('orchestration: pause then resume on tool approval', () => { + it('pauses for write_note approval, then finishes after allow', async () => { + const thread = makeApprovalThread(); + const orchestrator = new AgentThreadOrchestrator({ + agentThreads: new Map([[thread.threadId, thread]]), + createDynamicSubAgentThread: () => Promise.reject(new Error('unexpected sub-agent in approval test')), + tracing: NOOP_AGENT_TRACING, + logger: makeSilentLogger(), + }); + + const paused = await runTurn({ + orchestrator, + sendBatch: [{ type: EventType.USER_MESSAGE, content: 'hello' }], + }); + expect(paused.events).toMatchObject(EXPECTED_PAUSE_EVENTS); + expect(paused.result).toMatchObject(PAUSE_OUTPUT); + expect(paused.result.root_agent_error).toBeUndefined(); + + const resumed = await runTurn({ + orchestrator, + sendBatch: [ + { + type: EventType.USER_TOOL_APPROVAL, + thread_id: ROOT_ID, + tool_call_id: WRITE_NOTE_CALL_ID, + approval: { status: 'allow' }, + }, + ], + }); + expect(resumed.events).toMatchObject(EXPECTED_RESUME_EVENTS); + expect(resumed.result).toMatchObject(RESUME_OUTPUT); + expect(resumed.result.root_agent_error).toBeUndefined(); + }); +}); + +function makeApprovalThread(): AgentThread { + const agentDefinition: AgentDefinition = { + modelClient: makeApprovalThenTextLLM(ROOT_FINAL), + instruction: 'You are running in a test setup.', + messages: undefined, + modelParams: undefined, + responseFormat: undefined, + iterationLimit: undefined, + toolSets: [makeApprovalGatedWriteNoteToolSet()], + }; + + const agentThreadInput: AgentThreadConstructorInput = { + definition: agentDefinition, + threadId: ROOT_ID, + title: 'orchestration-approval', + parent: undefined, + agentInfo: undefined, + context: undefined, + currentContextUsage: undefined, + preComputedCompletion: undefined, + sandbox: undefined, + capabilities: undefined, + capabilityState: undefined, + tracing: NOOP_AGENT_TRACING, + logger: makeSilentLogger(), + }; + + return new AgentThread(agentThreadInput); +} diff --git a/packages/trueforge-core/tests/e2e/orchestrationWithTools.test.ts b/packages/trueforge-core/tests/orchestration/orchestrationWithTools.test.ts similarity index 59% rename from packages/trueforge-core/tests/e2e/orchestrationWithTools.test.ts rename to packages/trueforge-core/tests/orchestration/orchestrationWithTools.test.ts index 4942f3242..2c6c3a17b 100644 --- a/packages/trueforge-core/tests/e2e/orchestrationWithTools.test.ts +++ b/packages/trueforge-core/tests/orchestration/orchestrationWithTools.test.ts @@ -8,110 +8,62 @@ import { type AgentThreadOrchestratorInput, } from '../../src/core/runtime/AgentThreadOrchestrator'; import { NOOP_AGENT_TRACING } from '../../src/core/tracing/NoopAgentTracing'; -import { makeDummyLogger, makeRootLLM, makeTextLLM } from './helpers/helpers'; -import { expectTurn, runOrchestratorTurn } from './helpers/turnExpectations'; +import { makeSilentLogger } from '../core/harnessMocks'; +import { makeRootLLM, makeTextLLM, runTurn } from './helpers/helpers'; const ROOT_ID = 'thread_root'; const TOOL_CALL_ID = 'call-sub'; const CHILD_REPLY = 'hello from the child'; const ROOT_FINAL = 'How are you?'; -const EXPECTED = { - sendTypes: [InternalEventType.AGENT_CONTEXT_APPEND], - executeTrace: [ - // model.message is an empty stream shell; text lands in deltas / context / result. - { - type: EventType.MODEL_MESSAGE, - thread_id: ROOT_ID, - content: null, - }, - { - type: EventType.THREAD_CREATED, - thread_id: '', - title: 'worker', - parent: { thread_id: ROOT_ID, tool_call_id: TOOL_CALL_ID }, - }, - { - type: EventType.MODEL_MESSAGE, - thread_id: '', - content: null, - }, - { - type: EventType.TOOL_RESPONSE, - thread_id: ROOT_ID, - tool_call_id: TOOL_CALL_ID, - }, - { - type: InternalEventType.AGENT_DONE, - thread_id: '', - }, - { - type: EventType.MODEL_MESSAGE, - thread_id: ROOT_ID, - content: null, - }, - { - type: InternalEventType.AGENT_DONE, - thread_id: ROOT_ID, - }, - ], - result: { - output: { thread_id: ROOT_ID, content: ROOT_FINAL }, - required_actions: [], - root_agent_error: null, +const EXPECTED_EVENTS = [ + { type: EventType.MODEL_MESSAGE, thread_id: ROOT_ID }, + { type: EventType.MODEL_MESSAGE_DELTA, thread_id: ROOT_ID }, + { type: InternalEventType.AGENT_CONTEXT_APPEND, thread_id: ROOT_ID }, + { type: InternalEventType.AGENT_CONTEXT_APPEND, thread_id: ROOT_ID }, + { + type: EventType.THREAD_CREATED, + title: 'worker', + parent: { thread_id: ROOT_ID, tool_call_id: TOOL_CALL_ID }, }, - context: [ - { role: 'user', content: 'hello' }, - { - role: 'assistant', - content: null, - tool_calls: [ - { - id: TOOL_CALL_ID, - function: { - name: 'create_sub_agent', - arguments: JSON.stringify({ - name: 'worker', - input: 'do the delegated task', - }), - }, - }, - ], - }, - { - role: 'tool', - tool_call_id: TOOL_CALL_ID, - content: CHILD_REPLY, - }, - { - role: 'assistant', - content: ROOT_FINAL, - }, - ], + { type: EventType.MODEL_MESSAGE, thread_id: expect.any(String) }, + { type: EventType.MODEL_MESSAGE_DELTA, thread_id: expect.any(String), content: CHILD_REPLY }, + { type: InternalEventType.AGENT_CONTEXT_APPEND, thread_id: expect.any(String) }, + { type: EventType.TOOL_RESPONSE, thread_id: ROOT_ID, tool_call_id: TOOL_CALL_ID }, + { type: InternalEventType.AGENT_CONTEXT_APPEND, thread_id: ROOT_ID }, + { type: InternalEventType.AGENT_DONE, thread_id: expect.any(String), status: 'done' }, + { type: EventType.MODEL_MESSAGE, thread_id: ROOT_ID }, + { type: EventType.MODEL_MESSAGE_DELTA, thread_id: ROOT_ID, content: ROOT_FINAL }, + { type: InternalEventType.AGENT_CONTEXT_APPEND, thread_id: ROOT_ID }, + { type: InternalEventType.AGENT_DONE, thread_id: ROOT_ID, status: 'done' }, +]; + +const OUTPUT = { + output: { thread_id: ROOT_ID, content: ROOT_FINAL }, + required_actions: [], }; -describe('core E2E: orchestrator with dynamic sub-agent', () => { +describe('orchestration: dynamic sub-agent', () => { it('delegates via create_sub_agent, routes child result to parent, then finishes', async () => { - const logger = makeDummyLogger(); - const thread_1 = makeMainLLMThread(ROOT_ID, ROOT_FINAL, 'e2e-orchestration-with-tools'); + const thread_1 = makeMainLLMThread(ROOT_ID, ROOT_FINAL, 'orchestration-with-tools'); let orchestratorInput: AgentThreadOrchestratorInput = { agentThreads: new Map([[thread_1.threadId, thread_1]]), createDynamicSubAgentThread: createSubAgentThread, tracing: NOOP_AGENT_TRACING, - logger, + logger: makeSilentLogger(), }; const orchestrator = new AgentThreadOrchestrator(orchestratorInput); - const actual = await runOrchestratorTurn({ + const { events, result } = await runTurn({ orchestrator, - rootThread: thread_1, sendBatch: [{ type: EventType.USER_MESSAGE, content: 'hello' }], - logger, }); - expectTurn(actual, EXPECTED); + expect(events).toMatchObject(EXPECTED_EVENTS); + expect(result).toMatchObject(OUTPUT); + expect(result.root_agent_error).toBeUndefined(); }); }); @@ -143,7 +95,7 @@ function makeMainLLMThread(threadId: string, reply: string, title: string): Agen capabilityState: undefined, // Default tracing: NOOP_AGENT_TRACING, - logger: makeDummyLogger(), + logger: makeSilentLogger(), }; let agentThread = new AgentThread(agentThreadInput); @@ -175,6 +127,6 @@ const createSubAgentThread: CreateDynamicSubAgentThread = async ({ parentDefinit capabilities: undefined, capabilityState: undefined, tracing: NOOP_AGENT_TRACING, - logger: makeDummyLogger(), + logger: makeSilentLogger(), }); }; diff --git a/packages/trueforge-core/tsconfig.json b/packages/trueforge-core/tsconfig.json index 3883bd9f1..97562af5e 100644 --- a/packages/trueforge-core/tsconfig.json +++ b/packages/trueforge-core/tsconfig.json @@ -12,6 +12,6 @@ "openai/resources/chat": ["./node_modules/openai/resources/chat/index.d.ts"] } }, - "include": ["src/**/*", "tests/**/*", "tsup.config.ts", "jest.config.cjs", "jest.e2e.config.cjs"], + "include": ["src/**/*", "tests/**/*", "tsup.config.ts", "jest.config.cjs", "jest.orchestration.config.cjs"], "exclude": ["node_modules", "dist"] } From ee2791b239d411821e4ccf4fcd1b4e7d4a0606a5 Mon Sep 17 00:00:00 2001 From: Raman Tehlan Date: Tue, 1 Sep 2026 16:43:18 +0530 Subject: [PATCH 07/19] orchestration test: add note about the test --- .../tests/orchestration/helpers/helpers.ts | 4 + .../tests/orchestration/orchestration.test.ts | 17 ++- .../orchestrationApproval.test.ts | 53 +++++++- .../orchestrationWithTools.test.ts | 127 +++++++++++++----- 4 files changed, 168 insertions(+), 33 deletions(-) diff --git a/packages/trueforge-core/tests/orchestration/helpers/helpers.ts b/packages/trueforge-core/tests/orchestration/helpers/helpers.ts index b94c73615..1f1ad4b9f 100644 --- a/packages/trueforge-core/tests/orchestration/helpers/helpers.ts +++ b/packages/trueforge-core/tests/orchestration/helpers/helpers.ts @@ -228,3 +228,7 @@ export async function runTurn(input: { } return { events, result: step.value }; } + +export function llmCreateInputs(llm: ILLM): unknown[] { + return jest.mocked(llm).create.mock.calls.map(call => call[0]); +} diff --git a/packages/trueforge-core/tests/orchestration/orchestration.test.ts b/packages/trueforge-core/tests/orchestration/orchestration.test.ts index 4d3175d8b..af3aa5591 100644 --- a/packages/trueforge-core/tests/orchestration/orchestration.test.ts +++ b/packages/trueforge-core/tests/orchestration/orchestration.test.ts @@ -4,11 +4,13 @@ import { InternalEventType } from '../../src/core/runtime/AgentThread.types'; import { AgentThreadOrchestrator } from '../../src/core/runtime/AgentThreadOrchestrator'; import { NOOP_AGENT_TRACING } from '../../src/core/tracing/NoopAgentTracing'; import { makeSilentLogger } from '../core/harnessMocks'; -import { makeTextLLM, runTurn } from './helpers/helpers'; +import { llmCreateInputs, makeTextLLM, runTurn } from './helpers/helpers'; const THREAD_ID = 'main'; const REPLY = 'hello from the mocked model'; +const INSTRUCTION = 'You are running in a test setup.'; +/** One root thread, no tools: user message in, text reply out. */ const EXPECTED_EVENTS = [ { type: EventType.MODEL_MESSAGE, thread_id: THREAD_ID }, { type: EventType.MODEL_MESSAGE_DELTA, thread_id: THREAD_ID, content: REPLY }, @@ -21,6 +23,16 @@ const OUTPUT = { required_actions: [], }; +const EXPECTED_LLM_INPUT = [ + { + stream: true, + messages: [ + { role: 'system', content: expect.stringContaining(INSTRUCTION) }, + { role: 'user', content: 'hello' }, + ], + }, +]; + /** Root thread with a one-shot text LLM and no tool sets. */ function makeTextLlmThread(): AgentThread { return new AgentThread({ @@ -30,7 +42,7 @@ function makeTextLlmThread(): AgentThread { logger: makeSilentLogger(), definition: { modelClient: makeTextLLM(REPLY), - instruction: 'You are running in a test setup.', + instruction: INSTRUCTION, }, }); } @@ -55,5 +67,6 @@ describe('orchestration: mocked LLM and no tools', () => { expect(events).toMatchObject(EXPECTED_EVENTS); expect(result).toMatchObject(OUTPUT); expect(result.root_agent_error).toBeUndefined(); + expect(llmCreateInputs(thread.definition.modelClient)).toMatchObject(EXPECTED_LLM_INPUT); }); }); diff --git a/packages/trueforge-core/tests/orchestration/orchestrationApproval.test.ts b/packages/trueforge-core/tests/orchestration/orchestrationApproval.test.ts index be06249ad..de8d6ad9d 100644 --- a/packages/trueforge-core/tests/orchestration/orchestrationApproval.test.ts +++ b/packages/trueforge-core/tests/orchestration/orchestrationApproval.test.ts @@ -6,15 +6,29 @@ import { AgentThreadOrchestrator } from '../../src/core/runtime/AgentThreadOrche import { NOOP_AGENT_TRACING } from '../../src/core/tracing/NoopAgentTracing'; import { makeSilentLogger } from '../core/harnessMocks'; import { + llmCreateInputs, makeApprovalGatedWriteNoteToolSet, makeApprovalThenTextLLM, runTurn, + WRITE_NOTE_ARGUMENTS, WRITE_NOTE_CALL_ID, + WRITE_NOTE_RESULT, + WRITE_NOTE_TOOL_NAME, } from './helpers/helpers'; const ROOT_ID = 'thread_root'; const ROOT_FINAL = 'note saved'; +const INSTRUCTION = 'You are running in a test setup.'; +const WRITE_NOTE_TOOLS = [ + { function: { name: 'call_tool' } }, + { function: { name: 'get_tool_info' } }, + { function: { name: 'get_tool_output_schema' } }, + { function: { name: 'list_tools' } }, + { function: { name: WRITE_NOTE_TOOL_NAME } }, +]; + +/** Pause on write_note approval, then resume after allow and finish. */ const EXPECTED_PAUSE_EVENTS = [ { type: EventType.MODEL_MESSAGE, thread_id: ROOT_ID }, { type: EventType.MODEL_MESSAGE_DELTA, thread_id: ROOT_ID }, @@ -37,6 +51,17 @@ const PAUSE_OUTPUT = { ], }; +const EXPECTED_PAUSE_LLM_INPUT = [ + { + stream: true, + tools: WRITE_NOTE_TOOLS, + messages: [ + { role: 'system', content: expect.stringContaining(INSTRUCTION) }, + { role: 'user', content: 'hello' }, + ], + }, +]; + const EXPECTED_RESUME_EVENTS = [ { type: EventType.TOOL_RESPONSE, thread_id: ROOT_ID, tool_call_id: WRITE_NOTE_CALL_ID }, { type: InternalEventType.AGENT_CONTEXT_APPEND, thread_id: ROOT_ID }, @@ -51,6 +76,27 @@ const RESUME_OUTPUT = { required_actions: [], }; +const EXPECTED_RESUME_LLM_INPUT = { + stream: true, + tools: WRITE_NOTE_TOOLS, + messages: [ + { role: 'system', content: expect.stringContaining(INSTRUCTION) }, + { role: 'user', content: 'hello' }, + { + role: 'assistant', + content: null, + tool_calls: [ + { + id: WRITE_NOTE_CALL_ID, + type: 'function', + function: { name: WRITE_NOTE_TOOL_NAME, arguments: WRITE_NOTE_ARGUMENTS }, + }, + ], + }, + { role: 'tool', tool_call_id: WRITE_NOTE_CALL_ID, content: WRITE_NOTE_RESULT }, + ], +}; + describe('orchestration: pause then resume on tool approval', () => { it('pauses for write_note approval, then finishes after allow', async () => { const thread = makeApprovalThread(); @@ -68,6 +114,7 @@ describe('orchestration: pause then resume on tool approval', () => { expect(paused.events).toMatchObject(EXPECTED_PAUSE_EVENTS); expect(paused.result).toMatchObject(PAUSE_OUTPUT); expect(paused.result.root_agent_error).toBeUndefined(); + expect(llmCreateInputs(thread.definition.modelClient)).toMatchObject(EXPECTED_PAUSE_LLM_INPUT); const resumed = await runTurn({ orchestrator, @@ -83,13 +130,17 @@ describe('orchestration: pause then resume on tool approval', () => { expect(resumed.events).toMatchObject(EXPECTED_RESUME_EVENTS); expect(resumed.result).toMatchObject(RESUME_OUTPUT); expect(resumed.result.root_agent_error).toBeUndefined(); + expect(llmCreateInputs(thread.definition.modelClient)).toMatchObject([ + ...EXPECTED_PAUSE_LLM_INPUT, + EXPECTED_RESUME_LLM_INPUT, + ]); }); }); function makeApprovalThread(): AgentThread { const agentDefinition: AgentDefinition = { modelClient: makeApprovalThenTextLLM(ROOT_FINAL), - instruction: 'You are running in a test setup.', + instruction: INSTRUCTION, messages: undefined, modelParams: undefined, responseFormat: undefined, diff --git a/packages/trueforge-core/tests/orchestration/orchestrationWithTools.test.ts b/packages/trueforge-core/tests/orchestration/orchestrationWithTools.test.ts index 2c6c3a17b..0f9b42807 100644 --- a/packages/trueforge-core/tests/orchestration/orchestrationWithTools.test.ts +++ b/packages/trueforge-core/tests/orchestration/orchestrationWithTools.test.ts @@ -1,6 +1,7 @@ import type { AgentDefinition, CreateDynamicSubAgentThread } from '../../src/core'; import { DynamicSubAgents } from '../../src/core/capabilities/builtins/DynamicSubAgents'; import { EventType } from '../../src/core/events/schema'; +import type { ILLM } from '../../src/core/llm/ILLM'; import { AgentThread } from '../../src/core/runtime/AgentThread'; import { InternalEventType, type AgentThreadConstructorInput } from '../../src/core/runtime/AgentThread.types'; import { @@ -9,13 +10,24 @@ import { } from '../../src/core/runtime/AgentThreadOrchestrator'; import { NOOP_AGENT_TRACING } from '../../src/core/tracing/NoopAgentTracing'; import { makeSilentLogger } from '../core/harnessMocks'; -import { makeRootLLM, makeTextLLM, runTurn } from './helpers/helpers'; +import { llmCreateInputs, makeRootLLM, makeTextLLM, runTurn } from './helpers/helpers'; const ROOT_ID = 'thread_root'; const TOOL_CALL_ID = 'call-sub'; const CHILD_REPLY = 'hello from the child'; const ROOT_FINAL = 'How are you?'; +const INSTRUCTION = 'You are running in a test setup.'; +const CHILD_TASK = 'do the delegated task'; +const ROOT_TOOLS = [ + { function: { name: 'call_tool' } }, + { function: { name: 'get_tool_info' } }, + { function: { name: 'get_tool_output_schema' } }, + { function: { name: 'list_tools' } }, + { function: { name: 'create_sub_agent' } }, +]; + +/** Root delegates via create_sub_agent; child result returns to parent; root finishes. */ const EXPECTED_EVENTS = [ { type: EventType.MODEL_MESSAGE, thread_id: ROOT_ID }, { type: EventType.MODEL_MESSAGE_DELTA, thread_id: ROOT_ID }, @@ -43,9 +55,87 @@ const OUTPUT = { required_actions: [], }; +const EXPECTED_ROOT_LLM_INPUT = [ + { + stream: true, + tools: ROOT_TOOLS, + messages: [ + { role: 'system', content: expect.stringContaining(INSTRUCTION) }, + { role: 'user', content: 'hello' }, + ], + }, + { + stream: true, + tools: ROOT_TOOLS, + messages: [ + { role: 'system', content: expect.stringContaining(INSTRUCTION) }, + { role: 'user', content: 'hello' }, + { + role: 'assistant', + content: null, + tool_calls: [ + { + id: TOOL_CALL_ID, + type: 'function', + function: { + name: 'create_sub_agent', + arguments: JSON.stringify({ name: 'worker', input: CHILD_TASK }), + }, + }, + ], + }, + { role: 'tool', tool_call_id: TOOL_CALL_ID, content: CHILD_REPLY }, + ], + }, +]; + +const EXPECTED_CHILD_LLM_INPUT = [ + { + stream: true, + messages: [ + { role: 'system', content: expect.stringContaining('sub-agent') }, + { role: 'user', content: CHILD_TASK }, + ], + }, +]; + describe('orchestration: dynamic sub-agent', () => { it('delegates via create_sub_agent, routes child result to parent, then finishes', async () => { const thread_1 = makeMainLLMThread(ROOT_ID, ROOT_FINAL, 'orchestration-with-tools'); + let childLLM: ILLM | undefined; + + const createSubAgentThread: CreateDynamicSubAgentThread = async ({ + parentDefinition, + request, + threadId, + parent, + }) => { + childLLM = makeTextLLM(CHILD_REPLY); + const agentDefinition: AgentDefinition = { + modelClient: childLLM, + instruction: undefined, + messages: [{ role: 'user', content: request.input }], + modelParams: parentDefinition.modelParams, + responseFormat: undefined, + iterationLimit: parentDefinition.iterationLimit, + toolSets: undefined, + }; + return new AgentThread({ + definition: agentDefinition, + threadId, + title: request.name, + parent, + agentInfo: request, + context: undefined, + currentContextUsage: undefined, + preComputedCompletion: undefined, + sandbox: undefined, + capabilities: undefined, + capabilityState: undefined, + tracing: NOOP_AGENT_TRACING, + logger: makeSilentLogger(), + }); + }; let orchestratorInput: AgentThreadOrchestratorInput = { agentThreads: new Map([[thread_1.threadId, thread_1]]), @@ -64,6 +154,11 @@ describe('orchestration: dynamic sub-agent', () => { expect(events).toMatchObject(EXPECTED_EVENTS); expect(result).toMatchObject(OUTPUT); expect(result.root_agent_error).toBeUndefined(); + expect(llmCreateInputs(thread_1.definition.modelClient)).toMatchObject(EXPECTED_ROOT_LLM_INPUT); + if (childLLM === undefined) { + throw new Error('expected child LLM to be created'); + } + expect(llmCreateInputs(childLLM)).toMatchObject(EXPECTED_CHILD_LLM_INPUT); }); }); @@ -71,7 +166,7 @@ function makeMainLLMThread(threadId: string, reply: string, title: string): Agen let agentDefinition: AgentDefinition = { // This is an instance if ILLM modelClient: makeRootLLM(reply), - instruction: 'You are running in a test setup.', + instruction: INSTRUCTION, // Undefined messages: undefined, modelParams: undefined, @@ -102,31 +197,3 @@ function makeMainLLMThread(threadId: string, reply: string, title: string): Agen return agentThread; } - -const createSubAgentThread: CreateDynamicSubAgentThread = async ({ parentDefinition, request, threadId, parent }) => { - const agentDefinition: AgentDefinition = { - modelClient: makeTextLLM(CHILD_REPLY), // Child has text only LLM, - // Not sure if this should be taken from the parent, or left alone - instruction: undefined, - messages: [{ role: 'user', content: request.input }], - modelParams: parentDefinition.modelParams, - responseFormat: undefined, - iterationLimit: parentDefinition.iterationLimit, - toolSets: undefined, // No parents tools sent to the child - }; - return new AgentThread({ - definition: agentDefinition, - threadId, - title: request.name, - parent, - agentInfo: request, - context: undefined, - currentContextUsage: undefined, - preComputedCompletion: undefined, - sandbox: undefined, - capabilities: undefined, - capabilityState: undefined, - tracing: NOOP_AGENT_TRACING, - logger: makeSilentLogger(), - }); -}; From afef9bb424e56f86b1e3692332a7557207fd3b2d Mon Sep 17 00:00:00 2001 From: Raman Tehlan Date: Wed, 2 Sep 2026 16:17:08 +0530 Subject: [PATCH 08/19] orchestration test: add note about the test --- .../orchestrationWithTools.test.ts | 33 +++++++++++-------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/packages/trueforge-core/tests/orchestration/orchestrationWithTools.test.ts b/packages/trueforge-core/tests/orchestration/orchestrationWithTools.test.ts index 0f9b42807..19e5d2c5a 100644 --- a/packages/trueforge-core/tests/orchestration/orchestrationWithTools.test.ts +++ b/packages/trueforge-core/tests/orchestration/orchestrationWithTools.test.ts @@ -10,7 +10,7 @@ import { } from '../../src/core/runtime/AgentThreadOrchestrator'; import { NOOP_AGENT_TRACING } from '../../src/core/tracing/NoopAgentTracing'; import { makeSilentLogger } from '../core/harnessMocks'; -import { llmCreateInputs, makeRootLLM, makeTextLLM, runTurn } from './helpers/helpers'; +import { createSubAgentStream, llmCreateInputs, makeTextLLM, runTurn, textReplyStream } from './helpers/helpers'; const ROOT_ID = 'thread_root'; const TOOL_CALL_ID = 'call-sub'; @@ -163,20 +163,25 @@ describe('orchestration: dynamic sub-agent', () => { }); function makeMainLLMThread(threadId: string, reply: string, title: string): AgentThread { - let agentDefinition: AgentDefinition = { - // This is an instance if ILLM - modelClient: makeRootLLM(reply), - instruction: INSTRUCTION, - // Undefined - messages: undefined, - modelParams: undefined, - responseFormat: undefined, - iterationLimit: undefined, - toolSets: [new DynamicSubAgents({ tracing: NOOP_AGENT_TRACING })], - }; - let agentThreadInput: AgentThreadConstructorInput = { - definition: agentDefinition, + // AgentDefinition + definition: { + // This is an instance if ILLM + modelClient: { + create: jest + .fn() + .mockImplementationOnce(() => createSubAgentStream()) + .mockImplementation(() => textReplyStream(reply)), + createNonStream: jest.fn(), + }, + instruction: INSTRUCTION, + // Undefined + messages: undefined, + modelParams: undefined, + responseFormat: undefined, + iterationLimit: undefined, + toolSets: [new DynamicSubAgents({ tracing: NOOP_AGENT_TRACING })], + }, threadId: threadId, title: title, // Undefined From 6ccde5805aec5d61d0377729244dc5865597fa57 Mon Sep 17 00:00:00 2001 From: Raman Tehlan Date: Wed, 2 Sep 2026 16:23:46 +0530 Subject: [PATCH 09/19] unify the orchestration test flow --- .../orchestrationWithTools.test.ts | 86 +++++++++---------- 1 file changed, 42 insertions(+), 44 deletions(-) diff --git a/packages/trueforge-core/tests/orchestration/orchestrationWithTools.test.ts b/packages/trueforge-core/tests/orchestration/orchestrationWithTools.test.ts index 19e5d2c5a..e3ce2c38e 100644 --- a/packages/trueforge-core/tests/orchestration/orchestrationWithTools.test.ts +++ b/packages/trueforge-core/tests/orchestration/orchestrationWithTools.test.ts @@ -10,7 +10,7 @@ import { } from '../../src/core/runtime/AgentThreadOrchestrator'; import { NOOP_AGENT_TRACING } from '../../src/core/tracing/NoopAgentTracing'; import { makeSilentLogger } from '../core/harnessMocks'; -import { createSubAgentStream, llmCreateInputs, makeTextLLM, runTurn, textReplyStream } from './helpers/helpers'; +import { createSubAgentStream, llmCreateInputs, runTurn, textReplyStream } from './helpers/helpers'; const ROOT_ID = 'thread_root'; const TOOL_CALL_ID = 'call-sub'; @@ -101,7 +101,42 @@ const EXPECTED_CHILD_LLM_INPUT = [ describe('orchestration: dynamic sub-agent', () => { it('delegates via create_sub_agent, routes child result to parent, then finishes', async () => { - const thread_1 = makeMainLLMThread(ROOT_ID, ROOT_FINAL, 'orchestration-with-tools'); + let agentThreadInput: AgentThreadConstructorInput = { + // AgentDefinition + definition: { + // This is an instance if ILLM + modelClient: { + create: jest + .fn() + .mockImplementationOnce(() => createSubAgentStream()) + .mockImplementation(() => textReplyStream(ROOT_FINAL)), + createNonStream: jest.fn(), + }, + instruction: INSTRUCTION, + // Undefined + messages: undefined, + modelParams: undefined, + responseFormat: undefined, + iterationLimit: undefined, + toolSets: [new DynamicSubAgents({ tracing: NOOP_AGENT_TRACING })], + }, + threadId: ROOT_ID, + title: 'orchestration-with-tools', + // Undefined + parent: undefined, + agentInfo: undefined, + context: undefined, + currentContextUsage: undefined, + preComputedCompletion: undefined, + sandbox: undefined, + capabilities: undefined, + capabilityState: undefined, + // Default + tracing: NOOP_AGENT_TRACING, + logger: makeSilentLogger(), + }; + + let thread_1 = new AgentThread(agentThreadInput); let childLLM: ILLM | undefined; const createSubAgentThread: CreateDynamicSubAgentThread = async ({ @@ -110,7 +145,11 @@ describe('orchestration: dynamic sub-agent', () => { threadId, parent, }) => { - childLLM = makeTextLLM(CHILD_REPLY); + childLLM = { + create: jest.fn().mockImplementation(() => textReplyStream(CHILD_REPLY)), + createNonStream: jest.fn().mockImplementation(() => textReplyStream(CHILD_REPLY)), + }; + const agentDefinition: AgentDefinition = { modelClient: childLLM, instruction: undefined, @@ -161,44 +200,3 @@ describe('orchestration: dynamic sub-agent', () => { expect(llmCreateInputs(childLLM)).toMatchObject(EXPECTED_CHILD_LLM_INPUT); }); }); - -function makeMainLLMThread(threadId: string, reply: string, title: string): AgentThread { - let agentThreadInput: AgentThreadConstructorInput = { - // AgentDefinition - definition: { - // This is an instance if ILLM - modelClient: { - create: jest - .fn() - .mockImplementationOnce(() => createSubAgentStream()) - .mockImplementation(() => textReplyStream(reply)), - createNonStream: jest.fn(), - }, - instruction: INSTRUCTION, - // Undefined - messages: undefined, - modelParams: undefined, - responseFormat: undefined, - iterationLimit: undefined, - toolSets: [new DynamicSubAgents({ tracing: NOOP_AGENT_TRACING })], - }, - threadId: threadId, - title: title, - // Undefined - parent: undefined, - agentInfo: undefined, - context: undefined, - currentContextUsage: undefined, - preComputedCompletion: undefined, - sandbox: undefined, - capabilities: undefined, - capabilityState: undefined, - // Default - tracing: NOOP_AGENT_TRACING, - logger: makeSilentLogger(), - }; - - let agentThread = new AgentThread(agentThreadInput); - - return agentThread; -} From 29df9f35b6dccf0c3f8b62e0f6e61a0adc280fe0 Mon Sep 17 00:00:00 2001 From: Raman Tehlan Date: Wed, 2 Sep 2026 16:49:15 +0530 Subject: [PATCH 10/19] test: remove vscode debug script --- .gitignore | 2 ++ .vscode/launch.json | 18 ------------------ 2 files changed, 2 insertions(+), 18 deletions(-) delete mode 100644 .vscode/launch.json diff --git a/.gitignore b/.gitignore index 6ba4f9250..a6013f660 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,5 @@ service-account*.json # Local Cursor scratch (committed rules under .cursor/rules/ stay tracked) .cursor/notes/ .cursor/plans/ + +.vscode diff --git a/.vscode/launch.json b/.vscode/launch.json deleted file mode 100644 index 4037fdd27..000000000 --- a/.vscode/launch.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "version": "0.2.0", - "configurations": [ - { - "type": "pwa-node", - "request": "launch", - "name": "Debug trueforge-core orchestration (current file)", - "cwd": "${workspaceFolder}/packages/trueforge-core", - "runtimeExecutable": "pnpm", - "runtimeArgs": ["test:orchestration", "--", "--runInBand", "--testTimeout", "0", "${file}"], - "console": "integratedTerminal", - "autoAttachChildProcesses": true, - "skipFiles": ["/**", "**/node_modules/**"], - "sourceMaps": true, - "resolveSourceMapLocations": ["${workspaceFolder}/packages/trueforge-core/**", "!**/node_modules/**"] - } - ] -} From 195aaa4236fb6261d147b5e1b38fac7c8fb66bea Mon Sep 17 00:00:00 2001 From: Raman Tehlan Date: Wed, 2 Sep 2026 17:02:05 +0530 Subject: [PATCH 11/19] test: remove extra pnpm command for the tests --- package.json | 2 - packages/trueforge-core/jest.config.cjs | 3 +- .../jest.orchestration.config.cjs | 37 ------------------- packages/trueforge-core/package.json | 4 +- .../tests/orchestration/README.md | 11 +++--- packages/trueforge-core/tsconfig.json | 2 +- 6 files changed, 8 insertions(+), 51 deletions(-) delete mode 100644 packages/trueforge-core/jest.orchestration.config.cjs diff --git a/package.json b/package.json index 19070f0ec..238f8811b 100644 --- a/package.json +++ b/package.json @@ -40,8 +40,6 @@ "test:frontend": "pnpm --filter frontend test", "test:chart-version": "bash tests/scripts/resolve-chart-version.test.sh", "test:trueforge-core": "pnpm --filter @truefoundry/trueforge-core test", - "test:trueforge-core:orchestration": "pnpm --filter @truefoundry/trueforge-core test:orchestration", - "test:trueforge-core:orchestration:debug": "pnpm --filter @truefoundry/trueforge-core test:orchestration:debug", "test:trueforge": "pnpm --filter @truefoundry/trueforge test", "test:local-sandbox:contract": "pnpm --filter @truefoundry/trueforge test:local-sandbox:contract", "smoke:local-sandbox": "pnpm --filter @truefoundry/trueforge smoke:local-sandbox", diff --git a/packages/trueforge-core/jest.config.cjs b/packages/trueforge-core/jest.config.cjs index 31df920f6..935b21b7e 100644 --- a/packages/trueforge-core/jest.config.cjs +++ b/packages/trueforge-core/jest.config.cjs @@ -37,6 +37,5 @@ module.exports = { roots: ['/tests'], testMatch: ['**/tests/**/*.test.ts'], // Compile-time suites are enforced by `tsc --noEmit`, not the Jest runner. - // Orchestration tests live under tests/orchestration and are run via jest.orchestration.config.cjs. - testPathIgnorePatterns: ['\\.compile\\.test\\.ts$', '/tests/orchestration/'], + testPathIgnorePatterns: ['\\.compile\\.test\\.ts$'], }; diff --git a/packages/trueforge-core/jest.orchestration.config.cjs b/packages/trueforge-core/jest.orchestration.config.cjs deleted file mode 100644 index d042e3bbf..000000000 --- a/packages/trueforge-core/jest.orchestration.config.cjs +++ /dev/null @@ -1,37 +0,0 @@ -/** @type {import('jest').Config} */ -module.exports = { - testEnvironment: 'node', - transform: { - '^.+\\.tsx?$': [ - '@swc/jest', - { - jsc: { - parser: { syntax: 'typescript', decorators: true }, - target: 'es2022', - }, - module: { type: 'commonjs' }, - sourceMaps: 'inline', - }, - ], - '^.+\\.js$': [ - '@swc/jest', - { - jsc: { - parser: { syntax: 'ecmascript' }, - target: 'es2022', - }, - module: { type: 'commonjs' }, - sourceMaps: 'inline', - }, - ], - }, - transformIgnorePatterns: [], - moduleNameMapper: { - '^(\\.{1,2}/.*)\\.js$': '$1', - }, - setupFilesAfterEnv: ['/tests/setup.ts'], - testTimeout: 60_000, - maxWorkers: 1, - roots: ['/tests/orchestration'], - testMatch: ['/tests/orchestration/**/*.test.ts'], -}; diff --git a/packages/trueforge-core/package.json b/packages/trueforge-core/package.json index ac0ba17b8..b1f2a1c1a 100644 --- a/packages/trueforge-core/package.json +++ b/packages/trueforge-core/package.json @@ -95,9 +95,7 @@ "build:pkg": "node scripts/write-dist-package-json.mjs", "build:check": "node scripts/check-dist.mjs", "typecheck": "pnpm run build:gen && tsc --noEmit", - "test": "pnpm run build:gen && jest --config jest.config.cjs && jest --config jest.orchestration.config.cjs", - "test:orchestration": "pnpm run build:gen && jest --config jest.orchestration.config.cjs", - "test:orchestration:debug": "pnpm run build:gen && node --inspect-brk ./node_modules/jest/bin/jest.js --config jest.orchestration.config.cjs --runInBand --testTimeout 0", + "test": "pnpm run build:gen && jest --config jest.config.cjs", "pack:dry": "pnpm pack --dry-run" }, "dependencies": { diff --git a/packages/trueforge-core/tests/orchestration/README.md b/packages/trueforge-core/tests/orchestration/README.md index 86cb0d025..390171aa3 100644 --- a/packages/trueforge-core/tests/orchestration/README.md +++ b/packages/trueforge-core/tests/orchestration/README.md @@ -53,7 +53,6 @@ flowchart LR | `orchestration.test.ts` | **Program 1** - single root thread, text-only reply, full assertions | | `orchestrationWithTools.test.ts` | **Program 2** - root delegates to sub-agent via `create_sub_agent` | | `helpers.ts` | Mock LLM streams and approval-gated tools | -| `jest.orchestration.config.cjs` | Jest config scoped to this folder | ## Core components under test @@ -320,23 +319,23 @@ internal.agent.done ← root finished (last event) From `packages/trueforge-core`: ```bash -pnpm test:orchestration +pnpm test ``` Single file: ```bash -pnpm test:orchestration -- orchestration.test.ts -pnpm test:orchestration -- orchestrationWithTools.test.ts +pnpm test -- orchestration.test.ts +pnpm test -- orchestrationWithTools.test.ts ``` From repo root: ```bash -pnpm test:trueforge-core:orchestration +pnpm test:trueforge-core ``` -Orchestration tests use `jest.orchestration.config.cjs` (`maxWorkers: 1`, 60s timeout). Unit tests under `tests/` (excluding `tests/orchestration/`) run separately via `jest.config.cjs`. +These files run with the rest of `@truefoundry/trueforge-core` via `jest.config.cjs`. Threads and the orchestrator still take a Winston logger (required by the runtime). These tests use `makeSilentLogger()` from `tests/core/harnessMocks.ts`, so the suite does not print turn flow. diff --git a/packages/trueforge-core/tsconfig.json b/packages/trueforge-core/tsconfig.json index 97562af5e..85ca45efe 100644 --- a/packages/trueforge-core/tsconfig.json +++ b/packages/trueforge-core/tsconfig.json @@ -12,6 +12,6 @@ "openai/resources/chat": ["./node_modules/openai/resources/chat/index.d.ts"] } }, - "include": ["src/**/*", "tests/**/*", "tsup.config.ts", "jest.config.cjs", "jest.orchestration.config.cjs"], + "include": ["src/**/*", "tests/**/*", "tsup.config.ts", "jest.config.cjs"], "exclude": ["node_modules", "dist"] } From 83d32e3e685cefae16941b18c0674e4e881f2403 Mon Sep 17 00:00:00 2001 From: Raman Tehlan Date: Wed, 2 Sep 2026 17:03:55 +0530 Subject: [PATCH 12/19] test: remove test readme --- .../tests/orchestration/README.md | 386 ------------------ 1 file changed, 386 deletions(-) delete mode 100644 packages/trueforge-core/tests/orchestration/README.md diff --git a/packages/trueforge-core/tests/orchestration/README.md b/packages/trueforge-core/tests/orchestration/README.md deleted file mode 100644 index 390171aa3..000000000 --- a/packages/trueforge-core/tests/orchestration/README.md +++ /dev/null @@ -1,386 +0,0 @@ -# Orchestration tests - -End-to-end tests for `AgentThreadOrchestrator` and `AgentThread` in `@truefoundry/trueforge-core`. - -These tests wire the real orchestration loop with **mocked LLMs** and **no database**. They exist to learn and verify how a turn flows through the harness before adding persistence (`SessionHandle`), HTTP, or real model providers. - -## What we are testing - -| Layer | In scope | Out of scope | -| --------------------------------- | ------------------------------------------------------------------------- | ------------------------------------------- | -| `AgentThreadOrchestrator.send` | Route input to threads, validate, append context | Postgres / Redis store writes | -| `AgentThreadOrchestrator.execute` | Run leaf threads, merge streams, spawn sub-agents, return terminal result | `TurnHandle.stream` persistence | -| `AgentThread` | LLM loop, tool execution, context mutations | Real OpenAI / Vercel AI calls | -| Sub-agent lifecycle | `create_sub_agent` tool → child thread → result back to parent | Full `SessionHandle` resolver / spec wiring | - -**Goal:** prove the orchestrator correctly coordinates one root thread (Program 1) and a root + dynamic child thread (Program 2). - -## Why this design - -Production creates the orchestrator inside `SessionHandle.createTurn`: - -```text -resolve definitions → build AgentThread map → new AgentThreadOrchestrator → send → persist → execute (via TurnHandle) -``` - -These orchestration tests **skip the store and session layer** and talk to the orchestrator directly. That keeps the surface area small while still exercising the same `send` / `execute` contract production uses. - -```mermaid -flowchart LR - subgraph production["Production path"] - SH[SessionHandle] - Store[(ISessionStore)] - OrchP[AgentThreadOrchestrator] - SH --> Store - SH --> OrchP - end - - subgraph orch["Orchestration tests"] - Test[Jest test] - OrchE[AgentThreadOrchestrator] - MockLLM[Mock ILLM] - Test --> OrchE - OrchE --> MockLLM - end - - OrchP -. same class .- OrchE -``` - -## Files - -| File | Role | -| -------------------------------- | -------------------------------------------------------------------- | -| `orchestration.test.ts` | **Program 1** - single root thread, text-only reply, full assertions | -| `orchestrationWithTools.test.ts` | **Program 2** - root delegates to sub-agent via `create_sub_agent` | -| `helpers.ts` | Mock LLM streams and approval-gated tools | - -## Core components under test - -### `AgentThread` - -One conversation thread. Holds: - -- **`definition`** - `modelClient` (`ILLM`), optional `instruction`, `toolSets`, etc. -- **`context`** - LLM message history (user, assistant, tool messages) -- **`send(messages)`** - append user input, approvals, or tool responses to context (no LLM call) -- **`execute({ signal })`** - run the state machine: LLM → tools → pause or done - -### `AgentThreadOrchestrator` - -Owns a `Map` and coordinates a turn: - -- **`send(batch)`** - fan out messages to the right threads, validate, delegate to each thread's `send` -- **`execute({ signal })`** - run **leaf** threads in parallel (up to 5), merge event streams, handle sub-agent creation/completion -- **`createDynamicSubAgentThread`** - factory callback invoked when the root calls `create_sub_agent`; must return a new `AgentThread` (not called at construction time) - -### `CreateDynamicSubAgentThread` - -```ts -(input: { - parentDefinition: AgentDefinition; - request: AgentInfo; // { type: 'dynamic', name, input, model? } - threadId: string; // orchestrator already minted this - parent: AgentParent; // { thread_id, tool_call_id } - signal: AbortSignal; -}) => Promise; -``` - -Pass the **function reference** to the orchestrator. Do not call it yourself. - -## Turn lifecycle: `send` then `execute` - -These are separate steps on purpose (same as production: send before commit, then execute). - -```mermaid -sequenceDiagram - participant Test - participant Orch as AgentThreadOrchestrator - participant Thread as AgentThread - participant LLM as Mock ILLM - - Test->>Orch: send([USER_MESSAGE]) - Orch->>Thread: send(messages) - Thread-->>Orch: AGENT_CONTEXT_APPEND - Orch-->>Test: yield append events - - Note over Test,LLM: send does NOT call the model - - Test->>Orch: execute({ signal }) - loop until AGENT_DONE or pause - Orch->>Thread: execute({ signal }) - Thread->>LLM: create(streaming) - LLM-->>Thread: chunks / tool_calls - Thread-->>Orch: model.message.delta, model.message, ... - Orch-->>Test: yield execution events - end - Orch-->>Test: return AgentThreadExecutionResult -``` - -**Important:** `send` returns an async generator. You must consume it with `for await`; otherwise the user message never lands in context. - -**Important:** `execute` also returns an async generator. The **return value** (final assistant output, required pauses, errors) is only available after the last `next()` when `done === true`. - -## Mock LLM helpers (`helpers.ts`) - -| Helper | Behavior | -| ------------------------- | ---------------------------------------------------------------------------- | -| `textReplyStream(text)` | One streaming chunk + stop completion with fixed text | -| `makeTextLLM(text)` | `ILLM` that always replies with `text` (used for child threads) | -| `createSubAgentStream()` | First root call: stream a `create_sub_agent` tool call | -| `makeRootLLM(finalReply)` | First `create()` → sub-agent tool call; every later call → `finalReply` text | - -Root and child threads use **different** `ILLM` instances so each can follow its own scripted sequence. - ---- - -## Program 1: text-only happy path - -**File:** `orchestration.test.ts` - -### Setup - -| Piece | Value | -| ----------------------------- | -------------------------------------------- | -| Root thread id | `"main"` | -| LLM | `makeTextLLM("hello from the mocked model")` | -| Tool sets | none | -| `createDynamicSubAgentThread` | rejects if ever called | -| Tracing | `NOOP_AGENT_TRACING` | -| Logger | silent (`makeSilentLogger`) | - -### Data flow - -```mermaid -flowchart TD - A["send: USER_MESSAGE 'hello'"] --> B["context: user message appended"] - B --> C["execute: llm-call-required"] - C --> D["Mock LLM streams text reply"] - D --> E["context: assistant message appended"] - E --> F["AGENT_DONE on root"] - F --> G["execute returns output + empty required_actions"] -``` - -### Expected event types - -**After `send`:** - -```text -internal.agent.context.append -``` - -**During `execute` (order may include duplicates / internal appends):** - -```text -model.message.delta -model.message -internal.agent.done ← last yielded event -``` - -**Must NOT appear:** - -```text -thread.created -tool.response -``` - -### Passing expectations (assertions) - -- `step.value.output.content` === `"hello from the mocked model"` -- `step.value.required_actions` === `[]` -- `step.value.root_agent_error` is undefined -- Root snapshot context contains user `"hello"` and assistant reply - ---- - -## Program 2: sub-agent delegation - -**File:** `orchestrationWithTools.test.ts` - -### Setup - -| Piece | Root thread | Child thread | -| ---------------- | ----------------------------- | ----------------------------------------------- | -| Thread id | `"thread_1"` (fixed) | minted by orchestrator at runtime | -| LLM | `makeRootLLM("How are you?")` | `makeTextLLM("hello from the child")` | -| Tool sets | `[new DynamicSubAgents(...)]` | `undefined` (no nested sub-agents) | -| Instruction | test setup string | `undefined` (harness adds `SUB_AGENT_IDENTITY`) | -| Initial messages | none | `[{ role: 'user', content: request.input }]` | -| Parent link | none | `{ thread_id, tool_call_id }` from orchestrator | - -`createSubAgentThread` is a top-level `CreateDynamicSubAgentThread` implementation (mirrors a simplified `SessionHandle.makeCreateDynamicSubAgentThread`). - -### Scripted LLM behavior - -1. **Root call 1** - model returns `create_sub_agent` with `{ name: 'worker', input: '...' }` -2. **Child call 1** - model returns `"hello from the child"` -3. **Root call 2** - model returns `"How are you?"` - -### Thread tree over time - -```mermaid -flowchart TD - subgraph phase1["After root LLM call 1"] - R1["thread_1 (root)
open create_sub_agent tool call"] - end - - subgraph phase2["After sub-agent created"] - R2["thread_1 (root)
waiting on tool call"] - C["child thread (leaf)
runs execute"] - R2 --- C - end - - subgraph phase3["After child AGENT_DONE"] - R3["thread_1 (root, leaf again)
tool result appended"] - end - - phase1 --> phase2 --> phase3 -``` - -Only **leaf** threads run. While the child exists, the root is paused (not a leaf). When the child finishes, the orchestrator: - -1. Yields `tool.response` on the parent -2. `send()`s the child's result into the parent as a tool message -3. Removes the child from the thread map -4. Resumes the root for LLM call 2 - -### Data flow - -```mermaid -sequenceDiagram - participant Test - participant Orch as Orchestrator - participant Root as thread_1 - participant Child as sub-agent - participant RootLLM as makeRootLLM - participant ChildLLM as makeTextLLM - - Test->>Orch: send(USER_MESSAGE) - Test->>Orch: execute() - - Root->>RootLLM: create() #1 - RootLLM-->>Root: create_sub_agent tool call - Root-->>Orch: internal.agent.create_subagent - Orch->>Orch: createSubAgentThread(...) - Orch-->>Test: thread.created - - Child->>ChildLLM: create() - ChildLLM-->>Child: "hello from the child" - Child-->>Orch: model.message, AGENT_DONE (child) - - Orch-->>Test: tool.response (parent) - Orch->>Root: send(tool message with child result) - - Root->>RootLLM: create() #2 - RootLLM-->>Root: "How are you?" - Root-->>Orch: model.message, AGENT_DONE (root) - Orch-->>Test: return { output: "How are you?", ... } -``` - -### Expected event types (from logging) - -Typical `execute` event sequence: - -```text -model.message / model.message.delta ← root tool call -internal.agent.context.append ← (internal, may repeat) -thread.created ← child registered -model.message / model.message.delta ← child reply -tool.response ← child result routed to parent -internal.agent.done ← child finished (thread_id = child) -model.message / model.message.delta ← root final reply -internal.agent.done ← root finished (last event) -``` - -`internal.agent.create_subagent` is handled inside the orchestrator and is **not** yielded to the test consumer. - -### Expected final state - -**`execute` return value:** - -| Field | Expected | -| ------------------ | --------------------------------------------------- | -| `output.content` | `"How are you?"` (root final reply, not child text) | -| `required_actions` | `[]` | -| `root_agent_error` | undefined | - -**Root thread context (after send + execute):** - -```text -1. user: "hello" -2. assistant: tool_call create_sub_agent (id: call-sub) -3. tool: "hello from the child" -4. assistant: "How are you?" -``` - ---- - -## Running tests - -From `packages/trueforge-core`: - -```bash -pnpm test -``` - -Single file: - -```bash -pnpm test -- orchestration.test.ts -pnpm test -- orchestrationWithTools.test.ts -``` - -From repo root: - -```bash -pnpm test:trueforge-core -``` - -These files run with the rest of `@truefoundry/trueforge-core` via `jest.config.cjs`. - -Threads and the orchestrator still take a Winston logger (required by the runtime). These tests use `makeSilentLogger()` from `tests/core/harnessMocks.ts`, so the suite does not print turn flow. - -## Relationship to production - -| Orchestration test | Production equivalent | -| -------------------------------------- | ------------------------------------------------------ | -| `new AgentThread({ definition, ... })` | `SessionHandle.buildThreads` + resolver | -| `createSubAgentThread` callback | `SessionHandle.makeCreateDynamicSubAgentThread` | -| `orchestrator.send` + `execute` | `SessionHandle.createTurn` + `TurnHandle.stream` | -| In-memory `thread.toSnapshot()` | `ISessionStore.createTurn` / persisted context appends | -| `NOOP_AGENT_TRACING` | `resolver.createTracing()` | - -Production adds: store persistence, turn records, event folding for SSE, sandbox resolution, full builtin capabilities from `AgentSpec`, and MCP servers beyond `DynamicSubAgents`. - -## Planned coverage (not yet implemented) - -| Program | Scenario | -| ------- | -------------------------------------------------------------------------------------------------- | -| **3** | Pause on `tool.approval.required` or `tool.response.required`, then resume with `send` + `execute` | -| **4** | Reject user message while sub-agent is live (`InvalidAgentSendInputError`) | -| **5** | MCP auth required (`internal.mcp.auth_required` merge across parallel sub-agents) | - -## Quick reference: orchestrator inputs - -```ts -new AgentThreadOrchestrator({ - agentThreads: new Map([[rootThreadId, rootThread]]), - createDynamicSubAgentThread, // function reference, not a call - tracing: NOOP_AGENT_TRACING, - logger, -}); -``` - -Every turn: - -```ts -for await (const _ of orchestrator.send(input)) { - /* collect appends */ -} -const it = orchestrator.execute({ signal }); -let step = await it.next(); -while (!step.done) { - // step.value is a streamed execution event - step = await it.next(); -} -// step.value is AgentThreadExecutionResult -``` From 4067a46477e02ce965ebfb904a8d10d12007ebc6 Mon Sep 17 00:00:00 2001 From: Raman Tehlan Date: Wed, 2 Sep 2026 18:27:31 +0530 Subject: [PATCH 13/19] test: review comments --- .../tests/orchestration/README.md | 386 ++++++++++++++++++ .../orchestrationApproval.test.ts | 28 +- .../orchestrationWithTools.test.ts | 23 -- 3 files changed, 399 insertions(+), 38 deletions(-) create mode 100644 packages/trueforge-core/tests/orchestration/README.md diff --git a/packages/trueforge-core/tests/orchestration/README.md b/packages/trueforge-core/tests/orchestration/README.md new file mode 100644 index 000000000..390171aa3 --- /dev/null +++ b/packages/trueforge-core/tests/orchestration/README.md @@ -0,0 +1,386 @@ +# Orchestration tests + +End-to-end tests for `AgentThreadOrchestrator` and `AgentThread` in `@truefoundry/trueforge-core`. + +These tests wire the real orchestration loop with **mocked LLMs** and **no database**. They exist to learn and verify how a turn flows through the harness before adding persistence (`SessionHandle`), HTTP, or real model providers. + +## What we are testing + +| Layer | In scope | Out of scope | +| --------------------------------- | ------------------------------------------------------------------------- | ------------------------------------------- | +| `AgentThreadOrchestrator.send` | Route input to threads, validate, append context | Postgres / Redis store writes | +| `AgentThreadOrchestrator.execute` | Run leaf threads, merge streams, spawn sub-agents, return terminal result | `TurnHandle.stream` persistence | +| `AgentThread` | LLM loop, tool execution, context mutations | Real OpenAI / Vercel AI calls | +| Sub-agent lifecycle | `create_sub_agent` tool → child thread → result back to parent | Full `SessionHandle` resolver / spec wiring | + +**Goal:** prove the orchestrator correctly coordinates one root thread (Program 1) and a root + dynamic child thread (Program 2). + +## Why this design + +Production creates the orchestrator inside `SessionHandle.createTurn`: + +```text +resolve definitions → build AgentThread map → new AgentThreadOrchestrator → send → persist → execute (via TurnHandle) +``` + +These orchestration tests **skip the store and session layer** and talk to the orchestrator directly. That keeps the surface area small while still exercising the same `send` / `execute` contract production uses. + +```mermaid +flowchart LR + subgraph production["Production path"] + SH[SessionHandle] + Store[(ISessionStore)] + OrchP[AgentThreadOrchestrator] + SH --> Store + SH --> OrchP + end + + subgraph orch["Orchestration tests"] + Test[Jest test] + OrchE[AgentThreadOrchestrator] + MockLLM[Mock ILLM] + Test --> OrchE + OrchE --> MockLLM + end + + OrchP -. same class .- OrchE +``` + +## Files + +| File | Role | +| -------------------------------- | -------------------------------------------------------------------- | +| `orchestration.test.ts` | **Program 1** - single root thread, text-only reply, full assertions | +| `orchestrationWithTools.test.ts` | **Program 2** - root delegates to sub-agent via `create_sub_agent` | +| `helpers.ts` | Mock LLM streams and approval-gated tools | + +## Core components under test + +### `AgentThread` + +One conversation thread. Holds: + +- **`definition`** - `modelClient` (`ILLM`), optional `instruction`, `toolSets`, etc. +- **`context`** - LLM message history (user, assistant, tool messages) +- **`send(messages)`** - append user input, approvals, or tool responses to context (no LLM call) +- **`execute({ signal })`** - run the state machine: LLM → tools → pause or done + +### `AgentThreadOrchestrator` + +Owns a `Map` and coordinates a turn: + +- **`send(batch)`** - fan out messages to the right threads, validate, delegate to each thread's `send` +- **`execute({ signal })`** - run **leaf** threads in parallel (up to 5), merge event streams, handle sub-agent creation/completion +- **`createDynamicSubAgentThread`** - factory callback invoked when the root calls `create_sub_agent`; must return a new `AgentThread` (not called at construction time) + +### `CreateDynamicSubAgentThread` + +```ts +(input: { + parentDefinition: AgentDefinition; + request: AgentInfo; // { type: 'dynamic', name, input, model? } + threadId: string; // orchestrator already minted this + parent: AgentParent; // { thread_id, tool_call_id } + signal: AbortSignal; +}) => Promise; +``` + +Pass the **function reference** to the orchestrator. Do not call it yourself. + +## Turn lifecycle: `send` then `execute` + +These are separate steps on purpose (same as production: send before commit, then execute). + +```mermaid +sequenceDiagram + participant Test + participant Orch as AgentThreadOrchestrator + participant Thread as AgentThread + participant LLM as Mock ILLM + + Test->>Orch: send([USER_MESSAGE]) + Orch->>Thread: send(messages) + Thread-->>Orch: AGENT_CONTEXT_APPEND + Orch-->>Test: yield append events + + Note over Test,LLM: send does NOT call the model + + Test->>Orch: execute({ signal }) + loop until AGENT_DONE or pause + Orch->>Thread: execute({ signal }) + Thread->>LLM: create(streaming) + LLM-->>Thread: chunks / tool_calls + Thread-->>Orch: model.message.delta, model.message, ... + Orch-->>Test: yield execution events + end + Orch-->>Test: return AgentThreadExecutionResult +``` + +**Important:** `send` returns an async generator. You must consume it with `for await`; otherwise the user message never lands in context. + +**Important:** `execute` also returns an async generator. The **return value** (final assistant output, required pauses, errors) is only available after the last `next()` when `done === true`. + +## Mock LLM helpers (`helpers.ts`) + +| Helper | Behavior | +| ------------------------- | ---------------------------------------------------------------------------- | +| `textReplyStream(text)` | One streaming chunk + stop completion with fixed text | +| `makeTextLLM(text)` | `ILLM` that always replies with `text` (used for child threads) | +| `createSubAgentStream()` | First root call: stream a `create_sub_agent` tool call | +| `makeRootLLM(finalReply)` | First `create()` → sub-agent tool call; every later call → `finalReply` text | + +Root and child threads use **different** `ILLM` instances so each can follow its own scripted sequence. + +--- + +## Program 1: text-only happy path + +**File:** `orchestration.test.ts` + +### Setup + +| Piece | Value | +| ----------------------------- | -------------------------------------------- | +| Root thread id | `"main"` | +| LLM | `makeTextLLM("hello from the mocked model")` | +| Tool sets | none | +| `createDynamicSubAgentThread` | rejects if ever called | +| Tracing | `NOOP_AGENT_TRACING` | +| Logger | silent (`makeSilentLogger`) | + +### Data flow + +```mermaid +flowchart TD + A["send: USER_MESSAGE 'hello'"] --> B["context: user message appended"] + B --> C["execute: llm-call-required"] + C --> D["Mock LLM streams text reply"] + D --> E["context: assistant message appended"] + E --> F["AGENT_DONE on root"] + F --> G["execute returns output + empty required_actions"] +``` + +### Expected event types + +**After `send`:** + +```text +internal.agent.context.append +``` + +**During `execute` (order may include duplicates / internal appends):** + +```text +model.message.delta +model.message +internal.agent.done ← last yielded event +``` + +**Must NOT appear:** + +```text +thread.created +tool.response +``` + +### Passing expectations (assertions) + +- `step.value.output.content` === `"hello from the mocked model"` +- `step.value.required_actions` === `[]` +- `step.value.root_agent_error` is undefined +- Root snapshot context contains user `"hello"` and assistant reply + +--- + +## Program 2: sub-agent delegation + +**File:** `orchestrationWithTools.test.ts` + +### Setup + +| Piece | Root thread | Child thread | +| ---------------- | ----------------------------- | ----------------------------------------------- | +| Thread id | `"thread_1"` (fixed) | minted by orchestrator at runtime | +| LLM | `makeRootLLM("How are you?")` | `makeTextLLM("hello from the child")` | +| Tool sets | `[new DynamicSubAgents(...)]` | `undefined` (no nested sub-agents) | +| Instruction | test setup string | `undefined` (harness adds `SUB_AGENT_IDENTITY`) | +| Initial messages | none | `[{ role: 'user', content: request.input }]` | +| Parent link | none | `{ thread_id, tool_call_id }` from orchestrator | + +`createSubAgentThread` is a top-level `CreateDynamicSubAgentThread` implementation (mirrors a simplified `SessionHandle.makeCreateDynamicSubAgentThread`). + +### Scripted LLM behavior + +1. **Root call 1** - model returns `create_sub_agent` with `{ name: 'worker', input: '...' }` +2. **Child call 1** - model returns `"hello from the child"` +3. **Root call 2** - model returns `"How are you?"` + +### Thread tree over time + +```mermaid +flowchart TD + subgraph phase1["After root LLM call 1"] + R1["thread_1 (root)
open create_sub_agent tool call"] + end + + subgraph phase2["After sub-agent created"] + R2["thread_1 (root)
waiting on tool call"] + C["child thread (leaf)
runs execute"] + R2 --- C + end + + subgraph phase3["After child AGENT_DONE"] + R3["thread_1 (root, leaf again)
tool result appended"] + end + + phase1 --> phase2 --> phase3 +``` + +Only **leaf** threads run. While the child exists, the root is paused (not a leaf). When the child finishes, the orchestrator: + +1. Yields `tool.response` on the parent +2. `send()`s the child's result into the parent as a tool message +3. Removes the child from the thread map +4. Resumes the root for LLM call 2 + +### Data flow + +```mermaid +sequenceDiagram + participant Test + participant Orch as Orchestrator + participant Root as thread_1 + participant Child as sub-agent + participant RootLLM as makeRootLLM + participant ChildLLM as makeTextLLM + + Test->>Orch: send(USER_MESSAGE) + Test->>Orch: execute() + + Root->>RootLLM: create() #1 + RootLLM-->>Root: create_sub_agent tool call + Root-->>Orch: internal.agent.create_subagent + Orch->>Orch: createSubAgentThread(...) + Orch-->>Test: thread.created + + Child->>ChildLLM: create() + ChildLLM-->>Child: "hello from the child" + Child-->>Orch: model.message, AGENT_DONE (child) + + Orch-->>Test: tool.response (parent) + Orch->>Root: send(tool message with child result) + + Root->>RootLLM: create() #2 + RootLLM-->>Root: "How are you?" + Root-->>Orch: model.message, AGENT_DONE (root) + Orch-->>Test: return { output: "How are you?", ... } +``` + +### Expected event types (from logging) + +Typical `execute` event sequence: + +```text +model.message / model.message.delta ← root tool call +internal.agent.context.append ← (internal, may repeat) +thread.created ← child registered +model.message / model.message.delta ← child reply +tool.response ← child result routed to parent +internal.agent.done ← child finished (thread_id = child) +model.message / model.message.delta ← root final reply +internal.agent.done ← root finished (last event) +``` + +`internal.agent.create_subagent` is handled inside the orchestrator and is **not** yielded to the test consumer. + +### Expected final state + +**`execute` return value:** + +| Field | Expected | +| ------------------ | --------------------------------------------------- | +| `output.content` | `"How are you?"` (root final reply, not child text) | +| `required_actions` | `[]` | +| `root_agent_error` | undefined | + +**Root thread context (after send + execute):** + +```text +1. user: "hello" +2. assistant: tool_call create_sub_agent (id: call-sub) +3. tool: "hello from the child" +4. assistant: "How are you?" +``` + +--- + +## Running tests + +From `packages/trueforge-core`: + +```bash +pnpm test +``` + +Single file: + +```bash +pnpm test -- orchestration.test.ts +pnpm test -- orchestrationWithTools.test.ts +``` + +From repo root: + +```bash +pnpm test:trueforge-core +``` + +These files run with the rest of `@truefoundry/trueforge-core` via `jest.config.cjs`. + +Threads and the orchestrator still take a Winston logger (required by the runtime). These tests use `makeSilentLogger()` from `tests/core/harnessMocks.ts`, so the suite does not print turn flow. + +## Relationship to production + +| Orchestration test | Production equivalent | +| -------------------------------------- | ------------------------------------------------------ | +| `new AgentThread({ definition, ... })` | `SessionHandle.buildThreads` + resolver | +| `createSubAgentThread` callback | `SessionHandle.makeCreateDynamicSubAgentThread` | +| `orchestrator.send` + `execute` | `SessionHandle.createTurn` + `TurnHandle.stream` | +| In-memory `thread.toSnapshot()` | `ISessionStore.createTurn` / persisted context appends | +| `NOOP_AGENT_TRACING` | `resolver.createTracing()` | + +Production adds: store persistence, turn records, event folding for SSE, sandbox resolution, full builtin capabilities from `AgentSpec`, and MCP servers beyond `DynamicSubAgents`. + +## Planned coverage (not yet implemented) + +| Program | Scenario | +| ------- | -------------------------------------------------------------------------------------------------- | +| **3** | Pause on `tool.approval.required` or `tool.response.required`, then resume with `send` + `execute` | +| **4** | Reject user message while sub-agent is live (`InvalidAgentSendInputError`) | +| **5** | MCP auth required (`internal.mcp.auth_required` merge across parallel sub-agents) | + +## Quick reference: orchestrator inputs + +```ts +new AgentThreadOrchestrator({ + agentThreads: new Map([[rootThreadId, rootThread]]), + createDynamicSubAgentThread, // function reference, not a call + tracing: NOOP_AGENT_TRACING, + logger, +}); +``` + +Every turn: + +```ts +for await (const _ of orchestrator.send(input)) { + /* collect appends */ +} +const it = orchestrator.execute({ signal }); +let step = await it.next(); +while (!step.done) { + // step.value is a streamed execution event + step = await it.next(); +} +// step.value is AgentThreadExecutionResult +``` diff --git a/packages/trueforge-core/tests/orchestration/orchestrationApproval.test.ts b/packages/trueforge-core/tests/orchestration/orchestrationApproval.test.ts index de8d6ad9d..dfdcf8f2f 100644 --- a/packages/trueforge-core/tests/orchestration/orchestrationApproval.test.ts +++ b/packages/trueforge-core/tests/orchestration/orchestrationApproval.test.ts @@ -29,7 +29,7 @@ const WRITE_NOTE_TOOLS = [ ]; /** Pause on write_note approval, then resume after allow and finish. */ -const EXPECTED_PAUSE_EVENTS = [ +const EXPECTED_TURN_1_EVENTS = [ { type: EventType.MODEL_MESSAGE, thread_id: ROOT_ID }, { type: EventType.MODEL_MESSAGE_DELTA, thread_id: ROOT_ID }, { type: InternalEventType.AGENT_CONTEXT_APPEND, thread_id: ROOT_ID }, @@ -40,7 +40,7 @@ const EXPECTED_PAUSE_EVENTS = [ }, ]; -const PAUSE_OUTPUT = { +const TURN_1_OUTPUT = { output: null, required_actions: [ { @@ -51,9 +51,8 @@ const PAUSE_OUTPUT = { ], }; -const EXPECTED_PAUSE_LLM_INPUT = [ +const EXPECTED_TURN_1_INPUT = [ { - stream: true, tools: WRITE_NOTE_TOOLS, messages: [ { role: 'system', content: expect.stringContaining(INSTRUCTION) }, @@ -62,7 +61,7 @@ const EXPECTED_PAUSE_LLM_INPUT = [ }, ]; -const EXPECTED_RESUME_EVENTS = [ +const EXPECTED_TURN_2_EVENTS = [ { type: EventType.TOOL_RESPONSE, thread_id: ROOT_ID, tool_call_id: WRITE_NOTE_CALL_ID }, { type: InternalEventType.AGENT_CONTEXT_APPEND, thread_id: ROOT_ID }, { type: EventType.MODEL_MESSAGE, thread_id: ROOT_ID }, @@ -71,13 +70,12 @@ const EXPECTED_RESUME_EVENTS = [ { type: InternalEventType.AGENT_DONE, thread_id: ROOT_ID, status: 'done' }, ]; -const RESUME_OUTPUT = { +const TURN_2_OUTPUT = { output: { thread_id: ROOT_ID, content: ROOT_FINAL }, required_actions: [], }; -const EXPECTED_RESUME_LLM_INPUT = { - stream: true, +const EXPECTED_TURN_2_OUTPUT = { tools: WRITE_NOTE_TOOLS, messages: [ { role: 'system', content: expect.stringContaining(INSTRUCTION) }, @@ -111,10 +109,10 @@ describe('orchestration: pause then resume on tool approval', () => { orchestrator, sendBatch: [{ type: EventType.USER_MESSAGE, content: 'hello' }], }); - expect(paused.events).toMatchObject(EXPECTED_PAUSE_EVENTS); - expect(paused.result).toMatchObject(PAUSE_OUTPUT); + expect(paused.events).toMatchObject(EXPECTED_TURN_1_EVENTS); + expect(paused.result).toMatchObject(TURN_1_OUTPUT); expect(paused.result.root_agent_error).toBeUndefined(); - expect(llmCreateInputs(thread.definition.modelClient)).toMatchObject(EXPECTED_PAUSE_LLM_INPUT); + expect(llmCreateInputs(thread.definition.modelClient)).toMatchObject(EXPECTED_TURN_1_INPUT); const resumed = await runTurn({ orchestrator, @@ -127,12 +125,12 @@ describe('orchestration: pause then resume on tool approval', () => { }, ], }); - expect(resumed.events).toMatchObject(EXPECTED_RESUME_EVENTS); - expect(resumed.result).toMatchObject(RESUME_OUTPUT); + expect(resumed.events).toMatchObject(EXPECTED_TURN_2_EVENTS); + expect(resumed.result).toMatchObject(TURN_2_OUTPUT); expect(resumed.result.root_agent_error).toBeUndefined(); expect(llmCreateInputs(thread.definition.modelClient)).toMatchObject([ - ...EXPECTED_PAUSE_LLM_INPUT, - EXPECTED_RESUME_LLM_INPUT, + ...EXPECTED_TURN_1_INPUT, + EXPECTED_TURN_2_OUTPUT, ]); }); }); diff --git a/packages/trueforge-core/tests/orchestration/orchestrationWithTools.test.ts b/packages/trueforge-core/tests/orchestration/orchestrationWithTools.test.ts index e3ce2c38e..68381cfa3 100644 --- a/packages/trueforge-core/tests/orchestration/orchestrationWithTools.test.ts +++ b/packages/trueforge-core/tests/orchestration/orchestrationWithTools.test.ts @@ -19,14 +19,6 @@ const ROOT_FINAL = 'How are you?'; const INSTRUCTION = 'You are running in a test setup.'; const CHILD_TASK = 'do the delegated task'; -const ROOT_TOOLS = [ - { function: { name: 'call_tool' } }, - { function: { name: 'get_tool_info' } }, - { function: { name: 'get_tool_output_schema' } }, - { function: { name: 'list_tools' } }, - { function: { name: 'create_sub_agent' } }, -]; - /** Root delegates via create_sub_agent; child result returns to parent; root finishes. */ const EXPECTED_EVENTS = [ { type: EventType.MODEL_MESSAGE, thread_id: ROOT_ID }, @@ -57,16 +49,12 @@ const OUTPUT = { const EXPECTED_ROOT_LLM_INPUT = [ { - stream: true, - tools: ROOT_TOOLS, messages: [ { role: 'system', content: expect.stringContaining(INSTRUCTION) }, { role: 'user', content: 'hello' }, ], }, { - stream: true, - tools: ROOT_TOOLS, messages: [ { role: 'system', content: expect.stringContaining(INSTRUCTION) }, { role: 'user', content: 'hello' }, @@ -89,16 +77,6 @@ const EXPECTED_ROOT_LLM_INPUT = [ }, ]; -const EXPECTED_CHILD_LLM_INPUT = [ - { - stream: true, - messages: [ - { role: 'system', content: expect.stringContaining('sub-agent') }, - { role: 'user', content: CHILD_TASK }, - ], - }, -]; - describe('orchestration: dynamic sub-agent', () => { it('delegates via create_sub_agent, routes child result to parent, then finishes', async () => { let agentThreadInput: AgentThreadConstructorInput = { @@ -197,6 +175,5 @@ describe('orchestration: dynamic sub-agent', () => { if (childLLM === undefined) { throw new Error('expected child LLM to be created'); } - expect(llmCreateInputs(childLLM)).toMatchObject(EXPECTED_CHILD_LLM_INPUT); }); }); From f091e63e0d07000cda6061f8b81c8abee7179048 Mon Sep 17 00:00:00 2001 From: Raman Tehlan Date: Wed, 2 Sep 2026 18:35:28 +0530 Subject: [PATCH 14/19] test: simplify approval test --- .../orchestrationApproval.test.ts | 69 ++++++++++--------- 1 file changed, 36 insertions(+), 33 deletions(-) diff --git a/packages/trueforge-core/tests/orchestration/orchestrationApproval.test.ts b/packages/trueforge-core/tests/orchestration/orchestrationApproval.test.ts index dfdcf8f2f..23b65b0d6 100644 --- a/packages/trueforge-core/tests/orchestration/orchestrationApproval.test.ts +++ b/packages/trueforge-core/tests/orchestration/orchestrationApproval.test.ts @@ -1,4 +1,3 @@ -import type { AgentDefinition } from '../../src/core'; import { EventType } from '../../src/core/events/schema'; import { AgentThread } from '../../src/core/runtime/AgentThread'; import { InternalEventType, type AgentThreadConstructorInput } from '../../src/core/runtime/AgentThread.types'; @@ -8,12 +7,13 @@ import { makeSilentLogger } from '../core/harnessMocks'; import { llmCreateInputs, makeApprovalGatedWriteNoteToolSet, - makeApprovalThenTextLLM, runTurn, + textReplyStream, WRITE_NOTE_ARGUMENTS, WRITE_NOTE_CALL_ID, WRITE_NOTE_RESULT, WRITE_NOTE_TOOL_NAME, + writeNoteToolCallStream, } from './helpers/helpers'; const ROOT_ID = 'thread_root'; @@ -97,7 +97,38 @@ const EXPECTED_TURN_2_OUTPUT = { describe('orchestration: pause then resume on tool approval', () => { it('pauses for write_note approval, then finishes after allow', async () => { - const thread = makeApprovalThread(); + const agentThreadInput: AgentThreadConstructorInput = { + definition: { + modelClient: { + create: jest + .fn() + .mockImplementationOnce(() => writeNoteToolCallStream()) + .mockImplementation(() => textReplyStream(ROOT_FINAL)), + createNonStream: jest.fn(), + }, + instruction: INSTRUCTION, + messages: undefined, + modelParams: undefined, + responseFormat: undefined, + iterationLimit: undefined, + toolSets: [makeApprovalGatedWriteNoteToolSet()], + }, + threadId: ROOT_ID, + title: 'orchestration-approval', + parent: undefined, + agentInfo: undefined, + context: undefined, + currentContextUsage: undefined, + preComputedCompletion: undefined, + sandbox: undefined, + capabilities: undefined, + capabilityState: undefined, + tracing: NOOP_AGENT_TRACING, + logger: makeSilentLogger(), + }; + + const thread = new AgentThread(agentThreadInput); + const orchestrator = new AgentThreadOrchestrator({ agentThreads: new Map([[thread.threadId, thread]]), createDynamicSubAgentThread: () => Promise.reject(new Error('unexpected sub-agent in approval test')), @@ -109,6 +140,8 @@ describe('orchestration: pause then resume on tool approval', () => { orchestrator, sendBatch: [{ type: EventType.USER_MESSAGE, content: 'hello' }], }); + + // Asserts expect(paused.events).toMatchObject(EXPECTED_TURN_1_EVENTS); expect(paused.result).toMatchObject(TURN_1_OUTPUT); expect(paused.result.root_agent_error).toBeUndefined(); @@ -134,33 +167,3 @@ describe('orchestration: pause then resume on tool approval', () => { ]); }); }); - -function makeApprovalThread(): AgentThread { - const agentDefinition: AgentDefinition = { - modelClient: makeApprovalThenTextLLM(ROOT_FINAL), - instruction: INSTRUCTION, - messages: undefined, - modelParams: undefined, - responseFormat: undefined, - iterationLimit: undefined, - toolSets: [makeApprovalGatedWriteNoteToolSet()], - }; - - const agentThreadInput: AgentThreadConstructorInput = { - definition: agentDefinition, - threadId: ROOT_ID, - title: 'orchestration-approval', - parent: undefined, - agentInfo: undefined, - context: undefined, - currentContextUsage: undefined, - preComputedCompletion: undefined, - sandbox: undefined, - capabilities: undefined, - capabilityState: undefined, - tracing: NOOP_AGENT_TRACING, - logger: makeSilentLogger(), - }; - - return new AgentThread(agentThreadInput); -} From b3106a6e4292bcfa6246de54e145ae74f2be9197 Mon Sep 17 00:00:00 2001 From: Raman Tehlan Date: Wed, 2 Sep 2026 18:56:36 +0530 Subject: [PATCH 15/19] test: test context of event too in orchestration --- .../tests/orchestration/orchestration.test.ts | 6 ++- .../orchestrationApproval.test.ts | 30 +++++++++++-- .../orchestrationWithTools.test.ts | 43 ++++++++++++++++--- 3 files changed, 69 insertions(+), 10 deletions(-) diff --git a/packages/trueforge-core/tests/orchestration/orchestration.test.ts b/packages/trueforge-core/tests/orchestration/orchestration.test.ts index af3aa5591..30d43c270 100644 --- a/packages/trueforge-core/tests/orchestration/orchestration.test.ts +++ b/packages/trueforge-core/tests/orchestration/orchestration.test.ts @@ -14,7 +14,11 @@ const INSTRUCTION = 'You are running in a test setup.'; const EXPECTED_EVENTS = [ { type: EventType.MODEL_MESSAGE, thread_id: THREAD_ID }, { type: EventType.MODEL_MESSAGE_DELTA, thread_id: THREAD_ID, content: REPLY }, - { type: InternalEventType.AGENT_CONTEXT_APPEND, thread_id: THREAD_ID }, + { + type: InternalEventType.AGENT_CONTEXT_APPEND, + thread_id: THREAD_ID, + context: [{ role: 'assistant', content: REPLY }], + }, { type: InternalEventType.AGENT_DONE, thread_id: THREAD_ID, status: 'done' }, ]; diff --git a/packages/trueforge-core/tests/orchestration/orchestrationApproval.test.ts b/packages/trueforge-core/tests/orchestration/orchestrationApproval.test.ts index 23b65b0d6..f078e4048 100644 --- a/packages/trueforge-core/tests/orchestration/orchestrationApproval.test.ts +++ b/packages/trueforge-core/tests/orchestration/orchestrationApproval.test.ts @@ -32,7 +32,23 @@ const WRITE_NOTE_TOOLS = [ const EXPECTED_TURN_1_EVENTS = [ { type: EventType.MODEL_MESSAGE, thread_id: ROOT_ID }, { type: EventType.MODEL_MESSAGE_DELTA, thread_id: ROOT_ID }, - { type: InternalEventType.AGENT_CONTEXT_APPEND, thread_id: ROOT_ID }, + { + type: InternalEventType.AGENT_CONTEXT_APPEND, + thread_id: ROOT_ID, + context: [ + { + role: 'assistant', + content: null, + tool_calls: [ + { + id: WRITE_NOTE_CALL_ID, + type: 'function', + function: { name: WRITE_NOTE_TOOL_NAME, arguments: WRITE_NOTE_ARGUMENTS }, + }, + ], + }, + ], + }, { type: EventType.TOOL_APPROVAL_REQUIRED, thread_id: ROOT_ID, @@ -63,10 +79,18 @@ const EXPECTED_TURN_1_INPUT = [ const EXPECTED_TURN_2_EVENTS = [ { type: EventType.TOOL_RESPONSE, thread_id: ROOT_ID, tool_call_id: WRITE_NOTE_CALL_ID }, - { type: InternalEventType.AGENT_CONTEXT_APPEND, thread_id: ROOT_ID }, + { + type: InternalEventType.AGENT_CONTEXT_APPEND, + thread_id: ROOT_ID, + context: [{ role: 'tool', tool_call_id: WRITE_NOTE_CALL_ID, content: WRITE_NOTE_RESULT }], + }, { type: EventType.MODEL_MESSAGE, thread_id: ROOT_ID }, { type: EventType.MODEL_MESSAGE_DELTA, thread_id: ROOT_ID, content: ROOT_FINAL }, - { type: InternalEventType.AGENT_CONTEXT_APPEND, thread_id: ROOT_ID }, + { + type: InternalEventType.AGENT_CONTEXT_APPEND, + thread_id: ROOT_ID, + context: [{ role: 'assistant', content: ROOT_FINAL }], + }, { type: InternalEventType.AGENT_DONE, thread_id: ROOT_ID, status: 'done' }, ]; diff --git a/packages/trueforge-core/tests/orchestration/orchestrationWithTools.test.ts b/packages/trueforge-core/tests/orchestration/orchestrationWithTools.test.ts index 68381cfa3..e69633eae 100644 --- a/packages/trueforge-core/tests/orchestration/orchestrationWithTools.test.ts +++ b/packages/trueforge-core/tests/orchestration/orchestrationWithTools.test.ts @@ -19,12 +19,31 @@ const ROOT_FINAL = 'How are you?'; const INSTRUCTION = 'You are running in a test setup.'; const CHILD_TASK = 'do the delegated task'; +const CREATE_SUB_AGENT_ARGS = JSON.stringify({ name: 'worker', input: CHILD_TASK }); + /** Root delegates via create_sub_agent; child result returns to parent; root finishes. */ const EXPECTED_EVENTS = [ { type: EventType.MODEL_MESSAGE, thread_id: ROOT_ID }, { type: EventType.MODEL_MESSAGE_DELTA, thread_id: ROOT_ID }, - { type: InternalEventType.AGENT_CONTEXT_APPEND, thread_id: ROOT_ID }, - { type: InternalEventType.AGENT_CONTEXT_APPEND, thread_id: ROOT_ID }, + { + type: InternalEventType.AGENT_CONTEXT_APPEND, + thread_id: ROOT_ID, + context: [ + { + role: 'assistant', + content: null, + tool_calls: [ + { + id: TOOL_CALL_ID, + type: 'function', + function: { name: 'create_sub_agent', arguments: CREATE_SUB_AGENT_ARGS }, + }, + ], + }, + ], + }, + // create_sub_agent tool path yields an empty append before THREAD_CREATED. + { type: InternalEventType.AGENT_CONTEXT_APPEND, thread_id: ROOT_ID, context: [] }, { type: EventType.THREAD_CREATED, title: 'worker', @@ -32,13 +51,25 @@ const EXPECTED_EVENTS = [ }, { type: EventType.MODEL_MESSAGE, thread_id: expect.any(String) }, { type: EventType.MODEL_MESSAGE_DELTA, thread_id: expect.any(String), content: CHILD_REPLY }, - { type: InternalEventType.AGENT_CONTEXT_APPEND, thread_id: expect.any(String) }, + { + type: InternalEventType.AGENT_CONTEXT_APPEND, + thread_id: expect.any(String), + context: [{ role: 'assistant', content: CHILD_REPLY }], + }, { type: EventType.TOOL_RESPONSE, thread_id: ROOT_ID, tool_call_id: TOOL_CALL_ID }, - { type: InternalEventType.AGENT_CONTEXT_APPEND, thread_id: ROOT_ID }, + { + type: InternalEventType.AGENT_CONTEXT_APPEND, + thread_id: ROOT_ID, + context: [{ role: 'tool', tool_call_id: TOOL_CALL_ID, content: CHILD_REPLY }], + }, { type: InternalEventType.AGENT_DONE, thread_id: expect.any(String), status: 'done' }, { type: EventType.MODEL_MESSAGE, thread_id: ROOT_ID }, { type: EventType.MODEL_MESSAGE_DELTA, thread_id: ROOT_ID, content: ROOT_FINAL }, - { type: InternalEventType.AGENT_CONTEXT_APPEND, thread_id: ROOT_ID }, + { + type: InternalEventType.AGENT_CONTEXT_APPEND, + thread_id: ROOT_ID, + context: [{ role: 'assistant', content: ROOT_FINAL }], + }, { type: InternalEventType.AGENT_DONE, thread_id: ROOT_ID, status: 'done' }, ]; @@ -67,7 +98,7 @@ const EXPECTED_ROOT_LLM_INPUT = [ type: 'function', function: { name: 'create_sub_agent', - arguments: JSON.stringify({ name: 'worker', input: CHILD_TASK }), + arguments: CREATE_SUB_AGENT_ARGS, }, }, ], From 99ad0e2038aa58724a1a7210a0b7e658799d0834 Mon Sep 17 00:00:00 2001 From: Raman Tehlan Date: Wed, 2 Sep 2026 19:05:06 +0530 Subject: [PATCH 16/19] test: flaton the orchestration test --- .../tests/orchestration/orchestration.test.ts | 44 ++++++++++++------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/packages/trueforge-core/tests/orchestration/orchestration.test.ts b/packages/trueforge-core/tests/orchestration/orchestration.test.ts index 30d43c270..c252ae372 100644 --- a/packages/trueforge-core/tests/orchestration/orchestration.test.ts +++ b/packages/trueforge-core/tests/orchestration/orchestration.test.ts @@ -4,7 +4,7 @@ import { InternalEventType } from '../../src/core/runtime/AgentThread.types'; import { AgentThreadOrchestrator } from '../../src/core/runtime/AgentThreadOrchestrator'; import { NOOP_AGENT_TRACING } from '../../src/core/tracing/NoopAgentTracing'; import { makeSilentLogger } from '../core/harnessMocks'; -import { llmCreateInputs, makeTextLLM, runTurn } from './helpers/helpers'; +import { llmCreateInputs, runTurn, textReplyStream } from './helpers/helpers'; const THREAD_ID = 'main'; const REPLY = 'hello from the mocked model'; @@ -37,23 +37,35 @@ const EXPECTED_LLM_INPUT = [ }, ]; -/** Root thread with a one-shot text LLM and no tool sets. */ -function makeTextLlmThread(): AgentThread { - return new AgentThread({ - threadId: THREAD_ID, - title: 'orchestration', - tracing: NOOP_AGENT_TRACING, - logger: makeSilentLogger(), - definition: { - modelClient: makeTextLLM(REPLY), - instruction: INSTRUCTION, - }, - }); -} - describe('orchestration: mocked LLM and no tools', () => { it('sends a user message and finishes the thread with a text reply', async () => { - const thread = makeTextLlmThread(); + const thread = new AgentThread({ + definition: { + modelClient: { + create: jest.fn().mockImplementation(() => textReplyStream(REPLY)), + createNonStream: jest.fn().mockImplementation(() => textReplyStream(REPLY)), + }, + instruction: INSTRUCTION, + messages: undefined, + modelParams: undefined, + responseFormat: undefined, + iterationLimit: undefined, + toolSets: [], + }, + threadId: THREAD_ID, + title: 'orchestration', + parent: undefined, + agentInfo: undefined, + context: undefined, + currentContextUsage: undefined, + preComputedCompletion: undefined, + sandbox: undefined, + capabilities: undefined, + capabilityState: undefined, + tracing: NOOP_AGENT_TRACING, + logger: makeSilentLogger(), + }); + // Orchestrator owns the thread map and fans send/execute across live threads. // This case has only the root thread, so sub-agent creation must never run. const orchestrator = new AgentThreadOrchestrator({ From 9086f0d588801bb29c3240e42f16a7a34944e926 Mon Sep 17 00:00:00 2001 From: Raman Tehlan Date: Wed, 2 Sep 2026 19:13:06 +0530 Subject: [PATCH 17/19] test: move tools from tool set to system capability --- .../tests/orchestration/orchestration.test.ts | 2 +- .../orchestrationApproval.test.ts | 23 +++++++++++-------- .../orchestrationWithTools.test.ts | 6 ++--- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/packages/trueforge-core/tests/orchestration/orchestration.test.ts b/packages/trueforge-core/tests/orchestration/orchestration.test.ts index c252ae372..7e4d657f4 100644 --- a/packages/trueforge-core/tests/orchestration/orchestration.test.ts +++ b/packages/trueforge-core/tests/orchestration/orchestration.test.ts @@ -50,7 +50,7 @@ describe('orchestration: mocked LLM and no tools', () => { modelParams: undefined, responseFormat: undefined, iterationLimit: undefined, - toolSets: [], + toolSets: undefined, }, threadId: THREAD_ID, title: 'orchestration', diff --git a/packages/trueforge-core/tests/orchestration/orchestrationApproval.test.ts b/packages/trueforge-core/tests/orchestration/orchestrationApproval.test.ts index f078e4048..8a27d2321 100644 --- a/packages/trueforge-core/tests/orchestration/orchestrationApproval.test.ts +++ b/packages/trueforge-core/tests/orchestration/orchestrationApproval.test.ts @@ -20,13 +20,7 @@ const ROOT_ID = 'thread_root'; const ROOT_FINAL = 'note saved'; const INSTRUCTION = 'You are running in a test setup.'; -const WRITE_NOTE_TOOLS = [ - { function: { name: 'call_tool' } }, - { function: { name: 'get_tool_info' } }, - { function: { name: 'get_tool_output_schema' } }, - { function: { name: 'list_tools' } }, - { function: { name: WRITE_NOTE_TOOL_NAME } }, -]; +const WRITE_NOTE_TOOLS = [{ function: { name: WRITE_NOTE_TOOL_NAME } }]; /** Pause on write_note approval, then resume after allow and finish. */ const EXPECTED_TURN_1_EVENTS = [ @@ -135,7 +129,7 @@ describe('orchestration: pause then resume on tool approval', () => { modelParams: undefined, responseFormat: undefined, iterationLimit: undefined, - toolSets: [makeApprovalGatedWriteNoteToolSet()], + toolSets: undefined, }, threadId: ROOT_ID, title: 'orchestration-approval', @@ -145,7 +139,18 @@ describe('orchestration: pause then resume on tool approval', () => { currentContextUsage: undefined, preComputedCompletion: undefined, sandbox: undefined, - capabilities: undefined, + capabilities: [ + { + systemToolSets: [makeApprovalGatedWriteNoteToolSet()], + preSendProcessors: undefined, + preLLMProcessors: undefined, + preLLMEphemeralProcessors: undefined, + postToolCallProcessors: undefined, + toolResponseProcessors: undefined, + instructionBuilders: undefined, + // state: {key: string, load(state: JsonValue): void} + }, + ], capabilityState: undefined, tracing: NOOP_AGENT_TRACING, logger: makeSilentLogger(), diff --git a/packages/trueforge-core/tests/orchestration/orchestrationWithTools.test.ts b/packages/trueforge-core/tests/orchestration/orchestrationWithTools.test.ts index e69633eae..6db326c38 100644 --- a/packages/trueforge-core/tests/orchestration/orchestrationWithTools.test.ts +++ b/packages/trueforge-core/tests/orchestration/orchestrationWithTools.test.ts @@ -1,5 +1,5 @@ import type { AgentDefinition, CreateDynamicSubAgentThread } from '../../src/core'; -import { DynamicSubAgents } from '../../src/core/capabilities/builtins/DynamicSubAgents'; +import { dynamicSubAgents } from '../../src/core/capabilities/builtins/DynamicSubAgents'; import { EventType } from '../../src/core/events/schema'; import type { ILLM } from '../../src/core/llm/ILLM'; import { AgentThread } from '../../src/core/runtime/AgentThread'; @@ -127,7 +127,7 @@ describe('orchestration: dynamic sub-agent', () => { modelParams: undefined, responseFormat: undefined, iterationLimit: undefined, - toolSets: [new DynamicSubAgents({ tracing: NOOP_AGENT_TRACING })], + toolSets: undefined, }, threadId: ROOT_ID, title: 'orchestration-with-tools', @@ -138,7 +138,7 @@ describe('orchestration: dynamic sub-agent', () => { currentContextUsage: undefined, preComputedCompletion: undefined, sandbox: undefined, - capabilities: undefined, + capabilities: [dynamicSubAgents({ sandboxAvailable: false, tracing: NOOP_AGENT_TRACING })], capabilityState: undefined, // Default tracing: NOOP_AGENT_TRACING, From 14dda847fb9f230ebe571d0de19d9676ab3212f9 Mon Sep 17 00:00:00 2001 From: Raman Tehlan Date: Wed, 2 Sep 2026 19:47:24 +0530 Subject: [PATCH 18/19] test: improve normal test --- .../tests/orchestration/helpers/helpers.ts | 29 ------------------- .../tests/orchestration/orchestration.test.ts | 2 +- .../orchestrationApproval.test.ts | 2 +- 3 files changed, 2 insertions(+), 31 deletions(-) diff --git a/packages/trueforge-core/tests/orchestration/helpers/helpers.ts b/packages/trueforge-core/tests/orchestration/helpers/helpers.ts index 1f1ad4b9f..bc0c3c5d0 100644 --- a/packages/trueforge-core/tests/orchestration/helpers/helpers.ts +++ b/packages/trueforge-core/tests/orchestration/helpers/helpers.ts @@ -84,24 +84,6 @@ export async function* createSubAgentStream() { }; } -/** ILLM that always streams `text` and then stops. */ -export function makeTextLLM(text: string): ILLM { - return { - create: jest.fn().mockImplementation(() => textReplyStream(text)), - createNonStream: jest.fn().mockImplementation(() => textReplyStream(text)), - }; -} - -export function makeRootLLM(finalReply: string): ILLM { - return { - create: jest - .fn() - .mockImplementationOnce(() => createSubAgentStream()) - .mockImplementation(() => textReplyStream(finalReply)), - createNonStream: jest.fn(), - }; -} - // eslint-disable-next-line @typescript-eslint/require-await -- async generator fixture, not awaiting I/O export async function* writeNoteToolCallStream() { yield { @@ -151,17 +133,6 @@ export async function* writeNoteToolCallStream() { }; } -/** First create() requests write_note; later calls stream `finalReply`. */ -export function makeApprovalThenTextLLM(finalReply: string): ILLM { - return { - create: jest - .fn() - .mockImplementationOnce(() => writeNoteToolCallStream()) - .mockImplementation(() => textReplyStream(finalReply)), - createNonStream: jest.fn(), - }; -} - function makeWriteNoteSource(): ToolSource { return { name: 'notes', diff --git a/packages/trueforge-core/tests/orchestration/orchestration.test.ts b/packages/trueforge-core/tests/orchestration/orchestration.test.ts index 7e4d657f4..3dceffddb 100644 --- a/packages/trueforge-core/tests/orchestration/orchestration.test.ts +++ b/packages/trueforge-core/tests/orchestration/orchestration.test.ts @@ -1,3 +1,4 @@ +/** One root thread, no tools: user message in, text reply out. */ import { EventType } from '../../src/core/events/schema'; import { AgentThread } from '../../src/core/runtime/AgentThread'; import { InternalEventType } from '../../src/core/runtime/AgentThread.types'; @@ -10,7 +11,6 @@ const THREAD_ID = 'main'; const REPLY = 'hello from the mocked model'; const INSTRUCTION = 'You are running in a test setup.'; -/** One root thread, no tools: user message in, text reply out. */ const EXPECTED_EVENTS = [ { type: EventType.MODEL_MESSAGE, thread_id: THREAD_ID }, { type: EventType.MODEL_MESSAGE_DELTA, thread_id: THREAD_ID, content: REPLY }, diff --git a/packages/trueforge-core/tests/orchestration/orchestrationApproval.test.ts b/packages/trueforge-core/tests/orchestration/orchestrationApproval.test.ts index 8a27d2321..7c455543d 100644 --- a/packages/trueforge-core/tests/orchestration/orchestrationApproval.test.ts +++ b/packages/trueforge-core/tests/orchestration/orchestrationApproval.test.ts @@ -1,3 +1,4 @@ +/** Pause on write_note approval, then resume after allow and finish. */ import { EventType } from '../../src/core/events/schema'; import { AgentThread } from '../../src/core/runtime/AgentThread'; import { InternalEventType, type AgentThreadConstructorInput } from '../../src/core/runtime/AgentThread.types'; @@ -22,7 +23,6 @@ const INSTRUCTION = 'You are running in a test setup.'; const WRITE_NOTE_TOOLS = [{ function: { name: WRITE_NOTE_TOOL_NAME } }]; -/** Pause on write_note approval, then resume after allow and finish. */ const EXPECTED_TURN_1_EVENTS = [ { type: EventType.MODEL_MESSAGE, thread_id: ROOT_ID }, { type: EventType.MODEL_MESSAGE_DELTA, thread_id: ROOT_ID }, From 5c5d88787f7471ca290e26dd381514ba0440de58 Mon Sep 17 00:00:00 2001 From: Raman Tehlan Date: Thu, 3 Sep 2026 15:23:17 +0530 Subject: [PATCH 19/19] test: add deny flow to the approval test --- .../tests/orchestration/README.md | 386 ------------------ .../tests/orchestration/helpers/helpers.ts | 35 +- .../orchestrationApproval.test.ts | 311 +++++++++----- 3 files changed, 224 insertions(+), 508 deletions(-) delete mode 100644 packages/trueforge-core/tests/orchestration/README.md diff --git a/packages/trueforge-core/tests/orchestration/README.md b/packages/trueforge-core/tests/orchestration/README.md deleted file mode 100644 index 390171aa3..000000000 --- a/packages/trueforge-core/tests/orchestration/README.md +++ /dev/null @@ -1,386 +0,0 @@ -# Orchestration tests - -End-to-end tests for `AgentThreadOrchestrator` and `AgentThread` in `@truefoundry/trueforge-core`. - -These tests wire the real orchestration loop with **mocked LLMs** and **no database**. They exist to learn and verify how a turn flows through the harness before adding persistence (`SessionHandle`), HTTP, or real model providers. - -## What we are testing - -| Layer | In scope | Out of scope | -| --------------------------------- | ------------------------------------------------------------------------- | ------------------------------------------- | -| `AgentThreadOrchestrator.send` | Route input to threads, validate, append context | Postgres / Redis store writes | -| `AgentThreadOrchestrator.execute` | Run leaf threads, merge streams, spawn sub-agents, return terminal result | `TurnHandle.stream` persistence | -| `AgentThread` | LLM loop, tool execution, context mutations | Real OpenAI / Vercel AI calls | -| Sub-agent lifecycle | `create_sub_agent` tool → child thread → result back to parent | Full `SessionHandle` resolver / spec wiring | - -**Goal:** prove the orchestrator correctly coordinates one root thread (Program 1) and a root + dynamic child thread (Program 2). - -## Why this design - -Production creates the orchestrator inside `SessionHandle.createTurn`: - -```text -resolve definitions → build AgentThread map → new AgentThreadOrchestrator → send → persist → execute (via TurnHandle) -``` - -These orchestration tests **skip the store and session layer** and talk to the orchestrator directly. That keeps the surface area small while still exercising the same `send` / `execute` contract production uses. - -```mermaid -flowchart LR - subgraph production["Production path"] - SH[SessionHandle] - Store[(ISessionStore)] - OrchP[AgentThreadOrchestrator] - SH --> Store - SH --> OrchP - end - - subgraph orch["Orchestration tests"] - Test[Jest test] - OrchE[AgentThreadOrchestrator] - MockLLM[Mock ILLM] - Test --> OrchE - OrchE --> MockLLM - end - - OrchP -. same class .- OrchE -``` - -## Files - -| File | Role | -| -------------------------------- | -------------------------------------------------------------------- | -| `orchestration.test.ts` | **Program 1** - single root thread, text-only reply, full assertions | -| `orchestrationWithTools.test.ts` | **Program 2** - root delegates to sub-agent via `create_sub_agent` | -| `helpers.ts` | Mock LLM streams and approval-gated tools | - -## Core components under test - -### `AgentThread` - -One conversation thread. Holds: - -- **`definition`** - `modelClient` (`ILLM`), optional `instruction`, `toolSets`, etc. -- **`context`** - LLM message history (user, assistant, tool messages) -- **`send(messages)`** - append user input, approvals, or tool responses to context (no LLM call) -- **`execute({ signal })`** - run the state machine: LLM → tools → pause or done - -### `AgentThreadOrchestrator` - -Owns a `Map` and coordinates a turn: - -- **`send(batch)`** - fan out messages to the right threads, validate, delegate to each thread's `send` -- **`execute({ signal })`** - run **leaf** threads in parallel (up to 5), merge event streams, handle sub-agent creation/completion -- **`createDynamicSubAgentThread`** - factory callback invoked when the root calls `create_sub_agent`; must return a new `AgentThread` (not called at construction time) - -### `CreateDynamicSubAgentThread` - -```ts -(input: { - parentDefinition: AgentDefinition; - request: AgentInfo; // { type: 'dynamic', name, input, model? } - threadId: string; // orchestrator already minted this - parent: AgentParent; // { thread_id, tool_call_id } - signal: AbortSignal; -}) => Promise; -``` - -Pass the **function reference** to the orchestrator. Do not call it yourself. - -## Turn lifecycle: `send` then `execute` - -These are separate steps on purpose (same as production: send before commit, then execute). - -```mermaid -sequenceDiagram - participant Test - participant Orch as AgentThreadOrchestrator - participant Thread as AgentThread - participant LLM as Mock ILLM - - Test->>Orch: send([USER_MESSAGE]) - Orch->>Thread: send(messages) - Thread-->>Orch: AGENT_CONTEXT_APPEND - Orch-->>Test: yield append events - - Note over Test,LLM: send does NOT call the model - - Test->>Orch: execute({ signal }) - loop until AGENT_DONE or pause - Orch->>Thread: execute({ signal }) - Thread->>LLM: create(streaming) - LLM-->>Thread: chunks / tool_calls - Thread-->>Orch: model.message.delta, model.message, ... - Orch-->>Test: yield execution events - end - Orch-->>Test: return AgentThreadExecutionResult -``` - -**Important:** `send` returns an async generator. You must consume it with `for await`; otherwise the user message never lands in context. - -**Important:** `execute` also returns an async generator. The **return value** (final assistant output, required pauses, errors) is only available after the last `next()` when `done === true`. - -## Mock LLM helpers (`helpers.ts`) - -| Helper | Behavior | -| ------------------------- | ---------------------------------------------------------------------------- | -| `textReplyStream(text)` | One streaming chunk + stop completion with fixed text | -| `makeTextLLM(text)` | `ILLM` that always replies with `text` (used for child threads) | -| `createSubAgentStream()` | First root call: stream a `create_sub_agent` tool call | -| `makeRootLLM(finalReply)` | First `create()` → sub-agent tool call; every later call → `finalReply` text | - -Root and child threads use **different** `ILLM` instances so each can follow its own scripted sequence. - ---- - -## Program 1: text-only happy path - -**File:** `orchestration.test.ts` - -### Setup - -| Piece | Value | -| ----------------------------- | -------------------------------------------- | -| Root thread id | `"main"` | -| LLM | `makeTextLLM("hello from the mocked model")` | -| Tool sets | none | -| `createDynamicSubAgentThread` | rejects if ever called | -| Tracing | `NOOP_AGENT_TRACING` | -| Logger | silent (`makeSilentLogger`) | - -### Data flow - -```mermaid -flowchart TD - A["send: USER_MESSAGE 'hello'"] --> B["context: user message appended"] - B --> C["execute: llm-call-required"] - C --> D["Mock LLM streams text reply"] - D --> E["context: assistant message appended"] - E --> F["AGENT_DONE on root"] - F --> G["execute returns output + empty required_actions"] -``` - -### Expected event types - -**After `send`:** - -```text -internal.agent.context.append -``` - -**During `execute` (order may include duplicates / internal appends):** - -```text -model.message.delta -model.message -internal.agent.done ← last yielded event -``` - -**Must NOT appear:** - -```text -thread.created -tool.response -``` - -### Passing expectations (assertions) - -- `step.value.output.content` === `"hello from the mocked model"` -- `step.value.required_actions` === `[]` -- `step.value.root_agent_error` is undefined -- Root snapshot context contains user `"hello"` and assistant reply - ---- - -## Program 2: sub-agent delegation - -**File:** `orchestrationWithTools.test.ts` - -### Setup - -| Piece | Root thread | Child thread | -| ---------------- | ----------------------------- | ----------------------------------------------- | -| Thread id | `"thread_1"` (fixed) | minted by orchestrator at runtime | -| LLM | `makeRootLLM("How are you?")` | `makeTextLLM("hello from the child")` | -| Tool sets | `[new DynamicSubAgents(...)]` | `undefined` (no nested sub-agents) | -| Instruction | test setup string | `undefined` (harness adds `SUB_AGENT_IDENTITY`) | -| Initial messages | none | `[{ role: 'user', content: request.input }]` | -| Parent link | none | `{ thread_id, tool_call_id }` from orchestrator | - -`createSubAgentThread` is a top-level `CreateDynamicSubAgentThread` implementation (mirrors a simplified `SessionHandle.makeCreateDynamicSubAgentThread`). - -### Scripted LLM behavior - -1. **Root call 1** - model returns `create_sub_agent` with `{ name: 'worker', input: '...' }` -2. **Child call 1** - model returns `"hello from the child"` -3. **Root call 2** - model returns `"How are you?"` - -### Thread tree over time - -```mermaid -flowchart TD - subgraph phase1["After root LLM call 1"] - R1["thread_1 (root)
open create_sub_agent tool call"] - end - - subgraph phase2["After sub-agent created"] - R2["thread_1 (root)
waiting on tool call"] - C["child thread (leaf)
runs execute"] - R2 --- C - end - - subgraph phase3["After child AGENT_DONE"] - R3["thread_1 (root, leaf again)
tool result appended"] - end - - phase1 --> phase2 --> phase3 -``` - -Only **leaf** threads run. While the child exists, the root is paused (not a leaf). When the child finishes, the orchestrator: - -1. Yields `tool.response` on the parent -2. `send()`s the child's result into the parent as a tool message -3. Removes the child from the thread map -4. Resumes the root for LLM call 2 - -### Data flow - -```mermaid -sequenceDiagram - participant Test - participant Orch as Orchestrator - participant Root as thread_1 - participant Child as sub-agent - participant RootLLM as makeRootLLM - participant ChildLLM as makeTextLLM - - Test->>Orch: send(USER_MESSAGE) - Test->>Orch: execute() - - Root->>RootLLM: create() #1 - RootLLM-->>Root: create_sub_agent tool call - Root-->>Orch: internal.agent.create_subagent - Orch->>Orch: createSubAgentThread(...) - Orch-->>Test: thread.created - - Child->>ChildLLM: create() - ChildLLM-->>Child: "hello from the child" - Child-->>Orch: model.message, AGENT_DONE (child) - - Orch-->>Test: tool.response (parent) - Orch->>Root: send(tool message with child result) - - Root->>RootLLM: create() #2 - RootLLM-->>Root: "How are you?" - Root-->>Orch: model.message, AGENT_DONE (root) - Orch-->>Test: return { output: "How are you?", ... } -``` - -### Expected event types (from logging) - -Typical `execute` event sequence: - -```text -model.message / model.message.delta ← root tool call -internal.agent.context.append ← (internal, may repeat) -thread.created ← child registered -model.message / model.message.delta ← child reply -tool.response ← child result routed to parent -internal.agent.done ← child finished (thread_id = child) -model.message / model.message.delta ← root final reply -internal.agent.done ← root finished (last event) -``` - -`internal.agent.create_subagent` is handled inside the orchestrator and is **not** yielded to the test consumer. - -### Expected final state - -**`execute` return value:** - -| Field | Expected | -| ------------------ | --------------------------------------------------- | -| `output.content` | `"How are you?"` (root final reply, not child text) | -| `required_actions` | `[]` | -| `root_agent_error` | undefined | - -**Root thread context (after send + execute):** - -```text -1. user: "hello" -2. assistant: tool_call create_sub_agent (id: call-sub) -3. tool: "hello from the child" -4. assistant: "How are you?" -``` - ---- - -## Running tests - -From `packages/trueforge-core`: - -```bash -pnpm test -``` - -Single file: - -```bash -pnpm test -- orchestration.test.ts -pnpm test -- orchestrationWithTools.test.ts -``` - -From repo root: - -```bash -pnpm test:trueforge-core -``` - -These files run with the rest of `@truefoundry/trueforge-core` via `jest.config.cjs`. - -Threads and the orchestrator still take a Winston logger (required by the runtime). These tests use `makeSilentLogger()` from `tests/core/harnessMocks.ts`, so the suite does not print turn flow. - -## Relationship to production - -| Orchestration test | Production equivalent | -| -------------------------------------- | ------------------------------------------------------ | -| `new AgentThread({ definition, ... })` | `SessionHandle.buildThreads` + resolver | -| `createSubAgentThread` callback | `SessionHandle.makeCreateDynamicSubAgentThread` | -| `orchestrator.send` + `execute` | `SessionHandle.createTurn` + `TurnHandle.stream` | -| In-memory `thread.toSnapshot()` | `ISessionStore.createTurn` / persisted context appends | -| `NOOP_AGENT_TRACING` | `resolver.createTracing()` | - -Production adds: store persistence, turn records, event folding for SSE, sandbox resolution, full builtin capabilities from `AgentSpec`, and MCP servers beyond `DynamicSubAgents`. - -## Planned coverage (not yet implemented) - -| Program | Scenario | -| ------- | -------------------------------------------------------------------------------------------------- | -| **3** | Pause on `tool.approval.required` or `tool.response.required`, then resume with `send` + `execute` | -| **4** | Reject user message while sub-agent is live (`InvalidAgentSendInputError`) | -| **5** | MCP auth required (`internal.mcp.auth_required` merge across parallel sub-agents) | - -## Quick reference: orchestrator inputs - -```ts -new AgentThreadOrchestrator({ - agentThreads: new Map([[rootThreadId, rootThread]]), - createDynamicSubAgentThread, // function reference, not a call - tracing: NOOP_AGENT_TRACING, - logger, -}); -``` - -Every turn: - -```ts -for await (const _ of orchestrator.send(input)) { - /* collect appends */ -} -const it = orchestrator.execute({ signal }); -let step = await it.next(); -while (!step.done) { - // step.value is a streamed execution event - step = await it.next(); -} -// step.value is AgentThreadExecutionResult -``` diff --git a/packages/trueforge-core/tests/orchestration/helpers/helpers.ts b/packages/trueforge-core/tests/orchestration/helpers/helpers.ts index bc0c3c5d0..e4d40d1af 100644 --- a/packages/trueforge-core/tests/orchestration/helpers/helpers.ts +++ b/packages/trueforge-core/tests/orchestration/helpers/helpers.ts @@ -133,7 +133,7 @@ export async function* writeNoteToolCallStream() { }; } -function makeWriteNoteSource(): ToolSource { +function makeWriteNoteSource(callTool: ToolSource['callTool']): ToolSource { return { name: 'notes', id: 'notes', @@ -154,7 +154,7 @@ function makeWriteNoteSource(): ToolSource { }, wasInitialized: undefined, }), - callTool: () => Promise.resolve(toolResultResponse({ text: WRITE_NOTE_RESULT })), + callTool, toolCallInfo: () => Promise.resolve({ type: 'mcp', @@ -165,18 +165,25 @@ function makeWriteNoteSource(): ToolSource { }; } -/** User tool set that pauses until write_note is approved. */ -export function makeApprovalGatedWriteNoteToolSet(): IToolSet { - return new ToolSet({ - source: makeWriteNoteSource(), - selectors: { - enableTools: ['@all'], - disableTools: [], - preloadTools: [], - requireApprovalForTools: [WRITE_NOTE_TOOL_NAME], - }, - preload: true, - }); +/** Approval-gated write_note tool set; `callTool` spy proves allow runs the source and deny does not. */ +export function makeApprovalGatedWriteNoteToolSet(): { + toolSet: IToolSet; + callTool: jest.Mock; +} { + const callTool = jest.fn(() => Promise.resolve(toolResultResponse({ text: WRITE_NOTE_RESULT }))); + return { + toolSet: new ToolSet({ + source: makeWriteNoteSource(callTool), + selectors: { + enableTools: ['@all'], + disableTools: [], + preloadTools: [], + requireApprovalForTools: [WRITE_NOTE_TOOL_NAME], + }, + preload: true, + }), + callTool, + }; } /** Consume send() then execute(); return raw events and the generator result. */ diff --git a/packages/trueforge-core/tests/orchestration/orchestrationApproval.test.ts b/packages/trueforge-core/tests/orchestration/orchestrationApproval.test.ts index 7c455543d..9260b2524 100644 --- a/packages/trueforge-core/tests/orchestration/orchestrationApproval.test.ts +++ b/packages/trueforge-core/tests/orchestration/orchestrationApproval.test.ts @@ -1,4 +1,4 @@ -/** Pause on write_note approval, then resume after allow and finish. */ +/** Pause on write_note approval, then resume after allow or deny. */ import { EventType } from '../../src/core/events/schema'; import { AgentThread } from '../../src/core/runtime/AgentThread'; import { InternalEventType, type AgentThreadConstructorInput } from '../../src/core/runtime/AgentThread.types'; @@ -18,7 +18,11 @@ import { } from './helpers/helpers'; const ROOT_ID = 'thread_root'; -const ROOT_FINAL = 'note saved'; +const DENY_REASON = 'not allowed in this test'; +/** ToolSet deny → isError path wraps the text payload again for context. */ +const DENY_TOOL_CONTENT = JSON.stringify({ + error: [{ type: 'text', text: JSON.stringify({ error: `User denied tool call: ${DENY_REASON}` }) }], +}); const INSTRUCTION = 'You are running in a test setup.'; const WRITE_NOTE_TOOLS = [{ function: { name: WRITE_NOTE_TOOL_NAME } }]; @@ -71,128 +75,219 @@ const EXPECTED_TURN_1_INPUT = [ }, ]; -const EXPECTED_TURN_2_EVENTS = [ - { type: EventType.TOOL_RESPONSE, thread_id: ROOT_ID, tool_call_id: WRITE_NOTE_CALL_ID }, - { - type: InternalEventType.AGENT_CONTEXT_APPEND, - thread_id: ROOT_ID, - context: [{ role: 'tool', tool_call_id: WRITE_NOTE_CALL_ID, content: WRITE_NOTE_RESULT }], - }, - { type: EventType.MODEL_MESSAGE, thread_id: ROOT_ID }, - { type: EventType.MODEL_MESSAGE_DELTA, thread_id: ROOT_ID, content: ROOT_FINAL }, - { - type: InternalEventType.AGENT_CONTEXT_APPEND, - thread_id: ROOT_ID, - context: [{ role: 'assistant', content: ROOT_FINAL }], - }, - { type: InternalEventType.AGENT_DONE, thread_id: ROOT_ID, status: 'done' }, -]; +describe('orchestration: pause then resume on tool approval', () => { + describe('allow', () => { + const ROOT_FINAL = 'note saved'; -const TURN_2_OUTPUT = { - output: { thread_id: ROOT_ID, content: ROOT_FINAL }, - required_actions: [], -}; + const EXPECTED_TURN_2_EVENTS = [ + { type: EventType.TOOL_RESPONSE, thread_id: ROOT_ID, tool_call_id: WRITE_NOTE_CALL_ID }, + { + type: InternalEventType.AGENT_CONTEXT_APPEND, + thread_id: ROOT_ID, + context: [{ role: 'tool', tool_call_id: WRITE_NOTE_CALL_ID, content: WRITE_NOTE_RESULT }], + }, + { type: EventType.MODEL_MESSAGE, thread_id: ROOT_ID }, + { type: EventType.MODEL_MESSAGE_DELTA, thread_id: ROOT_ID, content: ROOT_FINAL }, + { + type: InternalEventType.AGENT_CONTEXT_APPEND, + thread_id: ROOT_ID, + context: [{ role: 'assistant', content: ROOT_FINAL }], + }, + { type: InternalEventType.AGENT_DONE, thread_id: ROOT_ID, status: 'done' }, + ]; -const EXPECTED_TURN_2_OUTPUT = { - tools: WRITE_NOTE_TOOLS, - messages: [ - { role: 'system', content: expect.stringContaining(INSTRUCTION) }, - { role: 'user', content: 'hello' }, - { - role: 'assistant', - content: null, - tool_calls: [ - { - id: WRITE_NOTE_CALL_ID, - type: 'function', - function: { name: WRITE_NOTE_TOOL_NAME, arguments: WRITE_NOTE_ARGUMENTS }, - }, - ], - }, - { role: 'tool', tool_call_id: WRITE_NOTE_CALL_ID, content: WRITE_NOTE_RESULT }, - ], -}; + const TURN_2_OUTPUT = { + output: { thread_id: ROOT_ID, content: ROOT_FINAL }, + required_actions: [], + }; -describe('orchestration: pause then resume on tool approval', () => { - it('pauses for write_note approval, then finishes after allow', async () => { - const agentThreadInput: AgentThreadConstructorInput = { - definition: { - modelClient: { - create: jest - .fn() - .mockImplementationOnce(() => writeNoteToolCallStream()) - .mockImplementation(() => textReplyStream(ROOT_FINAL)), - createNonStream: jest.fn(), - }, - instruction: INSTRUCTION, - messages: undefined, - modelParams: undefined, - responseFormat: undefined, - iterationLimit: undefined, - toolSets: undefined, - }, - threadId: ROOT_ID, - title: 'orchestration-approval', - parent: undefined, - agentInfo: undefined, - context: undefined, - currentContextUsage: undefined, - preComputedCompletion: undefined, - sandbox: undefined, - capabilities: [ + const EXPECTED_TURN_2_INPUT = { + tools: WRITE_NOTE_TOOLS, + messages: [ + { role: 'system', content: expect.stringContaining(INSTRUCTION) }, + { role: 'user', content: 'hello' }, { - systemToolSets: [makeApprovalGatedWriteNoteToolSet()], - preSendProcessors: undefined, - preLLMProcessors: undefined, - preLLMEphemeralProcessors: undefined, - postToolCallProcessors: undefined, - toolResponseProcessors: undefined, - instructionBuilders: undefined, - // state: {key: string, load(state: JsonValue): void} + role: 'assistant', + content: null, + tool_calls: [ + { + id: WRITE_NOTE_CALL_ID, + type: 'function', + function: { name: WRITE_NOTE_TOOL_NAME, arguments: WRITE_NOTE_ARGUMENTS }, + }, + ], }, + { role: 'tool', tool_call_id: WRITE_NOTE_CALL_ID, content: WRITE_NOTE_RESULT }, ], - capabilityState: undefined, - tracing: NOOP_AGENT_TRACING, - logger: makeSilentLogger(), }; - const thread = new AgentThread(agentThreadInput); + it('pauses for write_note approval, then finishes after allow', async () => { + const { orchestrator, thread, callTool } = makeApprovalHarness(ROOT_FINAL); - const orchestrator = new AgentThreadOrchestrator({ - agentThreads: new Map([[thread.threadId, thread]]), - createDynamicSubAgentThread: () => Promise.reject(new Error('unexpected sub-agent in approval test')), - tracing: NOOP_AGENT_TRACING, - logger: makeSilentLogger(), - }); + const paused = await runTurn({ + orchestrator, + sendBatch: [{ type: EventType.USER_MESSAGE, content: 'hello' }], + }); + expect(paused.events).toMatchObject(EXPECTED_TURN_1_EVENTS); + expect(paused.result).toMatchObject(TURN_1_OUTPUT); + expect(paused.result.root_agent_error).toBeUndefined(); + expect(llmCreateInputs(thread.definition.modelClient)).toMatchObject(EXPECTED_TURN_1_INPUT); + expect(callTool).not.toHaveBeenCalled(); - const paused = await runTurn({ - orchestrator, - sendBatch: [{ type: EventType.USER_MESSAGE, content: 'hello' }], + const resumed = await runTurn({ + orchestrator, + sendBatch: [ + { + type: EventType.USER_TOOL_APPROVAL, + thread_id: ROOT_ID, + tool_call_id: WRITE_NOTE_CALL_ID, + approval: { status: 'allow' }, + }, + ], + }); + expect(resumed.events).toMatchObject(EXPECTED_TURN_2_EVENTS); + expect(resumed.result).toMatchObject(TURN_2_OUTPUT); + expect(resumed.result.root_agent_error).toBeUndefined(); + expect(callTool).toHaveBeenCalledTimes(1); + expect(llmCreateInputs(thread.definition.modelClient)).toMatchObject([ + ...EXPECTED_TURN_1_INPUT, + EXPECTED_TURN_2_INPUT, + ]); }); + }); + + describe('deny', () => { + const ROOT_FINAL = 'ok, I will not write the note'; + + const EXPECTED_TURN_2_EVENTS = [ + { type: EventType.TOOL_RESPONSE, thread_id: ROOT_ID, tool_call_id: WRITE_NOTE_CALL_ID }, + { + type: InternalEventType.AGENT_CONTEXT_APPEND, + thread_id: ROOT_ID, + context: [{ role: 'tool', tool_call_id: WRITE_NOTE_CALL_ID, content: DENY_TOOL_CONTENT }], + }, + { type: EventType.MODEL_MESSAGE, thread_id: ROOT_ID }, + { type: EventType.MODEL_MESSAGE_DELTA, thread_id: ROOT_ID, content: ROOT_FINAL }, + { + type: InternalEventType.AGENT_CONTEXT_APPEND, + thread_id: ROOT_ID, + context: [{ role: 'assistant', content: ROOT_FINAL }], + }, + { type: InternalEventType.AGENT_DONE, thread_id: ROOT_ID, status: 'done' }, + ]; - // Asserts - expect(paused.events).toMatchObject(EXPECTED_TURN_1_EVENTS); - expect(paused.result).toMatchObject(TURN_1_OUTPUT); - expect(paused.result.root_agent_error).toBeUndefined(); - expect(llmCreateInputs(thread.definition.modelClient)).toMatchObject(EXPECTED_TURN_1_INPUT); + const TURN_2_OUTPUT = { + output: { thread_id: ROOT_ID, content: ROOT_FINAL }, + required_actions: [], + }; - const resumed = await runTurn({ - orchestrator, - sendBatch: [ + const EXPECTED_TURN_2_INPUT = { + tools: WRITE_NOTE_TOOLS, + messages: [ + { role: 'system', content: expect.stringContaining(INSTRUCTION) }, + { role: 'user', content: 'hello' }, { - type: EventType.USER_TOOL_APPROVAL, - thread_id: ROOT_ID, - tool_call_id: WRITE_NOTE_CALL_ID, - approval: { status: 'allow' }, + role: 'assistant', + content: null, + tool_calls: [ + { + id: WRITE_NOTE_CALL_ID, + type: 'function', + function: { name: WRITE_NOTE_TOOL_NAME, arguments: WRITE_NOTE_ARGUMENTS }, + }, + ], }, + { role: 'tool', tool_call_id: WRITE_NOTE_CALL_ID, content: DENY_TOOL_CONTENT }, ], + }; + + it('pauses for write_note approval, then finishes after deny without running the tool', async () => { + const { orchestrator, thread, callTool } = makeApprovalHarness(ROOT_FINAL); + + const paused = await runTurn({ + orchestrator, + sendBatch: [{ type: EventType.USER_MESSAGE, content: 'hello' }], + }); + expect(paused.events).toMatchObject(EXPECTED_TURN_1_EVENTS); + expect(paused.result).toMatchObject(TURN_1_OUTPUT); + expect(callTool).not.toHaveBeenCalled(); + + // Deny is a new turn (send + execute), same as allow — turn 1 already stopped at approval. + const resumed = await runTurn({ + orchestrator, + sendBatch: [ + { + type: EventType.USER_TOOL_APPROVAL, + thread_id: ROOT_ID, + tool_call_id: WRITE_NOTE_CALL_ID, + approval: { status: 'deny', reason: DENY_REASON }, + }, + ], + }); + expect(resumed.events).toMatchObject(EXPECTED_TURN_2_EVENTS); + expect(resumed.result).toMatchObject(TURN_2_OUTPUT); + expect(resumed.result.root_agent_error).toBeUndefined(); + expect(callTool).not.toHaveBeenCalled(); + expect(llmCreateInputs(thread.definition.modelClient)).toMatchObject([ + ...EXPECTED_TURN_1_INPUT, + EXPECTED_TURN_2_INPUT, + ]); }); - expect(resumed.events).toMatchObject(EXPECTED_TURN_2_EVENTS); - expect(resumed.result).toMatchObject(TURN_2_OUTPUT); - expect(resumed.result.root_agent_error).toBeUndefined(); - expect(llmCreateInputs(thread.definition.modelClient)).toMatchObject([ - ...EXPECTED_TURN_1_INPUT, - EXPECTED_TURN_2_OUTPUT, - ]); }); }); + +function makeApprovalHarness(finalReply: string): { + orchestrator: AgentThreadOrchestrator; + thread: AgentThread; + callTool: jest.Mock; +} { + const { toolSet, callTool } = makeApprovalGatedWriteNoteToolSet(); + const agentThreadInput: AgentThreadConstructorInput = { + definition: { + modelClient: { + create: jest + .fn() + .mockImplementationOnce(() => writeNoteToolCallStream()) + .mockImplementation(() => textReplyStream(finalReply)), + createNonStream: jest.fn(), + }, + instruction: INSTRUCTION, + messages: undefined, + modelParams: undefined, + responseFormat: undefined, + iterationLimit: undefined, + toolSets: undefined, + }, + threadId: ROOT_ID, + title: 'orchestration-approval', + parent: undefined, + agentInfo: undefined, + context: undefined, + currentContextUsage: undefined, + preComputedCompletion: undefined, + sandbox: undefined, + capabilities: [ + { + systemToolSets: [toolSet], + preSendProcessors: undefined, + preLLMProcessors: undefined, + preLLMEphemeralProcessors: undefined, + postToolCallProcessors: undefined, + toolResponseProcessors: undefined, + instructionBuilders: undefined, + }, + ], + capabilityState: undefined, + tracing: NOOP_AGENT_TRACING, + logger: makeSilentLogger(), + }; + + const thread = new AgentThread(agentThreadInput); + const orchestrator = new AgentThreadOrchestrator({ + agentThreads: new Map([[thread.threadId, thread]]), + createDynamicSubAgentThread: () => Promise.reject(new Error('unexpected sub-agent in approval test')), + tracing: NOOP_AGENT_TRACING, + logger: makeSilentLogger(), + }); + return { orchestrator, thread, callTool }; +}