Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 48 additions & 0 deletions packages/agent-core-v2/docs/state-manifest.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -721,6 +721,12 @@ export interface AgentStateSnapshot {
readonly skillPath?: string;
readonly skillSource?: 'project' | 'user' | 'extra' | 'builtin';
}[];
readonly attachments?: readonly /* PromptFileAttachment — packages/agent-core-v2/src/agent/contextMemory/types.ts */ {
readonly name: string;
readonly mediaType: string;
readonly size: number;
readonly path: string;
}[];
} | /* SkillActivationOrigin — packages/agent-core-v2/src/agent/contextMemory/types.ts */ {
readonly kind: 'skill_activation';
readonly activationId: string;
Expand All @@ -730,6 +736,12 @@ export interface AgentStateSnapshot {
readonly skillType?: string;
readonly skillPath?: string;
readonly skillSource?: 'project' | 'user' | 'extra' | 'builtin';
readonly attachments?: readonly /* PromptFileAttachment — packages/agent-core-v2/src/agent/contextMemory/types.ts */ {
readonly name: string;
readonly mediaType: string;
readonly size: number;
readonly path: string;
}[];
} | /* PluginCommandOrigin — packages/agent-core-v2/src/agent/contextMemory/types.ts */ {
readonly kind: 'plugin_command';
readonly activationId: string;
Expand Down Expand Up @@ -854,6 +866,12 @@ export interface AgentStateSnapshot {
readonly skillPath?: string;
readonly skillSource?: 'project' | 'user' | 'extra' | 'builtin';
}[];
readonly attachments?: readonly /* PromptFileAttachment — packages/agent-core-v2/src/agent/contextMemory/types.ts */ {
readonly name: string;
readonly mediaType: string;
readonly size: number;
readonly path: string;
}[];
} | /* SkillActivationOrigin — packages/agent-core-v2/src/agent/contextMemory/types.ts */ {
readonly kind: 'skill_activation';
readonly activationId: string;
Expand All @@ -863,6 +881,12 @@ export interface AgentStateSnapshot {
readonly skillType?: string;
readonly skillPath?: string;
readonly skillSource?: 'project' | 'user' | 'extra' | 'builtin';
readonly attachments?: readonly /* PromptFileAttachment — packages/agent-core-v2/src/agent/contextMemory/types.ts */ {
readonly name: string;
readonly mediaType: string;
readonly size: number;
readonly path: string;
}[];
} | /* PluginCommandOrigin — packages/agent-core-v2/src/agent/contextMemory/types.ts */ {
readonly kind: 'plugin_command';
readonly activationId: string;
Expand Down Expand Up @@ -919,6 +943,12 @@ export interface AgentStateSnapshot {
readonly skillPath?: string;
readonly skillSource?: 'project' | 'user' | 'extra' | 'builtin';
}[];
readonly attachments?: readonly /* PromptFileAttachment — packages/agent-core-v2/src/agent/contextMemory/types.ts */ {
readonly name: string;
readonly mediaType: string;
readonly size: number;
readonly path: string;
}[];
} | /* SkillActivationOrigin — packages/agent-core-v2/src/agent/contextMemory/types.ts */ {
readonly kind: 'skill_activation';
readonly activationId: string;
Expand All @@ -928,6 +958,12 @@ export interface AgentStateSnapshot {
readonly skillType?: string;
readonly skillPath?: string;
readonly skillSource?: 'project' | 'user' | 'extra' | 'builtin';
readonly attachments?: readonly /* PromptFileAttachment — packages/agent-core-v2/src/agent/contextMemory/types.ts */ {
readonly name: string;
readonly mediaType: string;
readonly size: number;
readonly path: string;
}[];
} | /* PluginCommandOrigin — packages/agent-core-v2/src/agent/contextMemory/types.ts */ {
readonly kind: 'plugin_command';
readonly activationId: string;
Expand Down Expand Up @@ -1062,6 +1098,12 @@ export interface AgentStateSnapshot {
readonly skillPath?: string;
readonly skillSource?: 'project' | 'user' | 'extra' | 'builtin';
}[];
readonly attachments?: readonly /* PromptFileAttachment — packages/agent-core-v2/src/agent/contextMemory/types.ts */ {
readonly name: string;
readonly mediaType: string;
readonly size: number;
readonly path: string;
}[];
} | /* SkillActivationOrigin — packages/agent-core-v2/src/agent/contextMemory/types.ts */ {
readonly kind: 'skill_activation';
readonly activationId: string;
Expand All @@ -1071,6 +1113,12 @@ export interface AgentStateSnapshot {
readonly skillType?: string;
readonly skillPath?: string;
readonly skillSource?: 'project' | 'user' | 'extra' | 'builtin';
readonly attachments?: readonly /* PromptFileAttachment — packages/agent-core-v2/src/agent/contextMemory/types.ts */ {
readonly name: string;
readonly mediaType: string;
readonly size: number;
readonly path: string;
}[];
} | /* PluginCommandOrigin — packages/agent-core-v2/src/agent/contextMemory/types.ts */ {
readonly kind: 'plugin_command';
readonly activationId: string;
Expand Down
9 changes: 9 additions & 0 deletions packages/agent-core-v2/src/agent/contextMemory/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,17 @@ import type { AgentTaskStatus } from '#/agent/task/task';

export type SkillSource = 'project' | 'user' | 'extra' | 'builtin';

export interface PromptFileAttachment {
readonly name: string;
readonly mediaType: string;
readonly size: number;
readonly path: string;
}

export interface UserPromptOrigin {
readonly kind: 'user';
readonly skillActivations?: readonly BundledSkillActivation[];
readonly attachments?: readonly PromptFileAttachment[];
}

export const USER_PROMPT_ORIGIN: UserPromptOrigin = { kind: 'user' };
Expand All @@ -29,6 +37,7 @@ export interface SkillActivationOrigin {
readonly skillType?: string | undefined;
readonly skillPath?: string | undefined;
readonly skillSource?: SkillSource | undefined;
readonly attachments?: readonly PromptFileAttachment[];
}

export interface PluginCommandOrigin {
Expand Down
2 changes: 1 addition & 1 deletion packages/agent-core-v2/src/agent/loop/loopService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -475,7 +475,7 @@ export class AgentLoopService extends Disposable implements IAgentLoopService {
turnId: job.turn.id,
origin,
prompt: isDisplayablePromptOrigin(origin) ? turnPromptText(job.seed.input, origin) : undefined,
promptAttachments: turnPromptAttachments(job.seed.input),
promptAttachments: turnPromptAttachments(job.seed.input, origin),
}),
);
void this.runTurn(job.turn, job.ready).then(job.result.resolve, job.result.reject);
Expand Down
22 changes: 20 additions & 2 deletions packages/agent-core-v2/src/agent/loop/turnEvents.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,24 @@ export type TurnInterruptReason =
| 'filtered'
| 'blocked';

export interface TurnPromptAttachmentFile {
readonly kind: 'file';
readonly name: string;
readonly mediaType: string;
readonly size: number;
readonly path: string;
}

export type TurnPromptAttachment =
| { readonly kind: 'image' | 'video' | 'audio'; readonly fileId: string }
| TurnPromptAttachmentFile;

export interface TurnStartedPayload {
readonly agentId: string;
readonly turnId: number;
readonly origin: PromptOrigin;
readonly prompt?: string;
readonly promptAttachments?: readonly { kind: 'image' | 'video' | 'audio'; fileId: string }[];
readonly promptAttachments?: readonly TurnPromptAttachment[];
}

export class TurnStarted extends AgentEvent2<TurnStartedPayload> {
Expand All @@ -45,8 +57,9 @@ export function turnPromptText(

export function turnPromptAttachments(
input: readonly ContentPart[],
origin?: PromptOrigin,
): TurnStartedPayload['promptAttachments'] {
const attachments: { kind: 'image' | 'video' | 'audio'; fileId: string }[] = [];
const attachments: TurnPromptAttachment[] = [];
const promptMediaFileId = (url: string, id: string | undefined): string | undefined => {
const fileId = parseDaemonFileUrl(url)?.fileId;
if (id === undefined) return fileId;
Expand All @@ -64,6 +77,11 @@ export function turnPromptAttachments(
if (fileId !== undefined) attachments.push({ kind: 'audio', fileId });
}
}
if (origin?.kind === 'user' || origin?.kind === 'skill_activation') {
for (const attachment of origin.attachments ?? []) {
attachments.push({ kind: 'file', ...attachment });
}
}
return attachments.length > 0 ? attachments : undefined;
}

Expand Down
12 changes: 11 additions & 1 deletion packages/agent-core-v2/src/agent/prompt/promptService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -152,14 +152,24 @@ function mergeSteerMessages(records: readonly Record[]): ContextMessage {
const skillActivations = records.flatMap((item) =>
item.message.origin?.kind === 'user' ? (item.message.origin.skillActivations ?? []) : [],
);
const attachments = records.flatMap((item) =>
item.message.origin?.kind === 'user' ? (item.message.origin.attachments ?? []) : [],
);
return {
role: 'user',
content: [
...records.flatMap((item) => item.message.content.slice(0, bundledSkillBlockCount(item.message))),
...records.flatMap((item) => stripBundledSkillBlocks(item.message)),
],
toolCalls: [],
origin: skillActivations.length === 0 ? USER_PROMPT_ORIGIN : { kind: 'user', skillActivations },
origin:
skillActivations.length === 0 && attachments.length === 0
? USER_PROMPT_ORIGIN
: {
kind: 'user',
skillActivations: skillActivations.length === 0 ? undefined : skillActivations,
attachments: attachments.length === 0 ? undefined : attachments,
},
};
}

Expand Down
3 changes: 3 additions & 0 deletions packages/agent-core-v2/src/features/skill/skill.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
import type { ContentPart } from '#/kosong/contract/message';
import type { PromptFileAttachment } from '#/agent/contextMemory/types';

export interface SkillActivationInput {
readonly name: string;
readonly args?: string;
readonly content?: readonly ContentPart[];
readonly attachments?: readonly PromptFileAttachment[];
}

export interface PromptSkillActivation {
Expand All @@ -14,6 +16,7 @@ export interface PromptSkillActivation {
export interface PromptWithSkillsInput {
readonly input: readonly ContentPart[];
readonly skills: readonly PromptSkillActivation[];
readonly attachments?: readonly PromptFileAttachment[];
}

export interface PromptWithSkillsResult {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ export class SkillRuntime {
skillPath: skill.path,
skillSource: skill.source,
skillArgs: input.args,
attachments: input.attachments,
},
content,
);
Expand Down Expand Up @@ -134,6 +135,7 @@ export class SkillRuntime {
origin: {
kind: 'user',
skillActivations: prepared.map((activation) => activation.entry),
attachments: input.attachments,
},
});
if (handle.state === 'pending') {
Expand Down
1 change: 1 addition & 0 deletions packages/agent-core-v2/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -586,6 +586,7 @@ export {
compressImageForModel,
gateImageFormatParts,
IMAGE_BYTE_BUDGET,
MAX_IMAGE_DECODE_BYTES,
MAX_IMAGE_EDGE_PX,
READ_IMAGE_BYTE_BUDGET,
resolveMaxImageEdgePx,
Expand Down
93 changes: 92 additions & 1 deletion packages/agent-core-v2/test/agent/loop/loop.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -845,7 +845,7 @@ describe('Agent loop', () => {
});

it('carries kimi-file prompt attachments on turn.started, falling back to the URL file id', async () => {
const payloads: Array<readonly { kind: string; fileId: string }[] | undefined> = [];
const payloads: Array<TurnStarted['promptAttachments']> = [];
const subscription = ctx.get(IEventBus).subscribe(TurnStarted, (event) => {
payloads.push(event.promptAttachments);
});
Expand Down Expand Up @@ -883,6 +883,97 @@ describe('Agent loop', () => {
],
]);
});

it('carries origin file attachments on turn.started promptAttachments', async () => {
const payloads: Array<TurnStarted['promptAttachments']> = [];
const subscription = ctx.get(IEventBus).subscribe(TurnStarted, (event) => {
payloads.push(event.promptAttachments);
});
ctx.mockNextResponse({ type: 'text', text: 'seen' });

const turn = (
await loop.enqueue(
new MessageStepRequest(
{
role: 'user',
content: [
{ type: 'image_url', imageUrl: { url: 'kimi-file://file_1', id: 'file_1' } },
{ type: 'text', text: 'summarize' },
],
toolCalls: [],
origin: {
kind: 'user',
attachments: [
{
name: 'report.pdf',
mediaType: 'application/pdf',
size: 42,
path: '/data/report.pdf',
},
],
},
},
{ admission: 'newTurn' },
),
).assigned
).turn;
await turn.result;
subscription.dispose();

expect(payloads).toEqual([
[
{ kind: 'image', fileId: 'file_1' },
{
kind: 'file',
name: 'report.pdf',
mediaType: 'application/pdf',
size: 42,
path: '/data/report.pdf',
},
],
]);
});

it('carries skill activation file attachments on turn.started promptAttachments', async () => {
const payloads: Array<TurnStarted['promptAttachments']> = [];
const subscription = ctx.get(IEventBus).subscribe(TurnStarted, (event) => {
payloads.push(event.promptAttachments);
});
ctx.mockNextResponse({ type: 'text', text: 'seen' });

const turn = (
await loop.enqueue(
new MessageStepRequest(
{
role: 'user',
content: [{ type: 'text', text: 'User activated the skill "check".' }],
toolCalls: [],
origin: {
kind: 'skill_activation',
activationId: 'act_1',
skillName: 'check',
trigger: 'user-slash',
attachments: [
{
name: 'note.txt',
mediaType: 'text/plain',
size: 21,
path: '/data/note.txt',
},
],
},
},
{ admission: 'newTurn' },
),
).assigned
).turn;
await turn.result;
subscription.dispose();

expect(payloads).toEqual([
[{ kind: 'file', name: 'note.txt', mediaType: 'text/plain', size: 21, path: '/data/note.txt' }],
]);
});
});

describe('turn telemetry', () => {
Expand Down
Loading
Loading