diff --git a/packages/agent-core-v2/docs/state-manifest.d.ts b/packages/agent-core-v2/docs/state-manifest.d.ts index 70f917a720..3f5fd7a681 100644 --- a/packages/agent-core-v2/docs/state-manifest.d.ts +++ b/packages/agent-core-v2/docs/state-manifest.d.ts @@ -721,6 +721,12 @@ export interface AgentStateSnapshot { readonly skillPath?: string; readonly skillSource?: 'project' | 'user' | 'extra' | 'builtin'; }[]; + readonly attachments?: readonly /* PromptFileAttachment — packages/agent-core-v2/src/agent/contextMemory/types.ts */ { + readonly name: string; + readonly mediaType: string; + readonly size: number; + readonly path: string; + }[]; } | /* SkillActivationOrigin — packages/agent-core-v2/src/agent/contextMemory/types.ts */ { readonly kind: 'skill_activation'; readonly activationId: string; @@ -730,6 +736,12 @@ export interface AgentStateSnapshot { readonly skillType?: string; readonly skillPath?: string; readonly skillSource?: 'project' | 'user' | 'extra' | 'builtin'; + readonly attachments?: readonly /* PromptFileAttachment — packages/agent-core-v2/src/agent/contextMemory/types.ts */ { + readonly name: string; + readonly mediaType: string; + readonly size: number; + readonly path: string; + }[]; } | /* PluginCommandOrigin — packages/agent-core-v2/src/agent/contextMemory/types.ts */ { readonly kind: 'plugin_command'; readonly activationId: string; @@ -854,6 +866,12 @@ export interface AgentStateSnapshot { readonly skillPath?: string; readonly skillSource?: 'project' | 'user' | 'extra' | 'builtin'; }[]; + readonly attachments?: readonly /* PromptFileAttachment — packages/agent-core-v2/src/agent/contextMemory/types.ts */ { + readonly name: string; + readonly mediaType: string; + readonly size: number; + readonly path: string; + }[]; } | /* SkillActivationOrigin — packages/agent-core-v2/src/agent/contextMemory/types.ts */ { readonly kind: 'skill_activation'; readonly activationId: string; @@ -863,6 +881,12 @@ export interface AgentStateSnapshot { readonly skillType?: string; readonly skillPath?: string; readonly skillSource?: 'project' | 'user' | 'extra' | 'builtin'; + readonly attachments?: readonly /* PromptFileAttachment — packages/agent-core-v2/src/agent/contextMemory/types.ts */ { + readonly name: string; + readonly mediaType: string; + readonly size: number; + readonly path: string; + }[]; } | /* PluginCommandOrigin — packages/agent-core-v2/src/agent/contextMemory/types.ts */ { readonly kind: 'plugin_command'; readonly activationId: string; @@ -919,6 +943,12 @@ export interface AgentStateSnapshot { readonly skillPath?: string; readonly skillSource?: 'project' | 'user' | 'extra' | 'builtin'; }[]; + readonly attachments?: readonly /* PromptFileAttachment — packages/agent-core-v2/src/agent/contextMemory/types.ts */ { + readonly name: string; + readonly mediaType: string; + readonly size: number; + readonly path: string; + }[]; } | /* SkillActivationOrigin — packages/agent-core-v2/src/agent/contextMemory/types.ts */ { readonly kind: 'skill_activation'; readonly activationId: string; @@ -928,6 +958,12 @@ export interface AgentStateSnapshot { readonly skillType?: string; readonly skillPath?: string; readonly skillSource?: 'project' | 'user' | 'extra' | 'builtin'; + readonly attachments?: readonly /* PromptFileAttachment — packages/agent-core-v2/src/agent/contextMemory/types.ts */ { + readonly name: string; + readonly mediaType: string; + readonly size: number; + readonly path: string; + }[]; } | /* PluginCommandOrigin — packages/agent-core-v2/src/agent/contextMemory/types.ts */ { readonly kind: 'plugin_command'; readonly activationId: string; @@ -1062,6 +1098,12 @@ export interface AgentStateSnapshot { readonly skillPath?: string; readonly skillSource?: 'project' | 'user' | 'extra' | 'builtin'; }[]; + readonly attachments?: readonly /* PromptFileAttachment — packages/agent-core-v2/src/agent/contextMemory/types.ts */ { + readonly name: string; + readonly mediaType: string; + readonly size: number; + readonly path: string; + }[]; } | /* SkillActivationOrigin — packages/agent-core-v2/src/agent/contextMemory/types.ts */ { readonly kind: 'skill_activation'; readonly activationId: string; @@ -1071,6 +1113,12 @@ export interface AgentStateSnapshot { readonly skillType?: string; readonly skillPath?: string; readonly skillSource?: 'project' | 'user' | 'extra' | 'builtin'; + readonly attachments?: readonly /* PromptFileAttachment — packages/agent-core-v2/src/agent/contextMemory/types.ts */ { + readonly name: string; + readonly mediaType: string; + readonly size: number; + readonly path: string; + }[]; } | /* PluginCommandOrigin — packages/agent-core-v2/src/agent/contextMemory/types.ts */ { readonly kind: 'plugin_command'; readonly activationId: string; diff --git a/packages/agent-core-v2/src/agent/contextMemory/types.ts b/packages/agent-core-v2/src/agent/contextMemory/types.ts index 6907ddc189..a925dbc089 100644 --- a/packages/agent-core-v2/src/agent/contextMemory/types.ts +++ b/packages/agent-core-v2/src/agent/contextMemory/types.ts @@ -4,9 +4,17 @@ import type { AgentTaskStatus } from '#/agent/task/task'; export type SkillSource = 'project' | 'user' | 'extra' | 'builtin'; +export interface PromptFileAttachment { + readonly name: string; + readonly mediaType: string; + readonly size: number; + readonly path: string; +} + export interface UserPromptOrigin { readonly kind: 'user'; readonly skillActivations?: readonly BundledSkillActivation[]; + readonly attachments?: readonly PromptFileAttachment[]; } export const USER_PROMPT_ORIGIN: UserPromptOrigin = { kind: 'user' }; @@ -29,6 +37,7 @@ export interface SkillActivationOrigin { readonly skillType?: string | undefined; readonly skillPath?: string | undefined; readonly skillSource?: SkillSource | undefined; + readonly attachments?: readonly PromptFileAttachment[]; } export interface PluginCommandOrigin { diff --git a/packages/agent-core-v2/src/agent/loop/loopService.ts b/packages/agent-core-v2/src/agent/loop/loopService.ts index 1f1668cb74..1f33dd26bd 100644 --- a/packages/agent-core-v2/src/agent/loop/loopService.ts +++ b/packages/agent-core-v2/src/agent/loop/loopService.ts @@ -475,7 +475,7 @@ export class AgentLoopService extends Disposable implements IAgentLoopService { turnId: job.turn.id, origin, prompt: isDisplayablePromptOrigin(origin) ? turnPromptText(job.seed.input, origin) : undefined, - promptAttachments: turnPromptAttachments(job.seed.input), + promptAttachments: turnPromptAttachments(job.seed.input, origin), }), ); void this.runTurn(job.turn, job.ready).then(job.result.resolve, job.result.reject); diff --git a/packages/agent-core-v2/src/agent/loop/turnEvents.ts b/packages/agent-core-v2/src/agent/loop/turnEvents.ts index aa6ab6630f..68e88f9d17 100644 --- a/packages/agent-core-v2/src/agent/loop/turnEvents.ts +++ b/packages/agent-core-v2/src/agent/loop/turnEvents.ts @@ -16,12 +16,24 @@ export type TurnInterruptReason = | 'filtered' | 'blocked'; +export interface TurnPromptAttachmentFile { + readonly kind: 'file'; + readonly name: string; + readonly mediaType: string; + readonly size: number; + readonly path: string; +} + +export type TurnPromptAttachment = + | { readonly kind: 'image' | 'video' | 'audio'; readonly fileId: string } + | TurnPromptAttachmentFile; + export interface TurnStartedPayload { readonly agentId: string; readonly turnId: number; readonly origin: PromptOrigin; readonly prompt?: string; - readonly promptAttachments?: readonly { kind: 'image' | 'video' | 'audio'; fileId: string }[]; + readonly promptAttachments?: readonly TurnPromptAttachment[]; } export class TurnStarted extends AgentEvent2 { @@ -45,8 +57,9 @@ export function turnPromptText( export function turnPromptAttachments( input: readonly ContentPart[], + origin?: PromptOrigin, ): TurnStartedPayload['promptAttachments'] { - const attachments: { kind: 'image' | 'video' | 'audio'; fileId: string }[] = []; + const attachments: TurnPromptAttachment[] = []; const promptMediaFileId = (url: string, id: string | undefined): string | undefined => { const fileId = parseDaemonFileUrl(url)?.fileId; if (id === undefined) return fileId; @@ -64,6 +77,11 @@ export function turnPromptAttachments( if (fileId !== undefined) attachments.push({ kind: 'audio', fileId }); } } + if (origin?.kind === 'user' || origin?.kind === 'skill_activation') { + for (const attachment of origin.attachments ?? []) { + attachments.push({ kind: 'file', ...attachment }); + } + } return attachments.length > 0 ? attachments : undefined; } diff --git a/packages/agent-core-v2/src/agent/prompt/promptService.ts b/packages/agent-core-v2/src/agent/prompt/promptService.ts index 7bfb1308b1..793034a528 100644 --- a/packages/agent-core-v2/src/agent/prompt/promptService.ts +++ b/packages/agent-core-v2/src/agent/prompt/promptService.ts @@ -152,6 +152,9 @@ function mergeSteerMessages(records: readonly Record[]): ContextMessage { const skillActivations = records.flatMap((item) => item.message.origin?.kind === 'user' ? (item.message.origin.skillActivations ?? []) : [], ); + const attachments = records.flatMap((item) => + item.message.origin?.kind === 'user' ? (item.message.origin.attachments ?? []) : [], + ); return { role: 'user', content: [ @@ -159,7 +162,14 @@ function mergeSteerMessages(records: readonly Record[]): ContextMessage { ...records.flatMap((item) => stripBundledSkillBlocks(item.message)), ], toolCalls: [], - origin: skillActivations.length === 0 ? USER_PROMPT_ORIGIN : { kind: 'user', skillActivations }, + origin: + skillActivations.length === 0 && attachments.length === 0 + ? USER_PROMPT_ORIGIN + : { + kind: 'user', + skillActivations: skillActivations.length === 0 ? undefined : skillActivations, + attachments: attachments.length === 0 ? undefined : attachments, + }, }; } diff --git a/packages/agent-core-v2/src/features/skill/skill.ts b/packages/agent-core-v2/src/features/skill/skill.ts index 5bbb71a25b..8c7e12ffc6 100644 --- a/packages/agent-core-v2/src/features/skill/skill.ts +++ b/packages/agent-core-v2/src/features/skill/skill.ts @@ -1,9 +1,11 @@ import type { ContentPart } from '#/kosong/contract/message'; +import type { PromptFileAttachment } from '#/agent/contextMemory/types'; export interface SkillActivationInput { readonly name: string; readonly args?: string; readonly content?: readonly ContentPart[]; + readonly attachments?: readonly PromptFileAttachment[]; } export interface PromptSkillActivation { @@ -14,6 +16,7 @@ export interface PromptSkillActivation { export interface PromptWithSkillsInput { readonly input: readonly ContentPart[]; readonly skills: readonly PromptSkillActivation[]; + readonly attachments?: readonly PromptFileAttachment[]; } export interface PromptWithSkillsResult { diff --git a/packages/agent-core-v2/src/features/skill/skillAgentRuntime.ts b/packages/agent-core-v2/src/features/skill/skillAgentRuntime.ts index 9dfb2b9156..8e9533eef3 100644 --- a/packages/agent-core-v2/src/features/skill/skillAgentRuntime.ts +++ b/packages/agent-core-v2/src/features/skill/skillAgentRuntime.ts @@ -76,6 +76,7 @@ export class SkillRuntime { skillPath: skill.path, skillSource: skill.source, skillArgs: input.args, + attachments: input.attachments, }, content, ); @@ -134,6 +135,7 @@ export class SkillRuntime { origin: { kind: 'user', skillActivations: prepared.map((activation) => activation.entry), + attachments: input.attachments, }, }); if (handle.state === 'pending') { diff --git a/packages/agent-core-v2/src/index.ts b/packages/agent-core-v2/src/index.ts index 48a14a09ac..d5a646d23c 100644 --- a/packages/agent-core-v2/src/index.ts +++ b/packages/agent-core-v2/src/index.ts @@ -586,6 +586,7 @@ export { compressImageForModel, gateImageFormatParts, IMAGE_BYTE_BUDGET, + MAX_IMAGE_DECODE_BYTES, MAX_IMAGE_EDGE_PX, READ_IMAGE_BYTE_BUDGET, resolveMaxImageEdgePx, diff --git a/packages/agent-core-v2/test/agent/loop/loop.test.ts b/packages/agent-core-v2/test/agent/loop/loop.test.ts index ba34b39b3a..81607619e0 100644 --- a/packages/agent-core-v2/test/agent/loop/loop.test.ts +++ b/packages/agent-core-v2/test/agent/loop/loop.test.ts @@ -845,7 +845,7 @@ describe('Agent loop', () => { }); it('carries kimi-file prompt attachments on turn.started, falling back to the URL file id', async () => { - const payloads: Array = []; + const payloads: Array = []; const subscription = ctx.get(IEventBus).subscribe(TurnStarted, (event) => { payloads.push(event.promptAttachments); }); @@ -883,6 +883,97 @@ describe('Agent loop', () => { ], ]); }); + + it('carries origin file attachments on turn.started promptAttachments', async () => { + const payloads: Array = []; + const subscription = ctx.get(IEventBus).subscribe(TurnStarted, (event) => { + payloads.push(event.promptAttachments); + }); + ctx.mockNextResponse({ type: 'text', text: 'seen' }); + + const turn = ( + await loop.enqueue( + new MessageStepRequest( + { + role: 'user', + content: [ + { type: 'image_url', imageUrl: { url: 'kimi-file://file_1', id: 'file_1' } }, + { type: 'text', text: 'summarize' }, + ], + toolCalls: [], + origin: { + kind: 'user', + attachments: [ + { + name: 'report.pdf', + mediaType: 'application/pdf', + size: 42, + path: '/data/report.pdf', + }, + ], + }, + }, + { admission: 'newTurn' }, + ), + ).assigned + ).turn; + await turn.result; + subscription.dispose(); + + expect(payloads).toEqual([ + [ + { kind: 'image', fileId: 'file_1' }, + { + kind: 'file', + name: 'report.pdf', + mediaType: 'application/pdf', + size: 42, + path: '/data/report.pdf', + }, + ], + ]); + }); + + it('carries skill activation file attachments on turn.started promptAttachments', async () => { + const payloads: Array = []; + const subscription = ctx.get(IEventBus).subscribe(TurnStarted, (event) => { + payloads.push(event.promptAttachments); + }); + ctx.mockNextResponse({ type: 'text', text: 'seen' }); + + const turn = ( + await loop.enqueue( + new MessageStepRequest( + { + role: 'user', + content: [{ type: 'text', text: 'User activated the skill "check".' }], + toolCalls: [], + origin: { + kind: 'skill_activation', + activationId: 'act_1', + skillName: 'check', + trigger: 'user-slash', + attachments: [ + { + name: 'note.txt', + mediaType: 'text/plain', + size: 21, + path: '/data/note.txt', + }, + ], + }, + }, + { admission: 'newTurn' }, + ), + ).assigned + ).turn; + await turn.result; + subscription.dispose(); + + expect(payloads).toEqual([ + [{ kind: 'file', name: 'note.txt', mediaType: 'text/plain', size: 21, path: '/data/note.txt' }], + ]); + }); }); describe('turn telemetry', () => { diff --git a/packages/agent-core-v2/test/agent/prompt/promptService.test.ts b/packages/agent-core-v2/test/agent/prompt/promptService.test.ts index ea7ea539ca..6ebf93f5a4 100644 --- a/packages/agent-core-v2/test/agent/prompt/promptService.test.ts +++ b/packages/agent-core-v2/test/agent/prompt/promptService.test.ts @@ -466,6 +466,46 @@ describe('AgentPromptService', () => { ]); }); + it('concatenates origin file attachments when steering queued prompts', async () => { + const { prompt, context, loop } = harness(); + const active = await prompt.enqueue({ message: message('active') }); + await active.launched; + const one = await prompt.enqueue({ + message: { + role: 'user', + content: [{ type: 'text', text: 'one' }], + toolCalls: [], + origin: { + kind: 'user', + attachments: [{ name: 'a.txt', mediaType: 'text/plain', size: 1, path: '/data/a.txt' }], + }, + }, + }); + const two = await prompt.enqueue({ + message: { + role: 'user', + content: [{ type: 'text', text: 'two' }], + toolCalls: [], + origin: { + kind: 'user', + attachments: [{ name: 'b.txt', mediaType: 'text/plain', size: 2, path: '/data/b.txt' }], + }, + }, + }); + + await prompt.steer([one.id, two.id]); + loop.drainNextBatch(context); + + const merged = context + .get() + .find((entry) => entry.origin?.kind === 'user' && entry.origin.attachments !== undefined); + expect(merged?.origin?.kind === 'user' && merged.origin.attachments).toEqual([ + { name: 'a.txt', mediaType: 'text/plain', size: 1, path: '/data/a.txt' }, + { name: 'b.txt', mediaType: 'text/plain', size: 2, path: '/data/b.txt' }, + ]); + expect(merged?.origin?.kind === 'user' && merged.origin.skillActivations).toBeUndefined(); + }); + it('restarts the queue after restoring a steer raced by the active turn settling', async () => { const { prompt, loop } = harness({ manualTurnResult: true }); const active = await prompt.enqueue({ message: message('active') }); diff --git a/packages/kap-server/src/lib/promptMedia.ts b/packages/kap-server/src/lib/promptMedia.ts index 03e5799e00..6b5484ffc5 100644 --- a/packages/kap-server/src/lib/promptMedia.ts +++ b/packages/kap-server/src/lib/promptMedia.ts @@ -1,7 +1,7 @@ import { createHash } from 'node:crypto'; -import { createWriteStream } from 'node:fs'; -import { mkdir, stat, writeFile } from 'node:fs/promises'; -import { join } from 'node:path'; +import { createReadStream, createWriteStream, type Stats } from 'node:fs'; +import { mkdir, readFile, realpath, stat, writeFile } from 'node:fs/promises'; +import { basename, extname, isAbsolute, join } from 'node:path'; import { Readable } from 'node:stream'; import { pipeline } from 'node:stream/promises'; @@ -15,6 +15,7 @@ import { Error2, fileNotFoundError, isModelAcceptedImageMime, + MAX_IMAGE_DECODE_BYTES, normalizeImageMime, persistOriginalImage, resolveEffectiveImageMime, @@ -25,7 +26,14 @@ import { type ISessionMediaStore, type ImageCompressionTelemetry, type ITelemetryService, + type PromptFileAttachment, } from '@moonshot-ai/agent-core-v2'; +import { sniffMediaFromMagic } from '@moonshot-ai/agent-core-v2/agent/media/file-type'; +import { + IMAGE_MIME_BY_SUFFIX, + VIDEO_MIME_BY_SUFFIX, +} from '@moonshot-ai/agent-core-v2/agent/media/mediaRef'; +import { isSensitiveFile } from '@moonshot-ai/agent-core-v2/tool/path-access'; import type { PromptSubmission } from '../protocol/rest-prompt'; @@ -34,7 +42,7 @@ type WireContent = PromptSubmission['content']; export async function assertPromptFileRefs(content: WireContent, store: IFileService): Promise { for (const part of content) { if (part.type === 'file') { - await store.get(part.file_id); + if (part.file_id !== undefined) await store.get(part.file_id); } else if ((part.type === 'image' || part.type === 'video') && part.source.kind === 'file') { const file = await store.get(part.source.file_id); assertMediaFile(file, part.type); @@ -42,6 +50,44 @@ export async function assertPromptFileRefs(content: WireContent, store: IFileSer } } +export async function assertPromptPathRefs(content: WireContent): Promise { + for (const part of content) { + const path = promptPartPath(part); + if (path === undefined) continue; + if (!isAbsolute(path)) { + throw new Error2('validation.failed', `attachment path must be absolute: ${path}`); + } + const { resolvedPath } = await statAttachmentFile(path); + if (isSensitiveFile(resolvedPath)) { + throw new Error2('validation.failed', `attachment path is a sensitive file: ${path}`); + } + } +} + +export function contentHasPathRefs(content: WireContent): boolean { + return content.some((part) => promptPartPath(part) !== undefined); +} + +function promptPartPath(part: WireContent[number]): string | undefined { + if (part.type === 'file') return part.path; + if ((part.type === 'image' || part.type === 'video') && part.source.kind === 'path') { + return part.source.path; + } + return undefined; +} + +async function statAttachmentFile(sourcePath: string): Promise<{ resolvedPath: string; info: Stats }> { + const resolvedPath = await realpath(sourcePath).catch(() => undefined); + if (resolvedPath === undefined) throw fileNotFoundError(sourcePath); + const info = await stat(resolvedPath).catch(() => undefined); + if (info === undefined || !info.isFile()) throw fileNotFoundError(sourcePath); + return { resolvedPath, info }; +} + +function isFsError(error: unknown): boolean { + return error instanceof Error && typeof (error as NodeJS.ErrnoException).code === 'string'; +} + export async function assertPromptSessionMediaRefs( content: WireContent, store: ISessionMediaStore, @@ -78,6 +124,7 @@ export interface ResolvePromptMediaOptions { export interface PromptMediaPreparation { readonly content: WireContent; + readonly attachments: readonly PromptFileAttachment[]; readonly discard: () => Promise; } @@ -117,6 +164,7 @@ export async function resolvePromptMediaFiles( }; const telemetryFor = (source: string): ImageCompressionTelemetry | undefined => options.telemetry === undefined ? undefined : { client: options.telemetry, source }; + const attachments: PromptFileAttachment[] = []; const content: WireContent = []; try { for (const part of input) { @@ -139,6 +187,9 @@ export async function resolvePromptMediaFiles( ? buildUnsupportedImageNotice(effectiveMime) : buildAttachedFileNotice(name, effectiveMime, bytes.length, persisted), }); + if (persisted !== null) { + attachments.push({ name, mediaType: effectiveMime, size: bytes.length, path: persisted }); + } changed = true; continue; } @@ -194,12 +245,122 @@ export async function resolvePromptMediaFiles( } if (part.type === 'file') { + if (part.path !== undefined) { + const sourcePath = part.path; + const { info } = await statAttachmentFile(sourcePath); + const name = part.name ?? basename(sourcePath); + const mediaType = part.media_type ?? 'application/octet-stream'; + content.push({ + type: 'text', + text: buildAttachedFileNotice(name, mediaType, info.size, sourcePath), + }); + attachments.push({ name, mediaType, size: info.size, path: sourcePath }); + changed = true; + continue; + } + if (part.file_id === undefined) { + throw new Error2('validation.failed', 'file part requires file_id or path'); + } const file = await store.get(part.file_id); const attachedPath = await materializeAttachmentToDir(file, await resolveAttachmentsDir()); content.push({ type: 'text', text: buildAttachedFileNotice(file.meta.name, file.meta.media_type, file.meta.size, attachedPath), }); + attachments.push({ + name: file.meta.name, + mediaType: file.meta.media_type, + size: file.meta.size, + path: attachedPath, + }); + changed = true; + continue; + } + + if (part.type === 'image' && part.source.kind === 'path') { + const sourcePath = part.source.path; + const { resolvedPath, info } = await statAttachmentFile(sourcePath); + if (info.size > MAX_IMAGE_DECODE_BYTES) { + throw new Error2( + 'validation.failed', + `${sourcePath} is ${info.size} bytes, over the ${MAX_IMAGE_DECODE_BYTES}-byte image decode limit — attach it as a file instead`, + ); + } + const data = await readFile(resolvedPath).catch((error: unknown) => { + if (isFsError(error)) throw fileNotFoundError(sourcePath); + throw error; + }); + const name = basename(sourcePath); + const declared = pathMediaMime(sourcePath, data, 'image'); + if (!declared.startsWith('image/')) { + throw new Error2('validation.failed', `${sourcePath} is ${declared}, not an image`); + } + let mediaType = resolveEffectiveImageMime(declared, data); + if (!isModelAcceptedImageMime(mediaType)) { + content.push({ + type: 'text', + text: buildAttachedFileNotice(name, mediaType, data.length, sourcePath), + }); + attachments.push({ name, mediaType, size: data.length, path: sourcePath }); + changed = true; + continue; + } + mediaType = normalizeImageMime(mediaType); + const compressed = await compressImageForModel(data, mediaType, { + telemetry: telemetryFor('prompt_file'), + }); + if (compressed.changed) { + content.push({ + type: 'text', + text: buildImageCompressionCaption({ + original: { + width: compressed.originalWidth, + height: compressed.originalHeight, + byteLength: compressed.originalByteLength, + mimeType: mediaType, + }, + final: { + width: compressed.width, + height: compressed.height, + byteLength: compressed.finalByteLength, + mimeType: compressed.mimeType, + }, + originalPath: sourcePath, + }), + }); + } + const saved = await store.save( + Readable.from(compressed.changed ? Buffer.from(compressed.data) : data), + compressed.changed ? compressedUploadName(name, compressed.mimeType) : name, + { mimeType: compressed.changed ? compressed.mimeType : mediaType }, + ); + ownedFileIds.add(saved.id); + content.push({ + type: 'image', + source: { kind: 'url', url: buildDaemonFileUrl(saved.id) }, + }); + changed = true; + continue; + } + + if (part.type === 'video' && part.source.kind === 'path') { + const sourcePath = part.source.path; + const { resolvedPath } = await statAttachmentFile(sourcePath); + const mediaType = pathMediaMime(sourcePath, undefined, 'video'); + if (!mediaType.startsWith('video/')) { + throw new Error2('validation.failed', `${sourcePath} is ${mediaType}, not a video`); + } + const saved = await store + .save(createReadStream(resolvedPath), basename(sourcePath), { mimeType: mediaType }) + .catch((error: unknown) => { + if (isFsError(error)) throw fileNotFoundError(sourcePath); + throw error; + }); + ownedFileIds.add(saved.id); + content.push({ + type: 'video', + source: { kind: 'url', url: buildDaemonFileUrl(saved.id) }, + }); changed = true; continue; } @@ -227,6 +388,14 @@ export async function resolvePromptMediaFiles( ? buildUnsupportedImageNotice(mediaType, file.meta.name) : buildAttachedFileNotice(file.meta.name, mediaType, file.meta.size, persisted), }); + if (persisted !== null) { + attachments.push({ + name: file.meta.name, + mediaType, + size: file.meta.size, + path: persisted, + }); + } changed = true; continue; } @@ -280,7 +449,7 @@ export async function resolvePromptMediaFiles( }); changed = true; } - return { content: changed ? content : input, discard }; + return { content: changed ? content : input, attachments, discard }; } catch (error) { await discard(); throw error; @@ -336,6 +505,24 @@ function imageExtensionForMime(mediaType: string): string { return ext.length > 0 ? ext : 'img'; } +function pathMediaMime( + sourcePath: string, + data: Uint8Array | undefined, + kind: 'image' | 'video', +): string { + const suffix = extname(sourcePath).toLowerCase(); + if (kind === 'image') { + if (suffix === '.svg') return 'image/svg+xml'; + const declared = IMAGE_MIME_BY_SUFFIX[suffix]; + if (declared !== undefined) return declared; + } else { + const declared = VIDEO_MIME_BY_SUFFIX[suffix]; + if (declared !== undefined) return declared; + } + const sniffed = data === undefined ? null : sniffMediaFromMagic(data); + return sniffed?.mimeType ?? 'application/octet-stream'; +} + function buildAttachedFileNotice(name: string, mediaType: string, size: number, path: string): string { return `Attached file "${name}" (${mediaType}, ${size} bytes): ${path} — open it with the Read tool`; } diff --git a/packages/kap-server/src/protocol/events-zod.ts b/packages/kap-server/src/protocol/events-zod.ts index 98b1206579..334ed2fa53 100644 --- a/packages/kap-server/src/protocol/events-zod.ts +++ b/packages/kap-server/src/protocol/events-zod.ts @@ -711,7 +711,18 @@ export const turnStartedEventSchema = z.object({ prompt: z.string().optional(), promptId: z.string().optional(), promptAttachments: z - .array(z.object({ kind: z.enum(['image', 'video', 'audio']), fileId: z.string() })) + .array( + z.union([ + z.object({ kind: z.enum(['image', 'video', 'audio']), fileId: z.string() }), + z.object({ + kind: z.literal('file'), + name: z.string(), + mediaType: z.string(), + size: z.number(), + path: z.string(), + }), + ]), + ) .optional(), }); diff --git a/packages/kap-server/src/protocol/message.ts b/packages/kap-server/src/protocol/message.ts index eb40a2644f..3f048b8e84 100644 --- a/packages/kap-server/src/protocol/message.ts +++ b/packages/kap-server/src/protocol/message.ts @@ -40,6 +40,7 @@ export const imageSourceSchema = z.discriminatedUnion('kind', [ }), z.object({ kind: z.literal('file'), file_id: z.string().min(1) }), z.object({ kind: z.literal('session_media'), file_id: z.string().min(1) }), + z.object({ kind: z.literal('path'), path: z.string().min(1) }), ]); export type ImageSource = z.infer; @@ -55,13 +56,33 @@ export const videoContentSchema = z.object({ }); export type VideoContent = z.infer; -export const fileContentSchema = z.object({ - type: z.literal('file'), - file_id: z.string().min(1), - name: z.string(), - media_type: z.string().min(1), - size: z.number().int().nonnegative(), -}); +export const fileContentSchema = z + .object({ + type: z.literal('file'), + file_id: z.string().min(1).optional(), + path: z.string().min(1).optional(), + name: z.string().optional(), + media_type: z.string().min(1).optional(), + size: z.number().int().nonnegative().optional(), + }) + .superRefine((part, ctx) => { + const hasFileId = part.file_id !== undefined; + const hasPath = part.path !== undefined; + if (hasFileId === hasPath) { + ctx.addIssue({ + code: 'custom', + message: 'exactly one of file_id or path is required', + path: hasFileId ? ['path'] : ['file_id'], + }); + return; + } + if (hasPath) return; + for (const key of ['name', 'media_type', 'size'] as const) { + if (part[key] === undefined) { + ctx.addIssue({ code: 'custom', message: `${key} is required with file_id`, path: [key] }); + } + } + }); export type FileContent = z.infer; export const thinkingContentSchema = z.object({ diff --git a/packages/kap-server/src/routes/prompts.ts b/packages/kap-server/src/routes/prompts.ts index 3d558cca73..476d59726b 100644 --- a/packages/kap-server/src/routes/prompts.ts +++ b/packages/kap-server/src/routes/prompts.ts @@ -5,6 +5,7 @@ import { IAgentLifecycleService, IAgentPermissionModeService, IAgentProfileService, + IAgentRuntimeBindingService, IAgentToolPolicyService, IAgentPromptService, agentContextOf, @@ -51,7 +52,9 @@ import { z } from 'zod'; import { errEnvelope, okEnvelope } from '../envelope'; import { assertPromptFileRefs, + assertPromptPathRefs, assertPromptSessionMediaRefs, + contentHasPathRefs, contentToCoreParts, resolvePromptMediaFiles, type PromptMediaPreparation, @@ -116,6 +119,7 @@ async function resolvePromptFromSession(session: ISessionScopeHandle, agentId?: profile: agent.accessor.get(IAgentProfileService), toolPolicy: agent.accessor.get(IAgentToolPolicyService), permissionMode: agent.accessor.get(IAgentPermissionModeService), + binding: agent.accessor.get(IAgentRuntimeBindingService), }; } @@ -201,6 +205,7 @@ export function registerPromptsRoutes(app: PromptRouteHost, core: Scope): void { [ErrorCode.AUTH_TOKEN_UNAUTHORIZED]: { detailsSchema: authProviderDetailsSchema }, [ErrorCode.AUTH_MODEL_NOT_RESOLVED]: { detailsSchema: authModelDetailsSchema }, [ErrorCode.SESSION_NOT_FOUND]: {}, + [ErrorCode.FILE_NOT_FOUND]: {}, [ErrorCode.PROMPT_ID_CONFLICT]: {}, [ErrorCode.PROMPT_ALREADY_COMPLETED]: { dataSchema: z.object({ aborted: z.literal(false) }) }, }, @@ -214,8 +219,19 @@ export function registerPromptsRoutes(app: PromptRouteHost, core: Scope): void { let reservation: PromptReservation | undefined; let enqueued = false; try { - await assertPromptFileRefs(req.body.content, core.accessor.get(IFileService)); const session = await resolveSession(core, session_id); + let resolved: Awaited> | undefined; + if (contentHasPathRefs(req.body.content)) { + resolved = await resolvePromptFromSession(session, req.body.agent_id); + if (resolved.binding.get().runtimeId !== 'local') { + throw new Error2( + ErrorCodes.REQUEST_INVALID, + 'file attachments by server-local path require the local runtime', + ); + } + } + await assertPromptFileRefs(req.body.content, core.accessor.get(IFileService)); + await assertPromptPathRefs(req.body.content); if (req.body.skills !== undefined) { if (req.body.prompt_id !== undefined) { throw new Error2( @@ -232,7 +248,7 @@ export function registerPromptsRoutes(app: PromptRouteHost, core: Scope): void { req.body.content, session.accessor.get(ISessionMediaStore), ); - const resolved = await resolvePromptFromSession(session, req.body.agent_id); + resolved ??= await resolvePromptFromSession(session, req.body.agent_id); reservation = reservePrompt(resolved.prompt, req.body.prompt_id); await resolved.auth.ensureReady(); @@ -256,6 +272,8 @@ export function registerPromptsRoutes(app: PromptRouteHost, core: Scope): void { }, ); const resolvedContent = preparedMedia.content; + const promptAttachments = + preparedMedia.attachments.length > 0 ? preparedMedia.attachments : undefined; let thinkingConsumed = false; if (req.body.profile !== undefined) { @@ -296,6 +314,7 @@ export function registerPromptsRoutes(app: PromptRouteHost, core: Scope): void { result = await resolved.skill.promptWithSkills({ input: parts, skills: req.body.skills, + attachments: promptAttachments, }); } catch (error) { settlement.dispose(); @@ -326,7 +345,7 @@ export function registerPromptsRoutes(app: PromptRouteHost, core: Scope): void { role: 'user', content: parts, toolCalls: [], - origin: { kind: 'user' }, + origin: { kind: 'user', attachments: promptAttachments }, }); enqueued = true; const staging = preparedMedia; diff --git a/packages/kap-server/src/routes/skills.ts b/packages/kap-server/src/routes/skills.ts index 513b03d539..ce9c8dd81d 100644 --- a/packages/kap-server/src/routes/skills.ts +++ b/packages/kap-server/src/routes/skills.ts @@ -6,6 +6,7 @@ import { ErrorCodes, EXTRA_SKILL_DIRS_SECTION, IAgentLifecycleService, + IAgentRuntimeBindingService, IBootstrapService, IConfigService, IFileService, @@ -42,7 +43,9 @@ import { z } from 'zod'; import { errEnvelope, okEnvelope } from '../envelope'; import { assertPromptFileRefs, + assertPromptPathRefs, assertPromptSessionMediaRefs, + contentHasPathRefs, contentToCoreParts, resolvePromptMediaFiles, type PromptMediaPreparation, @@ -50,6 +53,7 @@ import { import { requestLog } from '../lib/requestLog'; import { defineRoute } from '../middleware/defineRoute'; import { ErrorCode } from '../protocol/error-codes'; +import { ensureMainAgent as ensureMainAgentHandle } from '../transport/mainAgent'; import { activateSkillRequestSchema, activateSkillResultSchema, @@ -223,6 +227,15 @@ export function registerSkillsRoutes(app: SkillsRouteHost, core: Scope): void { const attachments = req.body.attachments ?? []; const attachmentParts: ContentPart[] = []; if (attachments.length > 0) { + if (contentHasPathRefs(attachments)) { + const mainAgent = await ensureMainAgentHandle(resolved.handle); + if (mainAgent.accessor.get(IAgentRuntimeBindingService).get().runtimeId !== 'local') { + throw new Error2( + ErrorCodes.REQUEST_INVALID, + 'file attachments by server-local path require the local runtime', + ); + } + } const catalog = resolved.handle.accessor.get(ISessionSkillCatalog); await catalog.ready; const skill = catalog.catalog.getSkill(parsed.id); @@ -236,6 +249,7 @@ export function registerSkillsRoutes(app: SkillsRouteHost, core: Scope): void { ); } await assertPromptFileRefs(attachments, core.accessor.get(IFileService)); + await assertPromptPathRefs(attachments); await assertPromptSessionMediaRefs( attachments, resolved.handle.accessor.get(ISessionMediaStore), @@ -255,10 +269,19 @@ export function registerSkillsRoutes(app: SkillsRouteHost, core: Scope): void { attachmentParts.push(...contentToCoreParts(preparedMedia.content)); } const context = await ensureMainAgent(resolved.handle); + const promptAttachments = + preparedMedia !== undefined && preparedMedia.attachments.length > 0 + ? preparedMedia.attachments + : undefined; await resolved.handle.accessor .get(IAgentLifecycleService) .resolve(context, AgentSkill) - .activate({ name: parsed.id, args: req.body.args, content: attachmentParts }); + .activate({ + name: parsed.id, + args: req.body.args, + content: attachmentParts, + attachments: promptAttachments, + }); await preparedMedia?.discard(); preparedMedia = undefined; requestLog(req)?.info({ session_id, skill_name: parsed.id }, 'skill activated'); @@ -365,6 +388,7 @@ function sendMappedError( case ErrorCodes.FILE_NOT_FOUND: reply.send(errEnvelope(ErrorCode.FILE_NOT_FOUND, err.message, requestId, err.stack)); return; + case ErrorCodes.REQUEST_INVALID: case ErrorCodes.VALIDATION_FAILED: reply.send(errEnvelope(ErrorCode.VALIDATION_FAILED, err.message, requestId, err.stack)); return; diff --git a/packages/kap-server/src/services/transcript/coreEventMap.ts b/packages/kap-server/src/services/transcript/coreEventMap.ts index 607785a29e..a46e602d17 100644 --- a/packages/kap-server/src/services/transcript/coreEventMap.ts +++ b/packages/kap-server/src/services/transcript/coreEventMap.ts @@ -339,18 +339,29 @@ export class AgentTranscriptProjector { turnId: number; origin: unknown; prompt?: string; - promptAttachments?: readonly { kind: 'image' | 'video' | 'audio'; fileId: string }[]; + promptAttachments?: readonly ( + | { kind: 'image' | 'video' | 'audio'; fileId: string } + | { kind: 'file'; name: string; mediaType: string; size: number; path: string } + )[]; }): TranscriptOperation[] { const n = event.turnId; const turnId = `t${n}`; const ops: TranscriptOperation[] = []; const attachmentIds: string[] = []; for (const input of event.promptAttachments ?? []) { - const attachment: TranscriptAttachment = { - attachmentId: `${turnId}.att${attachmentIds.length + 1}`, - mediaType: `${input.kind}/*`, - source: { kind: 'session_media', fileId: input.fileId }, - }; + const attachment: TranscriptAttachment = + input.kind === 'file' + ? { + attachmentId: `${turnId}.att${attachmentIds.length + 1}`, + mediaType: input.mediaType, + name: input.name, + size: input.size, + } + : { + attachmentId: `${turnId}.att${attachmentIds.length + 1}`, + mediaType: `${input.kind}/*`, + source: { kind: 'session_media', fileId: input.fileId }, + }; ops.push({ op: 'attachment.upsert', attachment }); attachmentIds.push(attachment.attachmentId); } diff --git a/packages/kap-server/test/prompts.test.ts b/packages/kap-server/test/prompts.test.ts index dac151b8f2..3c1a2b9794 100644 --- a/packages/kap-server/test/prompts.test.ts +++ b/packages/kap-server/test/prompts.test.ts @@ -1,20 +1,23 @@ -import { chmod, mkdir, mkdtemp, readFile, realpath, rm, writeFile } from 'node:fs/promises'; +import { chmod, mkdir, mkdtemp, open, readFile, readdir, realpath, rm, symlink, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { deflateSync } from 'node:zlib'; import { agentContextOf, + agentRuntimeBindingKey, IAgentTitlePromptSource, IAgentContextMemoryService, IAgentLifecycleService, IAgentPermissionModeService, IAgentProfileService, + IAgentStateService, IAgentToolPolicyService, IBootstrapService, IFileService, ISessionContext, ISessionMetadata, + MAX_IMAGE_DECODE_BYTES, closeSessionById, getLiveSessionById, } from '@moonshot-ai/agent-core-v2'; @@ -1022,6 +1025,283 @@ describe('server-v2 /api/v1 prompts', () => { expect(await readFile(attachedPath)).toEqual(scriptBytes); }); + it('attaches a server-local file by path without copying it', async () => { + const id = await createSession(home as string); + await createMainAgent(id); + const outside = await mkdtemp(join(tmpdir(), 'kimi-attach-path-')); + try { + const sourcePath = join(outside, 'notes.txt'); + const bytes = Buffer.from('path attachment body'); + await writeFile(sourcePath, bytes); + + const submitted = await call('POST', `/api/v1/sessions/${id}/prompts`, { + content: [ + { type: 'text', text: 'read this' }, + { type: 'file', path: sourcePath }, + ], + }); + expect(submitted.body.code).toBe(0); + + const content = submitted.body.data.content as Array<{ type: string; text?: string }>; + expect(content).toHaveLength(2); + expect(content[0]).toEqual({ type: 'text', text: 'read this' }); + expect(content[1]).toEqual({ + type: 'text', + text: `Attached file "notes.txt" (application/octet-stream, ${bytes.length} bytes): ${sourcePath} — open it with the Read tool`, + }); + + const session = getLiveSessionById(server!.core.accessor, id); + const attachmentsDir = join(session!.accessor.get(ISessionContext).sessionDir, 'attachments'); + await expect(readdir(attachmentsDir)).rejects.toMatchObject({ code: 'ENOENT' }); + + const main = session!.accessor.get(IAgentLifecycleService).handleOf('main')!; + await vi.waitFor(() => { + const memory = main.accessor.get(IAgentContextMemoryService).get(); + const promptMessage = memory.find((entry) => entry.origin?.kind === 'user'); + expect(promptMessage?.origin).toEqual({ + kind: 'user', + attachments: [ + { + name: 'notes.txt', + mediaType: 'application/octet-stream', + size: bytes.length, + path: sourcePath, + }, + ], + }); + }); + } finally { + await rm(outside, { recursive: true, force: true }); + } + }); + + it('rejects a relative attachment path', async () => { + const id = await createSession(home as string); + + const { body } = await call('POST', `/api/v1/sessions/${id}/prompts`, { + content: [{ type: 'file', path: 'relative/notes.txt' }], + }); + expect(body.code).toBe(40001); + }); + + it('rejects a sensitive attachment path', async () => { + const id = await createSession(home as string); + const secretPath = join(home as string, '.env'); + await writeFile(secretPath, 'TOKEN=secret'); + + const { body } = await call('POST', `/api/v1/sessions/${id}/prompts`, { + content: [{ type: 'file', path: secretPath }], + }); + expect(body.code).toBe(40001); + }); + + it('rejects a missing attachment path with 40407', async () => { + const id = await createSession(home as string); + + const { body } = await call('POST', `/api/v1/sessions/${id}/prompts`, { + content: [{ type: 'file', path: join(home as string, 'nope.txt') }], + }); + expect(body.code).toBe(40407); + }); + + it('rejects a symlink pointing at a sensitive file', async () => { + const id = await createSession(home as string); + await createMainAgent(id); + const secretPath = join(home as string, '.env'); + await writeFile(secretPath, 'TOKEN=secret'); + const linkPath = join(home as string, 'innocent.txt'); + await symlink(secretPath, linkPath); + + const { body } = await call('POST', `/api/v1/sessions/${id}/prompts`, { + content: [{ type: 'file', path: linkPath }], + }); + expect(body.code).toBe(40001); + }); + + it('rejects path attachments on a non-local runtime before touching the filesystem', async () => { + const id = await createSession(home as string); + await createMainAgent(id); + const session = getLiveSessionById(server!.core.accessor, id); + const main = session!.accessor.get(IAgentLifecycleService).handleOf('main')!; + main.accessor.get(IAgentStateService).set(agentRuntimeBindingKey, { + workspaceId: session!.accessor.get(ISessionContext).workspaceId, + runtimeId: 'fake-remote', + }); + + const sourcePath = join(home as string, 'note.txt'); + await writeFile(sourcePath, 'x'); + const existing = await call('POST', `/api/v1/sessions/${id}/prompts`, { + content: [{ type: 'file', path: sourcePath }], + }); + expect(existing.body.code).toBe(40001); + expect(existing.body.msg).toContain('local runtime'); + + const missing = await call('POST', `/api/v1/sessions/${id}/prompts`, { + content: [{ type: 'file', path: join(home as string, 'nope.txt') }], + }); + expect(missing.body.code).toBe(40001); + + const uploadBytes = Buffer.from('upload unaffected'); + const uploaded = await uploadFile(uploadBytes, 'text/plain', 'up.txt'); + const upload = await call('POST', `/api/v1/sessions/${id}/prompts`, { + content: [ + { + type: 'file', + file_id: uploaded.id, + name: 'up.txt', + media_type: 'text/plain', + size: uploadBytes.length, + }, + ], + }); + expect(upload.body.code).toBe(0); + }); + + it('rejects an upload file part missing metadata', async () => { + const id = await createSession(home as string); + const uploaded = await uploadFile(Buffer.from('x'), 'text/plain', 'x.txt'); + + const { body } = await call('POST', `/api/v1/sessions/${id}/prompts`, { + content: [{ type: 'file', file_id: uploaded.id }], + }); + expect(body.code).toBe(40001); + }); + + it('rejects a file part carrying both file_id and path', async () => { + const id = await createSession(home as string); + const sourcePath = join(home as string, 'both.txt'); + await writeFile(sourcePath, 'x'); + const uploaded = await uploadFile(Buffer.from('x'), 'text/plain', 'x.txt'); + + const { body } = await call('POST', `/api/v1/sessions/${id}/prompts`, { + content: [ + { + type: 'file', + file_id: uploaded.id, + path: sourcePath, + name: 'both.txt', + media_type: 'text/plain', + size: 1, + }, + ], + }); + expect(body.code).toBe(40001); + }); + + it('carries a server-local image by path as an internal kimi-file reference', async () => { + const id = await createSession(home as string); + await createMainAgent(id); + const outside = await mkdtemp(join(tmpdir(), 'kimi-attach-img-')); + try { + const smallPng = solidPng(10, 10); + const sourcePath = join(outside, 'small.png'); + await writeFile(sourcePath, smallPng); + + const submitted = await call('POST', `/api/v1/sessions/${id}/prompts`, { + content: [{ type: 'image', source: { kind: 'path', path: sourcePath } }], + }); + expect(submitted.body.code).toBe(0); + + const content = submitted.body.data.content as Array>; + expect(content).toHaveLength(1); + const image = content[0] as { type: string; source: { kind: string; file_id: string } }; + expect(image.type).toBe('image'); + expect(image.source.kind).toBe('session_media'); + await expectSessionMedia(server!, id, `${image.source.file_id}.png`, smallPng); + expect(JSON.stringify(content)).not.toContain('kimi-file://'); + expect(JSON.stringify(content)).not.toContain(sourcePath); + } finally { + await rm(outside, { recursive: true, force: true }); + } + }); + + it('compresses a server-local image by path and captions the original path', async () => { + const id = await createSession(home as string); + await createMainAgent(id); + const outside = await mkdtemp(join(tmpdir(), 'kimi-attach-big-')); + try { + const bigPng = solidPng(3600, 1800); + const sourcePath = join(outside, 'big.png'); + await writeFile(sourcePath, bigPng); + + const submitted = await call('POST', `/api/v1/sessions/${id}/prompts`, { + content: [{ type: 'image', source: { kind: 'path', path: sourcePath } }], + }); + expect(submitted.body.code).toBe(0); + + const content = submitted.body.data.content as Array>; + expect(content).toHaveLength(2); + const caption = content[0] as { type: string; text: string }; + expect(caption.type).toBe('text'); + expect(caption.text).toContain('Image compressed'); + expect(caption.text).toContain(`saved at "${sourcePath}"`); + expect(await readFile(sourcePath)).toEqual(bigPng); + + const image = content[1] as { type: string; source: { kind: string; file_id: string } }; + expect(image.type).toBe('image'); + expect(image.source.kind).toBe('session_media'); + const mediaPath = join(sessionMediaDir(server!, id), `${image.source.file_id}.png`); + expect(pngDimensions(await readFileEventually(mediaPath))).toEqual({ width: 2000, height: 1000 }); + + const session = getLiveSessionById(server!.core.accessor, id); + const originalsDir = join(session!.accessor.get(ISessionContext).sessionDir, 'media-originals'); + await expect(readdir(originalsDir)).rejects.toMatchObject({ code: 'ENOENT' }); + } finally { + await rm(outside, { recursive: true, force: true }); + } + }); + + it('carries a server-local video by path as an internal kimi-file reference', async () => { + const id = await createSession(home as string); + await createMainAgent(id); + const outside = await mkdtemp(join(tmpdir(), 'kimi-attach-vid-')); + try { + const videoBytes = Buffer.from('tiny fake mp4 bytes'); + const sourcePath = join(outside, 'clip.mp4'); + await writeFile(sourcePath, videoBytes); + + const submitted = await call('POST', `/api/v1/sessions/${id}/prompts`, { + content: [{ type: 'video', source: { kind: 'path', path: sourcePath } }], + }); + expect(submitted.body.code).toBe(0); + + const content = submitted.body.data.content as Array>; + expect(content).toHaveLength(1); + const video = content[0] as { type: string; source: { kind: string; file_id: string } }; + expect(video.type).toBe('video'); + expect(video.source.kind).toBe('session_media'); + await expectSessionMedia(server!, id, `${video.source.file_id}.mp4`, videoBytes); + } finally { + await rm(outside, { recursive: true, force: true }); + } + }); + + it('rejects a mis-kinded server-local media path', async () => { + const id = await createSession(home as string); + await createMainAgent(id); + const sourcePath = join(home as string, 'notes.txt'); + await writeFile(sourcePath, 'plain text'); + + const { body } = await call('POST', `/api/v1/sessions/${id}/prompts`, { + content: [{ type: 'video', source: { kind: 'path', path: sourcePath } }], + }); + expect(body.code).toBe(40001); + }); + + it('rejects an over-limit server-local image with 40001', async () => { + const id = await createSession(home as string); + await createMainAgent(id); + const sourcePath = join(home as string, 'huge.png'); + const handle = await open(sourcePath, 'w'); + await handle.truncate(MAX_IMAGE_DECODE_BYTES + 1); + await handle.close(); + + const { body } = await call('POST', `/api/v1/sessions/${id}/prompts`, { + content: [{ type: 'image', source: { kind: 'path', path: sourcePath } }], + }); + expect(body.code).toBe(40001); + }); + it('returns 40402 when aborting a prompt that already settled', async () => { const id = await createSession(home as string); await createMainAgent(id); diff --git a/packages/kap-server/test/services/transcript.test.ts b/packages/kap-server/test/services/transcript.test.ts index 932377b071..41ad0d1c10 100644 --- a/packages/kap-server/test/services/transcript.test.ts +++ b/packages/kap-server/test/services/transcript.test.ts @@ -217,6 +217,58 @@ describe('AgentTranscriptProjector', () => { }); }); + it('projects turn.started file promptAttachments into path-sourced attachment entities', () => { + const projector = new AgentTranscriptProjector('main', TEST_SESSION_ID); + const tx = new AgentTranscript('main'); + const ops: TranscriptOperation[] = []; + const feed = (event: ProjectorBusEvent): void => { + const mapped = projector.map(event); + ops.push(...mapped); + tx.apply(mapped); + }; + + feed( + ev({ + type: 'turn.started', + turnId: 0, + origin: { kind: 'user' }, + prompt: 'summarize this', + promptAttachments: [ + { + kind: 'file', + name: 'report.pdf', + mediaType: 'application/pdf', + size: 1234, + path: '/data/report.pdf', + }, + ], + }), + ); + feed(ev({ type: 'turn.ended', turnId: 0, reason: 'completed' })); + + expect(ops.filter((op) => op.op === 'attachment.upsert')).toEqual([ + { + op: 'attachment.upsert', + attachment: { + attachmentId: 't0.att1', + mediaType: 'application/pdf', + name: 'report.pdf', + size: 1234, + }, + }, + ]); + + const turn = turnOps('t0', tx.getItems()); + expect(turn.prompt).toBe('summarize this'); + expect(turn.attachmentIds).toEqual(['t0.att1']); + expect(tx.getAttachment('t0.att1')).toEqual({ + attachmentId: 't0.att1', + mediaType: 'application/pdf', + name: 'report.pdf', + size: 1234, + }); + }); + it('places late-attach deltas into the engine-reported active step', () => { const tx = new AgentTranscript('main'); const projector = new AgentTranscriptProjector('main', TEST_SESSION_ID, { diff --git a/packages/kap-server/test/skills.test.ts b/packages/kap-server/test/skills.test.ts index 03971e1c1d..54638a00db 100644 --- a/packages/kap-server/test/skills.test.ts +++ b/packages/kap-server/test/skills.test.ts @@ -301,6 +301,81 @@ describe('server-v2 /api/v1 skills', () => { expect(body.code).toBe(40407); }); + it('activates a skill with a server-local file attachment by path', async () => { + const id = await createSession(); + await createMainAgent(id); + const noteBytes = Buffer.from('path attachment note'); + const sourcePath = join(home as string, 'note.txt'); + await writeFile(sourcePath, noteBytes); + + const { body } = await postJson<{ activated: boolean; skill_name: string }>( + `/api/v1/sessions/${id}/skills/update-config:activate`, + { attachments: [{ type: 'file', path: sourcePath }] }, + ); + expect(body.code).toBe(0); + expect(body.data).toEqual({ activated: true, skill_name: 'update-config' }); + + const messages = await getJson<{ + items: Array<{ role: string; content: Array<{ type: string; text?: string }> }>; + }>(`/api/v1/sessions/${id}/messages`); + const userMsg = messages.body.data.items.find( + (m) => + m.role === 'user' && + m.content.some((part) => part.text?.includes('User activated the skill')), + ); + expect(userMsg).toBeDefined(); + const notice = userMsg!.content[1]; + expect(notice).toEqual({ + type: 'text', + text: `Attached file "note.txt" (application/octet-stream, ${noteBytes.length} bytes): ${sourcePath} — open it with the Read tool`, + }); + + const transcript = await getJson<{ + items: Array<{ kind: string; attachmentIds?: string[] }>; + attachments: Array<{ + attachmentId: string; + mediaType: string; + name?: string; + size?: number; + source?: unknown; + }>; + }>(`/api/v1/sessions/${id}/transcript?agent_id=main`); + const transcriptAttachments = transcript.body.data.attachments; + expect(transcriptAttachments).toHaveLength(1); + expect(transcriptAttachments[0]).toMatchObject({ + mediaType: 'application/octet-stream', + name: 'note.txt', + size: noteBytes.length, + }); + expect(transcriptAttachments[0]).not.toHaveProperty('source'); + const turn = transcript.body.data.items.find((item) => item.kind === 'turn'); + expect(turn?.attachmentIds).toEqual([transcriptAttachments[0]!.attachmentId]); + }); + + it('rejects a relative attachment path on skill activation (40001)', async () => { + const id = await createSession(); + await createMainAgent(id); + + const { body } = await postJson( + `/api/v1/sessions/${id}/skills/update-config:activate`, + { attachments: [{ type: 'file', path: 'relative/note.txt' }] }, + ); + expect(body.code).toBe(40001); + }); + + it('rejects a sensitive attachment path on skill activation (40001)', async () => { + const id = await createSession(); + await createMainAgent(id); + const secretPath = join(home as string, '.env'); + await writeFile(secretPath, 'TOKEN=secret'); + + const { body } = await postJson( + `/api/v1/sessions/${id}/skills/update-config:activate`, + { attachments: [{ type: 'file', path: secretPath }] }, + ); + expect(body.code).toBe(40001); + }); + it('rejects an unknown skill with attachments before materializing them (40415)', async () => { const id = await createSession(); await createMainAgent(id); diff --git a/packages/protocol/src/__tests__/message.test.ts b/packages/protocol/src/__tests__/message.test.ts index 4939430b9f..0ec6da3e9d 100644 --- a/packages/protocol/src/__tests__/message.test.ts +++ b/packages/protocol/src/__tests__/message.test.ts @@ -108,6 +108,31 @@ describe('messageContentSchema variants', () => { expect(parsed.size).toBe(12345); }); + it('parses file content by server-local path', () => { + const parsed = fileContentSchema.parse({ type: 'file', path: '/data/doc.pdf' }); + expect(parsed.path).toBe('/data/doc.pdf'); + const parsedSource = imageContentSchema.parse({ + type: 'image', + source: { kind: 'path', path: '/data/pic.png' }, + }); + expect(parsedSource.source.kind).toBe('path'); + }); + + it('rejects file content with both file_id and path, or neither', () => { + expect( + fileContentSchema.safeParse({ + type: 'file', + file_id: 'file_01', + path: '/data/doc.pdf', + name: 'doc.pdf', + media_type: 'application/pdf', + size: 1, + }).success, + ).toBe(false); + expect(fileContentSchema.safeParse({ type: 'file' }).success).toBe(false); + expect(fileContentSchema.safeParse({ type: 'file', file_id: 'file_01' }).success).toBe(false); + }); + it('parses thinking content', () => { const parsed = thinkingContentSchema.parse({ type: 'thinking', diff --git a/packages/protocol/src/events.ts b/packages/protocol/src/events.ts index 8e22d7c85f..95939852e7 100644 --- a/packages/protocol/src/events.ts +++ b/packages/protocol/src/events.ts @@ -683,6 +683,17 @@ export interface WarningEvent { readonly code?: string; } +/** A prompt-carried transcript attachment: a session-media reference or a typed file attachment. */ +export type TurnPromptAttachment = + | { readonly kind: 'image' | 'video' | 'audio'; readonly fileId: string } + | { + readonly kind: 'file'; + readonly name: string; + readonly mediaType: string; + readonly size: number; + readonly path: string; + }; + export interface TurnStartedEvent { readonly type: 'turn.started'; readonly turnId: number; @@ -690,8 +701,8 @@ export interface TurnStartedEvent { readonly prompt?: string; /** The prompt record id when the turn was opened by a prompt submission. */ readonly promptId?: string; - /** Session-media references carried by the prompt (transcript attachments). */ - readonly promptAttachments?: readonly { kind: 'image' | 'video' | 'audio'; fileId: string }[]; + /** Session-media references and file attachments carried by the prompt (transcript attachments). */ + readonly promptAttachments?: readonly TurnPromptAttachment[]; } export interface TurnEndedEvent { @@ -1666,7 +1677,18 @@ export const turnStartedEventSchema = z.object({ prompt: z.string().optional(), promptId: z.string().optional(), promptAttachments: z - .array(z.object({ kind: z.enum(['image', 'video', 'audio']), fileId: z.string() })) + .array( + z.union([ + z.object({ kind: z.enum(['image', 'video', 'audio']), fileId: z.string() }), + z.object({ + kind: z.literal('file'), + name: z.string(), + mediaType: z.string(), + size: z.number(), + path: z.string(), + }), + ]), + ) .optional(), }) satisfies z.ZodType; diff --git a/packages/protocol/src/message.ts b/packages/protocol/src/message.ts index f33905e678..acf640f824 100644 --- a/packages/protocol/src/message.ts +++ b/packages/protocol/src/message.ts @@ -44,6 +44,9 @@ export const imageSourceSchema = z.discriminatedUnion('kind', [ // Stored prompt/message projections address the Session-owned canonical // copy; `file` remains the transient upload form accepted on submission. z.object({ kind: z.literal('session_media'), file_id: z.string().min(1) }), + // Zero-copy attach of a server-local absolute path (desktop clients); the + // daemon validates and reads the file in place — local runtime only. + z.object({ kind: z.literal('path'), path: z.string().min(1) }), ]); export type ImageSource = z.infer; @@ -60,13 +63,36 @@ export const videoContentSchema = z.object({ }); export type VideoContent = z.infer; -export const fileContentSchema = z.object({ - type: z.literal('file'), - file_id: z.string().min(1), - name: z.string(), - media_type: z.string().min(1), - size: z.number().int().nonnegative(), -}); +// A file part either references an uploaded file (`file_id`, with the +// client-supplied metadata) or attaches a server-local absolute `path` +// (zero-copy; the daemon fills name/media_type/size from stat). +export const fileContentSchema = z + .object({ + type: z.literal('file'), + file_id: z.string().min(1).optional(), + path: z.string().min(1).optional(), + name: z.string().optional(), + media_type: z.string().min(1).optional(), + size: z.number().int().nonnegative().optional(), + }) + .superRefine((part, ctx) => { + const hasFileId = part.file_id !== undefined; + const hasPath = part.path !== undefined; + if (hasFileId === hasPath) { + ctx.addIssue({ + code: 'custom', + message: 'exactly one of file_id or path is required', + path: hasFileId ? ['path'] : ['file_id'], + }); + return; + } + if (hasPath) return; + for (const key of ['name', 'media_type', 'size'] as const) { + if (part[key] === undefined) { + ctx.addIssue({ code: 'custom', message: `${key} is required with file_id`, path: [key] }); + } + } + }); export type FileContent = z.infer; export const thinkingContentSchema = z.object({ diff --git a/packages/transcript/src/history/groupTurns.ts b/packages/transcript/src/history/groupTurns.ts index aaff109951..cd788f0b54 100644 --- a/packages/transcript/src/history/groupTurns.ts +++ b/packages/transcript/src/history/groupTurns.ts @@ -136,6 +136,16 @@ export function groupMessagesIntoSnapshot( ids.push(entity.attachmentId); } } + for (const attachment of originFileAttachments(message)) { + const entity: TranscriptAttachment = { + attachmentId: `att_${attachments.length + 1}`, + mediaType: attachment.mediaType, + name: attachment.name, + size: attachment.size, + }; + attachments.push(entity); + ids.push(entity.attachmentId); + } return { text: texts.join(''), attachmentIds: ids.length > 0 ? ids : undefined }; }; @@ -434,6 +444,28 @@ function bundledSkillActivations(message: HistoryMessage): readonly BundledSkill ); } +interface OriginFileAttachment { + readonly name: string; + readonly mediaType: string; + readonly size: number; + readonly path: string; +} + +function originFileAttachments(message: HistoryMessage): readonly OriginFileAttachment[] { + if (message.origin?.kind !== 'user' && message.origin?.kind !== 'skill_activation') return []; + const attachments = (message.origin as { readonly attachments?: unknown }).attachments; + if (!Array.isArray(attachments)) return []; + return attachments.filter( + (attachment): attachment is OriginFileAttachment => + typeof attachment === 'object' && + attachment !== null && + typeof (attachment as { name?: unknown }).name === 'string' && + typeof (attachment as { mediaType?: unknown }).mediaType === 'string' && + typeof (attachment as { size?: unknown }).size === 'number' && + typeof (attachment as { path?: unknown }).path === 'string', + ); +} + function textOf(message: HistoryMessage): string { return (message.content ?? []) .filter((part): part is { readonly type: 'text'; readonly text: string } => part.type === 'text' && 'text' in part) diff --git a/packages/transcript/test/layers.test.ts b/packages/transcript/test/layers.test.ts index c457f9c3eb..8fa504400d 100644 --- a/packages/transcript/test/layers.test.ts +++ b/packages/transcript/test/layers.test.ts @@ -806,6 +806,77 @@ describe('groupMessagesIntoSnapshot (cold path)', () => { expect(firstTurn.attachmentIds).toEqual(['att_1', 'att_2', 'att_3']); }); + it('folds origin file attachments on the opening user message into path-sourced entities', () => { + const snapshot = groupMessagesIntoSnapshot([ + { + role: 'user', + content: [ + { + type: 'text', + text: 'Attached file "report.pdf" (application/pdf, 128 bytes): /data/report.pdf — open it with the Read tool', + }, + ], + toolCalls: [], + origin: { + kind: 'user', + attachments: [ + { name: 'report.pdf', mediaType: 'application/pdf', size: 128, path: '/data/report.pdf' }, + ], + } as { kind: string }, + }, + { role: 'assistant', content: [{ type: 'text', text: 'done' }], toolCalls: [] }, + ]); + + expect(snapshot.attachments).toEqual([ + { + attachmentId: 'att_1', + mediaType: 'application/pdf', + name: 'report.pdf', + size: 128, + }, + ]); + const turn = snapshot.items[0]; + if (turn?.kind !== 'turn') throw new Error('expected turn'); + expect(turn.attachmentIds).toEqual(['att_1']); + }); + + it('folds origin file attachments on a skill activation message into path-sourced entities', () => { + const snapshot = groupMessagesIntoSnapshot([ + { + role: 'user', + content: [ + { + type: 'text', + text: 'User activated the skill "update-config".', + }, + ], + toolCalls: [], + origin: { + kind: 'skill_activation', + activationId: 'act_1', + skillName: 'update-config', + trigger: 'user-slash', + attachments: [ + { name: 'note.txt', mediaType: 'text/plain', size: 21, path: '/data/note.txt' }, + ], + } as { kind: string }, + }, + { role: 'assistant', content: [{ type: 'text', text: 'done' }], toolCalls: [] }, + ]); + + expect(snapshot.attachments).toEqual([ + { + attachmentId: 'att_1', + mediaType: 'text/plain', + name: 'note.txt', + size: 21, + }, + ]); + const turn = snapshot.items[1]; + if (turn?.kind !== 'turn') throw new Error('expected turn'); + expect(turn.attachmentIds).toEqual(['att_1']); + }); + it('maps persisted kimi-file media refs to attachments', () => { const snapshot = groupMessagesIntoSnapshot([ {