diff --git a/apps/server/src/browser.ts b/apps/server/src/browser.ts index 0be1cd1c4..643d6f8bd 100644 --- a/apps/server/src/browser.ts +++ b/apps/server/src/browser.ts @@ -7,6 +7,7 @@ import type { Config } from "./config.ts"; import type { Store } from "./db.ts"; import { AppError } from "./errors.ts"; import type { Files } from "./files.ts"; +import { clipText } from "./text.ts"; const sessionSchema = z.object({ id: z.string(), @@ -220,7 +221,7 @@ export class BrowserService { return { sessionId: id, ...page, - text: page.text.slice(0, 30_000), + text: clipText(page.text, 30_000), truncated: page.truncated || page.text.length > 30_000, }; }); diff --git a/apps/server/src/engine/conversation.ts b/apps/server/src/engine/conversation.ts index e2c0a57b7..72b5e8568 100644 --- a/apps/server/src/engine/conversation.ts +++ b/apps/server/src/engine/conversation.ts @@ -19,13 +19,9 @@ import { JevService } from "../jev/service.ts"; import { presentChoicesTool } from "../jev/tools.ts"; import { searchDescription, searchInputSchema, searchInstructions } from "../search.ts"; import type { AgentService } from "./service.ts"; +import { clipText } from "../text.ts"; import { tanstackAgent } from "./tanstack-agent.ts"; -function clipCalendarText(text: string, limit: number) { - const clipped = text.slice(0, limit); - return text.length > limit && /[\uD800-\uDBFF]$/.test(clipped) ? clipped.slice(0, -1) : clipped; -} - export class ConversationAgent extends AbstractAgent { constructor( private readonly config: Config, @@ -187,9 +183,9 @@ export class ConversationAgent extends AbstractAgent { ...range, events: events.map(({ attendees: _, ...event }) => ({ ...event, - title: clipCalendarText(event.title, 500), - location: clipCalendarText(event.location, 500), - description: clipCalendarText(event.description, 2000), + title: clipText(event.title, 500), + location: clipText(event.location, 500), + description: clipText(event.description, 2000), })), truncated: page.truncated || diff --git a/apps/server/src/engine/model.ts b/apps/server/src/engine/model.ts index 28c71a311..7cb9424cd 100644 --- a/apps/server/src/engine/model.ts +++ b/apps/server/src/engine/model.ts @@ -10,6 +10,7 @@ import { searchDescription, searchInputSchema, searchInstructions } from "../sea import type { AgentService } from "./service.ts"; import { tanstackAgent } from "./tanstack-agent.ts"; import type { TaskContext } from "./worker.ts"; +import { clipText } from "../text.ts"; export type TaskAnswer = { question?: string; answer: string }; @@ -186,7 +187,7 @@ export async function executeModelTask( kind: "web", title: source.title ?? source.url, url: source.url, - excerpt: source.excerpts.join("\n").slice(0, 500), + excerpt: clipText(source.excerpts.join("\n"), 500), }); } task = await ctx.checkpoint({ evidence }); @@ -216,11 +217,11 @@ export async function executeModelTask( kind: "web", title: page.title, url: page.url, - excerpt: page.text.slice(0, 500), + excerpt: clipText(page.text, 500), }, ], }); - return { ...page, text: page.text.slice(0, 30000) }; + return { ...page, text: clipText(page.text, 30000) }; }, ), ] @@ -400,7 +401,7 @@ export async function executeModelTask( }); }); if (runError) throw new Error(runError); - if (text) await ctx.event("step", "Agent update", text.slice(0, 12000)); + if (text) await ctx.event("step", "Agent update", clipText(text, 12000)); return ( outcome ?? { status: "waiting_input", diff --git a/apps/server/src/engine/service.ts b/apps/server/src/engine/service.ts index c0b16dbe8..4c60ebb33 100644 --- a/apps/server/src/engine/service.ts +++ b/apps/server/src/engine/service.ts @@ -43,6 +43,7 @@ import { withoutRelativeTimes, } from "./page-diff.ts"; import { LostLeaseError, type TaskContext, TaskWorker } from "./worker.ts"; +import { clipText } from "../text.ts"; const hash = (text: string) => createHash("sha256").update(text).digest("hex"); /** A change watch's last saved page: its hash, relative-time-free fingerprint and lines. */ @@ -735,7 +736,7 @@ export class AgentService { await this.db.insertIfAbsent(owner, "notifications", value); } mailEvidence(mail: Mail): Evidence { - return { id: mail.id, kind: "mail", title: mail.subject, excerpt: mail.body.slice(0, 400) }; + return { id: mail.id, kind: "mail", title: mail.subject, excerpt: clipText(mail.body, 400) }; } async artifact( owner: string, @@ -1140,7 +1141,7 @@ export class AgentService { checks: monitor.checks + 1, lastCheckedAt: date(), lastHash: currentHash, - lastValue: text.slice(0, 1000), + lastValue: clipText(text, 1000), nextCheckAt, error: null, }, @@ -1156,11 +1157,11 @@ export class AgentService { await ctx.event( "observation", previousHash ? "Checked for changes" : "Saved the first observation", - text.slice(0, 1000), + clipText(text, 1000), ); if (shouldNotify) { await ctx.guard(); - await ctx.event("result", "A meaningful change was found", changes || text.slice(0, 500)); + await ctx.event("result", "A meaningful change was found", changes || clipText(text, 500)); } const count = diff ? countPageDiff(diff) : ""; return { @@ -1184,7 +1185,7 @@ export class AgentService { title: monitor.title, body: changes ? `Changed at ${observation.url}\n${changes}` - : `Condition met at ${observation.url}: ${text.slice(0, 240)}`, + : `Condition met at ${observation.url}: ${clipText(text, 240)}`, key: `monitor:${monitor.id}:${alertSequence}:${currentHash}`, } : null, @@ -1196,7 +1197,7 @@ export class AgentService { kind: "web", title: observation.title, url: observation.url, - excerpt: text.slice(0, 600), + excerpt: clipText(text, 600), }, ], plan: task.plan.map((s) => ({ ...s, status: "succeeded" })), diff --git a/apps/server/src/jev/service.ts b/apps/server/src/jev/service.ts index c2fbd95a6..8195b0576 100644 --- a/apps/server/src/jev/service.ts +++ b/apps/server/src/jev/service.ts @@ -8,6 +8,7 @@ import { jevOptionSchema, jevPanelSchema, } from "../../../../packages/domain/src/jev.ts"; +import { clipText } from "../text.ts"; import type { Store } from "../db.ts"; import { type JevAdapter, rankJevOptions } from "./adapter.ts"; @@ -351,7 +352,7 @@ export class JevService { runId, kind, reference, - text: text.slice(0, 30_000), + text: clipText(text, 30_000), }); return; } diff --git a/apps/server/src/text.ts b/apps/server/src/text.ts new file mode 100644 index 000000000..a3489727a --- /dev/null +++ b/apps/server/src/text.ts @@ -0,0 +1,25 @@ +/** + * Clips untrusted text (page text, mail bodies, titles) at a UTF-16 limit + * without splitting a surrogate pair: a lone half of a pair is not a + * character, renders as a replacement character and breaks consumers that + * encode the result to UTF-8. When the cut lands between a pair's high and + * low half, the high half is dropped with it. + * + * Untrusted input may already contain a lone surrogate half — malformed + * upstream, a bad copy/paste — that no cut of ours produced. Each lone half + * is normalized to U+FFFD before the clip, the same treatment + * `String.prototype.toWellFormed()` applies, so the result is always + * well-formed UTF-16 and survives a UTF-8 encode even when no cut happens. + */ +const LONE_SURROGATE_HALF = + /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|[\uDC00-\uDFFF](? limit && /[\uD800-\uDBFF]$/.test(clipped) + ? clipped.slice(0, -1) + : clipped + ); +} diff --git a/tests/clip-text.test.ts b/tests/clip-text.test.ts new file mode 100644 index 000000000..dec0863a3 --- /dev/null +++ b/tests/clip-text.test.ts @@ -0,0 +1,61 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { clipText } from "../apps/server/src/text.ts"; + +// The clip sites carry untrusted page and mail text into tool results, evidence +// and notifications. A UTF-16 cut can land between a surrogate pair's halves; +// the lone half that survives is not a character (it renders as U+FFFD and +// breaks consumers that encode the value to UTF-8). +test("keeps a short string unchanged", () => { + assert.equal(clipText("hello", 10), "hello"); + assert.equal(clipText("hello", 5), "hello"); +}); + +test("clips a BMP string at the limit", () => { + assert.equal(clipText("hello world", 5), "hello"); +}); + +test("does not split an astral character at the cut", () => { + // U+1F44D (thumbs up) is the surrogate pair D83D DC4D in UTF-16. + const text = "a".repeat(239) + "\u{1F44D}".repeat(3); + const clipped = clipText(text, 240); + assert.equal(clipped.length, 239); + assert.ok(!/[\uD800-\uDBFF]$/.test(clipped), "must not end on a lone high surrogate"); + assert.ok(clipped.endsWith("a")); +}); + +test("keeps a pair that fits before the limit", () => { + const text = "ab\u{1F44D}cd"; + assert.equal(clipText(text, 4), "ab\u{1F44D}"); +}); + +test("clips exactly at a pair boundary without dropping a character", () => { + const text = "\u{1F44D}\u{1F44E}"; + assert.equal(clipText(text, 2), "\u{1F44D}"); + assert.equal(clipText(text, 4), "\u{1F44D}\u{1F44E}"); +}); + +test("does not trim when the string is exactly the limit", () => { + const text = "\u{1F44D}".repeat(6); // 12 UTF-16 units + assert.equal(clipText(text, 12), text); +}); + +test("normalizes a lone surrogate already present in the input", () => { + // "ab" + a lone high surrogate + "cd": the input is malformed without any + // cut, so repairing only cut-created halves would let it through. + const clipped = clipText("ab\uD800cd", 100); + assert.equal(clipped, "ab\uFFFDcd"); + // the result survives a UTF-8 round trip, e.g. a Postgres insert + assert.equal(Buffer.from(clipped, "utf8").toString("utf8"), clipped); +}); + +test("normalizes a lone low surrogate in the input", () => { + assert.equal(clipText("ab\uDC01cd", 100), "ab\uFFFDcd"); +}); + +test("normalizes lone halves but keeps a well-formed pair in the same text", () => { + // A lone high surrogate followed by another high surrogate, then a valid + // emoji pair: only the halves that lack a partner are replaced. + const clipped = clipText("a\uD800\uD801b\u{1F44D}c", 100); + assert.equal(clipped, "a\uFFFD\uFFFDb\u{1F44D}c"); +});