authorgravatar for me@t3.ggTheo Browne <me@t3.gg> 2026-09-24 21:52:01-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-09-24 21:52:01-07:00
log4f27a8463105662ddfc834e01bca375d5124be79
tree645c1a1561b0d095f277bd39d1efbb9238acb633
parent9957349c0a066a9efc657600fcb901c9b9cca34b
signature Signed by PGP key B5690EEEBB952194

fix(mcp): preview snapshots fit in the agent's tool output again (#13558)

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

3 files changed, 125 insertions(+), 40 deletions(-)

apps/server/src/mcp/McpHttpServer.test.ts+58-5
...@@ -248,7 +248,10 @@ it.effect.each([...@@ -248,7 +248,10 @@ it.effect.each([
248 const metadata = { ...page, title: `Snapshot ${call}`, screenshot };248 const metadata = { ...page, title: `Snapshot ${call}`, screenshot };
249 const { accessibilityTree: _tree, ...boundedMetadata } = metadata;249 const { accessibilityTree: _tree, ...boundedMetadata } = metadata;
250 expect(snapshot.isError).toBe(false);250 expect(snapshot.isError).toBe(false);
251 expect(snapshot.structuredContent).toEqual(metadata);251 expect(snapshot.structuredContent).toEqual({
252 ...boundedMetadata,
253 omitted: ["accessibilityTree (use interactiveElements locators or preview_evaluate)"],
254 });
252 const [identity, text, ...rest] = snapshot.content;255 const [identity, text, ...rest] = snapshot.content;
253 expect(identity?.type === "text" ? decodeJsonText(identity.text) : null).toEqual({256 expect(identity?.type === "text" ? decodeJsonText(identity.text) : null).toEqual({
254 url: page.url,257 url: page.url,
...@@ -287,7 +290,8 @@ it.effect.each([...@@ -287,7 +290,8 @@ it.effect.each([
287 "text",290 "text",
288 "image",291 "image",
289 ]);292 ]);
290 expect(nextDefault.structuredContent).toEqual({ ...page, title: "Snapshot 7", screenshot });293 expect(nextDefault.structuredContent).toMatchObject({ title: "Snapshot 7", screenshot });
294 expect(nextDefault.structuredContent).not.toHaveProperty("accessibilityTree");
291 expect(requests).toBe(7);295 expect(requests).toBe(7);
292 }),296 }),
293 ).pipe(Effect.provide(TestLayer)),297 ).pipe(Effect.provide(TestLayer)),
...@@ -341,6 +345,15 @@ it.effect("saves the snapshot PNG on request and reports its path", () =>...@@ -341,6 +345,15 @@ it.effect("saves the snapshot PNG on request and reports its path", () =>
341345
342 const unsaved = yield* callSnapshot({});346 const unsaved = yield* callSnapshot({});
343 expect(unsaved.structuredContent).not.toHaveProperty("screenshotPath");347 expect(unsaved.structuredContent).not.toHaveProperty("screenshotPath");
348
349 // A save without the image skips the page dump.
350 const pathOnly = yield* callSnapshot({ save: true, includeImage: false });
351 const saved = pathOnly.structuredContent as { readonly screenshotPath: string };
352 expect(saved).toEqual({ url: snapshotResult.url, screenshotPath: expect.any(String) });
353 expect(Buffer.from(yield* fileSystem.readFile(saved.screenshotPath)).toString()).toBe("png");
354 const [only, ...others] = pathOnly.content;
355 expect(others).toEqual([]);
356 expect(only?.type === "text" ? decodeJsonText(only.text) : null).toEqual(saved);
344 }),357 }),
345 ).pipe(Effect.provide(TestLayer)),358 ).pipe(Effect.provide(TestLayer)),
346);359);
...@@ -460,9 +473,10 @@ it.effect("keeps the snapshot text under the agent's output ceiling", () =>...@@ -460,9 +473,10 @@ it.effect("keeps the snapshot text under the agent's output ceiling", () =>
460 expect(parsed.consoleEntries[0]?.text).toBe("entry 60");473 expect(parsed.consoleEntries[0]?.text).toBe("entry 60");
461 expect(notice?.type === "text" ? notice.text : "").toContain("accessibilityTree");474 expect(notice?.type === "text" ? notice.text : "").toContain("accessibilityTree");
462 expect(notice?.type === "text" ? notice.text : "").toContain("60 older console entries");475 expect(notice?.type === "text" ? notice.text : "").toContain("60 older console entries");
463 // The structured result is untouched; only the text the agent reads is bounded.476 // Claude Code shows the model structuredContent instead of the text, so it is bounded too.
464 expect(snapshot.structuredContent).toMatchObject({477 expect(snapshot.structuredContent).toEqual({
465 accessibilityTree: oversized.accessibilityTree,478 ...parsed,
479 omitted: expect.arrayContaining(["60 older console entries"]),
466 });480 });
467 }),481 }),
468 ).pipe(Effect.provide(TestLayer)),482 ).pipe(Effect.provide(TestLayer)),
...@@ -500,6 +514,45 @@ it.effect("bounds the snapshot text even when nothing but logs and the title are...@@ -500,6 +514,45 @@ it.effect("bounds the snapshot text even when nothing but logs and the title are
500 ).pipe(Effect.provide(TestLayer)),514 ).pipe(Effect.provide(TestLayer)),
501);515);
502516
517it.effect("bounds page text made of wide characters before dropping locators", () =>
518 Effect.scoped(
519 Effect.gen(function* () {
520 // The character caps alone leave 8,000 three-byte characters, about 24 KB.
521 yield* serveSnapshots("mcp-wide-text-client", {
522 ...snapshotResult,
523 visibleText: "界".repeat(9_000),
524 interactiveElements: Array.from({ length: 20 }, (_, i) => ({
525 tag: "button",
526 role: "button",
527 name: `Button ${i}`,
528 selector: `#button-${i}`,
529 x: 0,
530 y: 0,
531 width: 10,
532 height: 10,
533 })),
534 });
535
536 const snapshot = yield* callSnapshot({ includeImage: false });
537
538 const [, text, notice] = snapshot.content;
539 const body = text?.type === "text" ? text.text : "";
540 expect(Buffer.byteLength(body, "utf8")).toBeLessThanOrEqual(
541 McpHttpServer.MAX_SNAPSHOT_TEXT_BYTES,
542 );
543 const parsed = decodeJsonText(body) as {
544 readonly visibleText: string;
545 readonly interactiveElements: ReadonlyArray<unknown>;
546 };
547 expect(parsed.visibleText).toMatch(/^界+…$/);
548 expect(parsed.interactiveElements).toHaveLength(20);
549 expect(notice?.type === "text" ? notice.text : "").toContain(
550 "visibleText after 4000 characters",
551 );
552 }),
553 ).pipe(Effect.provide(TestLayer)),
554);
555
503it.effect("sheds log entries before locators when every list is full", () =>556it.effect("sheds log entries before locators when every list is full", () =>
504 Effect.scoped(557 Effect.scoped(
505 Effect.gen(function* () {558 Effect.gen(function* () {
apps/server/src/mcp/McpHttpServer.ts+65-33
...@@ -114,12 +114,15 @@ const McpAuthMiddlewareLive = HttpRouter.middleware<{...@@ -114,12 +114,15 @@ const McpAuthMiddlewareLive = HttpRouter.middleware<{
114}>()(makeMcpAuthMiddleware).layer;114}>()(makeMcpAuthMiddleware).layer;
115115
116/**116/**
117 * Claude Code drops every MCP result above 25k tokens (~100 KB of text) and117 * Claude Code moves an MCP result above its output limit to a file and hands
118 * hands the agent a truncation notice instead, so a snapshot that carries the118 * the agent a notice instead, so a snapshot that carries the full
119 * full accessibility tree and 20 KB of page text loses its locators too. Keep119 * accessibility tree and page text loses its locators too. Claude Code also
120 * the text under that ceiling and tell the agent what was cut.120 * shows the model `structuredContent` in place of the text blocks when a
121 * result has both, so both carry the same bounded snapshot. Keep it near
122 * 20 KB and tell the agent what was cut. The short `omitted` notes may go a
123 * little over; the provider limit is far above this.
121 */124 */
122export const MAX_SNAPSHOT_TEXT_BYTES = 60_000;125export const MAX_SNAPSHOT_TEXT_BYTES = 20_000;
123const MAX_SNAPSHOT_VISIBLE_TEXT_CHARS = 8_000;126const MAX_SNAPSHOT_VISIBLE_TEXT_CHARS = 8_000;
124const MAX_SNAPSHOT_ELEMENT_NAME_CHARS = 200;127const MAX_SNAPSHOT_ELEMENT_NAME_CHARS = 200;
125const MAX_SNAPSHOT_LOG_ENTRIES = 40;128const MAX_SNAPSHOT_LOG_ENTRIES = 40;
...@@ -164,12 +167,11 @@ type SnapshotMetadata = {...@@ -164,12 +167,11 @@ type SnapshotMetadata = {
164/**167/**
165 * Drops the accessibility tree, shortens page text, element names, identifiers,168 * Drops the accessibility tree, shortens page text, element names, identifiers,
166 * and log strings, keeps only the newest log entries, and finally sheds169 * and log strings, keeps only the newest log entries, and finally sheds
167 * interactive elements until the JSON fits. Returns the text plus notes on170 * interactive elements until the JSON fits. Returns the bounded value, its
168 * what is missing so the agent can reach for preview_evaluate.171 * text, and notes on what is missing so the agent can reach for
172 * preview_evaluate.
169 */173 */
170const boundSnapshotMetadata = (174const boundSnapshotMetadata = (metadata: SnapshotMetadata) => {
171 metadata: SnapshotMetadata,
172): { readonly text: string; readonly omitted: ReadonlyArray<string> } => {
173 const omitted: Array<string> = [];175 const omitted: Array<string> = [];
174 const { accessibilityTree, ...withoutTree } = metadata;176 const { accessibilityTree, ...withoutTree } = metadata;
175 if (accessibilityTree !== undefined) {177 if (accessibilityTree !== undefined) {
...@@ -198,16 +200,10 @@ const boundSnapshotMetadata = (...@@ -198,16 +200,10 @@ const boundSnapshotMetadata = (
198 ) {200 ) {
199 omitted.push(`element names longer than ${MAX_SNAPSHOT_ELEMENT_NAME_CHARS} characters`);201 omitted.push(`element names longer than ${MAX_SNAPSHOT_ELEMENT_NAME_CHARS} characters`);
200 }202 }
201 if (metadata.visibleText.length > MAX_SNAPSHOT_VISIBLE_TEXT_CHARS) {
202 omitted.push(
203 `visibleText after ${MAX_SNAPSHOT_VISIBLE_TEXT_CHARS} characters (use preview_evaluate for more)`,
204 );
205 }
206 const bounded = {203 const bounded = {
207 ...withoutTree,204 ...withoutTree,
208 url: cutText(metadata.url, MAX_SNAPSHOT_IDENTIFIER_CHARS),205 url: cutText(metadata.url, MAX_SNAPSHOT_IDENTIFIER_CHARS),
209 title: cutText(metadata.title, MAX_SNAPSHOT_IDENTIFIER_CHARS),206 title: cutText(metadata.title, MAX_SNAPSHOT_IDENTIFIER_CHARS),
210 visibleText: cutText(metadata.visibleText, MAX_SNAPSHOT_VISIBLE_TEXT_CHARS),
211 interactiveElements: metadata.interactiveElements.map((element) => ({207 interactiveElements: metadata.interactiveElements.map((element) => ({
212 ...element,208 ...element,
213 name: cutText(element.name, MAX_SNAPSHOT_ELEMENT_NAME_CHARS),209 name: cutText(element.name, MAX_SNAPSHOT_ELEMENT_NAME_CHARS),
...@@ -218,9 +214,10 @@ const boundSnapshotMetadata = (...@@ -218,9 +214,10 @@ const boundSnapshotMetadata = (
218 };214 };
219215
220 // Per-field caps do not sum below the ceiling: three log arrays of 40 capped216 // Per-field caps do not sum below the ceiling: three log arrays of 40 capped
221 // entries alone can pass 60 KB. Shed the least useful lists first, halving217 // entries alone can pass 60 KB, and the caps count characters, not bytes.
222 // one list per round, until the JSON fits. With every list empty the rest218 // Halve one thing per round until the JSON fits: logs first, then page
223 // is bounded by the identifier and visibleText caps, so this terminates.219 // text, then the locators. The identifier caps bound the rest, so this
220 // terminates.
224 const shedOrder = [221 const shedOrder = [
225 "actionTimeline",222 "actionTimeline",
226 "networkEntries",223 "networkEntries",
...@@ -239,31 +236,51 @@ const boundSnapshotMetadata = (...@@ -239,31 +236,51 @@ const boundSnapshotMetadata = (
239 networkEntries: 0,236 networkEntries: 0,
240 actionTimeline: 0,237 actionTimeline: 0,
241 };238 };
242 let text = encodeJsonText({ ...bounded, ...lists });239 let visibleTextChars = Math.min(metadata.visibleText.length, MAX_SNAPSHOT_VISIBLE_TEXT_CHARS);
240 const value = () => ({
241 ...bounded,
242 visibleText: cutText(metadata.visibleText, visibleTextChars),
243 ...lists,
244 });
245 let text = encodeJsonText(value());
243 while (utf8Length(text) > MAX_SNAPSHOT_TEXT_BYTES) {246 while (utf8Length(text) > MAX_SNAPSHOT_TEXT_BYTES) {
244 // Elements carry the locators, so they go last; logs shed newest-last.247 // Elements carry the locators, so they go last; logs shed newest-last.
245 const key =248 const key =
246 shedOrder.find(249 shedOrder.find(
247 (candidate) => candidate !== "interactiveElements" && lists[candidate].length > 0,250 (candidate) => candidate !== "interactiveElements" && lists[candidate].length > 0,
248 ) ?? (lists.interactiveElements.length > 0 ? "interactiveElements" : undefined);251 ) ??
252 (visibleTextChars > 0
253 ? "visibleText"
254 : lists.interactiveElements.length > 0
255 ? "interactiveElements"
256 : undefined);
249 if (key === undefined) break;257 if (key === undefined) break;
250 const keep = Math.floor(lists[key].length / 2);258 if (key === "visibleText") {
251 dropped[key] += lists[key].length - keep;259 visibleTextChars = Math.floor(visibleTextChars / 2);
252 // slice(-0) keeps everything, so spell out the empty case.260 } else {
253 lists[key] =261 const keep = Math.floor(lists[key].length / 2);
254 keep === 0262 dropped[key] += lists[key].length - keep;
255 ? []263 // slice(-0) keeps everything, so spell out the empty case.
256 : key === "interactiveElements"264 lists[key] =
257 ? lists[key].slice(0, keep)265 keep === 0
258 : lists[key].slice(-keep);266 ? []
259 text = encodeJsonText({ ...bounded, ...lists });267 : key === "interactiveElements"
268 ? lists[key].slice(0, keep)
269 : lists[key].slice(-keep);
270 }
271 text = encodeJsonText(value());
272 }
273 if (visibleTextChars < metadata.visibleText.length) {
274 omitted.push(
275 `visibleText after ${visibleTextChars} characters (use preview_evaluate for more)`,
276 );
260 }277 }
261 for (const key of shedOrder) {278 for (const key of shedOrder) {
262 if (dropped[key] > 0) {279 if (dropped[key] > 0) {
263 omitted.push(`${dropped[key]} of ${bounded[key].length} ${key}`);280 omitted.push(`${dropped[key]} of ${bounded[key].length} ${key}`);
264 }281 }
265 }282 }
266 return { text, omitted };283 return { value: value(), text, omitted };
267};284};
268285
269export class PreviewScreenshotSaveError extends Schema.TaggedError<PreviewScreenshotSaveError>()(286export class PreviewScreenshotSaveError extends Schema.TaggedError<PreviewScreenshotSaveError>()(
...@@ -396,6 +413,18 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot...@@ -396,6 +413,18 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot
396 const png = new Uint8Array(Buffer.from(screenshot.data, "base64"));413 const png = new Uint8Array(Buffer.from(screenshot.data, "base64"));
397 const screenshotPath =414 const screenshotPath =
398 payload?.save === true ? yield* saveScreenshot(snapshot.url, png) : undefined;415 payload?.save === true ? yield* saveScreenshot(snapshot.url, png) : undefined;
416 if (screenshotPath !== undefined && payload?.includeImage === false) {
417 // The agent only wants a file to show the user. The url keeps the site icon on the tool row.
418 const saved = {
419 url: cutText(snapshot.url, MAX_SNAPSHOT_IDENTIFIER_CHARS),
420 screenshotPath,
421 };
422 return new McpSchema.CallToolResult({
423 isError: false,
424 structuredContent: saved,
425 content: [{ type: "text", text: encodeJsonText(saved) }],
426 });
427 }
399 const metadata = {428 const metadata = {
400 ...page,429 ...page,
401 screenshot: {430 screenshot: {
...@@ -408,7 +437,10 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot...@@ -408,7 +437,10 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot
408 const bounded = boundSnapshotMetadata(metadata);437 const bounded = boundSnapshotMetadata(metadata);
409 return new McpSchema.CallToolResult({438 return new McpSchema.CallToolResult({
410 isError: false,439 isError: false,
411 structuredContent: metadata,440 structuredContent:
441 bounded.omitted.length === 0
442 ? bounded.value
443 : { ...bounded.value, omitted: bounded.omitted },
412 content: [444 content: [
413 // Keep the page identity readable even if a provider truncates the snapshot.445 // Keep the page identity readable even if a provider truncates the snapshot.
414 {446 {
apps/server/src/mcp/toolkits/preview/tools.ts+2-2
...@@ -119,7 +119,7 @@ const PreviewSetAppearanceTool = safeBrowserTool(...@@ -119,7 +119,7 @@ const PreviewSetAppearanceTool = safeBrowserTool(
119export const PreviewSnapshotTool = readonlyBrowserTool(119export const PreviewSnapshotTool = readonlyBrowserTool(
120 Tool.make("preview_snapshot", {120 Tool.make("preview_snapshot", {
121 description:121 description:
122 "Inspect a page before interacting. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab. Returns page state, semantic elements, diagnostics, action history, and a PNG screenshot. Set includeImage=false for text-only output with the same page metadata. Set save=true to also write the PNG to disk and get screenshotPath back; embed that path in your reply as ![alt](screenshotPath) so the user sees it. This is the only way to show the user a screenshot; the image in the tool result is not saved anywhere.",122 "Inspect a page before interacting. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab. Returns page state, semantic elements, diagnostics, action history, and a PNG screenshot. The text is capped near 20 KB and lists what it omitted; use preview_evaluate to read more. Set includeImage=false for text-only output with the same page metadata. Set save=true to also write the PNG to disk and get screenshotPath back; with includeImage=false, save=true returns only the url and screenshotPath. Embed that path in your reply as ![alt](screenshotPath) so the user sees it. This is the only way to show the user a screenshot; the image in the tool result is not saved anywhere.",
123 parameters: Schema.Struct({123 parameters: Schema.Struct({
124 ...PreviewAutomationTabTargetInput.fields,124 ...PreviewAutomationTabTargetInput.fields,
125 includeImage: Schema.optional(125 includeImage: Schema.optional(
...@@ -131,7 +131,7 @@ export const PreviewSnapshotTool = readonlyBrowserTool(...@@ -131,7 +131,7 @@ export const PreviewSnapshotTool = readonlyBrowserTool(
131 save: Schema.optional(131 save: Schema.optional(
132 Schema.Boolean.annotate({132 Schema.Boolean.annotate({
133 description:133 description:
134 "Write the screenshot PNG to disk and return its absolute path as screenshotPath. Defaults to false.",134 "Write the screenshot PNG to disk and return its absolute path as screenshotPath. With includeImage=false, return only the url and screenshotPath. Defaults to false.",
135 }),135 }),
136 ),136 ),
137 }),137 }),