From 29c5ecd0ee288489d254905dc6aa9844be36db8e Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sun, 6 Sep 2026 16:37:52 -0700 Subject: [PATCH] fix(mcp): allow text-only preview snapshots (#10232) --- apps/server/src/mcp/McpHttpServer.test.ts | 204 ++++++++++++++---- apps/server/src/mcp/McpHttpServer.ts | 14 +- .../src/mcp/toolkits/preview/handlers.ts | 6 +- apps/server/src/mcp/toolkits/preview/tools.ts | 12 +- 4 files changed, 191 insertions(+), 45 deletions(-) diff --git a/apps/server/src/mcp/McpHttpServer.test.ts b/apps/server/src/mcp/McpHttpServer.test.ts index fa2880f9c364eb78838cd7f9be1b22ab8c084838..5627d144cc48a27697251c52f00ea6494e7a1f8d 100644 --- a/apps/server/src/mcp/McpHttpServer.test.ts +++ b/apps/server/src/mcp/McpHttpServer.test.ts @@ -2,8 +2,10 @@ import { expect, it } from "@effect/vitest"; import { NodeHttpServer } from "@effect/platform-node"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { EnvironmentId, PreviewTabId, ProviderInstanceId, ThreadId } from "@t3tools/contracts"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; import { McpProtocol, McpSchema, McpServer } from "effect/unstable/ai"; import { HttpBody, HttpClient, HttpRouter, HttpServerResponse } from "effect/unstable/http"; @@ -16,6 +18,7 @@ const environmentId = EnvironmentId.make("environment-mcp-test"); const threadId = ThreadId.make("thread-mcp-test"); const tabId = PreviewTabId.make("tab-mcp-test"); const alternateTabId = PreviewTabId.make("tab-mcp-alternate"); +const encodeJsonText = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); const invocation = { environmentId, threadId, @@ -51,52 +54,179 @@ it("normalizes empty successful notification responses to accepted", () => { expect(resultResponse.status).toBe(200); }); -it.effect("returns bounded structural preview snapshot failures", () => +it.effect.each([{}, { includeImage: false }])( + "returns bounded structural preview snapshot failures %#", + (input) => + Effect.scoped( + Effect.gen(function* () { + const server = yield* McpServer.McpServer; + const broker = yield* PreviewAutomationBroker.PreviewAutomationBroker; + const events = yield* broker.connect({ + clientId: "mcp-failure-client", + environmentId, + }); + yield* Stream.runForEach(events, (event) => + event.type === "connected" + ? Effect.void + : broker.respond({ + clientId: "mcp-failure-client", + connectionId: event.connectionId, + requestId: event.request.requestId, + ok: false, + error: { + _tag: "PreviewAutomationExecutionError", + message: "sensitive renderer failure", + detail: { consoleOutput: "sensitive browser output" }, + }, + }), + ).pipe(Effect.forkScoped); + yield* Effect.yieldNow; + + const snapshot = yield* server + .callTool({ name: "preview_snapshot", arguments: input }) + .pipe( + Effect.provideService(McpInvocationContext.McpInvocationContext, invocation), + Effect.provideService(McpSchema.McpServerClient, client), + ); + + expect(snapshot.isError).toBe(true); + expect(snapshot.content).toEqual([{ type: "text", text: "Preview snapshot failed." }]); + expect(snapshot.structuredContent).toEqual({ + error: { + _tag: "PreviewAutomationExecutionError", + operation: "snapshot", + failureCount: 1, + }, + }); + }), + ).pipe(Effect.provide(TestLayer)), +); + +it.effect.each([ + { mode: "default", input: {}, images: true }, + { mode: "explicit image", input: { includeImage: true }, images: true }, + { mode: "text only", input: { includeImage: false }, images: false }, +])("returns fresh $mode snapshots on repeated MCP calls", ({ input, images }) => Effect.scoped( Effect.gen(function* () { const server = yield* McpServer.McpServer; const broker = yield* PreviewAutomationBroker.PreviewAutomationBroker; - const events = yield* broker.connect({ - clientId: "mcp-failure-client", - environmentId, - }); - yield* Stream.runForEach(events, (event) => - event.type === "connected" - ? Effect.void - : broker.respond({ - clientId: "mcp-failure-client", - connectionId: event.connectionId, - requestId: event.request.requestId, - ok: false, - error: { - _tag: "PreviewAutomationExecutionError", - message: "sensitive renderer failure", - detail: { consoleOutput: "sensitive browser output" }, - }, - }), - ).pipe(Effect.forkScoped); - yield* Effect.yieldNow; - - const snapshot = yield* server - .callTool({ name: "preview_snapshot", arguments: {} }) - .pipe( - Effect.provideService(McpInvocationContext.McpInvocationContext, invocation), - Effect.provideService(McpSchema.McpServerClient, client), - ); - - expect(snapshot.isError).toBe(true); - expect(snapshot.content).toEqual([{ type: "text", text: "Preview snapshot failed." }]); - expect(snapshot.structuredContent).toEqual({ - error: { - _tag: "PreviewAutomationExecutionError", + const connected = yield* Deferred.make(); + const png = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jRZkAAAAASUVORK5CYII="; + const page = { + url: "http://example.test/", + loading: false, + visibleText: "Save your changes", + interactiveElements: [ + { + tag: "button", + role: "button", + name: "Save", + selector: "#save", + x: 0, + y: 0, + width: 20, + height: 10, + }, + ], + accessibilityTree: { role: "document", name: "Example" }, + consoleEntries: [], + networkEntries: [], + actionTimeline: [], + }; + const screenshot = { mimeType: "image/png", width: 1, height: 1 }; + let requests = 0; + const events = yield* broker.connect({ clientId: "mcp-image-option-client", environmentId }); + yield* Stream.runForEach(events, (event) => { + if (event.type === "connected") return Deferred.succeed(connected, undefined); + requests += 1; + expect(event.request).toMatchObject({ operation: "snapshot", - failureCount: 1, - }, - }); + tabId: alternateTabId, + threadId, + }); + expect(event.request.input).toEqual({}); + return broker.respond({ + clientId: "mcp-image-option-client", + connectionId: event.connectionId, + requestId: event.request.requestId, + ok: true, + result: { + ...page, + title: `Snapshot ${requests}`, + screenshot: { ...screenshot, data: png }, + }, + }); + }).pipe(Effect.forkScoped); + yield* Deferred.await(connected); + + for (const call of [1, 2, 3, 4, 5, 6]) { + const snapshot = yield* server + .callTool({ + name: "preview_snapshot", + arguments: { ...input, tabId: alternateTabId }, + }) + .pipe( + Effect.provideService(McpInvocationContext.McpInvocationContext, invocation), + Effect.provideService(McpSchema.McpServerClient, client), + ); + const metadata = { ...page, title: `Snapshot ${call}`, screenshot }; + expect(snapshot.isError).toBe(false); + expect(snapshot.structuredContent).toEqual(metadata); + expect(snapshot.content).toEqual([ + { type: "text", text: encodeJsonText(snapshot.structuredContent) }, + ...(images + ? [ + { + type: "image", + mimeType: "image/png", + data: new Uint8Array(Buffer.from(png, "base64")), + }, + ] + : []), + ]); + } + + // Output selection belongs to this call, not the MCP session's history. + const nextDefault = yield* server + .callTool({ + name: "preview_snapshot", + arguments: { tabId: alternateTabId }, + }) + .pipe( + Effect.provideService(McpInvocationContext.McpInvocationContext, invocation), + Effect.provideService(McpSchema.McpServerClient, client), + ); + expect(nextDefault.content.map((content) => content.type)).toEqual(["text", "image"]); + expect(nextDefault.structuredContent).toEqual({ ...page, title: "Snapshot 7", screenshot }); + expect(requests).toBe(7); }), ).pipe(Effect.provide(TestLayer)), ); +it.effect("rejects non-boolean snapshot image options before selecting a browser host", () => + Effect.gen(function* () { + const server = yield* McpServer.McpServer; + for (const includeImage of ["false", 0, null]) { + const result = yield* server + .callTool({ + name: "preview_snapshot", + arguments: { includeImage }, + }) + .pipe( + Effect.provideService(McpInvocationContext.McpInvocationContext, invocation), + Effect.provideService(McpSchema.McpServerClient, client), + ); + expect(result.isError).toBe(true); + expect(result.content).toEqual([{ type: "text", text: "Preview snapshot failed." }]); + expect(result.structuredContent).toEqual({ + error: { _tag: "AiError", operation: "snapshot", failureCount: 1 }, + }); + } + }).pipe(Effect.provide(TestLayer)), +); + it.effect("terminates HTTP MCP sessions with DELETE", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/mcp/McpHttpServer.ts b/apps/server/src/mcp/McpHttpServer.ts index 44ca928e63bb22cb49f34e4c5390a1a8dfcd03b4..79bd631c03cab175f164dd210a887684bbace1ce 100644 --- a/apps/server/src/mcp/McpHttpServer.ts +++ b/apps/server/src/mcp/McpHttpServer.ts @@ -187,11 +187,15 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot structuredContent: metadata, content: [ { type: "text", text: JSON.stringify(metadata) }, - { - type: "image", - data: new Uint8Array(Buffer.from(screenshot.data, "base64")), - mimeType: screenshot.mimeType, - }, + ...(payload?.includeImage === false + ? [] + : [ + { + type: "image" as const, + data: new Uint8Array(Buffer.from(screenshot.data, "base64")), + mimeType: screenshot.mimeType, + }, + ]), ], }), ); diff --git a/apps/server/src/mcp/toolkits/preview/handlers.ts b/apps/server/src/mcp/toolkits/preview/handlers.ts index 17da780148854278f59a8f72cd74c8be554cb768..c501ee08711d66fa2dc722624bab27ae133827ca 100644 --- a/apps/server/src/mcp/toolkits/preview/handlers.ts +++ b/apps/server/src/mcp/toolkits/preview/handlers.ts @@ -77,7 +77,11 @@ const handlers = { invokeTargeted("resize", input, input.timeoutMs), preview_set_appearance: (input) => invokeTargeted("setColorScheme", input), - preview_snapshot: (input) => invokeTargeted("snapshot", input ?? {}), + preview_snapshot: (input) => { + // Output selection is MCP-only; the browser still produces a complete snapshot. + const { includeImage: _includeImage, ...operationInput } = input ?? {}; + return invokeTargeted("snapshot", operationInput); + }, preview_click: (input) => invokeTargeted("click", input, input.timeoutMs).pipe(Effect.as({})), preview_type: (input) => invokeTargeted("type", input, input.timeoutMs).pipe(Effect.as({})), diff --git a/apps/server/src/mcp/toolkits/preview/tools.ts b/apps/server/src/mcp/toolkits/preview/tools.ts index 33528d8bb38c3e4d282eace47eb9f3b0eb35543a..ab8d1580bb61cde104e2f20aa6b4bb24a8644f8f 100644 --- a/apps/server/src/mcp/toolkits/preview/tools.ts +++ b/apps/server/src/mcp/toolkits/preview/tools.ts @@ -111,8 +111,16 @@ export const PreviewSetAppearanceTool = safeBrowserTool( export const PreviewSnapshotTool = readonlyBrowserTool( Tool.make("preview_snapshot", { description: - "Inspect a page before interacting. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab. Returns page state, semantic elements, diagnostics, action history, and a PNG screenshot.", - parameters: PreviewAutomationTabTargetInput, + "Inspect a page before interacting. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab. Returns page state, semantic elements, diagnostics, action history, and a PNG screenshot. Set includeImage=false for text-only output with the same page metadata.", + parameters: Schema.Struct({ + ...PreviewAutomationTabTargetInput.fields, + includeImage: Schema.optional( + Schema.Boolean.annotate({ + description: + "Include the PNG image in the tool response. Defaults to true. Set false for text-only output.", + }), + ), + }), success: PreviewAutomationSnapshot, failure: PreviewAutomationError, dependencies, -- 2.54.0