authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-09-06 16:37:52-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-09-06 16:37:52-07:00
log29c5ecd0ee288489d254905dc6aa9844be36db8e
tree6bab03f3d32b89f67cc415818a1c419b9c8a17cc
parent86070cbc7430957e542f5b1d4c6a38359329002d
signature Signed by PGP key B5690EEEBB952194

fix(mcp): allow text-only preview snapshots (#10232)

4 files changed, 187 insertions(+), 41 deletions(-)

apps/server/src/mcp/McpHttpServer.test.ts+163-33
...@@ -2,8 +2,10 @@ import { expect, it } from "@effect/vitest";...@@ -2,8 +2,10 @@ import { expect, it } from "@effect/vitest";
2import { NodeHttpServer } from "@effect/platform-node";2import { NodeHttpServer } from "@effect/platform-node";
3import * as NodeServices from "@effect/platform-node/NodeServices";3import * as NodeServices from "@effect/platform-node/NodeServices";
4import { EnvironmentId, PreviewTabId, ProviderInstanceId, ThreadId } from "@t3tools/contracts";4import { EnvironmentId, PreviewTabId, ProviderInstanceId, ThreadId } from "@t3tools/contracts";
5import * as Deferred from "effect/Deferred";
5import * as Effect from "effect/Effect";6import * as Effect from "effect/Effect";
6import * as Layer from "effect/Layer";7import * as Layer from "effect/Layer";
8import * as Schema from "effect/Schema";
7import * as Stream from "effect/Stream";9import * as Stream from "effect/Stream";
8import { McpProtocol, McpSchema, McpServer } from "effect/unstable/ai";10import { McpProtocol, McpSchema, McpServer } from "effect/unstable/ai";
9import { HttpBody, HttpClient, HttpRouter, HttpServerResponse } from "effect/unstable/http";11import { HttpBody, HttpClient, HttpRouter, HttpServerResponse } from "effect/unstable/http";
...@@ -16,6 +18,7 @@ const environmentId = EnvironmentId.make("environment-mcp-test");...@@ -16,6 +18,7 @@ const environmentId = EnvironmentId.make("environment-mcp-test");
16const threadId = ThreadId.make("thread-mcp-test");18const threadId = ThreadId.make("thread-mcp-test");
17const tabId = PreviewTabId.make("tab-mcp-test");19const tabId = PreviewTabId.make("tab-mcp-test");
18const alternateTabId = PreviewTabId.make("tab-mcp-alternate");20const alternateTabId = PreviewTabId.make("tab-mcp-alternate");
21const encodeJsonText = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown));
19const invocation = {22const invocation = {
20 environmentId,23 environmentId,
21 threadId,24 threadId,
...@@ -51,52 +54,179 @@ it("normalizes empty successful notification responses to accepted", () => {...@@ -51,52 +54,179 @@ it("normalizes empty successful notification responses to accepted", () => {
51 expect(resultResponse.status).toBe(200);54 expect(resultResponse.status).toBe(200);
52});55});
5356
54it.effect("returns bounded structural preview snapshot failures", () =>57it.effect.each([{}, { includeImage: false }])(
58 "returns bounded structural preview snapshot failures %#",
59 (input) =>
60 Effect.scoped(
61 Effect.gen(function* () {
62 const server = yield* McpServer.McpServer;
63 const broker = yield* PreviewAutomationBroker.PreviewAutomationBroker;
64 const events = yield* broker.connect({
65 clientId: "mcp-failure-client",
66 environmentId,
67 });
68 yield* Stream.runForEach(events, (event) =>
69 event.type === "connected"
70 ? Effect.void
71 : broker.respond({
72 clientId: "mcp-failure-client",
73 connectionId: event.connectionId,
74 requestId: event.request.requestId,
75 ok: false,
76 error: {
77 _tag: "PreviewAutomationExecutionError",
78 message: "sensitive renderer failure",
79 detail: { consoleOutput: "sensitive browser output" },
80 },
81 }),
82 ).pipe(Effect.forkScoped);
83 yield* Effect.yieldNow;
84
85 const snapshot = yield* server
86 .callTool({ name: "preview_snapshot", arguments: input })
87 .pipe(
88 Effect.provideService(McpInvocationContext.McpInvocationContext, invocation),
89 Effect.provideService(McpSchema.McpServerClient, client),
90 );
91
92 expect(snapshot.isError).toBe(true);
93 expect(snapshot.content).toEqual([{ type: "text", text: "Preview snapshot failed." }]);
94 expect(snapshot.structuredContent).toEqual({
95 error: {
96 _tag: "PreviewAutomationExecutionError",
97 operation: "snapshot",
98 failureCount: 1,
99 },
100 });
101 }),
102 ).pipe(Effect.provide(TestLayer)),
103);
104
105it.effect.each([
106 { mode: "default", input: {}, images: true },
107 { mode: "explicit image", input: { includeImage: true }, images: true },
108 { mode: "text only", input: { includeImage: false }, images: false },
109])("returns fresh $mode snapshots on repeated MCP calls", ({ input, images }) =>
55 Effect.scoped(110 Effect.scoped(
56 Effect.gen(function* () {111 Effect.gen(function* () {
57 const server = yield* McpServer.McpServer;112 const server = yield* McpServer.McpServer;
58 const broker = yield* PreviewAutomationBroker.PreviewAutomationBroker;113 const broker = yield* PreviewAutomationBroker.PreviewAutomationBroker;
59 const events = yield* broker.connect({114 const connected = yield* Deferred.make<void>();
60 clientId: "mcp-failure-client",115 const png =
61 environmentId,116 "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jRZkAAAAASUVORK5CYII=";
62 });117 const page = {
63 yield* Stream.runForEach(events, (event) =>118 url: "http://example.test/",
64 event.type === "connected"119 loading: false,
65 ? Effect.void120 visibleText: "Save your changes",
66 : broker.respond({121 interactiveElements: [
67 clientId: "mcp-failure-client",122 {
68 connectionId: event.connectionId,123 tag: "button",
69 requestId: event.request.requestId,124 role: "button",
70 ok: false,125 name: "Save",
71 error: {126 selector: "#save",
72 _tag: "PreviewAutomationExecutionError",127 x: 0,
73 message: "sensitive renderer failure",128 y: 0,
74 detail: { consoleOutput: "sensitive browser output" },129 width: 20,
75 },130 height: 10,
76 }),131 },
77 ).pipe(Effect.forkScoped);132 ],
78 yield* Effect.yieldNow;133 accessibilityTree: { role: "document", name: "Example" },
134 consoleEntries: [],
135 networkEntries: [],
136 actionTimeline: [],
137 };
138 const screenshot = { mimeType: "image/png", width: 1, height: 1 };
139 let requests = 0;
140 const events = yield* broker.connect({ clientId: "mcp-image-option-client", environmentId });
141 yield* Stream.runForEach(events, (event) => {
142 if (event.type === "connected") return Deferred.succeed(connected, undefined);
143 requests += 1;
144 expect(event.request).toMatchObject({
145 operation: "snapshot",
146 tabId: alternateTabId,
147 threadId,
148 });
149 expect(event.request.input).toEqual({});
150 return broker.respond({
151 clientId: "mcp-image-option-client",
152 connectionId: event.connectionId,
153 requestId: event.request.requestId,
154 ok: true,
155 result: {
156 ...page,
157 title: `Snapshot ${requests}`,
158 screenshot: { ...screenshot, data: png },
159 },
160 });
161 }).pipe(Effect.forkScoped);
162 yield* Deferred.await(connected);
79163
80 const snapshot = yield* server164 for (const call of [1, 2, 3, 4, 5, 6]) {
81 .callTool({ name: "preview_snapshot", arguments: {} })165 const snapshot = yield* server
166 .callTool({
167 name: "preview_snapshot",
168 arguments: { ...input, tabId: alternateTabId },
169 })
170 .pipe(
171 Effect.provideService(McpInvocationContext.McpInvocationContext, invocation),
172 Effect.provideService(McpSchema.McpServerClient, client),
173 );
174 const metadata = { ...page, title: `Snapshot ${call}`, screenshot };
175 expect(snapshot.isError).toBe(false);
176 expect(snapshot.structuredContent).toEqual(metadata);
177 expect(snapshot.content).toEqual([
178 { type: "text", text: encodeJsonText(snapshot.structuredContent) },
179 ...(images
180 ? [
181 {
182 type: "image",
183 mimeType: "image/png",
184 data: new Uint8Array(Buffer.from(png, "base64")),
185 },
186 ]
187 : []),
188 ]);
189 }
190
191 // Output selection belongs to this call, not the MCP session's history.
192 const nextDefault = yield* server
193 .callTool({
194 name: "preview_snapshot",
195 arguments: { tabId: alternateTabId },
196 })
82 .pipe(197 .pipe(
83 Effect.provideService(McpInvocationContext.McpInvocationContext, invocation),198 Effect.provideService(McpInvocationContext.McpInvocationContext, invocation),
84 Effect.provideService(McpSchema.McpServerClient, client),199 Effect.provideService(McpSchema.McpServerClient, client),
85 );200 );
86201 expect(nextDefault.content.map((content) => content.type)).toEqual(["text", "image"]);
87 expect(snapshot.isError).toBe(true);202 expect(nextDefault.structuredContent).toEqual({ ...page, title: "Snapshot 7", screenshot });
88 expect(snapshot.content).toEqual([{ type: "text", text: "Preview snapshot failed." }]);203 expect(requests).toBe(7);
89 expect(snapshot.structuredContent).toEqual({
90 error: {
91 _tag: "PreviewAutomationExecutionError",
92 operation: "snapshot",
93 failureCount: 1,
94 },
95 });
96 }),204 }),
97 ).pipe(Effect.provide(TestLayer)),205 ).pipe(Effect.provide(TestLayer)),
98);206);
99207
208it.effect("rejects non-boolean snapshot image options before selecting a browser host", () =>
209 Effect.gen(function* () {
210 const server = yield* McpServer.McpServer;
211 for (const includeImage of ["false", 0, null]) {
212 const result = yield* server
213 .callTool({
214 name: "preview_snapshot",
215 arguments: { includeImage },
216 })
217 .pipe(
218 Effect.provideService(McpInvocationContext.McpInvocationContext, invocation),
219 Effect.provideService(McpSchema.McpServerClient, client),
220 );
221 expect(result.isError).toBe(true);
222 expect(result.content).toEqual([{ type: "text", text: "Preview snapshot failed." }]);
223 expect(result.structuredContent).toEqual({
224 error: { _tag: "AiError", operation: "snapshot", failureCount: 1 },
225 });
226 }
227 }).pipe(Effect.provide(TestLayer)),
228);
229
100it.effect("terminates HTTP MCP sessions with DELETE", () =>230it.effect("terminates HTTP MCP sessions with DELETE", () =>
101 Effect.scoped(231 Effect.scoped(
102 Effect.gen(function* () {232 Effect.gen(function* () {
apps/server/src/mcp/McpHttpServer.ts+9-5
...@@ -187,11 +187,15 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot...@@ -187,11 +187,15 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot
187 structuredContent: metadata,187 structuredContent: metadata,
188 content: [188 content: [
189 { type: "text", text: JSON.stringify(metadata) },189 { type: "text", text: JSON.stringify(metadata) },
190 {190 ...(payload?.includeImage === false
191 type: "image",191 ? []
192 data: new Uint8Array(Buffer.from(screenshot.data, "base64")),192 : [
193 mimeType: screenshot.mimeType,193 {
194 },194 type: "image" as const,
195 data: new Uint8Array(Buffer.from(screenshot.data, "base64")),
196 mimeType: screenshot.mimeType,
197 },
198 ]),
195 ],199 ],
196 }),200 }),
197 );201 );
apps/server/src/mcp/toolkits/preview/handlers.ts+5-1
...@@ -77,7 +77,11 @@ const handlers = {...@@ -77,7 +77,11 @@ const handlers = {
77 invokeTargeted<PreviewAutomationResizeResult>("resize", input, input.timeoutMs),77 invokeTargeted<PreviewAutomationResizeResult>("resize", input, input.timeoutMs),
78 preview_set_appearance: (input) =>78 preview_set_appearance: (input) =>
79 invokeTargeted<PreviewAutomationSetColorSchemeResult>("setColorScheme", input),79 invokeTargeted<PreviewAutomationSetColorSchemeResult>("setColorScheme", input),
80 preview_snapshot: (input) => invokeTargeted<PreviewAutomationSnapshot>("snapshot", input ?? {}),80 preview_snapshot: (input) => {
81 // Output selection is MCP-only; the browser still produces a complete snapshot.
82 const { includeImage: _includeImage, ...operationInput } = input ?? {};
83 return invokeTargeted<PreviewAutomationSnapshot>("snapshot", operationInput);
84 },
81 preview_click: (input) =>85 preview_click: (input) =>
82 invokeTargeted<void>("click", input, input.timeoutMs).pipe(Effect.as({})),86 invokeTargeted<void>("click", input, input.timeoutMs).pipe(Effect.as({})),
83 preview_type: (input) => invokeTargeted<void>("type", input, input.timeoutMs).pipe(Effect.as({})),87 preview_type: (input) => invokeTargeted<void>("type", input, input.timeoutMs).pipe(Effect.as({})),
apps/server/src/mcp/toolkits/preview/tools.ts+10-2
...@@ -111,8 +111,16 @@ export const PreviewSetAppearanceTool = safeBrowserTool(...@@ -111,8 +111,16 @@ export const PreviewSetAppearanceTool = safeBrowserTool(
111export const PreviewSnapshotTool = readonlyBrowserTool(111export const PreviewSnapshotTool = readonlyBrowserTool(
112 Tool.make("preview_snapshot", {112 Tool.make("preview_snapshot", {
113 description:113 description:
114 "Inspect a page before interacting. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab. Returns page state, semantic elements, diagnostics, action history, and a PNG screenshot.",114 "Inspect a page before interacting. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab. Returns page state, semantic elements, diagnostics, action history, and a PNG screenshot. Set includeImage=false for text-only output with the same page metadata.",
115 parameters: PreviewAutomationTabTargetInput,115 parameters: Schema.Struct({
116 ...PreviewAutomationTabTargetInput.fields,
117 includeImage: Schema.optional(
118 Schema.Boolean.annotate({
119 description:
120 "Include the PNG image in the tool response. Defaults to true. Set false for text-only output.",
121 }),
122 ),
123 }),
116 success: PreviewAutomationSnapshot,124 success: PreviewAutomationSnapshot,
117 failure: PreviewAutomationError,125 failure: PreviewAutomationError,
118 dependencies,126 dependencies,