authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-09-06 16:37:52-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-09-06 16:37:52-07:00
log29c5ecd0ee288489d254905dc6aa9844be36db8e
tree6bab03f3d32b89f67cc415818a1c419b9c8a17cc
parent86070cbc7430957e542f5b1d4c6a38359329002d
signature Signed by PGP key B5690EEEBB952194

fix(mcp): allow text-only preview snapshots (#10232)

4 files changed, 187 insertions(+), 41 deletions(-)

apps/server/src/mcp/McpHttpServer.test.ts+163-33
......@@ -2,8 +2,10 @@ import { expect, it } from "@effect/vitest";
22import { NodeHttpServer } from "@effect/platform-node";
33import * as NodeServices from "@effect/platform-node/NodeServices";
44import { EnvironmentId, PreviewTabId, ProviderInstanceId, ThreadId } from "@t3tools/contracts";
5import * as Deferred from "effect/Deferred";
56import * as Effect from "effect/Effect";
67import * as Layer from "effect/Layer";
8import * as Schema from "effect/Schema";
79import * as Stream from "effect/Stream";
810import { McpProtocol, McpSchema, McpServer } from "effect/unstable/ai";
911import { HttpBody, HttpClient, HttpRouter, HttpServerResponse } from "effect/unstable/http";
......@@ -16,6 +18,7 @@ const environmentId = EnvironmentId.make("environment-mcp-test");
1618const threadId = ThreadId.make("thread-mcp-test");
1719const tabId = PreviewTabId.make("tab-mcp-test");
1820const alternateTabId = PreviewTabId.make("tab-mcp-alternate");
21const encodeJsonText = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown));
1922const invocation = {
2023 environmentId,
2124 threadId,
......@@ -51,52 +54,179 @@ it("normalizes empty successful notification responses to accepted", () => {
5154 expect(resultResponse.status).toBe(200);
5255});
5356
54it.effect("returns bounded structural preview snapshot failures", () =>
57it.effect.each([{}, { includeImage: false }])(
58 "returns bounded structural preview snapshot failures %#",
59 (input) =>
60 Effect.scoped(
61 Effect.gen(function* () {
62 const server = yield* McpServer.McpServer;
63 const broker = yield* PreviewAutomationBroker.PreviewAutomationBroker;
64 const events = yield* broker.connect({
65 clientId: "mcp-failure-client",
66 environmentId,
67 });
68 yield* Stream.runForEach(events, (event) =>
69 event.type === "connected"
70 ? Effect.void
71 : broker.respond({
72 clientId: "mcp-failure-client",
73 connectionId: event.connectionId,
74 requestId: event.request.requestId,
75 ok: false,
76 error: {
77 _tag: "PreviewAutomationExecutionError",
78 message: "sensitive renderer failure",
79 detail: { consoleOutput: "sensitive browser output" },
80 },
81 }),
82 ).pipe(Effect.forkScoped);
83 yield* Effect.yieldNow;
84
85 const snapshot = yield* server
86 .callTool({ name: "preview_snapshot", arguments: input })
87 .pipe(
88 Effect.provideService(McpInvocationContext.McpInvocationContext, invocation),
89 Effect.provideService(McpSchema.McpServerClient, client),
90 );
91
92 expect(snapshot.isError).toBe(true);
93 expect(snapshot.content).toEqual([{ type: "text", text: "Preview snapshot failed." }]);
94 expect(snapshot.structuredContent).toEqual({
95 error: {
96 _tag: "PreviewAutomationExecutionError",
97 operation: "snapshot",
98 failureCount: 1,
99 },
100 });
101 }),
102 ).pipe(Effect.provide(TestLayer)),
103);
104
105it.effect.each([
106 { mode: "default", input: {}, images: true },
107 { mode: "explicit image", input: { includeImage: true }, images: true },
108 { mode: "text only", input: { includeImage: false }, images: false },
109])("returns fresh $mode snapshots on repeated MCP calls", ({ input, images }) =>
55110 Effect.scoped(
56111 Effect.gen(function* () {
57112 const server = yield* McpServer.McpServer;
58113 const broker = yield* PreviewAutomationBroker.PreviewAutomationBroker;
59 const events = yield* broker.connect({
60 clientId: "mcp-failure-client",
61 environmentId,
62 });
63 yield* Stream.runForEach(events, (event) =>
64 event.type === "connected"
65 ? Effect.void
66 : broker.respond({
67 clientId: "mcp-failure-client",
68 connectionId: event.connectionId,
69 requestId: event.request.requestId,
70 ok: false,
71 error: {
72 _tag: "PreviewAutomationExecutionError",
73 message: "sensitive renderer failure",
74 detail: { consoleOutput: "sensitive browser output" },
75 },
76 }),
77 ).pipe(Effect.forkScoped);
78 yield* Effect.yieldNow;
114 const connected = yield* Deferred.make<void>();
115 const png =
116 "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jRZkAAAAASUVORK5CYII=";
117 const page = {
118 url: "http://example.test/",
119 loading: false,
120 visibleText: "Save your changes",
121 interactiveElements: [
122 {
123 tag: "button",
124 role: "button",
125 name: "Save",
126 selector: "#save",
127 x: 0,
128 y: 0,
129 width: 20,
130 height: 10,
131 },
132 ],
133 accessibilityTree: { role: "document", name: "Example" },
134 consoleEntries: [],
135 networkEntries: [],
136 actionTimeline: [],
137 };
138 const screenshot = { mimeType: "image/png", width: 1, height: 1 };
139 let requests = 0;
140 const events = yield* broker.connect({ clientId: "mcp-image-option-client", environmentId });
141 yield* Stream.runForEach(events, (event) => {
142 if (event.type === "connected") return Deferred.succeed(connected, undefined);
143 requests += 1;
144 expect(event.request).toMatchObject({
145 operation: "snapshot",
146 tabId: alternateTabId,
147 threadId,
148 });
149 expect(event.request.input).toEqual({});
150 return broker.respond({
151 clientId: "mcp-image-option-client",
152 connectionId: event.connectionId,
153 requestId: event.request.requestId,
154 ok: true,
155 result: {
156 ...page,
157 title: `Snapshot ${requests}`,
158 screenshot: { ...screenshot, data: png },
159 },
160 });
161 }).pipe(Effect.forkScoped);
162 yield* Deferred.await(connected);
79163
80 const snapshot = yield* server
81 .callTool({ name: "preview_snapshot", arguments: {} })
164 for (const call of [1, 2, 3, 4, 5, 6]) {
165 const snapshot = yield* server
166 .callTool({
167 name: "preview_snapshot",
168 arguments: { ...input, tabId: alternateTabId },
169 })
170 .pipe(
171 Effect.provideService(McpInvocationContext.McpInvocationContext, invocation),
172 Effect.provideService(McpSchema.McpServerClient, client),
173 );
174 const metadata = { ...page, title: `Snapshot ${call}`, screenshot };
175 expect(snapshot.isError).toBe(false);
176 expect(snapshot.structuredContent).toEqual(metadata);
177 expect(snapshot.content).toEqual([
178 { type: "text", text: encodeJsonText(snapshot.structuredContent) },
179 ...(images
180 ? [
181 {
182 type: "image",
183 mimeType: "image/png",
184 data: new Uint8Array(Buffer.from(png, "base64")),
185 },
186 ]
187 : []),
188 ]);
189 }
190
191 // Output selection belongs to this call, not the MCP session's history.
192 const nextDefault = yield* server
193 .callTool({
194 name: "preview_snapshot",
195 arguments: { tabId: alternateTabId },
196 })
82197 .pipe(
83198 Effect.provideService(McpInvocationContext.McpInvocationContext, invocation),
84199 Effect.provideService(McpSchema.McpServerClient, client),
85200 );
86
87 expect(snapshot.isError).toBe(true);
88 expect(snapshot.content).toEqual([{ type: "text", text: "Preview snapshot failed." }]);
89 expect(snapshot.structuredContent).toEqual({
90 error: {
91 _tag: "PreviewAutomationExecutionError",
92 operation: "snapshot",
93 failureCount: 1,
94 },
95 });
201 expect(nextDefault.content.map((content) => content.type)).toEqual(["text", "image"]);
202 expect(nextDefault.structuredContent).toEqual({ ...page, title: "Snapshot 7", screenshot });
203 expect(requests).toBe(7);
96204 }),
97205 ).pipe(Effect.provide(TestLayer)),
98206);
99207
208it.effect("rejects non-boolean snapshot image options before selecting a browser host", () =>
209 Effect.gen(function* () {
210 const server = yield* McpServer.McpServer;
211 for (const includeImage of ["false", 0, null]) {
212 const result = yield* server
213 .callTool({
214 name: "preview_snapshot",
215 arguments: { includeImage },
216 })
217 .pipe(
218 Effect.provideService(McpInvocationContext.McpInvocationContext, invocation),
219 Effect.provideService(McpSchema.McpServerClient, client),
220 );
221 expect(result.isError).toBe(true);
222 expect(result.content).toEqual([{ type: "text", text: "Preview snapshot failed." }]);
223 expect(result.structuredContent).toEqual({
224 error: { _tag: "AiError", operation: "snapshot", failureCount: 1 },
225 });
226 }
227 }).pipe(Effect.provide(TestLayer)),
228);
229
100230it.effect("terminates HTTP MCP sessions with DELETE", () =>
101231 Effect.scoped(
102232 Effect.gen(function* () {
apps/server/src/mcp/McpHttpServer.ts+9-5
......@@ -187,11 +187,15 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot
187187 structuredContent: metadata,
188188 content: [
189189 { type: "text", text: JSON.stringify(metadata) },
190 {
191 type: "image",
192 data: new Uint8Array(Buffer.from(screenshot.data, "base64")),
193 mimeType: screenshot.mimeType,
194 },
190 ...(payload?.includeImage === false
191 ? []
192 : [
193 {
194 type: "image" as const,
195 data: new Uint8Array(Buffer.from(screenshot.data, "base64")),
196 mimeType: screenshot.mimeType,
197 },
198 ]),
195199 ],
196200 }),
197201 );
apps/server/src/mcp/toolkits/preview/handlers.ts+5-1
......@@ -77,7 +77,11 @@ const handlers = {
7777 invokeTargeted<PreviewAutomationResizeResult>("resize", input, input.timeoutMs),
7878 preview_set_appearance: (input) =>
7979 invokeTargeted<PreviewAutomationSetColorSchemeResult>("setColorScheme", input),
80 preview_snapshot: (input) => invokeTargeted<PreviewAutomationSnapshot>("snapshot", input ?? {}),
80 preview_snapshot: (input) => {
81 // Output selection is MCP-only; the browser still produces a complete snapshot.
82 const { includeImage: _includeImage, ...operationInput } = input ?? {};
83 return invokeTargeted<PreviewAutomationSnapshot>("snapshot", operationInput);
84 },
8185 preview_click: (input) =>
8286 invokeTargeted<void>("click", input, input.timeoutMs).pipe(Effect.as({})),
8387 preview_type: (input) => invokeTargeted<void>("type", input, input.timeoutMs).pipe(Effect.as({})),
apps/server/src/mcp/toolkits/preview/tools.ts+10-2
......@@ -111,8 +111,16 @@ export const PreviewSetAppearanceTool = safeBrowserTool(
111111export const PreviewSnapshotTool = readonlyBrowserTool(
112112 Tool.make("preview_snapshot", {
113113 description:
114 "Inspect a page before interacting. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab. Returns page state, semantic elements, diagnostics, action history, and a PNG screenshot.",
115 parameters: PreviewAutomationTabTargetInput,
114 "Inspect a page before interacting. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab. Returns page state, semantic elements, diagnostics, action history, and a PNG screenshot. Set includeImage=false for text-only output with the same page metadata.",
115 parameters: Schema.Struct({
116 ...PreviewAutomationTabTargetInput.fields,
117 includeImage: Schema.optional(
118 Schema.Boolean.annotate({
119 description:
120 "Include the PNG image in the tool response. Defaults to true. Set false for text-only output.",
121 }),
122 ),
123 }),
116124 success: PreviewAutomationSnapshot,
117125 failure: PreviewAutomationError,
118126 dependencies,