authorgravatar for maria@kuuro.netmaria <maria@kuuro.net> 2026-09-11 01:04:14-03:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-09-11 01:04:14-03:00
log6c69534a5800a95e1640b0571580561ef71ec461
tree121319a9cb7a609df623601b1f4e9fb196231975
parent02297e3dbd896ef619d5c916938c751e343e76a8
signature Signed by PGP key B5690EEEBB952194

fix(preview): render website favicons for browser tool activity (#11032)

9 files changed, 320 insertions(+), 78 deletions(-)

apps/server/src/mcp/McpHttpServer.test.ts+33-14
...@@ -250,7 +250,10 @@ it.effect.each([...@@ -250,7 +250,10 @@ it.effect.each([
250 const { accessibilityTree: _tree, ...boundedMetadata } = metadata;250 const { accessibilityTree: _tree, ...boundedMetadata } = metadata;
251 expect(snapshot.isError).toBe(false);251 expect(snapshot.isError).toBe(false);
252 expect(snapshot.structuredContent).toEqual(metadata);252 expect(snapshot.structuredContent).toEqual(metadata);
253 const [text, ...rest] = snapshot.content;253 const [identity, text, ...rest] = snapshot.content;
254 expect(identity?.type === "text" ? decodeJsonText(identity.text) : null).toEqual({
255 url: page.url,
256 });
254 expect(text?.type === "text" ? decodeJsonText(text.text) : null).toEqual(boundedMetadata);257 expect(text?.type === "text" ? decodeJsonText(text.text) : null).toEqual(boundedMetadata);
255 expect(rest).toEqual([258 expect(rest).toEqual([
256 {259 {
...@@ -279,7 +282,12 @@ it.effect.each([...@@ -279,7 +282,12 @@ it.effect.each([
279 Effect.provideService(McpInvocationContext.McpInvocationContext, invocation),282 Effect.provideService(McpInvocationContext.McpInvocationContext, invocation),
280 Effect.provideService(McpSchema.McpServerClient, client),283 Effect.provideService(McpSchema.McpServerClient, client),
281 );284 );
282 expect(nextDefault.content.map((content) => content.type)).toEqual(["text", "text", "image"]);285 expect(nextDefault.content.map((content) => content.type)).toEqual([
286 "text",
287 "text",
288 "text",
289 "image",
290 ]);
283 expect(nextDefault.structuredContent).toEqual({ ...page, title: "Snapshot 7", screenshot });291 expect(nextDefault.structuredContent).toEqual({ ...page, title: "Snapshot 7", screenshot });
284 expect(requests).toBe(7);292 expect(requests).toBe(7);
285 }),293 }),
...@@ -329,7 +337,7 @@ it.effect("saves the snapshot PNG on request and reports its path", () =>...@@ -329,7 +337,7 @@ it.effect("saves the snapshot PNG on request and reports its path", () =>
329 /^browser-screenshot-example-test-[0-9a-z]+-[0-9a-f]{8}\.png$/,337 /^browser-screenshot-example-test-[0-9a-z]+-[0-9a-f]{8}\.png$/,
330 );338 );
331 expect(Buffer.from(yield* fileSystem.readFile(screenshotPath!)).toString()).toBe("png");339 expect(Buffer.from(yield* fileSystem.readFile(screenshotPath!)).toString()).toBe("png");
332 const text = snapshot.content.find((content) => content.type === "text");340 const [, text] = snapshot.content;
333 expect(text?.type === "text" ? text.text : "").toContain(screenshotPath);341 expect(text?.type === "text" ? text.text : "").toContain(screenshotPath);
334342
335 const unsaved = yield* callSnapshot({});343 const unsaved = yield* callSnapshot({});
...@@ -429,7 +437,10 @@ it.effect("keeps the snapshot text under the agent's output ceiling", () =>...@@ -429,7 +437,10 @@ it.effect("keeps the snapshot text under the agent's output ceiling", () =>
429 const snapshot = yield* callSnapshot({ includeImage: false });437 const snapshot = yield* callSnapshot({ includeImage: false });
430438
431 expect(snapshot.isError).toBe(false);439 expect(snapshot.isError).toBe(false);
432 const [text, notice] = snapshot.content;440 const [identity, text, notice] = snapshot.content;
441 expect(identity?.type === "text" ? decodeJsonText(identity.text) : null).toEqual({
442 url: oversized.url,
443 });
433 expect(text?.type).toBe("text");444 expect(text?.type).toBe("text");
434 const body = text?.type === "text" ? text.text : "";445 const body = text?.type === "text" ? text.text : "";
435 expect(Buffer.byteLength(body, "utf8")).toBeLessThanOrEqual(446 expect(Buffer.byteLength(body, "utf8")).toBeLessThanOrEqual(
...@@ -471,7 +482,7 @@ it.effect("bounds the snapshot text even when nothing but logs and the title are...@@ -471,7 +482,7 @@ it.effect("bounds the snapshot text even when nothing but logs and the title are
471482
472 const snapshot = yield* callSnapshot({ includeImage: false });483 const snapshot = yield* callSnapshot({ includeImage: false });
473484
474 const [text] = snapshot.content;485 const [, text] = snapshot.content;
475 const body = text?.type === "text" ? text.text : "";486 const body = text?.type === "text" ? text.text : "";
476 expect(Buffer.byteLength(body, "utf8")).toBeLessThanOrEqual(487 expect(Buffer.byteLength(body, "utf8")).toBeLessThanOrEqual(
477 McpHttpServer.MAX_SNAPSHOT_TEXT_BYTES,488 McpHttpServer.MAX_SNAPSHOT_TEXT_BYTES,
...@@ -482,7 +493,7 @@ it.effect("bounds the snapshot text even when nothing but logs and the title are...@@ -482,7 +493,7 @@ it.effect("bounds the snapshot text even when nothing but logs and the title are
482 };493 };
483 expect(parsed.title.length).toBe(2_049);494 expect(parsed.title.length).toBe(2_049);
484 expect(parsed.consoleEntries[0]?.text.length).toBe(501);495 expect(parsed.consoleEntries[0]?.text.length).toBe(501);
485 const notice = snapshot.content[1];496 const notice = snapshot.content[2];
486 const noticeText = notice?.type === "text" ? notice.text : "";497 const noticeText = notice?.type === "text" ? notice.text : "";
487 expect(noticeText).toContain("url or title after 2048 characters");498 expect(noticeText).toContain("url or title after 2048 characters");
488 expect(noticeText).toContain("console entries text after 500 characters");499 expect(noticeText).toContain("console entries text after 500 characters");
...@@ -533,7 +544,7 @@ it.effect("sheds log entries before locators when every list is full", () =>...@@ -533,7 +544,7 @@ it.effect("sheds log entries before locators when every list is full", () =>
533544
534 const snapshot = yield* callSnapshot({ includeImage: false });545 const snapshot = yield* callSnapshot({ includeImage: false });
535546
536 const [text, notice] = snapshot.content;547 const [, text, notice] = snapshot.content;
537 const body = text?.type === "text" ? text.text : "";548 const body = text?.type === "text" ? text.text : "";
538 expect(Buffer.byteLength(body, "utf8")).toBeLessThanOrEqual(549 expect(Buffer.byteLength(body, "utf8")).toBeLessThanOrEqual(
539 McpHttpServer.MAX_SNAPSHOT_TEXT_BYTES,550 McpHttpServer.MAX_SNAPSHOT_TEXT_BYTES,
...@@ -615,6 +626,10 @@ it.effect("registers annotated tools and preserves authenticated request context...@@ -615,6 +626,10 @@ it.effect("registers annotated tools and preserves authenticated request context
615 Effect.gen(function* () {626 Effect.gen(function* () {
616 const server = yield* McpServer.McpServer;627 const server = yield* McpServer.McpServer;
617 const broker = yield* PreviewAutomationBroker.PreviewAutomationBroker;628 const broker = yield* PreviewAutomationBroker.PreviewAutomationBroker;
629 const toolIcon = {
630 _tag: "website" as const,
631 pageUrl: "http://example.test/",
632 };
618 const routedRequests: Array<{633 const routedRequests: Array<{
619 readonly operation: string;634 readonly operation: string;
620 readonly tabId?: string | undefined;635 readonly tabId?: string | undefined;
...@@ -664,7 +679,7 @@ it.effect("registers annotated tools and preserves authenticated request context...@@ -664,7 +679,7 @@ it.effect("registers annotated tools and preserves authenticated request context
664 expect(clickTool?.tool.annotations?.readOnlyHint).toBe(false);679 expect(clickTool?.tool.annotations?.readOnlyHint).toBe(false);
665 expect(clickTool?.tool.annotations?.destructiveHint).toBe(true);680 expect(clickTool?.tool.annotations?.destructiveHint).toBe(true);
666 expect(clickTool?.tool.annotations?.openWorldHint).toBe(true);681 expect(clickTool?.tool.annotations?.openWorldHint).toBe(true);
667 expect(clickTool?.tool.outputSchema).toEqual({682 expect(clickTool?.tool.outputSchema).toMatchObject({
668 type: "object",683 type: "object",
669 additionalProperties: false,684 additionalProperties: false,
670 description: "The preview action completed successfully.",685 description: "The preview action completed successfully.",
...@@ -721,10 +736,12 @@ it.effect("registers annotated tools and preserves authenticated request context...@@ -721,10 +736,12 @@ it.effect("registers annotated tools and preserves authenticated request context
721 Effect.provideService(McpSchema.McpServerClient, client),736 Effect.provideService(McpSchema.McpServerClient, client),
722 );737 );
723 expect(evaluated.isError).toBe(false);738 expect(evaluated.isError).toBe(false);
724 expect(evaluated.structuredContent).toEqual({ value: ["Connect", "Continue"] });739 expect(evaluated.structuredContent).toEqual({ value: ["Connect", "Continue"], toolIcon });
725 expect(evaluated.content).toEqual([740 const evaluatedText = evaluated.content[0];
726 { type: "text", text: '{"value":["Connect","Continue"]}' },741 expect(evaluatedText?.type === "text" ? decodeJsonText(evaluatedText.text) : null).toEqual({
727 ]);742 toolIcon,
743 value: ["Connect", "Continue"],
744 });
728745
729 const actionRequests = [746 const actionRequests = [
730 { name: "preview_click", arguments: { x: 10, y: 10 } },747 { name: "preview_click", arguments: { x: 10, y: 10 } },
...@@ -741,8 +758,10 @@ it.effect("registers annotated tools and preserves authenticated request context...@@ -741,8 +758,10 @@ it.effect("registers annotated tools and preserves authenticated request context
741 Effect.provideService(McpSchema.McpServerClient, client),758 Effect.provideService(McpSchema.McpServerClient, client),
742 );759 );
743 expect(result.isError).toBe(false);760 expect(result.isError).toBe(false);
744 expect(result.structuredContent).toEqual({});761 expect(result.structuredContent).toEqual({ toolIcon });
745 expect(result.content).toEqual([{ type: "text", text: "{}" }]);762 expect(routedRequests.at(-1)?.operation).toBe("status");
763 const text = result.content[0];
764 expect(text?.type === "text" ? decodeJsonText(text.text) : null).toEqual({ toolIcon });
746 }765 }
747 }),766 }),
748 ).pipe(Effect.provide(TestLayer)),767 ).pipe(Effect.provide(TestLayer)),
apps/server/src/mcp/McpHttpServer.ts+7
...@@ -410,6 +410,13 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot...@@ -410,6 +410,13 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot
410 isError: false,410 isError: false,
411 structuredContent: metadata,411 structuredContent: metadata,
412 content: [412 content: [
413 // Keep the page identity readable even if a provider truncates the snapshot.
414 {
415 type: "text",
416 text: encodeJsonText({
417 url: cutText(snapshot.url, MAX_SNAPSHOT_IDENTIFIER_CHARS),
418 }),
419 },
413 { type: "text", text: bounded.text },420 { type: "text", text: bounded.text },
414 ...(bounded.omitted.length === 0421 ...(bounded.omitted.length === 0
415 ? []422 ? []
apps/server/src/mcp/PreviewAutomationBroker.test.ts+61-41
...@@ -127,50 +127,70 @@ it.effect("targets multiple tabs explicitly while retaining a default tab", () =...@@ -127,50 +127,70 @@ it.effect("targets multiple tabs explicitly while retaining a default tab", () =
127 ),127 ),
128);128);
129129
130it.effect("does not let an older response replace a newer explicit tab target", () =>130it.effect.each([true, false])(
131 Effect.scoped(131 "keeps an older target stable while a newer explicit tab responds (implicit: %s)",
132 Effect.gen(function* () {132 (implicit) =>
133 const broker = yield* makeBroker;133 Effect.scoped(
134 const olderTabId = PreviewTabId.make("tab-older-request");134 Effect.gen(function* () {
135 const newerTabId = PreviewTabId.make("tab-newer-request");135 const broker = yield* makeBroker;
136 const releaseOlderResponse = yield* Deferred.make<void>();136 const olderTabId = PreviewTabId.make("tab-older-request");
137 const routedRequests: RoutedRequest[] = [];137 const newerTabId = PreviewTabId.make("tab-newer-request");
138 const requests = requestsFrom(yield* broker.connect(makeHost()));138 const releaseOlderResponse = yield* Deferred.make<void>();
139 yield* Stream.runForEach(requests, (request) => {139 const routedRequests: RoutedRequest[] = [];
140 routedRequests.push(request);140 const requests = requestsFrom(yield* broker.connect(makeHost()));
141 const response = Effect.gen(function* () {141 yield* Stream.runForEach(requests, (request) => {
142 if (request.tabId === olderTabId) {142 routedRequests.push(request);
143 yield* Deferred.await(releaseOlderResponse);143 const response = Effect.gen(function* () {
144 }144 if (request.tabId === olderTabId && request.operation === "snapshot") {
145 yield* broker.respond({145 yield* Deferred.await(releaseOlderResponse);
146 clientId: "client-1",146 }
147 connectionId: request.connectionId,147 yield* broker.respond({
148 requestId: request.requestId,148 clientId: "client-1",
149 ok: true,149 connectionId: request.connectionId,
150 result: { url: "http://localhost:3200" },150 requestId: request.requestId,
151 ok: true,
152 result: { url: "http://localhost:3200" },
153 });
154 if (request.tabId === newerTabId) {
155 yield* Deferred.succeed(releaseOlderResponse, undefined);
156 }
151 });157 });
152 if (request.tabId === newerTabId) {158 return response.pipe(Effect.forkScoped, Effect.asVoid);
153 yield* Deferred.succeed(releaseOlderResponse, undefined);159 }).pipe(Effect.forkScoped);
154 }160 yield* Effect.yieldNow;
161
162 yield* broker.invoke({ scope, operation: "status", input: {}, tabId: olderTabId });
163 let capturedTabId: PreviewTabId | undefined;
164 const older = yield* broker
165 .invoke({
166 scope,
167 operation: "snapshot",
168 input: {},
169 ...(implicit ? {} : { tabId: olderTabId }),
170 onTargetTab: (tabId) => {
171 capturedTabId = tabId;
172 },
173 })
174 .pipe(Effect.forkScoped);
175 yield* Effect.yieldNow;
176 const newer = yield* broker
177 .invoke({ scope, operation: "snapshot", input: {}, tabId: newerTabId })
178 .pipe(Effect.forkScoped);
179 yield* Fiber.join(newer);
180 yield* Fiber.join(older);
181 yield* broker.invoke({
182 scope,
183 operation: "status",
184 input: {},
185 tabId: olderTabId,
186 updateCurrentTab: false,
155 });187 });
156 return response.pipe(Effect.forkScoped, Effect.asVoid);188 yield* broker.invoke({ scope, operation: "snapshot", input: {} });
157 }).pipe(Effect.forkScoped);
158 yield* Effect.yieldNow;
159
160 const older = yield* broker
161 .invoke({ scope, operation: "snapshot", input: {}, tabId: olderTabId })
162 .pipe(Effect.forkScoped);
163 yield* Effect.yieldNow;
164 const newer = yield* broker
165 .invoke({ scope, operation: "snapshot", input: {}, tabId: newerTabId })
166 .pipe(Effect.forkScoped);
167 yield* Fiber.join(newer);
168 yield* Fiber.join(older);
169 yield* broker.invoke({ scope, operation: "snapshot", input: {} });
170189
171 expect(routedRequests.at(-1)?.tabId).toBe(newerTabId);190 expect(routedRequests.at(-1)?.tabId).toBe(newerTabId);
172 }),191 expect(capturedTabId).toBe(olderTabId);
173 ),192 }),
193 ),
174);194);
175195
176it.effect("tracks the tab returned by a targeted recording stop", () =>196it.effect("tracks the tab returned by a targeted recording stop", () =>
apps/server/src/mcp/PreviewAutomationBroker.ts+6
...@@ -44,6 +44,10 @@ export interface PreviewAutomationInvokeInput {...@@ -44,6 +44,10 @@ export interface PreviewAutomationInvokeInput {
44 readonly input: unknown;44 readonly input: unknown;
45 readonly tabId?: PreviewTabId;45 readonly tabId?: PreviewTabId;
46 readonly timeoutMs?: number;46 readonly timeoutMs?: number;
47 /** Background metadata reads must not change the agent's current tab. */
48 readonly updateCurrentTab?: boolean;
49 /** Capture the routed tab before another request changes the current assignment. */
50 readonly onTargetTab?: (tabId: PreviewTabId | undefined) => void;
47}51}
4852
49export class PreviewAutomationBroker extends Context.Service<53export class PreviewAutomationBroker extends Context.Service<
...@@ -541,6 +545,7 @@ export const make = Effect.gen(function* PreviewAutomationBrokerMake() {...@@ -541,6 +545,7 @@ export const make = Effect.gen(function* PreviewAutomationBrokerMake() {
541 });545 });
542 }546 }
543 const { connection, requestId, requestContext, requestSequence } = route;547 const { connection, requestId, requestContext, requestSequence } = route;
548 input.onTargetTab?.(requestContext.tabId);
544 const removePending = SynchronizedRef.update(state, (next) => {549 const removePending = SynchronizedRef.update(state, (next) => {
545 if (!next.pending.has(requestId)) return next;550 if (!next.pending.has(requestId)) return next;
546 const pending = new Map(next.pending);551 const pending = new Map(next.pending);
...@@ -575,6 +580,7 @@ export const make = Effect.gen(function* PreviewAutomationBrokerMake() {...@@ -575,6 +580,7 @@ export const make = Effect.gen(function* PreviewAutomationBrokerMake() {
575 });580 });
576 });581 });
577 const result = yield* awaitResponse().pipe(Effect.ensuring(removePending));582 const result = yield* awaitResponse().pipe(Effect.ensuring(removePending));
583 if (input.updateCurrentTab === false) return result;
578 const responseTabId = readResultTabId(result);584 const responseTabId = readResultTabId(result);
579 const resultTabId = responseTabId === undefined ? input.tabId : responseTabId;585 const resultTabId = responseTabId === undefined ? input.tabId : responseTabId;
580 if (resultTabId === undefined) return result;586 if (resultTabId === undefined) return result;
apps/server/src/mcp/toolkits/preview/handlers.ts+52-17
...@@ -7,6 +7,7 @@ import {...@@ -7,6 +7,7 @@ import {
7 PreviewAutomationRecordingTransferError,7 PreviewAutomationRecordingTransferError,
8 PreviewAutomationRecordingDesktopUpdateRequiredError,8 PreviewAutomationRecordingDesktopUpdateRequiredError,
9 PreviewAutomationRecordingArtifact,9 PreviewAutomationRecordingArtifact,
10 type ToolActivityIcon,
10 type ThreadId,11 type ThreadId,
11 type PreviewAutomationOperation,12 type PreviewAutomationOperation,
12 type PreviewAutomationOpenInput,13 type PreviewAutomationOpenInput,
...@@ -55,22 +56,47 @@ const invoke = Effect.fn("PreviewToolkit.invoke")(function* <A>(...@@ -55,22 +56,47 @@ const invoke = Effect.fn("PreviewToolkit.invoke")(function* <A>(
55 timeoutMs?: number,56 timeoutMs?: number,
56 tabId?: PreviewTabId,57 tabId?: PreviewTabId,
57): Effect.fn.Return<58): Effect.fn.Return<
58 A,59 { result: A; toolIcon?: ToolActivityIcon },
59 import("@t3tools/contracts").PreviewAutomationError,60 import("@t3tools/contracts").PreviewAutomationError,
60 McpInvocationContext.McpInvocationContext | PreviewAutomationBroker.PreviewAutomationBroker61 McpInvocationContext.McpInvocationContext | PreviewAutomationBroker.PreviewAutomationBroker
61> {62> {
62 const scope = yield* McpInvocationContext.requireMcpCapability("preview");63 const scope = yield* McpInvocationContext.requireMcpCapability("preview");
63 const broker = yield* PreviewAutomationBroker.PreviewAutomationBroker;64 const broker = yield* PreviewAutomationBroker.PreviewAutomationBroker;
64 return yield* broker.invoke<A>({65 let targetTabId = tabId;
66 const result = yield* broker.invoke<A>({
67 onTargetTab: (resolvedTabId) => {
68 targetTabId = resolvedTabId;
69 },
65 scope,70 scope,
66 operation,71 operation,
67 input,72 input,
68 ...(timeoutMs === undefined ? {} : { timeoutMs }),73 ...(timeoutMs === undefined ? {} : { timeoutMs }),
69 ...(tabId === undefined ? {} : { tabId }),74 ...(tabId === undefined ? {} : { tabId }),
70 });75 });
76 if (["status", "open", "navigate", "snapshot"].includes(operation)) return { result };
77 const statusTabId =
78 (operation !== "evaluate" && typeof result === "object" && result !== null
79 ? (result as { tabId?: PreviewTabId }).tabId
80 : undefined) ?? targetTabId;
81 const page = yield* broker
82 .invoke<PreviewAutomationStatus>({
83 scope,
84 operation: "status",
85 input: {},
86 timeoutMs: 500,
87 updateCurrentTab: false,
88 ...(statusTabId === undefined ? {} : { tabId: statusTabId }),
89 })
90 .pipe(Effect.catch(() => Effect.succeed(null)));
91 return {
92 result,
93 ...(page?.url && /^https?:\/\//i.test(page.url) && page.url.length <= 4096
94 ? { toolIcon: { _tag: "website" as const, pageUrl: page.url } }
95 : {}),
96 };
71});97});
7298
73const invokeTargeted = <A>(99const invokeTargeted = <A extends object>(
74 operation: PreviewAutomationOperation,100 operation: PreviewAutomationOperation,
75 input: {101 input: {
76 readonly tabId?: PreviewTabId | undefined;102 readonly tabId?: PreviewTabId | undefined;
...@@ -79,7 +105,12 @@ const invokeTargeted = <A>(...@@ -79,7 +105,12 @@ const invokeTargeted = <A>(
79 timeoutMs?: number,105 timeoutMs?: number,
80) => {106) => {
81 const { tabId, ...operationInput } = input;107 const { tabId, ...operationInput } = input;
82 return invoke<A>(operation, operationInput, timeoutMs, tabId);108 return invoke<A>(operation, operationInput, timeoutMs, tabId).pipe(
109 Effect.map(({ result, toolIcon }) => ({
110 ...result,
111 ...(toolIcon ? { toolIcon } : {}),
112 })),
113 );
83};114};
84115
85const UploadedRecordingArtifact = Schema.Struct({116const UploadedRecordingArtifact = Schema.Struct({
...@@ -170,28 +201,32 @@ const handlers = {...@@ -170,28 +201,32 @@ const handlers = {
170 const { includeImage: _includeImage, save: _save, ...operationInput } = input ?? {};201 const { includeImage: _includeImage, save: _save, ...operationInput } = input ?? {};
171 return invokeTargeted<PreviewAutomationSnapshot>("snapshot", operationInput);202 return invokeTargeted<PreviewAutomationSnapshot>("snapshot", operationInput);
172 },203 },
173 preview_click: (input) =>204 preview_click: (input) => invokeTargeted<object>("click", input, input.timeoutMs),
174 invokeTargeted<void>("click", input, input.timeoutMs).pipe(Effect.as({})),205 preview_type: (input) => invokeTargeted<object>("type", input, input.timeoutMs),
175 preview_type: (input) => invokeTargeted<void>("type", input, input.timeoutMs).pipe(Effect.as({})),206 preview_press: (input) => invokeTargeted<object>("press", input),
176 preview_press: (input) => invokeTargeted<void>("press", input).pipe(Effect.as({})),207 preview_scroll: (input) => invokeTargeted<object>("scroll", input),
177 preview_scroll: (input) => invokeTargeted<void>("scroll", input).pipe(Effect.as({})),208 preview_evaluate: ({ tabId, ...input }) =>
178 preview_evaluate: (input) =>209 invoke<unknown>("evaluate", input, undefined, tabId).pipe(
179 invokeTargeted<unknown>("evaluate", input).pipe(210 Effect.map(({ result, toolIcon }) => ({
180 Effect.map((result) => ({ value: result ?? null })),211 value: result ?? null,
212 ...(toolIcon ? { toolIcon } : {}),
213 })),
181 ),214 ),
182 preview_wait_for: (input) =>215 preview_wait_for: (input) => invokeTargeted<object>("waitFor", input, input.timeoutMs),
183 invokeTargeted<void>("waitFor", input, input.timeoutMs).pipe(Effect.as({})),
184 preview_recording_start: (input) =>216 preview_recording_start: (input) =>
185 invokeTargeted<PreviewAutomationRecordingStatus>("recordingStart", input ?? {}),217 invokeTargeted<PreviewAutomationRecordingStatus>("recordingStart", input ?? {}),
186 preview_recording_stop: (input) =>218 preview_recording_stop: (input) =>
187 Effect.gen(function* () {219 Effect.gen(function* () {
188 const scope = yield* McpInvocationContext.requireMcpCapability("preview");220 const scope = yield* McpInvocationContext.requireMcpCapability("preview");
189 const response = yield* invokeTargeted<unknown>(221 const { tabId, ...operationInput } = input;
222 const response = yield* invoke<unknown>(
190 "recordingStop",223 "recordingStop",
191 { ...input, transferToEnvironment: true },224 { ...operationInput, transferToEnvironment: true },
192 PREVIEW_RECORDING_STOP_TIMEOUT_MS,225 PREVIEW_RECORDING_STOP_TIMEOUT_MS,
226 tabId,
193 );227 );
194 return yield* claimPreviewRecording(scope.threadId, response);228 const artifact = yield* claimPreviewRecording(scope.threadId, response.result);
229 return { ...artifact, ...(response.toolIcon ? { toolIcon: response.toolIcon } : {}) };
195 }),230 }),
196} satisfies Parameters<typeof PreviewToolkit.toLayer>[0];231} satisfies Parameters<typeof PreviewToolkit.toLayer>[0];
197232
apps/server/src/mcp/toolkits/preview/tools.test.ts+1
...@@ -67,6 +67,7 @@ it("exports exact object result schemas for preview actions", () => {...@@ -67,6 +67,7 @@ it("exports exact object result schemas for preview actions", () => {
67 for (const name of actionNames) {67 for (const name of actionNames) {
68 expect(Tool.getJsonSchemaFromSchema(PreviewToolkit.tools[name].successSchema)).toEqual({68 expect(Tool.getJsonSchemaFromSchema(PreviewToolkit.tools[name].successSchema)).toEqual({
69 type: "object",69 type: "object",
70 properties: { toolIcon: expect.any(Object) },
70 additionalProperties: false,71 additionalProperties: false,
71 description: "The preview action completed successfully.",72 description: "The preview action completed successfully.",
72 });73 });
apps/server/src/mcp/toolkits/preview/tools.ts+12-5
...@@ -1,4 +1,5 @@...@@ -1,4 +1,5 @@
1import {1import {
2 ToolActivityIcon,
2 PreviewAutomationClickInput,3 PreviewAutomationClickInput,
3 PreviewAutomationError,4 PreviewAutomationError,
4 PreviewAutomationEvaluateInput,5 PreviewAutomationEvaluateInput,
...@@ -31,7 +32,9 @@ const dependencies = [...@@ -31,7 +32,9 @@ const dependencies = [
31 PreviewAutomationBroker.PreviewAutomationBroker,32 PreviewAutomationBroker.PreviewAutomationBroker,
32];33];
3334
34const PreviewActionResult = Schema.Record(Schema.String, Schema.Never).annotate({35const presentationFields = { toolIcon: Schema.optional(ToolActivityIcon) };
36
37const PreviewActionResult = Schema.Struct(presentationFields).annotate({
35 description: "The preview action completed successfully.",38 description: "The preview action completed successfully.",
36});39});
3740
...@@ -89,7 +92,7 @@ const PreviewResizeTool = safeBrowserTool(...@@ -89,7 +92,7 @@ const PreviewResizeTool = safeBrowserTool(
89 description:92 description:
90 "Resize a collaborative browser tab, optionally selected by tabId. Use {mode:'fill'}, {mode:'freeform',width:1024,height:768}, or {mode:'preset',preset:'iphone-12-pro',orientation:'portrait'}. This changes CSS layout breakpoints without changing the desktop browser user agent.",93 "Resize a collaborative browser tab, optionally selected by tabId. Use {mode:'fill'}, {mode:'freeform',width:1024,height:768}, or {mode:'preset',preset:'iphone-12-pro',orientation:'portrait'}. This changes CSS layout breakpoints without changing the desktop browser user agent.",
91 parameters: PreviewAutomationResizeInput,94 parameters: PreviewAutomationResizeInput,
92 success: PreviewAutomationResizeResult,95 success: Schema.Struct({ ...PreviewAutomationResizeResult.fields, ...presentationFields }),
93 failure: PreviewAutomationError,96 failure: PreviewAutomationError,
94 dependencies,97 dependencies,
95 })98 })
...@@ -102,7 +105,10 @@ const PreviewSetAppearanceTool = safeBrowserTool(...@@ -102,7 +105,10 @@ const PreviewSetAppearanceTool = safeBrowserTool(
102 description:105 description:
103 "Emulate prefers-color-scheme in a collaborative browser tab, optionally selected by tabId. Use {colorScheme:'dark'} or {colorScheme:'light'} to preview the page in that appearance, and {colorScheme:'system'} to clear the override and follow the OS appearance.",106 "Emulate prefers-color-scheme in a collaborative browser tab, optionally selected by tabId. Use {colorScheme:'dark'} or {colorScheme:'light'} to preview the page in that appearance, and {colorScheme:'system'} to clear the override and follow the OS appearance.",
104 parameters: PreviewAutomationSetColorSchemeInput,107 parameters: PreviewAutomationSetColorSchemeInput,
105 success: PreviewAutomationSetColorSchemeResult,108 success: Schema.Struct({
109 ...PreviewAutomationSetColorSchemeResult.fields,
110 ...presentationFields,
111 }),
106 failure: PreviewAutomationError,112 failure: PreviewAutomationError,
107 dependencies,113 dependencies,
108 })114 })
...@@ -185,6 +191,7 @@ const PreviewScrollTool = safeBrowserTool(...@@ -185,6 +191,7 @@ const PreviewScrollTool = safeBrowserTool(
185 * null valid instead of failing only for non-object expressions.191 * null valid instead of failing only for non-object expressions.
186 */192 */
187export const PreviewEvaluateResult = Schema.Struct({193export const PreviewEvaluateResult = Schema.Struct({
194 ...presentationFields,
188 value: Schema.Unknown.annotate({195 value: Schema.Unknown.annotate({
189 description: "The JSON-serializable value the expression produced, or null.",196 description: "The JSON-serializable value the expression produced, or null.",
190 }),197 }),
...@@ -217,7 +224,7 @@ const PreviewRecordingStartTool = safeBrowserTool(...@@ -217,7 +224,7 @@ const PreviewRecordingStartTool = safeBrowserTool(
217 description:224 description:
218 "Start recording the collaborative browser tab selected by tabId, or this agent session's current tab when omitted.",225 "Start recording the collaborative browser tab selected by tabId, or this agent session's current tab when omitted.",
219 parameters: PreviewAutomationTabTargetInput,226 parameters: PreviewAutomationTabTargetInput,
220 success: PreviewAutomationRecordingStatus,227 success: Schema.Struct({ ...PreviewAutomationRecordingStatus.fields, ...presentationFields }),
221 failure: PreviewAutomationError,228 failure: PreviewAutomationError,
222 dependencies,229 dependencies,
223 }).annotate(Tool.Title, "Start browser recording"),230 }).annotate(Tool.Title, "Start browser recording"),
...@@ -228,7 +235,7 @@ const PreviewRecordingStopTool = safeBrowserTool(...@@ -228,7 +235,7 @@ const PreviewRecordingStopTool = safeBrowserTool(
228 description:235 description:
229 "Stop recording the collaborative browser tab selected by tabId, or this agent session's current tab when omitted, and transfer the compressed recording once (up to 50 MiB) to an evidence file readable in this agent's environment. Returns its environment-local path after transfer succeeds.",236 "Stop recording the collaborative browser tab selected by tabId, or this agent session's current tab when omitted, and transfer the compressed recording once (up to 50 MiB) to an evidence file readable in this agent's environment. Returns its environment-local path after transfer succeeds.",
230 parameters: PreviewAutomationTabTargetInput,237 parameters: PreviewAutomationTabTargetInput,
231 success: PreviewAutomationRecordingArtifact,238 success: Schema.Struct({ ...PreviewAutomationRecordingArtifact.fields, ...presentationFields }),
232 failure: PreviewAutomationError,239 failure: PreviewAutomationError,
233 dependencies: [...dependencies, FileSystem.FileSystem, ServerConfig.ServerConfig],240 dependencies: [...dependencies, FileSystem.FileSystem, ServerConfig.ServerConfig],
234 }).annotate(Tool.Title, "Stop browser recording"),241 }).annotate(Tool.Title, "Stop browser recording"),
apps/server/src/orchestration/ActivityPayloadProjection.test.ts+78
...@@ -249,6 +249,84 @@ describe("projectActivityPayload", () => {...@@ -249,6 +249,84 @@ describe("projectActivityPayload", () => {
249 expect(JSON.stringify(projected.payload).length).toBeLessThan(500);249 expect(JSON.stringify(projected.payload).length).toBeLessThan(500);
250 });250 });
251251
252 it.each([
253 {
254 item: {
255 server: "t3-code",
256 tool: "preview_open",
257 result: { structuredContent: { url: "https://example.com/" } },
258 },
259 },
260 {
261 toolName: "mcp__t3-code__preview_navigate",
262 result: { content: '{"url":"https://example.com/"}' },
263 },
264 { tool: "t3-code_preview_status", state: { output: '{"url":"https://example.com/"}' } },
265 {
266 toolName: "mcp__t3_code__preview_snapshot",
267 result: {
268 content: [
269 { type: "text", text: '{"url":"https://example.com/"}' },
270 { type: "text", text: "Snapshot text was bounded. Omitted: accessibilityTree." },
271 ],
272 },
273 },
274 {
275 toolName: "mcp__t3-code__preview_click",
276 result: { content: '{"toolIcon":{"_tag":"website","pageUrl":"https://example.com/"}}' },
277 },
278 {
279 toolName: "mcp__t3_code__preview_snapshot",
280 result: { content: '{"url":"https://example.com/"}\n{"accessibilityTree":"truncated' },
281 },
282 ...[false, true].map((truncated) => ({
283 toolName: "mcp__t3_code__preview_snapshot",
284 result: {
285 content: JSON.stringify({
286 content: [{ type: "text", text: '{"url":"https://example.com/"}' }],
287 structuredContent: { url: "https://example.com/", visibleText: "page" },
288 }).slice(0, truncated ? -5 : undefined),
289 },
290 })),
291 ...[
292 "type",
293 "press",
294 "scroll",
295 "resize",
296 "set_appearance",
297 "evaluate",
298 "wait_for",
299 "recording_start",
300 "recording_stop",
301 ].map((action) => ({
302 toolName: `mcp__t3_code__preview_${action}`,
303 result: { content: '{"toolIcon":{"_tag":"website","pageUrl":"https://example.com/"}}' },
304 })),
305 ])("preserves the preview page favicon through result slimming", (data) => {
306 const projected = projectActivityPayload(activity({ itemType: "mcp_tool_call", data }));
307 const icon = { _tag: "website", pageUrl: "https://example.com/" };
308 expect(projected.payload).toMatchObject({ toolIcon: icon });
309 expect(projectActivityPayload(projected).payload).toMatchObject({ toolIcon: icon });
310 });
311
312 it.each([
313 { toolName: "mcp__other__preview_open", result: { content: '{"url":"https://example.com/"}' } },
314 {
315 toolName: "mcp__t3-code__preview_evaluate",
316 result: { content: '{"url":"https://example.com/"}' },
317 },
318 {
319 toolName: "mcp__t3-code__preview_open",
320 result: { isError: true, content: '{"url":"https://example.com/"}' },
321 },
322 { toolName: "mcp__t3-code__preview_open", result: { content: "malformed JSON" } },
323 { toolName: "mcp__t3-code__preview_open", result: { content: '{"url":"about:blank"}' } },
324 ])("keeps the fallback for unrelated tools, failed navigation, and missing page URLs", (data) => {
325 expect(
326 projectActivityPayload(activity({ itemType: "mcp_tool_call", data })).payload,
327 ).not.toHaveProperty("toolIcon");
328 });
329
252 it("passes task lifecycle payloads (no data field) through untouched", () => {330 it("passes task lifecycle payloads (no data field) through untouched", () => {
253 const source = activity({331 const source = activity({
254 taskId: "task-9",332 taskId: "task-9",
apps/server/src/orchestration/ActivityPayloadProjection.ts+70-1
...@@ -5,6 +5,7 @@ import type {...@@ -5,6 +5,7 @@ import type {
5 OrchestrationThreadDetailSnapshot,5 OrchestrationThreadDetailSnapshot,
6} from "@t3tools/contracts";6} from "@t3tools/contracts";
7import { isWorkspaceImagePreviewPath } from "@t3tools/shared/filePreview";7import { isWorkspaceImagePreviewPath } from "@t3tools/shared/filePreview";
8import { extractJsonObject } from "@t3tools/shared/schemaJson";
89
9function asRecord(value: unknown): Record<string, unknown> | null {10function asRecord(value: unknown): Record<string, unknown> | null {
10 return value !== null && typeof value === "object" && !Array.isArray(value)11 return value !== null && typeof value === "object" && !Array.isArray(value)
...@@ -241,6 +242,70 @@ function summarizeMcpResult(result: unknown): Record<string, unknown> | undefine...@@ -241,6 +242,70 @@ function summarizeMcpResult(result: unknown): Record<string, unknown> | undefine
241 return summary ? { content: summary } : undefined;242 return summary ? { content: summary } : undefined;
242}243}
243244
245/** Reuse the page URL already returned by preview tools before slimming their output. */
246function projectPreviewToolMetadata(data: Record<string, unknown>, status: unknown) {
247 const item = asRecord(data.item);
248 const name = item ? `mcp__${item.server}__${item.tool}` : (data.toolName ?? data.tool);
249 if (
250 typeof name !== "string" ||
251 !/^(?:mcp__)?(?:t3-code|t3_code|t3code)_{1,2}preview_(?:open|navigate|status|snapshot|click|type|press|scroll|resize|set_appearance|evaluate|wait_for|recording_start|recording_stop)$/.test(
252 name,
253 )
254 )
255 return {};
256 const state = asRecord(data.state);
257 const result = item?.result ?? data.result ?? state?.output;
258 const record = asRecord(result);
259 if (
260 status === "failed" ||
261 status === "declined" ||
262 state?.status === "error" ||
263 item?.error != null ||
264 record?.isError === true ||
265 record?.is_error === true
266 )
267 return {};
268
269 let page = record;
270 let output: unknown = result;
271 for (let depth = 0; depth < 3; depth += 1) {
272 if (page?.isError === true || page?.is_error === true) return {};
273 const structured = asRecord(page?.structuredContent);
274 if (structured) {
275 page = structured;
276 break;
277 }
278 const text = extractMcpResultText(output)?.slice(0, 2 * 1024 * 1024);
279 if (!text) break;
280 try {
281 page = asRecord(JSON.parse(extractJsonObject(text)));
282 } catch {
283 // A truncated MCP envelope can still contain a complete first text block.
284 const firstBlock = /^\s*\{\s*"content"\s*:\s*\[\s*/.exec(text);
285 if (!firstBlock) return {};
286 try {
287 const block = asRecord(JSON.parse(extractJsonObject(text.slice(firstBlock[0].length))));
288 page = block?.type === "text" ? { content: [block] } : null;
289 } catch {
290 return {};
291 }
292 }
293 output = page;
294 }
295 const rawUrl = asTrimmedString(
296 asRecord(page?.toolIcon)?.pageUrl ??
297 (/preview_(?:open|navigate|status|snapshot)$/.test(name) ? page?.url : undefined),
298 );
299 if (!rawUrl || rawUrl.length > 4096) return {};
300 try {
301 const url = new URL(rawUrl);
302 if (url.protocol !== "http:" && url.protocol !== "https:") return {};
303 return { toolIcon: { _tag: "website", pageUrl: url.href } };
304 } catch {
305 return {};
306 }
307}
308
244/**309/**
245 * MCP tool calls carry full tool results (`data.item.result` on Codex,310 * MCP tool calls carry full tool results (`data.item.result` on Codex,
246 * `data.result` on Claude/OpenCode) that used to bypass slimming entirely to311 * `data.result` on Claude/OpenCode) that used to bypass slimming entirely to
...@@ -367,10 +432,14 @@ export function projectActivityPayload(...@@ -367,10 +432,14 @@ export function projectActivityPayload(
367 }432 }
368433
369 const itemStatus = asRecord(data.item)?.status;434 const itemStatus = asRecord(data.item)?.status;
370 const projectedPayload =435 const statusPayload =
371 payload.status === "completed" && (itemStatus === "failed" || itemStatus === "declined")436 payload.status === "completed" && (itemStatus === "failed" || itemStatus === "declined")
372 ? { ...payload, status: itemStatus }437 ? { ...payload, status: itemStatus }
373 : payload;438 : payload;
439 const projectedPayload = {
440 ...projectPreviewToolMetadata(data, statusPayload.status),
441 ...statusPayload,
442 };
374 const questionInput = projectQuestionToolInput(data, payload.title);443 const questionInput = projectQuestionToolInput(data, payload.title);
375444
376 if (payload.itemType === "mcp_tool_call") {445 if (payload.itemType === "mcp_tool_call") {