1import {
2 OrchestratorMcpFailure,
3 ToolActivityIcon,
4 PreviewAutomationClickInput,
5 PreviewAutomationDialogInput,
6 PreviewAutomationDragInput,
7 PreviewAutomationError,
8 PreviewAutomationEvaluateInput,
9 PreviewAutomationHoverInput,
10 PreviewAutomationNavigateInput,
11 PreviewAutomationOpenInput,
12 PreviewAutomationPressInput,
13 PreviewAutomationRecordingArtifact,
14 PreviewAutomationRecordingStatus,
15 PreviewAutomationResizeInput,
16 PreviewAutomationResizeResult,
17 PreviewAutomationScrollInput,
18 PreviewAutomationSelectInput,
19 PreviewAutomationSelectResult,
20 PreviewAutomationSetColorSchemeInput,
21 PreviewAutomationSetColorSchemeResult,
22 PreviewAutomationSnapshot,
23 PreviewAutomationStatus,
24 PreviewAutomationTabTargetInput,
25 PreviewAutomationTypeInput,
26 PreviewAutomationUploadInput,
27 PreviewAutomationWaitForInput,
28} from "@t3tools/contracts";
29import * as Schema from "effect/Schema";
30import * as FileSystem from "effect/FileSystem";
31import { Tool, Toolkit } from "effect/ai";
32
33import * as McpInvocationContext from "../../McpInvocationContext.ts";
34import * as PreviewAutomationBroker from "../../PreviewAutomationBroker.ts";
35import * as ServerConfig from "../../../config.ts";
36import * as ThreadManagementService from "../../../orchestration-v2/ThreadManagementService.ts";
37
38const dependencies = [
39 McpInvocationContext.McpInvocationContext,
40 ThreadManagementService.ThreadManagementService,
41 PreviewAutomationBroker.PreviewAutomationBroker,
42];
43
44/** What a browser action fails with, including the access gate's refusal. */
45const PreviewToolFailure = Schema.Union([PreviewAutomationError, OrchestratorMcpFailure]);
46
47const presentationFields = { toolIcon: Schema.optional(ToolActivityIcon) };
48
49const PreviewActionResult = Schema.Struct(presentationFields).annotate({
50 description: "The preview action completed successfully.",
51});
52
53/** Drives the real browser and can destroy page state. */
54const browserTool = <T extends Tool.Any>(tool: T): T =>
55 tool.annotate(Tool.OpenWorld, true).annotate(Tool.Destructive, true) as T;
56
57/** Same open-world browser access, but the action does not destroy page state. */
58const safeBrowserTool = <T extends Tool.Any>(tool: T): T =>
59 tool.annotate(Tool.OpenWorld, true).annotate(Tool.Destructive, false) as T;
60
61/** A safe browser action that only observes, so it is also repeatable. */
62const readonlyBrowserTool = <T extends Tool.Any>(tool: T): T =>
63 safeBrowserTool(tool).annotate(Tool.Readonly, true).annotate(Tool.Idempotent, true) as T;
64
65const PreviewStatusTool = Tool.make("preview_status", {
66 description:
67 "Report whether a collaborative browser tab is automation-capable, including its control owner, pending dialog, URL, title, visibility, loading state, viewport mode, and measured CSS-pixel size. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab, or the tab the user is viewing when you have none. Server hosts also list every tab in the thread (tabs) with its owner, including tabs the user opened, and the browser profiles preview_open accepts. You can read any listed tab; act only on your own tabs, or on an unclaimed tab while no human controls it.",
68 parameters: PreviewAutomationTabTargetInput,
69 success: PreviewAutomationStatus,
70 failure: PreviewToolFailure,
71 dependencies,
72})
73 .annotate(Tool.Title, "Get preview status")
74 .annotate(Tool.Readonly, true)
75 .annotate(Tool.Destructive, false)
76 .annotate(Tool.Idempotent, true);
77
78const PreviewOpenTool = browserTool(
79 Tool.make("preview_open", {
80 description:
81 "Initialize a collaborative browser tab and open its thread-bound inline preview by default. Set open=false for background-only automation. Pass tabId to reuse a specific existing tab, set reuseExistingTab=false to create another tab, or omit both to use this agent session's current tab. Parallel subagents sharing a provider session must each open with reuseExistingTab=false and pass their returned tabId on every call. Pass profileId (an id or name from preview_status profiles) to open under a browser profile and its saved logins; omit it for the user's default profile. Another agent session's tabs can be read but not operated.",
82 parameters: PreviewAutomationOpenInput,
83 success: PreviewAutomationStatus,
84 failure: PreviewToolFailure,
85 dependencies,
86 })
87 .annotate(Tool.Title, "Open browser preview")
88 .annotate(Tool.Destructive, false),
89);
90
91const PreviewDialogTool = browserTool(
92 Tool.make("preview_dialog", {
93 description:
94 "Accept or dismiss the server browser dialog reported by preview_status. For a prompt, supply promptText when accepting. Requires this agent to own the tab, or the tab to be unclaimed with no human in control. Desktop hosts may not support this operation.",
95 parameters: PreviewAutomationDialogInput,
96 success: PreviewAutomationStatus,
97 failure: PreviewToolFailure,
98 dependencies,
99 }).annotate(Tool.Title, "Resolve browser dialog"),
100);
101
102const PreviewNavigateTool = safeBrowserTool(
103 Tool.make("preview_navigate", {
104 description:
105 "Navigate a collaborative browser tab. Pass tabId to target a specific tab, plus {url:'https://t3.chat'} for a website or {target:{kind:'environment-port',port:5173}} for a dev server. Exactly one of url or target is required.",
106 parameters: PreviewAutomationNavigateInput,
107 success: PreviewAutomationStatus,
108 failure: PreviewToolFailure,
109 dependencies,
110 }).annotate(Tool.Title, "Navigate browser preview"),
111);
112
113const PreviewResizeTool = safeBrowserTool(
114 Tool.make("preview_resize", {
115 description:
116 "Resize a collaborative browser tab, optionally selected by tabId. Use {mode:'fill'}, {mode:'freeform',width:1024,height:768}, or {mode:'preset',preset:'iphone-12-pro',orientation:'portrait'}. This changes CSS layout breakpoints without changing the desktop browser user agent.",
117 parameters: PreviewAutomationResizeInput,
118 success: Schema.Struct({ ...PreviewAutomationResizeResult.fields, ...presentationFields }),
119 failure: PreviewToolFailure,
120 dependencies,
121 })
122 .annotate(Tool.Title, "Resize browser viewport")
123 .annotate(Tool.Idempotent, true),
124);
125
126const PreviewSetAppearanceTool = safeBrowserTool(
127 Tool.make("preview_set_appearance", {
128 description:
129 "Emulate prefers-color-scheme in a collaborative browser tab, optionally selected by tabId. Use {colorScheme:'dark'} or {colorScheme:'light'} to preview the page in that appearance, and {colorScheme:'system'} to clear the override and follow the OS appearance.",
130 parameters: PreviewAutomationSetColorSchemeInput,
131 success: Schema.Struct({
132 ...PreviewAutomationSetColorSchemeResult.fields,
133 ...presentationFields,
134 }),
135 failure: PreviewToolFailure,
136 dependencies,
137 })
138 .annotate(Tool.Title, "Set preview appearance")
139 .annotate(Tool.Idempotent, true),
140);
141
142// Not read-only: save=true writes the screenshot to disk.
143export const PreviewSnapshotTool = safeBrowserTool(
144 Tool.make("preview_snapshot", {
145 description:
146 "Inspect a page before interacting. Pass tabId to inspect a specific tab, including one the user opened (see preview_status tabs), even while they control it; omit it to use this agent session's current tab, or the tab the user is viewing when you have none. Returns page state, semantic elements, diagnostics, action history, and screenshot dimensions. Server snapshots include an accessibilityTree with refs; pass locator=aria-ref=<ref> to target one exact element, including inside frames. Refresh refs after navigation, another snapshot, or human takeover. The text is capped near 20 KB and lists what it omitted; use preview_evaluate to read more. Set includeImage=true only when you need to see the page; the image stays in tool history. Set save=true to write the PNG to disk and get back only the url and screenshotPath. Embed that path in your reply as ![alt](screenshotPath) so the user sees it. This is the only way to show the user a screenshot; the image in the tool result is not saved anywhere.",
147 parameters: Schema.Struct({
148 ...PreviewAutomationTabTargetInput.fields,
149 includeImage: Schema.optional(
150 Schema.Boolean.annotate({
151 description:
152 "Include the PNG image in the tool response. Defaults to false. Set true only when you need to see the page.",
153 }),
154 ),
155 save: Schema.optional(
156 Schema.Boolean.annotate({
157 description:
158 "Write the screenshot PNG to disk and return its absolute path as screenshotPath. Unless includeImage=true, return only the url and screenshotPath. Defaults to false.",
159 }),
160 ),
161 }),
162 success: PreviewAutomationSnapshot,
163 failure: PreviewToolFailure,
164 dependencies,
165 }).annotate(Tool.Title, "Inspect browser page"),
166);
167
168const PreviewClickTool = browserTool(
169 Tool.make("preview_click", {
170 description:
171 "Click exactly one target in the tab selected by tabId, or this agent session's current tab when omitted. Prefer a Playwright locator; selector accepts legacy CSS; x and y must be supplied together. Set button=right for a context menu or clickCount=2 for a double-click; server browser tabs only.",
172 parameters: PreviewAutomationClickInput,
173 success: PreviewActionResult,
174 failure: PreviewToolFailure,
175 dependencies,
176 }).annotate(Tool.Title, "Click preview page"),
177);
178
179const PreviewTypeTool = browserTool(
180 Tool.make("preview_type", {
181 description:
182 "Insert literal text into one input in the tab selected by tabId, or this agent session's current tab when omitted. Prefer a Playwright locator; set clear=true to replace existing text.",
183 parameters: PreviewAutomationTypeInput,
184 success: PreviewActionResult,
185 failure: PreviewToolFailure,
186 dependencies,
187 }).annotate(Tool.Title, "Type into preview page"),
188);
189
190const PreviewHoverTool = safeBrowserTool(
191 Tool.make("preview_hover", {
192 description:
193 "Move the mouse over exactly one target in the tab selected by tabId, or this agent session's current tab when omitted, to reveal hover menus and tooltips. Server browser tabs only.",
194 parameters: PreviewAutomationHoverInput,
195 success: PreviewActionResult,
196 failure: PreviewToolFailure,
197 dependencies,
198 }).annotate(Tool.Title, "Hover preview page"),
199);
200
201const PreviewSelectTool = browserTool(
202 Tool.make("preview_select", {
203 description:
204 "Choose options in one native <select> in the tab selected by tabId, or this agent session's current tab when omitted, by option value or visible label. Custom dropdowns are not <select>; click them open and click the option instead. Server browser tabs only.",
205 parameters: PreviewAutomationSelectInput,
206 success: Schema.Struct({
207 ...PreviewAutomationSelectResult.fields,
208 ...presentationFields,
209 }),
210 failure: PreviewToolFailure,
211 dependencies,
212 }).annotate(Tool.Title, "Select preview option"),
213);
214
215const PreviewDragTool = browserTool(
216 Tool.make("preview_drag", {
217 description:
218 "Drag one element onto another in the tab selected by tabId, or this agent session's current tab when omitted. Server browser tabs only.",
219 parameters: PreviewAutomationDragInput,
220 success: PreviewActionResult,
221 failure: PreviewToolFailure,
222 dependencies,
223 }).annotate(Tool.Title, "Drag in preview page"),
224);
225
226const PreviewUploadTool = browserTool(
227 Tool.make("preview_upload", {
228 description:
229 "Give files on the environment to the page in the tab selected by tabId, or this agent session's current tab when omitted. After clicking an upload control, preview_status reports the open fileChooser; call this with absolute paths to answer it, or with an empty list to cancel. Pass a locator for an <input type=file> to set its files without a picker. Server browser tabs only.",
230 parameters: PreviewAutomationUploadInput,
231 success: PreviewActionResult,
232 failure: PreviewToolFailure,
233 dependencies,
234 }).annotate(Tool.Title, "Upload files to preview page"),
235);
236
237const PreviewPressTool = browserTool(
238 Tool.make("preview_press", {
239 description:
240 "Press one keyboard key in the tab selected by tabId, or this agent session's current tab when omitted. Examples: {key:'Enter'}, {key:'Escape'}, or {key:'a',modifiers:['Meta']}.",
241 parameters: PreviewAutomationPressInput,
242 success: PreviewActionResult,
243 failure: PreviewToolFailure,
244 dependencies,
245 }).annotate(Tool.Title, "Press key in preview page"),
246);
247
248const PreviewScrollTool = safeBrowserTool(
249 Tool.make("preview_scroll", {
250 description:
251 "Scroll the tab selected by tabId, or this agent session's current tab when omitted. Positive deltaY scrolls down and positive deltaX scrolls right; a locator/selector targets a container.",
252 parameters: PreviewAutomationScrollInput,
253 success: PreviewActionResult,
254 failure: PreviewToolFailure,
255 dependencies,
256 }).annotate(Tool.Title, "Scroll preview page"),
257);
258
259/**
260 * MCP `structuredContent` must be a JSON object, and Claude Code rejects the
261 * whole result when it is not. Wrapping keeps arrays, strings, numbers, and
262 * null valid instead of failing only for non-object expressions.
263 */
264export const PreviewEvaluateResult = Schema.Struct({
265 ...presentationFields,
266 value: Schema.Unknown.annotate({
267 description: "The JSON-serializable value the expression produced, or null.",
268 }),
269}).annotate({ description: "The evaluated expression result." });
270
271const PreviewEvaluateTool = browserTool(
272 Tool.make("preview_evaluate", {
273 description:
274 "Evaluate JavaScript in the tab selected by tabId, or this agent session's current tab when omitted. Needs the same control as clicking: it works on your own tabs and on the user's tab only while nobody controls it. To read a tab the user is driving, use preview_snapshot. Returns {value} with a serializable result up to 64 KB; the expression may mutate page state.",
275 parameters: PreviewAutomationEvaluateInput,
276 success: PreviewEvaluateResult,
277 failure: PreviewToolFailure,
278 dependencies,
279 }).annotate(Tool.Title, "Evaluate JavaScript in preview"),
280);
281
282const PreviewWaitForTool = readonlyBrowserTool(
283 Tool.make("preview_wait_for", {
284 description:
285 "Wait in the tab selected by tabId, or this agent session's current tab when omitted, until all supplied locator, selector, text, and URL conditions match.",
286 parameters: PreviewAutomationWaitForInput,
287 success: PreviewActionResult,
288 failure: PreviewToolFailure,
289 dependencies,
290 }).annotate(Tool.Title, "Wait for preview page condition"),
291);
292
293const PreviewRecordingStartTool = safeBrowserTool(
294 Tool.make("preview_recording_start", {
295 description:
296 "Start recording the collaborative browser tab selected by tabId, or this agent session's current tab when omitted.",
297 parameters: PreviewAutomationTabTargetInput,
298 success: Schema.Struct({ ...PreviewAutomationRecordingStatus.fields, ...presentationFields }),
299 failure: PreviewToolFailure,
300 dependencies,
301 }).annotate(Tool.Title, "Start browser recording"),
302);
303
304const PreviewRecordingStopTool = safeBrowserTool(
305 Tool.make("preview_recording_stop", {
306 description:
307 "Stop recording the collaborative browser tab selected by tabId, or this agent session's current tab when omitted, and transfer the compressed recording once (up to 50 MiB) to an evidence file readable in this agent's environment. Returns its environment-local path after transfer succeeds.",
308 parameters: PreviewAutomationTabTargetInput,
309 success: Schema.Struct({ ...PreviewAutomationRecordingArtifact.fields, ...presentationFields }),
310 failure: PreviewToolFailure,
311 dependencies: [...dependencies, FileSystem.FileSystem, ServerConfig.ServerConfig],
312 }).annotate(Tool.Title, "Stop browser recording"),
313);
314
315export const PreviewToolkit = Toolkit.make(
316 PreviewDialogTool,
317 PreviewStatusTool,
318 PreviewOpenTool,
319 PreviewNavigateTool,
320 PreviewResizeTool,
321 PreviewSetAppearanceTool,
322 PreviewSnapshotTool,
323 PreviewClickTool,
324 PreviewTypeTool,
325 PreviewHoverTool,
326 PreviewSelectTool,
327 PreviewDragTool,
328 PreviewUploadTool,
329 PreviewPressTool,
330 PreviewScrollTool,
331 PreviewEvaluateTool,
332 PreviewWaitForTool,
333 PreviewRecordingStartTool,
334 PreviewRecordingStopTool,
335);
336
337export const PreviewStandardToolkit = Toolkit.make(
338 PreviewDialogTool,
339 PreviewStatusTool,
340 PreviewOpenTool,
341 PreviewNavigateTool,
342 PreviewResizeTool,
343 PreviewSetAppearanceTool,
344 PreviewClickTool,
345 PreviewTypeTool,
346 PreviewHoverTool,
347 PreviewSelectTool,
348 PreviewDragTool,
349 PreviewUploadTool,
350 PreviewPressTool,
351 PreviewScrollTool,
352 PreviewEvaluateTool,
353 PreviewWaitForTool,
354 PreviewRecordingStartTool,
355 PreviewRecordingStopTool,
356);
357
358export const PreviewSnapshotToolkit = Toolkit.make(PreviewSnapshotTool);