1import {
2 type DeviceError,
3 type DeviceHostId,
4 type DeviceId,
5 type DevicePlatform,
6 type DeviceSummary,
7 DeviceToolUnavailableError,
8 LOCAL_DEVICE_HOST_ID,
9} from "@t3tools/contracts";
10import * as Effect from "effect/Effect";
11import * as Path from "effect/Path";
12import * as HostProcess from "@t3tools/shared/HostProcess";
13import * as ServerConfig from "../../../config.ts";
14import { ensureAgentDeviceShim } from "../../../device/AgentDeviceShim.ts";
15import { nodeRuntimeUnavailableMessage } from "@t3tools/shared/nodeRuntime";
16
17import * as DeviceService from "../../../device/DeviceService.ts";
18import * as McpInvocationContext from "../../McpInvocationContext.ts";
19import * as McpToolAccess from "../../McpToolAccess.ts";
20import { DeviceScreenshotToolkit, DeviceStandardToolkit, DeviceToolkit } from "./tools.ts";
21
22/** The flags that pin every agent-device command to one device. */
23export function agentDeviceTargetArgs(device: DeviceSummary): ReadonlyArray<string> {
24 return device.platform === "ios"
25 ? ["--platform", "ios", "--udid", device.id]
26 : ["--platform", "android", "--serial", device.id];
27}
28
29/**
30 * Just-in-time guidance returned from `device_open`. This is the one place
31 * the agent learns how to drive the device, so it lives with the tool result
32 * rather than in the always-on prompt block; threads that never open a device
33 * never pay for it.
34 */
35export function agentDeviceQuickStart(
36 device: DeviceSummary,
37 targetArgs = agentDeviceTargetArgs(device),
38 command = "agent-device",
39): string {
40 const executable = /^[a-zA-Z0-9_./:-]+$/.test(command)
41 ? command
42 : "'" + command.replaceAll("'", "'\"'\"'") + "'";
43 const target = targetArgs
44 .map((arg) =>
45 /^[a-zA-Z0-9_./:-]+$/.test(arg) ? arg : "'" + arg.replaceAll("'", "'\"'\"'") + "'",
46 )
47 .join(" ");
48 const platformNotes =
49 device.platform === "ios"
50 ? "First use builds an XCTest runner and can take a couple of minutes; later commands are fast."
51 : "The Android snapshot helper installs itself on first use.";
52 return [
53 `The user is watching ${device.name} (${device.version}) in the Device panel.`,
54 `Drive it with ${executable}. Use this exact executable path; login shells may reset PATH. Always pass ${target}.`,
55 "Typical loop:",
56 ` ${executable} open <bundle-or-package-id> ${target} # or: open <app> <deep-link-url>`,
57 ` ${executable} snapshot -i ${target} # accessibility tree with @eN refs`,
58 ` ${executable} click @e3 ${target}`,
59 ` ${executable} fill @e5 "text" ${target}`,
60 ` ${executable} screenshot /tmp/shot.png ${target} # or call device_screenshot`,
61 ` ${executable} install <app> <path-to-.app-or-.apk> ${target}`,
62 `Prefer snapshot refs over coordinates. Run ${executable} help for workflow guides and ${executable} <command> --help for flags.`,
63 "Prefer agent-device for driving this device. simctl, adb, and xcrun remain available for anything it does not cover.",
64 "For remote hosts, arrange builds, app installation, and any Metro reverse forwarding yourself. T3 provides discovery, streaming, and control only.",
65 "Keep the returned --config and --session flags on every command. Other hosts can be used concurrently; opening one does not switch these commands.",
66 platformNotes,
67 ].join("\n");
68}
69
70const requireDeviceAccess = McpInvocationContext.requireThreadMcpCapability("device").pipe(
71 Effect.mapError(
72 () =>
73 new DeviceToolUnavailableError({
74 reason: "Agent device access is turned off for this environment.",
75 }),
76 ),
77);
78
79const pickDevice = (
80 devices: ReadonlyArray<DeviceSummary>,
81 input: {
82 readonly deviceId?: DeviceId | undefined;
83 readonly platform?: DevicePlatform | undefined;
84 readonly hostId?: DeviceHostId | undefined;
85 },
86): Effect.Effect<DeviceSummary, DeviceToolUnavailableError> =>
87 Effect.gen(function* () {
88 const hostId = input.hostId ?? LOCAL_DEVICE_HOST_ID;
89 if (input.deviceId !== undefined) {
90 const match = devices.find(
91 (device) => device.hostId === hostId && device.id === input.deviceId,
92 );
93 if (match) return match;
94 return yield* new DeviceToolUnavailableError({
95 reason: `No device ${input.deviceId} on host ${hostId}. Call device_list for current ids.`,
96 });
97 }
98 const candidates = devices.filter(
99 (device) =>
100 device.hostId === hostId &&
101 (input.platform === undefined || device.platform === input.platform),
102 );
103 if (candidates.length === 0) {
104 return yield* new DeviceToolUnavailableError({
105 reason:
106 input.platform === undefined
107 ? "No simulators or emulators were found. Call device_list to see why."
108 : `No ${input.platform} devices were found on host ${hostId}. Call device_list to see why.`,
109 });
110 }
111 const platforms = new Set(candidates.map((device) => device.platform));
112 if (input.platform === undefined && platforms.size > 1) {
113 return yield* new DeviceToolUnavailableError({
114 reason: "Both iOS and Android devices are available; pass platform or deviceId.",
115 });
116 }
117 return candidates.find((device) => device.booted) ?? candidates[0]!;
118 });
119
120const toolError = (error: DeviceError | DeviceToolUnavailableError) => error;
121
122const handlers = {
123 device_list: McpToolAccess.readsAsCaller((input) =>
124 Effect.gen(function* () {
125 const scope = yield* requireDeviceAccess;
126 const devices = yield* DeviceService.DeviceService;
127 const state = yield* devices.list;
128 if (state.hostStatus === "disabled") {
129 return yield* new DeviceToolUnavailableError({
130 reason:
131 "Device support is off. Ask the user to enable it in the Device panel before installing or starting device tools.",
132 });
133 }
134 const hostId = input?.hostId;
135 const open = state.sessions
136 .filter((session) => session.threadId === scope.thread.threadId)
137 .map((session) => ({ hostId: session.hostId, deviceId: session.deviceId }));
138 return {
139 hostStatuses: Object.fromEntries(
140 Object.entries(state.hostStatuses).filter(([id]) => !hostId || id === hostId),
141 ),
142 hosts: hostId ? state.hosts.filter((host) => host.id === hostId) : state.hosts,
143 devices: hostId
144 ? state.devices.filter((device) => device.hostId === hostId)
145 : state.devices,
146 open,
147 };
148 }).pipe(Effect.mapError(toolError)),
149 ),
150 device_open: McpToolAccess.actsAsCaller((input) =>
151 Effect.gen(function* () {
152 const scope = yield* requireDeviceAccess;
153 const devices = yield* DeviceService.DeviceService;
154 const state = yield* devices.list;
155 if (state.hostStatus === "disabled") {
156 return yield* new DeviceToolUnavailableError({
157 reason:
158 "Device support is off. Ask the user to enable it in the Device panel before installing or starting device tools.",
159 });
160 }
161 const target = yield* pickDevice(state.devices, input);
162 // Resolve consent and agent connectivity before booting or registering a session.
163 const agentArgs = yield* devices.agentTarget({
164 threadId: scope.thread.threadId,
165 hostId: target.hostId,
166 deviceId: target.id,
167 });
168 const session = yield* devices.open({
169 threadId: scope.thread.threadId,
170 hostId: target.hostId,
171 deviceId: target.id,
172 platform: target.platform,
173 });
174 const after = yield* devices.state;
175 const device =
176 after.devices.find(
177 (candidate) => candidate.hostId === session.hostId && candidate.id === session.deviceId,
178 ) ?? target;
179 const targetArgs = [...agentDeviceTargetArgs(device), ...agentArgs];
180 const config = yield* ServerConfig.ServerConfig;
181 const path = yield* Path.Path;
182 const platform = yield* HostProcess.Platform;
183 const shimDir = yield* ensureAgentDeviceShim({
184 entryPath: yield* devices.agentCli,
185 stateDir: config.stateDir,
186 }).pipe(
187 Effect.mapError(
188 (error) =>
189 new DeviceToolUnavailableError({
190 reason:
191 error._tag === "NodeRuntimeUnavailableError"
192 ? nodeRuntimeUnavailableMessage("Device automation")
193 : "Could not prepare the agent-device launcher.",
194 cause: error,
195 }),
196 ),
197 );
198 const command = path.join(
199 shimDir,
200 platform === "win32" ? "agent-device.cmd" : "agent-device",
201 );
202 return {
203 device,
204 agentDevice: { command, targetArgs },
205 quickStart: agentDeviceQuickStart(device, targetArgs, command),
206 };
207 }).pipe(Effect.mapError(toolError)),
208 ),
209 device_screenshot: McpToolAccess.readsAsCaller((input) =>
210 Effect.gen(function* () {
211 const scope = yield* requireDeviceAccess;
212 const devices = yield* DeviceService.DeviceService;
213 const sessions = yield* devices.sessionsForThread(scope.thread.threadId);
214 // Only devices this thread opened: another thread's device is not this agent's to watch.
215 const hostId =
216 input.deviceId === undefined ? input.hostId : (input.hostId ?? LOCAL_DEVICE_HOST_ID);
217 const target = sessions.findLast(
218 (session) =>
219 (hostId === undefined || session.hostId === hostId) &&
220 (input.deviceId === undefined || session.deviceId === input.deviceId),
221 );
222 if (!target) {
223 return yield* new DeviceToolUnavailableError({
224 reason:
225 input.deviceId === undefined
226 ? "No device is open in this thread. Call device_open first."
227 : `Device ${input.deviceId} on host ${hostId} is not open in this thread. Call device_open first.`,
228 });
229 }
230 const shot = yield* devices.screenshot(target);
231 return {
232 device: shot.device,
233 screenshot: {
234 mimeType: "image/png" as const,
235 data: Buffer.from(shot.png).toString("base64"),
236 ...pngDimensions(shot.png),
237 },
238 };
239 }).pipe(Effect.mapError(toolError)),
240 ),
241 device_close: McpToolAccess.actsAsCaller((input) =>
242 Effect.gen(function* () {
243 const scope = yield* requireDeviceAccess;
244 const devices = yield* DeviceService.DeviceService;
245 yield* devices.close({
246 threadId: scope.thread.threadId,
247 ...(input.hostId === undefined ? {} : { hostId: input.hostId }),
248 ...(input.deviceId === undefined ? {} : { deviceId: input.deviceId }),
249 ...(input.shutdown === undefined ? {} : { shutdown: input.shutdown }),
250 });
251 return {};
252 }).pipe(Effect.mapError(toolError)),
253 ),
254} satisfies McpToolAccess.Handlers<typeof DeviceToolkit.tools>;
255
256/** Width and height from the IHDR chunk; a PNG that lacks one reports 0×0. */
257export function pngDimensions(png: Uint8Array): { width: number; height: number } {
258 if (png.length < 24) return { width: 0, height: 0 };
259 const view = new DataView(png.buffer, png.byteOffset, png.byteLength);
260 const isPng =
261 view.getUint32(0) === 0x89504e47 &&
262 view.getUint32(4) === 0x0d0a1a0a &&
263 view.getUint32(12) === 0x49484452;
264 return isPng
265 ? { width: view.getUint32(16), height: view.getUint32(20) }
266 : { width: 0, height: 0 };
267}
268
269const { device_screenshot, ...standardHandlers } = handlers;
270
271export const layerStandard = McpToolAccess.toLayer(DeviceStandardToolkit, standardHandlers);
272
273export const layerScreenshot = McpToolAccess.toLayer(DeviceScreenshotToolkit, {
274 device_screenshot,
275});