authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-06-27 08:18:28-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-06-27 08:18:28-07:00
log44fb34ad5383badfb02c9f7daa1d8486277eeb01
tree3ce4b090b77b1eb13c0f1d12b0cdf3f286c75be9
parent6245c547c2d88e26434ac7b8e08213f2d9ef8577
signature Signed by PGP key B5690EEEBB952194

Stabilize preview browser surfaces, automation, and recording (#3565)

22 files changed, 1248 insertions(+), 207 deletions(-)

apps/server/src/mcp/McpHttpServer.test.ts+48-40
......@@ -15,6 +15,7 @@ import * as PreviewAutomationBroker from "./PreviewAutomationBroker.ts";
1515const environmentId = EnvironmentId.make("environment-mcp-test");
1616const threadId = ThreadId.make("thread-mcp-test");
1717const tabId = PreviewTabId.make("tab-mcp-test");
18const alternateTabId = PreviewTabId.make("tab-mcp-alternate");
1819const invocation = {
1920 environmentId,
2021 threadId,
......@@ -154,49 +155,53 @@ it.effect("registers annotated tools and preserves authenticated request context
154155 Effect.gen(function* () {
155156 const server = yield* McpServer.McpServer;
156157 const broker = yield* PreviewAutomationBroker.PreviewAutomationBroker;
158 const routedRequests: Array<{
159 readonly operation: string;
160 readonly tabId?: string | undefined;
161 }> = [];
157162 const events = yield* broker.connect({
158163 clientId: "mcp-test-client",
159164 environmentId,
160165 });
161 yield* Stream.runForEach(events, (event) =>
162 event.type === "connected"
163 ? Effect.void
164 : broker.respond({
165 clientId: "mcp-test-client",
166 connectionId: event.connectionId,
167 requestId: event.request.requestId,
168 ok: true,
169 result:
170 event.request.operation === "snapshot"
171 ? {
172 url: "http://example.test/",
173 title: "Example",
174 loading: false,
175 visibleText: "Example",
176 interactiveElements: [],
177 accessibilityTree: {},
178 consoleEntries: [],
179 networkEntries: [],
180 actionTimeline: [],
181 screenshot: {
182 mimeType: "image/png",
183 data: Buffer.from("png").toString("base64"),
184 width: 10,
185 height: 5,
186 },
187 }
188 : event.request.operation === "press"
189 ? undefined
190 : {
191 available: true,
192 visible: true,
193 tabId,
194 url: "http://example.test/",
195 title: "Example",
196 loading: false,
197 },
198 }),
199 ).pipe(Effect.forkScoped);
166 yield* Stream.runForEach(events, (event) => {
167 if (event.type === "connected") return Effect.void;
168 routedRequests.push(event.request);
169 return broker.respond({
170 clientId: "mcp-test-client",
171 connectionId: event.connectionId,
172 requestId: event.request.requestId,
173 ok: true,
174 result:
175 event.request.operation === "snapshot"
176 ? {
177 url: "http://example.test/",
178 title: "Example",
179 loading: false,
180 visibleText: "Example",
181 interactiveElements: [],
182 accessibilityTree: {},
183 consoleEntries: [],
184 networkEntries: [],
185 actionTimeline: [],
186 screenshot: {
187 mimeType: "image/png",
188 data: Buffer.from("png").toString("base64"),
189 width: 10,
190 height: 5,
191 },
192 }
193 : event.request.operation === "press"
194 ? undefined
195 : {
196 available: true,
197 visible: true,
198 tabId,
199 url: "http://example.test/",
200 title: "Example",
201 loading: false,
202 },
203 });
204 }).pipe(Effect.forkScoped);
200205 yield* Effect.yieldNow;
201206
202207 const statusTool = server.tools.find(({ tool }) => tool.name === "preview_status");
......@@ -239,7 +244,7 @@ it.effect("registers annotated tools and preserves authenticated request context
239244 expect(malformed.isError).toBe(true);
240245
241246 const snapshot = yield* server
242 .callTool({ name: "preview_snapshot", arguments: {} })
247 .callTool({ name: "preview_snapshot", arguments: { tabId: alternateTabId } })
243248 .pipe(
244249 Effect.provideService(McpInvocationContext.McpInvocationContext, invocation),
245250 Effect.provideService(McpSchema.McpServerClient, client),
......@@ -249,6 +254,9 @@ it.effect("registers annotated tools and preserves authenticated request context
249254 expect(snapshot.structuredContent).toMatchObject({
250255 screenshot: { mimeType: "image/png", width: 10, height: 5 },
251256 });
257 expect(routedRequests.find(({ operation }) => operation === "snapshot")?.tabId).toBe(
258 alternateTabId,
259 );
252260
253261 const press = yield* server
254262 .callTool({ name: "preview_press", arguments: { key: "Enter" } })
apps/server/src/mcp/PreviewAutomationBroker.test.ts+242-6
......@@ -15,6 +15,7 @@ import {
1515 type PreviewAutomationStreamEvent,
1616} from "@t3tools/contracts";
1717import * as Effect from "effect/Effect";
18import * as Deferred from "effect/Deferred";
1819import * as Fiber from "effect/Fiber";
1920import * as Result from "effect/Result";
2021import * as Stream from "effect/Stream";
......@@ -84,6 +85,187 @@ it.effect("atomically registers a connected host and correlates its response", (
8485 ),
8586);
8687
88it.effect("targets multiple tabs explicitly while retaining a default tab", () =>
89 Effect.scoped(
90 Effect.gen(function* () {
91 const broker = yield* makeBroker;
92 const appTabId = PreviewTabId.make("tab-web-app");
93 const simulatorTabId = PreviewTabId.make("tab-ios-simulator");
94 const openedTabIds = [appTabId, simulatorTabId];
95 let openIndex = 0;
96 const routedRequests: RoutedRequest[] = [];
97 const requests = requestsFrom(yield* broker.connect(makeHost()));
98 yield* Stream.runForEach(requests, (request) => {
99 routedRequests.push(request);
100 return broker.respond({
101 clientId: "client-1",
102 connectionId: request.connectionId,
103 requestId: request.requestId,
104 ok: true,
105 result:
106 request.operation === "open"
107 ? { available: true, tabId: openedTabIds[openIndex++] }
108 : { url: "http://localhost:3200" },
109 });
110 }).pipe(Effect.forkScoped);
111 yield* Effect.yieldNow;
112
113 yield* broker.invoke({ scope, operation: "open", input: { reuseExistingTab: false } });
114 yield* broker.invoke({ scope, operation: "open", input: { reuseExistingTab: false } });
115 yield* broker.invoke({ scope, operation: "snapshot", input: {} });
116 yield* broker.invoke({ scope, operation: "snapshot", input: {}, tabId: appTabId });
117 yield* broker.invoke({ scope, operation: "snapshot", input: {} });
118
119 expect(routedRequests).toHaveLength(5);
120 expect(routedRequests[0]?.tabId).toBeUndefined();
121 expect(routedRequests[1]?.tabId).toBe(appTabId);
122 expect(routedRequests[2]?.tabId).toBe(simulatorTabId);
123 expect(routedRequests[2]?.tabIdExplicit).toBe(false);
124 expect(routedRequests[3]?.tabId).toBe(appTabId);
125 expect(routedRequests[3]?.tabIdExplicit).toBe(true);
126 expect(routedRequests[4]?.tabId).toBe(appTabId);
127 }),
128 ),
129);
130
131it.effect("does not let an older response replace a newer explicit tab target", () =>
132 Effect.scoped(
133 Effect.gen(function* () {
134 const broker = yield* makeBroker;
135 const olderTabId = PreviewTabId.make("tab-older-request");
136 const newerTabId = PreviewTabId.make("tab-newer-request");
137 const releaseOlderResponse = yield* Deferred.make<void>();
138 const routedRequests: RoutedRequest[] = [];
139 const requests = requestsFrom(yield* broker.connect(makeHost()));
140 yield* Stream.runForEach(requests, (request) => {
141 routedRequests.push(request);
142 const response = Effect.gen(function* () {
143 if (request.tabId === olderTabId) {
144 yield* Deferred.await(releaseOlderResponse);
145 }
146 yield* broker.respond({
147 clientId: "client-1",
148 connectionId: request.connectionId,
149 requestId: request.requestId,
150 ok: true,
151 result: { url: "http://localhost:3200" },
152 });
153 if (request.tabId === newerTabId) {
154 yield* Deferred.succeed(releaseOlderResponse, undefined);
155 }
156 });
157 return response.pipe(Effect.forkScoped, Effect.asVoid);
158 }).pipe(Effect.forkScoped);
159 yield* Effect.yieldNow;
160
161 const older = yield* broker
162 .invoke({ scope, operation: "snapshot", input: {}, tabId: olderTabId })
163 .pipe(Effect.forkScoped);
164 yield* Effect.yieldNow;
165 const newer = yield* broker
166 .invoke({ scope, operation: "snapshot", input: {}, tabId: newerTabId })
167 .pipe(Effect.forkScoped);
168 yield* Fiber.join(newer);
169 yield* Fiber.join(older);
170 yield* broker.invoke({ scope, operation: "snapshot", input: {} });
171
172 expect(routedRequests.at(-1)?.tabId).toBe(newerTabId);
173 }),
174 ),
175);
176
177it.effect("does not replace the default tab with a globally stopped recording tab", () =>
178 Effect.scoped(
179 Effect.gen(function* () {
180 const broker = yield* makeBroker;
181 const browsingTabId = PreviewTabId.make("tab-session-b");
182 const recordingTabId = PreviewTabId.make("tab-session-a-recording");
183 const routedRequests: RoutedRequest[] = [];
184 const requests = requestsFrom(yield* broker.connect(makeHost()));
185 yield* Stream.runForEach(requests, (request) => {
186 routedRequests.push(request);
187 return broker.respond({
188 clientId: "client-1",
189 connectionId: request.connectionId,
190 requestId: request.requestId,
191 ok: true,
192 result:
193 request.operation === "open"
194 ? { available: true, tabId: browsingTabId }
195 : request.operation === "recordingStop"
196 ? { id: "recording-1", tabId: recordingTabId }
197 : { url: "http://localhost:3200" },
198 });
199 }).pipe(Effect.forkScoped);
200 yield* Effect.yieldNow;
201
202 yield* broker.invoke({ scope, operation: "open", input: {} });
203 yield* broker.invoke({ scope, operation: "recordingStop", input: {} });
204 yield* broker.invoke({ scope, operation: "snapshot", input: {} });
205
206 expect(routedRequests.at(-1)?.tabId).toBe(browsingTabId);
207 }),
208 ),
209);
210
211it.effect("does not let a no-tab response suppress an earlier tab decision", () =>
212 Effect.scoped(
213 Effect.gen(function* () {
214 const broker = yield* makeBroker;
215 const initialTabId = PreviewTabId.make("tab-initial");
216 const openedTabId = PreviewTabId.make("tab-opened-late");
217 const releaseOpenResponse = yield* Deferred.make<void>();
218 const routedRequests: RoutedRequest[] = [];
219 const requests = requestsFrom(yield* broker.connect(makeHost()));
220 yield* Stream.runForEach(requests, (request) => {
221 routedRequests.push(request);
222 const marker =
223 typeof request.input === "object" && request.input !== null && "marker" in request.input
224 ? request.input.marker
225 : undefined;
226 const response = Effect.gen(function* () {
227 if (marker === "older") {
228 yield* Deferred.await(releaseOpenResponse);
229 }
230 yield* broker.respond({
231 clientId: "client-1",
232 connectionId: request.connectionId,
233 requestId: request.requestId,
234 ok: true,
235 result:
236 request.operation === "open"
237 ? { available: true, tabId: marker === "older" ? openedTabId : initialTabId }
238 : { url: "http://localhost:3200" },
239 });
240 if (marker === "newer") {
241 yield* Deferred.succeed(releaseOpenResponse, undefined);
242 }
243 });
244 return response.pipe(Effect.forkScoped, Effect.asVoid);
245 }).pipe(Effect.forkScoped);
246 yield* Effect.yieldNow;
247
248 yield* broker.invoke({ scope, operation: "open", input: {} });
249 const older = yield* broker
250 .invoke({
251 scope,
252 operation: "open",
253 input: { marker: "older", reuseExistingTab: false },
254 })
255 .pipe(Effect.forkScoped);
256 yield* Effect.yieldNow;
257 const newer = yield* broker
258 .invoke({ scope, operation: "snapshot", input: { marker: "newer" } })
259 .pipe(Effect.forkScoped);
260 yield* Fiber.join(newer);
261 yield* Fiber.join(older);
262 yield* broker.invoke({ scope, operation: "snapshot", input: {} });
263
264 expect(routedRequests.at(-1)?.tabId).toBe(openedTabId);
265 }),
266 ),
267);
268
87269it.effect("announces a live replacement stream before delivering requests", () =>
88270 Effect.scoped(
89271 Effect.gen(function* () {
......@@ -632,7 +814,9 @@ it.effect("fails over a pinned provider session only after its host disconnects"
632814 Effect.scoped(
633815 Effect.gen(function* () {
634816 const broker = yield* makeBroker;
817 const firstTabId = PreviewTabId.make("tab-on-first-host");
635818 let firstConnectionId = "";
819 let secondRoutedTabId: PreviewTabId | undefined;
636820 const firstRequests = requestsFrom(
637821 yield* broker.connect(makeHost({ clientId: "client-first" })),
638822 (connectionId) => {
......@@ -648,18 +832,19 @@ it.effect("fails over a pinned provider session only after its host disconnects"
648832 connectionId: request.connectionId,
649833 requestId: request.requestId,
650834 ok: true,
651 result: "first",
835 result: request.operation === "open" ? { host: "first", tabId: firstTabId } : "first",
652836 }),
653837 ).pipe(Effect.forkScoped);
654 yield* Stream.runForEach(secondRequests, (request) =>
655 broker.respond({
838 yield* Stream.runForEach(secondRequests, (request) => {
839 secondRoutedTabId = request.tabId;
840 return broker.respond({
656841 clientId: "client-second",
657842 connectionId: request.connectionId,
658843 requestId: request.requestId,
659844 ok: true,
660845 result: "second",
661 }),
662 ).pipe(Effect.forkScoped);
846 });
847 }).pipe(Effect.forkScoped);
663848 yield* Effect.yieldNow;
664849
665850 yield* broker.focusHost({
......@@ -668,7 +853,10 @@ it.effect("fails over a pinned provider session only after its host disconnects"
668853 connectionId: firstConnectionId,
669854 focused: true,
670855 });
671 expect(yield* broker.invoke<string>({ scope, operation: "status", input: {} })).toBe("first");
856 expect(yield* broker.invoke({ scope, operation: "open", input: {} })).toEqual({
857 host: "first",
858 tabId: firstTabId,
859 });
672860
673861 yield* Fiber.interrupt(firstConsumer);
674862 yield* Effect.yieldNow;
......@@ -676,6 +864,7 @@ it.effect("fails over a pinned provider session only after its host disconnects"
676864 expect(yield* broker.invoke<string>({ scope, operation: "status", input: {} })).toBe(
677865 "second",
678866 );
867 expect(secondRoutedTabId).toBeUndefined();
679868 }),
680869 ),
681870);
......@@ -740,6 +929,53 @@ it.effect("keeps a replacement stream authoritative when the old stream finalize
740929 ),
741930);
742931
932it.effect("does not carry a tab id across a replacement automation stream", () =>
933 Effect.scoped(
934 Effect.gen(function* () {
935 const broker = yield* makeBroker;
936 const openedTabId = PreviewTabId.make("tab-first-webcontents");
937 const firstRequests = requestsFrom(yield* broker.connect(makeHost()));
938 yield* Stream.runForEach(firstRequests, (request) =>
939 broker.respond({
940 clientId: "client-1",
941 connectionId: request.connectionId,
942 requestId: request.requestId,
943 ok: true,
944 result:
945 request.operation === "open"
946 ? { host: "first", tabId: openedTabId }
947 : { host: "first" },
948 }),
949 ).pipe(Effect.forkScoped);
950 yield* Effect.yieldNow;
951
952 expect(yield* broker.invoke({ scope, operation: "open", input: {} })).toEqual({
953 host: "first",
954 tabId: openedTabId,
955 });
956
957 const routedRequests: RoutedRequest[] = [];
958 const replacementRequests = requestsFrom(yield* broker.connect(makeHost()));
959 yield* Stream.runForEach(replacementRequests, (request) => {
960 routedRequests.push(request);
961 return broker.respond({
962 clientId: "client-1",
963 connectionId: request.connectionId,
964 requestId: request.requestId,
965 ok: true,
966 result: "replacement",
967 });
968 }).pipe(Effect.forkScoped);
969 yield* Effect.yieldNow;
970
971 expect(yield* broker.invoke<string>({ scope, operation: "status", input: {} })).toBe(
972 "replacement",
973 );
974 expect(routedRequests.at(-1)?.tabId).toBeUndefined();
975 }),
976 ),
977);
978
743979it.effect("fails requests assigned to the stream that is replaced", () =>
744980 Effect.scoped(
745981 Effect.gen(function* () {
apps/server/src/mcp/PreviewAutomationBroker.ts+56-6
......@@ -13,13 +13,13 @@ import {
1313 PreviewAutomationTargetNotEditableError,
1414 PreviewAutomationTimeoutError,
1515 PreviewAutomationUnsupportedClientError,
16 PreviewTabId,
1617 type PreviewAutomationError,
1718 type PreviewAutomationOperation,
1819 type PreviewAutomationHost,
1920 type PreviewAutomationHostFocus,
2021 type PreviewAutomationResponse,
2122 type PreviewAutomationStreamEvent,
22 type PreviewTabId,
2323} from "@t3tools/contracts";
2424import * as Context from "effect/Context";
2525import * as Clock from "effect/Clock";
......@@ -29,6 +29,7 @@ import * as Effect from "effect/Effect";
2929import * as Layer from "effect/Layer";
3030import * as Option from "effect/Option";
3131import * as Queue from "effect/Queue";
32import * as Schema from "effect/Schema";
3233import * as Stream from "effect/Stream";
3334import * as SynchronizedRef from "effect/SynchronizedRef";
3435
......@@ -79,6 +80,8 @@ interface HostAssignment {
7980 readonly connectionId: ClientConnection["connectionId"];
8081 readonly queue: ClientConnection["queue"];
8182 readonly expiresAt: number;
83 readonly tabId?: PreviewTabId;
84 readonly tabSequence?: number;
8285}
8386
8487interface PreviewAutomationRequestErrorContext {
......@@ -144,6 +147,14 @@ const selectorDiagnosticsFromInput = (
144147const hostAssignmentKey = (scope: McpInvocationContext.McpInvocationScope): string =>
145148 `${scope.environmentId}\u0000${scope.providerSessionId}`;
146149
150const isPreviewTabId = Schema.is(PreviewTabId);
151
152const readResultTabId = (result: unknown): PreviewTabId | null | undefined => {
153 if (typeof result !== "object" || result === null || !("tabId" in result)) return undefined;
154 const tabId = result.tabId;
155 return tabId === null || isPreviewTabId(tabId) ? tabId : undefined;
156};
157
147158const supportsOperation = (
148159 connection: ClientConnection,
149160 operation: PreviewAutomationOperation,
......@@ -454,15 +465,24 @@ export const make = Effect.gen(function* PreviewAutomationBrokerMake() {
454465 if (!hasLiveAssignment) assignments.delete(assignmentKey);
455466 return [undefined, { ...current, assignments }] as const;
456467 }
468 const canReuseAssignedTab =
469 assigned !== undefined &&
470 assigned.connectionId === connection.connectionId &&
471 assigned.queue === connection.queue;
457472 assignments.set(assignmentKey, {
458473 clientId: connection.clientId,
459474 connectionId: connection.connectionId,
460475 queue: connection.queue,
461476 expiresAt: input.scope.expiresAt,
477 ...(canReuseAssignedTab && assigned.tabId !== undefined ? { tabId: assigned.tabId } : {}),
478 ...(canReuseAssignedTab && assigned.tabSequence !== undefined
479 ? { tabSequence: assigned.tabSequence }
480 : {}),
462481 });
463482
464 const requestId = `preview-${current.requestSequence}`;
465 const tabId = input.tabId;
483 const requestSequence = current.requestSequence;
484 const requestId = `preview-${requestSequence}`;
485 const tabId = input.tabId ?? (canReuseAssignedTab ? assigned.tabId : undefined);
466486 const selectorDiagnostics = selectorDiagnosticsFromInput(input.input);
467487 const context: PreviewAutomationRequestErrorContext = {
468488 operation: input.operation,
......@@ -480,7 +500,7 @@ export const make = Effect.gen(function* PreviewAutomationBrokerMake() {
480500 const pending = new Map(current.pending);
481501 pending.set(requestId, { queue: connection.queue, deferred, context });
482502 return [
483 { connection, requestId, requestContext: context },
503 { connection, requestId, requestContext: context, requestSequence },
484504 { ...current, assignments, pending, requestSequence: current.requestSequence + 1 },
485505 ] as const;
486506 });
......@@ -493,7 +513,7 @@ export const make = Effect.gen(function* PreviewAutomationBrokerMake() {
493513 providerInstanceId: input.scope.providerInstanceId,
494514 });
495515 }
496 const { connection, requestId, requestContext } = route;
516 const { connection, requestId, requestContext, requestSequence } = route;
497517 const removePending = SynchronizedRef.update(state, (next) => {
498518 if (!next.pending.has(requestId)) return next;
499519 const pending = new Map(next.pending);
......@@ -508,6 +528,7 @@ export const make = Effect.gen(function* PreviewAutomationBrokerMake() {
508528 requestId,
509529 threadId: input.scope.threadId,
510530 tabId: requestContext.tabId,
531 tabIdExplicit: input.tabId !== undefined,
511532 operation: input.operation,
512533 input: input.input,
513534 timeoutMs,
......@@ -526,7 +547,36 @@ export const make = Effect.gen(function* PreviewAutomationBrokerMake() {
526547 onSome: (value) => Effect.succeed(value as A),
527548 });
528549 });
529 return yield* awaitResponse().pipe(Effect.ensuring(removePending));
550 const result = yield* awaitResponse().pipe(Effect.ensuring(removePending));
551 // A stop artifact identifies the globally recorded tab, not the caller's browsing target.
552 const responseTabId = input.operation === "recordingStop" ? undefined : readResultTabId(result);
553 const resultTabId = responseTabId === undefined ? input.tabId : responseTabId;
554 if (resultTabId === undefined) return result;
555 const assignmentKey = hostAssignmentKey(input.scope);
556 yield* SynchronizedRef.update(state, (current) => {
557 const assignment = current.assignments.get(assignmentKey);
558 if (
559 !assignment ||
560 assignment.connectionId !== connection.connectionId ||
561 assignment.queue !== connection.queue ||
562 (assignment.tabSequence ?? -1) > requestSequence
563 ) {
564 return current;
565 }
566 const assignments = new Map(current.assignments);
567 if (resultTabId === null) {
568 const { tabId: _tabId, ...withoutTabId } = assignment;
569 assignments.set(assignmentKey, { ...withoutTabId, tabSequence: requestSequence });
570 } else {
571 assignments.set(assignmentKey, {
572 ...assignment,
573 ...(resultTabId === undefined ? {} : { tabId: resultTabId }),
574 tabSequence: requestSequence,
575 });
576 }
577 return { ...current, assignments };
578 });
579 return result;
530580 });
531581
532582 return PreviewAutomationBroker.of({ connect, focusHost, respond, invoke });
apps/server/src/mcp/toolkits/preview/handlers.ts+33-13
......@@ -6,6 +6,7 @@ import type {
66 PreviewAutomationResizeResult,
77 PreviewAutomationSnapshot,
88 PreviewAutomationStatus,
9 PreviewTabId,
910} from "@t3tools/contracts";
1011
1112import * as McpInvocationContext from "../../McpInvocationContext.ts";
......@@ -16,6 +17,7 @@ const invoke = Effect.fn("PreviewToolkit.invoke")(function* <A>(
1617 operation: PreviewAutomationOperation,
1718 input: unknown,
1819 timeoutMs?: number,
20 tabId?: PreviewTabId,
1921): Effect.fn.Return<
2022 A,
2123 import("@t3tools/contracts").PreviewAutomationError,
......@@ -28,31 +30,49 @@ const invoke = Effect.fn("PreviewToolkit.invoke")(function* <A>(
2830 operation,
2931 input,
3032 ...(timeoutMs === undefined ? {} : { timeoutMs }),
33 ...(tabId === undefined ? {} : { tabId }),
3134 });
3235});
3336
37const invokeTargeted = <A>(
38 operation: PreviewAutomationOperation,
39 input: {
40 readonly tabId?: PreviewTabId | undefined;
41 readonly [key: string]: unknown;
42 },
43 timeoutMs?: number,
44) => {
45 const { tabId, ...operationInput } = input;
46 return invoke<A>(operation, operationInput, timeoutMs, tabId);
47};
48
3449const handlers = {
35 preview_status: () => invoke<PreviewAutomationStatus>("status", {}),
50 preview_status: (input) => invokeTargeted<PreviewAutomationStatus>("status", input ?? {}),
3651 preview_open: (input) =>
37 invoke<PreviewAutomationStatus>("open", {
52 invokeTargeted<PreviewAutomationStatus>("open", {
3853 ...input,
3954 show: input.show ?? true,
4055 reuseExistingTab: input.reuseExistingTab ?? true,
4156 }),
42 preview_navigate: (input) => invoke<PreviewAutomationStatus>("navigate", input, input.timeoutMs),
57 preview_navigate: (input) =>
58 invokeTargeted<PreviewAutomationStatus>("navigate", input, input.timeoutMs),
4359 preview_resize: (input) =>
44 invoke<PreviewAutomationResizeResult>("resize", input, input.timeoutMs),
45 preview_snapshot: () => invoke<PreviewAutomationSnapshot>("snapshot", {}),
46 preview_click: (input) => invoke<void>("click", input, input.timeoutMs).pipe(Effect.as(null)),
47 preview_type: (input) => invoke<void>("type", input, input.timeoutMs).pipe(Effect.as(null)),
48 preview_press: (input) => invoke<void>("press", input).pipe(Effect.as(null)),
49 preview_scroll: (input) => invoke<void>("scroll", input).pipe(Effect.as(null)),
60 invokeTargeted<PreviewAutomationResizeResult>("resize", input, input.timeoutMs),
61 preview_snapshot: (input) => invokeTargeted<PreviewAutomationSnapshot>("snapshot", input ?? {}),
62 preview_click: (input) =>
63 invokeTargeted<void>("click", input, input.timeoutMs).pipe(Effect.as(null)),
64 preview_type: (input) =>
65 invokeTargeted<void>("type", input, input.timeoutMs).pipe(Effect.as(null)),
66 preview_press: (input) => invokeTargeted<void>("press", input).pipe(Effect.as(null)),
67 preview_scroll: (input) => invokeTargeted<void>("scroll", input).pipe(Effect.as(null)),
5068 preview_evaluate: (input) =>
51 invoke<unknown>("evaluate", input).pipe(Effect.map((result) => result ?? null)),
69 invokeTargeted<unknown>("evaluate", input).pipe(Effect.map((result) => result ?? null)),
5270 preview_wait_for: (input) =>
53 invoke<void>("waitFor", input, input.timeoutMs).pipe(Effect.as(null)),
54 preview_recording_start: () => invoke<PreviewAutomationRecordingStatus>("recordingStart", {}),
55 preview_recording_stop: () => invoke<PreviewAutomationRecordingArtifact>("recordingStop", {}),
71 invokeTargeted<void>("waitFor", input, input.timeoutMs).pipe(Effect.as(null)),
72 preview_recording_start: (input) =>
73 invokeTargeted<PreviewAutomationRecordingStatus>("recordingStart", input ?? {}),
74 preview_recording_stop: (input) =>
75 invokeTargeted<PreviewAutomationRecordingArtifact>("recordingStop", input ?? {}),
5676} satisfies Parameters<typeof PreviewToolkit.toLayer>[0];
5777
5878const { preview_snapshot, ...standardHandlers } = handlers;
apps/server/src/mcp/toolkits/preview/tools.test.ts+4
......@@ -27,6 +27,10 @@ it("exports provider-compatible object schemas with described parameters", () =>
2727 expect(schema.type, `${tool.name} must export a top-level object schema`).toBe("object");
2828 expect(schema.anyOf, `${tool.name} must not export a root anyOf`).toBeUndefined();
2929 expect(schema.oneOf, `${tool.name} must not export a root oneOf`).toBeUndefined();
30 expect(
31 schema.properties?.tabId,
32 `${tool.name} must allow an explicit collaborative browser tab target`,
33 ).toBeDefined();
3034 for (const [field, fieldSchema] of Object.entries(schema.properties ?? {})) {
3135 expect(
3236 schemaHasDescription(fieldSchema),
apps/server/src/mcp/toolkits/preview/tools.ts+17-12
......@@ -12,6 +12,7 @@ import {
1212 PreviewAutomationScrollInput,
1313 PreviewAutomationSnapshot,
1414 PreviewAutomationStatus,
15 PreviewAutomationTabTargetInput,
1516 PreviewAutomationTypeInput,
1617 PreviewAutomationWaitForInput,
1718} from "@t3tools/contracts";
......@@ -37,7 +38,8 @@ const readonlyBrowserTool = <T extends Tool.Any>(tool: T): T =>
3738
3839export const PreviewStatusTool = Tool.make("preview_status", {
3940 description:
40 "Report whether the scoped thread has an automation-capable desktop preview, including its active tab, URL, title, visibility, loading state, viewport mode, and measured CSS-pixel size.",
41 "Report whether a collaborative browser tab is automation-capable, including its URL, title, visibility, loading state, viewport mode, and measured CSS-pixel size. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab.",
42 parameters: PreviewAutomationTabTargetInput,
4143 success: PreviewAutomationStatus,
4244 failure: PreviewAutomationError,
4345 dependencies,
......@@ -50,7 +52,7 @@ export const PreviewStatusTool = Tool.make("preview_status", {
5052export const PreviewOpenTool = browserTool(
5153 Tool.make("preview_open", {
5254 description:
53 "Show and initialize the browser preview for the scoped thread, optionally reusing its current tab and navigating to a URL.",
55 "Show and initialize a collaborative browser tab. Pass tabId to reuse a specific existing tab, set reuseExistingTab=false to create another tab, or omit both to use this agent session's current tab.",
5456 parameters: PreviewAutomationOpenInput,
5557 success: PreviewAutomationStatus,
5658 failure: PreviewAutomationError,
......@@ -63,7 +65,7 @@ export const PreviewOpenTool = browserTool(
6365export const PreviewNavigateTool = safeBrowserTool(
6466 Tool.make("preview_navigate", {
6567 description:
66 "Navigate the active collaborative browser tab. Pass {url:'https://t3.chat'} for a website, or {target:{kind:'environment-port',port:5173}} for a dev server in the current environment. Exactly one of url or target is required. Defaults to waiting for page loading to stop.",
68 "Navigate a collaborative browser tab. Pass tabId to target a specific tab, plus {url:'https://t3.chat'} for a website or {target:{kind:'environment-port',port:5173}} for a dev server. Exactly one of url or target is required.",
6769 parameters: PreviewAutomationNavigateInput,
6870 success: PreviewAutomationStatus,
6971 failure: PreviewAutomationError,
......@@ -74,7 +76,7 @@ export const PreviewNavigateTool = safeBrowserTool(
7476export const PreviewResizeTool = safeBrowserTool(
7577 Tool.make("preview_resize", {
7678 description:
77 "Set the active collaborative browser tab to fill-panel sizing, an independently resizable freeform size, or a Chrome-standard device preset. Use {mode:'fill'}, {mode:'freeform',width:1024,height:768}, or {mode:'preset',preset:'iphone-12-pro',orientation:'portrait'}. This changes CSS layout breakpoints without changing the desktop browser user agent.",
79 "Resize a collaborative browser tab, optionally selected by tabId. Use {mode:'fill'}, {mode:'freeform',width:1024,height:768}, or {mode:'preset',preset:'iphone-12-pro',orientation:'portrait'}. This changes CSS layout breakpoints without changing the desktop browser user agent.",
7880 parameters: PreviewAutomationResizeInput,
7981 success: PreviewAutomationResizeResult,
8082 failure: PreviewAutomationError,
......@@ -87,7 +89,8 @@ export const PreviewResizeTool = safeBrowserTool(
8789export const PreviewSnapshotTool = readonlyBrowserTool(
8890 Tool.make("preview_snapshot", {
8991 description:
90 "Inspect the current page before interacting. Returns URL/title/loading state, visible text, semantic interactive elements with reusable selectors and coordinates, accessibility data, recent console/network failures, action history, and a PNG screenshot.",
92 "Inspect a page before interacting. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab. Returns page state, semantic elements, diagnostics, action history, and a PNG screenshot.",
93 parameters: PreviewAutomationTabTargetInput,
9194 success: PreviewAutomationSnapshot,
9295 failure: PreviewAutomationError,
9396 dependencies,
......@@ -97,7 +100,7 @@ export const PreviewSnapshotTool = readonlyBrowserTool(
97100export const PreviewClickTool = browserTool(
98101 Tool.make("preview_click", {
99102 description:
100 "Click exactly one page target. Prefer locator with a Playwright selector such as role=button[name='Send']; selector accepts legacy CSS; x and y are viewport CSS pixels and must be supplied together. Call preview_snapshot first when the target is unknown.",
103 "Click exactly one target in the tab selected by tabId, or this agent session's current tab when omitted. Prefer a Playwright locator; selector accepts legacy CSS; x and y must be supplied together.",
101104 parameters: PreviewAutomationClickInput,
102105 success: Schema.Null,
103106 failure: PreviewAutomationError,
......@@ -108,7 +111,7 @@ export const PreviewClickTool = browserTool(
108111export const PreviewTypeTool = browserTool(
109112 Tool.make("preview_type", {
110113 description:
111 "Insert literal text into one input. Prefer locator with a Playwright role/text selector; selector accepts legacy CSS. If neither is supplied, types into the currently focused element. Set clear=true to replace existing text.",
114 "Insert literal text into one input in the tab selected by tabId, or this agent session's current tab when omitted. Prefer a Playwright locator; set clear=true to replace existing text.",
112115 parameters: PreviewAutomationTypeInput,
113116 success: Schema.Null,
114117 failure: PreviewAutomationError,
......@@ -119,7 +122,7 @@ export const PreviewTypeTool = browserTool(
119122export const PreviewPressTool = browserTool(
120123 Tool.make("preview_press", {
121124 description:
122 "Press one keyboard key in the active page, for example {key:'Enter'}, {key:'Escape'}, or {key:'a',modifiers:['Meta']}. This targets the page's current focus.",
125 "Press one keyboard key in the tab selected by tabId, or this agent session's current tab when omitted. Examples: {key:'Enter'}, {key:'Escape'}, or {key:'a',modifiers:['Meta']}.",
123126 parameters: PreviewAutomationPressInput,
124127 success: Schema.Null,
125128 failure: PreviewAutomationError,
......@@ -130,7 +133,7 @@ export const PreviewPressTool = browserTool(
130133export const PreviewScrollTool = safeBrowserTool(
131134 Tool.make("preview_scroll", {
132135 description:
133 "Scroll by CSS pixels. Positive deltaY scrolls down and positive deltaX scrolls right. Without locator/selector it scrolls the viewport; otherwise it scrolls that container. At least one delta is required.",
136 "Scroll the tab selected by tabId, or this agent session's current tab when omitted. Positive deltaY scrolls down and positive deltaX scrolls right; a locator/selector targets a container.",
134137 parameters: PreviewAutomationScrollInput,
135138 success: Schema.Null,
136139 failure: PreviewAutomationError,
......@@ -141,7 +144,7 @@ export const PreviewScrollTool = safeBrowserTool(
141144export const PreviewEvaluateTool = browserTool(
142145 Tool.make("preview_evaluate", {
143146 description:
144 "Evaluate a JavaScript expression in the page's main frame and return a serializable result up to 64 KB. Prefer preview_snapshot and semantic click/type/wait tools; use this for inspection or interactions those tools cannot express. The expression may mutate page state.",
147 "Evaluate JavaScript in the tab selected by tabId, or this agent session's current tab when omitted. Returns a serializable result up to 64 KB; the expression may mutate page state.",
145148 parameters: PreviewAutomationEvaluateInput,
146149 success: Schema.Unknown,
147150 failure: PreviewAutomationError,
......@@ -152,7 +155,7 @@ export const PreviewEvaluateTool = browserTool(
152155export const PreviewWaitForTool = readonlyBrowserTool(
153156 Tool.make("preview_wait_for", {
154157 description:
155 "Wait until all supplied conditions match: a Playwright locator, legacy CSS selector, visible-text substring, and/or URL substring. Provide at least one condition. Defaults to 15 seconds, maximum 60 seconds.",
158 "Wait in the tab selected by tabId, or this agent session's current tab when omitted, until all supplied locator, selector, text, and URL conditions match.",
156159 parameters: PreviewAutomationWaitForInput,
157160 success: Schema.Null,
158161 failure: PreviewAutomationError,
......@@ -163,7 +166,8 @@ export const PreviewWaitForTool = readonlyBrowserTool(
163166export const PreviewRecordingStartTool = safeBrowserTool(
164167 Tool.make("preview_recording_start", {
165168 description:
166 "Start recording the active collaborative browser tab while keeping it interactive for both agent and human use.",
169 "Start recording the collaborative browser tab selected by tabId, or this agent session's current tab when omitted.",
170 parameters: PreviewAutomationTabTargetInput,
167171 success: PreviewAutomationRecordingStatus,
168172 failure: PreviewAutomationError,
169173 dependencies,
......@@ -173,6 +177,7 @@ export const PreviewRecordingStartTool = safeBrowserTool(
173177export const PreviewRecordingStopTool = safeBrowserTool(
174178 Tool.make("preview_recording_stop", {
175179 description: "Stop the active browser recording and save it as a local evidence artifact.",
180 parameters: PreviewAutomationTabTargetInput,
176181 success: PreviewAutomationRecordingArtifact,
177182 failure: PreviewAutomationError,
178183 dependencies,
apps/server/src/provider/Layers/CursorAdapter.test.ts+12-17
......@@ -1021,24 +1021,19 @@ cursorAdapterTestLayer("CursorAdapterLive", (it) => {
10211021 assert.equal(turnCompleted.payload.stopReason, "cancelled");
10221022 }
10231023
1024 const requests = yield* waitForJsonLogMatch(
1025 requestLogPath,
1026 (entry) => entry.method === "session/cancel",
1027 );
1024 const isCancelledApprovalResponse = (entry: Record<string, unknown>) =>
1025 !("method" in entry) &&
1026 typeof entry.result === "object" &&
1027 entry.result !== null &&
1028 "outcome" in entry.result &&
1029 typeof entry.result.outcome === "object" &&
1030 entry.result.outcome !== null &&
1031 "outcome" in entry.result.outcome &&
1032 entry.result.outcome.outcome === "cancelled";
1033 yield* waitForJsonLogMatch(requestLogPath, (entry) => entry.method === "session/cancel");
1034 const requests = yield* waitForJsonLogMatch(requestLogPath, isCancelledApprovalResponse);
10281035 assert.isTrue(requests.some((entry) => entry.method === "session/cancel"));
1029 assert.isTrue(
1030 requests.some(
1031 (entry) =>
1032 !("method" in entry) &&
1033 typeof entry.result === "object" &&
1034 entry.result !== null &&
1035 "outcome" in entry.result &&
1036 typeof entry.result.outcome === "object" &&
1037 entry.result.outcome !== null &&
1038 "outcome" in entry.result.outcome &&
1039 entry.result.outcome.outcome === "cancelled",
1040 ),
1041 );
1036 assert.isTrue(requests.some(isCancelledApprovalResponse));
10421037
10431038 yield* adapter.stopSession(threadId);
10441039 }),
apps/web/src/browser/BrowserSurfaceSlot.tsx+6-6
......@@ -1,8 +1,8 @@
11"use client";
22
3import { useEffect, useRef } from "react";
3import { useLayoutEffect, useRef } from "react";
44
5import { useBrowserSurfaceStore } from "./browserSurfaceStore";
5import { acquireBrowserSurface } from "./browserSurfaceStore";
66
77export function BrowserSurfaceSlot(props: {
88 readonly tabId: string;
......@@ -12,13 +12,13 @@ export function BrowserSurfaceSlot(props: {
1212 const { tabId, visible, className } = props;
1313 const elementRef = useRef<HTMLDivElement | null>(null);
1414
15 useEffect(() => {
15 useLayoutEffect(() => {
1616 const element = elementRef.current;
1717 if (!element) return;
18 const lease = acquireBrowserSurface(tabId);
1819 const update = () => {
1920 const rect = element.getBoundingClientRect();
20 useBrowserSurfaceStore.getState().present(
21 tabId,
21 lease.present(
2222 {
2323 x: Math.round(rect.x),
2424 y: Math.round(rect.y),
......@@ -37,7 +37,7 @@ export function BrowserSurfaceSlot(props: {
3737 observer.disconnect();
3838 window.removeEventListener("resize", update);
3939 window.removeEventListener("scroll", update, true);
40 useBrowserSurfaceStore.getState().hide(tabId);
40 lease.release();
4141 };
4242 }, [tabId, visible]);
4343
apps/web/src/browser/HostedBrowserWebview.tsx+20-27
......@@ -8,13 +8,13 @@ import { previewBridge } from "~/components/preview/previewBridge";
88import { usePreviewBridge } from "~/components/preview/usePreviewBridge";
99import { cn } from "~/lib/utils";
1010
11import { useActiveBrowserRecordingTabId } from "./browserRecording";
11import { stopBrowserRecording, useActiveBrowserRecordingTabId } from "./browserRecording";
1212import { resolveBrowserSurfacePanelRect, useBrowserSurfaceStore } from "./browserSurfaceStore";
1313import { browserViewportSettingKey } from "./browserViewportLayout";
14import { reconcileLockedAspectRatio } from "./browserDeviceToolbarState";
1514import { BrowserDeviceToolbar } from "./BrowserDeviceToolbar";
1615import { BrowserViewportResizeHandles } from "./BrowserViewportResizeHandles";
1716import { acquireDesktopTab, type AcquiredDesktopTab } from "./desktopTabLifetime";
17import { resolveHostedBrowserWebviewWrapperStyle } from "./hostedBrowserWebviewStyle";
1818import { usePreviewWebviewConfig } from "./previewWebviewConfigState";
1919import { useBrowserViewportResize } from "./useBrowserViewportResize";
2020
......@@ -46,7 +46,8 @@ export function HostedBrowserWebview(props: {
4646 const tabLeaseRef = useRef<AcquiredDesktopTab | null>(null);
4747 const wrapperRef = useRef<HTMLDivElement | null>(null);
4848 const webviewRef = useRef<ElectronWebview | null>(null);
49 const [lockedAspectRatio, setLockedAspectRatio] = useState<number | null>(null);
49 const [aspectRatioLocked, setAspectRatioLocked] = useState(false);
50 const activeRecordingTabId = useActiveBrowserRecordingTabId();
5051 const presentation = useBrowserSurfaceStore(
5152 useShallow((state) => {
5253 const current = state.byTabId[tabId];
......@@ -56,10 +57,13 @@ export function HostedBrowserWebview(props: {
5657 };
5758 }),
5859 );
59 const recording = useActiveBrowserRecordingTabId() === tabId;
60
6160 usePreviewBridge({ threadRef, tabId });
6261
62 useEffect(() => {
63 if (presentation.visible || activeRecordingTabId !== tabId) return;
64 void stopBrowserRecording(tabId).catch(() => undefined);
65 }, [activeRecordingTabId, presentation.visible, tabId]);
66
6367 useEffect(() => {
6468 const lease = acquireDesktopTab(tabId);
6569 tabLeaseRef.current = lease;
......@@ -115,9 +119,11 @@ export function HostedBrowserWebview(props: {
115119 const viewportHeight = viewport._tag === "fill" ? null : viewport.height;
116120 const viewportAspectRatio =
117121 viewportWidth === null || viewportHeight === null ? null : viewportWidth / viewportHeight;
118 useEffect(() => {
119 setLockedAspectRatio((current) => reconcileLockedAspectRatio(current, viewportAspectRatio));
120 }, [viewportAspectRatio]);
122 const lockedAspectRatio =
123 aspectRatioLocked && viewportAspectRatio !== null ? viewportAspectRatio : null;
124 const handleAspectRatioChange = useCallback((aspectRatio: number | null) => {
125 setAspectRatioLocked(aspectRatio !== null);
126 }, []);
121127 const hiddenSize =
122128 viewport._tag !== "fill"
123129 ? {
......@@ -170,24 +176,11 @@ export function HostedBrowserWebview(props: {
170176
171177 if (!config) return null;
172178
173 const wrapperStyle =
174 active && lastRect
175 ? {
176 left: lastRect.x,
177 top: lastRect.y,
178 width: lastRect.width,
179 height: lastRect.height,
180 zIndex: 30,
181 pointerEvents: "auto" as const,
182 }
183 : {
184 left: 0,
185 top: 0,
186 width: hiddenSize.width,
187 height: hiddenSize.height,
188 zIndex: recording ? 0 : -1,
189 pointerEvents: "none" as const,
190 };
179 const wrapperStyle = resolveHostedBrowserWebviewWrapperStyle({
180 active,
181 rect: lastRect,
182 hiddenSize,
183 });
191184
192185 return (
193186 <div
......@@ -203,7 +196,7 @@ export function HostedBrowserWebview(props: {
203196 setting={effectiveViewport}
204197 width={Math.max(1, Math.round(containerSize.width))}
205198 aspectRatio={lockedAspectRatio}
206 onAspectRatioChange={setLockedAspectRatio}
199 onAspectRatioChange={handleAspectRatioChange}
207200 onChange={commitViewportChange}
208201 />
209202 ) : null}
apps/web/src/browser/browserRecording.test.ts created+294
......@@ -0,0 +1,294 @@
1import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test";
2
3const { events, onFrame, registrySet, save, startScreencast, stopScreencast, surfaceState } =
4 vi.hoisted(() => {
5 const events: string[] = [];
6 const surfaceState = {
7 byTabId: {} as Record<string, unknown>,
8 };
9 return {
10 events,
11 onFrame: vi.fn(() => vi.fn()),
12 registrySet: vi.fn((_atom: unknown, value: string | null) => {
13 events.push(value === null ? "clear" : `publish:${value}`);
14 }),
15 save: vi.fn(async () => ({
16 id: "recording-test",
17 tabId: "recording-tab",
18 path: "/tmp/recording-test.webm",
19 mimeType: "video/webm" as const,
20 sizeBytes: 0,
21 createdAt: "2026-06-26T00:00:00.000Z",
22 })),
23 startScreencast: vi.fn(async () => {
24 events.push("start-screencast");
25 }),
26 stopScreencast: vi.fn(async () => undefined),
27 surfaceState,
28 };
29 });
30
31vi.mock("~/components/preview/previewBridge", () => ({
32 previewBridge: {
33 recording: { onFrame, save, startScreencast, stopScreencast },
34 },
35}));
36
37vi.mock("~/rpc/atomRegistry", () => ({
38 appAtomRegistry: { set: registrySet },
39}));
40
41vi.mock("./browserSurfaceStore", () => ({
42 useBrowserSurfaceStore: {
43 getState: () => surfaceState,
44 },
45}));
46
47import {
48 BROWSER_RECORDING_STARTUP_SETTLE_TIMEOUT_MS,
49 BrowserRecordingConflictError,
50 BrowserRecordingOperationError,
51 BrowserRecordingRequiresVisibleTabError,
52 startBrowserRecording,
53 stopBrowserRecording,
54} from "./browserRecording";
55
56class FakeMediaRecorder {
57 static isTypeSupported(): boolean {
58 return true;
59 }
60
61 state: RecordingState = "inactive";
62 private readonly listeners = new Map<string, Set<EventListenerOrEventListenerObject>>();
63
64 addEventListener(type: string, listener: EventListenerOrEventListenerObject): void {
65 const listeners = this.listeners.get(type) ?? new Set();
66 listeners.add(listener);
67 this.listeners.set(type, listeners);
68 }
69
70 start(): void {
71 this.state = "recording";
72 }
73
74 stop(): void {
75 this.state = "inactive";
76 for (const listener of this.listeners.get("stop") ?? []) {
77 if (typeof listener === "function") listener(new Event("stop"));
78 else listener.handleEvent(new Event("stop"));
79 }
80 }
81}
82
83describe("browser recording", () => {
84 beforeEach(() => {
85 events.length = 0;
86 surfaceState.byTabId = {
87 "recording-tab": {
88 visible: true,
89 rect: { x: 0, y: 0, width: 800, height: 600 },
90 content: { x: 0, y: 0, width: 800, height: 600, scale: 1, scrollLeft: 0, scrollTop: 0 },
91 },
92 };
93 vi.clearAllMocks();
94 vi.stubGlobal("window", globalThis);
95 vi.stubGlobal("MediaRecorder", FakeMediaRecorder as unknown as typeof MediaRecorder);
96 vi.stubGlobal("document", {
97 createElement: () => ({
98 width: 0,
99 height: 0,
100 captureStream: () => ({}),
101 getContext: () => ({ drawImage: vi.fn() }),
102 }),
103 });
104 });
105
106 afterEach(() => {
107 vi.useRealTimers();
108 vi.unstubAllGlobals();
109 });
110
111 it("starts recording for a visible tab", async () => {
112 await startBrowserRecording("recording-tab");
113
114 expect(events).toEqual(["start-screencast", "publish:recording-tab"]);
115
116 await stopBrowserRecording("recording-tab");
117 });
118
119 it("rejects recording for a hidden tab before starting screencast", async () => {
120 surfaceState.byTabId = {
121 "recording-tab": {
122 visible: false,
123 rect: { x: 0, y: 0, width: 800, height: 600 },
124 content: { x: 0, y: 0, width: 800, height: 600, scale: 1, scrollLeft: 0, scrollTop: 0 },
125 },
126 };
127
128 await expect(startBrowserRecording("recording-tab")).rejects.toBeInstanceOf(
129 BrowserRecordingRequiresVisibleTabError,
130 );
131
132 expect(startScreencast).not.toHaveBeenCalled();
133 expect(registrySet).not.toHaveBeenCalled();
134 });
135
136 it("does not report success for a second start while the first is still starting", async () => {
137 let finishStartingScreencast: (() => void) | undefined;
138 startScreencast.mockImplementationOnce(async () => {
139 events.push("start-screencast");
140 await new Promise<void>((resolve) => {
141 finishStartingScreencast = resolve;
142 });
143 });
144
145 const firstStart = startBrowserRecording("recording-tab");
146 await vi.waitFor(() => expect(startScreencast).toHaveBeenCalledOnce());
147
148 await expect(startBrowserRecording("recording-tab")).rejects.toBeInstanceOf(
149 BrowserRecordingConflictError,
150 );
151
152 finishStartingScreencast?.();
153 await firstStart;
154 await stopBrowserRecording("recording-tab");
155 });
156
157 it("does not report success for a start while the recording is stopping", async () => {
158 let finishStoppingScreencast: (() => void) | undefined;
159 stopScreencast.mockImplementationOnce(async () => {
160 await new Promise<void>((resolve) => {
161 finishStoppingScreencast = resolve;
162 });
163 return undefined;
164 });
165
166 await startBrowserRecording("recording-tab");
167 const stopPromise = stopBrowserRecording("recording-tab");
168 await vi.waitFor(() => expect(stopScreencast).toHaveBeenCalledOnce());
169
170 await expect(startBrowserRecording("recording-tab")).rejects.toBeInstanceOf(
171 BrowserRecordingConflictError,
172 );
173
174 finishStoppingScreencast?.();
175 await stopPromise;
176 });
177
178 it("shares an in-progress stop with duplicate callers", async () => {
179 let finishStoppingScreencast: (() => void) | undefined;
180 stopScreencast.mockImplementationOnce(async () => {
181 await new Promise<void>((resolve) => {
182 finishStoppingScreencast = resolve;
183 });
184 return undefined;
185 });
186
187 await startBrowserRecording("recording-tab");
188 const firstStop = stopBrowserRecording("recording-tab");
189 await vi.waitFor(() => expect(stopScreencast).toHaveBeenCalledOnce());
190 const duplicateStop = stopBrowserRecording("recording-tab");
191
192 finishStoppingScreencast?.();
193 const [firstArtifact, duplicateArtifact] = await Promise.all([firstStop, duplicateStop]);
194
195 expect(duplicateArtifact).toEqual(firstArtifact);
196 expect(stopScreencast).toHaveBeenCalledOnce();
197 expect(save).toHaveBeenCalledOnce();
198 });
199
200 it("stops a screencast that finishes starting after cancellation", async () => {
201 let finishStartingScreencast: (() => void) | undefined;
202 startScreencast.mockImplementationOnce(async () => {
203 events.push("start-screencast");
204 await new Promise<void>((resolve) => {
205 finishStartingScreencast = resolve;
206 });
207 });
208
209 const startPromise = startBrowserRecording("recording-tab");
210 const rejectedStart = expect(startPromise).rejects.toBeInstanceOf(
211 BrowserRecordingOperationError,
212 );
213 await vi.waitFor(() => expect(startScreencast).toHaveBeenCalledOnce());
214
215 const stopPromise = stopBrowserRecording("recording-tab");
216 await vi.waitFor(() => expect(stopScreencast).toHaveBeenCalledOnce());
217 finishStartingScreencast?.();
218
219 await rejectedStart;
220 await stopPromise;
221 expect(stopScreencast).toHaveBeenCalledTimes(2);
222 expect(events.at(-1)).toBe("clear");
223 });
224
225 it("does not release the recording slot until a cancelled start settles", async () => {
226 let finishStartingScreencast: (() => void) | undefined;
227 startScreencast.mockImplementationOnce(async () => {
228 events.push("start-screencast");
229 await new Promise<void>((resolve) => {
230 finishStartingScreencast = resolve;
231 });
232 });
233
234 const firstStart = startBrowserRecording("recording-tab");
235 const rejectedFirstStart = expect(firstStart).rejects.toBeInstanceOf(
236 BrowserRecordingOperationError,
237 );
238 await vi.waitFor(() => expect(startScreencast).toHaveBeenCalledOnce());
239
240 const stopPromise = stopBrowserRecording("recording-tab");
241 const restartAfterStop = stopPromise.then(() => startBrowserRecording("recording-tab"));
242 await new Promise((resolve) => setTimeout(resolve, 0));
243 const startCallsBeforeFirstSettled = startScreencast.mock.calls.length;
244
245 finishStartingScreencast?.();
246 await rejectedFirstStart;
247 await stopPromise;
248 await restartAfterStop;
249 await stopBrowserRecording("recording-tab");
250
251 expect(startCallsBeforeFirstSettled).toBe(1);
252 });
253
254 it("fails a stop that waits too long for startup without freeing the recording slot", async () => {
255 vi.useFakeTimers();
256 let finishStartingScreencast: (() => void) | undefined;
257 startScreencast.mockImplementationOnce(async () => {
258 events.push("start-screencast");
259 await new Promise<void>((resolve) => {
260 finishStartingScreencast = resolve;
261 });
262 });
263
264 const startPromise = startBrowserRecording("recording-tab");
265 const rejectedStart = expect(startPromise).rejects.toBeInstanceOf(
266 BrowserRecordingOperationError,
267 );
268 expect(startScreencast).toHaveBeenCalledOnce();
269
270 const stopPromise = stopBrowserRecording("recording-tab");
271 await Promise.resolve();
272 await Promise.resolve();
273 expect(stopScreencast).toHaveBeenCalledOnce();
274
275 const rejection = expect(stopPromise).rejects.toMatchObject({
276 operation: "wait-startup",
277 tabId: "recording-tab",
278 });
279 await vi.advanceTimersByTimeAsync(BROWSER_RECORDING_STARTUP_SETTLE_TIMEOUT_MS);
280
281 await rejection;
282 expect(save).not.toHaveBeenCalled();
283 await expect(startBrowserRecording("recording-tab")).rejects.toBeInstanceOf(
284 BrowserRecordingConflictError,
285 );
286
287 finishStartingScreencast?.();
288 await rejectedStart;
289 const cleanupResult = await stopBrowserRecording("recording-tab");
290 expect(cleanupResult).toBeNull();
291 expect(save).not.toHaveBeenCalled();
292 expect(events.at(-1)).toBe("clear");
293 });
294});
apps/web/src/browser/browserRecording.ts+201-49
......@@ -46,6 +46,17 @@ export class BrowserRecordingCanvasUnavailableError extends Schema.TaggedErrorCl
4646 }
4747}
4848
49export class BrowserRecordingRequiresVisibleTabError extends Schema.TaggedErrorClass<BrowserRecordingRequiresVisibleTabError>()(
50 "BrowserRecordingRequiresVisibleTabError",
51 {
52 tabId: Schema.String,
53 },
54) {
55 override get message(): string {
56 return `Browser recording requires tab ${this.tabId} to be visible.`;
57 }
58}
59
4960export class BrowserRecordingOperationError extends Schema.TaggedErrorClass<BrowserRecordingOperationError>()(
5061 "BrowserRecordingOperationError",
5162 {
......@@ -55,6 +66,7 @@ export class BrowserRecordingOperationError extends Schema.TaggedErrorClass<Brow
5566 "start-media-recorder",
5667 "start-screencast",
5768 "stop-screencast",
69 "wait-startup",
5870 "stop-media-recorder",
5971 "save-artifact",
6072 "cleanup",
......@@ -68,6 +80,16 @@ export class BrowserRecordingOperationError extends Schema.TaggedErrorClass<Brow
6880 }
6981}
7082
83const isBrowserRecordingOperationError = Schema.is(BrowserRecordingOperationError);
84
85type BrowserRecordingLifecycle =
86 | { readonly phase: "starting" }
87 | { readonly phase: "recording" }
88 | {
89 readonly phase: "stopping";
90 readonly stopPromise: Promise<DesktopPreviewRecordingArtifact | null>;
91 };
92
7193interface ActiveRecording {
7294 readonly tabId: string;
7395 readonly canvas: HTMLCanvasElement;
......@@ -76,6 +98,8 @@ interface ActiveRecording {
7698 readonly chunks: Blob[];
7799 readonly mimeType: string;
78100 readonly startedAt: string;
101 readonly startupSettled: Promise<void>;
102 lifecycle: BrowserRecordingLifecycle;
79103}
80104
81105const activeBrowserRecordingTabIdAtom = Atom.make<string | null>(null).pipe(
......@@ -90,6 +114,8 @@ export function useActiveBrowserRecordingTabId(): string | null {
90114let active: ActiveRecording | null = null;
91115let unsubscribeFrames: (() => void) | null = null;
92116
117export const BROWSER_RECORDING_STARTUP_SETTLE_TIMEOUT_MS = 5_000;
118
93119export function readActiveBrowserRecordingTabId(): string | null {
94120 return active?.tabId ?? null;
95121}
......@@ -131,17 +157,58 @@ const clearActiveRecording = (recording: ActiveRecording): void => {
131157 appAtomRegistry.set(activeBrowserRecordingTabIdAtom, null);
132158};
133159
160const recordingStartupCancelledError = (
161 recording: ActiveRecording,
162 cause: unknown = new Error(`Browser recording startup was cancelled for tab ${recording.tabId}.`),
163): BrowserRecordingOperationError =>
164 new BrowserRecordingOperationError({
165 operation: "start-screencast",
166 tabId: recording.tabId,
167 cause,
168 });
169
170const isRecordingStarting = (recording: ActiveRecording): boolean =>
171 active === recording && recording.lifecycle.phase === "starting";
172
173const waitForRecordingStartupToSettle = async (recording: ActiveRecording): Promise<void> => {
174 let timeout: ReturnType<typeof setTimeout> | null = null;
175 try {
176 await Promise.race([
177 recording.startupSettled,
178 new Promise<void>((_, reject) => {
179 timeout = setTimeout(() => {
180 reject(new Error(`Browser recording startup did not settle for tab ${recording.tabId}.`));
181 }, BROWSER_RECORDING_STARTUP_SETTLE_TIMEOUT_MS);
182 }),
183 ]);
184 } catch (cause) {
185 throw new BrowserRecordingOperationError({
186 operation: "wait-startup",
187 tabId: recording.tabId,
188 cause,
189 });
190 } finally {
191 if (timeout !== null) clearTimeout(timeout);
192 }
193};
194
195const isStartupWaitTimeout = (error: unknown): error is BrowserRecordingOperationError =>
196 isBrowserRecordingOperationError(error) && error.operation === "wait-startup";
197
134198export async function startBrowserRecording(tabId: string): Promise<string> {
135199 const bridge = previewBridge;
136200 if (!bridge) throw new BrowserRecordingUnavailableError({ tabId });
137201 if (active) {
138 if (active.tabId === tabId) return active.startedAt;
202 if (active.tabId === tabId && active.lifecycle.phase === "recording") {
203 return active.startedAt;
204 }
139205 throw new BrowserRecordingConflictError({
140206 requestedTabId: tabId,
141207 activeTabId: active.tabId,
142208 });
143209 }
144210 const surface = useBrowserSurfaceStore.getState().byTabId[tabId];
211 if (!surface?.visible) throw new BrowserRecordingRequiresVisibleTabError({ tabId });
145212 const recordingSize = surface?.content ?? surface?.rect;
146213 const canvas = document.createElement("canvas");
147214 canvas.width = Math.max(1, recordingSize?.width ?? 1280);
......@@ -171,65 +238,104 @@ export async function startBrowserRecording(tabId: string): Promise<string> {
171238 }
172239 const startedAt = new Date().toISOString();
173240 const chunks: Blob[] = [];
241 let settleStartup: (() => void) | undefined;
242 const startupSettled = new Promise<void>((resolve) => {
243 settleStartup = resolve;
244 });
174245 recorder.addEventListener("dataavailable", (event) => {
175246 if (event.data.size > 0) chunks.push(event.data);
176247 });
177 const recording = { tabId, canvas, context, recorder, chunks, mimeType, startedAt };
248 const recording: ActiveRecording = {
249 tabId,
250 canvas,
251 context,
252 recorder,
253 chunks,
254 mimeType,
255 startedAt,
256 startupSettled,
257 lifecycle: { phase: "starting" },
258 };
178259 active = recording;
179260 try {
180 unsubscribeFrames ??= bridge.recording.onFrame(drawFrame);
181 } catch (cause) {
182 clearActiveRecording(recording);
183 throw new BrowserRecordingOperationError({
184 operation: "subscribe-frames",
185 tabId,
186 cause,
187 });
188 }
189 try {
190 recorder.start(1_000);
191 } catch (cause) {
192 clearActiveRecording(recording);
193 throw new BrowserRecordingOperationError({
194 operation: "start-media-recorder",
195 tabId,
196 cause,
197 });
198 }
199 try {
200 await bridge.recording.startScreencast(tabId);
201 } catch (cause) {
202 let cleanupCause: unknown;
203261 try {
204 await stopMediaRecorder(recorder);
205 } catch (error) {
206 cleanupCause = error;
207 } finally {
262 unsubscribeFrames ??= bridge.recording.onFrame(drawFrame);
263 } catch (cause) {
208264 clearActiveRecording(recording);
265 throw new BrowserRecordingOperationError({
266 operation: "subscribe-frames",
267 tabId,
268 cause,
269 });
209270 }
210 throw new BrowserRecordingOperationError({
211 operation: "start-screencast",
212 tabId,
213 cause:
214 cleanupCause === undefined
215 ? cause
216 : new AggregateError(
217 [cause, cleanupCause],
218 `Browser recording start and cleanup failed for tab ${tabId}.`,
219 { cause },
220 ),
221 });
271 try {
272 recorder.start(1_000);
273 } catch (cause) {
274 clearActiveRecording(recording);
275 throw new BrowserRecordingOperationError({
276 operation: "start-media-recorder",
277 tabId,
278 cause,
279 });
280 }
281 if (!isRecordingStarting(recording)) {
282 throw recordingStartupCancelledError(recording);
283 }
284 try {
285 await bridge.recording.startScreencast(tabId);
286 } catch (cause) {
287 if (!isRecordingStarting(recording)) {
288 throw recordingStartupCancelledError(recording, cause);
289 }
290 let cleanupCause: unknown;
291 try {
292 await stopMediaRecorder(recorder);
293 } catch (error) {
294 cleanupCause = error;
295 } finally {
296 clearActiveRecording(recording);
297 }
298 throw new BrowserRecordingOperationError({
299 operation: "start-screencast",
300 tabId,
301 cause:
302 cleanupCause === undefined
303 ? cause
304 : new AggregateError(
305 [cause, cleanupCause],
306 `Browser recording start and cleanup failed for tab ${tabId}.`,
307 { cause },
308 ),
309 });
310 }
311 if (!isRecordingStarting(recording)) {
312 try {
313 await bridge.recording.stopScreencast(tabId);
314 } catch (cause) {
315 throw recordingStartupCancelledError(
316 recording,
317 new AggregateError(
318 [new Error(`Browser recording startup was cancelled for tab ${tabId}.`), cause],
319 `Browser recording startup cancellation failed for tab ${tabId}.`,
320 { cause },
321 ),
322 );
323 }
324 throw recordingStartupCancelledError(recording);
325 }
326 recording.lifecycle = { phase: "recording" };
327 appAtomRegistry.set(activeBrowserRecordingTabIdAtom, tabId);
328 return startedAt;
329 } finally {
330 settleStartup?.();
222331 }
223 appAtomRegistry.set(activeBrowserRecordingTabIdAtom, tabId);
224 return startedAt;
225332}
226333
227export async function stopBrowserRecording(
228 tabId: string,
229): Promise<DesktopPreviewRecordingArtifact | null> {
230 const bridge = previewBridge;
231 const recording = active;
232 if (!bridge || !recording || recording.tabId !== tabId) return null;
334const finalizeBrowserRecording = async (
335 bridge: NonNullable<typeof previewBridge>,
336 recording: ActiveRecording,
337): Promise<DesktopPreviewRecordingArtifact> => {
338 const { tabId } = recording;
233339 let result:
234340 | { readonly _tag: "Success"; readonly artifact: DesktopPreviewRecordingArtifact }
235341 | { readonly _tag: "Failure"; readonly error: unknown };
......@@ -243,6 +349,7 @@ export async function stopBrowserRecording(
243349 cause,
244350 });
245351 }
352 await waitForRecordingStartupToSettle(recording);
246353 try {
247354 await stopMediaRecorder(recording.recorder);
248355 } catch (cause) {
......@@ -271,6 +378,14 @@ export async function stopBrowserRecording(
271378 result = { _tag: "Failure", error };
272379 }
273380
381 if (result._tag === "Failure" && isStartupWaitTimeout(result.error)) {
382 // Do not clear `active` yet. The renderer-side start promise can still
383 // resolve later, and its cancellation path will call `stopScreencast`.
384 // Keeping the slot reserved prevents a newer recording for this tab from
385 // being started and then accidentally stopped by the older late cleanup.
386 throw result.error;
387 }
388
274389 let cleanupError: BrowserRecordingOperationError | undefined;
275390 try {
276391 await stopMediaRecorder(recording.recorder);
......@@ -300,4 +415,41 @@ export async function stopBrowserRecording(
300415 }
301416 if (cleanupError) throw cleanupError;
302417 return result.artifact;
418};
419
420const discardBrowserRecording = async (
421 bridge: NonNullable<typeof previewBridge>,
422 recording: ActiveRecording,
423): Promise<null> => {
424 try {
425 await bridge.recording.stopScreencast(recording.tabId).catch(() => undefined);
426 await stopMediaRecorder(recording.recorder).catch(() => undefined);
427 return null;
428 } finally {
429 clearActiveRecording(recording);
430 }
431};
432
433export function stopBrowserRecording(
434 tabId: string,
435): Promise<DesktopPreviewRecordingArtifact | null> {
436 const bridge = previewBridge;
437 const recording = active;
438 if (!bridge || !recording || recording.tabId !== tabId) return Promise.resolve(null);
439 if (recording.lifecycle.phase === "stopping") return recording.lifecycle.stopPromise;
440
441 const stopPromise = Promise.resolve()
442 .then(() => finalizeBrowserRecording(bridge, recording))
443 .catch((error) => {
444 if (isStartupWaitTimeout(error) && active === recording) {
445 const cleanupAfterStartup = recording.startupSettled.then(() =>
446 discardBrowserRecording(bridge, recording),
447 );
448 recording.lifecycle = { phase: "stopping", stopPromise: cleanupAfterStartup };
449 void cleanupAfterStartup.catch(() => undefined);
450 }
451 throw error;
452 });
453 recording.lifecycle = { phase: "stopping", stopPromise };
454 return stopPromise;
303455}
apps/web/src/browser/browserRecordingScope.test.ts+6-1
......@@ -3,9 +3,14 @@ import { describe, expect, it } from "vite-plus/test";
33import { resolveBrowserRecordingStopTarget } from "./browserRecordingScope";
44
55describe("resolveBrowserRecordingStopTarget", () => {
6 it("stops the active recording even after the requested tab changes", () => {
6 it("stops the active recording when no explicit tab was requested", () => {
77 expect(resolveBrowserRecordingStopTarget("tab-a")).toBe("tab-a");
88 expect(resolveBrowserRecordingStopTarget("tab-b")).toBe("tab-b");
99 expect(resolveBrowserRecordingStopTarget(null)).toBeNull();
1010 });
11
12 it("only stops an explicitly requested tab when it owns the recording", () => {
13 expect(resolveBrowserRecordingStopTarget("tab-a", "tab-a")).toBe("tab-a");
14 expect(resolveBrowserRecordingStopTarget("tab-a", "tab-b")).toBeNull();
15 });
1116});
apps/web/src/browser/browserRecordingScope.ts+6-2
......@@ -1,3 +1,7 @@
1export function resolveBrowserRecordingStopTarget(activeTabId: string | null): string | null {
2 return activeTabId;
1export function resolveBrowserRecordingStopTarget(
2 activeTabId: string | null,
3 requestedTabId?: string,
4): string | null {
5 if (activeTabId === null) return null;
6 return requestedTabId === undefined || requestedTabId === activeTabId ? activeTabId : null;
37}
apps/web/src/browser/browserSurfaceStore.test.ts+44-4
......@@ -1,8 +1,16 @@
1import { describe, expect, it } from "vite-plus/test";
1import { beforeEach, describe, expect, it } from "vite-plus/test";
22
3import { resolveBrowserSurfacePanelRect, useBrowserSurfaceStore } from "./browserSurfaceStore";
3import {
4 acquireBrowserSurface,
5 resolveBrowserSurfacePanelRect,
6 useBrowserSurfaceStore,
7} from "./browserSurfaceStore";
48
59describe("browserSurfaceStore", () => {
10 beforeEach(() => {
11 useBrowserSurfaceStore.setState({ byTabId: {} });
12 });
13
614 it("tracks content dimensions for a browser that has never been visible", () => {
715 const tabId = "hidden-browser-surface-content-test";
816 useBrowserSurfaceStore.getState().presentContent(tabId, {
......@@ -28,11 +36,43 @@ describe("browserSurfaceStore", () => {
2836 expect(
2937 resolveBrowserSurfacePanelRect(
3038 {
31 hidden: { rect: staleRect, visible: false, content: null, updatedAt: 1 },
32 active: { rect: liveRect, visible: true, content: null, updatedAt: 2 },
39 hidden: { rect: staleRect, visible: false, content: null, updatedAt: 1, owner: null },
40 active: { rect: liveRect, visible: true, content: null, updatedAt: 2, owner: null },
3341 },
3442 "hidden",
3543 ),
3644 ).toEqual(liveRect);
3745 });
46
47 it("ignores updates and releases from a stale surface lease", () => {
48 const tabId = "leased-browser-surface";
49 const staleRect = { x: 0, y: 0, width: 500, height: 700 };
50 const liveRect = { x: 10, y: 20, width: 900, height: 640 };
51 const staleLease = acquireBrowserSurface(tabId);
52 staleLease.present(staleRect, true);
53
54 const liveLease = acquireBrowserSurface(tabId);
55 liveLease.present(liveRect, true);
56 staleLease.present(staleRect, true);
57 staleLease.release();
58
59 expect(useBrowserSurfaceStore.getState().byTabId[tabId]).toMatchObject({
60 rect: liveRect,
61 visible: true,
62 });
63 });
64
65 it("hides a surface when its current lease is released", () => {
66 const tabId = "released-browser-surface";
67 const lease = acquireBrowserSurface(tabId);
68 lease.present({ x: 10, y: 20, width: 900, height: 640 }, true);
69
70 lease.release();
71 lease.present({ x: 0, y: 0, width: 1, height: 1 }, true);
72
73 expect(useBrowserSurfaceStore.getState().byTabId[tabId]).toMatchObject({
74 visible: false,
75 owner: null,
76 });
77 });
3878});
apps/web/src/browser/browserSurfaceStore.ts+56-7
......@@ -12,6 +12,7 @@ export interface BrowserSurfacePresentation {
1212 readonly visible: boolean;
1313 readonly content: BrowserSurfaceContentPresentation | null;
1414 readonly updatedAt: number;
15 readonly owner: symbol | null;
1516}
1617
1718export interface BrowserSurfaceContentPresentation {
......@@ -26,9 +27,20 @@ export interface BrowserSurfaceContentPresentation {
2627
2728interface BrowserSurfaceStoreState {
2829 readonly byTabId: Record<string, BrowserSurfacePresentation>;
29 readonly present: (tabId: string, rect: BrowserSurfaceRect, visible: boolean) => void;
30 readonly claim: (tabId: string, owner: symbol) => void;
31 readonly present: (
32 tabId: string,
33 owner: symbol,
34 rect: BrowserSurfaceRect,
35 visible: boolean,
36 ) => void;
3037 readonly presentContent: (tabId: string, content: BrowserSurfaceContentPresentation) => void;
31 readonly hide: (tabId: string) => void;
38 readonly release: (tabId: string, owner: symbol) => void;
39}
40
41export interface BrowserSurfaceLease {
42 readonly present: (rect: BrowserSurfaceRect, visible: boolean) => void;
43 readonly release: () => void;
3244}
3345
3446export function resolveBrowserSurfacePanelRect(
......@@ -60,14 +72,32 @@ const rectEquals = (left: BrowserSurfaceRect | null, right: BrowserSurfaceRect):
6072
6173export const useBrowserSurfaceStore = create<BrowserSurfaceStoreState>()((set) => ({
6274 byTabId: {},
63 present: (tabId, rect, visible) =>
75 claim: (tabId, owner) =>
76 set((state) => {
77 const current = state.byTabId[tabId];
78 if (current?.owner === owner) return state;
79 return {
80 byTabId: {
81 ...state.byTabId,
82 [tabId]: {
83 rect: current?.rect ?? null,
84 visible: false,
85 content: current?.content ?? null,
86 updatedAt: Date.now(),
87 owner,
88 },
89 },
90 };
91 }),
92 present: (tabId, owner, rect, visible) =>
6493 set((state) => {
6594 const current = state.byTabId[tabId];
95 if (current?.owner !== owner) return state;
6696 if (current && current.visible === visible && rectEquals(current.rect, rect)) return state;
6797 return {
6898 byTabId: {
6999 ...state.byTabId,
70 [tabId]: { rect, visible, content: current?.content ?? null, updatedAt: Date.now() },
100 [tabId]: { ...current, rect, visible, updatedAt: Date.now() },
71101 },
72102 };
73103 }),
......@@ -83,6 +113,7 @@ export const useBrowserSurfaceStore = create<BrowserSurfaceStoreState>()((set) =
83113 visible: false,
84114 content,
85115 updatedAt: Date.now(),
116 owner: null,
86117 },
87118 },
88119 };
......@@ -107,15 +138,33 @@ export const useBrowserSurfaceStore = create<BrowserSurfaceStoreState>()((set) =
107138 },
108139 };
109140 }),
110 hide: (tabId) =>
141 release: (tabId, owner) =>
111142 set((state) => {
112143 const current = state.byTabId[tabId];
113 if (!current || !current.visible) return state;
144 if (current?.owner !== owner) return state;
114145 return {
115146 byTabId: {
116147 ...state.byTabId,
117 [tabId]: { ...current, visible: false, updatedAt: Date.now() },
148 [tabId]: { ...current, visible: false, updatedAt: Date.now(), owner: null },
118149 },
119150 };
120151 }),
121152}));
153
154export function acquireBrowserSurface(tabId: string): BrowserSurfaceLease {
155 const owner = Symbol(`browser-surface:${tabId}`);
156 let released = false;
157 useBrowserSurfaceStore.getState().claim(tabId, owner);
158
159 return {
160 present: (rect, visible) => {
161 if (released) return;
162 useBrowserSurfaceStore.getState().present(tabId, owner, rect, visible);
163 },
164 release: () => {
165 if (released) return;
166 released = true;
167 useBrowserSurfaceStore.getState().release(tabId, owner);
168 },
169 };
170}
apps/web/src/browser/hostedBrowserWebviewStyle.test.ts created+43
......@@ -0,0 +1,43 @@
1import { describe, expect, it } from "vite-plus/test";
2
3import {
4 HIDDEN_BROWSER_WEBVIEW_OFFSET,
5 resolveHostedBrowserWebviewWrapperStyle,
6} from "./hostedBrowserWebviewStyle";
7
8describe("resolveHostedBrowserWebviewWrapperStyle", () => {
9 it("places an active webview on its presented surface", () => {
10 expect(
11 resolveHostedBrowserWebviewWrapperStyle({
12 active: true,
13 rect: { x: 12, y: 34, width: 800, height: 600 },
14 hiddenSize: { width: 1280, height: 800 },
15 }),
16 ).toEqual({
17 left: 12,
18 top: 34,
19 width: 800,
20 height: 600,
21 zIndex: 30,
22 pointerEvents: "auto",
23 });
24 });
25
26 it("keeps an inactive webview paintable while moving it offscreen", () => {
27 const style = resolveHostedBrowserWebviewWrapperStyle({
28 active: false,
29 rect: { x: 12, y: 34, width: 800, height: 600 },
30 hiddenSize: { width: 393, height: 852 },
31 });
32
33 expect(style).toEqual({
34 left: HIDDEN_BROWSER_WEBVIEW_OFFSET,
35 top: HIDDEN_BROWSER_WEBVIEW_OFFSET,
36 width: 393,
37 height: 852,
38 zIndex: -1,
39 pointerEvents: "none",
40 visibility: "visible",
41 });
42 });
43});
apps/web/src/browser/hostedBrowserWebviewStyle.ts created+49
......@@ -0,0 +1,49 @@
1import type { BrowserSurfaceRect } from "./browserSurfaceStore";
2
3export interface HostedBrowserWebviewSize {
4 readonly width: number;
5 readonly height: number;
6}
7
8export interface HostedBrowserWebviewWrapperStyle {
9 readonly left: number;
10 readonly top: number;
11 readonly width: number;
12 readonly height: number;
13 readonly zIndex: number;
14 readonly pointerEvents: "auto" | "none";
15 readonly visibility?: "visible";
16}
17
18export const HIDDEN_BROWSER_WEBVIEW_OFFSET = -100_000;
19
20export function resolveHostedBrowserWebviewWrapperStyle(input: {
21 readonly active: boolean;
22 readonly rect: BrowserSurfaceRect | null;
23 readonly hiddenSize: HostedBrowserWebviewSize;
24}): HostedBrowserWebviewWrapperStyle {
25 const { active, hiddenSize, rect } = input;
26 if (active && rect) {
27 return {
28 left: rect.x,
29 top: rect.y,
30 width: rect.width,
31 height: rect.height,
32 zIndex: 30,
33 pointerEvents: "auto",
34 };
35 }
36
37 return {
38 left: HIDDEN_BROWSER_WEBVIEW_OFFSET,
39 top: HIDDEN_BROWSER_WEBVIEW_OFFSET,
40 width: hiddenSize.width,
41 height: hiddenSize.height,
42 zIndex: -1,
43 pointerEvents: "none",
44 // Keep the guest CSS-visible even while physically offscreen. Electron
45 // webviews can keep metadata/status alive under `visibility:hidden` while
46 // CDP Runtime/Input commands stall, which breaks offscreen automation.
47 visibility: "visible",
48 };
49}
apps/web/src/components/preview/PreviewAutomationHosts.tsx+31-12
......@@ -56,6 +56,7 @@ import { createPreviewAutomationRequestConsumerAtom } from "./previewAutomationR
5656import { createPreviewAutomationClientId } from "./previewAutomationClientId";
5757import {
5858 needsPreviewAutomationSessionSync,
59 resolvePreviewAutomationOpenTab,
5960 resolvePreviewAutomationTarget,
6061} from "./previewAutomationTarget";
6162import { isPreviewViewportReady } from "./previewViewportReadiness";
......@@ -230,6 +231,16 @@ const currentStatus = async (
230231 };
231232};
232233
234const raiseAtomCommandFailure = (result: Parameters<typeof squashAtomCommandFailure>[0]): never => {
235 throw squashAtomCommandFailure(result);
236};
237
238const raisePreviewAutomationHostError = (
239 error: PreviewAutomationRecordingNotActiveError,
240): never => {
241 throw error;
242};
243
233244export function PreviewAutomationHosts() {
234245 const { environments } = useEnvironments();
235246 if (!isElectron || !previewBridge?.automation) return null;
......@@ -304,7 +315,7 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId })
304315 registry.refresh(previewEnvironment.list(listTarget));
305316 const result = await listPreviews(listTarget);
306317 if (result._tag === "Failure") {
307 throw squashAtomCommandFailure(result);
318 return raiseAtomCommandFailure(result);
308319 }
309320 reconcilePreviewServerSessions(threadRef, result.value.sessions);
310321 state = readThreadPreviewState(threadRef);
......@@ -332,8 +343,11 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId })
332343 return await currentStatus(threadRef, tabId);
333344 case "open": {
334345 const input = request.input as PreviewAutomationOpenInput;
335 let activeTabId =
336 (input.reuseExistingTab ?? true) ? (state.snapshot?.tabId ?? null) : null;
346 let activeTabId = resolvePreviewAutomationOpenTab(
347 state,
348 request.tabId,
349 input.reuseExistingTab ?? true,
350 );
337351 let activeSnapshot = activeTabId
338352 ? (state.sessions[activeTabId] ?? state.snapshot ?? undefined)
339353 : undefined;
......@@ -348,7 +362,7 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId })
348362 },
349363 });
350364 if (result._tag === "Failure") {
351 throw squashAtomCommandFailure(result);
365 return raiseAtomCommandFailure(result);
352366 }
353367 const snapshot = result.value;
354368 applyPreviewServerSnapshot(threadRef, snapshot);
......@@ -416,7 +430,7 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId })
416430 },
417431 });
418432 if (result._tag === "Failure") {
419 throw squashAtomCommandFailure(result);
433 return raiseAtomCommandFailure(result);
420434 }
421435 updatePreviewServerSnapshot(threadRef, result.value);
422436 const viewport = await waitForRenderedViewport(
......@@ -492,15 +506,20 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId })
492506 }
493507 case "recordingStop": {
494508 const recordingTabId = readActiveBrowserRecordingTabId();
495 const stopTabId = resolveBrowserRecordingStopTarget(recordingTabId);
509 const stopTabId = resolveBrowserRecordingStopTarget(
510 recordingTabId,
511 request.tabIdExplicit ? request.tabId : undefined,
512 );
496513 const artifact = stopTabId ? await stopBrowserRecording(stopTabId) : null;
497514 if (!artifact) {
498 throw new PreviewAutomationRecordingNotActiveError({
499 requestId: request.requestId,
500 environmentId,
501 threadId: request.threadId,
502 tabId,
503 });
515 return raisePreviewAutomationHostError(
516 new PreviewAutomationRecordingNotActiveError({
517 requestId: request.requestId,
518 environmentId,
519 threadId: request.threadId,
520 tabId,
521 }),
522 );
504523 }
505524 return artifact;
506525 }
apps/web/src/components/preview/previewAutomationTarget.test.ts+14
......@@ -3,6 +3,7 @@ import { describe, expect, it } from "vite-plus/test";
33
44import {
55 needsPreviewAutomationSessionSync,
6 resolvePreviewAutomationOpenTab,
67 resolvePreviewAutomationTarget,
78} from "./previewAutomationTarget";
89
......@@ -42,4 +43,17 @@ describe("preview automation target selection", () => {
4243 ),
4344 ).toEqual({ tabId: null, snapshot: null });
4445 });
46
47 it("reuses the provider session's pinned tab instead of the mutable UI tab", () => {
48 const uiActive = snapshot("tab-ui-active");
49 const agentTab = snapshot("tab-opened-by-agent");
50 const state = {
51 snapshot: uiActive,
52 sessions: { [uiActive.tabId]: uiActive, [agentTab.tabId]: agentTab },
53 };
54
55 expect(resolvePreviewAutomationOpenTab(state, agentTab.tabId, true)).toBe(agentTab.tabId);
56 expect(resolvePreviewAutomationOpenTab(state, undefined, true)).toBe(uiActive.tabId);
57 expect(resolvePreviewAutomationOpenTab(state, agentTab.tabId, false)).toBeNull();
58 });
4559});
apps/web/src/components/preview/previewAutomationTarget.ts+12
......@@ -23,3 +23,15 @@ export function resolvePreviewAutomationTarget(
2323 const snapshot = requestedTabId ? (state.sessions[requestedTabId] ?? null) : state.snapshot;
2424 return { tabId: snapshot?.tabId ?? null, snapshot };
2525}
26
27export function resolvePreviewAutomationOpenTab(
28 state: PreviewAutomationSessionIndex,
29 requestedTabId: string | undefined,
30 reuseExistingTab: boolean,
31): string | null {
32 if (!reuseExistingTab) return null;
33 if (requestedTabId !== undefined) {
34 return state.sessions[requestedTabId]?.tabId ?? null;
35 }
36 return state.snapshot?.tabId ?? null;
37}
packages/contracts/src/preview.test.ts+16
......@@ -11,6 +11,7 @@ import {
1111import {
1212 PreviewAutomationHost,
1313 PreviewAutomationError,
14 PreviewAutomationOpenInput,
1415 PreviewAutomationResizeInput,
1516 PreviewAutomationResizeResult,
1617 PreviewAutomationStatus,
......@@ -22,6 +23,7 @@ const decodeNavStatus = Schema.decodeUnknownSync(PreviewNavStatus);
2223const decodeServer = Schema.decodeUnknownSync(DiscoveredLocalServer);
2324const decodeViewport = Schema.decodeUnknownSync(PreviewViewportSetting);
2425const decodeResizeInput = Schema.decodeUnknownSync(PreviewAutomationResizeInput);
26const decodeOpenInput = Schema.decodeUnknownSync(PreviewAutomationOpenInput);
2527const decodeResizeResult = Schema.decodeUnknownSync(PreviewAutomationResizeResult);
2628const decodeAutomationHost = Schema.decodeUnknownSync(PreviewAutomationHost);
2729const decodeAutomationError = Schema.decodeUnknownSync(PreviewAutomationError);
......@@ -128,6 +130,20 @@ describe("PreviewAutomationResizeInput", () => {
128130 });
129131});
130132
133describe("preview automation tab targeting", () => {
134 it("accepts an explicit tab and rejects contradictory open behavior", () => {
135 expect(decodeResizeInput({ tabId: "tab-app", mode: "fill" })).toMatchObject({
136 tabId: "tab-app",
137 mode: "fill",
138 });
139 expect(decodeOpenInput({ tabId: "tab-app", reuseExistingTab: true })).toMatchObject({
140 tabId: "tab-app",
141 reuseExistingTab: true,
142 });
143 expect(() => decodeOpenInput({ tabId: "tab-app", reuseExistingTab: false })).toThrow();
144 });
145});
146
131147describe("PreviewAutomationHost", () => {
132148 it("accepts legacy hosts and current operation advertisements", () => {
133149 expect(decodeAutomationHost({ clientId: "legacy", environmentId: "environment-1" })).toEqual({
packages/contracts/src/previewAutomation.ts+38-5
......@@ -50,6 +50,21 @@ export const PREVIEW_AUTOMATION_OPERATIONS = [
5050export const PreviewAutomationOperation = Schema.Literals(PREVIEW_AUTOMATION_OPERATIONS);
5151export type PreviewAutomationOperation = typeof PreviewAutomationOperation.Type;
5252
53const PreviewAutomationTabTargetFields = {
54 tabId: Schema.optional(
55 PreviewTabId.annotate({
56 description:
57 "Exact collaborative browser tab to target. Omit to use this agent session's current tab.",
58 }),
59 ).annotate({
60 description:
61 "Exact collaborative browser tab to target. Omit to use this agent session's current tab.",
62 }),
63};
64
65export const PreviewAutomationTabTargetInput = Schema.Struct(PreviewAutomationTabTargetFields);
66export type PreviewAutomationTabTargetInput = typeof PreviewAutomationTabTargetInput.Type;
67
5368export const PreviewAutomationStatus = Schema.Struct({
5469 available: Schema.Boolean,
5570 visible: Schema.Boolean,
......@@ -65,6 +80,7 @@ export const PreviewAutomationStatus = Schema.Struct({
6580export type PreviewAutomationStatus = typeof PreviewAutomationStatus.Type;
6681
6782export const PreviewAutomationOpenInput = Schema.Struct({
83 ...PreviewAutomationTabTargetFields,
6884 url: Schema.optional(BoundedUrl).annotate({
6985 description:
7086 "Optional initial page URL, for example https://t3.chat or localhost:5173. Omit to open a blank tab.",
......@@ -77,13 +93,21 @@ export const PreviewAutomationOpenInput = Schema.Struct({
7793 reuseExistingTab: Schema.optional(
7894 Schema.Boolean.annotate({
7995 description:
80 "Reuse the thread's active browser tab when available. Defaults to true; set false to create a new tab.",
96 "Reuse tabId when supplied, otherwise this agent session's current tab. Defaults to true; set false to create a new tab.",
8197 }),
8298 ),
83}).annotate({
84 description:
85 "Opens the collaborative browser for the current thread. Use preview_navigate afterward when readiness waiting matters.",
86});
99})
100 .check(
101 Schema.makeFilter(
102 (input) =>
103 !(input.tabId !== undefined && input.reuseExistingTab === false) ||
104 "tabId cannot be combined with reuseExistingTab=false.",
105 ),
106 )
107 .annotate({
108 description:
109 "Opens the collaborative browser for the current thread. Use preview_navigate afterward when readiness waiting matters.",
110 });
87111export type PreviewAutomationOpenInput = typeof PreviewAutomationOpenInput.Type;
88112
89113export const BrowserNavigationTarget = Schema.Union([
......@@ -117,6 +141,7 @@ export const BrowserNavigationTarget = Schema.Union([
117141export type BrowserNavigationTarget = typeof BrowserNavigationTarget.Type;
118142
119143export const PreviewAutomationNavigateInput = Schema.Struct({
144 ...PreviewAutomationTabTargetFields,
120145 url: Schema.optional(BoundedUrl).annotate({
121146 description:
122147 "Website URL, for example https://t3.chat. Use this for public pages and directly reachable URLs.",
......@@ -155,6 +180,7 @@ export const PreviewAutomationNavigateInput = Schema.Struct({
155180export type PreviewAutomationNavigateInput = typeof PreviewAutomationNavigateInput.Type;
156181
157182export const PreviewAutomationResizeInput = Schema.Struct({
183 ...PreviewAutomationTabTargetFields,
158184 mode: Schema.Literals(["fill", "freeform", "preset"]).annotate({
159185 description:
160186 "Viewport mode: fill follows the preview panel, freeform uses exact independently resizable dimensions, and preset uses a named device size.",
......@@ -239,6 +265,7 @@ const LegacySelector = TrimmedNonEmptyString.annotate({
239265});
240266
241267export const PreviewAutomationClickInput = Schema.Struct({
268 ...PreviewAutomationTabTargetFields,
242269 selector: Schema.optional(LegacySelector).annotate({
243270 description:
244271 "Legacy CSS selector such as button[type='submit']. Prefer locator for resilient role/text targeting.",
......@@ -277,6 +304,7 @@ export const PreviewAutomationClickInput = Schema.Struct({
277304export type PreviewAutomationClickInput = typeof PreviewAutomationClickInput.Type;
278305
279306export const PreviewAutomationTypeInput = Schema.Struct({
307 ...PreviewAutomationTabTargetFields,
280308 text: Schema.String.annotate({ description: "Literal text to insert." }),
281309 selector: Schema.optional(LegacySelector).annotate({
282310 description: "Legacy CSS selector for the input. Prefer locator.",
......@@ -306,6 +334,7 @@ export const PreviewAutomationTypeInput = Schema.Struct({
306334export type PreviewAutomationTypeInput = typeof PreviewAutomationTypeInput.Type;
307335
308336export const PreviewAutomationPressInput = Schema.Struct({
337 ...PreviewAutomationTabTargetFields,
309338 key: Schema.String.check(Schema.isTrimmed())
310339 .check(
311340 Schema.isNonEmpty({
......@@ -326,6 +355,7 @@ export const PreviewAutomationPressInput = Schema.Struct({
326355export type PreviewAutomationPressInput = typeof PreviewAutomationPressInput.Type;
327356
328357export const PreviewAutomationScrollInput = Schema.Struct({
358 ...PreviewAutomationTabTargetFields,
329359 deltaX: Schema.optional(
330360 Schema.Finite.annotate({
331361 description: "Horizontal scroll delta in CSS pixels. Positive scrolls right. Defaults to 0.",
......@@ -360,6 +390,7 @@ export const PreviewAutomationScrollInput = Schema.Struct({
360390export type PreviewAutomationScrollInput = typeof PreviewAutomationScrollInput.Type;
361391
362392export const PreviewAutomationEvaluateInput = Schema.Struct({
393 ...PreviewAutomationTabTargetFields,
363394 expression: Schema.String.check(Schema.isTrimmed())
364395 .check(
365396 Schema.isNonEmpty({
......@@ -388,6 +419,7 @@ export const PreviewAutomationEvaluateInput = Schema.Struct({
388419export type PreviewAutomationEvaluateInput = typeof PreviewAutomationEvaluateInput.Type;
389420
390421export const PreviewAutomationWaitForInput = Schema.Struct({
422 ...PreviewAutomationTabTargetFields,
391423 selector: Schema.optional(LegacySelector).annotate({
392424 description: "Legacy CSS selector that must match an element. Prefer locator.",
393425 }),
......@@ -537,6 +569,7 @@ export const PreviewAutomationRequest = Schema.Struct({
537569 requestId: TrimmedNonEmptyString,
538570 threadId: ThreadId,
539571 tabId: Schema.optional(PreviewTabId),
572 tabIdExplicit: Schema.optional(Schema.Boolean),
540573 operation: PreviewAutomationOperation,
541574 input: Schema.Unknown,
542575 timeoutMs: Schema.Int.check(Schema.isGreaterThan(0)),