authorgravatar for 71498452+StiensWout@users.noreply.github.comWout Stiens <71498452+StiensWout@users.noreply.github.com> 2026-09-01 21:52:36+02:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-09-01 12:52:36-07:00
logef7014d851f56bb037a9da963095ffd883c7fa08
treed237fbf682bdb17fe762bc39340cb597543cc935
parentc17d02cff98f2e7b590d4c3a5775d1faa47c2e7b
signature Signed by PGP key B5690EEEBB952194

fix(preview): restore recording and macOS rendering after Electron 43 (#9001)

Co-authored-by: Guillermo Casanova <guillermo.casanova.b@gmail.com>

11 files changed, 1068 insertions(+), 335 deletions(-)

apps/desktop/src/ipc/methods/preview.ts+5-10
......@@ -11,7 +11,6 @@ import {
1111 DesktopPreviewConfigInputSchema,
1212 DesktopPreviewNavigateInputSchema,
1313 DesktopPreviewRecordingArtifactSchema,
14 DesktopPreviewRecordingSourceSchema,
1514 DesktopPreviewRecordingSaveInputSchema,
1615 DesktopPreviewRegisterWebviewInputSchema,
1716 DesktopPreviewScreenshotArtifactSchema,
......@@ -174,15 +173,11 @@ export const cancelPickElement = tabMethod(
174173 "desktop.ipc.preview.cancelPickElement",
175174 (manager, tabId) => manager.cancelPickElement(tabId),
176175);
177export const startRecording = DesktopIpc.makeIpcMethod({
178 channel: IpcChannels.PREVIEW_RECORDING_START_CHANNEL,
179 payload: DesktopPreviewTabInputSchema,
180 result: DesktopPreviewRecordingSourceSchema,
181 handler: Effect.fn("desktop.ipc.preview.startRecording")(function* ({ tabId }) {
182 const manager = yield* PreviewManager.PreviewManager;
183 return yield* manager.startRecording(tabId);
184 }),
185});
176export const startRecording = tabMethod(
177 IpcChannels.PREVIEW_RECORDING_START_CHANNEL,
178 "desktop.ipc.preview.startRecording",
179 (manager, tabId) => manager.startRecording(tabId),
180);
186181export const stopRecording = tabMethod(
187182 IpcChannels.PREVIEW_RECORDING_STOP_CHANNEL,
188183 "desktop.ipc.preview.stopRecording",
apps/desktop/src/preview/Manager.test.ts+375-136
......@@ -1,4 +1,5 @@
11import { it as effectIt } from "@effect/vitest";
2import { DESKTOP_PREVIEW_RECORDING_CAPTURE_TRIGGER } from "@t3tools/contracts";
23import type { DesktopPreviewRecordingFrame } from "@t3tools/contracts";
34import { HostProcessPlatform } from "@t3tools/shared/hostProcess";
45import * as Cause from "effect/Cause";
......@@ -123,7 +124,7 @@ const {
123124} = vi.hoisted(() => ({
124125 browserWindowConstructor: vi.fn(),
125126 createFromPath: vi.fn((): { readonly isEmpty: () => boolean } => ({ isEmpty: () => false })),
126 fromId: vi.fn((_id?: number) => null),
127 fromId: vi.fn<(_id?: number) => Electron.WebContents | null>((_id?: number) => null),
127128 getFocusedWebContents: vi.fn(() => null),
128129 mkdir: vi.fn((_path: string) => undefined),
129130 showItemInFolder: vi.fn(),
......@@ -209,14 +210,52 @@ interface TestCapturedPreviewImage {
209210 readonly getSize: () => { readonly width: number; readonly height: number };
210211}
211212
213type TestDisplayMediaHandler = (
214 request: { readonly frame: { readonly frameTreeNodeId: number } | null },
215 callback: (streams: { video?: unknown }) => void,
216) => void;
217
218interface TestHostWebContents {
219 readonly id: number;
220 readonly mainFrame: { readonly frameTreeNodeId: number };
221 readonly executeJavaScript: ReturnType<typeof vi.fn>;
222 readonly isDestroyed: () => boolean;
223 readonly session: {
224 readonly setDisplayMediaRequestHandler: ReturnType<typeof vi.fn>;
225 };
226 readonly displayMediaHandler: () => TestDisplayMediaHandler | undefined;
227}
228
229type TestPreviewWebContents = Electron.WebContents & {
230 readonly setBackgroundThrottling: ReturnType<typeof vi.fn<(enabled: boolean) => void>>;
231};
232
233const makeTestHostWebContents = (): TestHostWebContents => {
234 let handler: TestDisplayMediaHandler | undefined;
235 return {
236 id: 7,
237 mainFrame: { frameTreeNodeId: 7 },
238 executeJavaScript: vi.fn(async () => true),
239 isDestroyed: () => false,
240 session: {
241 setDisplayMediaRequestHandler: vi.fn((next: TestDisplayMediaHandler) => {
242 handler = next;
243 }),
244 },
245 displayMediaHandler: () => handler,
246 };
247};
248
212249const makeTestPreviewWebContents = (
213250 capturePage: () => Promise<TestCapturedPreviewImage>,
214251 id = 42,
215) =>
216 ({
252 hostWebContents: TestHostWebContents = makeTestHostWebContents(),
253) => {
254 const setBackgroundThrottling = vi.fn<(enabled: boolean) => void>();
255 return {
217256 id,
218 hostWebContents: { id: 7 },
219 getMediaSourceId: vi.fn(() => `tab:${id}`),
257 mainFrame: { routingId: id },
258 hostWebContents,
220259 executeJavaScript: vi.fn(async () => ({ width: 1280, height: 720 })),
221260 isDestroyed: () => false,
222261 getType: () => "webview",
......@@ -226,6 +265,7 @@ const makeTestPreviewWebContents = (
226265 getZoomFactor: () => 1,
227266 setZoomFactor: vi.fn(),
228267 setAudioMuted: vi.fn(),
268 setBackgroundThrottling,
229269 isCurrentlyAudible: () => false,
230270 on: vi.fn(),
231271 off: vi.fn(),
......@@ -241,7 +281,45 @@ const makeTestPreviewWebContents = (
241281 off: vi.fn(),
242282 },
243283 capturePage,
244 }) as never;
284 } as unknown as TestPreviewWebContents;
285};
286
287/** Two ready tabs (41, 42) sharing one window, so they contend for the single display-media slot. */
288const setupRecordingRaceTabs = (manager: PreviewManager.PreviewManager["Service"]) =>
289 Effect.gen(function* () {
290 const capturePage = vi.fn(async () => ({
291 toJPEG: () => Buffer.from("unused-recording-frame"),
292 getSize: () => ({ width: 1280, height: 720 }),
293 }));
294 const host = makeTestHostWebContents();
295 const destroyedIds = new Set<number>();
296 const makeWebContents = (id: number) =>
297 Object.assign(makeTestPreviewWebContents(capturePage, id, host), {
298 executeJavaScript: vi.fn(async () => ({ width: 1280, height: 720 })),
299 isDestroyed: () => destroyedIds.has(id),
300 });
301 const webContentsById = new Map([
302 [41, makeWebContents(41)],
303 [42, makeWebContents(42)],
304 ]);
305 fromId.mockImplementation((id) =>
306 id === undefined ? null : (webContentsById.get(id) ?? null),
307 );
308 yield* manager.createTab("tab_race_a");
309 yield* manager.createTab("tab_race_b");
310 yield* manager.registerWebview("tab_race_a", 41);
311 yield* manager.registerWebview("tab_race_b", 42);
312 const grants: Array<{ video?: unknown }> = [];
313 return {
314 host,
315 grants,
316 destroy: (id: number) => destroyedIds.add(id),
317 takeGrant: (frame = host.mainFrame) =>
318 host.displayMediaHandler()?.({ frame }, (value) => {
319 grants.push(value);
320 }),
321 };
322 });
245323
246324const TEST_FAVICON = "data:image/png;base64,cG5n";
247325
......@@ -1865,7 +1943,7 @@ describe("PreviewManager", () => {
18651943 ),
18661944 );
18671945
1868 effectIt.effect("keeps window unthrottled until the final frame capture stops", () =>
1946 effectIt.effect("keeps every recorded guest unthrottled until its frame capture stops", () =>
18691947 withManager((manager) =>
18701948 Effect.gen(function* () {
18711949 const setBackgroundThrottling = vi.fn();
......@@ -1873,9 +1951,12 @@ describe("PreviewManager", () => {
18731951 toJPEG: () => Buffer.from("recording-frame"),
18741952 getSize: () => ({ width: 1280, height: 720 }),
18751953 }));
1954 const host = makeTestHostWebContents();
1955 const firstWebContents = makeTestPreviewWebContents(capturePage, 41, host);
1956 const secondWebContents = makeTestPreviewWebContents(capturePage, 42, host);
18761957 const webContentsById = new Map([
1877 [41, makeTestPreviewWebContents(capturePage, 41)],
1878 [42, makeTestPreviewWebContents(capturePage, 42)],
1958 [41, firstWebContents],
1959 [42, secondWebContents],
18791960 ]);
18801961 fromId.mockImplementation((id) =>
18811962 id === undefined ? null : (webContentsById.get(id) ?? null),
......@@ -1892,14 +1973,21 @@ describe("PreviewManager", () => {
18921973 } as never);
18931974
18941975 yield* manager.startRecording("tab_capture_throttling_1");
1976 // The first renderer takes its grant, freeing the arm slot for the second tab.
1977 host.displayMediaHandler()?.({ frame: host.mainFrame }, () => {});
18951978 yield* manager.startRecording("tab_capture_throttling_2");
18961979 expect(setBackgroundThrottling.mock.calls).toEqual([[false]]);
1980 expect(firstWebContents.setBackgroundThrottling.mock.calls).toEqual([[false]]);
1981 expect(secondWebContents.setBackgroundThrottling.mock.calls).toEqual([[false]]);
18971982
18981983 yield* manager.stopRecording("tab_capture_throttling_1");
18991984 expect(setBackgroundThrottling.mock.calls).toEqual([[false]]);
1985 expect(firstWebContents.setBackgroundThrottling.mock.calls).toEqual([[false], [true]]);
1986 expect(secondWebContents.setBackgroundThrottling.mock.calls).toEqual([[false]]);
19001987
19011988 yield* manager.stopRecording("tab_capture_throttling_2");
19021989 expect(setBackgroundThrottling.mock.calls).toEqual([[false], [true]]);
1990 expect(secondWebContents.setBackgroundThrottling.mock.calls).toEqual([[false], [true]]);
19031991 }),
19041992 ),
19051993 );
......@@ -1953,6 +2041,48 @@ describe("PreviewManager", () => {
19532041 ),
19542042 );
19552043
2044 effectIt.effect("rolls back window throttling when a recorded guest cannot be unthrottled", () =>
2045 withManager((manager) =>
2046 Effect.gen(function* () {
2047 const setWindowBackgroundThrottling = vi.fn();
2048 const capturePage = vi.fn(async () => ({
2049 toJPEG: () => Buffer.from("recording-frame"),
2050 getSize: () => ({ width: 1280, height: 720 }),
2051 }));
2052 const wc = makeTestPreviewWebContents(capturePage);
2053 fromId.mockReturnValue(wc);
2054
2055 yield* manager.createTab("tab_guest_throttling_failure");
2056 yield* manager.registerWebview("tab_guest_throttling_failure", 42);
2057 yield* manager.setMainWindow({
2058 isDestroyed: () => false,
2059 once: vi.fn(),
2060 webContents: { setBackgroundThrottling: setWindowBackgroundThrottling },
2061 } as never);
2062
2063 wc.setBackgroundThrottling.mockImplementationOnce(() => {
2064 throw new Error("guest throttling update failed");
2065 });
2066 const failedStart = yield* Effect.exit(
2067 manager.startRecording("tab_guest_throttling_failure"),
2068 );
2069 expect(Exit.isFailure(failedStart)).toBe(true);
2070 expect(setWindowBackgroundThrottling.mock.calls).toEqual([[false], [true]]);
2071 expect(wc.setBackgroundThrottling.mock.calls).toEqual([[false]]);
2072
2073 yield* manager.startRecording("tab_guest_throttling_failure");
2074 yield* manager.stopRecording("tab_guest_throttling_failure");
2075 expect(setWindowBackgroundThrottling.mock.calls).toEqual([
2076 [false],
2077 [true],
2078 [false],
2079 [true],
2080 ]);
2081 expect(wc.setBackgroundThrottling.mock.calls).toEqual([[false], [false], [true]]);
2082 }),
2083 ),
2084 );
2085
19562086 effectIt.effect("does not publish a replacement window when capture reconciliation fails", () =>
19572087 withManager((manager) =>
19582088 Effect.gen(function* () {
......@@ -2031,9 +2161,10 @@ describe("PreviewManager", () => {
20312161 toJPEG: () => Buffer.from("recording-frame"),
20322162 getSize: () => ({ width: 1280, height: 720 }),
20332163 }));
2164 const host = makeTestHostWebContents();
20342165 const webContentsById = new Map([
2035 [42, makeTestPreviewWebContents(capturePage, 42)],
2036 [43, makeTestPreviewWebContents(capturePage, 43)],
2166 [42, makeTestPreviewWebContents(capturePage, 42, host)],
2167 [43, makeTestPreviewWebContents(capturePage, 43, host)],
20372168 ]);
20382169 fromId.mockImplementation((id) =>
20392170 id === undefined ? null : (webContentsById.get(id) ?? null),
......@@ -2065,6 +2196,10 @@ describe("PreviewManager", () => {
20652196 yield* Effect.yieldNow;
20662197 yield* Effect.yieldNow;
20672198
2199 const grants: Array<{ video?: unknown }> = [];
2200 host.displayMediaHandler()?.({ frame: host.mainFrame }, (value) => grants.push(value));
2201 expect(grants).toEqual([{}]);
2202
20682203 yield* manager.setMainWindow({
20692204 isDestroyed: () => false,
20702205 once: vi.fn(),
......@@ -2075,7 +2210,60 @@ describe("PreviewManager", () => {
20752210 ),
20762211 );
20772212
2078 effectIt.effect("returns native media sources for concurrent preview recordings", () =>
2213 effectIt.effect("does not arm recording after the main window closes during warmup", () =>
2214 withManager((manager) =>
2215 Effect.gen(function* () {
2216 let closeMainWindow: (() => void) | undefined;
2217 let finishWarmup!: (image: TestCapturedPreviewImage) => void;
2218 let markWarmupStarted!: () => void;
2219 const warmupStarted = new Promise<void>((resolve) => {
2220 markWarmupStarted = resolve;
2221 });
2222 const capturedImage = {
2223 toJPEG: () => Buffer.from("recording-frame"),
2224 getSize: () => ({ width: 1280, height: 720 }),
2225 };
2226 const capturePage = vi.fn(
2227 () =>
2228 new Promise<TestCapturedPreviewImage>((resolve) => {
2229 markWarmupStarted();
2230 finishWarmup = resolve;
2231 }),
2232 );
2233 const host = makeTestHostWebContents();
2234 fromId.mockReturnValue(makeTestPreviewWebContents(capturePage, 42, host));
2235
2236 yield* manager.createTab("tab_window_close_warmup");
2237 yield* manager.registerWebview("tab_window_close_warmup", 42);
2238 yield* manager.setMainWindow({
2239 isDestroyed: () => false,
2240 once: vi.fn((event: string, listener: () => void) => {
2241 if (event === "closed") closeMainWindow = listener;
2242 }),
2243 webContents: { setBackgroundThrottling: vi.fn() },
2244 } as never);
2245
2246 const start = yield* manager
2247 .startRecording("tab_window_close_warmup")
2248 .pipe(Effect.forkChild({ startImmediately: true }));
2249 yield* Effect.promise(() => warmupStarted);
2250 closeMainWindow?.();
2251 finishWarmup(capturedImage);
2252
2253 const exit = yield* Fiber.await(start);
2254 expect(Exit.isFailure(exit)).toBe(true);
2255 if (Exit.isFailure(exit)) {
2256 expect(Option.getOrThrow(Cause.findErrorOption(exit.cause))).toMatchObject({
2257 _tag: "PreviewMainWindowClosedError",
2258 tabId: "tab_window_close_warmup",
2259 });
2260 }
2261 expect(host.session.setDisplayMediaRequestHandler).not.toHaveBeenCalled();
2262 }),
2263 ),
2264 );
2265
2266 effectIt.effect("grants each concurrent preview recording its own tab frame", () =>
20792267 withManager((manager) =>
20802268 Effect.gen(function* () {
20812269 const firstJpeg = Buffer.from("first-recording-frame");
......@@ -2090,6 +2278,8 @@ describe("PreviewManager", () => {
20902278 }));
20912279 const firstSendCommand = vi.fn(async () => undefined);
20922280 const secondSendCommand = vi.fn(async () => undefined);
2281 // Both webviews live in the same window, so they share one display-media handler.
2282 const host = makeTestHostWebContents();
20932283 const makeWebContents = (
20942284 id: number,
20952285 capturePage: typeof firstCapturePage,
......@@ -2097,8 +2287,8 @@ describe("PreviewManager", () => {
20972287 ) =>
20982288 ({
20992289 id,
2100 hostWebContents: { id: 7 },
2101 getMediaSourceId: vi.fn(() => `tab:${id}`),
2290 mainFrame: { routingId: id },
2291 hostWebContents: host,
21022292 executeJavaScript: vi.fn(async () =>
21032293 id === 41 ? { width: 800, height: 600 } : { width: 390, height: 844 },
21042294 ),
......@@ -2110,6 +2300,7 @@ describe("PreviewManager", () => {
21102300 getZoomFactor: () => 1,
21112301 setZoomFactor: vi.fn(),
21122302 setAudioMuted: vi.fn(),
2303 setBackgroundThrottling: vi.fn(),
21132304 isCurrentlyAudible: () => false,
21142305 on: vi.fn(),
21152306 off: vi.fn(),
......@@ -2137,15 +2328,19 @@ describe("PreviewManager", () => {
21372328 yield* manager.createTab("tab_2");
21382329 yield* manager.registerWebview("tab_1", 41);
21392330 yield* manager.registerWebview("tab_2", 42);
2140 const sources = yield* Effect.all(
2141 [manager.startRecording("tab_1"), manager.startRecording("tab_2")],
2142 { concurrency: 2 },
2143 );
21442331
2145 expect(sources).toEqual([
2146 { sourceId: "tab:41", width: 800, height: 600 },
2147 { sourceId: "tab:42", width: 390, height: 844 },
2148 ]);
2332 const grants: Array<{ video?: unknown }> = [];
2333 const takeGrant = () =>
2334 host.displayMediaHandler()?.({ frame: host.mainFrame }, (value) => {
2335 grants.push(value);
2336 });
2337
2338 yield* manager.startRecording("tab_1");
2339 takeGrant();
2340 yield* manager.startRecording("tab_2");
2341 takeGrant();
2342 expect(grants).toEqual([{ video: { routingId: 41 } }, { video: { routingId: 42 } }]);
2343
21492344 expect(firstCapturePage).toHaveBeenCalledOnce();
21502345 expect(secondCapturePage).toHaveBeenCalledOnce();
21512346 expect(firstSendCommand).not.toHaveBeenCalledWith(
......@@ -2165,121 +2360,158 @@ describe("PreviewManager", () => {
21652360 ),
21662361 );
21672362
2168 effectIt.effect("continues native recording when the source warmup fails", () =>
2363 effectIt.effect("requests display media with a fresh renderer gesture", () =>
21692364 withManager((manager) =>
21702365 Effect.gen(function* () {
2171 const capturePage = vi.fn(async () => {
2172 throw new Error("source is not ready");
2173 });
2174 const getMediaSourceId = vi.fn(() => "tab:42");
2175 const webContents = Object.assign(makeTestPreviewWebContents(capturePage), {
2176 executeJavaScript: vi.fn(async () => ({ width: 1280, height: 720 })),
2177 getMediaSourceId,
2178 });
2179 fromId.mockReturnValue(webContents);
2366 const { host, takeGrant } = yield* setupRecordingRaceTabs(manager);
21802367
2181 yield* manager.createTab("tab_recording_warmup_failure");
2182 yield* manager.registerWebview("tab_recording_warmup_failure", 42);
2368 yield* manager.startRecording("tab_race_a");
2369
2370 expect(host.executeJavaScript).toHaveBeenCalledWith(
2371 expect.stringContaining(DESKTOP_PREVIEW_RECORDING_CAPTURE_TRIGGER),
2372 true,
2373 );
2374 expect(host.executeJavaScript).toHaveBeenCalledWith(
2375 expect.stringContaining("tab_race_a"),
2376 true,
2377 );
2378 takeGrant();
2379 yield* manager.stopRecording("tab_race_a");
2380 }),
2381 ),
2382 );
21832383
2184 expect(yield* manager.startRecording("tab_recording_warmup_failure")).toEqual({
2185 sourceId: "tab:42",
2186 width: 1280,
2187 height: 720,
2384 // Runs on the real clock: an earlier queueing design only settled under TestClock and stalled the
2385 // losing start forever in the desktop app.
2386 effectIt.live("settles both starts when two tabs race for the capture stream", () =>
2387 withManager((manager) =>
2388 Effect.gen(function* () {
2389 const { host, grants, takeGrant } = yield* setupRecordingRaceTabs(manager);
2390
2391 const exits = yield* Effect.all(
2392 [
2393 Effect.exit(manager.startRecording("tab_race_a")),
2394 Effect.exit(manager.startRecording("tab_race_b")),
2395 ],
2396 { concurrency: 2 },
2397 );
2398
2399 const [exitA, exitB] = exits;
2400 // Exactly one start owns the stream; the other fails fast instead of hanging.
2401 expect(exits.filter(Exit.isSuccess)).toHaveLength(1);
2402 const loserExit = Exit.isSuccess(exitA) ? exitB : exitA;
2403 if (Exit.isSuccess(loserExit)) return;
2404 expect(Option.getOrThrow(Cause.findErrorOption(loserExit.cause))).toMatchObject({
2405 _tag: "PreviewRecordingArmConflictError",
21882406 });
2189 expect(capturePage).toHaveBeenCalledTimes(2);
2190 expect(getMediaSourceId).toHaveBeenCalledOnce();
21912407
2192 yield* manager.stopRecording("tab_recording_warmup_failure");
2408 // The single grant goes to the tab that actually won the slot, never the other one.
2409 takeGrant();
2410 expect(grants).toEqual([{ video: { routingId: Exit.isSuccess(exitA) ? 41 : 42 } }]);
2411 expect(host.session.setDisplayMediaRequestHandler).toHaveBeenCalledOnce();
21932412 }),
21942413 ),
21952414 );
21962415
2197 effectIt.effect("reports invalid native recording dimensions as a structured error", () =>
2416 effectIt.effect("releases an armed slot that the renderer never redeemed", () =>
21982417 withManager((manager) =>
21992418 Effect.gen(function* () {
2200 const capturePage = vi.fn(async () => ({
2201 toJPEG: () => Buffer.from("unused-recording-frame"),
2202 getSize: () => ({ width: 1280, height: 720 }),
2203 }));
2204 const getMediaSourceId = vi.fn(() => "tab:42");
2205 const webContents = Object.assign(makeTestPreviewWebContents(capturePage), {
2206 executeJavaScript: vi.fn(async () => ({ width: 0, height: 720 })),
2207 getMediaSourceId,
2419 const { grants, takeGrant } = yield* setupRecordingRaceTabs(manager);
2420
2421 yield* manager.startRecording("tab_race_a");
2422 const blocked = yield* Effect.exit(manager.startRecording("tab_race_b"));
2423 if (Exit.isSuccess(blocked)) throw new Error("expected the second tab to be refused");
2424 expect(Option.getOrThrow(Cause.findErrorOption(blocked.cause))).toMatchObject({
2425 _tag: "PreviewRecordingArmConflictError",
2426 tabId: "tab_race_b",
2427 armedTabId: "tab_race_a",
22082428 });
2209 fromId.mockReturnValue(webContents);
22102429
2211 yield* manager.createTab("tab_invalid_recording_size");
2212 yield* manager.registerWebview("tab_invalid_recording_size", 42);
2213 const exit = yield* Effect.exit(manager.startRecording("tab_invalid_recording_size"));
2430 // Nothing ever captured the armed tab, so the slot goes stale and stops blocking starts.
2431 yield* TestClock.adjust(10_000);
2432 yield* manager.startRecording("tab_race_b");
2433 takeGrant();
2434 expect(grants).toEqual([{ video: { routingId: 42 } }]);
22142435
2215 expect(Exit.isFailure(exit)).toBe(true);
2216 if (Exit.isSuccess(exit)) return;
2217 expect(Option.getOrThrow(Cause.findErrorOption(exit.cause))).toMatchObject({
2218 _tag: "PreviewRecordingSourceSizeUnavailableError",
2219 tabId: "tab_invalid_recording_size",
2220 webContentsId: 42,
2221 });
2222 expect(getMediaSourceId).not.toHaveBeenCalled();
2436 yield* manager.stopRecording("tab_race_a");
2437 yield* manager.stopRecording("tab_race_b");
22232438 }),
22242439 ),
22252440 );
22262441
2227 effectIt.effect("keeps a newer recording lease when an earlier start fails", () =>
2442 effectIt.effect("denies a display-media request that arrives after the arm went stale", () =>
22282443 withManager((manager) =>
22292444 Effect.gen(function* () {
2230 const setBackgroundThrottling = vi.fn();
2231 const capturePage = vi.fn(async () => ({
2232 toJPEG: () => Buffer.from("unused-recording-frame"),
2233 getSize: () => ({ width: 1280, height: 720 }),
2234 }));
2235 let markMeasurementStarted!: () => void;
2236 const measurementStarted = new Promise<void>((resolve) => {
2237 markMeasurementStarted = resolve;
2238 });
2239 let rejectFirstMeasurement!: (error: Error) => void;
2240 const executeJavaScript = vi
2241 .fn<(expression: string, userGesture?: boolean) => Promise<unknown>>()
2242 .mockImplementationOnce(() => {
2243 markMeasurementStarted();
2244 return new Promise((_, reject) => {
2245 rejectFirstMeasurement = reject;
2246 });
2247 })
2248 .mockResolvedValue({ width: 1280, height: 720 });
2249 const webContents = Object.assign(makeTestPreviewWebContents(capturePage), {
2250 executeJavaScript,
2445 const { grants, takeGrant } = yield* setupRecordingRaceTabs(manager);
2446
2447 yield* manager.startRecording("tab_race_a");
2448 yield* TestClock.adjust(10_000);
2449 // The handler cannot read a clock, so the expiry fiber must have dropped the frame.
2450 takeGrant();
2451 expect(grants).toEqual([{}]);
2452
2453 yield* manager.stopRecording("tab_race_a");
2454 }),
2455 ),
2456 );
2457
2458 effectIt.effect("only lets the host frame that armed a recording claim its stream", () =>
2459 withManager((manager) =>
2460 Effect.gen(function* () {
2461 const { grants, takeGrant } = yield* setupRecordingRaceTabs(manager);
2462
2463 yield* manager.startRecording("tab_race_a");
2464 takeGrant({ frameTreeNodeId: 999 });
2465 takeGrant();
2466
2467 expect(grants).toEqual([{}, { video: { routingId: 41 } }]);
2468 yield* manager.stopRecording("tab_race_a");
2469 }),
2470 ),
2471 );
2472
2473 effectIt.effect("reclaims the arm slot from a destroyed webContents", () =>
2474 withManager((manager) =>
2475 Effect.gen(function* () {
2476 const { grants, takeGrant, destroy } = yield* setupRecordingRaceTabs(manager);
2477
2478 yield* manager.startRecording("tab_race_a");
2479 destroy(41);
2480 yield* manager.startRecording("tab_race_b");
2481 takeGrant();
2482 expect(grants).toEqual([{ video: { routingId: 42 } }]);
2483
2484 yield* manager.stopRecording("tab_race_b");
2485 }),
2486 ),
2487 );
2488
2489 effectIt.effect("continues native recording when the source warmup fails", () =>
2490 withManager((manager) =>
2491 Effect.gen(function* () {
2492 const capturePage = vi.fn(async () => {
2493 throw new Error("source is not ready");
2494 });
2495 const host = makeTestHostWebContents();
2496 const webContents = Object.assign(makeTestPreviewWebContents(capturePage, 42, host), {
2497 executeJavaScript: vi.fn(async () => ({ width: 1280, height: 720 })),
22512498 });
22522499 fromId.mockReturnValue(webContents);
22532500
2254 yield* manager.createTab("tab_recording_start_race");
2255 yield* manager.registerWebview("tab_recording_start_race", 42);
2256 yield* manager.setMainWindow({
2257 isDestroyed: () => false,
2258 once: vi.fn(),
2259 webContents: { setBackgroundThrottling },
2260 } as never);
2501 yield* manager.createTab("tab_recording_warmup_failure");
2502 yield* manager.registerWebview("tab_recording_warmup_failure", 42);
22612503
2262 const firstStart = yield* manager
2263 .startRecording("tab_recording_start_race")
2264 .pipe(Effect.forkChild({ startImmediately: true }));
2265 yield* Effect.promise(() => measurementStarted);
2266 const secondStart = yield* manager
2267 .startRecording("tab_recording_start_race")
2268 .pipe(Effect.forkChild({ startImmediately: true }));
2269 yield* Effect.yieldNow;
2270 expect(executeJavaScript).toHaveBeenCalledOnce();
2504 yield* manager.startRecording("tab_recording_warmup_failure");
2505 expect(capturePage).toHaveBeenCalledTimes(2);
22712506
2272 rejectFirstMeasurement(new Error("first measurement failed"));
2273 const firstExit = yield* Fiber.await(firstStart);
2274 expect(Exit.isFailure(firstExit)).toBe(true);
2275 expect(yield* Fiber.join(secondStart)).toEqual({
2276 sourceId: "tab:42",
2277 width: 1280,
2278 height: 720,
2279 });
2507 // The armed tab answers exactly one display-media request, then further requests are denied.
2508 const handler = host.displayMediaHandler();
2509 const streams: Array<{ video?: unknown }> = [];
2510 handler?.({ frame: host.mainFrame }, (value) => streams.push(value));
2511 handler?.({ frame: host.mainFrame }, (value) => streams.push(value));
2512 expect(streams).toEqual([{ video: { routingId: 42 } }, {}]);
22802513
2281 yield* manager.stopRecording("tab_recording_start_race");
2282 expect(setBackgroundThrottling.mock.calls).toEqual([[false], [true], [false], [true]]);
2514 yield* manager.stopRecording("tab_recording_warmup_failure");
22832515 }),
22842516 ),
22852517 );
......@@ -2287,25 +2519,26 @@ describe("PreviewManager", () => {
22872519 effectIt.effect("serializes recording source acquisition with webview replacement", () =>
22882520 withManager((manager) =>
22892521 Effect.gen(function* () {
2290 const capturePage = vi.fn(async () => ({
2522 const capturedImage = {
22912523 toJPEG: () => Buffer.from("unused-recording-frame"),
22922524 getSize: () => ({ width: 1280, height: 720 }),
2293 }));
2294 let markMeasurementStarted!: () => void;
2295 const measurementStarted = new Promise<void>((resolve) => {
2296 markMeasurementStarted = resolve;
2297 });
2298 let finishMeasurement!: (size: { readonly width: number; readonly height: number }) => void;
2299 const executeJavaScript = vi.fn(
2300 () =>
2301 new Promise<{ readonly width: number; readonly height: number }>((resolve) => {
2302 markMeasurementStarted();
2303 finishMeasurement = resolve;
2304 }),
2305 );
2306 const initialWebContents = Object.assign(makeTestPreviewWebContents(capturePage, 42), {
2307 executeJavaScript,
2308 });
2525 };
2526 let markWarmupStarted!: () => void;
2527 const warmupStarted = new Promise<void>((resolve) => {
2528 markWarmupStarted = resolve;
2529 });
2530 let finishWarmup!: (image: TestCapturedPreviewImage) => void;
2531 const capturePage = vi
2532 .fn<() => Promise<TestCapturedPreviewImage>>()
2533 .mockImplementationOnce(
2534 () =>
2535 new Promise<TestCapturedPreviewImage>((resolve) => {
2536 markWarmupStarted();
2537 finishWarmup = resolve;
2538 }),
2539 )
2540 .mockResolvedValue(capturedImage);
2541 const initialWebContents = makeTestPreviewWebContents(capturePage, 42);
23092542 const replacementOn = vi.fn();
23102543 const replacementWebContents = Object.assign(makeTestPreviewWebContents(capturePage, 43), {
23112544 on: replacementOn,
......@@ -2321,22 +2554,25 @@ describe("PreviewManager", () => {
23212554 const start = yield* manager
23222555 .startRecording("tab_recording_replacement_race")
23232556 .pipe(Effect.forkChild({ startImmediately: true }));
2324 yield* Effect.promise(() => measurementStarted);
2557 yield* Effect.promise(() => warmupStarted);
23252558 const replacement = yield* manager
23262559 .registerWebview("tab_recording_replacement_race", 43)
23272560 .pipe(Effect.forkChild({ startImmediately: true }));
23282561 yield* Effect.yieldNow;
23292562 expect(replacementOn).not.toHaveBeenCalled();
23302563
2331 finishMeasurement({ width: 1280, height: 720 });
2332 expect(yield* Fiber.join(start)).toEqual({
2333 sourceId: "tab:42",
2334 width: 1280,
2335 height: 720,
2336 });
2564 finishWarmup(capturedImage);
2565 yield* Fiber.join(start);
23372566 yield* Fiber.join(replacement);
23382567 expect(replacementOn).toHaveBeenCalled();
2568 expect(initialWebContents.setBackgroundThrottling.mock.calls).toEqual([[false]]);
2569 expect(replacementWebContents.setBackgroundThrottling.mock.calls).toEqual([[false]]);
23392570 yield* manager.stopRecording("tab_recording_replacement_race");
2571 expect(initialWebContents.setBackgroundThrottling.mock.calls).toEqual([[false], [true]]);
2572 expect(replacementWebContents.setBackgroundThrottling.mock.calls).toEqual([
2573 [false],
2574 [true],
2575 ]);
23402576 }),
23412577 ),
23422578 );
......@@ -2345,7 +2581,9 @@ describe("PreviewManager", () => {
23452581 withManager((manager) =>
23462582 Effect.gen(function* () {
23472583 const setBackgroundThrottling = vi.fn();
2348 const mainWindowWebContents = { setBackgroundThrottling };
2584 const mainWindowWebContents = Object.assign(makeTestHostWebContents(), {
2585 setBackgroundThrottling,
2586 });
23492587 const jpeg = Buffer.from("shared-preview-frame");
23502588 const capturePage = vi.fn(async () => ({
23512589 toJPEG: () => jpeg,
......@@ -2353,8 +2591,8 @@ describe("PreviewManager", () => {
23532591 }));
23542592 fromId.mockReturnValue({
23552593 id: 42,
2594 mainFrame: { routingId: 42 },
23562595 hostWebContents: mainWindowWebContents,
2357 getMediaSourceId: vi.fn(() => "tab:42"),
23582596 executeJavaScript: vi.fn(async () => ({ width: 1280, height: 720 })),
23592597 isDestroyed: () => false,
23602598 getType: () => "webview",
......@@ -2364,6 +2602,7 @@ describe("PreviewManager", () => {
23642602 getZoomFactor: () => 1,
23652603 setZoomFactor: vi.fn(),
23662604 setAudioMuted: vi.fn(),
2605 setBackgroundThrottling: vi.fn(),
23672606 isCurrentlyAudible: () => false,
23682607 on: vi.fn(),
23692608 off: vi.fn(),
apps/desktop/src/preview/Manager.ts+229-48
......@@ -5,6 +5,7 @@
55 * elements live in the renderer; we only attach listeners and forward state
66 * here). Single layer-scoped browser session partition.
77 */
8import { DESKTOP_PREVIEW_RECORDING_CAPTURE_TRIGGER } from "@t3tools/contracts";
89import type {
910 DesktopPreviewAnnotationTheme,
1011 DesktopPreviewAutomationStatus,
......@@ -16,7 +17,6 @@ import type {
1617 PreviewAnnotationSubmissionResult,
1718 DesktopPreviewRecordingArtifact,
1819 DesktopPreviewRecordingFrame,
19 DesktopPreviewRecordingSource,
2020 DesktopPreviewScreenshotArtifact,
2121 DesktopPreviewTabDefaults,
2222 PreviewAutomationClickInput,
......@@ -109,8 +109,8 @@ const MAX_EVALUATION_BYTES = 64_000;
109109const MAX_VISIBLE_TEXT_LENGTH = 20_000;
110110const MAX_INTERACTIVE_ELEMENTS = 200;
111111const MAX_SCREENSHOT_WIDTH = 1280;
112const RECORDING_SOURCE_SIZE_EXPRESSION =
113 "({ width: Math.round(globalThis.innerWidth), height: Math.round(globalThis.innerHeight) })";
112/** How long an armed tab keeps the exclusive display-media slot before another tab may take it. */
113const RECORDING_ARM_GRACE_MS = 10_000;
114114const PICTURE_IN_PICTURE_FRAME_INTERVAL_MS = Math.ceil(1_000 / 12);
115115const PICTURE_IN_PICTURE_JPEG_QUALITY = 80;
116116const PICTURE_IN_PICTURE_INITIAL_WIDTH = 480;
......@@ -122,6 +122,8 @@ const DIAGNOSTIC_BUFFER_LIMIT = 200;
122122const MAX_ARTIFACT_SITE_SLUG_LENGTH = 80;
123123const AGENT_CURSOR_MOVE_MS = 160;
124124const AGENT_CURSOR_CLICK_LEAD_MS = 40;
125const requestRecordingCaptureExpression = (tabId: string): string =>
126 `globalThis[${JSON.stringify(DESKTOP_PREVIEW_RECORDING_CAPTURE_TRIGGER)}]?.(${JSON.stringify(tabId)}) === true`;
125127const encodeUnknownJson = Schema.encodeUnknownEffect(Schema.fromJsonString(Schema.Unknown));
126128const DEFAULT_ANNOTATION_THEME: DesktopPreviewAnnotationTheme = {
127129 colorScheme: "light",
......@@ -390,6 +392,7 @@ type FrameCaptureConsumer = "picture-in-picture" | "recording";
390392interface FrameCaptureSession {
391393 readonly scope: Scope.Closeable | null;
392394 readonly consumers: ReadonlySet<FrameCaptureConsumer>;
395 readonly unthrottledWebContentsIds: ReadonlySet<number>;
393396 readonly lastPictureInPictureFrame: Buffer | null;
394397}
395398
......@@ -400,6 +403,14 @@ interface PictureInPictureSession {
400403 readonly initializationScope: Scope.Closeable;
401404}
402405
406/** The tab whose frame the next `getDisplayMedia()` request is allowed to capture. */
407interface PendingRecording {
408 readonly tabId: string;
409 readonly webContents: Electron.WebContents;
410 readonly requestingFrameTreeNodeId: number;
411 readonly armedAtMillis: number;
412}
413
403414interface PickSession {
404415 readonly cancel: Effect.Effect<void>;
405416}
......@@ -591,6 +602,11 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
591602 const pictureInPictureAspectRatiosRef = yield* Ref.make<ReadonlyMap<string, number>>(new Map());
592603 const pictureInPictureMutationSemaphore = yield* Semaphore.make(1);
593604 const closingTabIdsRef = yield* Ref.make<ReadonlySet<string>>(new Set());
605 // Tab recording uses `setDisplayMediaRequestHandler` because Electron's legacy
606 // `getMediaSourceId` + `chromeMediaSource: "tab"` capture path was removed upstream
607 // (electron#44618) and now always rejects with NotAllowedError.
608 let pendingRecording: PendingRecording | null = null;
609 const displayMediaHandlerSessions = new WeakSet<Session>();
594610 let frameCaptureWindowOpen = true;
595611 let currentMainWindow: BrowserWindow | undefined;
596612 let mainWindowCleanupFiber: Fiber.Fiber<void, never> | undefined;
......@@ -665,6 +681,65 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
665681 if (Option.isNone(mainWindow)) return;
666682 yield* setWindowBackgroundThrottling(mainWindow.value, enabled);
667683 });
684 const setFrameCaptureWebContentsBackgroundThrottling = Effect.fnUntraced(function* (
685 wc: Electron.WebContents,
686 enabled: boolean,
687 ) {
688 if (wc.isDestroyed()) return;
689 yield* attempt(
690 {
691 operation: "frameCapture.setBackgroundThrottling",
692 webContentsId: wc.id,
693 },
694 () => wc.setBackgroundThrottling(enabled),
695 );
696 });
697 const restoreFrameCaptureWebContentsBackgroundThrottling = Effect.fnUntraced(function* (
698 webContentsIds: ReadonlySet<number>,
699 ) {
700 yield* Effect.forEach(
701 webContentsIds,
702 (webContentsId) => {
703 const wc = webContents.fromId(webContentsId);
704 if (!wc || wc.isDestroyed()) return Effect.void;
705 return setFrameCaptureWebContentsBackgroundThrottling(wc, true).pipe(
706 Effect.retry({ times: 2 }),
707 Effect.catch((error) =>
708 Effect.logWarning("Failed to restore preview webview frame capture throttling.", {
709 webContentsId,
710 error,
711 }),
712 ),
713 );
714 },
715 { concurrency: "unbounded", discard: true },
716 );
717 });
718 const keepFrameCaptureWebContentsUnthrottled = Effect.fnUntraced(function* (
719 tabId: string,
720 wc: Electron.WebContents,
721 ) {
722 yield* SynchronizedRef.modifyEffect(frameCaptureSessionsRef, (sessions) => {
723 const current = sessions.get(tabId);
724 if (!current || current.unthrottledWebContentsIds.has(wc.id)) {
725 return Effect.succeed([undefined, sessions] as const);
726 }
727 return setFrameCaptureWebContentsBackgroundThrottling(wc, false).pipe(
728 Effect.map(
729 () =>
730 [
731 undefined,
732 replaceMap(sessions, (copy) => {
733 copy.set(tabId, {
734 ...current,
735 unthrottledWebContentsIds: new Set([...current.unthrottledWebContentsIds, wc.id]),
736 });
737 }),
738 ] as const,
739 ),
740 );
741 });
742 });
668743 const stopFrameCapture = Effect.fn("PreviewManager.stopFrameCapture")(function* (
669744 tabId: string,
670745 consumer: FrameCaptureConsumer,
......@@ -694,6 +769,9 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
694769 const remainingSessions = replaceMap(sessions, (copy) => {
695770 copy.delete(tabId);
696771 });
772 yield* restoreFrameCaptureWebContentsBackgroundThrottling(
773 current.unthrottledWebContentsIds,
774 );
697775 if (remainingSessions.size === 0) {
698776 yield* setFrameCaptureBackgroundThrottling(true).pipe(
699777 Effect.retry({ times: 2 }),
......@@ -713,6 +791,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
713791 });
714792
715793 const stopAllRecordings = Effect.fn("PreviewManager.stopAllRecordings")(function* () {
794 pendingRecording = null;
716795 const sessions = yield* SynchronizedRef.get(frameCaptureSessionsRef);
717796 yield* Effect.forEach(sessions.keys(), (tabId) => stopFrameCapture(tabId, "recording"), {
718797 concurrency: "unbounded",
......@@ -1889,6 +1968,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
18891968
18901969 const closeTabUnlocked = Effect.fn("PreviewManager.closeTabUnlocked")(function* (tabId: string) {
18911970 if (!(yield* SynchronizedRef.get(tabsRef)).has(tabId)) return;
1971 clearPendingRecording(tabId);
18921972 yield* Effect.all(
18931973 [
18941974 cancelPickElement(tabId),
......@@ -1980,6 +2060,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
19802060 const attached = yield* Ref.get(attachedRef);
19812061 const annotationTheme = yield* Ref.get(annotationThemeRef);
19822062 const currentAttachment = attached.get(webContentsId);
2063 yield* keepFrameCaptureWebContentsUnthrottled(tabId, wc);
19832064 if (tab.webContentsId === webContentsId && currentAttachment?.webContents === wc) {
19842065 // The guest we already own re-announced itself, so nothing about the tab
19852066 // changed. Only push its zoom back down — Chromium may have just handed
......@@ -1996,6 +2077,8 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
19962077 ? tab.webContentsId
19972078 : null;
19982079 if (replacedWebContentsId !== null) {
2080 // The replaced guest can no longer redeem a display-media grant.
2081 clearPendingRecording(tabId);
19992082 yield* Effect.all(
20002083 [
20012084 detachControlSession(replacedWebContentsId),
......@@ -2728,7 +2811,6 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
27282811 ) {
27292812 // Recording keeps only the activity lease. Picture-in-picture owns the
27302813 // capturePage loop and tolerates transient compositor warmup failures.
2731 yield* requireWebContents(tabId);
27322814 const captureNextFrame = Effect.sleep(PICTURE_IN_PICTURE_FRAME_INTERVAL_MS).pipe(
27332815 Effect.andThen(capturePreviewFrame(tabId)),
27342816 Effect.catch((error) =>
......@@ -2749,11 +2831,15 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
27492831 if (!tab || (yield* Ref.get(closingTabIdsRef)).has(tabId)) {
27502832 return yield* new PreviewTabNotFoundError({ tabId });
27512833 }
2834 const wc = yield* requireWebContents(tabId);
27522835 const current = sessions.get(tabId);
27532836 if (current) {
27542837 if (current.consumers.has(consumer)) {
27552838 return [false, sessions] as const;
27562839 }
2840 if (!current.unthrottledWebContentsIds.has(wc.id)) {
2841 yield* setFrameCaptureWebContentsBackgroundThrottling(wc, false);
2842 }
27572843 let scope = current.scope;
27582844 if (consumer === "picture-in-picture" && scope === null) {
27592845 scope = yield* Scope.fork(parentScope, "sequential");
......@@ -2766,6 +2852,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
27662852 ...current,
27672853 scope,
27682854 consumers: new Set([...current.consumers, consumer]),
2855 unthrottledWebContentsIds: new Set([...current.unthrottledWebContentsIds, wc.id]),
27692856 });
27702857 }),
27712858 ] as const;
......@@ -2773,6 +2860,13 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
27732860 if (sessions.size === 0) {
27742861 yield* setFrameCaptureBackgroundThrottling(false);
27752862 }
2863 yield* setFrameCaptureWebContentsBackgroundThrottling(wc, false).pipe(
2864 Effect.onError(() =>
2865 sessions.size === 0
2866 ? setFrameCaptureBackgroundThrottling(true).pipe(Effect.ignore)
2867 : Effect.void,
2868 ),
2869 );
27762870 const scope =
27772871 consumer === "picture-in-picture" ? yield* Scope.fork(parentScope, "sequential") : null;
27782872 if (scope !== null) {
......@@ -2784,6 +2878,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
27842878 copy.set(tabId, {
27852879 scope,
27862880 consumers: new Set([consumer]),
2881 unthrottledWebContentsIds: new Set([wc.id]),
27872882 lastPictureInPictureFrame: null,
27882883 });
27892884 }),
......@@ -3099,6 +3194,83 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
30993194 return yield* Effect.failCause(initializationExit.cause);
31003195 });
31013196
3197 /** Only drops the armed target when it still belongs to `tabId`, so tabs cannot clobber each other. */
3198 const clearPendingRecording = (tabId: string) => {
3199 if (pendingRecording?.tabId === tabId) pendingRecording = null;
3200 };
3201
3202 /**
3203 * Claims the single arm slot for `tabId`. A display-media request carries no tab identity, so the
3204 * slot is exclusive: a second tab arming before the first request lands would redirect the first
3205 * renderer's stream. Rather than queue (which can only ever stall a start), a colliding start
3206 * fails fast and the renderer can retry. An arm the renderer never redeemed goes stale after
3207 * `RECORDING_ARM_GRACE_MS` so it cannot hold the slot forever.
3208 */
3209 const armPendingRecording = Effect.fn("PreviewManager.armPendingRecording")(function* (
3210 tabId: string,
3211 wc: Electron.WebContents,
3212 requestingFrameTreeNodeId: number,
3213 ) {
3214 const now = yield* Clock.currentTimeMillis;
3215 const previous = pendingRecording;
3216 if (
3217 previous !== null &&
3218 previous.tabId !== tabId &&
3219 !previous.webContents.isDestroyed() &&
3220 now - previous.armedAtMillis < RECORDING_ARM_GRACE_MS
3221 ) {
3222 return yield* new PreviewRecordingArmConflictError({
3223 tabId,
3224 webContentsId: wc.id,
3225 armedTabId: previous.tabId,
3226 });
3227 }
3228 const armed: PendingRecording = {
3229 tabId,
3230 webContents: wc,
3231 requestingFrameTreeNodeId,
3232 armedAtMillis: now,
3233 };
3234 pendingRecording = armed;
3235 // The handler callback is sync and cannot read a clock, so expiry is driven from here.
3236 // Identity compare: a re-arm replaces the object, and this fiber must not clobber it.
3237 yield* Effect.forkIn(
3238 Effect.sleep(RECORDING_ARM_GRACE_MS).pipe(
3239 Effect.andThen(
3240 Effect.sync(() => {
3241 if (pendingRecording === armed) pendingRecording = null;
3242 }),
3243 ),
3244 ),
3245 parentScope,
3246 );
3247 });
3248
3249 // Installed once per session: answers the renderer's `getDisplayMedia()` with the tab that
3250 // `startRecording` armed, and denies anything else so pages cannot capture on their own.
3251 const installDisplayMediaRequestHandler = (session: Session) => {
3252 if (displayMediaHandlerSessions.has(session)) return;
3253 displayMediaHandlerSessions.add(session);
3254 session.setDisplayMediaRequestHandler((request, callback) => {
3255 const armed = pendingRecording;
3256 if (!armed) {
3257 callback({});
3258 return;
3259 }
3260 if (armed.webContents.isDestroyed()) {
3261 pendingRecording = null;
3262 callback({});
3263 return;
3264 }
3265 if (request.frame?.frameTreeNodeId !== armed.requestingFrameTreeNodeId) {
3266 callback({});
3267 return;
3268 }
3269 pendingRecording = null;
3270 callback({ video: armed.webContents.mainFrame });
3271 });
3272 };
3273
31023274 const startRecording = Effect.fn("PreviewManager.startRecording")(function* (tabId: string) {
31033275 if ((yield* Ref.get(closingTabIdsRef)).has(tabId)) {
31043276 return yield* new PreviewTabNotFoundError({ tabId });
......@@ -3112,31 +3284,6 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
31123284 if (requestWebContents === null) {
31133285 return yield* new PreviewMainWindowClosedError({ tabId });
31143286 }
3115 const measuredSize = yield* attemptPromise(
3116 {
3117 operation: "recording.measureSource",
3118 tabId,
3119 webContentsId: wc.id,
3120 },
3121 () => wc.executeJavaScript(RECORDING_SOURCE_SIZE_EXPRESSION, true),
3122 );
3123 if (
3124 typeof measuredSize !== "object" ||
3125 measuredSize === null ||
3126 !("width" in measuredSize) ||
3127 !("height" in measuredSize) ||
3128 typeof measuredSize.width !== "number" ||
3129 typeof measuredSize.height !== "number" ||
3130 !Number.isInteger(measuredSize.width) ||
3131 !Number.isInteger(measuredSize.height) ||
3132 measuredSize.width <= 0 ||
3133 measuredSize.height <= 0
3134 ) {
3135 return yield* new PreviewRecordingSourceSizeUnavailableError({
3136 tabId,
3137 webContentsId: wc.id,
3138 });
3139 }
31403287 yield* attemptPromise(
31413288 {
31423289 operation: "recording.warmSource",
......@@ -3152,25 +3299,44 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
31523299 webContentsId: wc.id,
31533300 });
31543301 }
3155 const sourceId = yield* attempt(
3302 if (!frameCaptureWindowOpen || requestWebContents.isDestroyed()) {
3303 return yield* new PreviewMainWindowClosedError({ tabId });
3304 }
3305 installDisplayMediaRequestHandler(requestWebContents.session);
3306 yield* armPendingRecording(tabId, wc, requestWebContents.mainFrame.frameTreeNodeId);
3307 const captureRequested = yield* attemptPromise(
31563308 {
3157 operation: "recording.getMediaSourceId",
3309 operation: "recording.requestCapture",
31583310 tabId,
3159 webContentsId: wc.id,
3311 webContentsId: requestWebContents.id,
31603312 },
3161 () => wc.getMediaSourceId(requestWebContents),
3313 () =>
3314 requestWebContents.executeJavaScript(requestRecordingCaptureExpression(tabId), true),
31623315 );
3163 return {
3164 sourceId,
3165 width: measuredSize.width,
3166 height: measuredSize.height,
3167 } satisfies DesktopPreviewRecordingSource;
3168 }).pipe(Effect.onError(() => stopFrameCapture(tabId, "recording").pipe(Effect.ignore))),
3316 if (captureRequested !== true) {
3317 return yield* new PreviewRecordingCaptureUnavailableError({
3318 tabId,
3319 webContentsId: requestWebContents.id,
3320 });
3321 }
3322 }).pipe(
3323 Effect.onError(() => {
3324 clearPendingRecording(tabId);
3325 return stopFrameCapture(tabId, "recording").pipe(Effect.ignore);
3326 }),
3327 ),
31693328 );
31703329 });
31713330
31723331 const stopRecording = Effect.fn("PreviewManager.stopRecording")(function* (tabId: string) {
3173 yield* withTabLifecycleLock(tabId, stopFrameCapture(tabId, "recording"));
3332 // Clearing runs under the tab lock so it cannot land before an in-flight start arms.
3333 yield* withTabLifecycleLock(
3334 tabId,
3335 Effect.suspend(() => {
3336 clearPendingRecording(tabId);
3337 return stopFrameCapture(tabId, "recording");
3338 }),
3339 );
31743340 });
31753341
31763342 const saveRecording = Effect.fn("PreviewManager.saveRecording")(function* (
......@@ -3980,12 +4146,28 @@ export class PreviewMainWindowClosedError extends Schema.TaggedErrorClass<Previe
39804146 }
39814147}
39824148
3983export class PreviewRecordingSourceSizeUnavailableError extends Schema.TaggedErrorClass<PreviewRecordingSourceSizeUnavailableError>()(
3984 "PreviewRecordingSourceSizeUnavailableError",
3985 { tabId: Schema.String, webContentsId: Schema.Number },
4149export class PreviewRecordingArmConflictError extends Schema.TaggedErrorClass<PreviewRecordingArmConflictError>()(
4150 "PreviewRecordingArmConflictError",
4151 {
4152 tabId: Schema.String,
4153 webContentsId: Schema.Number,
4154 armedTabId: Schema.String,
4155 },
39864156) {
39874157 override get message(): string {
3988 return `Preview media source dimensions are unavailable for tab ${this.tabId}`;
4158 return `Preview tab ${this.armedTabId} is still claiming the capture stream, so recording could not start for tab ${this.tabId}`;
4159 }
4160}
4161
4162export class PreviewRecordingCaptureUnavailableError extends Schema.TaggedErrorClass<PreviewRecordingCaptureUnavailableError>()(
4163 "PreviewRecordingCaptureUnavailableError",
4164 {
4165 tabId: Schema.String,
4166 webContentsId: Schema.Number,
4167 },
4168) {
4169 override get message(): string {
4170 return `Preview recording capture is unavailable for tab ${this.tabId} in WebContents ${this.webContentsId}`;
39894171 }
39904172}
39914173
......@@ -4196,7 +4378,8 @@ export const PreviewManagerError = Schema.Union([
41964378 PreviewWebContentsNotFoundError,
41974379 PreviewWebviewNotInitializedError,
41984380 PreviewMainWindowClosedError,
4199 PreviewRecordingSourceSizeUnavailableError,
4381 PreviewRecordingArmConflictError,
4382 PreviewRecordingCaptureUnavailableError,
42004383 PreviewOperationError,
42014384 PreviewArtifactPathOutsideDirectoryError,
42024385 PreviewArtifactImageLoadError,
......@@ -4274,9 +4457,7 @@ export class PreviewManager extends Context.Service<
42744457 readonly copyArtifactToClipboard: (path: string) => Effect.Effect<void, PreviewManagerError>;
42754458 readonly openPictureInPicture: (tabId: string) => Effect.Effect<void, PreviewManagerError>;
42764459 readonly closePictureInPicture: (tabId: string) => Effect.Effect<void, PreviewManagerError>;
4277 readonly startRecording: (
4278 tabId: string,
4279 ) => Effect.Effect<DesktopPreviewRecordingSource, PreviewManagerError>;
4460 readonly startRecording: (tabId: string) => Effect.Effect<void, PreviewManagerError>;
42804461 readonly stopRecording: (tabId: string) => Effect.Effect<void, PreviewManagerError>;
42814462 readonly saveRecording: (
42824463 tabId: string,
apps/web/src/browser/HostedBrowserWebview.tsx+5-1
......@@ -6,7 +6,7 @@ import { useCallback, useEffect, useRef, useState } from "react";
66
77import { previewBridge } from "~/components/preview/previewBridge";
88import { usePreviewBridge } from "~/components/preview/usePreviewBridge";
9import { cn } from "~/lib/utils";
9import { cn, isMacPlatform } from "~/lib/utils";
1010
1111import { resolveBrowserSurfacePanelRect, useBrowserSurfaceStore } from "./browserSurfaceStore";
1212import { useActiveBrowserRecordingTabIds } from "./browserRecording";
......@@ -241,6 +241,10 @@ export function HostedBrowserWebview(props: {
241241 const wrapperStyle = resolveHostedBrowserWebviewWrapperStyle({
242242 active,
243243 renderingActive,
244 // Electron 43 can permanently blank a macOS webview after `visibility: hidden`.
245 // Inactive macOS guests intentionally remain paintable offscreen; other platforms still
246 // suspend them, and automation continues to see the macOS guests as inactive.
247 keepPaintableWhenInactive: isMacPlatform(navigator.platform),
244248 cornerRadius: presentation.cornerRadius,
245249 rect: lastRect,
246250 hiddenSize,
apps/web/src/browser/browserRecording.test.ts+195-60
......@@ -1,37 +1,57 @@
1import { EnvironmentId, ThreadId } from "@t3tools/contracts";
1import {
2 DESKTOP_PREVIEW_RECORDING_CAPTURE_TRIGGER,
3 EnvironmentId,
4 ThreadId,
5} from "@t3tools/contracts";
26import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test";
37
4const { clientSettings, events, getUserMedia, registrySet, save, startScreencast, stopScreencast } =
5 vi.hoisted(() => {
6 const events: string[] = [];
7 return {
8 clientSettings: { browserRecordingFrameRate: 30 as 30 | 60 },
9 events,
10 getUserMedia: vi.fn(),
11 registrySet: vi.fn((_atom: unknown, value: { readonly tabIds: ReadonlySet<string> }) => {
12 events.push(
13 value.tabIds.size === 0 ? "clear" : `publish:${Array.from(value.tabIds).join(",")}`,
14 );
15 }),
16 save: vi.fn(async (tabId: string) => ({
17 id: "recording-test",
18 tabId,
19 path: "/tmp/recording-test.webm",
20 mimeType: "video/webm" as const,
21 sizeBytes: 0,
22 createdAt: "2026-06-26T00:00:00.000Z",
23 })),
24 startScreencast: vi.fn(async (tabId: string) => {
25 events.push("start-screencast");
26 return { sourceId: `source:${tabId}`, width: 1000, height: 620 };
27 }),
28 stopScreencast: vi.fn(async () => undefined),
29 };
30 });
8const {
9 clientSettings,
10 events,
11 getDisplayMedia,
12 registrySet,
13 requestDisplayMediaCapture,
14 save,
15 startScreencast,
16 stopScreencast,
17} = vi.hoisted(() => {
18 const events: string[] = [];
19 return {
20 clientSettings: { browserRecordingFrameRate: 30 as 30 | 60 },
21 events,
22 getDisplayMedia: vi.fn(),
23 requestDisplayMediaCapture: vi.fn((_tabId: string) => undefined),
24 registrySet: vi.fn((_atom: unknown, value: { readonly tabIds: ReadonlySet<string> }) => {
25 events.push(
26 value.tabIds.size === 0 ? "clear" : `publish:${Array.from(value.tabIds).join(",")}`,
27 );
28 }),
29 save: vi.fn(async (tabId: string) => ({
30 id: "recording-test",
31 tabId,
32 path: "/tmp/recording-test.webm",
33 mimeType: "video/webm" as const,
34 sizeBytes: 0,
35 createdAt: "2026-06-26T00:00:00.000Z",
36 })),
37 startScreencast: vi.fn(async (_tabId: string) => {
38 events.push("start-screencast");
39 }),
40 stopScreencast: vi.fn(async () => undefined),
41 };
42});
3143
3244vi.mock("~/components/preview/previewBridge", () => ({
3345 previewBridge: {
34 recording: { onFrame: vi.fn(), save, startScreencast, stopScreencast },
46 recording: {
47 onFrame: vi.fn(),
48 save,
49 startScreencast: async (tabId: string) => {
50 await startScreencast(tabId);
51 requestDisplayMediaCapture(tabId);
52 },
53 stopScreencast,
54 },
3555 },
3656}));
3757
......@@ -50,6 +70,7 @@ import {
5070 BrowserRecordingCaptureTimeoutError,
5171 BrowserRecordingConflictError,
5272 BrowserRecordingFormatUnavailableError,
73 BrowserRecordingStartCancelledError,
5374 findActiveBrowserRecordingRuntimeTabId,
5475 readActiveBrowserRecordingTabIds,
5576 readActiveBrowserRecordingTargets,
......@@ -122,10 +143,16 @@ describe("browser recording", () => {
122143 });
123144 vi.stubGlobal("cancelAnimationFrame", vi.fn());
124145 vi.stubGlobal("MediaRecorder", FakeMediaRecorder as unknown as typeof MediaRecorder);
125 getUserMedia.mockResolvedValue({
146 getDisplayMedia.mockResolvedValue({
126147 getTracks: () => [{ stop: vi.fn() }],
127148 });
128 vi.stubGlobal("navigator", { mediaDevices: { getUserMedia } });
149 requestDisplayMediaCapture.mockImplementation((tabId: string) => {
150 const trigger = Reflect.get(globalThis, DESKTOP_PREVIEW_RECORDING_CAPTURE_TRIGGER);
151 if (typeof trigger !== "function" || trigger(tabId) !== true) {
152 throw new Error(`No pending display-media capture for ${tabId}.`);
153 }
154 });
155 vi.stubGlobal("navigator", { mediaDevices: { getDisplayMedia } });
129156 useBrowserSurfaceStore.setState({ activityByTabId: {}, byTabId: {} });
130157 });
131158
......@@ -142,13 +169,20 @@ describe("browser recording", () => {
142169 expect(startupEvents).toEqual(["publish:recording-tab", "start-screencast"]);
143170 });
144171
172 it("routes gesture-free starts through the desktop capture trigger", async () => {
173 await startBrowserRecording("automation-recording-tab");
174
175 expect(requestDisplayMediaCapture).toHaveBeenCalledWith("automation-recording-tab");
176 expect(getDisplayMedia).toHaveBeenCalledOnce();
177 await stopBrowserRecording("automation-recording-tab");
178 });
179
145180 it("paints and holds a hidden browser surface for the recording lifetime", async () => {
146181 startScreencast.mockImplementationOnce(async (tabId: string) => {
147182 expect(animationFrameCount).toBe(2);
148183 expect(useBrowserSurfaceStore.getState().activityByTabId[tabId]).toBe(1);
149 return { sourceId: `source:${tabId}`, width: 1000, height: 620 };
150184 });
151 getUserMedia.mockImplementationOnce(async () => {
185 getDisplayMedia.mockImplementationOnce(async () => {
152186 expect(animationFrameCount).toBe(2);
153187 expect(useBrowserSurfaceStore.getState().activityByTabId["background-tab"]).toBe(1);
154188 return { getTracks: () => [{ stop: vi.fn() }] };
......@@ -178,26 +212,16 @@ describe("browser recording", () => {
178212 await stopBrowserRecording("hidden-window-tab");
179213 });
180214
181 it("records the native tab stream at the preview's current dimensions", async () => {
215 it("records the native tab stream armed by the main process", async () => {
182216 const stopTrack = vi.fn();
183217 const stream = { getTracks: () => [{ stop: stopTrack }] } as unknown as MediaStream;
184 getUserMedia.mockResolvedValueOnce(stream);
218 getDisplayMedia.mockResolvedValueOnce(stream);
185219
186220 await startBrowserRecording("recording-tab");
187221
188 expect(getUserMedia).toHaveBeenCalledWith({
222 expect(getDisplayMedia).toHaveBeenCalledWith({
189223 audio: false,
190 video: {
191 mandatory: {
192 chromeMediaSource: "tab",
193 chromeMediaSourceId: "source:recording-tab",
194 minWidth: 1000,
195 maxWidth: 1000,
196 minHeight: 620,
197 maxHeight: 620,
198 maxFrameRate: 30,
199 },
200 },
224 video: { frameRate: { max: 30 } },
201225 });
202226 expect(FakeMediaRecorder.instances[0]?.stream).toBe(stream);
203227
......@@ -210,18 +234,16 @@ describe("browser recording", () => {
210234
211235 await startBrowserRecording("recording-tab");
212236
213 expect(getUserMedia).toHaveBeenCalledWith({
237 expect(getDisplayMedia).toHaveBeenCalledWith({
214238 audio: false,
215 video: {
216 mandatory: expect.objectContaining({ maxFrameRate: 60 }),
217 },
239 video: { frameRate: { max: 60 } },
218240 });
219241 await stopBrowserRecording("recording-tab");
220242 });
221243
222244 it("stops the native stream when MediaRecorder cleanup fails", async () => {
223245 const stopTrack = vi.fn();
224 getUserMedia.mockResolvedValueOnce({
246 getDisplayMedia.mockResolvedValueOnce({
225247 getTracks: () => [{ stop: stopTrack }],
226248 });
227249
......@@ -285,7 +307,7 @@ describe("browser recording", () => {
285307 });
286308
287309 it("releases the native capture lease when stream acquisition fails", async () => {
288 getUserMedia.mockRejectedValueOnce(new Error("capture failed"));
310 getDisplayMedia.mockRejectedValueOnce(new Error("capture failed"));
289311
290312 await expect(startBrowserRecording("recording-tab")).rejects.toMatchObject({
291313 operation: "capture-media-stream",
......@@ -300,7 +322,7 @@ describe("browser recording", () => {
300322 vi.useFakeTimers();
301323 let finishCapture!: (stream: MediaStream) => void;
302324 const stopTrack = vi.fn();
303 getUserMedia.mockImplementationOnce(
325 getDisplayMedia.mockImplementationOnce(
304326 () =>
305327 new Promise<MediaStream>((resolve) => {
306328 finishCapture = resolve;
......@@ -308,7 +330,7 @@ describe("browser recording", () => {
308330 );
309331
310332 const startPromise = startBrowserRecording("recording-tab");
311 await vi.waitFor(() => expect(getUserMedia).toHaveBeenCalledOnce());
333 await vi.waitFor(() => expect(getDisplayMedia).toHaveBeenCalledOnce());
312334 const rejection = expect(startPromise).rejects.toMatchObject({
313335 _tag: "BrowserRecordingCaptureTimeoutError",
314336 tabId: "recording-tab",
......@@ -356,6 +378,124 @@ describe("browser recording", () => {
356378 expect(save).toHaveBeenCalledTimes(2);
357379 });
358380
381 it("serializes display media grants for concurrent recording starts", async () => {
382 let finishFirstCapture!: (stream: MediaStream) => void;
383 const stream = { getTracks: () => [{ stop: vi.fn() }] } as unknown as MediaStream;
384 getDisplayMedia
385 .mockImplementationOnce(
386 () =>
387 new Promise<MediaStream>((resolve) => {
388 finishFirstCapture = resolve;
389 }),
390 )
391 .mockResolvedValueOnce(stream);
392
393 const firstStart = startBrowserRecording("recording-tab");
394 await vi.waitFor(() => expect(getDisplayMedia).toHaveBeenCalledOnce());
395 const secondStart = startBrowserRecording("recording-tab-2");
396 await vi.waitFor(() => expect(readActiveBrowserRecordingTabIds().size).toBe(2));
397
398 expect(startScreencast).toHaveBeenCalledTimes(1);
399 finishFirstCapture(stream);
400 await Promise.all([firstStart, secondStart]);
401
402 expect(startScreencast.mock.calls).toEqual([["recording-tab"], ["recording-tab-2"]]);
403 expect(getDisplayMedia).toHaveBeenCalledTimes(2);
404 await Promise.all([
405 stopBrowserRecording("recording-tab"),
406 stopBrowserRecording("recording-tab-2"),
407 ]);
408 });
409
410 it("cancels a queued recording when stopped before its media grant", async () => {
411 let finishFirstCapture!: (stream: MediaStream) => void;
412 const stream = { getTracks: () => [{ stop: vi.fn() }] } as unknown as MediaStream;
413 getDisplayMedia.mockImplementationOnce(
414 () =>
415 new Promise<MediaStream>((resolve) => {
416 finishFirstCapture = resolve;
417 }),
418 );
419
420 const firstStart = startBrowserRecording("recording-tab");
421 await vi.waitFor(() => expect(getDisplayMedia).toHaveBeenCalledOnce());
422 const secondStart = startBrowserRecording("recording-tab-2");
423 await vi.waitFor(() => expect(readActiveBrowserRecordingTabIds().size).toBe(2));
424
425 const secondStop = stopBrowserRecording("recording-tab-2");
426 await expect(secondStart).rejects.toBeInstanceOf(BrowserRecordingStartCancelledError);
427 await expect(secondStop).resolves.toBeNull();
428 expect(startScreencast).toHaveBeenCalledTimes(1);
429
430 finishFirstCapture(stream);
431 await firstStart;
432 await stopBrowserRecording("recording-tab");
433 expect(getDisplayMedia).toHaveBeenCalledOnce();
434 });
435
436 it("latches a stop that arrives before the start becomes queued", async () => {
437 let releaseDelayedPaint!: (timestamp: number) => void;
438 let frameId = 0;
439 vi.stubGlobal(
440 "requestAnimationFrame",
441 vi.fn((callback: FrameRequestCallback) => {
442 frameId += 1;
443 if (frameId === 1) releaseDelayedPaint = callback;
444 else callback(frameId);
445 return frameId;
446 }),
447 );
448 let finishBlockingCapture!: (stream: MediaStream) => void;
449 const stream = { getTracks: () => [{ stop: vi.fn() }] } as unknown as MediaStream;
450 getDisplayMedia.mockImplementationOnce(
451 () =>
452 new Promise<MediaStream>((resolve) => {
453 finishBlockingCapture = resolve;
454 }),
455 );
456
457 const delayedStart = startBrowserRecording("delayed-tab");
458 await vi.waitFor(() =>
459 expect(readActiveBrowserRecordingTabIds().has("delayed-tab")).toBe(true),
460 );
461 const blockingStart = startBrowserRecording("blocking-tab");
462 await vi.waitFor(() => expect(getDisplayMedia).toHaveBeenCalledOnce());
463
464 const delayedStop = stopBrowserRecording("delayed-tab");
465 releaseDelayedPaint(1);
466
467 await expect(delayedStart).rejects.toBeInstanceOf(BrowserRecordingStartCancelledError);
468 await expect(delayedStop).resolves.toBeNull();
469 expect(startScreencast).toHaveBeenCalledOnce();
470
471 finishBlockingCapture(stream);
472 await blockingStart;
473 await stopBrowserRecording("blocking-tab");
474 });
475
476 it("finishes an uncontended pre-grant start before stopping", async () => {
477 const animationFrames: FrameRequestCallback[] = [];
478 vi.stubGlobal(
479 "requestAnimationFrame",
480 vi.fn((callback: FrameRequestCallback) => {
481 animationFrames.push(callback);
482 return animationFrames.length;
483 }),
484 );
485
486 const startPromise = startBrowserRecording("recording-tab");
487 await vi.waitFor(() =>
488 expect(readActiveBrowserRecordingTabIds().has("recording-tab")).toBe(true),
489 );
490 const stopPromise = stopBrowserRecording("recording-tab");
491 expect(startScreencast).not.toHaveBeenCalled();
492
493 animationFrames.shift()?.(1);
494 animationFrames.shift()?.(2);
495 await startPromise;
496 await expect(stopPromise).resolves.toMatchObject({ tabId: "recording-tab" });
497 });
498
359499 it("keeps a recording reachable through its runtime id after a server epoch changes", async () => {
360500 const threadRef = {
361501 environmentId: EnvironmentId.make("environment-recording"),
......@@ -387,7 +527,6 @@ describe("browser recording", () => {
387527 await new Promise<void>((resolve) => {
388528 finishStartingScreencast = resolve;
389529 });
390 return { sourceId: "source:recording-tab", width: 1000, height: 620 };
391530 });
392531
393532 const firstStart = startBrowserRecording("recording-tab");
......@@ -452,7 +591,6 @@ describe("browser recording", () => {
452591 await new Promise<void>((resolve) => {
453592 finishStartingScreencast = resolve;
454593 });
455 return { sourceId: "source:recording-tab", width: 1000, height: 620 };
456594 });
457595
458596 const startPromise = startBrowserRecording("recording-tab");
......@@ -476,7 +614,6 @@ describe("browser recording", () => {
476614 await new Promise<void>((resolve) => {
477615 finishStartingScreencast = resolve;
478616 });
479 return { sourceId: "source:recording-tab", width: 1000, height: 620 };
480617 });
481618
482619 const firstStart = startBrowserRecording("recording-tab");
......@@ -503,7 +640,6 @@ describe("browser recording", () => {
503640 await new Promise<void>((resolve) => {
504641 finishStartingScreencast = resolve;
505642 });
506 return { sourceId: "source:recording-tab", width: 1000, height: 620 };
507643 });
508644 stopScreencast.mockRejectedValueOnce(new Error("initial stop failed"));
509645
......@@ -537,7 +673,6 @@ describe("browser recording", () => {
537673 await new Promise<void>((resolve) => {
538674 finishStartingScreencast = resolve;
539675 });
540 return { sourceId: "source:recording-tab", width: 1000, height: 620 };
541676 });
542677
543678 const startPromise = startBrowserRecording("recording-tab");
apps/web/src/browser/browserRecording.ts+204-62
......@@ -1,8 +1,5 @@
1import type {
2 DesktopPreviewRecordingArtifact,
3 DesktopPreviewRecordingSource,
4 ScopedThreadRef,
5} from "@t3tools/contracts";
1import { DESKTOP_PREVIEW_RECORDING_CAPTURE_TRIGGER } from "@t3tools/contracts";
2import type { DesktopPreviewRecordingArtifact, ScopedThreadRef } from "@t3tools/contracts";
63import { useAtomValue } from "@effect/atom-react";
74import * as Schema from "effect/Schema";
85import { Atom } from "effect/unstable/reactivity";
......@@ -36,6 +33,17 @@ export class BrowserRecordingConflictError extends Schema.TaggedErrorClass<Brows
3633 }
3734}
3835
36export class BrowserRecordingStartCancelledError extends Schema.TaggedErrorClass<BrowserRecordingStartCancelledError>()(
37 "BrowserRecordingStartCancelledError",
38 {
39 tabId: Schema.String,
40 },
41) {
42 override get message(): string {
43 return `Browser recording start was cancelled for tab ${this.tabId}.`;
44 }
45}
46
3947export class BrowserRecordingFormatUnavailableError extends Schema.TaggedErrorClass<BrowserRecordingFormatUnavailableError>()(
4048 "BrowserRecordingFormatUnavailableError",
4149 { tabId: Schema.String },
......@@ -82,9 +90,21 @@ export class BrowserRecordingOperationError extends Schema.TaggedErrorClass<Brow
8290
8391const isBrowserRecordingOperationError = Schema.is(BrowserRecordingOperationError);
8492const isBrowserRecordingCaptureTimeoutError = Schema.is(BrowserRecordingCaptureTimeoutError);
93export const isBrowserRecordingStartCancelledError = Schema.is(BrowserRecordingStartCancelledError);
94
95interface StartingBrowserRecordingLifecycle {
96 readonly phase: "starting";
97 queuedForGrant: boolean | null;
98 grantStarted: boolean;
99 stopRequestedBeforeGrant: boolean;
100 cancelledBeforeGrant: boolean;
101 readonly cancelledBeforeGrantSignal: Promise<void>;
102 readonly cancelBeforeGrant: () => void;
103 readonly setQueuedForGrant: (queued: boolean) => void;
104}
85105
86106type BrowserRecordingLifecycle =
87 | { readonly phase: "starting" }
107 | StartingBrowserRecordingLifecycle
88108 | { readonly phase: "recording" }
89109 | {
90110 readonly phase: "stopping";
......@@ -124,6 +144,53 @@ export function useActiveBrowserRecordingTabIds(): ReadonlySet<string> {
124144}
125145
126146const activeRecordings = new Map<string, ActiveRecording>();
147let displayMediaGrantTail = Promise.resolve();
148let displayMediaGrantQueueDepth = 0;
149
150const makeStartingBrowserRecordingLifecycle = (): StartingBrowserRecordingLifecycle => {
151 let signalCancellation!: () => void;
152 const cancelledBeforeGrantSignal = new Promise<void>((resolve) => {
153 signalCancellation = resolve;
154 });
155 const lifecycle: StartingBrowserRecordingLifecycle = {
156 phase: "starting",
157 queuedForGrant: null,
158 grantStarted: false,
159 stopRequestedBeforeGrant: false,
160 cancelledBeforeGrant: false,
161 cancelledBeforeGrantSignal,
162 cancelBeforeGrant: () => {
163 // Queue position is unknown during paint/settings warmup. Keep the stop request so a start
164 // that later turns out to be contended can still be cancelled before native capture.
165 lifecycle.stopRequestedBeforeGrant = true;
166 if (lifecycle.queuedForGrant && !lifecycle.grantStarted && !lifecycle.cancelledBeforeGrant) {
167 lifecycle.cancelledBeforeGrant = true;
168 signalCancellation();
169 }
170 },
171 setQueuedForGrant: (queued) => {
172 lifecycle.queuedForGrant = queued;
173 if (queued && lifecycle.stopRequestedBeforeGrant) lifecycle.cancelBeforeGrant();
174 },
175 };
176 return lifecycle;
177};
178
179const queueDisplayMediaGrant = <T>(
180 useGrant: () => Promise<T>,
181): { readonly queued: boolean; readonly result: Promise<T> } => {
182 const queued = displayMediaGrantQueueDepth > 0;
183 displayMediaGrantQueueDepth += 1;
184 const result = displayMediaGrantTail.then(useGrant);
185 const settleGrant = () => {
186 displayMediaGrantQueueDepth -= 1;
187 };
188 displayMediaGrantTail = result.then(
189 () => settleGrant(),
190 () => settleGrant(),
191 );
192 return { queued, result };
193};
127194
128195const publishActiveRecordingTabIds = (): void => {
129196 appAtomRegistry.set(activeBrowserRecordingTabIdsAtom, {
......@@ -184,23 +251,12 @@ const createMediaRecorder = (stream: MediaStream): MediaRecorder => {
184251 return mimeType ? new MediaRecorder(stream, { mimeType }) : new MediaRecorder(stream);
185252};
186253
187const captureTabMediaStream = (
188 source: DesktopPreviewRecordingSource,
189 frameRate: number,
190): Promise<MediaStream> =>
191 navigator.mediaDevices.getUserMedia({
254const captureTabMediaStream = (frameRate: number): Promise<MediaStream> =>
255 // The desktop main process routes this request to the tab that `startScreencast` armed, so the
256 // stream already arrives at that tab's native size and needs no source or dimension constraints.
257 navigator.mediaDevices.getDisplayMedia({
192258 audio: false,
193 video: {
194 mandatory: {
195 chromeMediaSource: "tab",
196 chromeMediaSourceId: source.sourceId,
197 minWidth: source.width,
198 maxWidth: source.width,
199 minHeight: source.height,
200 maxHeight: source.height,
201 maxFrameRate: frameRate,
202 },
203 } as unknown as MediaTrackConstraints,
259 video: { frameRate: { max: frameRate } },
204260 });
205261
206262const stopMediaRecorder = async (recorder: MediaRecorder | null): Promise<void> => {
......@@ -216,14 +272,75 @@ const stopMediaStream = (stream: MediaStream | null): void => {
216272 for (const track of stream?.getTracks() ?? []) track.stop();
217273};
218274
275interface PendingTabMediaCapture {
276 readonly start: () => void;
277}
278
279const pendingTabMediaCaptures = new Map<string, PendingTabMediaCapture>();
280
281const prepareTabMediaCapture = (tabId: string, frameRate: number) => {
282 let acceptStream = true;
283 let capturedStream: MediaStream | null = null;
284 let resolveCapture!: (stream: MediaStream | PromiseLike<MediaStream>) => void;
285 let rejectCapture!: (cause: unknown) => void;
286 const capturePromise = new Promise<MediaStream>((resolve, reject) => {
287 resolveCapture = resolve;
288 rejectCapture = reject;
289 }).then((stream) => {
290 capturedStream = stream;
291 if (!acceptStream) {
292 stopMediaStream(stream);
293 capturedStream = null;
294 }
295 return stream;
296 });
297 const pending: PendingTabMediaCapture = {
298 start: () => {
299 try {
300 // Electron invokes this callback through executeJavaScript(..., true), so even automated
301 // and delayed queued starts satisfy getDisplayMedia's transient-activation requirement.
302 resolveCapture(captureTabMediaStream(frameRate));
303 } catch (cause) {
304 rejectCapture(cause);
305 }
306 },
307 };
308 pendingTabMediaCaptures.set(tabId, pending);
309 return {
310 capturePromise,
311 cancel: () => {
312 acceptStream = false;
313 if (capturedStream) {
314 stopMediaStream(capturedStream);
315 capturedStream = null;
316 }
317 if (pendingTabMediaCaptures.get(tabId) === pending) pendingTabMediaCaptures.delete(tabId);
318 void capturePromise.catch(() => undefined);
319 },
320 };
321};
322
323const triggerTabMediaCapture = (tabId: unknown): boolean => {
324 if (typeof tabId !== "string") return false;
325 const pending = pendingTabMediaCaptures.get(tabId);
326 if (!pending) return false;
327 pendingTabMediaCaptures.delete(tabId);
328 pending.start();
329 return true;
330};
331
332Object.defineProperty(globalThis, DESKTOP_PREVIEW_RECORDING_CAPTURE_TRIGGER, {
333 configurable: true,
334 value: triggerTabMediaCapture,
335});
336
219337const captureTabMediaStreamWithTimeout = async (
220338 tabId: string,
221 source: DesktopPreviewRecordingSource,
222 frameRate: number,
339 capturePromise: Promise<MediaStream>,
223340): Promise<MediaStream> => {
224341 let acceptStream = true;
225342 let timeoutId: number | null = null;
226 const streamPromise = captureTabMediaStream(source, frameRate).then((stream) => {
343 const streamPromise = capturePromise.then((stream) => {
227344 if (!acceptStream) stopMediaStream(stream);
228345 return stream;
229346 });
......@@ -382,6 +499,7 @@ export async function startBrowserRecording(
382499 const startupSettled = new Promise<void>((resolve) => {
383500 settleStartup = resolve;
384501 });
502 const startingLifecycle = makeStartingBrowserRecordingLifecycle();
385503 const releaseSurfaceActivity = acquireBrowserSurfaceActivity(tabId);
386504 const recording: ActiveRecording = {
387505 tabId,
......@@ -393,7 +511,7 @@ export async function startBrowserRecording(
393511 releaseSurfaceActivity,
394512 stream: null,
395513 recorder: null,
396 lifecycle: { phase: "starting" },
514 lifecycle: startingLifecycle,
397515 };
398516 activeRecordings.set(tabId, recording);
399517 publishActiveRecordingTabIds();
......@@ -402,23 +520,9 @@ export async function startBrowserRecording(
402520 () => getClientSettings().browserRecordingFrameRate,
403521 );
404522 const [frameRate] = await Promise.all([frameRatePromise, waitForBrowserRecordingPaint()]);
405 let source: DesktopPreviewRecordingSource;
406 try {
407 source = await bridge.recording.startScreencast(tabId);
408 } catch (cause) {
409 if (!isRecordingStarting(recording)) {
410 throw recordingStartupCancelledError(recording, cause);
411 }
412 clearActiveRecording(recording);
413 throw new BrowserRecordingOperationError({
414 operation: "start-screencast",
415 tabId,
416 cause,
417 });
418 }
419523 const throwIfStartupCancelled = async (): Promise<void> => {
420 // A stop requested during startup should let startup finish so the
421 // caller receives a real artifact. Only replacement/removal cancels it.
524 // Once a grant starts, a stop lets startup finish so the caller receives an artifact.
525 // Only a contended start can be cancelled before it reaches native capture.
422526 if (activeRecordings.get(tabId) === recording) return;
423527 try {
424528 await bridge.recording.stopScreencast(tabId);
......@@ -434,30 +538,67 @@ export async function startBrowserRecording(
434538 }
435539 throw recordingStartupCancelledError(recording);
436540 };
437 await throwIfStartupCancelled();
438 try {
439 recording.stream = await captureTabMediaStreamWithTimeout(tabId, source, frameRate);
440 } catch (cause) {
441 const cleanupCause = await cleanupFailedRecordingStart(bridge, recording);
442 if (isBrowserRecordingCaptureTimeoutError(cause) && cleanupCause === undefined) throw cause;
443 throw new BrowserRecordingOperationError({
444 operation: "capture-media-stream",
445 tabId,
446 cause:
447 cleanupCause === undefined
448 ? cause
449 : new AggregateError(
450 [cause, cleanupCause],
451 `Browser media capture and cleanup failed for tab ${tabId}.`,
452 { cause },
453 ),
454 });
455 }
541 // The desktop process exposes one display-media grant at a time. Keep only the
542 // arm-to-capture handoff exclusive; acquired streams can record concurrently.
543 const grant = queueDisplayMediaGrant(async () => {
544 if (startingLifecycle.cancelledBeforeGrant) {
545 throw new BrowserRecordingStartCancelledError({ tabId });
546 }
547 startingLifecycle.grantStarted = true;
548 await throwIfStartupCancelled();
549 const capture = prepareTabMediaCapture(tabId, frameRate);
550 try {
551 await bridge.recording.startScreencast(tabId);
552 } catch (cause) {
553 capture.cancel();
554 if (!isRecordingStarting(recording)) {
555 throw recordingStartupCancelledError(recording, cause);
556 }
557 clearActiveRecording(recording);
558 throw new BrowserRecordingOperationError({
559 operation: "start-screencast",
560 tabId,
561 cause,
562 });
563 }
564 try {
565 await throwIfStartupCancelled();
566 } catch (cause) {
567 capture.cancel();
568 throw cause;
569 }
570 try {
571 recording.stream = await captureTabMediaStreamWithTimeout(tabId, capture.capturePromise);
572 return recording.stream;
573 } catch (cause) {
574 const cleanupCause = await cleanupFailedRecordingStart(bridge, recording);
575 if (isBrowserRecordingCaptureTimeoutError(cause) && cleanupCause === undefined) throw cause;
576 throw new BrowserRecordingOperationError({
577 operation: "capture-media-stream",
578 tabId,
579 cause:
580 cleanupCause === undefined
581 ? cause
582 : new AggregateError(
583 [cause, cleanupCause],
584 `Browser media capture and cleanup failed for tab ${tabId}.`,
585 { cause },
586 ),
587 });
588 }
589 });
590 startingLifecycle.setQueuedForGrant(grant.queued);
591 const stream = await Promise.race([
592 grant.result,
593 startingLifecycle.cancelledBeforeGrantSignal.then(() => {
594 throw new BrowserRecordingStartCancelledError({ tabId });
595 }),
596 ]);
456597 await throwIfStartupCancelled();
457598
458599 let recorder: MediaRecorder;
459600 try {
460 recorder = createMediaRecorder(recording.stream);
601 recorder = createMediaRecorder(stream);
461602 recording.recorder = recorder;
462603 recorder.addEventListener("dataavailable", (event) => {
463604 if (event.data.size > 0) chunks.push(event.data);
......@@ -639,6 +780,7 @@ export function stopBrowserRecording(
639780 const recording = activeRecordings.get(tabId);
640781 if (!bridge || !recording) return Promise.resolve(null);
641782 if (recording.lifecycle.phase === "stopping") return recording.lifecycle.stopPromise;
783 if (recording.lifecycle.phase === "starting") recording.lifecycle.cancelBeforeGrant();
642784
643785 const stopPromise = Promise.resolve()
644786 .then(() => finalizeBrowserRecording(bridge, recording))
apps/web/src/browser/hostedBrowserWebviewStyle.test.ts+21-1
......@@ -61,7 +61,7 @@ describe("resolveHostedBrowserWebviewWrapperStyle", () => {
6161 });
6262 });
6363
64 it("keeps an active background task paintable offscreen", () => {
64 it("keeps an active background task paintable behind the app", () => {
6565 const style = resolveHostedBrowserWebviewWrapperStyle({
6666 active: false,
6767 renderingActive: true,
......@@ -69,6 +69,26 @@ describe("resolveHostedBrowserWebviewWrapperStyle", () => {
6969 hiddenSize: { width: 1280, height: 800 },
7070 });
7171
72 expect(style).toEqual({
73 left: 0,
74 top: 0,
75 width: 1280,
76 height: 800,
77 zIndex: -1,
78 pointerEvents: "none",
79 visibility: "visible",
80 });
81 });
82
83 it("keeps an inactive webview paintable without marking it as rendering-active", () => {
84 const style = resolveHostedBrowserWebviewWrapperStyle({
85 active: false,
86 renderingActive: false,
87 keepPaintableWhenInactive: true,
88 rect: null,
89 hiddenSize: { width: 1280, height: 800 },
90 });
91
7292 expect(style).toEqual({
7393 left: HIDDEN_BROWSER_WEBVIEW_OFFSET,
7494 top: HIDDEN_BROWSER_WEBVIEW_OFFSET,
apps/web/src/browser/hostedBrowserWebviewStyle.ts+25-2
......@@ -21,11 +21,19 @@ export const HIDDEN_BROWSER_WEBVIEW_OFFSET = -100_000;
2121export function resolveHostedBrowserWebviewWrapperStyle(input: {
2222 readonly active: boolean;
2323 readonly renderingActive: boolean;
24 readonly keepPaintableWhenInactive?: boolean;
2425 readonly cornerRadius?: number;
2526 readonly rect: BrowserSurfaceRect | null;
2627 readonly hiddenSize: HostedBrowserWebviewSize;
2728}): HostedBrowserWebviewWrapperStyle {
28 const { active, cornerRadius = 0, hiddenSize, rect, renderingActive } = input;
29 const {
30 active,
31 cornerRadius = 0,
32 hiddenSize,
33 keepPaintableWhenInactive = false,
34 rect,
35 renderingActive,
36 } = input;
2937 if (active && rect) {
3038 return {
3139 left: rect.x,
......@@ -38,6 +46,21 @@ export function resolveHostedBrowserWebviewWrapperStyle(input: {
3846 };
3947 }
4048
49 if (renderingActive) {
50 // Electron stops compositing a guest that is fully outside the window, even
51 // when background throttling is disabled. Keep capture-active guests inside
52 // the viewport but behind the app so recordings receive complete frames.
53 return {
54 left: 0,
55 top: 0,
56 width: hiddenSize.width,
57 height: hiddenSize.height,
58 zIndex: -1,
59 pointerEvents: "none",
60 visibility: "visible",
61 };
62 }
63
4164 return {
4265 left: HIDDEN_BROWSER_WEBVIEW_OFFSET,
4366 top: HIDDEN_BROWSER_WEBVIEW_OFFSET,
......@@ -45,6 +68,6 @@ export function resolveHostedBrowserWebviewWrapperStyle(input: {
4568 height: hiddenSize.height,
4669 zIndex: -1,
4770 pointerEvents: "none",
48 visibility: renderingActive ? "visible" : "hidden",
71 visibility: keepPaintableWhenInactive ? "visible" : "hidden",
4972 };
5073}
apps/web/src/components/preview/PreviewView.test.tsx+1
......@@ -149,6 +149,7 @@ vi.mock("~/state/use-atom-command", () => ({
149149
150150vi.mock("~/browser/browserRecording", () => ({
151151 findActiveBrowserRecordingRuntimeTabId: vi.fn(() => null),
152 isBrowserRecordingStartCancelledError: vi.fn(() => false),
152153 startBrowserRecording: vi.fn(),
153154 stopBrowserRecording: vi.fn(),
154155 useActiveBrowserRecordingTabIds: () => new Set(),
apps/web/src/components/preview/PreviewView.tsx+4-1
......@@ -55,6 +55,7 @@ import { ZoomIndicator } from "./ZoomIndicator";
5555import { AgentBrowserCursor } from "./AgentBrowserCursor";
5656import {
5757 findActiveBrowserRecordingRuntimeTabId,
58 isBrowserRecordingStartCancelledError,
5859 startBrowserRecording,
5960 stopBrowserRecording,
6061 useActiveBrowserRecordingTabIds,
......@@ -398,10 +399,12 @@ export function PreviewView({
398399 }
399400 if (record) {
400401 void startBrowserRecording(runtimeTabId, threadRef, tabId).catch((error) => {
402 const description = error instanceof Error ? error.message : "An error occurred.";
403 if (isBrowserRecordingStartCancelledError(error)) return;
401404 toastManager.add({
402405 type: "error",
403406 title: "Unable to start recording",
404 description: error instanceof Error ? error.message : "An error occurred.",
407 description,
405408 });
406409 });
407410 return;
packages/contracts/src/ipc.ts+4-14
......@@ -734,19 +734,6 @@ export const DesktopPreviewRecordingFrameSchema: Schema.Codec<DesktopPreviewReco
734734 receivedAt: Schema.String,
735735 });
736736
737export interface DesktopPreviewRecordingSource {
738 sourceId: string;
739 width: number;
740 height: number;
741}
742
743export const DesktopPreviewRecordingSourceSchema: Schema.Codec<DesktopPreviewRecordingSource> =
744 Schema.Struct({
745 sourceId: Schema.String,
746 width: Schema.Int.check(Schema.isGreaterThan(0)),
747 height: Schema.Int.check(Schema.isGreaterThan(0)),
748 });
749
750737export interface DesktopPreviewRecordingArtifact {
751738 id: string;
752739 tabId: string;
......@@ -1173,6 +1160,9 @@ export interface DesktopBridge {
11731160 preview?: DesktopPreviewBridge;
11741161}
11751162
1163/** Renderer callback invoked by Electron with a fresh user gesture before display-media capture. */
1164export const DESKTOP_PREVIEW_RECORDING_CAPTURE_TRIGGER = "__t3DesktopPreviewRecordingCapture";
1165
11761166export interface DesktopPreviewBridge {
11771167 createTab: (tabId: string, defaults?: DesktopPreviewTabDefaults) => Promise<void>;
11781168 closeTab: (tabId: string) => Promise<void>;
......@@ -1228,7 +1218,7 @@ export interface DesktopPreviewBridge {
12281218 close: (tabId: string) => Promise<void>;
12291219 };
12301220 recording: {
1231 startScreencast: (tabId: string) => Promise<DesktopPreviewRecordingSource>;
1221 startScreencast: (tabId: string) => Promise<void>;
12321222 stopScreencast: (tabId: string) => Promise<void>;
12331223 save: (
12341224 tabId: string,