authorgravatar for maria@kuuro.netmaria <maria@kuuro.net> 2026-08-31 01:28:24-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-08-31 01:28:24-04:00
log3958111057c10c10350dd9c20ec2a2df00f504be
tree2345787328c2898485bd82d18352c77f8e7350f3
parent3f62e6fa65c2a2a91a367be4ee95da1b50007bde
signature Signed by PGP key B5690EEEBB952194

fix(preview): improve browser recording quality (#8839)

Co-authored-by: maria-rcks <254055478+maria-rcks@users.noreply.github.com>

15 files changed, 861 insertions(+), 793 deletions(-)

apps/desktop/src/ipc/methods/preview.ts+10-5
...@@ -11,6 +11,7 @@ import {...@@ -11,6 +11,7 @@ import {
11 DesktopPreviewConfigInputSchema,11 DesktopPreviewConfigInputSchema,
12 DesktopPreviewNavigateInputSchema,12 DesktopPreviewNavigateInputSchema,
13 DesktopPreviewRecordingArtifactSchema,13 DesktopPreviewRecordingArtifactSchema,
14 DesktopPreviewRecordingSourceSchema,
14 DesktopPreviewRecordingSaveInputSchema,15 DesktopPreviewRecordingSaveInputSchema,
15 DesktopPreviewRegisterWebviewInputSchema,16 DesktopPreviewRegisterWebviewInputSchema,
16 DesktopPreviewScreenshotArtifactSchema,17 DesktopPreviewScreenshotArtifactSchema,
...@@ -173,11 +174,15 @@ export const cancelPickElement = tabMethod(...@@ -173,11 +174,15 @@ export const cancelPickElement = tabMethod(
173 "desktop.ipc.preview.cancelPickElement",174 "desktop.ipc.preview.cancelPickElement",
174 (manager, tabId) => manager.cancelPickElement(tabId),175 (manager, tabId) => manager.cancelPickElement(tabId),
175);176);
176export const startRecording = tabMethod(177export const startRecording = DesktopIpc.makeIpcMethod({
177 IpcChannels.PREVIEW_RECORDING_START_CHANNEL,178 channel: IpcChannels.PREVIEW_RECORDING_START_CHANNEL,
178 "desktop.ipc.preview.startRecording",179 payload: DesktopPreviewTabInputSchema,
179 (manager, tabId) => manager.startRecording(tabId),180 result: DesktopPreviewRecordingSourceSchema,
180);181 handler: Effect.fn("desktop.ipc.preview.startRecording")(function* ({ tabId }) {
182 const manager = yield* PreviewManager.PreviewManager;
183 return yield* manager.startRecording(tabId);
184 }),
185});
181export const stopRecording = tabMethod(186export const stopRecording = tabMethod(
182 IpcChannels.PREVIEW_RECORDING_STOP_CHANNEL,187 IpcChannels.PREVIEW_RECORDING_STOP_CHANNEL,
183 "desktop.ipc.preview.stopRecording",188 "desktop.ipc.preview.stopRecording",
apps/desktop/src/preview/Manager.test.ts+180-319
...@@ -36,6 +36,14 @@ describe("fitPictureInPictureContentSize", () => {...@@ -36,6 +36,14 @@ describe("fitPictureInPictureContentSize", () => {
36 });36 });
37});37});
3838
39describe("recordingFileExtension", () => {
40 it("derives the artifact extension from the recorder's actual mime type", () => {
41 expect(PreviewManager.recordingFileExtension("video/mp4;codecs=avc1.640028")).toBe("mp4");
42 expect(PreviewManager.recordingFileExtension("video/webm;codecs=vp9")).toBe("webm");
43 expect(PreviewManager.recordingFileExtension("video/x-matroska")).toBe("matroska");
44 });
45});
46
39describe("isPreviewRefreshShortcut", () => {47describe("isPreviewRefreshShortcut", () => {
40 const input = (overrides: Partial<Electron.Input> = {}) =>48 const input = (overrides: Partial<Electron.Input> = {}) =>
41 ({49 ({
...@@ -207,6 +215,9 @@ const makeTestPreviewWebContents = (...@@ -207,6 +215,9 @@ const makeTestPreviewWebContents = (
207) =>215) =>
208 ({216 ({
209 id,217 id,
218 hostWebContents: { id: 7 },
219 getMediaSourceId: vi.fn(() => `tab:${id}`),
220 executeJavaScript: vi.fn(async () => ({ width: 1280, height: 720 })),
210 isDestroyed: () => false,221 isDestroyed: () => false,
211 getType: () => "webview",222 getType: () => "webview",
212 getURL: () => "https://example.com",223 getURL: () => "https://example.com",
...@@ -1666,7 +1677,9 @@ describe("PreviewManager", () => {...@@ -1666,7 +1677,9 @@ describe("PreviewManager", () => {
1666 const recreated = yield* Fiber.join(recreateFiber);1677 const recreated = yield* Fiber.join(recreateFiber);
1667 const registrationExit = yield* Fiber.await(registrationFiber);1678 const registrationExit = yield* Fiber.await(registrationFiber);
16681679
1669 for (const exit of [registrationExit, recordingExit]) {1680 for (const exit of [registrationExit, recordingExit] as ReadonlyArray<
1681 Exit.Exit<unknown, PreviewManager.PreviewManagerError>
1682 >) {
1670 expect(Exit.isFailure(exit)).toBe(true);1683 expect(Exit.isFailure(exit)).toBe(true);
1671 if (Exit.isSuccess(exit)) continue;1684 if (Exit.isSuccess(exit)) continue;
1672 expect(Option.getOrThrow(Cause.findErrorOption(exit.cause))).toMatchObject({1685 expect(Option.getOrThrow(Cause.findErrorOption(exit.cause))).toMatchObject({
...@@ -2062,7 +2075,7 @@ describe("PreviewManager", () => {...@@ -2062,7 +2075,7 @@ describe("PreviewManager", () => {
2062 ),2075 ),
2063 );2076 );
20642077
2065 effectIt.effect("captures hidden preview recordings independently for concurrent tabs", () =>2078 effectIt.effect("returns native media sources for concurrent preview recordings", () =>
2066 withManager((manager) =>2079 withManager((manager) =>
2067 Effect.gen(function* () {2080 Effect.gen(function* () {
2068 const firstJpeg = Buffer.from("first-recording-frame");2081 const firstJpeg = Buffer.from("first-recording-frame");
...@@ -2084,6 +2097,11 @@ describe("PreviewManager", () => {...@@ -2084,6 +2097,11 @@ describe("PreviewManager", () => {
2084 ) =>2097 ) =>
2085 ({2098 ({
2086 id,2099 id,
2100 hostWebContents: { id: 7 },
2101 getMediaSourceId: vi.fn(() => `tab:${id}`),
2102 executeJavaScript: vi.fn(async () =>
2103 id === 41 ? { width: 800, height: 600 } : { width: 390, height: 844 },
2104 ),
2087 isDestroyed: () => false,2105 isDestroyed: () => false,
2088 getType: () => "webview",2106 getType: () => "webview",
2089 getURL: () => `https://example.com/${id}`,2107 getURL: () => `https://example.com/${id}`,
...@@ -2115,41 +2133,21 @@ describe("PreviewManager", () => {...@@ -2115,41 +2133,21 @@ describe("PreviewManager", () => {
2115 fromId.mockImplementation((id) =>2133 fromId.mockImplementation((id) =>
2116 id === undefined ? null : (webContentsById.get(id) ?? null),2134 id === undefined ? null : (webContentsById.get(id) ?? null),
2117 );2135 );
2118 const frames: DesktopPreviewRecordingFrame[] = [];
2119
2120 yield* manager.subscribeRecordingFrames((frame) =>
2121 Effect.sync(() => {
2122 frames.push(frame);
2123 }),
2124 );
2125 yield* manager.createTab("tab_1");2136 yield* manager.createTab("tab_1");
2126 yield* manager.createTab("tab_2");2137 yield* manager.createTab("tab_2");
2127 yield* manager.registerWebview("tab_1", 41);2138 yield* manager.registerWebview("tab_1", 41);
2128 yield* manager.registerWebview("tab_2", 42);2139 yield* manager.registerWebview("tab_2", 42);
2129 yield* Effect.all([manager.startRecording("tab_1"), manager.startRecording("tab_2")], {2140 const sources = yield* Effect.all(
2130 concurrency: 2,2141 [manager.startRecording("tab_1"), manager.startRecording("tab_2")],
2131 discard: true,2142 { concurrency: 2 },
2132 });2143 );
21332144
2145 expect(sources).toEqual([
2146 { sourceId: "tab:41", width: 800, height: 600 },
2147 { sourceId: "tab:42", width: 390, height: 844 },
2148 ]);
2134 expect(firstCapturePage).toHaveBeenCalledOnce();2149 expect(firstCapturePage).toHaveBeenCalledOnce();
2135 expect(secondCapturePage).toHaveBeenCalledOnce();2150 expect(secondCapturePage).toHaveBeenCalledOnce();
2136 expect(frames).toHaveLength(2);
2137 expect(frames).toEqual(
2138 expect.arrayContaining([
2139 expect.objectContaining({
2140 tabId: "tab_1",
2141 data: firstJpeg.toString("base64"),
2142 width: 800,
2143 height: 600,
2144 }),
2145 expect.objectContaining({
2146 tabId: "tab_2",
2147 data: secondJpeg.toString("base64"),
2148 width: 390,
2149 height: 844,
2150 }),
2151 ]),
2152 );
2153 expect(firstSendCommand).not.toHaveBeenCalledWith(2151 expect(firstSendCommand).not.toHaveBeenCalledWith(
2154 "Page.startScreencast",2152 "Page.startScreencast",
2155 expect.anything(),2153 expect.anything(),
...@@ -2167,203 +2165,178 @@ describe("PreviewManager", () => {...@@ -2167,203 +2165,178 @@ describe("PreviewManager", () => {
2167 ),2165 ),
2168 );2166 );
21692167
2170 effectIt.effect("drops a captured frame when the tab webview changes during capture", () =>2168 effectIt.effect("continues native recording when the source warmup fails", () =>
2171 withManager((manager) =>2169 withManager((manager) =>
2172 Effect.gen(function* () {2170 Effect.gen(function* () {
2173 const staleImage: TestCapturedPreviewImage = {2171 const capturePage = vi.fn(async () => {
2174 toJPEG: vi.fn(() => Buffer.from("stale-recording-frame")),2172 throw new Error("source is not ready");
2175 getSize: vi.fn(() => ({ width: 1280, height: 720 })),
2176 };
2177 let markCaptureStarted!: () => void;
2178 const captureStarted = new Promise<void>((resolve) => {
2179 markCaptureStarted = resolve;
2180 });
2181 let resolveCapture: ((image: TestCapturedPreviewImage) => void) | undefined;
2182 const staleCapturePage = vi.fn(() => {
2183 markCaptureStarted();
2184 return new Promise<TestCapturedPreviewImage>((resolve) => {
2185 resolveCapture = resolve;
2186 });
2187 });2173 });
2188 const replacementCapturePage = vi.fn(async () => ({2174 const getMediaSourceId = vi.fn(() => "tab:42");
2189 toJPEG: () => Buffer.from("replacement-recording-frame"),2175 const webContents = Object.assign(makeTestPreviewWebContents(capturePage), {
2190 getSize: () => ({ width: 1280, height: 720 }),2176 executeJavaScript: vi.fn(async () => ({ width: 1280, height: 720 })),
2191 }));2177 getMediaSourceId,
2192 const initialWebContents = makeTestPreviewWebContents(staleCapturePage, 42);
2193 const replacementWebContents = makeTestPreviewWebContents(replacementCapturePage, 43);
2194 fromId.mockImplementation((webContentsId?: number) => {
2195 if (webContentsId === 42) return initialWebContents;
2196 if (webContentsId === 43) return replacementWebContents;
2197 return null;
2198 });2178 });
2199 const frames: DesktopPreviewRecordingFrame[] = [];2179 fromId.mockReturnValue(webContents);
22002180
2201 yield* manager.subscribeRecordingFrames((frame) =>2181 yield* manager.createTab("tab_recording_warmup_failure");
2202 Effect.sync(() => {2182 yield* manager.registerWebview("tab_recording_warmup_failure", 42);
2203 frames.push(frame);
2204 }),
2205 );
2206 yield* manager.createTab("tab_capture_replaced");
2207 yield* manager.registerWebview("tab_capture_replaced", 42);
2208 const recordingFiber = yield* manager
2209 .startRecording("tab_capture_replaced")
2210 .pipe(Effect.forkChild({ startImmediately: true }));
2211 yield* Effect.promise(() => captureStarted);
2212
2213 yield* manager.registerWebview("tab_capture_replaced", 43);
2214 resolveCapture?.(staleImage);
2215 yield* Fiber.join(recordingFiber);
22162183
2217 expect(staleImage.getSize).not.toHaveBeenCalled();2184 expect(yield* manager.startRecording("tab_recording_warmup_failure")).toEqual({
2218 expect(staleImage.toJPEG).not.toHaveBeenCalled();2185 sourceId: "tab:42",
2219 expect(frames).toHaveLength(0);2186 width: 1280,
2220 expect(replacementCapturePage).not.toHaveBeenCalled();2187 height: 720,
2188 });
2189 expect(capturePage).toHaveBeenCalledTimes(2);
2190 expect(getMediaSourceId).toHaveBeenCalledOnce();
22212191
2222 yield* manager.stopRecording("tab_capture_replaced");2192 yield* manager.stopRecording("tab_recording_warmup_failure");
2223 }),2193 }),
2224 ),2194 ),
2225 );2195 );
22262196
2227 effectIt.effect("keeps an in-flight frame when a capture consumer is added", () =>2197 effectIt.effect("reports invalid native recording dimensions as a structured error", () =>
2228 withManager((manager) =>2198 withManager((manager) =>
2229 Effect.gen(function* () {2199 Effect.gen(function* () {
2230 const image: TestCapturedPreviewImage = {2200 const capturePage = vi.fn(async () => ({
2231 toJPEG: vi.fn(() => Buffer.from("shared-in-flight-frame")),2201 toJPEG: () => Buffer.from("unused-recording-frame"),
2232 getSize: vi.fn(() => ({ width: 1280, height: 720 })),2202 getSize: () => ({ width: 1280, height: 720 }),
2233 };2203 }));
2234 let markCaptureStarted!: () => void;2204 const getMediaSourceId = vi.fn(() => "tab:42");
2235 const captureStarted = new Promise<void>((resolve) => {2205 const webContents = Object.assign(makeTestPreviewWebContents(capturePage), {
2236 markCaptureStarted = resolve;2206 executeJavaScript: vi.fn(async () => ({ width: 0, height: 720 })),
2237 });2207 getMediaSourceId,
2238 let resolveCapture: ((captured: TestCapturedPreviewImage) => void) | undefined;
2239 const capturePage = vi.fn(() => {
2240 markCaptureStarted();
2241 return new Promise<TestCapturedPreviewImage>((resolve) => {
2242 resolveCapture = resolve;
2243 });
2244 });
2245 fromId.mockReturnValue(makeTestPreviewWebContents(capturePage));
2246 const { pictureInPictureWindow, send } = makeTestPictureInPictureWindow();
2247 browserWindowConstructor.mockImplementation(function () {
2248 return pictureInPictureWindow;
2249 });2208 });
2250 const recordingFrames: DesktopPreviewRecordingFrame[] = [];2209 fromId.mockReturnValue(webContents);
2251 yield* manager.subscribeRecordingFrames((frame) =>
2252 Effect.sync(() => {
2253 recordingFrames.push(frame);
2254 }),
2255 );
2256
2257 yield* manager.createTab("tab_capture_consumer_added");
2258 yield* manager.registerWebview("tab_capture_consumer_added", 42);
2259 const recordingFiber = yield* manager
2260 .startRecording("tab_capture_consumer_added")
2261 .pipe(Effect.forkChild({ startImmediately: true }));
2262 yield* Effect.promise(() => captureStarted);
22632210
2264 yield* manager.openPictureInPicture("tab_capture_consumer_added");2211 yield* manager.createTab("tab_invalid_recording_size");
2265 resolveCapture?.(image);2212 yield* manager.registerWebview("tab_invalid_recording_size", 42);
2266 yield* Fiber.join(recordingFiber);2213 const exit = yield* Effect.exit(manager.startRecording("tab_invalid_recording_size"));
22672214
2268 expect(recordingFrames).toHaveLength(1);2215 expect(Exit.isFailure(exit)).toBe(true);
2269 expect(send).toHaveBeenCalledWith(2216 if (Exit.isSuccess(exit)) return;
2270 "desktop:preview-pip-frame",2217 expect(Option.getOrThrow(Cause.findErrorOption(exit.cause))).toMatchObject({
2271 expect.objectContaining({2218 _tag: "PreviewRecordingSourceSizeUnavailableError",
2272 tabId: "tab_capture_consumer_added",2219 tabId: "tab_invalid_recording_size",
2273 data: Buffer.from("shared-in-flight-frame").toString("base64"),2220 webContentsId: 42,
2274 }),2221 });
2275 );2222 expect(getMediaSourceId).not.toHaveBeenCalled();
2276
2277 yield* manager.stopRecording("tab_capture_consumer_added");
2278 yield* manager.closePictureInPicture("tab_capture_consumer_added");
2279 }),2223 }),
2280 ),2224 ),
2281 );2225 );
22822226
2283 effectIt.effect("emits debugger screencast frames only while recording is active", () =>2227 effectIt.effect("keeps a newer recording lease when an earlier start fails", () =>
2284 withManager((manager) =>2228 withManager((manager) =>
2285 Effect.gen(function* () {2229 Effect.gen(function* () {
2286 let debuggerMessage:2230 const setBackgroundThrottling = vi.fn();
2287 | ((event: unknown, method: string, params: Record<string, unknown>) => void)
2288 | undefined;
2289 const capturePage = vi.fn(async () => ({2231 const capturePage = vi.fn(async () => ({
2290 toJPEG: () => Buffer.from("scheduled-recording-frame"),2232 toJPEG: () => Buffer.from("unused-recording-frame"),
2291 getSize: () => ({ width: 1280, height: 720 }),2233 getSize: () => ({ width: 1280, height: 720 }),
2292 }));2234 }));
2293 const sendCommand = vi.fn(async (method: string) =>2235 let markMeasurementStarted!: () => void;
2294 method === "Runtime.evaluate" ? { result: { value: null } } : undefined,2236 const measurementStarted = new Promise<void>((resolve) => {
2295 );2237 markMeasurementStarted = resolve;
2296 fromId.mockReturnValue({2238 });
2297 id: 42,2239 let rejectFirstMeasurement!: (error: Error) => void;
2240 const executeJavaScript = vi
2241 .fn<(expression: string, userGesture?: boolean) => Promise<unknown>>()
2242 .mockImplementationOnce(() => {
2243 markMeasurementStarted();
2244 return new Promise((_, reject) => {
2245 rejectFirstMeasurement = reject;
2246 });
2247 })
2248 .mockResolvedValue({ width: 1280, height: 720 });
2249 const webContents = Object.assign(makeTestPreviewWebContents(capturePage), {
2250 executeJavaScript,
2251 });
2252 fromId.mockReturnValue(webContents);
2253
2254 yield* manager.createTab("tab_recording_start_race");
2255 yield* manager.registerWebview("tab_recording_start_race", 42);
2256 yield* manager.setMainWindow({
2298 isDestroyed: () => false,2257 isDestroyed: () => false,
2299 getType: () => "webview",2258 once: vi.fn(),
2300 getURL: () => "https://example.com",2259 webContents: { setBackgroundThrottling },
2301 getTitle: () => "Example",
2302 isLoading: () => false,
2303 isDevToolsOpened: () => false,
2304 getZoomFactor: () => 1,
2305 setZoomFactor: vi.fn(),
2306 setAudioMuted: vi.fn(),
2307 isCurrentlyAudible: () => false,
2308 on: vi.fn(),
2309 off: vi.fn(),
2310 ipc: { on: vi.fn(), off: vi.fn() },
2311 send: webviewSend,
2312 navigationHistory: { canGoBack: () => false, canGoForward: () => false },
2313 setWindowOpenHandler: vi.fn(),
2314 debugger: {
2315 isAttached: () => false,
2316 attach: vi.fn(),
2317 sendCommand,
2318 on: vi.fn(
2319 (
2320 event: string,
2321 listener: (event: unknown, method: string, params: Record<string, unknown>) => void,
2322 ) => {
2323 if (event === "message") debuggerMessage = listener;
2324 },
2325 ),
2326 off: vi.fn(),
2327 },
2328 capturePage,
2329 } as never);2260 } as never);
2330 const recordingFrames: DesktopPreviewRecordingFrame[] = [];
23312261
2332 yield* manager.subscribeRecordingFrames((frame) =>2262 const firstStart = yield* manager
2333 Effect.sync(() => {2263 .startRecording("tab_recording_start_race")
2334 recordingFrames.push(frame);2264 .pipe(Effect.forkChild({ startImmediately: true }));
2335 }),2265 yield* Effect.promise(() => measurementStarted);
2336 );2266 const secondStart = yield* manager
2337 yield* manager.createTab("tab_screencast_guard");2267 .startRecording("tab_recording_start_race")
2338 yield* manager.registerWebview("tab_screencast_guard", 42);2268 .pipe(Effect.forkChild({ startImmediately: true }));
2339 yield* manager.automationEvaluate("tab_screencast_guard", { expression: "null" });2269 yield* Effect.yieldNow;
2270 expect(executeJavaScript).toHaveBeenCalledOnce();
23402271
2341 debuggerMessage?.({}, "Page.screencastFrame", {2272 rejectFirstMeasurement(new Error("first measurement failed"));
2342 sessionId: 1,2273 const firstExit = yield* Fiber.await(firstStart);
2343 data: "inactive-frame",2274 expect(Exit.isFailure(firstExit)).toBe(true);
2344 metadata: { deviceWidth: 1280, deviceHeight: 720 },2275 expect(yield* Fiber.join(secondStart)).toEqual({
2276 sourceId: "tab:42",
2277 width: 1280,
2278 height: 720,
2345 });2279 });
2346 yield* Effect.yieldNow;
2347 expect(recordingFrames).toHaveLength(0);
23482280
2349 yield* manager.startRecording("tab_screencast_guard");2281 yield* manager.stopRecording("tab_recording_start_race");
2350 recordingFrames.length = 0;2282 expect(setBackgroundThrottling.mock.calls).toEqual([[false], [true], [false], [true]]);
2351 debuggerMessage?.({}, "Page.screencastFrame", {2283 }),
2352 sessionId: 2,2284 ),
2353 data: "active-frame",2285 );
2354 metadata: { deviceWidth: 1280, deviceHeight: 720 },2286
2287 effectIt.effect("serializes recording source acquisition with webview replacement", () =>
2288 withManager((manager) =>
2289 Effect.gen(function* () {
2290 const capturePage = vi.fn(async () => ({
2291 toJPEG: () => Buffer.from("unused-recording-frame"),
2292 getSize: () => ({ width: 1280, height: 720 }),
2293 }));
2294 let markMeasurementStarted!: () => void;
2295 const measurementStarted = new Promise<void>((resolve) => {
2296 markMeasurementStarted = resolve;
2355 });2297 });
2298 let finishMeasurement!: (size: { readonly width: number; readonly height: number }) => void;
2299 const executeJavaScript = vi.fn(
2300 () =>
2301 new Promise<{ readonly width: number; readonly height: number }>((resolve) => {
2302 markMeasurementStarted();
2303 finishMeasurement = resolve;
2304 }),
2305 );
2306 const initialWebContents = Object.assign(makeTestPreviewWebContents(capturePage, 42), {
2307 executeJavaScript,
2308 });
2309 const replacementOn = vi.fn();
2310 const replacementWebContents = Object.assign(makeTestPreviewWebContents(capturePage, 43), {
2311 on: replacementOn,
2312 });
2313 fromId.mockImplementation((id) => {
2314 if (id === 42) return initialWebContents;
2315 if (id === 43) return replacementWebContents;
2316 return null;
2317 });
2318
2319 yield* manager.createTab("tab_recording_replacement_race");
2320 yield* manager.registerWebview("tab_recording_replacement_race", 42);
2321 const start = yield* manager
2322 .startRecording("tab_recording_replacement_race")
2323 .pipe(Effect.forkChild({ startImmediately: true }));
2324 yield* Effect.promise(() => measurementStarted);
2325 const replacement = yield* manager
2326 .registerWebview("tab_recording_replacement_race", 43)
2327 .pipe(Effect.forkChild({ startImmediately: true }));
2356 yield* Effect.yieldNow;2328 yield* Effect.yieldNow;
2329 expect(replacementOn).not.toHaveBeenCalled();
23572330
2358 expect(recordingFrames).toEqual([2331 finishMeasurement({ width: 1280, height: 720 });
2359 expect.objectContaining({2332 expect(yield* Fiber.join(start)).toEqual({
2360 tabId: "tab_screencast_guard",2333 sourceId: "tab:42",
2361 data: "active-frame",2334 width: 1280,
2362 width: 1280,2335 height: 720,
2363 height: 720,2336 });
2364 }),2337 yield* Fiber.join(replacement);
2365 ]);2338 expect(replacementOn).toHaveBeenCalled();
2366 yield* manager.stopRecording("tab_screencast_guard");2339 yield* manager.stopRecording("tab_recording_replacement_race");
2367 }),2340 }),
2368 ),2341 ),
2369 );2342 );
...@@ -2381,6 +2354,8 @@ describe("PreviewManager", () => {...@@ -2381,6 +2354,8 @@ describe("PreviewManager", () => {
2381 fromId.mockReturnValue({2354 fromId.mockReturnValue({
2382 id: 42,2355 id: 42,
2383 hostWebContents: mainWindowWebContents,2356 hostWebContents: mainWindowWebContents,
2357 getMediaSourceId: vi.fn(() => "tab:42"),
2358 executeJavaScript: vi.fn(async () => ({ width: 1280, height: 720 })),
2384 isDestroyed: () => false,2359 isDestroyed: () => false,
2385 getType: () => "webview",2360 getType: () => "webview",
2386 getURL: () => "https://example.com",2361 getURL: () => "https://example.com",
...@@ -2494,21 +2469,21 @@ describe("PreviewManager", () => {...@@ -2494,21 +2469,21 @@ describe("PreviewManager", () => {
2494 const pictureInPictureFramesBeforeRecording = pictureInPictureSend.mock.calls.length;2469 const pictureInPictureFramesBeforeRecording = pictureInPictureSend.mock.calls.length;
24952470
2496 yield* manager.startRecording("tab_pip");2471 yield* manager.startRecording("tab_pip");
2497 expect(capturePage).toHaveBeenCalledOnce();2472 expect(capturePage).toHaveBeenCalledTimes(2);
2498 expect(recordingFrames).toHaveLength(0);2473 expect(recordingFrames).toHaveLength(0);
24992474
2500 yield* TestClock.adjust(100);2475 yield* TestClock.adjust(100);
2501 expect(capturePage).toHaveBeenCalledTimes(2);2476 expect(capturePage).toHaveBeenCalledTimes(3);
2502 expect(pictureInPictureSend).toHaveBeenCalledTimes(pictureInPictureFramesBeforeRecording);2477 expect(pictureInPictureSend).toHaveBeenCalledTimes(pictureInPictureFramesBeforeRecording);
2503 expect(recordingFrames).toHaveLength(1);2478 expect(recordingFrames).toHaveLength(0);
25042479
2505 yield* manager.stopRecording("tab_pip");2480 yield* manager.stopRecording("tab_pip");
2506 expect(setBackgroundThrottling.mock.calls).toEqual([[false]]);2481 expect(setBackgroundThrottling.mock.calls).toEqual([[false]]);
2507 const framesBeforePictureInPictureOnlyTick = pictureInPictureSend.mock.calls.length;2482 const framesBeforePictureInPictureOnlyTick = pictureInPictureSend.mock.calls.length;
2508 yield* TestClock.adjust(100);2483 yield* TestClock.adjust(100);
2509 expect(capturePage).toHaveBeenCalledTimes(3);2484 expect(capturePage).toHaveBeenCalledTimes(4);
2510 expect(pictureInPictureSend.mock.calls.length).toBe(framesBeforePictureInPictureOnlyTick);2485 expect(pictureInPictureSend.mock.calls.length).toBe(framesBeforePictureInPictureOnlyTick);
2511 expect(recordingFrames).toHaveLength(1);2486 expect(recordingFrames).toHaveLength(0);
25122487
2513 setBackgroundThrottling.mockImplementationOnce(() => {2488 setBackgroundThrottling.mockImplementationOnce(() => {
2514 throw new Error("picture-in-picture throttling restore failed");2489 throw new Error("picture-in-picture throttling restore failed");
...@@ -2524,7 +2499,7 @@ describe("PreviewManager", () => {...@@ -2524,7 +2499,7 @@ describe("PreviewManager", () => {
2524 ),2499 ),
2525 );2500 );
25262501
2527 effectIt.effect("starts picture-in-picture without an extra recording capture", () =>2502 effectIt.effect("keeps picture-in-picture capture separate from recording warmup", () =>
2528 withManager((manager) =>2503 withManager((manager) =>
2529 Effect.gen(function* () {2504 Effect.gen(function* () {
2530 const jpeg = Buffer.from("shared-preview-frame");2505 const jpeg = Buffer.from("shared-preview-frame");
...@@ -2548,15 +2523,16 @@ describe("PreviewManager", () => {...@@ -2548,15 +2523,16 @@ describe("PreviewManager", () => {
2548 yield* manager.registerWebview("tab_recording_then_pip", 42);2523 yield* manager.registerWebview("tab_recording_then_pip", 42);
2549 yield* manager.startRecording("tab_recording_then_pip");2524 yield* manager.startRecording("tab_recording_then_pip");
25502525
2551 expect(recordingFrames).toHaveLength(1);2526 expect(recordingFrames).toHaveLength(0);
2552 yield* manager.openPictureInPicture("tab_recording_then_pip");
2553 expect(capturePage).toHaveBeenCalledOnce();2527 expect(capturePage).toHaveBeenCalledOnce();
2554 expect(send).not.toHaveBeenCalled();2528 yield* manager.openPictureInPicture("tab_recording_then_pip");
2529 expect(capturePage).toHaveBeenCalledTimes(2);
2530 expect(send).toHaveBeenCalledOnce();
25552531
2556 yield* TestClock.adjust(100);2532 yield* TestClock.adjust(100);
25572533
2558 expect(capturePage).toHaveBeenCalledTimes(2);2534 expect(capturePage).toHaveBeenCalledTimes(3);
2559 expect(recordingFrames).toHaveLength(2);2535 expect(recordingFrames).toHaveLength(0);
2560 expect(send).toHaveBeenCalledOnce();2536 expect(send).toHaveBeenCalledOnce();
2561 yield* manager.closePictureInPicture("tab_recording_then_pip");2537 yield* manager.closePictureInPicture("tab_recording_then_pip");
2562 yield* manager.stopRecording("tab_recording_then_pip");2538 yield* manager.stopRecording("tab_recording_then_pip");
...@@ -2658,121 +2634,6 @@ describe("PreviewManager", () => {...@@ -2658,121 +2634,6 @@ describe("PreviewManager", () => {
2658 ),2634 ),
2659 );2635 );
26602636
2661 effectIt.effect("keeps recording cadence when pixels remain unchanged", () =>
2662 withManager((manager) =>
2663 Effect.gen(function* () {
2664 const unchanged = Buffer.from("unchanged-preview-frame");
2665 const capturePage = vi.fn(async () => ({
2666 toJPEG: () => unchanged,
2667 getSize: () => ({ width: 1280, height: 720 }),
2668 }));
2669 fromId.mockReturnValue(makeTestPreviewWebContents(capturePage));
2670 const frames: DesktopPreviewRecordingFrame[] = [];
2671
2672 yield* manager.subscribeRecordingFrames((frame) =>
2673 Effect.sync(() => {
2674 frames.push(frame);
2675 }),
2676 );
2677 yield* manager.createTab("tab_unchanged_frame");
2678 yield* manager.registerWebview("tab_unchanged_frame", 42);
2679 yield* manager.startRecording("tab_unchanged_frame");
2680
2681 expect(frames.map((frame) => frame.data)).toEqual([unchanged.toString("base64")]);
2682
2683 yield* TestClock.adjust(100);
2684
2685 expect(capturePage).toHaveBeenCalledTimes(2);
2686 expect(frames.map((frame) => frame.data)).toEqual([
2687 unchanged.toString("base64"),
2688 unchanged.toString("base64"),
2689 ]);
2690
2691 yield* TestClock.adjust(100);
2692
2693 expect(capturePage).toHaveBeenCalledTimes(3);
2694 expect(frames.map((frame) => frame.data)).toEqual([
2695 unchanged.toString("base64"),
2696 unchanged.toString("base64"),
2697 unchanged.toString("base64"),
2698 ]);
2699 yield* manager.stopRecording("tab_unchanged_frame");
2700 }),
2701 ),
2702 );
2703
2704 effectIt.effect("retries recording delivery when pixels remain unchanged", () =>
2705 withManager((manager) =>
2706 Effect.gen(function* () {
2707 const jpeg = Buffer.from("retry-recording-frame");
2708 const capturePage = vi.fn(async () => ({
2709 toJPEG: () => jpeg,
2710 getSize: () => ({ width: 1280, height: 720 }),
2711 }));
2712 fromId.mockReturnValue(makeTestPreviewWebContents(capturePage));
2713 let deliveries = 0;
2714
2715 yield* manager.subscribeRecordingFrames(() =>
2716 Effect.sync(() => {
2717 deliveries += 1;
2718 if (deliveries === 1) throw new Error("recording delivery failed");
2719 }),
2720 );
2721 yield* manager.createTab("tab_recording_delivery_retry");
2722 yield* manager.registerWebview("tab_recording_delivery_retry", 42);
2723 yield* manager.startRecording("tab_recording_delivery_retry");
2724 expect(deliveries).toBe(1);
2725
2726 yield* TestClock.adjust(100);
2727
2728 expect(deliveries).toBe(2);
2729 yield* manager.stopRecording("tab_recording_delivery_retry");
2730 }),
2731 ),
2732 );
2733
2734 effectIt.effect("retries a cold hidden-tab capture without dropping recording", () =>
2735 withManager((manager) =>
2736 Effect.gen(function* () {
2737 const jpeg = Buffer.from("recovered-preview-frame");
2738 const capturePage = vi.fn(async () => ({
2739 toJPEG: () => jpeg,
2740 getSize: () => ({ width: 1280, height: 720 }),
2741 }));
2742 capturePage.mockRejectedValueOnce(new Error("UnknownVizError"));
2743 fromId.mockReturnValue(makeTestPreviewWebContents(capturePage));
2744 const frames: DesktopPreviewRecordingFrame[] = [];
2745
2746 yield* manager.subscribeRecordingFrames((frame) =>
2747 Effect.sync(() => {
2748 frames.push(frame);
2749 }),
2750 );
2751 yield* manager.createTab("tab_cold_capture");
2752 yield* manager.registerWebview("tab_cold_capture", 42);
2753
2754 yield* manager.startRecording("tab_cold_capture");
2755
2756 expect(capturePage).toHaveBeenCalledOnce();
2757 expect(frames).toHaveLength(0);
2758
2759 yield* TestClock.adjust(100);
2760
2761 expect(capturePage).toHaveBeenCalledTimes(2);
2762 expect(frames).toEqual([
2763 expect.objectContaining({
2764 tabId: "tab_cold_capture",
2765 data: jpeg.toString("base64"),
2766 width: 1280,
2767 height: 720,
2768 }),
2769 ]);
2770
2771 yield* manager.stopRecording("tab_cold_capture");
2772 }),
2773 ),
2774 );
2775
2776 effectIt.effect("drops empty frames before picture-in-picture delivery", () =>2637 effectIt.effect("drops empty frames before picture-in-picture delivery", () =>
2777 withManager((manager) =>2638 withManager((manager) =>
2778 Effect.gen(function* () {2639 Effect.gen(function* () {
apps/desktop/src/preview/Manager.ts+150-62
...@@ -16,6 +16,7 @@ import type {...@@ -16,6 +16,7 @@ import type {
16 PreviewAnnotationSubmissionResult,16 PreviewAnnotationSubmissionResult,
17 DesktopPreviewRecordingArtifact,17 DesktopPreviewRecordingArtifact,
18 DesktopPreviewRecordingFrame,18 DesktopPreviewRecordingFrame,
19 DesktopPreviewRecordingSource,
19 DesktopPreviewScreenshotArtifact,20 DesktopPreviewScreenshotArtifact,
20 DesktopPreviewTabDefaults,21 DesktopPreviewTabDefaults,
21 PreviewAutomationClickInput,22 PreviewAutomationClickInput,
...@@ -108,8 +109,10 @@ const MAX_EVALUATION_BYTES = 64_000;...@@ -108,8 +109,10 @@ const MAX_EVALUATION_BYTES = 64_000;
108const MAX_VISIBLE_TEXT_LENGTH = 20_000;109const MAX_VISIBLE_TEXT_LENGTH = 20_000;
109const MAX_INTERACTIVE_ELEMENTS = 200;110const MAX_INTERACTIVE_ELEMENTS = 200;
110const MAX_SCREENSHOT_WIDTH = 1280;111const MAX_SCREENSHOT_WIDTH = 1280;
111const RECORDING_FRAME_INTERVAL_MS = Math.ceil(1_000 / 12);112const RECORDING_SOURCE_SIZE_EXPRESSION =
112const RECORDING_JPEG_QUALITY = 80;113 "({ width: Math.round(globalThis.innerWidth), height: Math.round(globalThis.innerHeight) })";
114const PICTURE_IN_PICTURE_FRAME_INTERVAL_MS = Math.ceil(1_000 / 12);
115const PICTURE_IN_PICTURE_JPEG_QUALITY = 80;
113const PICTURE_IN_PICTURE_INITIAL_WIDTH = 480;116const PICTURE_IN_PICTURE_INITIAL_WIDTH = 480;
114const PICTURE_IN_PICTURE_INITIAL_HEIGHT = 320;117const PICTURE_IN_PICTURE_INITIAL_HEIGHT = 320;
115const PICTURE_IN_PICTURE_MIN_WIDTH = 240;118const PICTURE_IN_PICTURE_MIN_WIDTH = 240;
...@@ -188,6 +191,12 @@ export const fitPictureInPictureContentSize = (...@@ -188,6 +191,12 @@ export const fitPictureInPictureContentSize = (
188 return [Math.round(width), Math.round(height)];191 return [Math.round(width), Math.round(height)];
189};192};
190193
194export const recordingFileExtension = (mimeType: string): string => {
195 const subtype = mimeType.split(";", 1)[0]?.trim().toLowerCase().split("/")[1] ?? "";
196 const extension = subtype.replace(/^x-/, "").replace(/[^a-z0-9]/g, "");
197 return extension || "video";
198};
199
191const artifactSiteSlug = (rawUrl: string): string => {200const artifactSiteSlug = (rawUrl: string): string => {
192 try {201 try {
193 const url = new URL(rawUrl);202 const url = new URL(rawUrl);
...@@ -379,7 +388,7 @@ interface ManagedListeners {...@@ -379,7 +388,7 @@ interface ManagedListeners {
379type FrameCaptureConsumer = "picture-in-picture" | "recording";388type FrameCaptureConsumer = "picture-in-picture" | "recording";
380389
381interface FrameCaptureSession {390interface FrameCaptureSession {
382 readonly scope: Scope.Closeable;391 readonly scope: Scope.Closeable | null;
383 readonly consumers: ReadonlySet<FrameCaptureConsumer>;392 readonly consumers: ReadonlySet<FrameCaptureConsumer>;
384 readonly lastPictureInPictureFrame: Buffer | null;393 readonly lastPictureInPictureFrame: Buffer | null;
385}394}
...@@ -670,10 +679,11 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function...@@ -670,10 +679,11 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
670 consumers.delete(consumer);679 consumers.delete(consumer);
671 if (consumers.size > 0) {680 if (consumers.size > 0) {
672 return [681 return [
673 undefined,682 consumer === "picture-in-picture" ? current.scope : undefined,
674 replaceMap(sessions, (copy) => {683 replaceMap(sessions, (copy) => {
675 copy.set(tabId, {684 copy.set(tabId, {
676 ...current,685 ...current,
686 scope: consumer === "picture-in-picture" ? null : current.scope,
677 consumers,687 consumers,
678 lastPictureInPictureFrame:688 lastPictureInPictureFrame:
679 consumer === "picture-in-picture" ? null : current.lastPictureInPictureFrame,689 consumer === "picture-in-picture" ? null : current.lastPictureInPictureFrame,
...@@ -2575,7 +2585,8 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function...@@ -2575,7 +2585,8 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
2575 tabId: string,2585 tabId: string,
2576 ) {2586 ) {
2577 const captureSession = (yield* SynchronizedRef.get(frameCaptureSessionsRef)).get(tabId);2587 const captureSession = (yield* SynchronizedRef.get(frameCaptureSessionsRef)).get(tabId);
2578 if (!captureSession) return;2588 if (!captureSession?.consumers.has("picture-in-picture") || captureSession.scope === null)
2589 return;
2579 const wc = yield* requireWebContents(tabId);2590 const wc = yield* requireWebContents(tabId);
2580 const image = yield* attemptPromise(2591 const image = yield* attemptPromise(
2581 {2592 {
...@@ -2621,15 +2632,14 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function...@@ -2621,15 +2632,14 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
2621 tabId,2632 tabId,
2622 webContentsId: wc.id,2633 webContentsId: wc.id,
2623 },2634 },
2624 () => image.toJPEG(RECORDING_JPEG_QUALITY),2635 () => image.toJPEG(PICTURE_IN_PICTURE_JPEG_QUALITY),
2625 );2636 );
2626 const frameSession = (yield* SynchronizedRef.get(frameCaptureSessionsRef)).get(tabId);2637 const frameSession = (yield* SynchronizedRef.get(frameCaptureSessionsRef)).get(tabId);
2627 if (frameSession?.scope !== captureSession.scope) return;2638 if (frameSession?.scope !== captureSession.scope) return;
2628 const recording = frameSession.consumers.has("recording");
2629 const pictureInPicture =2639 const pictureInPicture =
2630 frameSession.consumers.has("picture-in-picture") &&2640 frameSession.consumers.has("picture-in-picture") &&
2631 frameSession.lastPictureInPictureFrame?.equals(encoded) !== true;2641 frameSession.lastPictureInPictureFrame?.equals(encoded) !== true;
2632 if (!recording && !pictureInPicture) return;2642 if (!pictureInPicture) return;
2633 const receivedAt = yield* currentIso;2643 const receivedAt = yield* currentIso;
2634 const frame: DesktopPreviewRecordingFrame = {2644 const frame: DesktopPreviewRecordingFrame = {
2635 tabId,2645 tabId,
...@@ -2639,16 +2649,6 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function...@@ -2639,16 +2649,6 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
2639 receivedAt,2649 receivedAt,
2640 };2650 };
2641 const deliveries: Array<Effect.Effect<void>> = [];2651 const deliveries: Array<Effect.Effect<void>> = [];
2642 if (recording) {
2643 const listeners = yield* Ref.get(recordingFrameListenersRef);
2644 deliveries.push(
2645 Effect.forEach(
2646 listeners,
2647 (listener) => deliverEvent("recording-frame", frame.tabId, () => listener(frame)),
2648 { discard: true },
2649 ),
2650 );
2651 }
2652 if (pictureInPicture) {2652 if (pictureInPicture) {
2653 const pictureInPictureWindow = (yield* SynchronizedRef.get(pictureInPictureSessionsRef)).get(2653 const pictureInPictureWindow = (yield* SynchronizedRef.get(pictureInPictureSessionsRef)).get(
2654 tabId,2654 tabId,
...@@ -2726,12 +2726,10 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function...@@ -2726,12 +2726,10 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
2726 tabId: string,2726 tabId: string,
2727 consumer: FrameCaptureConsumer,2727 consumer: FrameCaptureConsumer,
2728 ) {2728 ) {
2729 // Validate the tab synchronously, but treat capturePage failures as2729 // Recording keeps only the activity lease. Picture-in-picture owns the
2730 // transient. Chromium can return UnknownVizError while a hidden guest is2730 // capturePage loop and tolerates transient compositor warmup failures.
2731 // warming its first compositor frame; the scheduled loop should keep the
2732 // consumer alive and recover instead of tearing recording/PiP back down.
2733 yield* requireWebContents(tabId);2731 yield* requireWebContents(tabId);
2734 const captureNextFrame = Effect.sleep(RECORDING_FRAME_INTERVAL_MS).pipe(2732 const captureNextFrame = Effect.sleep(PICTURE_IN_PICTURE_FRAME_INTERVAL_MS).pipe(
2735 Effect.andThen(capturePreviewFrame(tabId)),2733 Effect.andThen(capturePreviewFrame(tabId)),
2736 Effect.catch((error) =>2734 Effect.catch((error) =>
2737 Effect.logWarning("Background preview frame capture failed.", {2735 Effect.logWarning("Background preview frame capture failed.", {
...@@ -2740,48 +2738,60 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function...@@ -2740,48 +2738,60 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
2740 }),2738 }),
2741 ),2739 ),
2742 );2740 );
2743 const created = yield* SynchronizedRef.modifyEffect(frameCaptureSessionsRef, (sessions) => {2741 const captureInitialFrame = yield* SynchronizedRef.modifyEffect(
2744 return Effect.gen(function* () {2742 frameCaptureSessionsRef,
2745 if (!frameCaptureWindowOpen) {2743 (sessions) => {
2746 return yield* new PreviewMainWindowClosedError({ tabId });2744 return Effect.gen(function* () {
2747 }2745 if (!frameCaptureWindowOpen) {
2748 const tab = (yield* SynchronizedRef.get(tabsRef)).get(tabId);2746 return yield* new PreviewMainWindowClosedError({ tabId });
2749 if (!tab || (yield* Ref.get(closingTabIdsRef)).has(tabId)) {2747 }
2750 return yield* new PreviewTabNotFoundError({ tabId });2748 const tab = (yield* SynchronizedRef.get(tabsRef)).get(tabId);
2751 }2749 if (!tab || (yield* Ref.get(closingTabIdsRef)).has(tabId)) {
2752 const current = sessions.get(tabId);2750 return yield* new PreviewTabNotFoundError({ tabId });
2753 if (current) {2751 }
2754 if (current.consumers.has(consumer)) {2752 const current = sessions.get(tabId);
2755 return [false, sessions] as const;2753 if (current) {
2754 if (current.consumers.has(consumer)) {
2755 return [false, sessions] as const;
2756 }
2757 let scope = current.scope;
2758 if (consumer === "picture-in-picture" && scope === null) {
2759 scope = yield* Scope.fork(parentScope, "sequential");
2760 yield* Effect.forkIn(Effect.forever(captureNextFrame), scope);
2761 }
2762 return [
2763 consumer === "picture-in-picture",
2764 replaceMap(sessions, (copy) => {
2765 copy.set(tabId, {
2766 ...current,
2767 scope,
2768 consumers: new Set([...current.consumers, consumer]),
2769 });
2770 }),
2771 ] as const;
2772 }
2773 if (sessions.size === 0) {
2774 yield* setFrameCaptureBackgroundThrottling(false);
2775 }
2776 const scope =
2777 consumer === "picture-in-picture" ? yield* Scope.fork(parentScope, "sequential") : null;
2778 if (scope !== null) {
2779 yield* Effect.forkIn(Effect.forever(captureNextFrame), scope);
2756 }2780 }
2757 return [2781 return [
2758 false,2782 consumer === "picture-in-picture",
2759 replaceMap(sessions, (copy) => {2783 replaceMap(sessions, (copy) => {
2760 copy.set(tabId, {2784 copy.set(tabId, {
2761 ...current,2785 scope,
2762 consumers: new Set([...current.consumers, consumer]),2786 consumers: new Set([consumer]),
2787 lastPictureInPictureFrame: null,
2763 });2788 });
2764 }),2789 }),
2765 ] as const;2790 ] as const;
2766 }2791 });
2767 if (sessions.size === 0) {2792 },
2768 yield* setFrameCaptureBackgroundThrottling(false);2793 ).pipe(Effect.uninterruptible);
2769 }2794 if (!captureInitialFrame) return;
2770 const scope = yield* Scope.fork(parentScope, "sequential");
2771 yield* Effect.forkIn(Effect.forever(captureNextFrame), scope);
2772 return [
2773 true,
2774 replaceMap(sessions, (copy) => {
2775 copy.set(tabId, {
2776 scope,
2777 consumers: new Set([consumer]),
2778 lastPictureInPictureFrame: null,
2779 });
2780 }),
2781 ] as const;
2782 });
2783 }).pipe(Effect.uninterruptible);
2784 if (!created) return;
2785 yield* capturePreviewFrame(tabId).pipe(2795 yield* capturePreviewFrame(tabId).pipe(
2786 Effect.catch((error) =>2796 Effect.catch((error) =>
2787 Effect.logWarning("Initial background preview frame was not ready; capture will retry.", {2797 Effect.logWarning("Initial background preview frame was not ready; capture will retry.", {
...@@ -3090,11 +3100,77 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function...@@ -3090,11 +3100,77 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
3090 });3100 });
30913101
3092 const startRecording = Effect.fn("PreviewManager.startRecording")(function* (tabId: string) {3102 const startRecording = Effect.fn("PreviewManager.startRecording")(function* (tabId: string) {
3093 yield* startFrameCapture(tabId, "recording");3103 if ((yield* Ref.get(closingTabIdsRef)).has(tabId)) {
3104 return yield* new PreviewTabNotFoundError({ tabId });
3105 }
3106 return yield* withTabLifecycleLock(
3107 tabId,
3108 Effect.gen(function* () {
3109 yield* startFrameCapture(tabId, "recording");
3110 const wc = yield* requireWebContents(tabId);
3111 const requestWebContents = wc.hostWebContents;
3112 if (requestWebContents === null) {
3113 return yield* new PreviewMainWindowClosedError({ tabId });
3114 }
3115 const measuredSize = yield* attemptPromise(
3116 {
3117 operation: "recording.measureSource",
3118 tabId,
3119 webContentsId: wc.id,
3120 },
3121 () => wc.executeJavaScript(RECORDING_SOURCE_SIZE_EXPRESSION, true),
3122 );
3123 if (
3124 typeof measuredSize !== "object" ||
3125 measuredSize === null ||
3126 !("width" in measuredSize) ||
3127 !("height" in measuredSize) ||
3128 typeof measuredSize.width !== "number" ||
3129 typeof measuredSize.height !== "number" ||
3130 !Number.isInteger(measuredSize.width) ||
3131 !Number.isInteger(measuredSize.height) ||
3132 measuredSize.width <= 0 ||
3133 measuredSize.height <= 0
3134 ) {
3135 return yield* new PreviewRecordingSourceSizeUnavailableError({
3136 tabId,
3137 webContentsId: wc.id,
3138 });
3139 }
3140 yield* attemptPromise(
3141 {
3142 operation: "recording.warmSource",
3143 tabId,
3144 webContentsId: wc.id,
3145 },
3146 () => wc.capturePage().then(() => undefined),
3147 ).pipe(Effect.retry({ times: 1 }), Effect.ignore);
3148 const currentWebContents = yield* requireWebContents(tabId);
3149 if (currentWebContents !== wc || wc.isDestroyed()) {
3150 return yield* new PreviewWebContentsNotFoundError({
3151 tabId,
3152 webContentsId: wc.id,
3153 });
3154 }
3155 const sourceId = yield* attempt(
3156 {
3157 operation: "recording.getMediaSourceId",
3158 tabId,
3159 webContentsId: wc.id,
3160 },
3161 () => wc.getMediaSourceId(requestWebContents),
3162 );
3163 return {
3164 sourceId,
3165 width: measuredSize.width,
3166 height: measuredSize.height,
3167 } satisfies DesktopPreviewRecordingSource;
3168 }).pipe(Effect.onError(() => stopFrameCapture(tabId, "recording").pipe(Effect.ignore))),
3169 );
3094 });3170 });
30953171
3096 const stopRecording = Effect.fn("PreviewManager.stopRecording")(function* (tabId: string) {3172 const stopRecording = Effect.fn("PreviewManager.stopRecording")(function* (tabId: string) {
3097 yield* stopFrameCapture(tabId, "recording");3173 yield* withTabLifecycleLock(tabId, stopFrameCapture(tabId, "recording"));
3098 });3174 });
30993175
3100 const saveRecording = Effect.fn("PreviewManager.saveRecording")(function* (3176 const saveRecording = Effect.fn("PreviewManager.saveRecording")(function* (
...@@ -3104,7 +3180,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function...@@ -3104,7 +3180,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
3104 ) {3180 ) {
3105 const [createdAt, millis] = yield* Effect.all([currentIso, currentMillis]);3181 const [createdAt, millis] = yield* Effect.all([currentIso, currentMillis]);
3106 const id = `browser-recording-${millis.toString(36)}`;3182 const id = `browser-recording-${millis.toString(36)}`;
3107 const extension = mimeType.includes("mp4") ? "mp4" : "webm";3183 const extension = recordingFileExtension(mimeType);
3108 const artifactPath = path.join(resolvedArtifactDirectory, `${id}.${extension}`);3184 const artifactPath = path.join(resolvedArtifactDirectory, `${id}.${extension}`);
3109 yield* fileSystem.makeDirectory(resolvedArtifactDirectory, { recursive: true }).pipe(3185 yield* fileSystem.makeDirectory(resolvedArtifactDirectory, { recursive: true }).pipe(
3110 Effect.mapError(3186 Effect.mapError(
...@@ -3904,6 +3980,15 @@ export class PreviewMainWindowClosedError extends Schema.TaggedErrorClass<Previe...@@ -3904,6 +3980,15 @@ export class PreviewMainWindowClosedError extends Schema.TaggedErrorClass<Previe
3904 }3980 }
3905}3981}
39063982
3983export class PreviewRecordingSourceSizeUnavailableError extends Schema.TaggedErrorClass<PreviewRecordingSourceSizeUnavailableError>()(
3984 "PreviewRecordingSourceSizeUnavailableError",
3985 { tabId: Schema.String, webContentsId: Schema.Number },
3986) {
3987 override get message(): string {
3988 return `Preview media source dimensions are unavailable for tab ${this.tabId}`;
3989 }
3990}
3991
3907export class PreviewOperationError extends Schema.TaggedErrorClass<PreviewOperationError>()(3992export class PreviewOperationError extends Schema.TaggedErrorClass<PreviewOperationError>()(
3908 "PreviewOperationError",3993 "PreviewOperationError",
3909 {3994 {
...@@ -4111,6 +4196,7 @@ export const PreviewManagerError = Schema.Union([...@@ -4111,6 +4196,7 @@ export const PreviewManagerError = Schema.Union([
4111 PreviewWebContentsNotFoundError,4196 PreviewWebContentsNotFoundError,
4112 PreviewWebviewNotInitializedError,4197 PreviewWebviewNotInitializedError,
4113 PreviewMainWindowClosedError,4198 PreviewMainWindowClosedError,
4199 PreviewRecordingSourceSizeUnavailableError,
4114 PreviewOperationError,4200 PreviewOperationError,
4115 PreviewArtifactPathOutsideDirectoryError,4201 PreviewArtifactPathOutsideDirectoryError,
4116 PreviewArtifactImageLoadError,4202 PreviewArtifactImageLoadError,
...@@ -4188,7 +4274,9 @@ export class PreviewManager extends Context.Service<...@@ -4188,7 +4274,9 @@ export class PreviewManager extends Context.Service<
4188 readonly copyArtifactToClipboard: (path: string) => Effect.Effect<void, PreviewManagerError>;4274 readonly copyArtifactToClipboard: (path: string) => Effect.Effect<void, PreviewManagerError>;
4189 readonly openPictureInPicture: (tabId: string) => Effect.Effect<void, PreviewManagerError>;4275 readonly openPictureInPicture: (tabId: string) => Effect.Effect<void, PreviewManagerError>;
4190 readonly closePictureInPicture: (tabId: string) => Effect.Effect<void, PreviewManagerError>;4276 readonly closePictureInPicture: (tabId: string) => Effect.Effect<void, PreviewManagerError>;
4191 readonly startRecording: (tabId: string) => Effect.Effect<void, PreviewManagerError>;4277 readonly startRecording: (
4278 tabId: string,
4279 ) => Effect.Effect<DesktopPreviewRecordingSource, PreviewManagerError>;
4192 readonly stopRecording: (tabId: string) => Effect.Effect<void, PreviewManagerError>;4280 readonly stopRecording: (tabId: string) => Effect.Effect<void, PreviewManagerError>;
4193 readonly saveRecording: (4281 readonly saveRecording: (
4194 tabId: string,4282 tabId: string,
apps/desktop/src/settings/DesktopClientSettings.test.ts+1
...@@ -17,6 +17,7 @@ const clientSettings: ClientSettings = {...@@ -17,6 +17,7 @@ const clientSettings: ClientSettings = {
17 browserDefaultViewport: { _tag: "preset", width: 1024, height: 600, presetId: "nest-hub" },17 browserDefaultViewport: { _tag: "preset", width: 1024, height: 600, presetId: "nest-hub" },
18 browserDefaultZoomFactor: 1.25,18 browserDefaultZoomFactor: 1.25,
19 browserDefaultAppearance: "dark",19 browserDefaultAppearance: "dark",
20 browserRecordingFrameRate: 60,
20 browserAutoShowFloatingPreview: false,21 browserAutoShowFloatingPreview: false,
21 confirmQuit: true,22 confirmQuit: true,
22 confirmThreadArchive: true,23 confirmThreadArchive: true,
apps/web/src/browser/browserRecording.test.ts+228-260
...@@ -1,77 +1,37 @@...@@ -1,77 +1,37 @@
1import { EnvironmentId, ThreadId } from "@t3tools/contracts";1import { EnvironmentId, ThreadId } from "@t3tools/contracts";
2import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test";2import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test";
33
4const {4const { clientSettings, events, getUserMedia, registrySet, save, startScreencast, stopScreencast } =
5 events,5 vi.hoisted(() => {
6 frameSubscription,6 const events: string[] = [];
7 onFrame,7 return {
8 registrySet,8 clientSettings: { browserRecordingFrameRate: 30 as 30 | 60 },
9 save,9 events,
10 startScreencast,10 getUserMedia: vi.fn(),
11 stopScreencast,11 registrySet: vi.fn((_atom: unknown, value: { readonly tabIds: ReadonlySet<string> }) => {
12 surfaceState,12 events.push(
13} = vi.hoisted(() => {13 value.tabIds.size === 0 ? "clear" : `publish:${Array.from(value.tabIds).join(",")}`,
14 const events: string[] = [];14 );
15 type Frame = {15 }),
16 readonly tabId: string;16 save: vi.fn(async (tabId: string) => ({
17 readonly data: string;17 id: "recording-test",
18 readonly width: number;
19 readonly height: number;
20 readonly receivedAt: string;
21 };
22 const frameSubscription: { listener: ((frame: Frame) => void) | null } = {
23 listener: null,
24 };
25 const surfaceState = {
26 byTabId: {} as Record<string, unknown>,
27 };
28 return {
29 events,
30 frameSubscription,
31 onFrame: vi.fn((listener: (frame: Frame) => void) => {
32 frameSubscription.listener = listener;
33 return () => {
34 if (frameSubscription.listener === listener) frameSubscription.listener = null;
35 };
36 }),
37 registrySet: vi.fn((_atom: unknown, value: { readonly tabIds: ReadonlySet<string> }) => {
38 events.push(
39 value.tabIds.size === 0 ? "clear" : `publish:${Array.from(value.tabIds).join(",")}`,
40 );
41 }),
42 save: vi.fn(async (tabId: string) => ({
43 id: "recording-test",
44 tabId,
45 path: "/tmp/recording-test.webm",
46 mimeType: "video/webm" as const,
47 sizeBytes: 0,
48 createdAt: "2026-06-26T00:00:00.000Z",
49 })),
50 startScreencast: vi.fn(async (tabId: string) => {
51 events.push("start-screencast");
52 const surface = surfaceState.byTabId[tabId] as
53 | {
54 readonly content?: { readonly width: number; readonly height: number };
55 readonly rect?: { readonly width: number; readonly height: number };
56 }
57 | undefined;
58 const size = surface?.content ?? surface?.rect;
59 frameSubscription.listener?.({
60 tabId,18 tabId,
61 data: "initial-frame",19 path: "/tmp/recording-test.webm",
62 width: size?.width ?? 1280,20 mimeType: "video/webm" as const,
63 height: size?.height ?? 800,21 sizeBytes: 0,
64 receivedAt: "2026-06-26T00:00:00.000Z",22 createdAt: "2026-06-26T00:00:00.000Z",
65 });23 })),
66 }),24 startScreencast: vi.fn(async (tabId: string) => {
67 stopScreencast: vi.fn(async () => undefined),25 events.push("start-screencast");
68 surfaceState,26 return { sourceId: `source:${tabId}`, width: 1000, height: 620 };
69 };27 }),
70});28 stopScreencast: vi.fn(async () => undefined),
29 };
30 });
7131
72vi.mock("~/components/preview/previewBridge", () => ({32vi.mock("~/components/preview/previewBridge", () => ({
73 previewBridge: {33 previewBridge: {
74 recording: { onFrame, save, startScreencast, stopScreencast },34 recording: { onFrame: vi.fn(), save, startScreencast, stopScreencast },
75 },35 },
76}));36}));
7737
...@@ -79,32 +39,49 @@ vi.mock("~/rpc/atomRegistry", () => ({...@@ -79,32 +39,49 @@ vi.mock("~/rpc/atomRegistry", () => ({
79 appAtomRegistry: { set: registrySet },39 appAtomRegistry: { set: registrySet },
80}));40}));
8141
82vi.mock("./browserSurfaceStore", () => ({42vi.mock("~/hooks/useSettings", () => ({
83 useBrowserSurfaceStore: {43 ensureClientSettingsHydrated: vi.fn(async () => undefined),
84 getState: () => surfaceState,44 getClientSettings: () => clientSettings,
85 },
86}));45}));
8746
88import {47import {
89 BROWSER_RECORDING_FIRST_FRAME_SIZE_TIMEOUT_MS,48 BROWSER_RECORDING_PAINT_SETTLE_TIMEOUT_MS,
90 BROWSER_RECORDING_STARTUP_SETTLE_TIMEOUT_MS,49 BROWSER_RECORDING_STARTUP_SETTLE_TIMEOUT_MS,
50 BrowserRecordingCaptureTimeoutError,
91 BrowserRecordingConflictError,51 BrowserRecordingConflictError,
52 BrowserRecordingFormatUnavailableError,
92 findActiveBrowserRecordingRuntimeTabId,53 findActiveBrowserRecordingRuntimeTabId,
93 readActiveBrowserRecordingTabIds,54 readActiveBrowserRecordingTabIds,
94 readActiveBrowserRecordingTargets,55 readActiveBrowserRecordingTargets,
95 startBrowserRecording,56 startBrowserRecording,
96 stopBrowserRecording,57 stopBrowserRecording,
97} from "./browserRecording";58} from "./browserRecording";
59import { useBrowserSurfaceStore } from "./browserSurfaceStore";
98import { previewRuntimeTabId } from "./previewRuntimeTabId";60import { previewRuntimeTabId } from "./previewRuntimeTabId";
9961
100class FakeMediaRecorder {62class FakeMediaRecorder {
101 static isTypeSupported(): boolean {63 static readonly instances: FakeMediaRecorder[] = [];
102 return true;64 static supportedTypes = new Set(["video/webm;codecs=vp9"]);
65 static outputMimeType: string | undefined;
66 static stopError: unknown;
67 static isTypeSupported(type: string): boolean {
68 return this.supportedTypes.has(type);
103 }69 }
10470
105 state: RecordingState = "inactive";71 state: RecordingState = "inactive";
72 readonly mimeType: string;
73 readonly stream: MediaStream;
74 readonly options: MediaRecorderOptions | undefined;
106 private readonly listeners = new Map<string, Set<EventListenerOrEventListenerObject>>();75 private readonly listeners = new Map<string, Set<EventListenerOrEventListenerObject>>();
10776
77 constructor(stream: MediaStream, options?: MediaRecorderOptions) {
78 this.stream = stream;
79 this.options = options;
80 this.mimeType =
81 FakeMediaRecorder.outputMimeType ?? options?.mimeType ?? "video/browser-default";
82 FakeMediaRecorder.instances.push(this);
83 }
84
108 addEventListener(type: string, listener: EventListenerOrEventListenerObject): void {85 addEventListener(type: string, listener: EventListenerOrEventListenerObject): void {
109 const listeners = this.listeners.get(type) ?? new Set();86 const listeners = this.listeners.get(type) ?? new Set();
110 listeners.add(listener);87 listeners.add(listener);
...@@ -116,6 +93,7 @@ class FakeMediaRecorder {...@@ -116,6 +93,7 @@ class FakeMediaRecorder {
116 }93 }
11794
118 stop(): void {95 stop(): void {
96 if (FakeMediaRecorder.stopError !== undefined) throw FakeMediaRecorder.stopError;
119 this.state = "inactive";97 this.state = "inactive";
120 for (const listener of this.listeners.get("stop") ?? []) {98 for (const listener of this.listeners.get("stop") ?? []) {
121 if (typeof listener === "function") listener(new Event("stop"));99 if (typeof listener === "function") listener(new Event("stop"));
...@@ -124,52 +102,31 @@ class FakeMediaRecorder {...@@ -124,52 +102,31 @@ class FakeMediaRecorder {
124 }102 }
125}103}
126104
127const emitRecordingFrame = () => {
128 frameSubscription.listener?.({
129 tabId: "recording-tab",
130 data: "startup-frame",
131 width: 800,
132 height: 600,
133 receivedAt: "2026-06-26T00:00:00.000Z",
134 });
135};
136
137describe("browser recording", () => {105describe("browser recording", () => {
106 let animationFrameCount = 0;
107
138 beforeEach(() => {108 beforeEach(() => {
139 events.length = 0;109 events.length = 0;
140 frameSubscription.listener = null;
141 surfaceState.byTabId = {
142 "recording-tab": {
143 visible: true,
144 rect: { x: 0, y: 0, width: 800, height: 600 },
145 content: { x: 0, y: 0, width: 800, height: 600, scale: 1, scrollLeft: 0, scrollTop: 0 },
146 },
147 };
148 vi.clearAllMocks();110 vi.clearAllMocks();
111 FakeMediaRecorder.instances.length = 0;
112 FakeMediaRecorder.supportedTypes = new Set(["video/webm;codecs=vp9"]);
113 FakeMediaRecorder.outputMimeType = undefined;
114 FakeMediaRecorder.stopError = undefined;
115 clientSettings.browserRecordingFrameRate = 30;
116 animationFrameCount = 0;
149 vi.stubGlobal("window", globalThis);117 vi.stubGlobal("window", globalThis);
118 vi.stubGlobal("requestAnimationFrame", (callback: FrameRequestCallback) => {
119 animationFrameCount += 1;
120 callback(animationFrameCount);
121 return animationFrameCount;
122 });
123 vi.stubGlobal("cancelAnimationFrame", vi.fn());
150 vi.stubGlobal("MediaRecorder", FakeMediaRecorder as unknown as typeof MediaRecorder);124 vi.stubGlobal("MediaRecorder", FakeMediaRecorder as unknown as typeof MediaRecorder);
151 class ImmediateImage {125 getUserMedia.mockResolvedValue({
152 private loadListener: EventListenerOrEventListenerObject | undefined;126 getTracks: () => [{ stop: vi.fn() }],
153
154 addEventListener(type: string, listener: EventListenerOrEventListenerObject): void {
155 if (type === "load") this.loadListener = listener;
156 }
157
158 set src(_value: string) {
159 const event = new Event("load");
160 if (typeof this.loadListener === "function") this.loadListener(event);
161 else this.loadListener?.handleEvent(event);
162 }
163 }
164 vi.stubGlobal("Image", ImmediateImage as unknown as typeof Image);
165 vi.stubGlobal("document", {
166 createElement: () => ({
167 width: 0,
168 height: 0,
169 captureStream: () => ({}),
170 getContext: () => ({ drawImage: vi.fn(), fillRect: vi.fn(), fillStyle: "" }),
171 }),
172 });127 });
128 vi.stubGlobal("navigator", { mediaDevices: { getUserMedia } });
129 useBrowserSurfaceStore.setState({ activityByTabId: {}, byTabId: {} });
173 });130 });
174131
175 afterEach(() => {132 afterEach(() => {
...@@ -185,146 +142,189 @@ describe("browser recording", () => {...@@ -185,146 +142,189 @@ describe("browser recording", () => {
185 expect(startupEvents).toEqual(["publish:recording-tab", "start-screencast"]);142 expect(startupEvents).toEqual(["publish:recording-tab", "start-screencast"]);
186 });143 });
187144
188 it("records a hidden tab without requiring it to become visible", async () => {145 it("paints and holds a hidden browser surface for the recording lifetime", async () => {
189 surfaceState.byTabId = {146 startScreencast.mockImplementationOnce(async (tabId: string) => {
190 "recording-tab": {147 expect(animationFrameCount).toBe(2);
191 visible: false,148 expect(useBrowserSurfaceStore.getState().activityByTabId[tabId]).toBe(1);
192 rect: { x: 0, y: 0, width: 800, height: 600 },149 return { sourceId: `source:${tabId}`, width: 1000, height: 620 };
193 content: { x: 0, y: 0, width: 800, height: 600, scale: 1, scrollLeft: 0, scrollTop: 0 },150 });
194 },151 getUserMedia.mockImplementationOnce(async () => {
195 };152 expect(animationFrameCount).toBe(2);
153 expect(useBrowserSurfaceStore.getState().activityByTabId["background-tab"]).toBe(1);
154 return { getTracks: () => [{ stop: vi.fn() }] };
155 });
196156
197 await startBrowserRecording("recording-tab");157 await startBrowserRecording("background-tab");
198 const startupEvents = [...events];158 expect(useBrowserSurfaceStore.getState().activityByTabId["background-tab"]).toBe(1);
199159
200 expect(startScreencast).toHaveBeenCalledWith("recording-tab");160 await stopBrowserRecording("background-tab");
201 await stopBrowserRecording("recording-tab");161 expect(useBrowserSurfaceStore.getState().activityByTabId["background-tab"]).toBeUndefined();
202 expect(startupEvents).toEqual(["publish:recording-tab", "start-screencast"]);
203 });162 });
204163
205 it("fails startup instead of locking a fallback size when no frame arrives", async () => {164 it("bounds compositor warmup when animation frames are paused", async () => {
206 vi.useFakeTimers();165 vi.useFakeTimers();
207 startScreencast.mockImplementationOnce(async () => {166 const cancelAnimationFrame = vi.fn();
208 events.push("start-screencast");167 vi.stubGlobal(
209 });168 "requestAnimationFrame",
169 vi.fn(() => 42),
170 );
171 vi.stubGlobal("cancelAnimationFrame", cancelAnimationFrame);
210172
211 const startPromise = startBrowserRecording("recording-tab");173 const startPromise = startBrowserRecording("hidden-window-tab");
212 const rejection = expect(startPromise).rejects.toMatchObject({174 await vi.advanceTimersByTimeAsync(BROWSER_RECORDING_PAINT_SETTLE_TIMEOUT_MS);
213 operation: "wait-first-frame",175
214 tabId: "recording-tab",176 await startPromise;
177 expect(cancelAnimationFrame).toHaveBeenCalledWith(42);
178 await stopBrowserRecording("hidden-window-tab");
179 });
180
181 it("records the native tab stream at the preview's current dimensions", async () => {
182 const stopTrack = vi.fn();
183 const stream = { getTracks: () => [{ stop: stopTrack }] } as unknown as MediaStream;
184 getUserMedia.mockResolvedValueOnce(stream);
185
186 await startBrowserRecording("recording-tab");
187
188 expect(getUserMedia).toHaveBeenCalledWith({
189 audio: false,
190 video: {
191 mandatory: {
192 chromeMediaSource: "tab",
193 chromeMediaSourceId: "source:recording-tab",
194 minWidth: 1000,
195 maxWidth: 1000,
196 minHeight: 620,
197 maxHeight: 620,
198 maxFrameRate: 30,
199 },
200 },
215 });201 });
216 await Promise.resolve();202 expect(FakeMediaRecorder.instances[0]?.stream).toBe(stream);
217 await vi.advanceTimersByTimeAsync(BROWSER_RECORDING_FIRST_FRAME_SIZE_TIMEOUT_MS);
218203
219 await rejection;204 await stopBrowserRecording("recording-tab");
220 expect(stopScreencast).toHaveBeenCalledWith("recording-tab");205 expect(stopTrack).toHaveBeenCalledOnce();
221 expect(events.at(-1)).toBe("clear");
222 });206 });
223207
224 it("fixes hidden recording dimensions before MediaRecorder starts", async () => {208 it("uses the configured recording frame rate", async () => {
225 const drawImage = vi.fn();209 clientSettings.browserRecordingFrameRate = 60;
226 const fillRect = vi.fn();210
227 let capturedStreamSize: { readonly width: number; readonly height: number } | undefined;211 await startBrowserRecording("recording-tab");
228 const canvas = {212
229 width: 0,213 expect(getUserMedia).toHaveBeenCalledWith({
230 height: 0,214 audio: false,
231 captureStream: () => {215 video: {
232 capturedStreamSize = { width: canvas.width, height: canvas.height };216 mandatory: expect.objectContaining({ maxFrameRate: 60 }),
233 return {};
234 },217 },
235 getContext: () => ({ drawImage, fillRect, fillStyle: "" }),
236 };
237 vi.stubGlobal("document", {
238 createElement: () => canvas,
239 });218 });
240 surfaceState.byTabId = {};219 await stopBrowserRecording("recording-tab");
241 startScreencast.mockImplementationOnce(async (tabId: string) => {220 });
242 events.push("start-screencast");221
243 frameSubscription.listener?.({222 it("stops the native stream when MediaRecorder cleanup fails", async () => {
244 tabId,223 const stopTrack = vi.fn();
245 data: "captured-frame",224 getUserMedia.mockResolvedValueOnce({
246 width: 390,225 getTracks: () => [{ stop: stopTrack }],
247 height: 844,
248 receivedAt: "2026-06-26T00:00:00.000Z",
249 });
250 });226 });
251227
252 await startBrowserRecording("recording-tab");228 await startBrowserRecording("recording-tab");
229 FakeMediaRecorder.stopError = new Error("stop failed");
253230
254 expect(canvas).toMatchObject({ width: 390, height: 844 });231 await expect(stopBrowserRecording("recording-tab")).rejects.toMatchObject({
255 expect(capturedStreamSize).toEqual({ width: 390, height: 844 });232 operation: "cleanup",
256 expect(drawImage).toHaveBeenCalledWith(expect.anything(), 0, 0, 390, 844);
257
258 frameSubscription.listener?.({
259 tabId: "recording-tab",233 tabId: "recording-tab",
260 data: "different-sized-frame",
261 width: 1280,
262 height: 720,
263 receivedAt: "2026-06-26T00:00:01.000Z",
264 });234 });
235 expect(stopTrack).toHaveBeenCalledOnce();
236 });
265237
266 expect(canvas).toMatchObject({ width: 390, height: 844 });238 it("uses the best supported encoder and saves the recorder's actual format", async () => {
267 expect(fillRect).toHaveBeenLastCalledWith(0, 0, 390, 844);239 FakeMediaRecorder.supportedTypes = new Set([
240 "video/mp4;codecs=avc1.42e01e",
241 "video/webm;codecs=vp9",
242 "video/webm;codecs=av1",
243 ]);
244 FakeMediaRecorder.outputMimeType = "video/webm;codecs=av01";
268245
246 await startBrowserRecording("recording-tab");
269 await stopBrowserRecording("recording-tab");247 await stopBrowserRecording("recording-tab");
270 });
271248
272 it("draws the newest decoded frames without starving behind decode latency", async () => {249 expect(FakeMediaRecorder.instances[0]?.options).toEqual({
273 const drawImage = vi.fn();250 mimeType: "video/webm;codecs=av1",
274 class DeferredImage {251 });
275 static readonly instances: DeferredImage[] = [];252 expect(save).toHaveBeenCalledWith(
276 private loadListener: EventListenerOrEventListenerObject | undefined;253 "recording-tab",
254 "video/webm;codecs=av01",
255 expect.any(Uint8Array),
256 );
257 });
277258
278 constructor() {259 it("lets the browser select the format when no preferred encoding is supported", async () => {
279 DeferredImage.instances.push(this);260 FakeMediaRecorder.supportedTypes = new Set();
280 }261 FakeMediaRecorder.outputMimeType = "video/platform-default";
281262
282 addEventListener(type: string, listener: EventListenerOrEventListenerObject): void {263 await startBrowserRecording("recording-tab");
283 if (type === "load") this.loadListener = listener;264 await stopBrowserRecording("recording-tab");
284 }
285265
286 set src(_value: string) {}266 expect(FakeMediaRecorder.instances[0]?.options).toBeUndefined();
267 expect(save).toHaveBeenCalledWith(
268 "recording-tab",
269 "video/platform-default",
270 expect.any(Uint8Array),
271 );
272 });
287273
288 finishLoading(): void {274 it("reports when MediaRecorder provides no output format", async () => {
289 const event = new Event("load");275 FakeMediaRecorder.supportedTypes = new Set();
290 if (typeof this.loadListener === "function") this.loadListener(event);276 FakeMediaRecorder.outputMimeType = "";
291 else this.loadListener?.handleEvent(event);
292 }
293 }
294 vi.stubGlobal("Image", DeferredImage as unknown as typeof Image);
295 vi.stubGlobal("document", {
296 createElement: () => ({
297 width: 0,
298 height: 0,
299 captureStream: () => ({}),
300 getContext: () => ({ drawImage, fillRect: vi.fn(), fillStyle: "" }),
301 }),
302 });
303277
304 await startBrowserRecording("recording-tab");278 await startBrowserRecording("recording-tab");
305 frameSubscription.listener?.({279
280 await expect(stopBrowserRecording("recording-tab")).rejects.toBeInstanceOf(
281 BrowserRecordingFormatUnavailableError,
282 );
283 expect(save).not.toHaveBeenCalled();
284 expect(readActiveBrowserRecordingTabIds()).toEqual(new Set());
285 });
286
287 it("releases the native capture lease when stream acquisition fails", async () => {
288 getUserMedia.mockRejectedValueOnce(new Error("capture failed"));
289
290 await expect(startBrowserRecording("recording-tab")).rejects.toMatchObject({
291 operation: "capture-media-stream",
306 tabId: "recording-tab",292 tabId: "recording-tab",
307 data: "second-frame",
308 width: 800,
309 height: 600,
310 receivedAt: "2026-06-26T00:00:01.000Z",
311 });293 });
312 frameSubscription.listener?.({294
295 expect(stopScreencast).toHaveBeenCalledWith("recording-tab");
296 expect(events.at(-1)).toBe("clear");
297 });
298
299 it("times out stalled stream acquisition and stops a late stream", async () => {
300 vi.useFakeTimers();
301 let finishCapture!: (stream: MediaStream) => void;
302 const stopTrack = vi.fn();
303 getUserMedia.mockImplementationOnce(
304 () =>
305 new Promise<MediaStream>((resolve) => {
306 finishCapture = resolve;
307 }),
308 );
309
310 const startPromise = startBrowserRecording("recording-tab");
311 await vi.waitFor(() => expect(getUserMedia).toHaveBeenCalledOnce());
312 const rejection = expect(startPromise).rejects.toMatchObject({
313 _tag: "BrowserRecordingCaptureTimeoutError",
313 tabId: "recording-tab",314 tabId: "recording-tab",
314 data: "third-frame",315 timeoutMs: BROWSER_RECORDING_STARTUP_SETTLE_TIMEOUT_MS,
315 width: 800,
316 height: 600,
317 receivedAt: "2026-06-26T00:00:02.000Z",
318 });316 });
317 await vi.advanceTimersByTimeAsync(BROWSER_RECORDING_STARTUP_SETTLE_TIMEOUT_MS);
319318
320 DeferredImage.instances[1]?.finishLoading();319 await rejection;
321 expect(drawImage).toHaveBeenCalledOnce();320 await expect(startPromise).rejects.toBeInstanceOf(BrowserRecordingCaptureTimeoutError);
322 DeferredImage.instances[2]?.finishLoading();321 expect(stopScreencast).toHaveBeenCalledWith("recording-tab");
323 expect(drawImage).toHaveBeenCalledTimes(2);322 expect(readActiveBrowserRecordingTabIds()).toEqual(new Set());
324 DeferredImage.instances[0]?.finishLoading();323 expect(useBrowserSurfaceStore.getState().activityByTabId["recording-tab"]).toBeUndefined();
325 expect(drawImage).toHaveBeenCalledTimes(2);
326324
327 await stopBrowserRecording("recording-tab");325 finishCapture({ getTracks: () => [{ stop: stopTrack }] } as unknown as MediaStream);
326 await vi.advanceTimersByTimeAsync(0);
327 expect(stopTrack).toHaveBeenCalledOnce();
328 });328 });
329329
330 it("records separate tabs concurrently", async () => {330 it("records separate tabs concurrently", async () => {
...@@ -336,22 +336,12 @@ describe("browser recording", () => {...@@ -336,22 +336,12 @@ describe("browser recording", () => {
336 environmentId: EnvironmentId.make("environment-recording"),336 environmentId: EnvironmentId.make("environment-recording"),
337 threadId: ThreadId.make("thread-recording-second"),337 threadId: ThreadId.make("thread-recording-second"),
338 };338 };
339 surfaceState.byTabId = {
340 ...surfaceState.byTabId,
341 "recording-tab-2": {
342 visible: false,
343 rect: { x: 0, y: 0, width: 390, height: 844 },
344 content: { x: 0, y: 0, width: 390, height: 844, scale: 1, scrollLeft: 0, scrollTop: 0 },
345 },
346 };
347
348 await Promise.all([339 await Promise.all([
349 startBrowserRecording("recording-tab", firstThreadRef),340 startBrowserRecording("recording-tab", firstThreadRef),
350 startBrowserRecording("recording-tab-2", secondThreadRef),341 startBrowserRecording("recording-tab-2", secondThreadRef),
351 ]);342 ]);
352343
353 expect(startScreencast).toHaveBeenCalledTimes(2);344 expect(startScreencast).toHaveBeenCalledTimes(2);
354 expect(onFrame).toHaveBeenCalledOnce();
355 expect(events).toContain("publish:recording-tab,recording-tab-2");345 expect(events).toContain("publish:recording-tab,recording-tab-2");
356 expect(readActiveBrowserRecordingTabIds()).toEqual(346 expect(readActiveBrowserRecordingTabIds()).toEqual(
357 new Set(["recording-tab", "recording-tab-2"]),347 new Set(["recording-tab", "recording-tab-2"]),
...@@ -372,22 +362,6 @@ describe("browser recording", () => {...@@ -372,22 +362,6 @@ describe("browser recording", () => {
372 threadId: ThreadId.make("thread-recording-scoped"),362 threadId: ThreadId.make("thread-recording-scoped"),
373 };363 };
374 const runtimeTabId = previewRuntimeTabId(threadRef, "epoch-a", "tab_1");364 const runtimeTabId = previewRuntimeTabId(threadRef, "epoch-a", "tab_1");
375 surfaceState.byTabId = {
376 [runtimeTabId]: {
377 visible: false,
378 rect: { x: 0, y: 0, width: 1280, height: 800 },
379 content: {
380 x: 0,
381 y: 0,
382 width: 1280,
383 height: 800,
384 scale: 1,
385 scrollLeft: 0,
386 scrollTop: 0,
387 },
388 },
389 };
390
391 await startBrowserRecording(runtimeTabId, threadRef, "tab_1");365 await startBrowserRecording(runtimeTabId, threadRef, "tab_1");
392366
393 expect(startScreencast).toHaveBeenCalledWith(runtimeTabId);367 expect(startScreencast).toHaveBeenCalledWith(runtimeTabId);
...@@ -408,18 +382,12 @@ describe("browser recording", () => {...@@ -408,18 +382,12 @@ describe("browser recording", () => {
408382
409 it("does not report success for a second start while the first is still starting", async () => {383 it("does not report success for a second start while the first is still starting", async () => {
410 let finishStartingScreencast: (() => void) | undefined;384 let finishStartingScreencast: (() => void) | undefined;
411 startScreencast.mockImplementationOnce(async (tabId: string) => {385 startScreencast.mockImplementationOnce(async () => {
412 events.push("start-screencast");386 events.push("start-screencast");
413 frameSubscription.listener?.({
414 tabId,
415 data: "initial-frame",
416 width: 800,
417 height: 600,
418 receivedAt: "2026-06-26T00:00:00.000Z",
419 });
420 await new Promise<void>((resolve) => {387 await new Promise<void>((resolve) => {
421 finishStartingScreencast = resolve;388 finishStartingScreencast = resolve;
422 });389 });
390 return { sourceId: "source:recording-tab", width: 1000, height: 620 };
423 });391 });
424392
425 const firstStart = startBrowserRecording("recording-tab");393 const firstStart = startBrowserRecording("recording-tab");
...@@ -484,7 +452,7 @@ describe("browser recording", () => {...@@ -484,7 +452,7 @@ describe("browser recording", () => {
484 await new Promise<void>((resolve) => {452 await new Promise<void>((resolve) => {
485 finishStartingScreencast = resolve;453 finishStartingScreencast = resolve;
486 });454 });
487 emitRecordingFrame();455 return { sourceId: "source:recording-tab", width: 1000, height: 620 };
488 });456 });
489457
490 const startPromise = startBrowserRecording("recording-tab");458 const startPromise = startBrowserRecording("recording-tab");
...@@ -508,7 +476,7 @@ describe("browser recording", () => {...@@ -508,7 +476,7 @@ describe("browser recording", () => {
508 await new Promise<void>((resolve) => {476 await new Promise<void>((resolve) => {
509 finishStartingScreencast = resolve;477 finishStartingScreencast = resolve;
510 });478 });
511 emitRecordingFrame();479 return { sourceId: "source:recording-tab", width: 1000, height: 620 };
512 });480 });
513481
514 const firstStart = startBrowserRecording("recording-tab");482 const firstStart = startBrowserRecording("recording-tab");
...@@ -535,7 +503,7 @@ describe("browser recording", () => {...@@ -535,7 +503,7 @@ describe("browser recording", () => {
535 await new Promise<void>((resolve) => {503 await new Promise<void>((resolve) => {
536 finishStartingScreencast = resolve;504 finishStartingScreencast = resolve;
537 });505 });
538 emitRecordingFrame();506 return { sourceId: "source:recording-tab", width: 1000, height: 620 };
539 });507 });
540 stopScreencast.mockRejectedValueOnce(new Error("initial stop failed"));508 stopScreencast.mockRejectedValueOnce(new Error("initial stop failed"));
541509
...@@ -569,15 +537,14 @@ describe("browser recording", () => {...@@ -569,15 +537,14 @@ describe("browser recording", () => {
569 await new Promise<void>((resolve) => {537 await new Promise<void>((resolve) => {
570 finishStartingScreencast = resolve;538 finishStartingScreencast = resolve;
571 });539 });
572 emitRecordingFrame();540 return { sourceId: "source:recording-tab", width: 1000, height: 620 };
573 });541 });
574542
575 const startPromise = startBrowserRecording("recording-tab");543 const startPromise = startBrowserRecording("recording-tab");
576 expect(startScreencast).toHaveBeenCalledOnce();544 await vi.waitFor(() => expect(startScreencast).toHaveBeenCalledOnce());
577545
578 const stopPromise = stopBrowserRecording("recording-tab");546 const stopPromise = stopBrowserRecording("recording-tab");
579 await Promise.resolve();547 await vi.advanceTimersByTimeAsync(0);
580 await Promise.resolve();
581 expect(stopScreencast).not.toHaveBeenCalled();548 expect(stopScreencast).not.toHaveBeenCalled();
582549
583 const rejection = expect(stopPromise).rejects.toMatchObject({550 const rejection = expect(stopPromise).rejects.toMatchObject({
...@@ -593,6 +560,7 @@ describe("browser recording", () => {...@@ -593,6 +560,7 @@ describe("browser recording", () => {
593 );560 );
594561
595 finishStartingScreencast?.();562 finishStartingScreencast?.();
563 await vi.advanceTimersByTimeAsync(32);
596 await startPromise;564 await startPromise;
597 const cleanupResult = await stopBrowserRecording("recording-tab");565 const cleanupResult = await stopBrowserRecording("recording-tab");
598 expect(cleanupResult).toBeNull();566 expect(cleanupResult).toBeNull();
apps/web/src/browser/browserRecording.ts+171-143
...@@ -1,6 +1,6 @@...@@ -1,6 +1,6 @@
1import type {1import type {
2 DesktopPreviewRecordingArtifact,2 DesktopPreviewRecordingArtifact,
3 DesktopPreviewRecordingFrame,3 DesktopPreviewRecordingSource,
4 ScopedThreadRef,4 ScopedThreadRef,
5} from "@t3tools/contracts";5} from "@t3tools/contracts";
6import { useAtomValue } from "@effect/atom-react";6import { useAtomValue } from "@effect/atom-react";
...@@ -8,8 +8,10 @@ import * as Schema from "effect/Schema";...@@ -8,8 +8,10 @@ import * as Schema from "effect/Schema";
8import { Atom } from "effect/unstable/reactivity";8import { Atom } from "effect/unstable/reactivity";
99
10import { previewBridge } from "~/components/preview/previewBridge";10import { previewBridge } from "~/components/preview/previewBridge";
11import { ensureClientSettingsHydrated, getClientSettings } from "~/hooks/useSettings";
11import { appAtomRegistry } from "~/rpc/atomRegistry";12import { appAtomRegistry } from "~/rpc/atomRegistry";
12import { useBrowserSurfaceStore } from "./browserSurfaceStore";13
14import { acquireBrowserSurfaceActivity } from "./browserSurfaceStore";
1315
14export class BrowserRecordingUnavailableError extends Schema.TaggedErrorClass<BrowserRecordingUnavailableError>()(16export class BrowserRecordingUnavailableError extends Schema.TaggedErrorClass<BrowserRecordingUnavailableError>()(
15 "BrowserRecordingUnavailableError",17 "BrowserRecordingUnavailableError",
...@@ -34,16 +36,24 @@ export class BrowserRecordingConflictError extends Schema.TaggedErrorClass<Brows...@@ -34,16 +36,24 @@ export class BrowserRecordingConflictError extends Schema.TaggedErrorClass<Brows
34 }36 }
35}37}
3638
37export class BrowserRecordingCanvasUnavailableError extends Schema.TaggedErrorClass<BrowserRecordingCanvasUnavailableError>()(39export class BrowserRecordingFormatUnavailableError extends Schema.TaggedErrorClass<BrowserRecordingFormatUnavailableError>()(
38 "BrowserRecordingCanvasUnavailableError",40 "BrowserRecordingFormatUnavailableError",
41 { tabId: Schema.String },
42) {
43 override get message(): string {
44 return `MediaRecorder did not report an output format for tab ${this.tabId}.`;
45 }
46}
47
48export class BrowserRecordingCaptureTimeoutError extends Schema.TaggedErrorClass<BrowserRecordingCaptureTimeoutError>()(
49 "BrowserRecordingCaptureTimeoutError",
39 {50 {
40 tabId: Schema.String,51 tabId: Schema.String,
41 width: Schema.Number,52 timeoutMs: Schema.Number,
42 height: Schema.Number,
43 },53 },
44) {54) {
45 override get message(): string {55 override get message(): string {
46 return `Browser recording canvas ${this.width}x${this.height} is unavailable for tab ${this.tabId}.`;56 return `Browser recording media capture for tab ${this.tabId} did not settle within ${this.timeoutMs}ms.`;
47 }57 }
48}58}
4959
...@@ -52,11 +62,10 @@ export class BrowserRecordingOperationError extends Schema.TaggedErrorClass<Brow...@@ -52,11 +62,10 @@ export class BrowserRecordingOperationError extends Schema.TaggedErrorClass<Brow
52 {62 {
53 operation: Schema.Literals([63 operation: Schema.Literals([
54 "initialize-media-recorder",64 "initialize-media-recorder",
55 "subscribe-frames",65 "capture-media-stream",
56 "start-media-recorder",66 "start-media-recorder",
57 "start-screencast",67 "start-screencast",
58 "stop-screencast",68 "stop-screencast",
59 "wait-first-frame",
60 "wait-startup",69 "wait-startup",
61 "stop-media-recorder",70 "stop-media-recorder",
62 "save-artifact",71 "save-artifact",
...@@ -72,6 +81,7 @@ export class BrowserRecordingOperationError extends Schema.TaggedErrorClass<Brow...@@ -72,6 +81,7 @@ export class BrowserRecordingOperationError extends Schema.TaggedErrorClass<Brow
72}81}
7382
74const isBrowserRecordingOperationError = Schema.is(BrowserRecordingOperationError);83const isBrowserRecordingOperationError = Schema.is(BrowserRecordingOperationError);
84const isBrowserRecordingCaptureTimeoutError = Schema.is(BrowserRecordingCaptureTimeoutError);
7585
76type BrowserRecordingLifecycle =86type BrowserRecordingLifecycle =
77 | { readonly phase: "starting" }87 | { readonly phase: "starting" }
...@@ -82,23 +92,17 @@ type BrowserRecordingLifecycle =...@@ -82,23 +92,17 @@ type BrowserRecordingLifecycle =
82 };92 };
8393
84interface ActiveRecording {94interface ActiveRecording {
85 /** Desktop-scoped identity used by capture and surface stores. */95 /** Desktop-scoped identity used by the native capture lease. */
86 readonly tabId: string;96 readonly tabId: string;
87 /** Server-local identity returned by preview automation tools. */97 /** Server-local identity returned by preview automation tools. */
88 readonly serverTabId: string;98 readonly serverTabId: string;
89 readonly threadRef: ScopedThreadRef | null;99 readonly threadRef: ScopedThreadRef | null;
90 readonly canvas: HTMLCanvasElement;
91 readonly context: CanvasRenderingContext2D;
92 readonly chunks: Blob[];100 readonly chunks: Blob[];
93 readonly startedAt: string;101 readonly startedAt: string;
94 readonly startupSettled: Promise<void>;102 readonly startupSettled: Promise<void>;
95 readonly firstFrameSize: Promise<"frame" | "cancelled">;103 releaseSurfaceActivity: (() => void) | null;
96 readonly settleFirstFrameSize: (outcome: "frame" | "cancelled") => void;104 stream: MediaStream | null;
97 recorder: MediaRecorder | null;105 recorder: MediaRecorder | null;
98 mimeType: string | null;
99 frameSizeEstablished: boolean;
100 frameSequence: number;
101 lastDrawnFrameSequence: number;
102 lifecycle: BrowserRecordingLifecycle;106 lifecycle: BrowserRecordingLifecycle;
103}107}
104108
...@@ -120,7 +124,6 @@ export function useActiveBrowserRecordingTabIds(): ReadonlySet<string> {...@@ -120,7 +124,6 @@ export function useActiveBrowserRecordingTabIds(): ReadonlySet<string> {
120}124}
121125
122const activeRecordings = new Map<string, ActiveRecording>();126const activeRecordings = new Map<string, ActiveRecording>();
123let unsubscribeFrames: (() => void) | null = null;
124127
125const publishActiveRecordingTabIds = (): void => {128const publishActiveRecordingTabIds = (): void => {
126 appAtomRegistry.set(activeBrowserRecordingTabIdsAtom, {129 appAtomRegistry.set(activeBrowserRecordingTabIdsAtom, {
...@@ -129,7 +132,7 @@ const publishActiveRecordingTabIds = (): void => {...@@ -129,7 +132,7 @@ const publishActiveRecordingTabIds = (): void => {
129};132};
130133
131export const BROWSER_RECORDING_STARTUP_SETTLE_TIMEOUT_MS = 5_000;134export const BROWSER_RECORDING_STARTUP_SETTLE_TIMEOUT_MS = 5_000;
132export const BROWSER_RECORDING_FIRST_FRAME_SIZE_TIMEOUT_MS = 5_000;135export const BROWSER_RECORDING_PAINT_SETTLE_TIMEOUT_MS = 250;
133136
134export function readActiveBrowserRecordingTabIds(threadRef?: ScopedThreadRef): ReadonlySet<string> {137export function readActiveBrowserRecordingTabIds(threadRef?: ScopedThreadRef): ReadonlySet<string> {
135 const tabIds = new Set<string>();138 const tabIds = new Set<string>();
...@@ -167,55 +170,38 @@ export function findActiveBrowserRecordingRuntimeTabId(...@@ -167,55 +170,38 @@ export function findActiveBrowserRecordingRuntimeTabId(
167 );170 );
168}171}
169172
170const preferredMimeType = (): string => {173const preferredMimeTypes = [
171 const candidates = ["video/mp4;codecs=avc1.42E01E", "video/webm;codecs=vp9", "video/webm"];174 "video/webm;codecs=av1",
172 return candidates.find((candidate) => MediaRecorder.isTypeSupported(candidate)) ?? "video/webm";175 "video/webm;codecs=vp9",
176 "video/mp4;codecs=avc1.640028",
177 "video/mp4;codecs=avc1.42e01e",
178 "video/webm;codecs=vp8",
179 "video/webm",
180] as const;
181
182const createMediaRecorder = (stream: MediaStream): MediaRecorder => {
183 const mimeType = preferredMimeTypes.find((candidate) => MediaRecorder.isTypeSupported(candidate));
184 return mimeType ? new MediaRecorder(stream, { mimeType }) : new MediaRecorder(stream);
173};185};
174186
175const drawFrame = (frame: DesktopPreviewRecordingFrame): void => {187const captureTabMediaStream = (
176 const recording = activeRecordings.get(frame.tabId);188 source: DesktopPreviewRecordingSource,
177 if (!recording) return;189 frameRate: number,
178 if (190): Promise<MediaStream> =>
179 !Number.isFinite(frame.width) ||191 navigator.mediaDevices.getUserMedia({
180 !Number.isFinite(frame.height) ||192 audio: false,
181 frame.width <= 0 ||193 video: {
182 frame.height <= 0194 mandatory: {
183 ) {195 chromeMediaSource: "tab",
184 return;196 chromeMediaSourceId: source.sourceId,
185 }197 minWidth: source.width,
186 const width = Math.max(1, Math.round(frame.width));198 maxWidth: source.width,
187 const height = Math.max(1, Math.round(frame.height));199 minHeight: source.height,
188 if (!recording.frameSizeEstablished) {200 maxHeight: source.height,
189 recording.canvas.width = width;201 maxFrameRate: frameRate,
190 recording.canvas.height = height;202 },
191 recording.frameSizeEstablished = true;203 } as unknown as MediaTrackConstraints,
192 recording.settleFirstFrameSize("frame");204 });
193 }
194 const frameSequence = ++recording.frameSequence;
195 const image = new Image();
196 image.addEventListener(
197 "load",
198 () => {
199 if (
200 activeRecordings.get(frame.tabId) !== recording ||
201 frameSequence <= recording.lastDrawnFrameSequence
202 ) {
203 return;
204 }
205 recording.lastDrawnFrameSequence = frameSequence;
206 const scale = Math.min(recording.canvas.width / width, recording.canvas.height / height);
207 const targetWidth = width * scale;
208 const targetHeight = height * scale;
209 const targetX = (recording.canvas.width - targetWidth) / 2;
210 const targetY = (recording.canvas.height - targetHeight) / 2;
211 recording.context.fillStyle = "#000000";
212 recording.context.fillRect(0, 0, recording.canvas.width, recording.canvas.height);
213 recording.context.drawImage(image, targetX, targetY, targetWidth, targetHeight);
214 },
215 { once: true },
216 );
217 image.src = `data:image/jpeg;base64,${frame.data}`;
218};
219205
220const stopMediaRecorder = async (recorder: MediaRecorder | null): Promise<void> => {206const stopMediaRecorder = async (recorder: MediaRecorder | null): Promise<void> => {
221 if (!recorder || recorder.state === "inactive") return;207 if (!recorder || recorder.state === "inactive") return;
...@@ -226,17 +212,76 @@ const stopMediaRecorder = async (recorder: MediaRecorder | null): Promise<void>...@@ -226,17 +212,76 @@ const stopMediaRecorder = async (recorder: MediaRecorder | null): Promise<void>
226 await stopped;212 await stopped;
227};213};
228214
215const stopMediaStream = (stream: MediaStream | null): void => {
216 for (const track of stream?.getTracks() ?? []) track.stop();
217};
218
219const captureTabMediaStreamWithTimeout = async (
220 tabId: string,
221 source: DesktopPreviewRecordingSource,
222 frameRate: number,
223): Promise<MediaStream> => {
224 let acceptStream = true;
225 let timeoutId: number | null = null;
226 const streamPromise = captureTabMediaStream(source, frameRate).then((stream) => {
227 if (!acceptStream) stopMediaStream(stream);
228 return stream;
229 });
230 try {
231 return await Promise.race([
232 streamPromise,
233 new Promise<never>((_, reject) => {
234 timeoutId = window.setTimeout(
235 () =>
236 reject(
237 new BrowserRecordingCaptureTimeoutError({
238 tabId,
239 timeoutMs: BROWSER_RECORDING_STARTUP_SETTLE_TIMEOUT_MS,
240 }),
241 ),
242 BROWSER_RECORDING_STARTUP_SETTLE_TIMEOUT_MS,
243 );
244 }),
245 ]);
246 } finally {
247 acceptStream = false;
248 if (timeoutId !== null) window.clearTimeout(timeoutId);
249 }
250};
251
229const clearActiveRecording = (recording: ActiveRecording): void => {252const clearActiveRecording = (recording: ActiveRecording): void => {
253 recording.releaseSurfaceActivity?.();
254 recording.releaseSurfaceActivity = null;
230 if (activeRecordings.get(recording.tabId) !== recording) return;255 if (activeRecordings.get(recording.tabId) !== recording) return;
231 recording.settleFirstFrameSize("cancelled");
232 activeRecordings.delete(recording.tabId);256 activeRecordings.delete(recording.tabId);
233 if (activeRecordings.size === 0) {
234 unsubscribeFrames?.();
235 unsubscribeFrames = null;
236 }
237 publishActiveRecordingTabIds();257 publishActiveRecordingTabIds();
238};258};
239259
260const waitForBrowserRecordingPaint = async (): Promise<void> => {
261 let firstFrameId: number | null = null;
262 let secondFrameId: number | null = null;
263 let timeoutId: number | null = null;
264 const painted = new Promise<void>((resolve) => {
265 firstFrameId = window.requestAnimationFrame(() => {
266 firstFrameId = null;
267 secondFrameId = window.requestAnimationFrame(() => {
268 secondFrameId = null;
269 resolve();
270 });
271 });
272 });
273 const timedOut = new Promise<void>((resolve) => {
274 timeoutId = window.setTimeout(resolve, BROWSER_RECORDING_PAINT_SETTLE_TIMEOUT_MS);
275 });
276 try {
277 await Promise.race([painted, timedOut]);
278 } finally {
279 if (timeoutId !== null) window.clearTimeout(timeoutId);
280 if (firstFrameId !== null) window.cancelAnimationFrame(firstFrameId);
281 if (secondFrameId !== null) window.cancelAnimationFrame(secondFrameId);
282 }
283};
284
240const cleanupFailedRecordingStart = async (285const cleanupFailedRecordingStart = async (
241 bridge: NonNullable<typeof previewBridge>,286 bridge: NonNullable<typeof previewBridge>,
242 recording: ActiveRecording,287 recording: ActiveRecording,
...@@ -251,6 +296,11 @@ const cleanupFailedRecordingStart = async (...@@ -251,6 +296,11 @@ const cleanupFailedRecordingStart = async (
251 await stopMediaRecorder(recording.recorder);296 await stopMediaRecorder(recording.recorder);
252 } catch (error) {297 } catch (error) {
253 errors.push(error);298 errors.push(error);
299 }
300 try {
301 stopMediaStream(recording.stream);
302 } catch (error) {
303 errors.push(error);
254 } finally {304 } finally {
255 clearActiveRecording(recording);305 clearActiveRecording(recording);
256 }306 }
...@@ -276,19 +326,6 @@ const recordingStartupCancelledError = (...@@ -276,19 +326,6 @@ const recordingStartupCancelledError = (
276const isRecordingStarting = (recording: ActiveRecording): boolean =>326const isRecordingStarting = (recording: ActiveRecording): boolean =>
277 activeRecordings.get(recording.tabId) === recording && recording.lifecycle.phase === "starting";327 activeRecordings.get(recording.tabId) === recording && recording.lifecycle.phase === "starting";
278328
279const waitForFirstFrameSize = async (recording: ActiveRecording): Promise<boolean> => {
280 if (recording.frameSizeEstablished) return true;
281 let timeout: ReturnType<typeof setTimeout> | null = null;
282 const outcome = await Promise.race([
283 recording.firstFrameSize,
284 new Promise<"timeout">((resolve) => {
285 timeout = setTimeout(() => resolve("timeout"), BROWSER_RECORDING_FIRST_FRAME_SIZE_TIMEOUT_MS);
286 }),
287 ]);
288 if (timeout !== null) clearTimeout(timeout);
289 return outcome === "frame";
290};
291
292const waitForRecordingStartupToSettle = async (recording: ActiveRecording): Promise<void> => {329const waitForRecordingStartupToSettle = async (recording: ActiveRecording): Promise<void> => {
293 let timeout: ReturnType<typeof setTimeout> | null = null;330 let timeout: ReturnType<typeof setTimeout> | null = null;
294 try {331 try {
...@@ -339,62 +376,35 @@ export async function startBrowserRecording(...@@ -339,62 +376,35 @@ export async function startBrowserRecording(
339 activeTabId: activeLogicalRecording,376 activeTabId: activeLogicalRecording,
340 });377 });
341 }378 }
342 const surface = useBrowserSurfaceStore.getState().byTabId[tabId];
343 const recordingSize = surface?.content ?? surface?.rect;
344 const canvas = document.createElement("canvas");
345 canvas.width = Math.max(1, recordingSize?.width ?? 1280);
346 canvas.height = Math.max(1, recordingSize?.height ?? 800);
347 const context = canvas.getContext("2d", { alpha: false });
348 if (!context) {
349 throw new BrowserRecordingCanvasUnavailableError({
350 tabId,
351 width: canvas.width,
352 height: canvas.height,
353 });
354 }
355 const startedAt = new Date().toISOString();379 const startedAt = new Date().toISOString();
356 const chunks: Blob[] = [];380 const chunks: Blob[] = [];
357 let settleStartup: (() => void) | undefined;381 let settleStartup: (() => void) | undefined;
358 const startupSettled = new Promise<void>((resolve) => {382 const startupSettled = new Promise<void>((resolve) => {
359 settleStartup = resolve;383 settleStartup = resolve;
360 });384 });
361 let settleFirstFrameSize: ((outcome: "frame" | "cancelled") => void) | undefined;385 const releaseSurfaceActivity = acquireBrowserSurfaceActivity(tabId);
362 const firstFrameSize = new Promise<"frame" | "cancelled">((resolve) => {
363 settleFirstFrameSize = resolve;
364 });
365 const recording: ActiveRecording = {386 const recording: ActiveRecording = {
366 tabId,387 tabId,
367 serverTabId,388 serverTabId,
368 threadRef,389 threadRef,
369 canvas,
370 context,
371 chunks,390 chunks,
372 startedAt,391 startedAt,
373 startupSettled,392 startupSettled,
374 firstFrameSize,393 releaseSurfaceActivity,
375 settleFirstFrameSize: (outcome) => settleFirstFrameSize?.(outcome),394 stream: null,
376 recorder: null,395 recorder: null,
377 mimeType: null,
378 frameSizeEstablished: false,
379 frameSequence: 0,
380 lastDrawnFrameSequence: 0,
381 lifecycle: { phase: "starting" },396 lifecycle: { phase: "starting" },
382 };397 };
383 activeRecordings.set(tabId, recording);398 activeRecordings.set(tabId, recording);
384 publishActiveRecordingTabIds();399 publishActiveRecordingTabIds();
385 try {400 try {
401 const frameRatePromise = ensureClientSettingsHydrated().then(
402 () => getClientSettings().browserRecordingFrameRate,
403 );
404 const [frameRate] = await Promise.all([frameRatePromise, waitForBrowserRecordingPaint()]);
405 let source: DesktopPreviewRecordingSource;
386 try {406 try {
387 unsubscribeFrames ??= bridge.recording.onFrame(drawFrame);407 source = await bridge.recording.startScreencast(tabId);
388 } catch (cause) {
389 clearActiveRecording(recording);
390 throw new BrowserRecordingOperationError({
391 operation: "subscribe-frames",
392 tabId,
393 cause,
394 });
395 }
396 try {
397 await bridge.recording.startScreencast(tabId);
398 } catch (cause) {408 } catch (cause) {
399 if (!isRecordingStarting(recording)) {409 if (!isRecordingStarting(recording)) {
400 throw recordingStartupCancelledError(recording, cause);410 throw recordingStartupCancelledError(recording, cause);
...@@ -425,34 +435,29 @@ export async function startBrowserRecording(...@@ -425,34 +435,29 @@ export async function startBrowserRecording(
425 throw recordingStartupCancelledError(recording);435 throw recordingStartupCancelledError(recording);
426 };436 };
427 await throwIfStartupCancelled();437 await throwIfStartupCancelled();
428 const hasFirstFrame = await waitForFirstFrameSize(recording);438 try {
429 await throwIfStartupCancelled();439 recording.stream = await captureTabMediaStreamWithTimeout(tabId, source, frameRate);
430 if (!hasFirstFrame) {440 } catch (cause) {
431 const cause = new Error(`No valid recording frame arrived for tab ${tabId}.`);
432 const cleanupCause = await cleanupFailedRecordingStart(bridge, recording);441 const cleanupCause = await cleanupFailedRecordingStart(bridge, recording);
442 if (isBrowserRecordingCaptureTimeoutError(cause) && cleanupCause === undefined) throw cause;
433 throw new BrowserRecordingOperationError({443 throw new BrowserRecordingOperationError({
434 operation: "wait-first-frame",444 operation: "capture-media-stream",
435 tabId,445 tabId,
436 cause:446 cause:
437 cleanupCause === undefined447 cleanupCause === undefined
438 ? cause448 ? cause
439 : new AggregateError(449 : new AggregateError(
440 [cause, cleanupCause],450 [cause, cleanupCause],
441 `Browser recording frame wait and cleanup failed for tab ${tabId}.`,451 `Browser media capture and cleanup failed for tab ${tabId}.`,
442 { cause },452 { cause },
443 ),453 ),
444 });454 });
445 }455 }
456 await throwIfStartupCancelled();
446457
447 let mimeType: string;
448 let recorder: MediaRecorder;458 let recorder: MediaRecorder;
449 try {459 try {
450 mimeType = preferredMimeType();460 recorder = createMediaRecorder(recording.stream);
451 recorder = new MediaRecorder(canvas.captureStream(12), {
452 mimeType,
453 videoBitsPerSecond: 4_000_000,
454 });
455 recording.mimeType = mimeType;
456 recording.recorder = recorder;461 recording.recorder = recorder;
457 recorder.addEventListener("dataavailable", (event) => {462 recorder.addEventListener("dataavailable", (event) => {
458 if (event.data.size > 0) chunks.push(event.data);463 if (event.data.size > 0) chunks.push(event.data);
...@@ -520,7 +525,7 @@ const finalizeBrowserRecording = async (...@@ -520,7 +525,7 @@ const finalizeBrowserRecording = async (
520 cause,525 cause,
521 });526 });
522 }527 }
523 if (!recording.recorder || !recording.mimeType) {528 if (!recording.recorder) {
524 result = { _tag: "Success", artifact: null };529 result = { _tag: "Success", artifact: null };
525 } else {530 } else {
526 try {531 try {
...@@ -532,11 +537,17 @@ const finalizeBrowserRecording = async (...@@ -532,11 +537,17 @@ const finalizeBrowserRecording = async (
532 cause,537 cause,
533 });538 });
534 }539 }
540 const mimeType =
541 recording.recorder.mimeType ||
542 recording.chunks.find((chunk) => chunk.type.length > 0)?.type;
543 if (!mimeType) {
544 throw new BrowserRecordingFormatUnavailableError({ tabId });
545 }
535 try {546 try {
536 const blob = new Blob(recording.chunks, { type: recording.mimeType });547 const blob = new Blob(recording.chunks, { type: mimeType });
537 const artifact = await bridge.recording.save(548 const artifact = await bridge.recording.save(
538 tabId,549 tabId,
539 recording.mimeType,550 mimeType,
540 new Uint8Array(await blob.arrayBuffer()),551 new Uint8Array(await blob.arrayBuffer()),
541 );552 );
542 result = { _tag: "Success", artifact };553 result = { _tag: "Success", artifact };
...@@ -560,18 +571,34 @@ const finalizeBrowserRecording = async (...@@ -560,18 +571,34 @@ const finalizeBrowserRecording = async (
560 throw result.error;571 throw result.error;
561 }572 }
562573
563 let cleanupError: BrowserRecordingOperationError | undefined;574 const cleanupErrors: unknown[] = [];
564 try {575 try {
565 await stopMediaRecorder(recording.recorder);576 await stopMediaRecorder(recording.recorder);
566 } catch (cause) {577 } catch (cause) {
567 cleanupError = new BrowserRecordingOperationError({578 cleanupErrors.push(cause);
568 operation: "stop-media-recorder",579 }
569 tabId,580 try {
570 cause,581 stopMediaStream(recording.stream);
571 });582 } catch (cause) {
583 cleanupErrors.push(cause);
572 } finally {584 } finally {
573 clearActiveRecording(recording);585 clearActiveRecording(recording);
574 }586 }
587 const cleanupError =
588 cleanupErrors.length === 0
589 ? undefined
590 : new BrowserRecordingOperationError({
591 operation: "cleanup",
592 tabId,
593 cause:
594 cleanupErrors.length === 1
595 ? cleanupErrors[0]
596 : new AggregateError(
597 cleanupErrors,
598 `Browser recording media cleanup failed for tab ${tabId}.`,
599 { cause: cleanupErrors[0] },
600 ),
601 });
575602
576 if (result._tag === "Failure") {603 if (result._tag === "Failure") {
577 if (cleanupError) {604 if (cleanupError) {
...@@ -598,6 +625,7 @@ const discardBrowserRecording = async (...@@ -598,6 +625,7 @@ const discardBrowserRecording = async (
598 try {625 try {
599 await bridge.recording.stopScreencast(recording.tabId).catch(() => undefined);626 await bridge.recording.stopScreencast(recording.tabId).catch(() => undefined);
600 await stopMediaRecorder(recording.recorder).catch(() => undefined);627 await stopMediaRecorder(recording.recorder).catch(() => undefined);
628 stopMediaStream(recording.stream);
601 return null;629 return null;
602 } finally {630 } finally {
603 clearActiveRecording(recording);631 clearActiveRecording(recording);
apps/web/src/components/settings/IntegrationsSettings.tsx+48
...@@ -7,7 +7,9 @@...@@ -7,7 +7,9 @@
7 * @module IntegrationsSettings7 * @module IntegrationsSettings
8 */8 */
9import {9import {
10 BROWSER_RECORDING_FRAME_RATES,
10 DEFAULT_BROWSER_AUTO_SHOW_FLOATING_PREVIEW,11 DEFAULT_BROWSER_AUTO_SHOW_FLOATING_PREVIEW,
12 DEFAULT_BROWSER_RECORDING_FRAME_RATE,
11 DEFAULT_BROWSER_VIEWPORT,13 DEFAULT_BROWSER_VIEWPORT,
12 DEFAULT_PREVIEW_APPEARANCE,14 DEFAULT_PREVIEW_APPEARANCE,
13 DEFAULT_UNIFIED_SETTINGS,15 DEFAULT_UNIFIED_SETTINGS,
...@@ -355,6 +357,51 @@ function BrowserAppearanceSetting({ disabled }: { readonly disabled: boolean })...@@ -355,6 +357,51 @@ function BrowserAppearanceSetting({ disabled }: { readonly disabled: boolean })
355 );357 );
356}358}
357359
360function BrowserRecordingFrameRateSetting({ disabled }: { readonly disabled: boolean }) {
361 const frameRate = useClientSettings((settings) => settings.browserRecordingFrameRate);
362 const updateSettings = useUpdatePrimarySettings();
363
364 return (
365 <SettingsRow
366 {...searchableSetting("browser-recording-frame-rate")}
367 description="Maximum frame rate for browser recordings. 30 fps is the default and uses less CPU and storage; 60 fps captures smoother motion."
368 resetAction={
369 !disabled && frameRate !== DEFAULT_BROWSER_RECORDING_FRAME_RATE ? (
370 <SettingResetButton
371 label="browser recording frame rate"
372 onClick={() =>
373 updateSettings({ browserRecordingFrameRate: DEFAULT_BROWSER_RECORDING_FRAME_RATE })
374 }
375 />
376 ) : null
377 }
378 control={
379 <Select
380 disabled={disabled}
381 value={String(frameRate)}
382 onValueChange={(value) => {
383 const next = BROWSER_RECORDING_FRAME_RATES.find((rate) => String(rate) === value);
384 if (next !== undefined) {
385 updateSettings({ browserRecordingFrameRate: next });
386 }
387 }}
388 >
389 <SelectTrigger className="w-full sm:w-40" aria-label="Browser recording frame rate">
390 <SelectValue>{frameRate} fps</SelectValue>
391 </SelectTrigger>
392 <SelectPopup align="end" alignItemWithTrigger={false}>
393 {BROWSER_RECORDING_FRAME_RATES.map((rate) => (
394 <SelectItem hideIndicator key={rate} value={String(rate)}>
395 {rate} fps
396 </SelectItem>
397 ))}
398 </SelectPopup>
399 </Select>
400 }
401 />
402 );
403}
404
358function AgentBrowserAccessSetting() {405function AgentBrowserAccessSetting() {
359 const settings = usePrimarySettings();406 const settings = usePrimarySettings();
360 const updateSettings = useUpdatePrimarySettings();407 const updateSettings = useUpdatePrimarySettings();
...@@ -461,6 +508,7 @@ export function IntegrationsSettingsPanel() {...@@ -461,6 +508,7 @@ export function IntegrationsSettingsPanel() {
461 <BrowserViewportSetting disabled={previewDefaultsDisabled} />508 <BrowserViewportSetting disabled={previewDefaultsDisabled} />
462 <BrowserZoomSetting disabled={previewDefaultsDisabled} />509 <BrowserZoomSetting disabled={previewDefaultsDisabled} />
463 <BrowserAppearanceSetting disabled={previewDefaultsDisabled} />510 <BrowserAppearanceSetting disabled={previewDefaultsDisabled} />
511 <BrowserRecordingFrameRateSetting disabled={previewDefaultsDisabled} />
464 <BrowserAutoShowFloatingPreviewSetting disabled={previewDefaultsDisabled} />512 <BrowserAutoShowFloatingPreviewSetting disabled={previewDefaultsDisabled} />
465 </>513 </>
466 );514 );
apps/web/src/components/settings/SettingsPanels.logic.test.ts+8-1
...@@ -268,9 +268,16 @@ describe("getChangedBrowserSettingLabels", () => {...@@ -268,9 +268,16 @@ describe("getChangedBrowserSettingLabels", () => {
268 browserDefaultViewport: { _tag: "freeform", width: 900, height: 600 },268 browserDefaultViewport: { _tag: "freeform", width: 900, height: 600 },
269 browserDefaultZoomFactor: 1.5,269 browserDefaultZoomFactor: 1.5,
270 browserDefaultAppearance: "dark",270 browserDefaultAppearance: "dark",
271 browserRecordingFrameRate: 60,
271 browserAutoShowFloatingPreview: !DEFAULT_UNIFIED_SETTINGS.browserAutoShowFloatingPreview,272 browserAutoShowFloatingPreview: !DEFAULT_UNIFIED_SETTINGS.browserAutoShowFloatingPreview,
272 }),273 }),
273 ).toEqual(["Browser viewport", "Browser zoom", "Browser appearance", "Floating preview"]);274 ).toEqual([
275 "Browser viewport",
276 "Browser zoom",
277 "Browser appearance",
278 "Recording frame rate",
279 "Floating preview",
280 ]);
274 });281 });
275});282});
276283
apps/web/src/components/settings/SettingsPanels.logic.ts+4
...@@ -114,6 +114,7 @@ export type BrowserDefaultSettings = Pick<...@@ -114,6 +114,7 @@ export type BrowserDefaultSettings = Pick<
114 | "browserDefaultViewport"114 | "browserDefaultViewport"
115 | "browserDefaultZoomFactor"115 | "browserDefaultZoomFactor"
116 | "browserDefaultAppearance"116 | "browserDefaultAppearance"
117 | "browserRecordingFrameRate"
117 | "browserAutoShowFloatingPreview"118 | "browserAutoShowFloatingPreview"
118>;119>;
119120
...@@ -151,6 +152,9 @@ export function getChangedBrowserSettingLabels(settings: BrowserDefaultSettings)...@@ -151,6 +152,9 @@ export function getChangedBrowserSettingLabels(settings: BrowserDefaultSettings)
151 ...(settings.browserDefaultAppearance !== DEFAULT_UNIFIED_SETTINGS.browserDefaultAppearance152 ...(settings.browserDefaultAppearance !== DEFAULT_UNIFIED_SETTINGS.browserDefaultAppearance
152 ? ["Browser appearance"]153 ? ["Browser appearance"]
153 : []),154 : []),
155 ...(settings.browserRecordingFrameRate !== DEFAULT_UNIFIED_SETTINGS.browserRecordingFrameRate
156 ? ["Recording frame rate"]
157 : []),
154 ...(settings.browserAutoShowFloatingPreview !==158 ...(settings.browserAutoShowFloatingPreview !==
155 DEFAULT_UNIFIED_SETTINGS.browserAutoShowFloatingPreview159 DEFAULT_UNIFIED_SETTINGS.browserAutoShowFloatingPreview
156 ? ["Floating preview"]160 ? ["Floating preview"]
apps/web/src/components/settings/SettingsPanels.tsx+2
...@@ -553,6 +553,7 @@ export function useSettingsRestore(onRestored?: () => void) {...@@ -553,6 +553,7 @@ export function useSettingsRestore(onRestored?: () => void) {
553 settings.browserDefaultViewport,553 settings.browserDefaultViewport,
554 settings.browserDefaultZoomFactor,554 settings.browserDefaultZoomFactor,
555 settings.browserDefaultAppearance,555 settings.browserDefaultAppearance,
556 settings.browserRecordingFrameRate,
556 settings.browserAutoShowFloatingPreview,557 settings.browserAutoShowFloatingPreview,
557 settings.appearanceContrast,558 settings.appearanceContrast,
558 settings.enableAgentBrowserAccess,559 settings.enableAgentBrowserAccess,
...@@ -688,6 +689,7 @@ export function useSettingsRestore(onRestored?: () => void) {...@@ -688,6 +689,7 @@ export function useSettingsRestore(onRestored?: () => void) {
688 browserDefaultViewport: DEFAULT_UNIFIED_SETTINGS.browserDefaultViewport,689 browserDefaultViewport: DEFAULT_UNIFIED_SETTINGS.browserDefaultViewport,
689 browserDefaultZoomFactor: DEFAULT_UNIFIED_SETTINGS.browserDefaultZoomFactor,690 browserDefaultZoomFactor: DEFAULT_UNIFIED_SETTINGS.browserDefaultZoomFactor,
690 browserDefaultAppearance: DEFAULT_UNIFIED_SETTINGS.browserDefaultAppearance,691 browserDefaultAppearance: DEFAULT_UNIFIED_SETTINGS.browserDefaultAppearance,
692 browserRecordingFrameRate: DEFAULT_UNIFIED_SETTINGS.browserRecordingFrameRate,
691 browserAutoShowFloatingPreview: DEFAULT_UNIFIED_SETTINGS.browserAutoShowFloatingPreview,693 browserAutoShowFloatingPreview: DEFAULT_UNIFIED_SETTINGS.browserAutoShowFloatingPreview,
692 // Re-granted like any other default. The confirmation dialog lists it by694 // Re-granted like any other default. The confirmation dialog lists it by
693 // name, so a user restoring defaults is told the agent regains access695 // name, so a user restoring defaults is told the agent regains access
apps/web/src/components/settings/settingsSearch.test.ts+8
...@@ -98,4 +98,12 @@ describe("searchSettings", () => {...@@ -98,4 +98,12 @@ describe("searchSettings", () => {
98 targetId: "appearance",98 targetId: "appearance",
99 });99 });
100 });100 });
101
102 it("routes browser recording quality to integrations", () => {
103 expect(searchSettings("recording frame rate")[0]).toMatchObject({
104 id: "browser-recording-frame-rate",
105 to: "/settings/integrations",
106 targetId: "browser",
107 });
108 });
101});109});
apps/web/src/components/settings/settingsSearch.ts+6
...@@ -237,6 +237,12 @@ export const SETTINGS_SEARCH_ITEMS = [...@@ -237,6 +237,12 @@ export const SETTINGS_SEARCH_ITEMS = [
237 to: "/settings/integrations",237 to: "/settings/integrations",
238 targetId: "browser",238 targetId: "browser",
239 },239 },
240 {
241 id: "browser-recording-frame-rate",
242 title: "Browser recording frame rate",
243 to: "/settings/integrations",
244 targetId: "browser",
245 },
240 {246 {
241 id: "browser-auto-show-floating-preview",247 id: "browser-auto-show-floating-preview",
242 title: "Auto-show floating preview",248 title: "Auto-show floating preview",
packages/contracts/src/ipc.ts+14-1
...@@ -734,6 +734,19 @@ export const DesktopPreviewRecordingFrameSchema: Schema.Codec<DesktopPreviewReco...@@ -734,6 +734,19 @@ export const DesktopPreviewRecordingFrameSchema: Schema.Codec<DesktopPreviewReco
734 receivedAt: Schema.String,734 receivedAt: Schema.String,
735 });735 });
736736
737export interface DesktopPreviewRecordingSource {
738 sourceId: string;
739 width: number;
740 height: number;
741}
742
743export const DesktopPreviewRecordingSourceSchema: Schema.Codec<DesktopPreviewRecordingSource> =
744 Schema.Struct({
745 sourceId: Schema.String,
746 width: Schema.Int.check(Schema.isGreaterThan(0)),
747 height: Schema.Int.check(Schema.isGreaterThan(0)),
748 });
749
737export interface DesktopPreviewRecordingArtifact {750export interface DesktopPreviewRecordingArtifact {
738 id: string;751 id: string;
739 tabId: string;752 tabId: string;
...@@ -1215,7 +1228,7 @@ export interface DesktopPreviewBridge {...@@ -1215,7 +1228,7 @@ export interface DesktopPreviewBridge {
1215 close: (tabId: string) => Promise<void>;1228 close: (tabId: string) => Promise<void>;
1216 };1229 };
1217 recording: {1230 recording: {
1218 startScreencast: (tabId: string) => Promise<void>;1231 startScreencast: (tabId: string) => Promise<DesktopPreviewRecordingSource>;
1219 stopScreencast: (tabId: string) => Promise<void>;1232 stopScreencast: (tabId: string) => Promise<void>;
1220 save: (1233 save: (
1221 tabId: string,1234 tabId: string,
packages/contracts/src/settings.test.ts+20
...@@ -66,6 +66,26 @@ describe("ClientSettings word wrap", () => {...@@ -66,6 +66,26 @@ describe("ClientSettings word wrap", () => {
66 });66 });
67});67});
6868
69describe("ClientSettings browser recording frame rate", () => {
70 it("defaults to 30 fps", () => {
71 expect(decodeClientSettings({}).browserRecordingFrameRate).toBe(30);
72 });
73
74 it.each([30, 60])("accepts a supported frame rate: %s", (frameRate) => {
75 expect(
76 decodeClientSettings({ browserRecordingFrameRate: frameRate }).browserRecordingFrameRate,
77 ).toBe(frameRate);
78 expect(
79 decodeClientSettingsPatch({ browserRecordingFrameRate: frameRate }).browserRecordingFrameRate,
80 ).toBe(frameRate);
81 });
82
83 it.each([24, 59, 120])("rejects an unsupported frame rate: %s", (frameRate) => {
84 expect(() => decodeClientSettings({ browserRecordingFrameRate: frameRate })).toThrow();
85 expect(() => decodeClientSettingsPatch({ browserRecordingFrameRate: frameRate })).toThrow();
86 });
87});
88
69describe("ClientSettings glass opacity", () => {89describe("ClientSettings glass opacity", () => {
70 it("defaults to a readable translucent surface", () => {90 it("defaults to a readable translucent surface", () => {
71 expect(decodeClientSettings({}).glassOpacity).toBe(80);91 expect(decodeClientSettings({}).glassOpacity).toBe(80);
packages/contracts/src/settings.ts+11-2
...@@ -147,11 +147,16 @@ export type DefaultThemePreference = typeof DefaultThemePreference.Type;...@@ -147,11 +147,16 @@ export type DefaultThemePreference = typeof DefaultThemePreference.Type;
147/**147/**
148 * Defaults for the in-app preview browser, applied whenever a tab is opened148 * Defaults for the in-app preview browser, applied whenever a tab is opened
149 * without an explicit viewport/zoom/appearance — by the user opening a browser149 * without an explicit viewport/zoom/appearance — by the user opening a browser
150 * tab, or by an agent calling `preview_open` with no size. Client-local150 * tab, or by an agent calling `preview_open` with no size. Recording quality is
151 * because the Chromium guest they configure is desktop-local.151 * client-local for the same reason: the Chromium guest being captured belongs
152 * to the desktop app.
152 */153 */
153export const DEFAULT_BROWSER_VIEWPORT: PreviewViewportSetting = FILL_PREVIEW_VIEWPORT;154export const DEFAULT_BROWSER_VIEWPORT: PreviewViewportSetting = FILL_PREVIEW_VIEWPORT;
154export const DEFAULT_BROWSER_AUTO_SHOW_FLOATING_PREVIEW = true;155export const DEFAULT_BROWSER_AUTO_SHOW_FLOATING_PREVIEW = true;
156export const BROWSER_RECORDING_FRAME_RATES = [30, 60] as const;
157export const BrowserRecordingFrameRate = Schema.Literals(BROWSER_RECORDING_FRAME_RATES);
158export type BrowserRecordingFrameRate = typeof BrowserRecordingFrameRate.Type;
159export const DEFAULT_BROWSER_RECORDING_FRAME_RATE: BrowserRecordingFrameRate = 30;
155160
156export const ClientSettingsSchema = Schema.Struct({161export const ClientSettingsSchema = Schema.Struct({
157 appearanceContrast: AppearanceContrast.pipe(162 appearanceContrast: AppearanceContrast.pipe(
...@@ -166,6 +171,9 @@ export const ClientSettingsSchema = Schema.Struct({...@@ -166,6 +171,9 @@ export const ClientSettingsSchema = Schema.Struct({
166 browserDefaultAppearance: PreviewAppearancePreference.pipe(171 browserDefaultAppearance: PreviewAppearancePreference.pipe(
167 Schema.withDecodingDefault(Effect.succeed(DEFAULT_PREVIEW_APPEARANCE)),172 Schema.withDecodingDefault(Effect.succeed(DEFAULT_PREVIEW_APPEARANCE)),
168 ),173 ),
174 browserRecordingFrameRate: BrowserRecordingFrameRate.pipe(
175 Schema.withDecodingDefault(Effect.succeed(DEFAULT_BROWSER_RECORDING_FRAME_RATE)),
176 ),
169 /**177 /**
170 * Whether an agent opening a preview pops the floating mini player into178 * Whether an agent opening a preview pops the floating mini player into
171 * view. Only applies when the agent didn't ask either way — an explicit179 * view. Only applies when the agent didn't ask either way — an explicit
...@@ -926,6 +934,7 @@ export const ClientSettingsPatch = Schema.Struct({...@@ -926,6 +934,7 @@ export const ClientSettingsPatch = Schema.Struct({
926 browserDefaultViewport: Schema.optionalKey(PreviewViewportSetting),934 browserDefaultViewport: Schema.optionalKey(PreviewViewportSetting),
927 browserDefaultZoomFactor: Schema.optionalKey(PreviewZoomFactor),935 browserDefaultZoomFactor: Schema.optionalKey(PreviewZoomFactor),
928 browserDefaultAppearance: Schema.optionalKey(PreviewAppearancePreference),936 browserDefaultAppearance: Schema.optionalKey(PreviewAppearancePreference),
937 browserRecordingFrameRate: Schema.optionalKey(BrowserRecordingFrameRate),
929 browserAutoShowFloatingPreview: Schema.optionalKey(Schema.Boolean),938 browserAutoShowFloatingPreview: Schema.optionalKey(Schema.Boolean),
930 confirmQuit: Schema.optionalKey(Schema.Boolean),939 confirmQuit: Schema.optionalKey(Schema.Boolean),
931 confirmThreadArchive: Schema.optionalKey(Schema.Boolean),940 confirmThreadArchive: Schema.optionalKey(Schema.Boolean),