From 55c24273a025874cdb8a96a25ee0c59eeb4e3051 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sun, 20 Sep 2026 13:48:27 -0700 Subject: [PATCH] fix(desktop): align preview recording cursors and show input feedback (#12779) --- apps/desktop/src/ipc/channels.ts | 2 + apps/desktop/src/ipc/methods/preview.ts | 25 +- apps/desktop/src/preload.ts | 10 + apps/desktop/src/preview/GuestProtocol.ts | 5 + apps/desktop/src/preview/Manager.test.ts | 350 ++++++++++++++---- apps/desktop/src/preview/Manager.ts | 154 +++++++- apps/desktop/src/preview/PickPreload.ts | 82 ++++ apps/desktop/src/preview/RecordingCursor.ts | 197 ++++++++++ .../src/preview/RecordingInput.test.ts | 51 +++ apps/desktop/src/preview/RecordingInput.ts | 58 +++ apps/web/src/browser/browserRecording.ts | 25 +- .../src/browser/recordingCompositor.test.ts | 179 +++++++++ apps/web/src/browser/recordingCompositor.ts | 190 ++++++++++ .../components/preview/AgentBrowserCursor.tsx | 8 +- .../components/preview/PreviewView.test.tsx | 43 ++- .../src/components/preview/PreviewView.tsx | 11 +- .../settings/IntegrationsSettings.tsx | 39 ++ .../settings/SettingsPanels.logic.test.ts | 4 + .../settings/SettingsPanels.logic.ts | 4 + .../components/settings/SettingsPanels.tsx | 4 + .../src/components/settings/settingsSearch.ts | 12 + packages/contracts/src/ipc.ts | 25 ++ packages/contracts/src/settings.test.ts | 17 + packages/contracts/src/settings.ts | 8 + 24 files changed, 1397 insertions(+), 106 deletions(-) create mode 100644 apps/desktop/src/preview/RecordingCursor.ts create mode 100644 apps/desktop/src/preview/RecordingInput.test.ts create mode 100644 apps/desktop/src/preview/RecordingInput.ts create mode 100644 apps/web/src/browser/recordingCompositor.test.ts create mode 100644 apps/web/src/browser/recordingCompositor.ts diff --git a/apps/desktop/src/ipc/channels.ts b/apps/desktop/src/ipc/channels.ts index 226793657848940d45ffea840547c9d7283075fb..f49b6dbd968e364e3ae75a81a173f54da0c6ea92 100644 --- a/apps/desktop/src/ipc/channels.ts +++ b/apps/desktop/src/ipc/channels.ts @@ -113,3 +113,5 @@ export const PREVIEW_POINTER_EVENT_CHANNEL = "desktop:preview-pointer-event"; export const MAC_PERMISSION_HELPER_CHANNEL = "desktop:mac-permission-helper"; export const CHECK_SYSTEM_PERMISSION_CHANNEL = "desktop:check-system-permission"; + +export const PREVIEW_RECORDING_INPUT_CHANNEL = "desktop:preview-recording-input"; diff --git a/apps/desktop/src/ipc/methods/preview.ts b/apps/desktop/src/ipc/methods/preview.ts index 5fb7eff99fc6790e342513b23dc52bd65ebcfaa6..36cf7c9abe153c226adeed42c6f2b7878d0d142e 100644 --- a/apps/desktop/src/ipc/methods/preview.ts +++ b/apps/desktop/src/ipc/methods/preview.ts @@ -30,11 +30,13 @@ import { } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; import * as Schema from "effect/Schema"; +import * as Option from "effect/Option"; import * as NodeURL from "node:url"; import * as ElectronWindow from "../../electron/ElectronWindow.ts"; import * as BrowserImport from "../../preview/BrowserImport/BrowserImport.ts"; import * as PreviewManager from "../../preview/Manager.ts"; +import * as DesktopClientSettings from "../../settings/DesktopClientSettings.ts"; import { PREVIEW_WEBVIEW_PREFERENCES } from "../../preview/WebviewPreferences.ts"; import * as IpcChannels from "../channels.ts"; import * as DesktopIpc from "../DesktopIpc.ts"; @@ -50,6 +52,9 @@ export const installPreviewEventForwarding = Effect.fn( yield* manager.subscribeRecordingFrames((frame) => electronWindow.sendAll(IpcChannels.PREVIEW_RECORDING_FRAME_CHANNEL, frame), ); + yield* manager.subscribeRecordingInputs((event) => + electronWindow.sendAll(IpcChannels.PREVIEW_RECORDING_INPUT_CHANNEL, event), + ); yield* manager.subscribePointerEvents((event) => electronWindow.sendAll(IpcChannels.PREVIEW_POINTER_EVENT_CHANNEL, event), ); @@ -180,11 +185,21 @@ export const cancelPickElement = tabMethod( "desktop.ipc.preview.cancelPickElement", (manager, tabId) => manager.cancelPickElement(tabId), ); -export const startRecording = tabMethod( - IpcChannels.PREVIEW_RECORDING_START_CHANNEL, - "desktop.ipc.preview.startRecording", - (manager, tabId) => manager.startRecording(tabId), -); +export const startRecording = DesktopIpc.makeIpcMethod({ + channel: IpcChannels.PREVIEW_RECORDING_START_CHANNEL, + payload: DesktopPreviewTabInputSchema, + result: Schema.Void, + handler: Effect.fn("desktop.ipc.preview.startRecording")(function* ({ tabId }) { + const manager = yield* PreviewManager.PreviewManager; + const store = yield* DesktopClientSettings.DesktopClientSettings; + const settings = yield* store.get; + const options = Option.map(settings, (value) => ({ + showKeyPresses: value.browserRecordingShowKeyPresses, + showMousePresses: value.browserRecordingShowMousePresses, + })); + yield* manager.startRecording(tabId, Option.getOrUndefined(options)); + }), +}); export const stopRecording = tabMethod( IpcChannels.PREVIEW_RECORDING_STOP_CHANNEL, "desktop.ipc.preview.stopRecording", diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.ts index 885b31a64d6b6f452f20e4959e50fbd9bfeaee29..7256f610fbebf24517ff7c3a7d6d49f05dce11d3 100644 --- a/apps/desktop/src/preload.ts +++ b/apps/desktop/src/preload.ts @@ -1,6 +1,7 @@ import type { DesktopBridge, DesktopPreviewPointerEvent, + DesktopPreviewRecordingInputEvent, DesktopPreviewRecordingFrame, DesktopPreviewTabState, DesktopSnapShotEvent, @@ -327,6 +328,15 @@ contextBridge.exposeInMainWorld("desktopBridge", { ipcRenderer.invoke(IpcChannels.PREVIEW_PICTURE_IN_PICTURE_CLOSE_CHANNEL, { tabId }), }, recording: { + onInput: (listener) => { + const wrappedListener = (_event: Electron.IpcRendererEvent, event: unknown) => { + if (typeof event !== "object" || event === null) return; + listener(event as DesktopPreviewRecordingInputEvent); + }; + ipcRenderer.on(IpcChannels.PREVIEW_RECORDING_INPUT_CHANNEL, wrappedListener); + return () => + ipcRenderer.removeListener(IpcChannels.PREVIEW_RECORDING_INPUT_CHANNEL, wrappedListener); + }, startScreencast: (tabId) => ipcRenderer.invoke(IpcChannels.PREVIEW_RECORDING_START_CHANNEL, { tabId }), stopScreencast: (tabId) => diff --git a/apps/desktop/src/preview/GuestProtocol.ts b/apps/desktop/src/preview/GuestProtocol.ts index e63597b71efc11b2f04ab4948b1576c62ab28d8a..1a73bb30f29e707668d41c8b7455edc11e15230a 100644 --- a/apps/desktop/src/preview/GuestProtocol.ts +++ b/apps/desktop/src/preview/GuestProtocol.ts @@ -5,3 +5,8 @@ export const ANNOTATION_CAPTURED_CHANNEL = "preview:annotation-captured"; export const ANNOTATION_THEME_CHANNEL = "preview:annotation-theme"; export const HUMAN_INPUT_CHANNEL = "preview:human-input"; export const MOUSE_NAVIGATE_CHANNEL = "preview:mouse-navigate"; +export const RECORDING_CURSOR_CHANNEL = "preview:recording-cursor"; +export const RECORDING_POINTER_CHANNEL = "preview:recording-pointer"; +export const RECORDING_KEY_CHANNEL = "preview:recording-key"; +export const RECORDING_INPUT_CHANNEL = "preview:recording-input"; +export const RECORDING_CONTROLLER_CHANNEL = "preview:recording-controller"; diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index 7b76a1b8003ab4ddbb94e045abe1c94bf2f4f80a..304efeafefe9258e173e81da0566489d79cdf63d 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -1,7 +1,10 @@ import * as NodeVM from "node:vm"; import { it as effectIt } from "@effect/vitest"; import { DESKTOP_PREVIEW_RECORDING_CAPTURE_TRIGGER } from "@t3tools/contracts"; -import type { DesktopPreviewRecordingFrame } from "@t3tools/contracts"; +import type { + DesktopPreviewRecordingFrame, + DesktopPreviewRecordingInputEvent, +} from "@t3tools/contracts"; import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; import * as Cause from "effect/Cause"; import * as Deferred from "effect/Deferred"; @@ -2885,6 +2888,165 @@ describe("PreviewManager", () => { ), ); + effectIt.effect( + "restores the native cursor when recording startup fails, then allows a retry", + () => + withManager((manager) => + Effect.gen(function* () { + const host = makeTestHostWebContents(); + host.executeJavaScript.mockResolvedValueOnce(false); + let cursorActive = false; + const cursorAtCapture: boolean[] = []; + const contents = Object.assign( + makeTestPreviewWebContents( + async () => { + cursorAtCapture.push(cursorActive); + return { + toJPEG: () => Buffer.from("frame"), + getSize: () => ({ width: 800, height: 600 }), + }; + }, + 42, + host, + ), + { + send: (channel: string, active: unknown) => { + if (channel === "preview:recording-cursor") cursorActive = active === true; + }, + }, + ); + fromId.mockReturnValue(contents as never); + yield* manager.createTab("tab_cursor"); + yield* manager.registerWebview("tab_cursor", 42); + const failed = yield* Effect.exit(manager.startRecording("tab_cursor")); + expect(Exit.isFailure(failed)).toBe(true); + expect(cursorAtCapture).toEqual([true]); + expect(cursorActive).toBe(false); + + yield* manager.startRecording("tab_cursor"); + expect(cursorAtCapture).toEqual([true, true]); + expect(cursorActive).toBe(true); + yield* manager.stopRecording("tab_cursor"); + expect(cursorActive).toBe(false); + }), + ), + ); + + effectIt.effect("restores the recording cursor after navigation only while recording", () => + withManager((manager) => + Effect.gen(function* () { + const listeners = new Map void>(); + let cursorActive = false; + let cursorUpdated: (() => void) | undefined; + let inputOptions: unknown; + const options = { showKeyPresses: true, showMousePresses: false }; + const contents = Object.assign( + makeTestPreviewWebContents(async () => ({ + toJPEG: () => Buffer.from("frame"), + getSize: () => ({ width: 800, height: 600 }), + })), + { + on: (event: string, listener: () => void) => listeners.set(event, listener), + send: (channel: string, active: unknown, recordingOptions: unknown) => { + if (channel !== "preview:recording-cursor") return; + cursorActive = active === true; + inputOptions = recordingOptions; + cursorUpdated?.(); + }, + }, + ); + fromId.mockReturnValue(contents as never); + yield* manager.createTab("tab_cursor_reload"); + yield* manager.registerWebview("tab_cursor_reload", 42); + yield* manager.startRecording("tab_cursor_reload", options); + for (const recording of [true, false]) { + if (!recording) yield* manager.stopRecording("tab_cursor_reload"); + // A new document has lost the previous preload's cursor overlay. + cursorActive = false; + const restored = new Promise((resolve) => { + cursorUpdated = resolve; + }); + listeners.get("dom-ready")?.(); + yield* Effect.promise(() => restored); + cursorUpdated = undefined; + expect(cursorActive).toBe(recording); + expect(inputOptions).toEqual(recording ? options : undefined); + } + }), + ), + ); + + effectIt.effect("gates recording decorations and isolates failed subscribers", () => + withManager((manager) => + Effect.gen(function* () { + const callbacks = new Map< + string, + (event: unknown, input: unknown) => Fiber.Fiber | undefined + >(); + const contents = Object.assign( + makeTestPreviewWebContents(async () => ({ + toJPEG: () => Buffer.from("frame"), + getSize: () => ({ width: 800, height: 600 }), + })), + { + ipc: { + on: ( + channel: string, + callback: (event: unknown, input: unknown) => Fiber.Fiber | undefined, + ) => callbacks.set(channel, callback), + off: vi.fn(), + }, + }, + ); + fromId.mockReturnValue(contents as never); + yield* manager.createTab("tab_recording_input"); + yield* manager.registerWebview("tab_recording_input", 42); + const received: DesktopPreviewRecordingInputEvent[] = []; + yield* manager.subscribeRecordingInputs(() => Effect.die("renderer unavailable")); + yield* manager.subscribeRecordingInputs((event) => + Effect.sync(() => { + received.push(event); + }), + ); + const send = (input: unknown) => + Effect.gen(function* () { + const fiber = callbacks.get("preview:recording-input")?.(null, input); + if (fiber) yield* Fiber.join(fiber); + }); + const key = { type: "key", label: "⌘C", held: true, width: 800 }; + const pointer = { + type: "pointer", + phase: "down", + x: 120, + y: 80, + width: 800, + height: 600, + }; + yield* send(key); + expect(received).toEqual([]); + yield* manager.startRecording("tab_recording_input", { + showKeyPresses: true, + showMousePresses: false, + }); + yield* send(key); + yield* send(pointer); + yield* send({ ...key, width: 0 }); + expect(received).toEqual([{ tabId: "tab_recording_input", input: key }]); + yield* manager.stopRecording("tab_recording_input"); + yield* send(key); + expect(received).toHaveLength(1); + yield* manager.startRecording("tab_recording_input", { + showKeyPresses: false, + showMousePresses: true, + }); + yield* send(key); + yield* send(pointer); + expect(received.at(-1)).toEqual({ tabId: "tab_recording_input", input: pointer }); + expect(received).toHaveLength(2); + }), + ), + ); + effectIt.effect("continues native recording when the source warmup fails", () => withManager((manager) => Effect.gen(function* () { @@ -3853,88 +4015,108 @@ describe("PreviewManager", () => { ), ); - effectIt.effect("emits the resolved pointer target before dispatching an automation click", () => - withManager((manager) => - Effect.gen(function* () { - let humanInput: ((_event: unknown, signal: unknown) => void) | undefined; - const activity: string[] = []; - const sendCommand = vi.fn(async (method: string, params?: Record) => { - if (method === "Runtime.evaluate") { - return { - result: { - value: { width: 800, height: 600 }, - }, - }; - } - if (method === "Input.dispatchMouseEvent" && params?.type === "mousePressed") { - activity.push("mousePressed"); - humanInput?.({}, { kind: "pointer", x: params.x, y: params.y, button: 0 }); - } - return undefined; - }); - fromId.mockReturnValue({ - id: 42, - isDestroyed: () => false, - getType: () => "webview", - getURL: () => "https://example.com", - getTitle: () => "Example", - isLoading: () => false, - isDevToolsOpened: () => false, - getZoomFactor: () => 1, - setZoomFactor: vi.fn(), - setAudioMuted: vi.fn(), - isCurrentlyAudible: () => false, - on: vi.fn(), - off: vi.fn(), - ipc: { - on: vi.fn((channel: string, listener: typeof humanInput) => { - if (channel === "preview:human-input") humanInput = listener; - }), - off: vi.fn(), - }, - send: webviewSend, - navigationHistory: { canGoBack: () => false, canGoForward: () => false }, - setIgnoreMenuShortcuts: vi.fn(), - setWindowOpenHandler: vi.fn(), - debugger: { - isAttached: () => false, - attach: vi.fn(), - sendCommand, + effectIt.effect( + "records the resolved pointer target before dispatching an automation click", + () => + withManager((manager) => + Effect.gen(function* () { + let humanInput: ((_event: unknown, signal: unknown) => void) | undefined; + const activity: string[] = []; + const sendCommand = vi.fn(async (method: string, params?: Record) => { + if (method === "Runtime.evaluate") { + return { + result: { + value: { width: 800, height: 600 }, + }, + }; + } + if (method === "Input.dispatchMouseEvent" && params?.type === "mousePressed") { + activity.push("mousePressed"); + humanInput?.({}, { kind: "pointer", x: params.x, y: params.y, button: 0 }); + } + return undefined; + }); + fromId.mockReturnValue({ + id: 42, + hostWebContents: makeTestHostWebContents(), + capturePage: vi.fn(async () => ({ toPNG: () => Buffer.from("frame") })), + setBackgroundThrottling: vi.fn(), + isDestroyed: () => false, + getType: () => "webview", + getURL: () => "https://example.com", + getTitle: () => "Example", + isLoading: () => false, + isDevToolsOpened: () => false, + getZoomFactor: () => 1, + setZoomFactor: vi.fn(), + setAudioMuted: vi.fn(), + isCurrentlyAudible: () => false, on: vi.fn(), off: vi.fn(), - }, - } as never); + ipc: { + on: vi.fn((channel: string, listener: typeof humanInput) => { + if (channel === "preview:human-input") humanInput = listener; + }), + off: vi.fn(), + }, + send: webviewSend, + navigationHistory: { canGoBack: () => false, canGoForward: () => false }, + setIgnoreMenuShortcuts: vi.fn(), + setWindowOpenHandler: vi.fn(), + debugger: { + isAttached: () => false, + attach: vi.fn(), + sendCommand, + on: vi.fn(), + off: vi.fn(), + }, + } as never); - yield* manager.subscribePointerEvents((event) => - Effect.sync(() => { - activity.push(event.phase); - }), - ); - yield* manager.createTab("tab_1"); - yield* manager.registerWebview("tab_1", 42); - const click = yield* manager - .automationClick("tab_1", { x: 120, y: 80 }) - .pipe(Effect.forkChild({ startImmediately: true })); - yield* TestClock.adjust(200); - yield* Fiber.join(click); + yield* manager.subscribePointerEvents((event) => + Effect.sync(() => { + activity.push(event.phase); + }), + ); + yield* manager.createTab("tab_1"); + yield* manager.registerWebview("tab_1", 42); + yield* manager.startRecording("tab_1"); + const click = yield* manager + .automationClick("tab_1", { x: 120, y: 80 }) + .pipe(Effect.forkChild({ startImmediately: true })); + yield* TestClock.adjust(200); + yield* Fiber.join(click); - expect(activity).toEqual(["move", "click", "mousePressed"]); - expect(sendCommand).toHaveBeenCalledWith("Input.dispatchMouseEvent", { - type: "mousePressed", - x: 120, - y: 80, - button: "left", - clickCount: 1, - }); - expect(sendCommand).toHaveBeenCalledWith("Input.dispatchMouseEvent", { - type: "mouseReleased", - x: 120, - y: 80, - button: "left", - clickCount: 1, - }); - }), - ), + expect(activity).toEqual(["move", "click", "mousePressed"]); + expect( + webviewSend.mock.calls + .filter(([channel]) => channel === "preview:recording-controller") + .map(([, controller]) => controller), + ).toEqual(["agent", "none"]); + + const recordedPointer = webviewSend.mock.calls + .filter(([channel]) => channel === "preview:recording-pointer") + .map(([, event]) => event); + expect(recordedPointer).toEqual([ + expect.objectContaining({ phase: "move", x: 120, y: 80 }), + expect.objectContaining({ phase: "click", x: 120, y: 80 }), + ]); + expect(sendCommand).toHaveBeenCalledWith("Input.dispatchMouseEvent", { + type: "mousePressed", + x: 120, + y: 80, + button: "left", + clickCount: 1, + }); + yield* manager.stopRecording("tab_1"); + expect(sendCommand).toHaveBeenCalledWith("Input.dispatchMouseEvent", { + type: "mouseReleased", + x: 120, + y: 80, + button: "left", + clickCount: 1, + }); + }), + ), ); effectIt.effect("types in background webviews and enables native key input", () => @@ -4230,6 +4412,9 @@ describe("PreviewManager", () => { }); fromId.mockReturnValue({ id: 42, + hostWebContents: makeTestHostWebContents(), + capturePage: vi.fn(async () => ({ toPNG: () => Buffer.from("frame") })), + setBackgroundThrottling: vi.fn(), isDestroyed: () => false, getType: () => "webview", getURL: () => "https://example.com", @@ -4263,6 +4448,7 @@ describe("PreviewManager", () => { yield* manager.createTab("tab_1"); yield* manager.registerWebview("tab_1", 42); + yield* manager.startRecording("tab_1"); const click = yield* manager .automationClick("tab_1", { x: 120, y: 80 }) @@ -4270,6 +4456,12 @@ describe("PreviewManager", () => { yield* TestClock.adjust(200); const exit = yield* Fiber.await(click); expect(Exit.isFailure(exit)).toBe(true); + expect( + webviewSend.mock.calls + .filter(([channel]) => channel === "preview:recording-controller") + .map(([, controller]) => controller), + ).toEqual(["agent", "human", "none"]); + yield* manager.stopRecording("tab_1"); if (Exit.isSuccess(exit)) return; const error = Option.getOrThrow(Cause.findErrorOption(exit.cause)); expect(error).toMatchObject({ diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index a2e34fe547360f30dee5271579ee00212fcd126c..468c470653154dd354b8951a3479e37a356249df 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -6,7 +6,10 @@ * here). Single layer-scoped browser session partition. */ import * as NodeCrypto from "node:crypto"; -import { DESKTOP_PREVIEW_RECORDING_CAPTURE_TRIGGER } from "@t3tools/contracts"; +import { + DesktopPreviewRecordingInputSchema, + DESKTOP_PREVIEW_RECORDING_CAPTURE_TRIGGER, +} from "@t3tools/contracts"; import type { DesktopPreviewAnnotationTheme, DesktopPreviewAutomationStatus, @@ -18,6 +21,7 @@ import type { PreviewAnnotationSubmissionResult, DesktopPreviewRecordingArtifact, DesktopPreviewRecordingFrame, + DesktopPreviewRecordingInputEvent, DesktopPreviewScreenshotArtifact, DesktopPreviewTabDefaults, PreviewAutomationClickInput, @@ -71,6 +75,11 @@ import { ELEMENT_PICKED_CHANNEL, HUMAN_INPUT_CHANNEL, MOUSE_NAVIGATE_CHANNEL, + RECORDING_CURSOR_CHANNEL, + RECORDING_POINTER_CHANNEL, + RECORDING_KEY_CHANNEL, + RECORDING_INPUT_CHANNEL, + RECORDING_CONTROLLER_CHANNEL, START_PICK_CHANNEL, } from "./GuestProtocol.ts"; import { isPreviewAnnotationPayload } from "./PickedElementPayload.ts"; @@ -81,6 +90,7 @@ import { previewAutomationEditingCommandExpression, } from "./PreviewKeyboard.ts"; import { captureFavicon, safeHttpOrigin, selectFaviconCandidates } from "./FaviconCapture.ts"; +import { DEFAULT_RECORDING_INPUT_OPTIONS, type RecordingInputOptions } from "./RecordingInput.ts"; export type PreviewNavStatus = | { kind: "Idle" } @@ -437,6 +447,7 @@ interface ManagedListeners { type FrameCaptureConsumer = "picture-in-picture" | "recording"; interface FrameCaptureSession { + readonly recordingInputOptions?: RecordingInputOptions; readonly scope: Scope.Closeable | null; readonly consumers: ReadonlySet; readonly unthrottledWebContentsIds: ReadonlySet; @@ -485,6 +496,10 @@ interface BrowserDiagnostics { readonly requests: ReadonlyMap; } +const isRecordingInput = Schema.is(DesktopPreviewRecordingInputSchema); + +type RecordingInputListener = (event: DesktopPreviewRecordingInputEvent) => Effect.Effect; + type PointerEventListener = (event: DesktopPreviewPointerEvent) => Effect.Effect; interface ExpectedAgentInput { @@ -636,6 +651,9 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const attachedRef = yield* Ref.make>(new Map()); const listenersRef = yield* Ref.make>(new Set()); const pointerEventListenersRef = yield* Ref.make>(new Set()); + const recordingInputListenersRef = yield* Ref.make>( + new Set(), + ); const recordingFrameListenersRef = yield* Ref.make>( new Set(), ); @@ -821,6 +839,20 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function return Effect.succeed([undefined, sessions] as const); } return setFrameCaptureWebContentsBackgroundThrottling(wc, false).pipe( + Effect.tap(() => + Effect.gen(function* () { + if (!current.consumers.has("recording")) return; + const tab = (yield* SynchronizedRef.get(tabsRef)).get(tabId); + yield* attempt({ operation: "recording.cursor", tabId, webContentsId: wc.id }, () => + wc.send( + RECORDING_CURSOR_CHANNEL, + true, + current.recordingInputOptions, + tab?.controller, + ), + ); + }), + ), Effect.map( () => [ @@ -846,6 +878,15 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function if (!current || !current.consumers.has(consumer)) { return [undefined, sessions] as const; } + if (consumer === "recording") { + yield* Effect.forEach(current.unthrottledWebContentsIds, (id) => + attempt({ operation: "recording.cursor", tabId, webContentsId: id }, () => { + const contents = webContents.fromId(id); + if (contents && !contents.isDestroyed()) + contents.send(RECORDING_CURSOR_CHANNEL, false); + }).pipe(Effect.ignore), + ); + } const consumers = new Set(current.consumers); consumers.delete(consumer); if (consumers.size > 0) { @@ -896,7 +937,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }); const deliverEvent = ( - eventKind: "state-change" | "recording-frame" | "pointer-event", + eventKind: "state-change" | "recording-frame" | "recording-input" | "pointer-event", tabId: string, delivery: () => Effect.Effect, ) => @@ -933,6 +974,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const update = Effect.fn("PreviewManager.update")(function* ( tabId: string, patch: Partial, + humanPoint?: { readonly x: number; readonly y: number }, ) { const updatedAt = yield* currentIso; const next = yield* SynchronizedRef.modify(tabsRef, (tabs) => { @@ -950,7 +992,20 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function // can commit between the modify above and here, and republishing this // snapshot would roll the UI back to a value that writer will not send // again because it suppresses unchanged audibility. - if (Option.isSome(next)) yield* emitIfCurrent(tabId, next.value); + if (Option.isSome(next)) { + if (patch.controller !== undefined && next.value.webContentsId != null) { + const capture = (yield* SynchronizedRef.get(frameCaptureSessionsRef)).get(tabId); + const webContentsId = next.value.webContentsId; + if (capture?.consumers.has("recording")) { + yield* attempt({ operation: "recording.controller", tabId }, () => { + const contents = webContents.fromId(webContentsId); + if (contents && !contents.isDestroyed()) + contents.send(RECORDING_CONTROLLER_CHANNEL, patch.controller, humanPoint); + }).pipe(Effect.ignore); + } + } + yield* emitIfCurrent(tabId, next.value); + } }); /** @@ -1728,6 +1783,21 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const sync = () => runFork(syncState(true)); const syncNavigation = () => runFork(syncState(false, true)); const syncInPageNavigation = () => runFork(syncState(false)); + const restoreRecordingCursor = () => + runFork( + Effect.gen(function* () { + const session = (yield* SynchronizedRef.get(frameCaptureSessionsRef)).get(tabId); + if (!wc.isDestroyed()) { + const tab = (yield* SynchronizedRef.get(tabsRef)).get(tabId); + wc.send( + RECORDING_CURSOR_CHANNEL, + session?.consumers.has("recording") ?? false, + session?.recordingInputOptions, + tab?.controller, + ); + } + }), + ); const navigationStarted = ( event: Electron.Event, ) => { @@ -1860,13 +1930,37 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function copy.set(tabId, (epochs.get(tabId) ?? 0) + 1); }), ); - yield* update(tabId, { controller: "human" }); + yield* update( + tabId, + { controller: "human" }, + isPreviewInputSignal(rawSignal) && rawSignal.kind === "pointer" + ? { x: rawSignal.x, y: rawSignal.y } + : undefined, + ); yield* Effect.sleep(750); const tabs = yield* SynchronizedRef.get(tabsRef); if (tabs.get(tabId)?.controller === "human") { yield* update(tabId, { controller: "none" }); } }); + const recordingInput = (_event: unknown, input: unknown) => { + if (!isRecordingInput(input)) return; + return runFork( + Effect.gen(function* () { + const tab = (yield* SynchronizedRef.get(tabsRef)).get(tabId); + const capture = (yield* SynchronizedRef.get(frameCaptureSessionsRef)).get(tabId); + if (tab?.webContentsId !== wc.id || !capture?.consumers.has("recording")) return; + if (input.type === "key" && !capture.recordingInputOptions?.showKeyPresses) return; + if (input.type === "pointer" && !capture.recordingInputOptions?.showMousePresses) return; + const listeners = yield* Ref.get(recordingInputListenersRef); + yield* Effect.forEach( + listeners, + (listener) => deliverEvent("recording-input", tabId, () => listener({ tabId, input })), + { discard: true }, + ); + }), + ); + }; const humanInput = (_event: unknown, rawSignal?: unknown): void => { runFork(handleHumanInput(rawSignal)); }; @@ -1928,11 +2022,13 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function wc.off("page-favicon-updated", faviconUpdated as never); wc.off("did-start-loading", sync); wc.off("did-stop-loading", sync); + wc.off("dom-ready", restoreRecordingCursor); wc.off("did-fail-load", failed as never); wc.off("audio-state-changed", audioStateChanged); wc.off("did-create-window", windowCreated); wc.off("before-input-event", beforeInput); wc.ipc.off(HUMAN_INPUT_CHANNEL, humanInput); + wc.ipc.off(RECORDING_INPUT_CHANNEL, recordingInput); wc.ipc.off(MOUSE_NAVIGATE_CHANNEL, mouseNavigate); }).pipe(Effect.ignore), ); @@ -1948,9 +2044,11 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function wc.on("page-favicon-updated", faviconUpdated as never); wc.on("did-start-loading", sync); wc.on("did-stop-loading", sync); + wc.on("dom-ready", restoreRecordingCursor); wc.on("did-fail-load", failed as never); wc.on("audio-state-changed", audioStateChanged); wc.ipc.on(HUMAN_INPUT_CHANNEL, humanInput); + wc.ipc.on(RECORDING_INPUT_CHANNEL, recordingInput); wc.ipc.on(MOUSE_NAVIGATE_CHANNEL, mouseNavigate); wc.setWindowOpenHandler((details) => { if (previewWindowOpenAction(details) === "popup") { @@ -3405,7 +3503,10 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }); }; - const startRecording = Effect.fn("PreviewManager.startRecording")(function* (tabId: string) { + const startRecording = Effect.fn("PreviewManager.startRecording")(function* ( + tabId: string, + options: RecordingInputOptions = DEFAULT_RECORDING_INPUT_OPTIONS, + ) { if ((yield* Ref.get(closingTabIdsRef)).has(tabId)) { return yield* new PreviewTabNotFoundError({ tabId }); } @@ -3413,11 +3514,21 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function tabId, Effect.gen(function* () { yield* startFrameCapture(tabId, "recording"); + yield* SynchronizedRef.update(frameCaptureSessionsRef, (sessions) => + replaceMap(sessions, (copy) => { + const current = copy.get(tabId); + if (current) copy.set(tabId, { ...current, recordingInputOptions: options }); + }), + ); const wc = yield* requireWebContents(tabId); const requestWebContents = wc.hostWebContents; if (requestWebContents === null) { return yield* new PreviewMainWindowClosedError({ tabId }); } + const tab = (yield* SynchronizedRef.get(tabsRef)).get(tabId); + yield* attempt({ operation: "recording.cursor", tabId, webContentsId: wc.id }, () => + wc.send(RECORDING_CURSOR_CHANNEL, true, options, tab?.controller), + ); yield* attemptPromise( { operation: "recording.warmSource", @@ -3720,6 +3831,15 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const emitPointerEvent = Effect.fn("PreviewManager.emitPointerEvent")(function* ( event: DesktopPreviewPointerEvent, ) { + const recording = (yield* SynchronizedRef.get(frameCaptureSessionsRef)).get(event.tabId); + const tab = (yield* SynchronizedRef.get(tabsRef)).get(event.tabId); + const webContentsId = tab?.webContentsId; + if (recording?.consumers.has("recording") && webContentsId != null) { + yield* attempt({ operation: "recording.pointer", tabId: event.tabId }, () => { + const contents = webContents.fromId(webContentsId); + if (contents && !contents.isDestroyed()) contents.send(RECORDING_POINTER_CHANNEL, event); + }); + } const listeners = yield* Ref.get(pointerEventListenersRef); yield* Effect.forEach( listeners, @@ -4110,6 +4230,18 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const keySequence = makePreviewAutomationNativeKeySequence(input, { isMac: hostPlatform === "darwin", }); + const recording = (yield* SynchronizedRef.get(frameCaptureSessionsRef)).get(tabId); + if (recording?.consumers.has("recording") && recording.recordingInputOptions?.showKeyPresses) { + yield* attempt({ operation: "recording.key", tabId, webContentsId: wc.id }, () => + wc.send(RECORDING_KEY_CHANNEL, { + key: keySequence.signal.key || input.key, + metaKey: input.modifiers?.includes("Meta") ?? false, + ctrlKey: input.modifiers?.includes("Control") ?? false, + altKey: input.modifiers?.includes("Alt") ?? false, + shiftKey: input.modifiers?.includes("Shift") ?? false, + }), + ); + } // CDP keyboard dispatch follows the embedder's focused renderer, and // WebContents.focus() is a no-op for webview guests. Native input targets // this guest's widget directly, so Enter cannot submit the host composer. @@ -4470,6 +4602,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function Ref.set(expectedAgentInputsRef, new Map()), Ref.set(pointerEventListenersRef, new Set()), Ref.set(recordingFrameListenersRef, new Set()), + Ref.set(recordingInputListenersRef, new Set()), ], { discard: true }, ); @@ -4513,6 +4646,8 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function stopRecording, subscribePointerEvents: (listener: PointerEventListener) => subscribe(pointerEventListenersRef, listener), + subscribeRecordingInputs: (listener: RecordingInputListener) => + subscribe(recordingInputListenersRef, listener), subscribeRecordingFrames: (listener: RecordingFrameListener) => subscribe(recordingFrameListenersRef, listener), subscribeStateChanges: (listener: Listener) => subscribe(listenersRef, listener), @@ -4877,7 +5012,10 @@ export class PreviewManager extends Context.Service< readonly copyArtifactToClipboard: (path: string) => Effect.Effect; readonly openPictureInPicture: (tabId: string) => Effect.Effect; readonly closePictureInPicture: (tabId: string) => Effect.Effect; - readonly startRecording: (tabId: string) => Effect.Effect; + readonly startRecording: ( + tabId: string, + options?: RecordingInputOptions, + ) => Effect.Effect; readonly stopRecording: (tabId: string) => Effect.Effect; readonly saveRecording: ( tabId: string, @@ -4918,6 +5056,9 @@ export class PreviewManager extends Context.Service< readonly subscribePointerEvents: ( listener: PointerEventListener, ) => Effect.Effect; + readonly subscribeRecordingInputs: ( + listener: RecordingInputListener, + ) => Effect.Effect; readonly subscribeRecordingFrames: ( listener: RecordingFrameListener, ) => Effect.Effect; @@ -5011,6 +5152,7 @@ export const make = Effect.gen(function* PreviewManagerMake() { subscribeStateChanges: operations.subscribeStateChanges, subscribePointerEvents: operations.subscribePointerEvents, subscribeRecordingFrames: operations.subscribeRecordingFrames, + subscribeRecordingInputs: operations.subscribeRecordingInputs, }); }).pipe(Effect.withSpan("PreviewManager.make")); diff --git a/apps/desktop/src/preview/PickPreload.ts b/apps/desktop/src/preview/PickPreload.ts index 6155c4119ec8e955951c84d53f00c9e761fd824c..78351e9d79c3c96754ec0d638f2574fdd7275061 100644 --- a/apps/desktop/src/preview/PickPreload.ts +++ b/apps/desktop/src/preview/PickPreload.ts @@ -16,6 +16,8 @@ import type { import { resolveAnnotationSubmission } from "./AnnotationKeyboard.ts"; import { previewAnnotationStyles } from "./AnnotationStyles.generated.ts"; +import { installRecordingCursor } from "./RecordingCursor.ts"; +import { DEFAULT_RECORDING_INPUT_OPTIONS } from "./RecordingInput.ts"; import { ANNOTATION_CAPTURED_CHANNEL, ANNOTATION_THEME_CHANNEL, @@ -23,6 +25,11 @@ import { ELEMENT_PICKED_CHANNEL, HUMAN_INPUT_CHANNEL, MOUSE_NAVIGATE_CHANNEL, + RECORDING_CURSOR_CHANNEL, + RECORDING_POINTER_CHANNEL, + RECORDING_KEY_CHANNEL, + RECORDING_INPUT_CHANNEL, + RECORDING_CONTROLLER_CHANNEL, START_PICK_CHANNEL, } from "./GuestProtocol.ts"; const OVERLAY_ATTRIBUTE = "data-t3code-annotation-ui"; @@ -35,6 +42,80 @@ const ELEMENT_CONTEXT_TIMEOUT_MS = 5_000; const CONTENT_LAYER_Z_INDEX = 1; const CHROME_LAYER_Z_INDEX = 10; +let recordingCursor: ReturnType | null = null; +ipcRenderer.on( + RECORDING_CURSOR_CHANNEL, + (_event, active: unknown, inputOptions: unknown, controller: unknown) => { + if (active === true) { + const options = + typeof inputOptions === "object" && inputOptions !== null + ? { + showKeyPresses: + "showKeyPresses" in inputOptions && inputOptions.showKeyPresses === true, + showMousePresses: + "showMousePresses" in inputOptions && inputOptions.showMousePresses === true, + } + : DEFAULT_RECORDING_INPUT_OPTIONS; + recordingCursor ??= installRecordingCursor(document, window, options, (input) => + ipcRenderer.send(RECORDING_INPUT_CHANNEL, input), + ); + recordingCursor.setTheme(annotationTheme); + if (controller === "agent" || controller === "human" || controller === "none") + recordingCursor.setController(controller); + } else { + recordingCursor?.dispose(); + recordingCursor = null; + } + }, +); +ipcRenderer.on(RECORDING_CONTROLLER_CHANNEL, (_event, controller: unknown, point: unknown) => { + const humanPoint = + typeof point === "object" && + point !== null && + "x" in point && + typeof point.x === "number" && + Number.isFinite(point.x) && + "y" in point && + typeof point.y === "number" && + Number.isFinite(point.y) + ? { x: point.x, y: point.y } + : undefined; + if (controller === "agent" || controller === "human" || controller === "none") + recordingCursor?.setController(controller, humanPoint); +}); +ipcRenderer.on(RECORDING_KEY_CHANNEL, (_event, input: unknown) => { + if ( + typeof input !== "object" || + input === null || + !("key" in input) || + typeof input.key !== "string" + ) + return; + recordingCursor?.keyPress({ + key: input.key, + metaKey: "metaKey" in input && input.metaKey === true, + ctrlKey: "ctrlKey" in input && input.ctrlKey === true, + altKey: "altKey" in input && input.altKey === true, + shiftKey: "shiftKey" in input && input.shiftKey === true, + }); +}); +ipcRenderer.on(RECORDING_POINTER_CHANNEL, (_event, point: unknown) => { + if ( + typeof point === "object" && + point !== null && + "x" in point && + typeof point.x === "number" && + Number.isFinite(point.x) && + "y" in point && + typeof point.y === "number" && + Number.isFinite(point.y) + ) + recordingCursor?.move( + { x: point.x, y: point.y }, + "phase" in point && point.phase === "click" ? "click" : "move", + ); +}); + type AnnotationTool = "select" | "marquee" | "draw" | "erase"; interface SelectedElement { @@ -1361,6 +1442,7 @@ ipcRenderer.on(START_PICK_CHANNEL, (_event, theme: DesktopPreviewAnnotationTheme }); ipcRenderer.on(ANNOTATION_THEME_CHANNEL, (_event, theme: DesktopPreviewAnnotationTheme) => { annotationTheme = theme; + recordingCursor?.setTheme(theme); activeSession?.applyTheme(theme); }); ipcRenderer.on(CANCEL_PICK_CHANNEL, () => activeSession?.teardown(false)); diff --git a/apps/desktop/src/preview/RecordingCursor.ts b/apps/desktop/src/preview/RecordingCursor.ts new file mode 100644 index 0000000000000000000000000000000000000000..70b0a43be43ad7f6e8213e45ab27b760e2217d18 --- /dev/null +++ b/apps/desktop/src/preview/RecordingCursor.ts @@ -0,0 +1,197 @@ +import type { + DesktopPreviewAnnotationTheme, + DesktopPreviewRecordingInput, +} from "@t3tools/contracts"; + +import { + DEFAULT_RECORDING_INPUT_OPTIONS, + recordingKeyLabel, + recordingKeysAreSensitive, + type RecordingInputOptions, + type RecordingKeyPress, +} from "./RecordingInput.ts"; + +/** + * Chromium's capture cursor uses native window bounds, which do not follow a + * webview's CSS placement or scale. Draw it in the guest's coordinate space + * while recording, and make the native cursor transparent to avoid two cursors. + */ +export function installRecordingCursor( + document: Document, + window: Window, + options: RecordingInputOptions = DEFAULT_RECORDING_INPUT_OPTIONS, + emit: (input: DesktopPreviewRecordingInput) => void = () => {}, +) { + const style = document.createElement("style"); + style.textContent = + "html, html * { cursor: none !important; } @media (prefers-reduced-motion: reduce) { [data-t3code-recording-agent-cursor] { transition: none !important; } }"; + const cursor = document.createElement("div"); + cursor.setAttribute("aria-hidden", "true"); + cursor.setAttribute("data-t3code-recording-cursor", ""); + cursor.style.cssText = + "position:fixed;left:0;top:0;width:16px;height:24px;pointer-events:none;z-index:2147483647;display:none;"; + cursor.innerHTML = + ''; + const agentCursor = document.createElement("div"); + agentCursor.setAttribute("aria-hidden", "true"); + agentCursor.setAttribute("data-t3code-recording-agent-cursor", ""); + agentCursor.style.cssText = + "position:fixed;left:0;top:0;width:20px;height:20px;pointer-events:none;z-index:2147483647;display:none;filter:drop-shadow(0 1px 2px #0003);transition:transform 150ms ease-out,opacity 150ms ease-out;"; + // Match the MousePointer2 icon used by the live AgentBrowserCursor. + agentCursor.innerHTML = + ''; + document.documentElement.append(style, cursor, agentCursor); + let controller: "human" | "agent" | "none" = "none"; + let humanPoint: { readonly x: number; readonly y: number } | null = null; + const drawHuman = () => { + if (!humanPoint) return; + cursor.style.transform = `translate(${humanPoint.x}px, ${humanPoint.y}px)`; + cursor.style.display = "block"; + }; + let agentActive = false; + let agentTimer: number | undefined; + const setController = ( + next: typeof controller, + point?: { readonly x: number; readonly y: number }, + ) => { + if (point) humanPoint = point; + controller = next; + if (next === "agent") cursor.style.display = "none"; + if (next === "human") drawHuman(); + if (!agentActive) agentCursor.style.opacity = next === "human" ? "0.18" : "0.35"; + }; + const setTheme = ( + theme: Pick | null, + ) => { + agentCursor.style.setProperty("--recording-cursor-primary", theme?.primary ?? "#2563eb"); + agentCursor.style.setProperty("--recording-cursor-background", theme?.background ?? "white"); + }; + let lastKeyLabel: string | null = null; + let pointerHeld = false; + let pointerFrame: number | undefined; + let pendingPointer: DesktopPreviewRecordingInput | undefined; + const cancelPendingPointer = () => { + if (pointerFrame !== undefined) window.cancelAnimationFrame(pointerFrame); + pointerFrame = undefined; + pendingPointer = undefined; + }; + const keyPress = (input: RecordingKeyPress, held = false) => { + if (!options.showKeyPresses) return; + const label = recordingKeysAreSensitive(document) + ? null + : recordingKeyLabel(input, /Mac/.test(window.navigator.platform)); + lastKeyLabel = label; + emit({ type: "key", label, held, width: window.innerWidth }); + }; + const pointer = ( + point: { readonly x: number; readonly y: number }, + phase: "move" | "down" | "up" | "click", + ) => { + if (!options.showMousePresses || (phase === "move" && !pointerHeld)) return; + if (phase === "down") pointerHeld = true; + if (phase === "up") pointerHeld = false; + const input: DesktopPreviewRecordingInput = { + type: "pointer", + phase, + ...point, + width: window.innerWidth, + height: window.innerHeight, + }; + if (phase === "move") { + pendingPointer = input; + pointerFrame ??= window.requestAnimationFrame(() => { + pointerFrame = undefined; + if (pendingPointer) emit(pendingPointer); + pendingPointer = undefined; + }); + } else { + cancelPendingPointer(); + emit(input); + } + }; + const move = ( + point: { readonly x: number; readonly y: number }, + phase: "move" | "click" = "move", + ) => { + agentCursor.style.transform = `translate(${point.x}px, ${point.y}px)`; + agentCursor.style.display = "block"; + agentCursor.style.opacity = "1"; + agentActive = true; + window.clearTimeout(agentTimer); + agentTimer = window.setTimeout(() => { + agentActive = false; + agentCursor.style.opacity = controller === "human" ? "0.18" : "0.35"; + }, 700); + pointer(point, phase); + }; + const moveHuman = (point: { readonly x: number; readonly y: number }) => { + if (controller === "agent") return; + humanPoint = point; + drawHuman(); + }; + const pointerMove = (event: PointerEvent) => { + if (event.pointerType === "touch") return; + const point = { x: event.clientX, y: event.clientY }; + moveHuman(point); + pointer(point, "move"); + }; + const pointerDown = (event: PointerEvent) => { + if (event.pointerType === "touch") return; + moveHuman({ x: event.clientX, y: event.clientY }); + pointer({ x: event.clientX, y: event.clientY }, "down"); + }; + const pointerUp = (event: PointerEvent) => { + if (event.pointerType !== "touch") pointer({ x: event.clientX, y: event.clientY }, "up"); + }; + const keyDown = (event: KeyboardEvent) => { + if (event.isComposing || event.repeat) return; + keyPress(event, true); + }; + const keyUp = () => { + if (!options.showKeyPresses) return; + emit({ + type: "key", + label: recordingKeysAreSensitive(document) ? null : lastKeyLabel, + held: false, + width: window.innerWidth, + }); + }; + const hide = () => { + cursor.style.display = "none"; + pointerHeld = false; + cancelPendingPointer(); + if (options.showKeyPresses || options.showMousePresses) emit({ type: "clear" }); + }; + const leave = (event: PointerEvent) => { + if (event.relatedTarget === null) hide(); + }; + window.addEventListener("pointermove", pointerMove, true); + window.addEventListener("pointerdown", pointerDown, true); + window.addEventListener("pointerup", pointerUp, true); + window.addEventListener("pointercancel", pointerUp, true); + window.addEventListener("keydown", keyDown, true); + window.addEventListener("keyup", keyUp, true); + window.addEventListener("pointerout", leave, true); + window.addEventListener("blur", hide); + return { + move, + keyPress, + setController, + setTheme, + dispose: () => { + window.removeEventListener("pointermove", pointerMove, true); + window.removeEventListener("pointerdown", pointerDown, true); + window.removeEventListener("pointerup", pointerUp, true); + window.removeEventListener("pointercancel", pointerUp, true); + window.removeEventListener("keydown", keyDown, true); + window.removeEventListener("keyup", keyUp, true); + window.removeEventListener("pointerout", leave, true); + window.removeEventListener("blur", hide); + cancelPendingPointer(); + window.clearTimeout(agentTimer); + cursor.remove(); + agentCursor.remove(); + style.remove(); + }, + }; +} diff --git a/apps/desktop/src/preview/RecordingInput.test.ts b/apps/desktop/src/preview/RecordingInput.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..062685ec2934280e286e7665691cc0a0d71bc8d7 --- /dev/null +++ b/apps/desktop/src/preview/RecordingInput.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from "vite-plus/test"; +import { recordingKeyLabel, recordingKeysAreSensitive } from "./RecordingInput.ts"; + +const key = ( + value: string, + modifiers: Partial<{ + metaKey: boolean; + ctrlKey: boolean; + altKey: boolean; + shiftKey: boolean; + }> = {}, +) => ({ + key: value, + metaKey: false, + ctrlKey: false, + altKey: false, + shiftKey: false, + ...modifiers, +}); + +describe("recording key labels", () => { + it("formats macOS and other-platform shortcuts", () => { + expect(recordingKeyLabel(key("c", { metaKey: true }), true)).toBe("⌘C"); + expect(recordingKeyLabel(key("c", { ctrlKey: true }), false)).toBe("Ctrl + C"); + expect(recordingKeyLabel(key("Tab", { altKey: true, shiftKey: true }), true)).toBe("⌥⇧⇥"); + }); + it("shows held modifiers once and labels navigation keys", () => { + expect(recordingKeyLabel(key("Meta", { metaKey: true }), true)).toBe("⌘"); + expect(recordingKeyLabel(key("Shift", { shiftKey: true }), false)).toBe("Shift"); + expect(recordingKeyLabel(key("ArrowLeft"), true)).toBe("←"); + expect(recordingKeyLabel(key(" "), false)).toBe("Space"); + }); + it.each(["Dead", "Unidentified", "Process", ""])("excludes composition key %s", (value) => { + expect(recordingKeyLabel(key(value), true)).toBeNull(); + }); +}); + +describe("recording key privacy", () => { + const field = (type: string) => ({ tagName: "INPUT", getAttribute: () => type }); + const sensitive = (activeElement: unknown) => + recordingKeysAreSensitive({ activeElement } as Document); + it("excludes password fields and their shadow-root focus", () => { + expect(sensitive(field("password"))).toBe(true); + expect(sensitive({ shadowRoot: { activeElement: field("password") } })).toBe(true); + expect(sensitive(field("text"))).toBe(false); + }); + it("excludes iframe focus whose field cannot be inspected", () => { + expect(sensitive({ tagName: "IFRAME" })).toBe(true); + expect(sensitive({ tagName: "SECRET-FIELD" })).toBe(true); + }); +}); diff --git a/apps/desktop/src/preview/RecordingInput.ts b/apps/desktop/src/preview/RecordingInput.ts new file mode 100644 index 0000000000000000000000000000000000000000..a69102121d4ed56d72b8f8198d6822ad4b6de94c --- /dev/null +++ b/apps/desktop/src/preview/RecordingInput.ts @@ -0,0 +1,58 @@ +export interface RecordingInputOptions { + readonly showKeyPresses: boolean; + readonly showMousePresses: boolean; +} + +export const DEFAULT_RECORDING_INPUT_OPTIONS: RecordingInputOptions = { + showKeyPresses: false, + showMousePresses: false, +}; + +export interface RecordingKeyPress { + readonly key: string; + readonly metaKey: boolean; + readonly ctrlKey: boolean; + readonly altKey: boolean; + readonly shiftKey: boolean; +} + +/** Formats a single chord without duplicating a modifier pressed on its own. */ +export function recordingKeyLabel(input: RecordingKeyPress, isMac: boolean): string | null { + if (["Dead", "Process", "Unidentified", ""].includes(input.key)) return null; + const modifiers = [ + input.ctrlKey || input.key === "Control" ? (isMac ? "⌃" : "Ctrl") : null, + input.altKey || input.key === "Alt" ? (isMac ? "⌥" : "Alt") : null, + input.shiftKey || input.key === "Shift" ? (isMac ? "⇧" : "Shift") : null, + input.metaKey || input.key === "Meta" ? (isMac ? "⌘" : "Win") : null, + ].filter((value) => value !== null); + const labels: Record = { + Enter: "↵", + Tab: "⇥", + Backspace: "⌫", + Delete: "⌦", + Escape: "Esc", + ArrowUp: "↑", + ArrowDown: "↓", + ArrowLeft: "←", + ArrowRight: "→", + " ": "Space", + Space: "Space", + }; + if (!["Control", "Alt", "Shift", "Meta"].includes(input.key)) { + modifiers.push( + labels[input.key] ?? (input.key.length === 1 ? input.key.toUpperCase() : input.key), + ); + } + return modifiers.join(isMac ? "" : " + "); +} + +/** Unknown iframe or closed-shadow focus is excluded because its field type cannot be checked. */ +export function recordingKeysAreSensitive(document: Document): boolean { + let element = document.activeElement; + while (element?.shadowRoot?.activeElement) element = element.shadowRoot.activeElement; + return ( + element?.tagName === "IFRAME" || + element?.tagName.includes("-") === true || + element?.getAttribute("type")?.toLowerCase() === "password" + ); +} diff --git a/apps/web/src/browser/browserRecording.ts b/apps/web/src/browser/browserRecording.ts index 5ea10c2b05d5c4c662f3dc16ac6ecd79193a5578..0bbe4bd7bf182bd0158217c55ca3466e88bc6d9a 100644 --- a/apps/web/src/browser/browserRecording.ts +++ b/apps/web/src/browser/browserRecording.ts @@ -8,6 +8,8 @@ import { previewBridge } from "~/components/preview/previewBridge"; import { ensureClientSettingsHydrated, getClientSettings } from "~/hooks/useSettings"; import { appAtomRegistry } from "~/rpc/atomRegistry"; +import { createRecordingCompositor } from "./recordingCompositor"; + import { acquireBrowserSurfaceActivity } from "./browserSurfaceStore"; export class BrowserRecordingUnavailableError extends Schema.TaggedError()( @@ -123,6 +125,7 @@ interface ActiveRecording { releaseSurfaceActivity: (() => void) | null; stream: MediaStream | null; recorder: MediaRecorder | null; + compositor: Awaited>; savedBlob?: Blob; uploadPromise?: Promise; lifecycle: BrowserRecordingLifecycle; @@ -381,6 +384,8 @@ const captureTabMediaStreamWithTimeout = async ( }; const clearActiveRecording = (recording: ActiveRecording): void => { + recording.compositor?.dispose(); + recording.compositor = null; recording.releaseSurfaceActivity?.(); recording.releaseSurfaceActivity = null; if (activeRecordings.get(recording.tabId) !== recording) return; @@ -525,6 +530,7 @@ export async function startBrowserRecording( releaseSurfaceActivity, stream: null, recorder: null, + compositor: null, lifecycle: startingLifecycle, }; activeRecordings.set(tabId, recording); @@ -534,7 +540,8 @@ export async function startBrowserRecording( clearActiveRecording(recording); throw cause; }); - const frameRate = getClientSettings().browserRecordingFrameRate; + const settings = getClientSettings(); + const frameRate = settings.browserRecordingFrameRate; await waitForBrowserRecordingPaint(); const throwIfStartupCancelled = async (): Promise => { // Once a grant starts, a stop lets startup finish so the caller receives an artifact. @@ -614,7 +621,19 @@ export async function startBrowserRecording( let recorder: MediaRecorder; try { - recorder = createMediaRecorder(stream); + recording.compositor = await createRecordingCompositor( + stream, + { + showKeyPresses: settings.browserRecordingShowKeyPresses, + showMousePresses: settings.browserRecordingShowMousePresses, + frameRate, + }, + (listener) => + bridge.recording.onInput((event) => { + if (event.tabId === tabId) listener(event.input); + }), + ); + recorder = createMediaRecorder(recording.compositor?.stream ?? stream); recording.recorder = recorder; recorder.addEventListener("dataavailable", (event) => { if (event.data.size > 0) chunks.push(event.data); @@ -694,6 +713,8 @@ const finalizeBrowserRecording = async ( cause, }); } + recording.compositor?.dispose(); + recording.compositor = null; // Encoding has flushed; release native capture before materializing and saving the file. stopMediaStream(recording.stream); recording.stream = null; diff --git a/apps/web/src/browser/recordingCompositor.test.ts b/apps/web/src/browser/recordingCompositor.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..d10e69d24663220c2e0114344a6bdd9941cefc6c --- /dev/null +++ b/apps/web/src/browser/recordingCompositor.test.ts @@ -0,0 +1,179 @@ +import type { DesktopPreviewRecordingInput } from "@t3tools/contracts"; +import { afterEach, describe, expect, it, vi } from "vite-plus/test"; + +import { createRecordingCompositor, RecordingDecorations } from "./recordingCompositor"; + +const primaryColor = "oklch(0.65 0.2 310)"; +vi.mock("./annotationTheme", () => ({ + readPreviewAnnotationTheme: () => ({ primary: "oklch(0.65 0.2 310)" }), +})); + +const options = { showKeyPresses: true, showMousePresses: true, frameRate: 30 }; +const pointer = ( + phase: "move" | "down" | "up" | "click", + x = 100, +): DesktopPreviewRecordingInput => ({ + type: "pointer", + phase, + x, + y: 80, + width: 800, + height: 600, +}); +const context = () => ({ + save: vi.fn(), + restore: vi.fn(), + beginPath: vi.fn(), + fill: vi.fn(), + stroke: vi.fn(), + ellipse: vi.fn(), + roundRect: vi.fn(), + fillText: vi.fn(), + drawImage: vi.fn(), + measureText: () => ({ width: 40 }), + globalAlpha: 1, + strokeStyle: "", + fillStyle: "", +}); + +describe("recording decorations", () => { + it("keeps rings aligned through dragging and stops following the cursor after release", () => { + const decorations = new RecordingDecorations(options, primaryColor); + const ctx = context(); + decorations.apply(pointer("down"), 0); + decorations.apply(pointer("move", 120), 10); + decorations.draw(ctx as unknown as CanvasRenderingContext2D, 1600, 1200, 10); + expect(ctx.ellipse.mock.calls[0]?.slice(0, 4)).toEqual([240, 160, 40, 40]); + expect(ctx.strokeStyle).toBe(primaryColor); + expect(ctx.fillStyle).toBe(primaryColor); + expect(decorations.nextRedraw(10)).toBeNull(); + decorations.apply(pointer("up", 130), 20); + decorations.apply(pointer("move", 300), 30); + decorations.draw(ctx as unknown as CanvasRenderingContext2D, 1600, 1200, 320); + expect(ctx.ellipse.mock.calls[1]?.slice(0, 4)).toEqual([260, 160, 50, 50]); + ctx.ellipse.mockClear(); + decorations.draw(ctx as unknown as CanvasRenderingContext2D, 1600, 1200, 620); + expect(ctx.ellipse).not.toHaveBeenCalled(); + expect(decorations.nextRedraw(620)).toBeNull(); + }); + + it("pulses agent clicks and clears decorations on blur or navigation", () => { + const decorations = new RecordingDecorations(options, primaryColor); + const ctx = context(); + decorations.apply(pointer("click"), 0); + decorations.draw(ctx as unknown as CanvasRenderingContext2D, 800, 600, 300); + expect(ctx.ellipse).toHaveBeenCalledOnce(); + decorations.apply({ type: "clear" }, 301); + ctx.ellipse.mockClear(); + decorations.draw(ctx as unknown as CanvasRenderingContext2D, 800, 600, 302); + expect(ctx.ellipse).not.toHaveBeenCalled(); + expect(decorations.nextRedraw(302)).toBeNull(); + }); + + it("holds shortcut badges until release, then expires them even on a static page", () => { + const decorations = new RecordingDecorations(options, primaryColor); + const ctx = context(); + const key = { type: "key" as const, label: "⌘C", held: true, width: 800 }; + decorations.apply(key, 0); + decorations.draw(ctx as unknown as CanvasRenderingContext2D, 1600, 1200, 5000); + expect(ctx.fillText.mock.calls[0]?.slice(0, 3)).toEqual(["⌘C", 800, 1098]); + decorations.apply({ ...key, held: false }, 5000); + expect(decorations.nextRedraw(5500)).toBe(400); + ctx.fillText.mockClear(); + decorations.draw(ctx as unknown as CanvasRenderingContext2D, 1600, 1200, 5900); + expect(ctx.fillText).not.toHaveBeenCalled(); + }); + + it("removes the previous key badge on password focus", () => { + const decorations = new RecordingDecorations(options, primaryColor); + const ctx = context(); + decorations.apply({ type: "key", label: "A", held: true, width: 800 }, 0); + decorations.apply({ type: "key", label: null, held: true, width: 800 }, 1); + decorations.draw(ctx as unknown as CanvasRenderingContext2D, 800, 600, 2); + expect(ctx.fillText).not.toHaveBeenCalled(); + }); + + it("honors independent opt-in flags", () => { + const decorations = new RecordingDecorations( + { ...options, showMousePresses: false }, + primaryColor, + ); + const ctx = context(); + decorations.apply(pointer("down"), 0); + decorations.apply({ type: "key", label: "⌘C", held: true, width: 800 }, 0); + decorations.draw(ctx as unknown as CanvasRenderingContext2D, 800, 600, 1); + expect(ctx.ellipse).not.toHaveBeenCalled(); + expect(ctx.fillText).toHaveBeenCalledOnce(); + }); +}); + +describe("detached recording compositor", () => { + afterEach(() => vi.unstubAllGlobals()); + + it("keeps native capture when both decorations are off", async () => { + vi.stubGlobal("document", { + createElement: () => { + throw new Error("must not allocate"); + }, + }); + expect( + await createRecordingCompositor( + {} as MediaStream, + { + ...options, + showKeyPresses: false, + showMousePresses: false, + }, + () => { + throw new Error("must not subscribe"); + }, + ), + ).toBeNull(); + }); + + it.each([false, true])( + "releases the detached output on disposal or playback failure (%s)", + async (failPlayback) => { + const ctx = context(); + const stop = vi.fn(); + const unsubscribe = vi.fn(); + const cancelFrame = vi.fn(); + const source = { + getVideoTracks: () => [{ getSettings: () => ({ width: 800, height: 600 }) }], + } as unknown as MediaStream; + const stream = { getTracks: () => [{ stop }] } as unknown as MediaStream; + const canvas = { width: 0, height: 0, getContext: () => ctx, captureStream: () => stream }; + const video = { + muted: false, + playsInline: false, + srcObject: null as MediaStream | null, + readyState: 2, + videoWidth: 800, + videoHeight: 600, + pause: vi.fn(), + play: async () => { + if (failPlayback) throw new Error("play failed"); + }, + requestVideoFrameCallback: () => 1, + cancelVideoFrameCallback: cancelFrame, + }; + vi.stubGlobal("document", { + createElement: (tag: string) => (tag === "canvas" ? canvas : video), + }); + vi.stubGlobal("window", { clearTimeout: vi.fn(), setTimeout: vi.fn() }); + const compositor = createRecordingCompositor(source, options, () => unsubscribe); + if (failPlayback) await expect(compositor).rejects.toThrow("play failed"); + else { + const result = await compositor; + expect(result?.stream).toBe(stream); + expect(ctx.drawImage).toHaveBeenCalledOnce(); + result?.dispose(); + result?.dispose(); + } + expect(stop).toHaveBeenCalledOnce(); + expect(unsubscribe).toHaveBeenCalledOnce(); + expect(cancelFrame).toHaveBeenCalledWith(1); + expect(video.srcObject).toBeNull(); + }, + ); +}); diff --git a/apps/web/src/browser/recordingCompositor.ts b/apps/web/src/browser/recordingCompositor.ts new file mode 100644 index 0000000000000000000000000000000000000000..352154bd9aef2e3d7dc01299687e96e8ab6483b3 --- /dev/null +++ b/apps/web/src/browser/recordingCompositor.ts @@ -0,0 +1,190 @@ +import type { DesktopPreviewRecordingInput } from "@t3tools/contracts"; + +import { readPreviewAnnotationTheme } from "./annotationTheme"; + +interface RecordingDecorationOptions { + readonly showKeyPresses: boolean; + readonly showMousePresses: boolean; + readonly frameRate: number; +} + +/** Decorates a detached canvas; no recording UI is inserted into the preview page. */ +export async function createRecordingCompositor( + source: MediaStream, + options: RecordingDecorationOptions, + subscribe: (listener: (input: DesktopPreviewRecordingInput) => void) => () => void, +) { + if (!options.showKeyPresses && !options.showMousePresses) return null; + const canvas = document.createElement("canvas"); + const context = canvas.getContext("2d", { alpha: false }); + if (!context) throw new Error("Recording canvas is unavailable."); + const video = document.createElement("video"); + video.muted = true; + video.playsInline = true; + video.srcObject = source; + const settings = source.getVideoTracks()[0]?.getSettings(); + canvas.width = settings?.width ?? 1920; + canvas.height = settings?.height ?? 1080; + const decorations = new RecordingDecorations(options, readPreviewAnnotationTheme().primary); + let disposed = false; + let frameId: number | undefined; + let timer: number | undefined; + const draw = () => { + if (disposed || video.readyState < 2) return; + const width = video.videoWidth || canvas.width; + const height = video.videoHeight || canvas.height; + if (canvas.width !== width) canvas.width = width; + if (canvas.height !== height) canvas.height = height; + context.drawImage(video, 0, 0, width, height); + const now = performance.now(); + decorations.draw(context, width, height, now); + window.clearTimeout(timer); + const next = decorations.nextRedraw(now); + if (next !== null) timer = window.setTimeout(draw, next); + }; + const frame = () => { + if (disposed) return; + draw(); + frameId = video.requestVideoFrameCallback(frame); + }; + const output = canvas.captureStream(options.frameRate); + let unsubscribe: (() => void) | undefined; + const dispose = () => { + if (disposed) return; + disposed = true; + unsubscribe?.(); + window.clearTimeout(timer); + if (frameId !== undefined) video.cancelVideoFrameCallback(frameId); + video.pause(); + video.srcObject = null; + for (const track of output.getTracks()) track.stop(); + }; + try { + unsubscribe = subscribe((input) => { + decorations.apply(input, performance.now()); + draw(); + }); + frameId = video.requestVideoFrameCallback(frame); + await video.play(); + draw(); + return { stream: output, dispose }; + } catch (error) { + dispose(); + throw error; + } +} + +/** Keeps input timing and coordinates independent of native video frame delivery. */ +export class RecordingDecorations { + private ring: { + x: number; + y: number; + width: number; + height: number; + held: boolean; + releasedAt: number | null; + } | null = null; + private key: { label: string; width: number; expiresAt: number | null } | null = null; + + constructor( + private readonly options: RecordingDecorationOptions, + private readonly primaryColor: string, + ) {} + + apply(input: DesktopPreviewRecordingInput, now: number) { + if (input.type === "clear") { + this.ring = null; + this.key = null; + } else if (input.type === "key" && this.options.showKeyPresses) { + this.key = input.label + ? { label: input.label, width: input.width, expiresAt: input.held ? null : now + 900 } + : null; + } else if (input.type === "pointer" && this.options.showMousePresses) { + if (input.phase === "down" || input.phase === "click") { + this.ring = { + x: input.x, + y: input.y, + width: input.width, + height: input.height, + held: input.phase === "down", + releasedAt: input.phase === "click" ? now : null, + }; + } else if (this.ring?.held) { + this.ring = { + ...this.ring, + x: input.x, + y: input.y, + width: input.width, + height: input.height, + held: input.phase !== "up", + releasedAt: input.phase === "up" ? now : null, + }; + } + } + } + + nextRedraw(now: number): number | null { + if ( + this.ring?.releasedAt !== null && + this.ring?.releasedAt !== undefined && + now < this.ring.releasedAt + 600 + ) { + return 1000 / this.options.frameRate; + } + return this.key?.expiresAt !== null && + this.key?.expiresAt !== undefined && + now < this.key.expiresAt + ? this.key.expiresAt - now + : null; + } + + draw(context: CanvasRenderingContext2D, width: number, height: number, now: number) { + // Guest coordinates are CSS pixels; native frames include zoom and display scale. + const scale = width / (this.key?.width ?? this.ring?.width ?? 1280); + const ring = this.ring; + if (ring && (ring.held || (ring.releasedAt !== null && now < ring.releasedAt + 600))) { + const progress = ring.releasedAt === null ? 0 : Math.min(1, (now - ring.releasedAt) / 600); + context.save(); + const opacity = 0.9 * (1 - progress); + context.strokeStyle = this.primaryColor; + context.fillStyle = this.primaryColor; + context.lineWidth = 2 * scale; + context.beginPath(); + context.ellipse( + (ring.x * width) / ring.width, + (ring.y * height) / ring.height, + ((20 * width) / ring.width) * (1 + progress * 0.5), + ((20 * height) / ring.height) * (1 + progress * 0.5), + 0, + 0, + Math.PI * 2, + ); + context.globalAlpha = opacity * 0.15; + context.fill(); + context.globalAlpha = opacity; + context.stroke(); + context.restore(); + } + const key = this.key; + if (key && (key.expiresAt === null || now < key.expiresAt)) { + context.save(); + context.font = `500 ${26 * scale}px system-ui, sans-serif`; + const badgeWidth = Math.min( + width - 32 * scale, + context.measureText(key.label).width + 36 * scale, + ); + const badgeHeight = 54 * scale; + const left = (width - badgeWidth) / 2; + const top = height - 24 * scale - badgeHeight; + context.fillStyle = "rgba(32,32,34,.86)"; + context.beginPath(); + context.roundRect(left, top, badgeWidth, badgeHeight, 14 * scale); + context.fill(); + context.fillStyle = "white"; + context.textAlign = "center"; + context.textBaseline = "middle"; + context.fillText(key.label, width / 2, top + badgeHeight / 2, badgeWidth - 24 * scale); + context.restore(); + } + } +} diff --git a/apps/web/src/components/preview/AgentBrowserCursor.tsx b/apps/web/src/components/preview/AgentBrowserCursor.tsx index bc89daee45955c1577f1f2e8d35a74fa514de004..408f268fa0c4062c5b59dd8d44945eaa9005ee23 100644 --- a/apps/web/src/components/preview/AgentBrowserCursor.tsx +++ b/apps/web/src/components/preview/AgentBrowserCursor.tsx @@ -24,7 +24,6 @@ export function AgentBrowserCursor(props: { return ( (null); + const active = inactiveSequence !== event.sequence; useEffect(() => { - const timeout = window.setTimeout(() => setActive(false), CURSOR_ACTIVE_MS); + const timeout = window.setTimeout(() => setInactiveSequence(event.sequence), CURSOR_ACTIVE_MS); return () => window.clearTimeout(timeout); - }, []); + }, [event.sequence]); return (
({ pictureInPicture: false, showEmptyState: false, loading: false, + serverEpoch: null as string | null, + recordingTabIds: new Set(), + recordingRuntimeTabId: null as string | null, recordVisitForThread: vi.fn(), })); @@ -98,6 +101,7 @@ vi.mock("~/previewStateStore", () => ({ updatePreviewServerSnapshot: vi.fn(), useThreadPreviewState: () => ({ activeTabId: "tab-1", + serverEpoch: mocks.serverEpoch, desktopByTabId: { "tab-1": { hasWebContents: true, @@ -146,11 +150,11 @@ vi.mock("~/state/use-atom-command", () => ({ })); vi.mock("~/browser/browserRecording", () => ({ - findActiveBrowserRecordingRuntimeTabId: vi.fn(() => null), + findActiveBrowserRecordingRuntimeTabId: () => mocks.recordingRuntimeTabId, isBrowserRecordingStartCancelledError: vi.fn(() => false), startBrowserRecording: vi.fn(), stopBrowserRecording: vi.fn(), - useActiveBrowserRecordingTabIds: () => new Set(), + useActiveBrowserRecordingTabIds: () => mocks.recordingTabIds, })); vi.mock("~/browser/browserSurfaceStore", () => ({ @@ -244,7 +248,9 @@ vi.mock("./PreviewMoreMenu", () => ({ })); vi.mock("./PreviewUnreachable", () => ({ PreviewUnreachable: () => null })); vi.mock("./ZoomIndicator", () => ({ ZoomIndicator: () => null })); -vi.mock("./AgentBrowserCursor", () => ({ AgentBrowserCursor: () => null })); +vi.mock("./AgentBrowserCursor", () => ({ + AgentBrowserCursor: () => createElement("agent-cursor"), +})); vi.mock("~/browser/BrowserSurfaceSlot", () => ({ BrowserSurfaceSlot: () => null })); vi.mock("./usePreviewSession", () => ({ usePreviewSession: vi.fn() })); @@ -344,9 +350,38 @@ describe("PreviewView navigation", () => { mocks.pictureInPicture = false; mocks.showEmptyState = false; mocks.loading = false; + mocks.serverEpoch = null; + mocks.recordingTabIds = new Set(); + mocks.recordingRuntimeTabId = null; mocks.recordVisitForThread.mockClear(); }); + it("shows the cursor in a replacement browser while the old instance still records", async () => { + const document = installTestDom(); + const { createRoot } = await import("react-dom/client"); + const container = document.createElement("div"); + const root = createRoot(container as unknown as Element); + const hasCursor = (node: TestNode): boolean => + node.nodeName === "AGENT-CURSOR" || node.childNodes.some(hasCursor); + mocks.recordingTabIds.add(TEST_RUNTIME_TAB_ID); + mocks.recordingRuntimeTabId = TEST_RUNTIME_TAB_ID; + try { + await act(() => { + root.render(); + }); + expect(hasCursor(container)).toBe(false); + mocks.serverEpoch = "replacement-server"; + await act(() => { + root.render(); + }); + expect(hasCursor(container)).toBe(true); + expect(mocks.recordingTabIds.has(TEST_RUNTIME_TAB_ID)).toBe(true); + } finally { + await act(() => root.unmount()); + vi.unstubAllGlobals(); + } + }); + it("does not rerender while loading time passes", async () => { vi.useFakeTimers(); mocks.loading = true; diff --git a/apps/web/src/components/preview/PreviewView.tsx b/apps/web/src/components/preview/PreviewView.tsx index d190a78d48253a85601dd49bd2d6c450aa8a6cca..810e6d502804ebf2e029b21fbdfb0522a0e04d13 100644 --- a/apps/web/src/components/preview/PreviewView.tsx +++ b/apps/web/src/components/preview/PreviewView.tsx @@ -798,18 +798,17 @@ export function PreviewView({ {snapshot && desktopOverlay ? ( ) : null} - {runtimeTabId && desktopOverlay && !showEmptyState && !isUnreachable ? ( + {runtimeTabId && + desktopOverlay && + !showEmptyState && + !isUnreachable && + !activeRecordingTabIds.has(runtimeTabId) ? ( ) : null} - {controller !== "none" ? ( -
- {controller === "agent" ? "Agent controlling browser" : "Human control"} -
- ) : null} {navStatus._tag === "LoadFailed" ? (
settings.browserRecordingShowKeyPresses); + const showMouse = useClientSettings((settings) => settings.browserRecordingShowMousePresses); + const updateSettings = useUpdatePrimarySettings(); + return ( + <> + + updateSettings({ browserRecordingShowKeyPresses: Boolean(checked) }) + } + /> + } + /> + + updateSettings({ browserRecordingShowMousePresses: Boolean(checked) }) + } + /> + } + /> + + ); +} + function BrowserRecordingFrameRateSetting({ disabled }: { readonly disabled: boolean }) { const frameRate = useClientSettings((settings) => settings.browserRecordingFrameRate); const updateSettings = useUpdatePrimarySettings(); @@ -1326,6 +1364,7 @@ export function IntegrationsSettingsPanel() { + diff --git a/apps/web/src/components/settings/SettingsPanels.logic.test.ts b/apps/web/src/components/settings/SettingsPanels.logic.test.ts index d93db8d970b1da4daae5d390c96bf6cc015cf38b..3f9d233ed2789a9de74dec8e3d4330ba15914ee2 100644 --- a/apps/web/src/components/settings/SettingsPanels.logic.test.ts +++ b/apps/web/src/components/settings/SettingsPanels.logic.test.ts @@ -268,6 +268,8 @@ describe("getChangedBrowserSettingLabels", () => { browserDefaultZoomFactor: 1.5, browserDefaultAppearance: "dark", browserRecordingFrameRate: 60, + browserRecordingShowKeyPresses: true, + browserRecordingShowMousePresses: true, browserLinkTarget: "app", browserAutoShowFloatingPreview: !DEFAULT_UNIFIED_SETTINGS.browserAutoShowFloatingPreview, }), @@ -276,6 +278,8 @@ describe("getChangedBrowserSettingLabels", () => { "Browser zoom", "Browser appearance", "Recording frame rate", + "Recording key presses", + "Recording mouse presses", "Open links in", "Floating preview", ]); diff --git a/apps/web/src/components/settings/SettingsPanels.logic.ts b/apps/web/src/components/settings/SettingsPanels.logic.ts index 5cbcb190a97b608ccb275fe37b5db671847d983c..d5c6359c1c9f0cdec8451f3e40a2e6e66d310779 100644 --- a/apps/web/src/components/settings/SettingsPanels.logic.ts +++ b/apps/web/src/components/settings/SettingsPanels.logic.ts @@ -115,6 +115,8 @@ export type BrowserDefaultSettings = Pick< | "browserDefaultZoomFactor" | "browserDefaultAppearance" | "browserRecordingFrameRate" + | "browserRecordingShowKeyPresses" + | "browserRecordingShowMousePresses" | "browserLinkTarget" | "browserAutoShowFloatingPreview" >; @@ -156,6 +158,8 @@ export function getChangedBrowserSettingLabels(settings: BrowserDefaultSettings) ...(settings.browserRecordingFrameRate !== DEFAULT_UNIFIED_SETTINGS.browserRecordingFrameRate ? ["Recording frame rate"] : []), + ...(settings.browserRecordingShowKeyPresses ? ["Recording key presses"] : []), + ...(settings.browserRecordingShowMousePresses ? ["Recording mouse presses"] : []), ...(settings.browserLinkTarget !== DEFAULT_UNIFIED_SETTINGS.browserLinkTarget ? ["Open links in"] : []), diff --git a/apps/web/src/components/settings/SettingsPanels.tsx b/apps/web/src/components/settings/SettingsPanels.tsx index 85e1de588f942ea887b301cbff2d391fc3168b14..180ab45361dfc8cdc35f6c9148d696717ba3eb4a 100644 --- a/apps/web/src/components/settings/SettingsPanels.tsx +++ b/apps/web/src/components/settings/SettingsPanels.tsx @@ -635,6 +635,8 @@ export function useSettingsRestore(onRestored?: () => void) { settings.browserDefaultZoomFactor, settings.browserDefaultAppearance, settings.browserRecordingFrameRate, + settings.browserRecordingShowKeyPresses, + settings.browserRecordingShowMousePresses, settings.browserLinkTarget, settings.browserAutoShowFloatingPreview, settings.appearanceContrast, @@ -798,6 +800,8 @@ export function useSettingsRestore(onRestored?: () => void) { browserDefaultZoomFactor: DEFAULT_UNIFIED_SETTINGS.browserDefaultZoomFactor, browserDefaultAppearance: DEFAULT_UNIFIED_SETTINGS.browserDefaultAppearance, browserRecordingFrameRate: DEFAULT_UNIFIED_SETTINGS.browserRecordingFrameRate, + browserRecordingShowKeyPresses: DEFAULT_UNIFIED_SETTINGS.browserRecordingShowKeyPresses, + browserRecordingShowMousePresses: DEFAULT_UNIFIED_SETTINGS.browserRecordingShowMousePresses, browserLinkTarget: DEFAULT_UNIFIED_SETTINGS.browserLinkTarget, browserAutoShowFloatingPreview: DEFAULT_UNIFIED_SETTINGS.browserAutoShowFloatingPreview, // Re-granted like any other default. The confirmation dialog lists it by diff --git a/apps/web/src/components/settings/settingsSearch.ts b/apps/web/src/components/settings/settingsSearch.ts index 4532bd5f72fb8ff370975ca5ae5683530dbab5a9..88f0326ea9e0c47970ee30e602250e44baf937cb 100644 --- a/apps/web/src/components/settings/settingsSearch.ts +++ b/apps/web/src/components/settings/settingsSearch.ts @@ -615,6 +615,18 @@ export const SETTINGS_SEARCH_ITEMS = [ title: "Browser recording frame rate", to: "/settings/integrations", }, + { + id: "browser-recording-key-presses", + title: "Show key presses in recordings", + to: "/settings/integrations", + searchTerms: ["browser preview keyboard shortcuts keystrokes overlay capture"], + }, + { + id: "browser-recording-mouse-presses", + title: "Show mouse presses in recordings", + to: "/settings/integrations", + searchTerms: ["browser preview clicks buttons drag overlay capture"], + }, { id: "browser-link-target", title: "Open links in", diff --git a/packages/contracts/src/ipc.ts b/packages/contracts/src/ipc.ts index 7e738b97786b23c9f793cb79e6b5918e40cb0fe1..c67ecd27c0e1785701d737c9e82f402243493698 100644 --- a/packages/contracts/src/ipc.ts +++ b/packages/contracts/src/ipc.ts @@ -658,6 +658,30 @@ export interface DesktopPreviewPointerEvent { createdAt: string; } +/** Recording decorations are forwarded separately from the captured page pixels. */ +export const DesktopPreviewRecordingInputSchema = Schema.Union([ + Schema.Struct({ + type: Schema.Literal("pointer"), + phase: Schema.Literals(["move", "down", "up", "click"]), + x: Schema.Finite, + y: Schema.Finite, + width: Schema.Finite.check(Schema.isGreaterThan(0)), + height: Schema.Finite.check(Schema.isGreaterThan(0)), + }), + Schema.Struct({ + type: Schema.Literal("key"), + label: Schema.NullOr(Schema.String.check(Schema.isMaxLength(100))), + held: Schema.Boolean, + width: Schema.Finite.check(Schema.isGreaterThan(0)), + }), + Schema.Struct({ type: Schema.Literal("clear") }), +]); +export type DesktopPreviewRecordingInput = typeof DesktopPreviewRecordingInputSchema.Type; +export interface DesktopPreviewRecordingInputEvent { + readonly tabId: string; + readonly input: DesktopPreviewRecordingInput; +} + /** * Static config a renderer needs to mount a preview ``. Returned * atomically by `DesktopPreviewBridge.getPreviewConfig()` so the renderer @@ -1295,6 +1319,7 @@ export interface DesktopPreviewBridge { close: (tabId: string) => Promise; }; recording: { + onInput: (listener: (event: DesktopPreviewRecordingInputEvent) => void) => () => void; startScreencast: (tabId: string) => Promise; stopScreencast: (tabId: string) => Promise; save: ( diff --git a/packages/contracts/src/settings.test.ts b/packages/contracts/src/settings.test.ts index 3a9fc35c4c82cf7b49f79533975d44a76480ff8c..ff54b6c0aa533507ac1f5e5f26b24f222ac954e9 100644 --- a/packages/contracts/src/settings.test.ts +++ b/packages/contracts/src/settings.test.ts @@ -471,6 +471,23 @@ describe("ClientSettings browser recording frame rate", () => { }); }); +describe("ClientSettings recording input overlays", () => { + it("defaults both overlays off and accepts independent opt-ins", () => { + const settings = decodeClientSettings({}); + expect(settings.browserRecordingShowKeyPresses).toBe(false); + expect(settings.browserRecordingShowMousePresses).toBe(false); + expect( + decodeClientSettingsPatch({ + browserRecordingShowKeyPresses: true, + browserRecordingShowMousePresses: false, + }), + ).toMatchObject({ + browserRecordingShowKeyPresses: true, + browserRecordingShowMousePresses: false, + }); + }); +}); + describe("ClientSettings glass opacity", () => { it("defaults to a readable translucent surface", () => { expect(decodeClientSettings({}).glassOpacity).toBe(80); diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index 42e91a76ecaf1c99138c49c22adf482abe27f062..73a2a2517d05ff1268a7e0c2a08b5184c7f82a90 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -317,6 +317,12 @@ export const ClientSettingsSchema = Schema.Struct({ browserRecordingFrameRate: BrowserRecordingFrameRate.pipe( Schema.withDecodingDefault(Effect.succeed(DEFAULT_BROWSER_RECORDING_FRAME_RATE)), ), + browserRecordingShowKeyPresses: Schema.Boolean.pipe( + Schema.withDecodingDefault(Effect.succeed(false)), + ), + browserRecordingShowMousePresses: Schema.Boolean.pipe( + Schema.withDecodingDefault(Effect.succeed(false)), + ), /** * Where links clicked in a thread (chat markdown, terminal output) open. * Only the desktop app has an in-app browser, so other clients ignore "app". @@ -1525,6 +1531,8 @@ export const ClientSettingsPatch = Schema.Struct({ browserDefaultZoomFactor: Schema.optionalKey(PreviewZoomFactor), browserDefaultAppearance: Schema.optionalKey(PreviewAppearancePreference), browserRecordingFrameRate: Schema.optionalKey(BrowserRecordingFrameRate), + browserRecordingShowKeyPresses: Schema.optionalKey(Schema.Boolean), + browserRecordingShowMousePresses: Schema.optionalKey(Schema.Boolean), browserLinkTarget: Schema.optionalKey(BrowserLinkTarget), browserAutoShowFloatingPreview: Schema.optionalKey(Schema.Boolean), browserProfiles: Schema.optionalKey(Schema.Array(BrowserProfile)), -- 2.54.0