Commit 55c24273a025874cdb8a96a25ee0c59eeb4e3051 | the-knife

apps/desktop/src/ipc/channels.ts+2

...	@@ -113,3 +113,5 @@ export const PREVIEW_POINTER_EVENT_CHANNEL = "desktop:preview-pointer-event";	...	@@ -113,3 +113,5 @@ export const PREVIEW_POINTER_EVENT_CHANNEL = "desktop:preview-pointer-event";
113	export const MAC_PERMISSION_HELPER_CHANNEL = "desktop:mac-permission-helper";	113	export const MAC_PERMISSION_HELPER_CHANNEL = "desktop:mac-permission-helper";
114		114
115	export const CHECK_SYSTEM_PERMISSION_CHANNEL = "desktop:check-system-permission";	115	export const CHECK_SYSTEM_PERMISSION_CHANNEL = "desktop:check-system-permission";
		116
		117	export const PREVIEW_RECORDING_INPUT_CHANNEL = "desktop:preview-recording-input";

apps/desktop/src/ipc/methods/preview.ts+20-5

 @@ -30,11 +30,13 @@ import {
 } from "@t3tools/contracts";
 import * as Effect from "effect/Effect";
 import * as Schema from "effect/Schema";
+import * as Option from "effect/Option";
 import * as NodeURL from "node:url";
 import * as ElectronWindow from "../../electron/ElectronWindow.ts";
 import * as BrowserImport from "../../preview/BrowserImport/BrowserImport.ts";
 import * as PreviewManager from "../../preview/Manager.ts";
+import * as DesktopClientSettings from "../../settings/DesktopClientSettings.ts";
 import { PREVIEW_WEBVIEW_PREFERENCES } from "../../preview/WebviewPreferences.ts";
 import * as IpcChannels from "../channels.ts";
 import * as DesktopIpc from "../DesktopIpc.ts";
 @@ -50,6 +52,9 @@ export const installPreviewEventForwarding = Effect.fn(
   yield* manager.subscribeRecordingFrames((frame) =>
     electronWindow.sendAll(IpcChannels.PREVIEW_RECORDING_FRAME_CHANNEL, frame),
   );
+  yield* manager.subscribeRecordingInputs((event) =>
+    electronWindow.sendAll(IpcChannels.PREVIEW_RECORDING_INPUT_CHANNEL, event),
+  );
   yield* manager.subscribePointerEvents((event) =>
     electronWindow.sendAll(IpcChannels.PREVIEW_POINTER_EVENT_CHANNEL, event),
   );
 @@ -180,11 +185,21 @@ export const cancelPickElement = tabMethod(
   "desktop.ipc.preview.cancelPickElement",
   (manager, tabId) => manager.cancelPickElement(tabId),
 );
-export const startRecording = tabMethod(
+export const startRecording = DesktopIpc.makeIpcMethod({
-  IpcChannels.PREVIEW_RECORDING_START_CHANNEL,
+  channel: IpcChannels.PREVIEW_RECORDING_START_CHANNEL,
-  "desktop.ipc.preview.startRecording",
+  payload: DesktopPreviewTabInputSchema,
-  (manager, tabId) => manager.startRecording(tabId),
+  result: Schema.Void,
-);
+  handler: Effect.fn("desktop.ipc.preview.startRecording")(function* ({ tabId }) {
+    const manager = yield* PreviewManager.PreviewManager;
+    const store = yield* DesktopClientSettings.DesktopClientSettings;
+    const settings = yield* store.get;
+    const options = Option.map(settings, (value) => ({
+      showKeyPresses: value.browserRecordingShowKeyPresses,
+      showMousePresses: value.browserRecordingShowMousePresses,
+    }));
+    yield* manager.startRecording(tabId, Option.getOrUndefined(options));
+  }),
+});
 export const stopRecording = tabMethod(
   IpcChannels.PREVIEW_RECORDING_STOP_CHANNEL,
   "desktop.ipc.preview.stopRecording",

apps/desktop/src/preload.ts+10

 @@ -1,6 +1,7 @@
 import type {
   DesktopBridge,
   DesktopPreviewPointerEvent,
+  DesktopPreviewRecordingInputEvent,
   DesktopPreviewRecordingFrame,
   DesktopPreviewTabState,
   DesktopSnapShotEvent,
 @@ -327,6 +328,15 @@ contextBridge.exposeInMainWorld("desktopBridge", {
         ipcRenderer.invoke(IpcChannels.PREVIEW_PICTURE_IN_PICTURE_CLOSE_CHANNEL, { tabId }),
     },
     recording: {
+      onInput: (listener) => {
+        const wrappedListener = (_event: Electron.IpcRendererEvent, event: unknown) => {
+          if (typeof event !== "object" || event === null) return;
+          listener(event as DesktopPreviewRecordingInputEvent);
+        };
+        ipcRenderer.on(IpcChannels.PREVIEW_RECORDING_INPUT_CHANNEL, wrappedListener);
+        return () =>
+          ipcRenderer.removeListener(IpcChannels.PREVIEW_RECORDING_INPUT_CHANNEL, wrappedListener);
+      },
       startScreencast: (tabId) =>
         ipcRenderer.invoke(IpcChannels.PREVIEW_RECORDING_START_CHANNEL, { tabId }),
       stopScreencast: (tabId) =>

apps/desktop/src/preview/GuestProtocol.ts+5

...	@@ -5,3 +5,8 @@ export const ANNOTATION_CAPTURED_CHANNEL = "preview:annotation-captured";	...	@@ -5,3 +5,8 @@ export const ANNOTATION_CAPTURED_CHANNEL = "preview:annotation-captured";
5	export const ANNOTATION_THEME_CHANNEL = "preview:annotation-theme";	5	export const ANNOTATION_THEME_CHANNEL = "preview:annotation-theme";
6	export const HUMAN_INPUT_CHANNEL = "preview:human-input";	6	export const HUMAN_INPUT_CHANNEL = "preview:human-input";
7	export const MOUSE_NAVIGATE_CHANNEL = "preview:mouse-navigate";	7	export const MOUSE_NAVIGATE_CHANNEL = "preview:mouse-navigate";
		8	export const RECORDING_CURSOR_CHANNEL = "preview:recording-cursor";
		9	export const RECORDING_POINTER_CHANNEL = "preview:recording-pointer";
		10	export const RECORDING_KEY_CHANNEL = "preview:recording-key";
		11	export const RECORDING_INPUT_CHANNEL = "preview:recording-input";
		12	export const RECORDING_CONTROLLER_CHANNEL = "preview:recording-controller";

apps/desktop/src/preview/Manager.test.ts+272-80

...	@@ -1,7 +1,10 @@	...	@@ -1,7 +1,10 @@
1	import * as NodeVM from "node:vm";	1	import * as NodeVM from "node:vm";
2	import { it as effectIt } from "@effect/vitest";	2	import { it as effectIt } from "@effect/vitest";
3	import { DESKTOP_PREVIEW_RECORDING_CAPTURE_TRIGGER } from "@t3tools/contracts";	3	import { DESKTOP_PREVIEW_RECORDING_CAPTURE_TRIGGER } from "@t3tools/contracts";
4	import type { DesktopPreviewRecordingFrame } from "@t3tools/contracts";	4	import type {
		5	DesktopPreviewRecordingFrame,
		6	DesktopPreviewRecordingInputEvent,
		7	} from "@t3tools/contracts";
5	import { HostProcessPlatform } from "@t3tools/shared/hostProcess";	8	import { HostProcessPlatform } from "@t3tools/shared/hostProcess";
6	import * as Cause from "effect/Cause";	9	import * as Cause from "effect/Cause";
7	import * as Deferred from "effect/Deferred";	10	import * as Deferred from "effect/Deferred";
...	@@ -2885,6 +2888,165 @@ describe("PreviewManager", () => {	...	@@ -2885,6 +2888,165 @@ describe("PreviewManager", () => {
2885	),	2888	),
2886	);	2889	);
2887		2890
		2891	effectIt.effect(
		2892	"restores the native cursor when recording startup fails, then allows a retry",
		2893	() =>
		2894	withManager((manager) =>
		2895	Effect.gen(function* () {
		2896	const host = makeTestHostWebContents();
		2897	host.executeJavaScript.mockResolvedValueOnce(false);
		2898	let cursorActive = false;
		2899	const cursorAtCapture: boolean[] = [];
		2900	const contents = Object.assign(
		2901	makeTestPreviewWebContents(
		2902	async () => {
		2903	cursorAtCapture.push(cursorActive);
		2904	return {
		2905	toJPEG: () => Buffer.from("frame"),
		2906	getSize: () => ({ width: 800, height: 600 }),
		2907	};
		2908	},
		2909	42,
		2910	host,
		2911	),
		2912	{
		2913	send: (channel: string, active: unknown) => {
		2914	if (channel === "preview:recording-cursor") cursorActive = active === true;
		2915	},
		2916	},
		2917	);
		2918	fromId.mockReturnValue(contents as never);
		2919	yield* manager.createTab("tab_cursor");
		2920	yield* manager.registerWebview("tab_cursor", 42);
		2921	const failed = yield* Effect.exit(manager.startRecording("tab_cursor"));
		2922	expect(Exit.isFailure(failed)).toBe(true);
		2923	expect(cursorAtCapture).toEqual([true]);
		2924	expect(cursorActive).toBe(false);
		2925
		2926	yield* manager.startRecording("tab_cursor");
		2927	expect(cursorAtCapture).toEqual([true, true]);
		2928	expect(cursorActive).toBe(true);
		2929	yield* manager.stopRecording("tab_cursor");
		2930	expect(cursorActive).toBe(false);
		2931	}),
		2932	),
		2933	);
		2934
		2935	effectIt.effect("restores the recording cursor after navigation only while recording", () =>
		2936	withManager((manager) =>
		2937	Effect.gen(function* () {
		2938	const listeners = new Map<string, () => void>();
		2939	let cursorActive = false;
		2940	let cursorUpdated: (() => void) \| undefined;
		2941	let inputOptions: unknown;
		2942	const options = { showKeyPresses: true, showMousePresses: false };
		2943	const contents = Object.assign(
		2944	makeTestPreviewWebContents(async () => ({
		2945	toJPEG: () => Buffer.from("frame"),
		2946	getSize: () => ({ width: 800, height: 600 }),
		2947	})),
		2948	{
		2949	on: (event: string, listener: () => void) => listeners.set(event, listener),
		2950	send: (channel: string, active: unknown, recordingOptions: unknown) => {
		2951	if (channel !== "preview:recording-cursor") return;
		2952	cursorActive = active === true;
		2953	inputOptions = recordingOptions;
		2954	cursorUpdated?.();
		2955	},
		2956	},
		2957	);
		2958	fromId.mockReturnValue(contents as never);
		2959	yield* manager.createTab("tab_cursor_reload");
		2960	yield* manager.registerWebview("tab_cursor_reload", 42);
		2961	yield* manager.startRecording("tab_cursor_reload", options);
		2962	for (const recording of [true, false]) {
		2963	if (!recording) yield* manager.stopRecording("tab_cursor_reload");
		2964	// A new document has lost the previous preload's cursor overlay.
		2965	cursorActive = false;
		2966	const restored = new Promise<void>((resolve) => {
		2967	cursorUpdated = resolve;
		2968	});
		2969	listeners.get("dom-ready")?.();
		2970	yield* Effect.promise(() => restored);
		2971	cursorUpdated = undefined;
		2972	expect(cursorActive).toBe(recording);
		2973	expect(inputOptions).toEqual(recording ? options : undefined);
		2974	}
		2975	}),
		2976	),
		2977	);
		2978
		2979	effectIt.effect("gates recording decorations and isolates failed subscribers", () =>
		2980	withManager((manager) =>
		2981	Effect.gen(function* () {
		2982	const callbacks = new Map<
		2983	string,
		2984	(event: unknown, input: unknown) => Fiber.Fiber<void, never> \| undefined
		2985	>();
		2986	const contents = Object.assign(
		2987	makeTestPreviewWebContents(async () => ({
		2988	toJPEG: () => Buffer.from("frame"),
		2989	getSize: () => ({ width: 800, height: 600 }),
		2990	})),
		2991	{
		2992	ipc: {
		2993	on: (
		2994	channel: string,
		2995	callback: (event: unknown, input: unknown) => Fiber.Fiber<void, never> \| undefined,
		2996	) => callbacks.set(channel, callback),
		2997	off: vi.fn(),
		2998	},
		2999	},
		3000	);
		3001	fromId.mockReturnValue(contents as never);
		3002	yield* manager.createTab("tab_recording_input");
		3003	yield* manager.registerWebview("tab_recording_input", 42);
		3004	const received: DesktopPreviewRecordingInputEvent[] = [];
		3005	yield* manager.subscribeRecordingInputs(() => Effect.die("renderer unavailable"));
		3006	yield* manager.subscribeRecordingInputs((event) =>
		3007	Effect.sync(() => {
		3008	received.push(event);
		3009	}),
		3010	);
		3011	const send = (input: unknown) =>
		3012	Effect.gen(function* () {
		3013	const fiber = callbacks.get("preview:recording-input")?.(null, input);
		3014	if (fiber) yield* Fiber.join(fiber);
		3015	});
		3016	const key = { type: "key", label: "⌘C", held: true, width: 800 };
		3017	const pointer = {
		3018	type: "pointer",
		3019	phase: "down",
		3020	x: 120,
		3021	y: 80,
		3022	width: 800,
		3023	height: 600,
		3024	};
		3025	yield* send(key);
		3026	expect(received).toEqual([]);
		3027	yield* manager.startRecording("tab_recording_input", {
		3028	showKeyPresses: true,
		3029	showMousePresses: false,
		3030	});
		3031	yield* send(key);
		3032	yield* send(pointer);
		3033	yield* send({ ...key, width: 0 });
		3034	expect(received).toEqual([{ tabId: "tab_recording_input", input: key }]);
		3035	yield* manager.stopRecording("tab_recording_input");
		3036	yield* send(key);
		3037	expect(received).toHaveLength(1);
		3038	yield* manager.startRecording("tab_recording_input", {
		3039	showKeyPresses: false,
		3040	showMousePresses: true,
		3041	});
		3042	yield* send(key);
		3043	yield* send(pointer);
		3044	expect(received.at(-1)).toEqual({ tabId: "tab_recording_input", input: pointer });
		3045	expect(received).toHaveLength(2);
		3046	}),
		3047	),
		3048	);
		3049
2888	effectIt.effect("continues native recording when the source warmup fails", () =>	3050	effectIt.effect("continues native recording when the source warmup fails", () =>
2889	withManager((manager) =>	3051	withManager((manager) =>
2890	Effect.gen(function* () {	3052	Effect.gen(function* () {
...	@@ -3853,88 +4015,108 @@ describe("PreviewManager", () => {	...	@@ -3853,88 +4015,108 @@ describe("PreviewManager", () => {
3853	),	4015	),
3854	);	4016	);
3855		4017
3856	effectIt.effect("emits the resolved pointer target before dispatching an automation click", () =>	4018	effectIt.effect(
3857	withManager((manager) =>	4019	"records the resolved pointer target before dispatching an automation click",
3858	Effect.gen(function* () {	4020	() =>
3859	let humanInput: ((_event: unknown, signal: unknown) => void) \| undefined;	4021	withManager((manager) =>
3860	const activity: string[] = [];	4022	Effect.gen(function* () {
3861	const sendCommand = vi.fn(async (method: string, params?: Record<string, unknown>) => {	4023	let humanInput: ((_event: unknown, signal: unknown) => void) \| undefined;
3862	if (method === "Runtime.evaluate") {	4024	const activity: string[] = [];
3863	return {	4025	const sendCommand = vi.fn(async (method: string, params?: Record<string, unknown>) => {
3864	result: {	4026	if (method === "Runtime.evaluate") {
3865	value: { width: 800, height: 600 },	4027	return {
3866	},	4028	result: {
3867	};	4029	value: { width: 800, height: 600 },
3868	}	4030	},
3869	if (method === "Input.dispatchMouseEvent" && params?.type === "mousePressed") {	4031	};
3870	activity.push("mousePressed");	4032	}
3871	humanInput?.({}, { kind: "pointer", x: params.x, y: params.y, button: 0 });	4033	if (method === "Input.dispatchMouseEvent" && params?.type === "mousePressed") {
3872	}	4034	activity.push("mousePressed");
3873	return undefined;	4035	humanInput?.({}, { kind: "pointer", x: params.x, y: params.y, button: 0 });
3874	});	4036	}
3875	fromId.mockReturnValue({	4037	return undefined;
3876	id: 42,	4038	});
3877	isDestroyed: () => false,	4039	fromId.mockReturnValue({
3878	getType: () => "webview",	4040	id: 42,
3879	getURL: () => "https://example.com",	4041	hostWebContents: makeTestHostWebContents(),
3880	getTitle: () => "Example",	4042	capturePage: vi.fn(async () => ({ toPNG: () => Buffer.from("frame") })),
3881	isLoading: () => false,	4043	setBackgroundThrottling: vi.fn(),
3882	isDevToolsOpened: () => false,	4044	isDestroyed: () => false,
3883	getZoomFactor: () => 1,	4045	getType: () => "webview",
3884	setZoomFactor: vi.fn(),	4046	getURL: () => "https://example.com",
3885	setAudioMuted: vi.fn(),	4047	getTitle: () => "Example",
3886	isCurrentlyAudible: () => false,	4048	isLoading: () => false,
3887	on: vi.fn(),	4049	isDevToolsOpened: () => false,
3888	off: vi.fn(),	4050	getZoomFactor: () => 1,
3889	ipc: {	4051	setZoomFactor: vi.fn(),
3890	on: vi.fn((channel: string, listener: typeof humanInput) => {	4052	setAudioMuted: vi.fn(),
3891	if (channel === "preview:human-input") humanInput = listener;	4053	isCurrentlyAudible: () => false,
3892	}),
3893	off: vi.fn(),
3894	},
3895	send: webviewSend,
3896	navigationHistory: { canGoBack: () => false, canGoForward: () => false },
3897	setIgnoreMenuShortcuts: vi.fn(),
3898	setWindowOpenHandler: vi.fn(),
3899	debugger: {
3900	isAttached: () => false,
3901	attach: vi.fn(),
3902	sendCommand,
3903	on: vi.fn(),	4054	on: vi.fn(),
3904	off: vi.fn(),	4055	off: vi.fn(),
3905	},	4056	ipc: {
3906	} as never);	4057	on: vi.fn((channel: string, listener: typeof humanInput) => {
3907		4058	if (channel === "preview:human-input") humanInput = listener;
3908	yield* manager.subscribePointerEvents((event) =>	4059	}),
3909	Effect.sync(() => {	4060	off: vi.fn(),
3910	activity.push(event.phase);	4061	},
3911	}),	4062	send: webviewSend,
3912	);	4063	navigationHistory: { canGoBack: () => false, canGoForward: () => false },
3913	yield* manager.createTab("tab_1");	4064	setIgnoreMenuShortcuts: vi.fn(),
3914	yield* manager.registerWebview("tab_1", 42);	4065	setWindowOpenHandler: vi.fn(),
3915	const click = yield* manager	4066	debugger: {
3916	.automationClick("tab_1", { x: 120, y: 80 })	4067	isAttached: () => false,
3917	.pipe(Effect.forkChild({ startImmediately: true }));	4068	attach: vi.fn(),
3918	yield* TestClock.adjust(200);	4069	sendCommand,
3919	yield* Fiber.join(click);	4070	on: vi.fn(),
		4071	off: vi.fn(),
		4072	},
		4073	} as never);
3920		4074
3921	expect(activity).toEqual(["move", "click", "mousePressed"]);	4075	yield* manager.subscribePointerEvents((event) =>
3922	expect(sendCommand).toHaveBeenCalledWith("Input.dispatchMouseEvent", {	4076	Effect.sync(() => {
3923	type: "mousePressed",	4077	activity.push(event.phase);
3924	x: 120,	4078	}),
3925	y: 80,	4079	);
3926	button: "left",	4080	yield* manager.createTab("tab_1");
3927	clickCount: 1,	4081	yield* manager.registerWebview("tab_1", 42);
3928	});	4082	yield* manager.startRecording("tab_1");
3929	expect(sendCommand).toHaveBeenCalledWith("Input.dispatchMouseEvent", {	4083	const click = yield* manager
3930	type: "mouseReleased",	4084	.automationClick("tab_1", { x: 120, y: 80 })
3931	x: 120,	4085	.pipe(Effect.forkChild({ startImmediately: true }));
3932	y: 80,	4086	yield* TestClock.adjust(200);
3933	button: "left",	4087	yield* Fiber.join(click);
3934	clickCount: 1,	4088
3935	});	4089	expect(activity).toEqual(["move", "click", "mousePressed"]);
3936	}),	4090	expect(
3937	),	4091	webviewSend.mock.calls
		4092	.filter(([channel]) => channel === "preview:recording-controller")
		4093	.map(([, controller]) => controller),
		4094	).toEqual(["agent", "none"]);
		4095
		4096	const recordedPointer = webviewSend.mock.calls
		4097	.filter(([channel]) => channel === "preview:recording-pointer")
		4098	.map(([, event]) => event);
		4099	expect(recordedPointer).toEqual([
		4100	expect.objectContaining({ phase: "move", x: 120, y: 80 }),
		4101	expect.objectContaining({ phase: "click", x: 120, y: 80 }),
		4102	]);
		4103	expect(sendCommand).toHaveBeenCalledWith("Input.dispatchMouseEvent", {
		4104	type: "mousePressed",
		4105	x: 120,
		4106	y: 80,
		4107	button: "left",
		4108	clickCount: 1,
		4109	});
		4110	yield* manager.stopRecording("tab_1");
		4111	expect(sendCommand).toHaveBeenCalledWith("Input.dispatchMouseEvent", {
		4112	type: "mouseReleased",
		4113	x: 120,
		4114	y: 80,
		4115	button: "left",
		4116	clickCount: 1,
		4117	});
		4118	}),
		4119	),
3938	);	4120	);
3939		4121
3940	effectIt.effect("types in background webviews and enables native key input", () =>	4122	effectIt.effect("types in background webviews and enables native key input", () =>
...	@@ -4230,6 +4412,9 @@ describe("PreviewManager", () => {	...	@@ -4230,6 +4412,9 @@ describe("PreviewManager", () => {
4230	});	4412	});
4231	fromId.mockReturnValue({	4413	fromId.mockReturnValue({
4232	id: 42,	4414	id: 42,
		4415	hostWebContents: makeTestHostWebContents(),
		4416	capturePage: vi.fn(async () => ({ toPNG: () => Buffer.from("frame") })),
		4417	setBackgroundThrottling: vi.fn(),
4233	isDestroyed: () => false,	4418	isDestroyed: () => false,
4234	getType: () => "webview",	4419	getType: () => "webview",
4235	getURL: () => "https://example.com",	4420	getURL: () => "https://example.com",
...	@@ -4263,6 +4448,7 @@ describe("PreviewManager", () => {	...	@@ -4263,6 +4448,7 @@ describe("PreviewManager", () => {
4263		4448
4264	yield* manager.createTab("tab_1");	4449	yield* manager.createTab("tab_1");
4265	yield* manager.registerWebview("tab_1", 42);	4450	yield* manager.registerWebview("tab_1", 42);
		4451	yield* manager.startRecording("tab_1");
4266		4452
4267	const click = yield* manager	4453	const click = yield* manager
4268	.automationClick("tab_1", { x: 120, y: 80 })	4454	.automationClick("tab_1", { x: 120, y: 80 })
...	@@ -4270,6 +4456,12 @@ describe("PreviewManager", () => {	...	@@ -4270,6 +4456,12 @@ describe("PreviewManager", () => {
4270	yield* TestClock.adjust(200);	4456	yield* TestClock.adjust(200);
4271	const exit = yield* Fiber.await(click);	4457	const exit = yield* Fiber.await(click);
4272	expect(Exit.isFailure(exit)).toBe(true);	4458	expect(Exit.isFailure(exit)).toBe(true);
		4459	expect(
		4460	webviewSend.mock.calls
		4461	.filter(([channel]) => channel === "preview:recording-controller")
		4462	.map(([, controller]) => controller),
		4463	).toEqual(["agent", "human", "none"]);
		4464	yield* manager.stopRecording("tab_1");
4273	if (Exit.isSuccess(exit)) return;	4465	if (Exit.isSuccess(exit)) return;
4274	const error = Option.getOrThrow(Cause.findErrorOption(exit.cause));	4466	const error = Option.getOrThrow(Cause.findErrorOption(exit.cause));
4275	expect(error).toMatchObject({	4467	expect(error).toMatchObject({

apps/desktop/src/preview/Manager.ts+148-6

 @@ -6,7 +6,10 @@
  * here). Single layer-scoped browser session partition.
  */
 import * as NodeCrypto from "node:crypto";
-import { DESKTOP_PREVIEW_RECORDING_CAPTURE_TRIGGER } from "@t3tools/contracts";
+import {
+  DesktopPreviewRecordingInputSchema,
+  DESKTOP_PREVIEW_RECORDING_CAPTURE_TRIGGER,
+} from "@t3tools/contracts";
 import type {
   DesktopPreviewAnnotationTheme,
   DesktopPreviewAutomationStatus,
 @@ -18,6 +21,7 @@ import type {
   PreviewAnnotationSubmissionResult,
   DesktopPreviewRecordingArtifact,
   DesktopPreviewRecordingFrame,
+  DesktopPreviewRecordingInputEvent,
   DesktopPreviewScreenshotArtifact,
   DesktopPreviewTabDefaults,
   PreviewAutomationClickInput,
 @@ -71,6 +75,11 @@ import {
   ELEMENT_PICKED_CHANNEL,
   HUMAN_INPUT_CHANNEL,
   MOUSE_NAVIGATE_CHANNEL,
+  RECORDING_CURSOR_CHANNEL,
+  RECORDING_POINTER_CHANNEL,
+  RECORDING_KEY_CHANNEL,
+  RECORDING_INPUT_CHANNEL,
+  RECORDING_CONTROLLER_CHANNEL,
   START_PICK_CHANNEL,
 } from "./GuestProtocol.ts";
 import { isPreviewAnnotationPayload } from "./PickedElementPayload.ts";
 @@ -81,6 +90,7 @@ import {
   previewAutomationEditingCommandExpression,
 } from "./PreviewKeyboard.ts";
 import { captureFavicon, safeHttpOrigin, selectFaviconCandidates } from "./FaviconCapture.ts";
+import { DEFAULT_RECORDING_INPUT_OPTIONS, type RecordingInputOptions } from "./RecordingInput.ts";
 export type PreviewNavStatus =
   | { kind: "Idle" }
 @@ -437,6 +447,7 @@ interface ManagedListeners {
 type FrameCaptureConsumer = "picture-in-picture" | "recording";
 interface FrameCaptureSession {
+  readonly recordingInputOptions?: RecordingInputOptions;
   readonly scope: Scope.Closeable | null;
   readonly consumers: ReadonlySet<FrameCaptureConsumer>;
   readonly unthrottledWebContentsIds: ReadonlySet<number>;
 @@ -485,6 +496,10 @@ interface BrowserDiagnostics {
   readonly requests: ReadonlyMap<string, { url: string; method: string }>;
 }
+const isRecordingInput = Schema.is(DesktopPreviewRecordingInputSchema);
+type RecordingInputListener = (event: DesktopPreviewRecordingInputEvent) => Effect.Effect<void>;
 type PointerEventListener = (event: DesktopPreviewPointerEvent) => Effect.Effect<void>;
 interface ExpectedAgentInput {
 @@ -636,6 +651,9 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
   const attachedRef = yield* Ref.make<ReadonlyMap<number, ManagedListeners>>(new Map());
   const listenersRef = yield* Ref.make<ReadonlySet<Listener>>(new Set());
   const pointerEventListenersRef = yield* Ref.make<ReadonlySet<PointerEventListener>>(new Set());
+  const recordingInputListenersRef = yield* Ref.make<ReadonlySet<RecordingInputListener>>(
+    new Set(),
+  );
   const recordingFrameListenersRef = yield* Ref.make<ReadonlySet<RecordingFrameListener>>(
     new Set(),
   );
 @@ -821,6 +839,20 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
         return Effect.succeed([undefined, sessions] as const);
       }
       return setFrameCaptureWebContentsBackgroundThrottling(wc, false).pipe(
+        Effect.tap(() =>
+          Effect.gen(function* () {
+            if (!current.consumers.has("recording")) return;
+            const tab = (yield* SynchronizedRef.get(tabsRef)).get(tabId);
+            yield* attempt({ operation: "recording.cursor", tabId, webContentsId: wc.id }, () =>
+              wc.send(
+                RECORDING_CURSOR_CHANNEL,
+                true,
+                current.recordingInputOptions,
+                tab?.controller,
+              ),
+            );
+          }),
+        ),
         Effect.map(
           () =>
             [
 @@ -846,6 +878,15 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
         if (!current || !current.consumers.has(consumer)) {
           return [undefined, sessions] as const;
         }
+        if (consumer === "recording") {
+          yield* Effect.forEach(current.unthrottledWebContentsIds, (id) =>
+            attempt({ operation: "recording.cursor", tabId, webContentsId: id }, () => {
+              const contents = webContents.fromId(id);
+              if (contents && !contents.isDestroyed())
+                contents.send(RECORDING_CURSOR_CHANNEL, false);
+            }).pipe(Effect.ignore),
+          );
+        }
         const consumers = new Set(current.consumers);
         consumers.delete(consumer);
         if (consumers.size > 0) {
 @@ -896,7 +937,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
   });
   const deliverEvent = (
-    eventKind: "state-change" | "recording-frame" | "pointer-event",
+    eventKind: "state-change" | "recording-frame" | "recording-input" | "pointer-event",
     tabId: string,
     delivery: () => Effect.Effect<void>,
   ) =>
 @@ -933,6 +974,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
   const update = Effect.fn("PreviewManager.update")(function* (
     tabId: string,
     patch: Partial<PreviewTabState>,
+    humanPoint?: { readonly x: number; readonly y: number },
   ) {
     const updatedAt = yield* currentIso;
     const next = yield* SynchronizedRef.modify(tabsRef, (tabs) => {
 @@ -950,7 +992,20 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
     // can commit between the modify above and here, and republishing this
     // snapshot would roll the UI back to a value that writer will not send
     // again because it suppresses unchanged audibility.
-    if (Option.isSome(next)) yield* emitIfCurrent(tabId, next.value);
+    if (Option.isSome(next)) {
+      if (patch.controller !== undefined && next.value.webContentsId != null) {
+        const capture = (yield* SynchronizedRef.get(frameCaptureSessionsRef)).get(tabId);
+        const webContentsId = next.value.webContentsId;
+        if (capture?.consumers.has("recording")) {
+          yield* attempt({ operation: "recording.controller", tabId }, () => {
+            const contents = webContents.fromId(webContentsId);
+            if (contents && !contents.isDestroyed())
+              contents.send(RECORDING_CONTROLLER_CHANNEL, patch.controller, humanPoint);
+          }).pipe(Effect.ignore);
+        }
+      }
+      yield* emitIfCurrent(tabId, next.value);
+    }
   });
   /**
 @@ -1728,6 +1783,21 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
     const sync = () => runFork(syncState(true));
     const syncNavigation = () => runFork(syncState(false, true));
     const syncInPageNavigation = () => runFork(syncState(false));
+    const restoreRecordingCursor = () =>
+      runFork(
+        Effect.gen(function* () {
+          const session = (yield* SynchronizedRef.get(frameCaptureSessionsRef)).get(tabId);
+          if (!wc.isDestroyed()) {
+            const tab = (yield* SynchronizedRef.get(tabsRef)).get(tabId);
+            wc.send(
+              RECORDING_CURSOR_CHANNEL,
+              session?.consumers.has("recording") ?? false,
+              session?.recordingInputOptions,
+              tab?.controller,
+            );
+          }
+        }),
+      );
     const navigationStarted = (
       event: Electron.Event<Electron.WebContentsDidStartNavigationEventParams>,
     ) => {
 @@ -1860,13 +1930,37 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
           copy.set(tabId, (epochs.get(tabId) ?? 0) + 1);
         }),
       );
-      yield* update(tabId, { controller: "human" });
+      yield* update(
+        tabId,
+        { controller: "human" },
+        isPreviewInputSignal(rawSignal) && rawSignal.kind === "pointer"
+          ? { x: rawSignal.x, y: rawSignal.y }
+          : undefined,
+      );
       yield* Effect.sleep(750);
       const tabs = yield* SynchronizedRef.get(tabsRef);
       if (tabs.get(tabId)?.controller === "human") {
         yield* update(tabId, { controller: "none" });
       }
     });
+    const recordingInput = (_event: unknown, input: unknown) => {
+      if (!isRecordingInput(input)) return;
+      return runFork(
+        Effect.gen(function* () {
+          const tab = (yield* SynchronizedRef.get(tabsRef)).get(tabId);
+          const capture = (yield* SynchronizedRef.get(frameCaptureSessionsRef)).get(tabId);
+          if (tab?.webContentsId !== wc.id || !capture?.consumers.has("recording")) return;
+          if (input.type === "key" && !capture.recordingInputOptions?.showKeyPresses) return;
+          if (input.type === "pointer" && !capture.recordingInputOptions?.showMousePresses) return;
+          const listeners = yield* Ref.get(recordingInputListenersRef);
+          yield* Effect.forEach(
+            listeners,
+            (listener) => deliverEvent("recording-input", tabId, () => listener({ tabId, input })),
+            { discard: true },
+          );
+        }),
+      );
+    };
     const humanInput = (_event: unknown, rawSignal?: unknown): void => {
       runFork(handleHumanInput(rawSignal));
     };
 @@ -1928,11 +2022,13 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
         wc.off("page-favicon-updated", faviconUpdated as never);
         wc.off("did-start-loading", sync);
         wc.off("did-stop-loading", sync);
+        wc.off("dom-ready", restoreRecordingCursor);
         wc.off("did-fail-load", failed as never);
         wc.off("audio-state-changed", audioStateChanged);
         wc.off("did-create-window", windowCreated);
         wc.off("before-input-event", beforeInput);
         wc.ipc.off(HUMAN_INPUT_CHANNEL, humanInput);
+        wc.ipc.off(RECORDING_INPUT_CHANNEL, recordingInput);
         wc.ipc.off(MOUSE_NAVIGATE_CHANNEL, mouseNavigate);
       }).pipe(Effect.ignore),
     );
 @@ -1948,9 +2044,11 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
         wc.on("page-favicon-updated", faviconUpdated as never);
         wc.on("did-start-loading", sync);
         wc.on("did-stop-loading", sync);
+        wc.on("dom-ready", restoreRecordingCursor);
         wc.on("did-fail-load", failed as never);
         wc.on("audio-state-changed", audioStateChanged);
         wc.ipc.on(HUMAN_INPUT_CHANNEL, humanInput);
+        wc.ipc.on(RECORDING_INPUT_CHANNEL, recordingInput);
         wc.ipc.on(MOUSE_NAVIGATE_CHANNEL, mouseNavigate);
         wc.setWindowOpenHandler((details) => {
           if (previewWindowOpenAction(details) === "popup") {
 @@ -3405,7 +3503,10 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
     });
   };
-  const startRecording = Effect.fn("PreviewManager.startRecording")(function* (tabId: string) {
+  const startRecording = Effect.fn("PreviewManager.startRecording")(function* (
+    tabId: string,
+    options: RecordingInputOptions = DEFAULT_RECORDING_INPUT_OPTIONS,
+  ) {
     if ((yield* Ref.get(closingTabIdsRef)).has(tabId)) {
       return yield* new PreviewTabNotFoundError({ tabId });
     }
 @@ -3413,11 +3514,21 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
       tabId,
       Effect.gen(function* () {
         yield* startFrameCapture(tabId, "recording");
+        yield* SynchronizedRef.update(frameCaptureSessionsRef, (sessions) =>
+          replaceMap(sessions, (copy) => {
+            const current = copy.get(tabId);
+            if (current) copy.set(tabId, { ...current, recordingInputOptions: options });
+          }),
+        );
         const wc = yield* requireWebContents(tabId);
         const requestWebContents = wc.hostWebContents;
         if (requestWebContents === null) {
           return yield* new PreviewMainWindowClosedError({ tabId });
         }
+        const tab = (yield* SynchronizedRef.get(tabsRef)).get(tabId);
+        yield* attempt({ operation: "recording.cursor", tabId, webContentsId: wc.id }, () =>
+          wc.send(RECORDING_CURSOR_CHANNEL, true, options, tab?.controller),
+        );
         yield* attemptPromise(
           {
             operation: "recording.warmSource",
 @@ -3720,6 +3831,15 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
   const emitPointerEvent = Effect.fn("PreviewManager.emitPointerEvent")(function* (
     event: DesktopPreviewPointerEvent,
   ) {
+    const recording = (yield* SynchronizedRef.get(frameCaptureSessionsRef)).get(event.tabId);
+    const tab = (yield* SynchronizedRef.get(tabsRef)).get(event.tabId);
+    const webContentsId = tab?.webContentsId;
+    if (recording?.consumers.has("recording") && webContentsId != null) {
+      yield* attempt({ operation: "recording.pointer", tabId: event.tabId }, () => {
+        const contents = webContents.fromId(webContentsId);
+        if (contents && !contents.isDestroyed()) contents.send(RECORDING_POINTER_CHANNEL, event);
+      });
+    }
     const listeners = yield* Ref.get(pointerEventListenersRef);
     yield* Effect.forEach(
       listeners,
 @@ -4110,6 +4230,18 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
     const keySequence = makePreviewAutomationNativeKeySequence(input, {
       isMac: hostPlatform === "darwin",
     });
+    const recording = (yield* SynchronizedRef.get(frameCaptureSessionsRef)).get(tabId);
+    if (recording?.consumers.has("recording") && recording.recordingInputOptions?.showKeyPresses) {
+      yield* attempt({ operation: "recording.key", tabId, webContentsId: wc.id }, () =>
+        wc.send(RECORDING_KEY_CHANNEL, {
+          key: keySequence.signal.key || input.key,
+          metaKey: input.modifiers?.includes("Meta") ?? false,
+          ctrlKey: input.modifiers?.includes("Control") ?? false,
+          altKey: input.modifiers?.includes("Alt") ?? false,
+          shiftKey: input.modifiers?.includes("Shift") ?? false,
+        }),
+      );
+    }
     // CDP keyboard dispatch follows the embedder's focused renderer, and
     // WebContents.focus() is a no-op for webview guests. Native input targets
     // this guest's widget directly, so Enter cannot submit the host composer.
 @@ -4470,6 +4602,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
         Ref.set(expectedAgentInputsRef, new Map()),
         Ref.set(pointerEventListenersRef, new Set()),
         Ref.set(recordingFrameListenersRef, new Set()),
+        Ref.set(recordingInputListenersRef, new Set()),
       ],
       { discard: true },
     );
 @@ -4513,6 +4646,8 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
     stopRecording,
     subscribePointerEvents: (listener: PointerEventListener) =>
       subscribe(pointerEventListenersRef, listener),
+    subscribeRecordingInputs: (listener: RecordingInputListener) =>
+      subscribe(recordingInputListenersRef, listener),
     subscribeRecordingFrames: (listener: RecordingFrameListener) =>
       subscribe(recordingFrameListenersRef, listener),
     subscribeStateChanges: (listener: Listener) => subscribe(listenersRef, listener),
 @@ -4877,7 +5012,10 @@ export class PreviewManager extends Context.Service<
     readonly copyArtifactToClipboard: (path: string) => Effect.Effect<void, PreviewManagerError>;
     readonly openPictureInPicture: (tabId: string) => Effect.Effect<void, PreviewManagerError>;
     readonly closePictureInPicture: (tabId: string) => Effect.Effect<void, PreviewManagerError>;
-    readonly startRecording: (tabId: string) => Effect.Effect<void, PreviewManagerError>;
+    readonly startRecording: (
+      tabId: string,
+      options?: RecordingInputOptions,
+    ) => Effect.Effect<void, PreviewManagerError>;
     readonly stopRecording: (tabId: string) => Effect.Effect<void, PreviewManagerError>;
     readonly saveRecording: (
       tabId: string,
 @@ -4918,6 +5056,9 @@ export class PreviewManager extends Context.Service<
     readonly subscribePointerEvents: (
       listener: PointerEventListener,
     ) => Effect.Effect<void, never, Scope.Scope>;
+    readonly subscribeRecordingInputs: (
+      listener: RecordingInputListener,
+    ) => Effect.Effect<void, never, Scope.Scope>;
     readonly subscribeRecordingFrames: (
       listener: RecordingFrameListener,
     ) => Effect.Effect<void, never, Scope.Scope>;
 @@ -5011,6 +5152,7 @@ export const make = Effect.gen(function* PreviewManagerMake() {
     subscribeStateChanges: operations.subscribeStateChanges,
     subscribePointerEvents: operations.subscribePointerEvents,
     subscribeRecordingFrames: operations.subscribeRecordingFrames,
+    subscribeRecordingInputs: operations.subscribeRecordingInputs,
   });
 }).pipe(Effect.withSpan("PreviewManager.make"));

apps/desktop/src/preview/PickPreload.ts+82

...	@@ -16,6 +16,8 @@ import type {	...	@@ -16,6 +16,8 @@ import type {
16		16
17	import { resolveAnnotationSubmission } from "./AnnotationKeyboard.ts";	17	import { resolveAnnotationSubmission } from "./AnnotationKeyboard.ts";
18	import { previewAnnotationStyles } from "./AnnotationStyles.generated.ts";	18	import { previewAnnotationStyles } from "./AnnotationStyles.generated.ts";
		19	import { installRecordingCursor } from "./RecordingCursor.ts";
		20	import { DEFAULT_RECORDING_INPUT_OPTIONS } from "./RecordingInput.ts";
19	import {	21	import {
20	ANNOTATION_CAPTURED_CHANNEL,	22	ANNOTATION_CAPTURED_CHANNEL,
21	ANNOTATION_THEME_CHANNEL,	23	ANNOTATION_THEME_CHANNEL,
...	@@ -23,6 +25,11 @@ import {	...	@@ -23,6 +25,11 @@ import {
23	ELEMENT_PICKED_CHANNEL,	25	ELEMENT_PICKED_CHANNEL,
24	HUMAN_INPUT_CHANNEL,	26	HUMAN_INPUT_CHANNEL,
25	MOUSE_NAVIGATE_CHANNEL,	27	MOUSE_NAVIGATE_CHANNEL,
		28	RECORDING_CURSOR_CHANNEL,
		29	RECORDING_POINTER_CHANNEL,
		30	RECORDING_KEY_CHANNEL,
		31	RECORDING_INPUT_CHANNEL,
		32	RECORDING_CONTROLLER_CHANNEL,
26	START_PICK_CHANNEL,	33	START_PICK_CHANNEL,
27	} from "./GuestProtocol.ts";	34	} from "./GuestProtocol.ts";
28	const OVERLAY_ATTRIBUTE = "data-t3code-annotation-ui";	35	const OVERLAY_ATTRIBUTE = "data-t3code-annotation-ui";
...	@@ -35,6 +42,80 @@ const ELEMENT_CONTEXT_TIMEOUT_MS = 5_000;	...	@@ -35,6 +42,80 @@ const ELEMENT_CONTEXT_TIMEOUT_MS = 5_000;
35	const CONTENT_LAYER_Z_INDEX = 1;	42	const CONTENT_LAYER_Z_INDEX = 1;
36	const CHROME_LAYER_Z_INDEX = 10;	43	const CHROME_LAYER_Z_INDEX = 10;
37		44
		45	let recordingCursor: ReturnType<typeof installRecordingCursor> \| null = null;
		46	ipcRenderer.on(
		47	RECORDING_CURSOR_CHANNEL,
		48	(_event, active: unknown, inputOptions: unknown, controller: unknown) => {
		49	if (active === true) {
		50	const options =
		51	typeof inputOptions === "object" && inputOptions !== null
		52	? {
		53	showKeyPresses:
		54	"showKeyPresses" in inputOptions && inputOptions.showKeyPresses === true,
		55	showMousePresses:
		56	"showMousePresses" in inputOptions && inputOptions.showMousePresses === true,
		57	}
		58	: DEFAULT_RECORDING_INPUT_OPTIONS;
		59	recordingCursor ??= installRecordingCursor(document, window, options, (input) =>
		60	ipcRenderer.send(RECORDING_INPUT_CHANNEL, input),
		61	);
		62	recordingCursor.setTheme(annotationTheme);
		63	if (controller === "agent" \|\| controller === "human" \|\| controller === "none")
		64	recordingCursor.setController(controller);
		65	} else {
		66	recordingCursor?.dispose();
		67	recordingCursor = null;
		68	}
		69	},
		70	);
		71	ipcRenderer.on(RECORDING_CONTROLLER_CHANNEL, (_event, controller: unknown, point: unknown) => {
		72	const humanPoint =
		73	typeof point === "object" &&
		74	point !== null &&
		75	"x" in point &&
		76	typeof point.x === "number" &&
		77	Number.isFinite(point.x) &&
		78	"y" in point &&
		79	typeof point.y === "number" &&
		80	Number.isFinite(point.y)
		81	? { x: point.x, y: point.y }
		82	: undefined;
		83	if (controller === "agent" \|\| controller === "human" \|\| controller === "none")
		84	recordingCursor?.setController(controller, humanPoint);
		85	});
		86	ipcRenderer.on(RECORDING_KEY_CHANNEL, (_event, input: unknown) => {
		87	if (
		88	typeof input !== "object" \|\|
		89	input === null \|\|
		90	!("key" in input) \|\|
		91	typeof input.key !== "string"
		92	)
		93	return;
		94	recordingCursor?.keyPress({
		95	key: input.key,
		96	metaKey: "metaKey" in input && input.metaKey === true,
		97	ctrlKey: "ctrlKey" in input && input.ctrlKey === true,
		98	altKey: "altKey" in input && input.altKey === true,
		99	shiftKey: "shiftKey" in input && input.shiftKey === true,
		100	});
		101	});
		102	ipcRenderer.on(RECORDING_POINTER_CHANNEL, (_event, point: unknown) => {
		103	if (
		104	typeof point === "object" &&
		105	point !== null &&
		106	"x" in point &&
		107	typeof point.x === "number" &&
		108	Number.isFinite(point.x) &&
		109	"y" in point &&
		110	typeof point.y === "number" &&
		111	Number.isFinite(point.y)
		112	)
		113	recordingCursor?.move(
		114	{ x: point.x, y: point.y },
		115	"phase" in point && point.phase === "click" ? "click" : "move",
		116	);
		117	});
		118
38	type AnnotationTool = "select" \| "marquee" \| "draw" \| "erase";	119	type AnnotationTool = "select" \| "marquee" \| "draw" \| "erase";
39		120
40	interface SelectedElement {	121	interface SelectedElement {
...	@@ -1361,6 +1442,7 @@ ipcRenderer.on(START_PICK_CHANNEL, (_event, theme: DesktopPreviewAnnotationTheme	...	@@ -1361,6 +1442,7 @@ ipcRenderer.on(START_PICK_CHANNEL, (_event, theme: DesktopPreviewAnnotationTheme
1361	});	1442	});
1362	ipcRenderer.on(ANNOTATION_THEME_CHANNEL, (_event, theme: DesktopPreviewAnnotationTheme) => {	1443	ipcRenderer.on(ANNOTATION_THEME_CHANNEL, (_event, theme: DesktopPreviewAnnotationTheme) => {
1363	annotationTheme = theme;	1444	annotationTheme = theme;
		1445	recordingCursor?.setTheme(theme);
1364	activeSession?.applyTheme(theme);	1446	activeSession?.applyTheme(theme);
1365	});	1447	});
1366	ipcRenderer.on(CANCEL_PICK_CHANNEL, () => activeSession?.teardown(false));	1448	ipcRenderer.on(CANCEL_PICK_CHANNEL, () => activeSession?.teardown(false));

apps/desktop/src/preview/RecordingCursor.ts created+197

...	@@ -0,0 +1,197 @@
1	import type {
2	DesktopPreviewAnnotationTheme,
3	DesktopPreviewRecordingInput,
4	} from "@t3tools/contracts";
5
6	import {
7	DEFAULT_RECORDING_INPUT_OPTIONS,
8	recordingKeyLabel,
9	recordingKeysAreSensitive,
10	type RecordingInputOptions,
11	type RecordingKeyPress,
12	} from "./RecordingInput.ts";
13
14	/**
15	* Chromium's capture cursor uses native window bounds, which do not follow a
16	* webview's CSS placement or scale. Draw it in the guest's coordinate space
17	* while recording, and make the native cursor transparent to avoid two cursors.
18	*/
19	export function installRecordingCursor(
20	document: Document,
21	window: Window,
22	options: RecordingInputOptions = DEFAULT_RECORDING_INPUT_OPTIONS,
23	emit: (input: DesktopPreviewRecordingInput) => void = () => {},
24	) {
25	const style = document.createElement("style");
26	style.textContent =
27	"html, html * { cursor: none !important; } @media (prefers-reduced-motion: reduce) { [data-t3code-recording-agent-cursor] { transition: none !important; } }";
28	const cursor = document.createElement("div");
29	cursor.setAttribute("aria-hidden", "true");
30	cursor.setAttribute("data-t3code-recording-cursor", "");
31	cursor.style.cssText =
32	"position:fixed;left:0;top:0;width:16px;height:24px;pointer-events:none;z-index:2147483647;display:none;";
33	cursor.innerHTML =
34	'<svg xmlns="http://www.w3.org/2000/svg" width="16" height="24" viewBox="0 0 16 24"><path d="M1 1v18l4-4 4 8 3-1.5-4-8H15Z" fill="black" stroke="white" stroke-width="1.5" stroke-linejoin="round"/></svg>';
35	const agentCursor = document.createElement("div");
36	agentCursor.setAttribute("aria-hidden", "true");
37	agentCursor.setAttribute("data-t3code-recording-agent-cursor", "");
38	agentCursor.style.cssText =
39	"position:fixed;left:0;top:0;width:20px;height:20px;pointer-events:none;z-index:2147483647;display:none;filter:drop-shadow(0 1px 2px #0003);transition:transform 150ms ease-out,opacity 150ms ease-out;";
40	// Match the MousePointer2 icon used by the live AgentBrowserCursor.
41	agentCursor.innerHTML =
42	'<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="var(--recording-cursor-background,white)" stroke="var(--recording-cursor-primary,#2563eb)" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="transform:translate(-2px,-2px)"><path d="M4.037 4.688a.495.495 0 0 1 .651-.651l16 6.5a.5.5 0 0 1-.063.947l-6.124 1.58a2 2 0 0 0-1.438 1.435l-1.579 6.126a.5.5 0 0 1-.947.063z"/></svg>';
43	document.documentElement.append(style, cursor, agentCursor);
44	let controller: "human" \| "agent" \| "none" = "none";
45	let humanPoint: { readonly x: number; readonly y: number } \| null = null;
46	const drawHuman = () => {
47	if (!humanPoint) return;
48	cursor.style.transform = `translate(${humanPoint.x}px, ${humanPoint.y}px)`;
49	cursor.style.display = "block";
50	};
51	let agentActive = false;
52	let agentTimer: number \| undefined;
53	const setController = (
54	next: typeof controller,
55	point?: { readonly x: number; readonly y: number },
56	) => {
57	if (point) humanPoint = point;
58	controller = next;
59	if (next === "agent") cursor.style.display = "none";
60	if (next === "human") drawHuman();
61	if (!agentActive) agentCursor.style.opacity = next === "human" ? "0.18" : "0.35";
62	};
63	const setTheme = (
64	theme: Pick<DesktopPreviewAnnotationTheme, "primary" \| "background"> \| null,
65	) => {
66	agentCursor.style.setProperty("--recording-cursor-primary", theme?.primary ?? "#2563eb");
67	agentCursor.style.setProperty("--recording-cursor-background", theme?.background ?? "white");
68	};
69	let lastKeyLabel: string \| null = null;
70	let pointerHeld = false;
71	let pointerFrame: number \| undefined;
72	let pendingPointer: DesktopPreviewRecordingInput \| undefined;
73	const cancelPendingPointer = () => {
74	if (pointerFrame !== undefined) window.cancelAnimationFrame(pointerFrame);
75	pointerFrame = undefined;
76	pendingPointer = undefined;
77	};
78	const keyPress = (input: RecordingKeyPress, held = false) => {
79	if (!options.showKeyPresses) return;
80	const label = recordingKeysAreSensitive(document)
81	? null
82	: recordingKeyLabel(input, /Mac/.test(window.navigator.platform));
83	lastKeyLabel = label;
84	emit({ type: "key", label, held, width: window.innerWidth });
85	};
86	const pointer = (
87	point: { readonly x: number; readonly y: number },
88	phase: "move" \| "down" \| "up" \| "click",
89	) => {
90	if (!options.showMousePresses \|\| (phase === "move" && !pointerHeld)) return;
91	if (phase === "down") pointerHeld = true;
92	if (phase === "up") pointerHeld = false;
93	const input: DesktopPreviewRecordingInput = {
94	type: "pointer",
95	phase,
96	...point,
97	width: window.innerWidth,
98	height: window.innerHeight,
99	};
100	if (phase === "move") {
101	pendingPointer = input;
102	pointerFrame ??= window.requestAnimationFrame(() => {
103	pointerFrame = undefined;
104	if (pendingPointer) emit(pendingPointer);
105	pendingPointer = undefined;
106	});
107	} else {
108	cancelPendingPointer();
109	emit(input);
110	}
111	};
112	const move = (
113	point: { readonly x: number; readonly y: number },
114	phase: "move" \| "click" = "move",
115	) => {
116	agentCursor.style.transform = `translate(${point.x}px, ${point.y}px)`;
117	agentCursor.style.display = "block";
118	agentCursor.style.opacity = "1";
119	agentActive = true;
120	window.clearTimeout(agentTimer);
121	agentTimer = window.setTimeout(() => {
122	agentActive = false;
123	agentCursor.style.opacity = controller === "human" ? "0.18" : "0.35";
124	}, 700);
125	pointer(point, phase);
126	};
127	const moveHuman = (point: { readonly x: number; readonly y: number }) => {
128	if (controller === "agent") return;
129	humanPoint = point;
130	drawHuman();
131	};
132	const pointerMove = (event: PointerEvent) => {
133	if (event.pointerType === "touch") return;
134	const point = { x: event.clientX, y: event.clientY };
135	moveHuman(point);
136	pointer(point, "move");
137	};
138	const pointerDown = (event: PointerEvent) => {
139	if (event.pointerType === "touch") return;
140	moveHuman({ x: event.clientX, y: event.clientY });
141	pointer({ x: event.clientX, y: event.clientY }, "down");
142	};
143	const pointerUp = (event: PointerEvent) => {
144	if (event.pointerType !== "touch") pointer({ x: event.clientX, y: event.clientY }, "up");
145	};
146	const keyDown = (event: KeyboardEvent) => {
147	if (event.isComposing \|\| event.repeat) return;
148	keyPress(event, true);
149	};
150	const keyUp = () => {
151	if (!options.showKeyPresses) return;
152	emit({
153	type: "key",
154	label: recordingKeysAreSensitive(document) ? null : lastKeyLabel,
155	held: false,
156	width: window.innerWidth,
157	});
158	};
159	const hide = () => {
160	cursor.style.display = "none";
161	pointerHeld = false;
162	cancelPendingPointer();
163	if (options.showKeyPresses \|\| options.showMousePresses) emit({ type: "clear" });
164	};
165	const leave = (event: PointerEvent) => {
166	if (event.relatedTarget === null) hide();
167	};
168	window.addEventListener("pointermove", pointerMove, true);
169	window.addEventListener("pointerdown", pointerDown, true);
170	window.addEventListener("pointerup", pointerUp, true);
171	window.addEventListener("pointercancel", pointerUp, true);
172	window.addEventListener("keydown", keyDown, true);
173	window.addEventListener("keyup", keyUp, true);
174	window.addEventListener("pointerout", leave, true);
175	window.addEventListener("blur", hide);
176	return {
177	move,
178	keyPress,
179	setController,
180	setTheme,
181	dispose: () => {
182	window.removeEventListener("pointermove", pointerMove, true);
183	window.removeEventListener("pointerdown", pointerDown, true);
184	window.removeEventListener("pointerup", pointerUp, true);
185	window.removeEventListener("pointercancel", pointerUp, true);
186	window.removeEventListener("keydown", keyDown, true);
187	window.removeEventListener("keyup", keyUp, true);
188	window.removeEventListener("pointerout", leave, true);
189	window.removeEventListener("blur", hide);
190	cancelPendingPointer();
191	window.clearTimeout(agentTimer);
192	cursor.remove();
193	agentCursor.remove();
194	style.remove();
195	},
196	};
197	}

apps/desktop/src/preview/RecordingInput.test.ts created+51

...	@@ -0,0 +1,51 @@
1	import { describe, expect, it } from "vite-plus/test";
2	import { recordingKeyLabel, recordingKeysAreSensitive } from "./RecordingInput.ts";
3
4	const key = (
5	value: string,
6	modifiers: Partial<{
7	metaKey: boolean;
8	ctrlKey: boolean;
9	altKey: boolean;
10	shiftKey: boolean;
11	}> = {},
12	) => ({
13	key: value,
14	metaKey: false,
15	ctrlKey: false,
16	altKey: false,
17	shiftKey: false,
18	...modifiers,
19	});
20
21	describe("recording key labels", () => {
22	it("formats macOS and other-platform shortcuts", () => {
23	expect(recordingKeyLabel(key("c", { metaKey: true }), true)).toBe("⌘C");
24	expect(recordingKeyLabel(key("c", { ctrlKey: true }), false)).toBe("Ctrl + C");
25	expect(recordingKeyLabel(key("Tab", { altKey: true, shiftKey: true }), true)).toBe("⌥⇧⇥");
26	});
27	it("shows held modifiers once and labels navigation keys", () => {
28	expect(recordingKeyLabel(key("Meta", { metaKey: true }), true)).toBe("⌘");
29	expect(recordingKeyLabel(key("Shift", { shiftKey: true }), false)).toBe("Shift");
30	expect(recordingKeyLabel(key("ArrowLeft"), true)).toBe("←");
31	expect(recordingKeyLabel(key(" "), false)).toBe("Space");
32	});
33	it.each(["Dead", "Unidentified", "Process", ""])("excludes composition key %s", (value) => {
34	expect(recordingKeyLabel(key(value), true)).toBeNull();
35	});
36	});
37
38	describe("recording key privacy", () => {
39	const field = (type: string) => ({ tagName: "INPUT", getAttribute: () => type });
40	const sensitive = (activeElement: unknown) =>
41	recordingKeysAreSensitive({ activeElement } as Document);
42	it("excludes password fields and their shadow-root focus", () => {
43	expect(sensitive(field("password"))).toBe(true);
44	expect(sensitive({ shadowRoot: { activeElement: field("password") } })).toBe(true);
45	expect(sensitive(field("text"))).toBe(false);
46	});
47	it("excludes iframe focus whose field cannot be inspected", () => {
48	expect(sensitive({ tagName: "IFRAME" })).toBe(true);
49	expect(sensitive({ tagName: "SECRET-FIELD" })).toBe(true);
50	});
51	});

apps/desktop/src/preview/RecordingInput.ts created+58

...	@@ -0,0 +1,58 @@
1	export interface RecordingInputOptions {
2	readonly showKeyPresses: boolean;
3	readonly showMousePresses: boolean;
4	}
5
6	export const DEFAULT_RECORDING_INPUT_OPTIONS: RecordingInputOptions = {
7	showKeyPresses: false,
8	showMousePresses: false,
9	};
10
11	export interface RecordingKeyPress {
12	readonly key: string;
13	readonly metaKey: boolean;
14	readonly ctrlKey: boolean;
15	readonly altKey: boolean;
16	readonly shiftKey: boolean;
17	}
18
19	/** Formats a single chord without duplicating a modifier pressed on its own. */
20	export function recordingKeyLabel(input: RecordingKeyPress, isMac: boolean): string \| null {
21	if (["Dead", "Process", "Unidentified", ""].includes(input.key)) return null;
22	const modifiers = [
23	input.ctrlKey \|\| input.key === "Control" ? (isMac ? "⌃" : "Ctrl") : null,
24	input.altKey \|\| input.key === "Alt" ? (isMac ? "⌥" : "Alt") : null,
25	input.shiftKey \|\| input.key === "Shift" ? (isMac ? "⇧" : "Shift") : null,
26	input.metaKey \|\| input.key === "Meta" ? (isMac ? "⌘" : "Win") : null,
27	].filter((value) => value !== null);
28	const labels: Record<string, string> = {
29	Enter: "↵",
30	Tab: "⇥",
31	Backspace: "⌫",
32	Delete: "⌦",
33	Escape: "Esc",
34	ArrowUp: "↑",
35	ArrowDown: "↓",
36	ArrowLeft: "←",
37	ArrowRight: "→",
38	" ": "Space",
39	Space: "Space",
40	};
41	if (!["Control", "Alt", "Shift", "Meta"].includes(input.key)) {
42	modifiers.push(
43	labels[input.key] ?? (input.key.length === 1 ? input.key.toUpperCase() : input.key),
44	);
45	}
46	return modifiers.join(isMac ? "" : " + ");
47	}
48
49	/** Unknown iframe or closed-shadow focus is excluded because its field type cannot be checked. */
50	export function recordingKeysAreSensitive(document: Document): boolean {
51	let element = document.activeElement;
52	while (element?.shadowRoot?.activeElement) element = element.shadowRoot.activeElement;
53	return (
54	element?.tagName === "IFRAME" \|\|
55	element?.tagName.includes("-") === true \|\|
56	element?.getAttribute("type")?.toLowerCase() === "password"
57	);
58	}

apps/web/src/browser/browserRecording.ts+23-2

 @@ -8,6 +8,8 @@ import { previewBridge } from "~/components/preview/previewBridge";
 import { ensureClientSettingsHydrated, getClientSettings } from "~/hooks/useSettings";
 import { appAtomRegistry } from "~/rpc/atomRegistry";
+import { createRecordingCompositor } from "./recordingCompositor";
 import { acquireBrowserSurfaceActivity } from "./browserSurfaceStore";
 export class BrowserRecordingUnavailableError extends Schema.TaggedError<BrowserRecordingUnavailableError>()(
 @@ -123,6 +125,7 @@ interface ActiveRecording {
   releaseSurfaceActivity: (() => void) | null;
   stream: MediaStream | null;
   recorder: MediaRecorder | null;
+  compositor: Awaited<ReturnType<typeof createRecordingCompositor>>;
   savedBlob?: Blob;
   uploadPromise?: Promise<string>;
   lifecycle: BrowserRecordingLifecycle;
 @@ -381,6 +384,8 @@ const captureTabMediaStreamWithTimeout = async (
 };
 const clearActiveRecording = (recording: ActiveRecording): void => {
+  recording.compositor?.dispose();
+  recording.compositor = null;
   recording.releaseSurfaceActivity?.();
   recording.releaseSurfaceActivity = null;
   if (activeRecordings.get(recording.tabId) !== recording) return;
 @@ -525,6 +530,7 @@ export async function startBrowserRecording(
     releaseSurfaceActivity,
     stream: null,
     recorder: null,
+    compositor: null,
     lifecycle: startingLifecycle,
   };
   activeRecordings.set(tabId, recording);
 @@ -534,7 +540,8 @@ export async function startBrowserRecording(
       clearActiveRecording(recording);
       throw cause;
     });
-    const frameRate = getClientSettings().browserRecordingFrameRate;
+    const settings = getClientSettings();
+    const frameRate = settings.browserRecordingFrameRate;
     await waitForBrowserRecordingPaint();
     const throwIfStartupCancelled = async (): Promise<void> => {
       // Once a grant starts, a stop lets startup finish so the caller receives an artifact.
 @@ -614,7 +621,19 @@ export async function startBrowserRecording(
     let recorder: MediaRecorder;
     try {
-      recorder = createMediaRecorder(stream);
+      recording.compositor = await createRecordingCompositor(
+        stream,
+        {
+          showKeyPresses: settings.browserRecordingShowKeyPresses,
+          showMousePresses: settings.browserRecordingShowMousePresses,
+          frameRate,
+        },
+        (listener) =>
+          bridge.recording.onInput((event) => {
+            if (event.tabId === tabId) listener(event.input);
+          }),
+      );
+      recorder = createMediaRecorder(recording.compositor?.stream ?? stream);
       recording.recorder = recorder;
       recorder.addEventListener("dataavailable", (event) => {
         if (event.data.size > 0) chunks.push(event.data);
 @@ -694,6 +713,8 @@ const finalizeBrowserRecording = async (
           cause,
         });
       }
+      recording.compositor?.dispose();
+      recording.compositor = null;
       // Encoding has flushed; release native capture before materializing and saving the file.
       stopMediaStream(recording.stream);
       recording.stream = null;

apps/web/src/browser/recordingCompositor.test.ts created+179

...	@@ -0,0 +1,179 @@
1	import type { DesktopPreviewRecordingInput } from "@t3tools/contracts";
2	import { afterEach, describe, expect, it, vi } from "vite-plus/test";
3
4	import { createRecordingCompositor, RecordingDecorations } from "./recordingCompositor";
5
6	const primaryColor = "oklch(0.65 0.2 310)";
7	vi.mock("./annotationTheme", () => ({
8	readPreviewAnnotationTheme: () => ({ primary: "oklch(0.65 0.2 310)" }),
9	}));
10
11	const options = { showKeyPresses: true, showMousePresses: true, frameRate: 30 };
12	const pointer = (
13	phase: "move" \| "down" \| "up" \| "click",
14	x = 100,
15	): DesktopPreviewRecordingInput => ({
16	type: "pointer",
17	phase,
18	x,
19	y: 80,
20	width: 800,
21	height: 600,
22	});
23	const context = () => ({
24	save: vi.fn(),
25	restore: vi.fn(),
26	beginPath: vi.fn(),
27	fill: vi.fn(),
28	stroke: vi.fn(),
29	ellipse: vi.fn(),
30	roundRect: vi.fn(),
31	fillText: vi.fn(),
32	drawImage: vi.fn(),
33	measureText: () => ({ width: 40 }),
34	globalAlpha: 1,
35	strokeStyle: "",
36	fillStyle: "",
37	});
38
39	describe("recording decorations", () => {
40	it("keeps rings aligned through dragging and stops following the cursor after release", () => {
41	const decorations = new RecordingDecorations(options, primaryColor);
42	const ctx = context();
43	decorations.apply(pointer("down"), 0);
44	decorations.apply(pointer("move", 120), 10);
45	decorations.draw(ctx as unknown as CanvasRenderingContext2D, 1600, 1200, 10);
46	expect(ctx.ellipse.mock.calls[0]?.slice(0, 4)).toEqual([240, 160, 40, 40]);
47	expect(ctx.strokeStyle).toBe(primaryColor);
48	expect(ctx.fillStyle).toBe(primaryColor);
49	expect(decorations.nextRedraw(10)).toBeNull();
50	decorations.apply(pointer("up", 130), 20);
51	decorations.apply(pointer("move", 300), 30);
52	decorations.draw(ctx as unknown as CanvasRenderingContext2D, 1600, 1200, 320);
53	expect(ctx.ellipse.mock.calls[1]?.slice(0, 4)).toEqual([260, 160, 50, 50]);
54	ctx.ellipse.mockClear();
55	decorations.draw(ctx as unknown as CanvasRenderingContext2D, 1600, 1200, 620);
56	expect(ctx.ellipse).not.toHaveBeenCalled();
57	expect(decorations.nextRedraw(620)).toBeNull();
58	});
59
60	it("pulses agent clicks and clears decorations on blur or navigation", () => {
61	const decorations = new RecordingDecorations(options, primaryColor);
62	const ctx = context();
63	decorations.apply(pointer("click"), 0);
64	decorations.draw(ctx as unknown as CanvasRenderingContext2D, 800, 600, 300);
65	expect(ctx.ellipse).toHaveBeenCalledOnce();
66	decorations.apply({ type: "clear" }, 301);
67	ctx.ellipse.mockClear();
68	decorations.draw(ctx as unknown as CanvasRenderingContext2D, 800, 600, 302);
69	expect(ctx.ellipse).not.toHaveBeenCalled();
70	expect(decorations.nextRedraw(302)).toBeNull();
71	});
72
73	it("holds shortcut badges until release, then expires them even on a static page", () => {
74	const decorations = new RecordingDecorations(options, primaryColor);
75	const ctx = context();
76	const key = { type: "key" as const, label: "⌘C", held: true, width: 800 };
77	decorations.apply(key, 0);
78	decorations.draw(ctx as unknown as CanvasRenderingContext2D, 1600, 1200, 5000);
79	expect(ctx.fillText.mock.calls[0]?.slice(0, 3)).toEqual(["⌘C", 800, 1098]);
80	decorations.apply({ ...key, held: false }, 5000);
81	expect(decorations.nextRedraw(5500)).toBe(400);
82	ctx.fillText.mockClear();
83	decorations.draw(ctx as unknown as CanvasRenderingContext2D, 1600, 1200, 5900);
84	expect(ctx.fillText).not.toHaveBeenCalled();
85	});
86
87	it("removes the previous key badge on password focus", () => {
88	const decorations = new RecordingDecorations(options, primaryColor);
89	const ctx = context();
90	decorations.apply({ type: "key", label: "A", held: true, width: 800 }, 0);
91	decorations.apply({ type: "key", label: null, held: true, width: 800 }, 1);
92	decorations.draw(ctx as unknown as CanvasRenderingContext2D, 800, 600, 2);
93	expect(ctx.fillText).not.toHaveBeenCalled();
94	});
95
96	it("honors independent opt-in flags", () => {
97	const decorations = new RecordingDecorations(
98	{ ...options, showMousePresses: false },
99	primaryColor,
100	);
101	const ctx = context();
102	decorations.apply(pointer("down"), 0);
103	decorations.apply({ type: "key", label: "⌘C", held: true, width: 800 }, 0);
104	decorations.draw(ctx as unknown as CanvasRenderingContext2D, 800, 600, 1);
105	expect(ctx.ellipse).not.toHaveBeenCalled();
106	expect(ctx.fillText).toHaveBeenCalledOnce();
107	});
108	});
109
110	describe("detached recording compositor", () => {
111	afterEach(() => vi.unstubAllGlobals());
112
113	it("keeps native capture when both decorations are off", async () => {
114	vi.stubGlobal("document", {
115	createElement: () => {
116	throw new Error("must not allocate");
117	},
118	});
119	expect(
120	await createRecordingCompositor(
121	{} as MediaStream,
122	{
123	...options,
124	showKeyPresses: false,
125	showMousePresses: false,
126	},
127	() => {
128	throw new Error("must not subscribe");
129	},
130	),
131	).toBeNull();
132	});
133
134	it.each([false, true])(
135	"releases the detached output on disposal or playback failure (%s)",
136	async (failPlayback) => {
137	const ctx = context();
138	const stop = vi.fn();
139	const unsubscribe = vi.fn();
140	const cancelFrame = vi.fn();
141	const source = {
142	getVideoTracks: () => [{ getSettings: () => ({ width: 800, height: 600 }) }],
143	} as unknown as MediaStream;
144	const stream = { getTracks: () => [{ stop }] } as unknown as MediaStream;
145	const canvas = { width: 0, height: 0, getContext: () => ctx, captureStream: () => stream };
146	const video = {
147	muted: false,
148	playsInline: false,
149	srcObject: null as MediaStream \| null,
150	readyState: 2,
151	videoWidth: 800,
152	videoHeight: 600,
153	pause: vi.fn(),
154	play: async () => {
155	if (failPlayback) throw new Error("play failed");
156	},
157	requestVideoFrameCallback: () => 1,
158	cancelVideoFrameCallback: cancelFrame,
159	};
160	vi.stubGlobal("document", {
161	createElement: (tag: string) => (tag === "canvas" ? canvas : video),
162	});
163	vi.stubGlobal("window", { clearTimeout: vi.fn(), setTimeout: vi.fn() });
164	const compositor = createRecordingCompositor(source, options, () => unsubscribe);
165	if (failPlayback) await expect(compositor).rejects.toThrow("play failed");
166	else {
167	const result = await compositor;
168	expect(result?.stream).toBe(stream);
169	expect(ctx.drawImage).toHaveBeenCalledOnce();
170	result?.dispose();
171	result?.dispose();
172	}
173	expect(stop).toHaveBeenCalledOnce();
174	expect(unsubscribe).toHaveBeenCalledOnce();
175	expect(cancelFrame).toHaveBeenCalledWith(1);
176	expect(video.srcObject).toBeNull();
177	},
178	);
179	});

apps/web/src/browser/recordingCompositor.ts created+190

...	@@ -0,0 +1,190 @@
1	import type { DesktopPreviewRecordingInput } from "@t3tools/contracts";
2
3	import { readPreviewAnnotationTheme } from "./annotationTheme";
4
5	interface RecordingDecorationOptions {
6	readonly showKeyPresses: boolean;
7	readonly showMousePresses: boolean;
8	readonly frameRate: number;
9	}
10
11	/** Decorates a detached canvas; no recording UI is inserted into the preview page. */
12	export async function createRecordingCompositor(
13	source: MediaStream,
14	options: RecordingDecorationOptions,
15	subscribe: (listener: (input: DesktopPreviewRecordingInput) => void) => () => void,
16	) {
17	if (!options.showKeyPresses && !options.showMousePresses) return null;
18	const canvas = document.createElement("canvas");
19	const context = canvas.getContext("2d", { alpha: false });
20	if (!context) throw new Error("Recording canvas is unavailable.");
21	const video = document.createElement("video");
22	video.muted = true;
23	video.playsInline = true;
24	video.srcObject = source;
25	const settings = source.getVideoTracks()[0]?.getSettings();
26	canvas.width = settings?.width ?? 1920;
27	canvas.height = settings?.height ?? 1080;
28	const decorations = new RecordingDecorations(options, readPreviewAnnotationTheme().primary);
29	let disposed = false;
30	let frameId: number \| undefined;
31	let timer: number \| undefined;
32	const draw = () => {
33	if (disposed \|\| video.readyState < 2) return;
34	const width = video.videoWidth \|\| canvas.width;
35	const height = video.videoHeight \|\| canvas.height;
36	if (canvas.width !== width) canvas.width = width;
37	if (canvas.height !== height) canvas.height = height;
38	context.drawImage(video, 0, 0, width, height);
39	const now = performance.now();
40	decorations.draw(context, width, height, now);
41	window.clearTimeout(timer);
42	const next = decorations.nextRedraw(now);
43	if (next !== null) timer = window.setTimeout(draw, next);
44	};
45	const frame = () => {
46	if (disposed) return;
47	draw();
48	frameId = video.requestVideoFrameCallback(frame);
49	};
50	const output = canvas.captureStream(options.frameRate);
51	let unsubscribe: (() => void) \| undefined;
52	const dispose = () => {
53	if (disposed) return;
54	disposed = true;
55	unsubscribe?.();
56	window.clearTimeout(timer);
57	if (frameId !== undefined) video.cancelVideoFrameCallback(frameId);
58	video.pause();
59	video.srcObject = null;
60	for (const track of output.getTracks()) track.stop();
61	};
62	try {
63	unsubscribe = subscribe((input) => {
64	decorations.apply(input, performance.now());
65	draw();
66	});
67	frameId = video.requestVideoFrameCallback(frame);
68	await video.play();
69	draw();
70	return { stream: output, dispose };
71	} catch (error) {
72	dispose();
73	throw error;
74	}
75	}
76
77	/** Keeps input timing and coordinates independent of native video frame delivery. */
78	export class RecordingDecorations {
79	private ring: {
80	x: number;
81	y: number;
82	width: number;
83	height: number;
84	held: boolean;
85	releasedAt: number \| null;
86	} \| null = null;
87	private key: { label: string; width: number; expiresAt: number \| null } \| null = null;
88
89	constructor(
90	private readonly options: RecordingDecorationOptions,
91	private readonly primaryColor: string,
92	) {}
93
94	apply(input: DesktopPreviewRecordingInput, now: number) {
95	if (input.type === "clear") {
96	this.ring = null;
97	this.key = null;
98	} else if (input.type === "key" && this.options.showKeyPresses) {
99	this.key = input.label
100	? { label: input.label, width: input.width, expiresAt: input.held ? null : now + 900 }
101	: null;
102	} else if (input.type === "pointer" && this.options.showMousePresses) {
103	if (input.phase === "down" \|\| input.phase === "click") {
104	this.ring = {
105	x: input.x,
106	y: input.y,
107	width: input.width,
108	height: input.height,
109	held: input.phase === "down",
110	releasedAt: input.phase === "click" ? now : null,
111	};
112	} else if (this.ring?.held) {
113	this.ring = {
114	...this.ring,
115	x: input.x,
116	y: input.y,
117	width: input.width,
118	height: input.height,
119	held: input.phase !== "up",
120	releasedAt: input.phase === "up" ? now : null,
121	};
122	}
123	}
124	}
125
126	nextRedraw(now: number): number \| null {
127	if (
128	this.ring?.releasedAt !== null &&
129	this.ring?.releasedAt !== undefined &&
130	now < this.ring.releasedAt + 600
131	) {
132	return 1000 / this.options.frameRate;
133	}
134	return this.key?.expiresAt !== null &&
135	this.key?.expiresAt !== undefined &&
136	now < this.key.expiresAt
137	? this.key.expiresAt - now
138	: null;
139	}
140
141	draw(context: CanvasRenderingContext2D, width: number, height: number, now: number) {
142	// Guest coordinates are CSS pixels; native frames include zoom and display scale.
143	const scale = width / (this.key?.width ?? this.ring?.width ?? 1280);
144	const ring = this.ring;
145	if (ring && (ring.held \|\| (ring.releasedAt !== null && now < ring.releasedAt + 600))) {
146	const progress = ring.releasedAt === null ? 0 : Math.min(1, (now - ring.releasedAt) / 600);
147	context.save();
148	const opacity = 0.9 * (1 - progress);
149	context.strokeStyle = this.primaryColor;
150	context.fillStyle = this.primaryColor;
151	context.lineWidth = 2 * scale;
152	context.beginPath();
153	context.ellipse(
154	(ring.x * width) / ring.width,
155	(ring.y * height) / ring.height,
156	((20 * width) / ring.width) * (1 + progress * 0.5),
157	((20 * height) / ring.height) * (1 + progress * 0.5),
158	0,
159	0,
160	Math.PI * 2,
161	);
162	context.globalAlpha = opacity * 0.15;
163	context.fill();
164	context.globalAlpha = opacity;
165	context.stroke();
166	context.restore();
167	}
168	const key = this.key;
169	if (key && (key.expiresAt === null \|\| now < key.expiresAt)) {
170	context.save();
171	context.font = `500 ${26 * scale}px system-ui, sans-serif`;
172	const badgeWidth = Math.min(
173	width - 32 * scale,
174	context.measureText(key.label).width + 36 * scale,
175	);
176	const badgeHeight = 54 * scale;
177	const left = (width - badgeWidth) / 2;
178	const top = height - 24 * scale - badgeHeight;
179	context.fillStyle = "rgba(32,32,34,.86)";
180	context.beginPath();
181	context.roundRect(left, top, badgeWidth, badgeHeight, 14 * scale);
182	context.fill();
183	context.fillStyle = "white";
184	context.textAlign = "center";
185	context.textBaseline = "middle";
186	context.fillText(key.label, width / 2, top + badgeHeight / 2, badgeWidth - 24 * scale);
187	context.restore();
188	}
189	}
190	}

apps/web/src/components/preview/AgentBrowserCursor.tsx+4-4

 @@ -24,7 +24,6 @@ export function AgentBrowserCursor(props: {
   return (
     <AgentBrowserCursorEvent
-      key={event.sequence}
       event={event}
       content={content}
       zoomFactor={zoomFactor}
 @@ -46,12 +45,13 @@ function AgentBrowserCursorEvent(props: {
   readonly controller: BrowserController;
 }) {
   const { event, content, zoomFactor, controller } = props;
-  const [active, setActive] = useState(true);
+  const [inactiveSequence, setInactiveSequence] = useState<number | null>(null);
+  const active = inactiveSequence !== event.sequence;
   useEffect(() => {
-    const timeout = window.setTimeout(() => setActive(false), CURSOR_ACTIVE_MS);
+    const timeout = window.setTimeout(() => setInactiveSequence(event.sequence), CURSOR_ACTIVE_MS);
     return () => window.clearTimeout(timeout);
-  }, []);
+  }, [event.sequence]);
   return (
     <div

apps/web/src/components/preview/PreviewView.test.tsx+39-4

 @@ -7,7 +7,7 @@ import {
   FILL_PREVIEW_VIEWPORT,
   ThreadId,
 } from "@t3tools/contracts";
-import { act, Profiler } from "react";
+import { act, createElement, Profiler } from "react";
 import { renderToStaticMarkup } from "react-dom/server";
 import { beforeEach, describe, expect, it, vi } from "vite-plus/test";
 @@ -33,6 +33,9 @@ const mocks = vi.hoisted(() => ({
   pictureInPicture: false,
   showEmptyState: false,
   loading: false,
+  serverEpoch: null as string | null,
+  recordingTabIds: new Set<string>(),
+  recordingRuntimeTabId: null as string | null,
   recordVisitForThread: vi.fn(),
 }));
 @@ -98,6 +101,7 @@ vi.mock("~/previewStateStore", () => ({
   updatePreviewServerSnapshot: vi.fn(),
   useThreadPreviewState: () => ({
     activeTabId: "tab-1",
+    serverEpoch: mocks.serverEpoch,
     desktopByTabId: {
       "tab-1": {
         hasWebContents: true,
 @@ -146,11 +150,11 @@ vi.mock("~/state/use-atom-command", () => ({
 }));
 vi.mock("~/browser/browserRecording", () => ({
-  findActiveBrowserRecordingRuntimeTabId: vi.fn(() => null),
+  findActiveBrowserRecordingRuntimeTabId: () => mocks.recordingRuntimeTabId,
   isBrowserRecordingStartCancelledError: vi.fn(() => false),
   startBrowserRecording: vi.fn(),
   stopBrowserRecording: vi.fn(),
-  useActiveBrowserRecordingTabIds: () => new Set(),
+  useActiveBrowserRecordingTabIds: () => mocks.recordingTabIds,
 }));
 vi.mock("~/browser/browserSurfaceStore", () => ({
 @@ -244,7 +248,9 @@ vi.mock("./PreviewMoreMenu", () => ({
 }));
 vi.mock("./PreviewUnreachable", () => ({ PreviewUnreachable: () => null }));
 vi.mock("./ZoomIndicator", () => ({ ZoomIndicator: () => null }));
-vi.mock("./AgentBrowserCursor", () => ({ AgentBrowserCursor: () => null }));
+vi.mock("./AgentBrowserCursor", () => ({
+  AgentBrowserCursor: () => createElement("agent-cursor"),
+}));
 vi.mock("~/browser/BrowserSurfaceSlot", () => ({ BrowserSurfaceSlot: () => null }));
 vi.mock("./usePreviewSession", () => ({ usePreviewSession: vi.fn() }));
 @@ -344,9 +350,38 @@ describe("PreviewView navigation", () => {
     mocks.pictureInPicture = false;
     mocks.showEmptyState = false;
     mocks.loading = false;
+    mocks.serverEpoch = null;
+    mocks.recordingTabIds = new Set();
+    mocks.recordingRuntimeTabId = null;
     mocks.recordVisitForThread.mockClear();
   });
+  it("shows the cursor in a replacement browser while the old instance still records", async () => {
+    const document = installTestDom();
+    const { createRoot } = await import("react-dom/client");
+    const container = document.createElement("div");
+    const root = createRoot(container as unknown as Element);
+    const hasCursor = (node: TestNode): boolean =>
+      node.nodeName === "AGENT-CURSOR" || node.childNodes.some(hasCursor);
+    mocks.recordingTabIds.add(TEST_RUNTIME_TAB_ID);
+    mocks.recordingRuntimeTabId = TEST_RUNTIME_TAB_ID;
+    try {
+      await act(() => {
+        root.render(<PreviewView threadRef={TEST_THREAD_REF} tabId="tab-1" visible />);
+      });
+      expect(hasCursor(container)).toBe(false);
+      mocks.serverEpoch = "replacement-server";
+      await act(() => {
+        root.render(<PreviewView threadRef={TEST_THREAD_REF} tabId="tab-1" visible />);
+      });
+      expect(hasCursor(container)).toBe(true);
+      expect(mocks.recordingTabIds.has(TEST_RUNTIME_TAB_ID)).toBe(true);
+    } finally {
+      await act(() => root.unmount());
+      vi.unstubAllGlobals();
+    }
+  });
   it("does not rerender while loading time passes", async () => {
     vi.useFakeTimers();
     mocks.loading = true;

apps/web/src/components/preview/PreviewView.tsx+5-6

 @@ -798,18 +798,17 @@ export function PreviewView({
         {snapshot && desktopOverlay ? (
           <ZoomIndicator zoomFactor={desktopOverlay.zoomFactor} />
         ) : null}
-        {runtimeTabId && desktopOverlay && !showEmptyState && !isUnreachable ? (
+        {runtimeTabId &&
+        desktopOverlay &&
+        !showEmptyState &&
+        !isUnreachable &&
+        !activeRecordingTabIds.has(runtimeTabId) ? (
           <AgentBrowserCursor
             tabId={runtimeTabId}
             zoomFactor={desktopOverlay.zoomFactor}
             controller={controller}
           />
         ) : null}
-        {controller !== "none" ? (
-          <div className="pointer-events-none absolute left-3 top-3 z-40 rounded-full border border-border/70 bg-background/90 px-2.5 py-1 text-[11px] font-medium shadow-sm backdrop-blur">
-            {controller === "agent" ? "Agent controlling browser" : "Human control"}
-          </div>
-        ) : null}
         {navStatus._tag === "LoadFailed" ? (
           <div className="absolute inset-0 z-10 bg-background">
             <PreviewUnreachable

apps/web/src/components/settings/IntegrationsSettings.tsx+39

...	@@ -470,6 +470,44 @@ function BrowserAppearanceSetting({ disabled }: { readonly disabled: boolean })	...	@@ -470,6 +470,44 @@ function BrowserAppearanceSetting({ disabled }: { readonly disabled: boolean })
470	);	470	);
471	}	471	}
472		472
		473	function BrowserRecordingInputSettings({ disabled }: { readonly disabled: boolean }) {
		474	const showKeys = useClientSettings((settings) => settings.browserRecordingShowKeyPresses);
		475	const showMouse = useClientSettings((settings) => settings.browserRecordingShowMousePresses);
		476	const updateSettings = useUpdatePrimarySettings();
		477	return (
		478	<>
		479	<SettingsRow
		480	{...searchableSetting("browser-recording-key-presses")}
		481	description="Show pressed keys and shortcuts in new recordings. Password fields are excluded."
		482	control={
		483	<Switch
		484	disabled={disabled}
		485	checked={showKeys}
		486	aria-label="Show key presses in recordings"
		487	onCheckedChange={(checked) =>
		488	updateSettings({ browserRecordingShowKeyPresses: Boolean(checked) })
		489	}
		490	/>
		491	}
		492	/>
		493	<SettingsRow
		494	{...searchableSetting("browser-recording-mouse-presses")}
		495	description="Highlight mouse presses and held buttons in new recordings."
		496	control={
		497	<Switch
		498	disabled={disabled}
		499	checked={showMouse}
		500	aria-label="Show mouse presses in recordings"
		501	onCheckedChange={(checked) =>
		502	updateSettings({ browserRecordingShowMousePresses: Boolean(checked) })
		503	}
		504	/>
		505	}
		506	/>
		507	</>
		508	);
		509	}
		510
473	function BrowserRecordingFrameRateSetting({ disabled }: { readonly disabled: boolean }) {	511	function BrowserRecordingFrameRateSetting({ disabled }: { readonly disabled: boolean }) {
474	const frameRate = useClientSettings((settings) => settings.browserRecordingFrameRate);	512	const frameRate = useClientSettings((settings) => settings.browserRecordingFrameRate);
475	const updateSettings = useUpdatePrimarySettings();	513	const updateSettings = useUpdatePrimarySettings();
...	@@ -1326,6 +1364,7 @@ export function IntegrationsSettingsPanel() {	...	@@ -1326,6 +1364,7 @@ export function IntegrationsSettingsPanel() {
1326	<BrowserZoomSetting disabled={previewDefaultsDisabled} />	1364	<BrowserZoomSetting disabled={previewDefaultsDisabled} />
1327	<BrowserAppearanceSetting disabled={previewDefaultsDisabled} />	1365	<BrowserAppearanceSetting disabled={previewDefaultsDisabled} />
1328	<BrowserRecordingFrameRateSetting disabled={previewDefaultsDisabled} />	1366	<BrowserRecordingFrameRateSetting disabled={previewDefaultsDisabled} />
		1367	<BrowserRecordingInputSettings disabled={previewDefaultsDisabled} />
1329	<BrowserLinkTargetSetting disabled={previewDefaultsDisabled} />	1368	<BrowserLinkTargetSetting disabled={previewDefaultsDisabled} />
1330	<BrowserAutoShowFloatingPreviewSetting disabled={previewDefaultsDisabled} />	1369	<BrowserAutoShowFloatingPreviewSetting disabled={previewDefaultsDisabled} />
1331	</>	1370	</>

apps/web/src/components/settings/SettingsPanels.logic.test.ts+4

...	@@ -268,6 +268,8 @@ describe("getChangedBrowserSettingLabels", () => {	...	@@ -268,6 +268,8 @@ describe("getChangedBrowserSettingLabels", () => {
268	browserDefaultZoomFactor: 1.5,	268	browserDefaultZoomFactor: 1.5,
269	browserDefaultAppearance: "dark",	269	browserDefaultAppearance: "dark",
270	browserRecordingFrameRate: 60,	270	browserRecordingFrameRate: 60,
		271	browserRecordingShowKeyPresses: true,
		272	browserRecordingShowMousePresses: true,
271	browserLinkTarget: "app",	273	browserLinkTarget: "app",
272	browserAutoShowFloatingPreview: !DEFAULT_UNIFIED_SETTINGS.browserAutoShowFloatingPreview,	274	browserAutoShowFloatingPreview: !DEFAULT_UNIFIED_SETTINGS.browserAutoShowFloatingPreview,
273	}),	275	}),
...	@@ -276,6 +278,8 @@ describe("getChangedBrowserSettingLabels", () => {	...	@@ -276,6 +278,8 @@ describe("getChangedBrowserSettingLabels", () => {
276	"Browser zoom",	278	"Browser zoom",
277	"Browser appearance",	279	"Browser appearance",
278	"Recording frame rate",	280	"Recording frame rate",
		281	"Recording key presses",
		282	"Recording mouse presses",
279	"Open links in",	283	"Open links in",
280	"Floating preview",	284	"Floating preview",
281	]);	285	]);

apps/web/src/components/settings/SettingsPanels.logic.ts+4

...	@@ -115,6 +115,8 @@ export type BrowserDefaultSettings = Pick<	...	@@ -115,6 +115,8 @@ export type BrowserDefaultSettings = Pick<
115	\| "browserDefaultZoomFactor"	115	\| "browserDefaultZoomFactor"
116	\| "browserDefaultAppearance"	116	\| "browserDefaultAppearance"
117	\| "browserRecordingFrameRate"	117	\| "browserRecordingFrameRate"
		118	\| "browserRecordingShowKeyPresses"
		119	\| "browserRecordingShowMousePresses"
118	\| "browserLinkTarget"	120	\| "browserLinkTarget"
119	\| "browserAutoShowFloatingPreview"	121	\| "browserAutoShowFloatingPreview"
120	>;	122	>;
...	@@ -156,6 +158,8 @@ export function getChangedBrowserSettingLabels(settings: BrowserDefaultSettings)	...	@@ -156,6 +158,8 @@ export function getChangedBrowserSettingLabels(settings: BrowserDefaultSettings)
156	...(settings.browserRecordingFrameRate !== DEFAULT_UNIFIED_SETTINGS.browserRecordingFrameRate	158	...(settings.browserRecordingFrameRate !== DEFAULT_UNIFIED_SETTINGS.browserRecordingFrameRate
157	? ["Recording frame rate"]	159	? ["Recording frame rate"]
158	: []),	160	: []),
		161	...(settings.browserRecordingShowKeyPresses ? ["Recording key presses"] : []),
		162	...(settings.browserRecordingShowMousePresses ? ["Recording mouse presses"] : []),
159	...(settings.browserLinkTarget !== DEFAULT_UNIFIED_SETTINGS.browserLinkTarget	163	...(settings.browserLinkTarget !== DEFAULT_UNIFIED_SETTINGS.browserLinkTarget
160	? ["Open links in"]	164	? ["Open links in"]
161	: []),	165	: []),

apps/web/src/components/settings/SettingsPanels.tsx+4

apps/web/src/components/settings/settingsSearch.ts+12

...	@@ -615,6 +615,18 @@ export const SETTINGS_SEARCH_ITEMS = [	...	@@ -615,6 +615,18 @@ export const SETTINGS_SEARCH_ITEMS = [
615	title: "Browser recording frame rate",	615	title: "Browser recording frame rate",
616	to: "/settings/integrations",	616	to: "/settings/integrations",
617	},	617	},
		618	{
		619	id: "browser-recording-key-presses",
		620	title: "Show key presses in recordings",
		621	to: "/settings/integrations",
		622	searchTerms: ["browser preview keyboard shortcuts keystrokes overlay capture"],
		623	},
		624	{
		625	id: "browser-recording-mouse-presses",
		626	title: "Show mouse presses in recordings",
		627	to: "/settings/integrations",
		628	searchTerms: ["browser preview clicks buttons drag overlay capture"],
		629	},
618	{	630	{
619	id: "browser-link-target",	631	id: "browser-link-target",
620	title: "Open links in",	632	title: "Open links in",

packages/contracts/src/ipc.ts+25

...	@@ -658,6 +658,30 @@ export interface DesktopPreviewPointerEvent {	...	@@ -658,6 +658,30 @@ export interface DesktopPreviewPointerEvent {
658	createdAt: string;	658	createdAt: string;
659	}	659	}
660		660
		661	/** Recording decorations are forwarded separately from the captured page pixels. */
		662	export const DesktopPreviewRecordingInputSchema = Schema.Union([
		663	Schema.Struct({
		664	type: Schema.Literal("pointer"),
		665	phase: Schema.Literals(["move", "down", "up", "click"]),
		666	x: Schema.Finite,
		667	y: Schema.Finite,
		668	width: Schema.Finite.check(Schema.isGreaterThan(0)),
		669	height: Schema.Finite.check(Schema.isGreaterThan(0)),
		670	}),
		671	Schema.Struct({
		672	type: Schema.Literal("key"),
		673	label: Schema.NullOr(Schema.String.check(Schema.isMaxLength(100))),
		674	held: Schema.Boolean,
		675	width: Schema.Finite.check(Schema.isGreaterThan(0)),
		676	}),
		677	Schema.Struct({ type: Schema.Literal("clear") }),
		678	]);
		679	export type DesktopPreviewRecordingInput = typeof DesktopPreviewRecordingInputSchema.Type;
		680	export interface DesktopPreviewRecordingInputEvent {
		681	readonly tabId: string;
		682	readonly input: DesktopPreviewRecordingInput;
		683	}
		684
661	/**	685	/**
662	* Static config a renderer needs to mount a preview `<webview>`. Returned	686	* Static config a renderer needs to mount a preview `<webview>`. Returned
663	* atomically by `DesktopPreviewBridge.getPreviewConfig()` so the renderer	687	* atomically by `DesktopPreviewBridge.getPreviewConfig()` so the renderer
...	@@ -1295,6 +1319,7 @@ export interface DesktopPreviewBridge {	...	@@ -1295,6 +1319,7 @@ export interface DesktopPreviewBridge {
1295	close: (tabId: string) => Promise<void>;	1319	close: (tabId: string) => Promise<void>;
1296	};	1320	};
1297	recording: {	1321	recording: {
		1322	onInput: (listener: (event: DesktopPreviewRecordingInputEvent) => void) => () => void;
1298	startScreencast: (tabId: string) => Promise<void>;	1323	startScreencast: (tabId: string) => Promise<void>;
1299	stopScreencast: (tabId: string) => Promise<void>;	1324	stopScreencast: (tabId: string) => Promise<void>;
1300	save: (	1325	save: (

packages/contracts/src/settings.test.ts+17

...	@@ -471,6 +471,23 @@ describe("ClientSettings browser recording frame rate", () => {	...	@@ -471,6 +471,23 @@ describe("ClientSettings browser recording frame rate", () => {
471	});	471	});
472	});	472	});
473		473
		474	describe("ClientSettings recording input overlays", () => {
		475	it("defaults both overlays off and accepts independent opt-ins", () => {
		476	const settings = decodeClientSettings({});
		477	expect(settings.browserRecordingShowKeyPresses).toBe(false);
		478	expect(settings.browserRecordingShowMousePresses).toBe(false);
		479	expect(
		480	decodeClientSettingsPatch({
		481	browserRecordingShowKeyPresses: true,
		482	browserRecordingShowMousePresses: false,
		483	}),
		484	).toMatchObject({
		485	browserRecordingShowKeyPresses: true,
		486	browserRecordingShowMousePresses: false,
		487	});
		488	});
		489	});
		490
474	describe("ClientSettings glass opacity", () => {	491	describe("ClientSettings glass opacity", () => {
475	it("defaults to a readable translucent surface", () => {	492	it("defaults to a readable translucent surface", () => {
476	expect(decodeClientSettings({}).glassOpacity).toBe(80);	493	expect(decodeClientSettings({}).glassOpacity).toBe(80);

packages/contracts/src/settings.ts+8

 @@ -317,6 +317,12 @@ export const ClientSettingsSchema = Schema.Struct({
   browserRecordingFrameRate: BrowserRecordingFrameRate.pipe(
     Schema.withDecodingDefault(Effect.succeed(DEFAULT_BROWSER_RECORDING_FRAME_RATE)),
   ),
+  browserRecordingShowKeyPresses: Schema.Boolean.pipe(
+    Schema.withDecodingDefault(Effect.succeed(false)),
+  ),
+  browserRecordingShowMousePresses: Schema.Boolean.pipe(
+    Schema.withDecodingDefault(Effect.succeed(false)),
+  ),
   /**
    * Where links clicked in a thread (chat markdown, terminal output) open.
    * Only the desktop app has an in-app browser, so other clients ignore "app".
 @@ -1525,6 +1531,8 @@ export const ClientSettingsPatch = Schema.Struct({
   browserDefaultZoomFactor: Schema.optionalKey(PreviewZoomFactor),
   browserDefaultAppearance: Schema.optionalKey(PreviewAppearancePreference),
   browserRecordingFrameRate: Schema.optionalKey(BrowserRecordingFrameRate),
+  browserRecordingShowKeyPresses: Schema.optionalKey(Schema.Boolean),
+  browserRecordingShowMousePresses: Schema.optionalKey(Schema.Boolean),
   browserLinkTarget: Schema.optionalKey(BrowserLinkTarget),
   browserAutoShowFloatingPreview: Schema.optionalKey(Schema.Boolean),
   browserProfiles: Schema.optionalKey(Schema.Array(BrowserProfile)),

author	Julius Marminge <julius0216@outlook.com> 2026-09-20 13:48:27-07:00
committer	GitHub <noreply@github.com> 2026-09-20 13:48:27-07:00
log	55c24273a025874cdb8a96a25ee0c59eeb4e3051
tree	701b8681bf46985699d73b5ea448e412e349fc0a
parent	6a699f0f2fbd8847d7ec2df8d9245ce8c2eb8707
signature	Signed by PGP key B5690EEEBB952194

...	@@ -635,6 +635,8 @@ export function useSettingsRestore(onRestored?: () => void) {	...	@@ -635,6 +635,8 @@ export function useSettingsRestore(onRestored?: () => void) {
635	settings.browserDefaultZoomFactor,	635	settings.browserDefaultZoomFactor,
636	settings.browserDefaultAppearance,	636	settings.browserDefaultAppearance,
637	settings.browserRecordingFrameRate,	637	settings.browserRecordingFrameRate,
		638	settings.browserRecordingShowKeyPresses,
		639	settings.browserRecordingShowMousePresses,
638	settings.browserLinkTarget,	640	settings.browserLinkTarget,
639	settings.browserAutoShowFloatingPreview,	641	settings.browserAutoShowFloatingPreview,
640	settings.appearanceContrast,	642	settings.appearanceContrast,
...	@@ -798,6 +800,8 @@ export function useSettingsRestore(onRestored?: () => void) {	...	@@ -798,6 +800,8 @@ export function useSettingsRestore(onRestored?: () => void) {
798	browserDefaultZoomFactor: DEFAULT_UNIFIED_SETTINGS.browserDefaultZoomFactor,	800	browserDefaultZoomFactor: DEFAULT_UNIFIED_SETTINGS.browserDefaultZoomFactor,
799	browserDefaultAppearance: DEFAULT_UNIFIED_SETTINGS.browserDefaultAppearance,	801	browserDefaultAppearance: DEFAULT_UNIFIED_SETTINGS.browserDefaultAppearance,
800	browserRecordingFrameRate: DEFAULT_UNIFIED_SETTINGS.browserRecordingFrameRate,	802	browserRecordingFrameRate: DEFAULT_UNIFIED_SETTINGS.browserRecordingFrameRate,
		803	browserRecordingShowKeyPresses: DEFAULT_UNIFIED_SETTINGS.browserRecordingShowKeyPresses,
		804	browserRecordingShowMousePresses: DEFAULT_UNIFIED_SETTINGS.browserRecordingShowMousePresses,
801	browserLinkTarget: DEFAULT_UNIFIED_SETTINGS.browserLinkTarget,	805	browserLinkTarget: DEFAULT_UNIFIED_SETTINGS.browserLinkTarget,
802	browserAutoShowFloatingPreview: DEFAULT_UNIFIED_SETTINGS.browserAutoShowFloatingPreview,	806	browserAutoShowFloatingPreview: DEFAULT_UNIFIED_SETTINGS.browserAutoShowFloatingPreview,
803	// Re-granted like any other default. The confirmation dialog lists it by	807	// Re-granted like any other default. The confirmation dialog lists it by

fix(desktop): align preview recording cursors and show input feedback (#12779)