Commit 55c24273a025874cdb8a96a25ee0c59eeb4e3051 | the-knife

apps/desktop/src/ipc/channels.ts+2

...	...	@@ -113,3 +113,5 @@ export const PREVIEW_POINTER_EVENT_CHANNEL = "desktop:preview-pointer-event";
113	113	export const MAC_PERMISSION_HELPER_CHANNEL = "desktop:mac-permission-helper";
114	114
115	115	export const CHECK_SYSTEM_PERMISSION_CHANNEL = "desktop:check-system-permission";
	116
	117	export const PREVIEW_RECORDING_INPUT_CHANNEL = "desktop:preview-recording-input";

apps/desktop/src/ipc/methods/preview.ts+20-5

...	...	@@ -30,11 +30,13 @@ import {
30	30	} from "@t3tools/contracts";
31	31	import * as Effect from "effect/Effect";
32	32	import * as Schema from "effect/Schema";
	33	import * as Option from "effect/Option";
33	34	import * as NodeURL from "node:url";
34	35
35	36	import * as ElectronWindow from "../../electron/ElectronWindow.ts";
36	37	import * as BrowserImport from "../../preview/BrowserImport/BrowserImport.ts";
37	38	import * as PreviewManager from "../../preview/Manager.ts";
	39	import * as DesktopClientSettings from "../../settings/DesktopClientSettings.ts";
38	40	import { PREVIEW_WEBVIEW_PREFERENCES } from "../../preview/WebviewPreferences.ts";
39	41	import * as IpcChannels from "../channels.ts";
40	42	import * as DesktopIpc from "../DesktopIpc.ts";
...	...	@@ -50,6 +52,9 @@ export const installPreviewEventForwarding = Effect.fn(
50	52	yield* manager.subscribeRecordingFrames((frame) =>
51	53	electronWindow.sendAll(IpcChannels.PREVIEW_RECORDING_FRAME_CHANNEL, frame),
52	54	);
	55	yield* manager.subscribeRecordingInputs((event) =>
	56	electronWindow.sendAll(IpcChannels.PREVIEW_RECORDING_INPUT_CHANNEL, event),
	57	);
53	58	yield* manager.subscribePointerEvents((event) =>
54	59	electronWindow.sendAll(IpcChannels.PREVIEW_POINTER_EVENT_CHANNEL, event),
55	60	);
...	...	@@ -180,11 +185,21 @@ export const cancelPickElement = tabMethod(
180	185	"desktop.ipc.preview.cancelPickElement",
181	186	(manager, tabId) => manager.cancelPickElement(tabId),
182	187	);
183		export const startRecording = tabMethod(
184		IpcChannels.PREVIEW_RECORDING_START_CHANNEL,
185		"desktop.ipc.preview.startRecording",
186		(manager, tabId) => manager.startRecording(tabId),
187		);
	188	export const startRecording = DesktopIpc.makeIpcMethod({
	189	channel: IpcChannels.PREVIEW_RECORDING_START_CHANNEL,
	190	payload: DesktopPreviewTabInputSchema,
	191	result: Schema.Void,
	192	handler: Effect.fn("desktop.ipc.preview.startRecording")(function* ({ tabId }) {
	193	const manager = yield* PreviewManager.PreviewManager;
	194	const store = yield* DesktopClientSettings.DesktopClientSettings;
	195	const settings = yield* store.get;
	196	const options = Option.map(settings, (value) => ({
	197	showKeyPresses: value.browserRecordingShowKeyPresses,
	198	showMousePresses: value.browserRecordingShowMousePresses,
	199	}));
	200	yield* manager.startRecording(tabId, Option.getOrUndefined(options));
	201	}),
	202	});
188	203	export const stopRecording = tabMethod(
189	204	IpcChannels.PREVIEW_RECORDING_STOP_CHANNEL,
190	205	"desktop.ipc.preview.stopRecording",

apps/desktop/src/preload.ts+10

...	...	@@ -1,6 +1,7 @@
1	1	import type {
2	2	DesktopBridge,
3	3	DesktopPreviewPointerEvent,
	4	DesktopPreviewRecordingInputEvent,
4	5	DesktopPreviewRecordingFrame,
5	6	DesktopPreviewTabState,
6	7	DesktopSnapShotEvent,
...	...	@@ -327,6 +328,15 @@ contextBridge.exposeInMainWorld("desktopBridge", {
327	328	ipcRenderer.invoke(IpcChannels.PREVIEW_PICTURE_IN_PICTURE_CLOSE_CHANNEL, { tabId }),
328	329	},
329	330	recording: {
	331	onInput: (listener) => {
	332	const wrappedListener = (_event: Electron.IpcRendererEvent, event: unknown) => {
	333	if (typeof event !== "object" \|\| event === null) return;
	334	listener(event as DesktopPreviewRecordingInputEvent);
	335	};
	336	ipcRenderer.on(IpcChannels.PREVIEW_RECORDING_INPUT_CHANNEL, wrappedListener);
	337	return () =>
	338	ipcRenderer.removeListener(IpcChannels.PREVIEW_RECORDING_INPUT_CHANNEL, wrappedListener);
	339	},
330	340	startScreencast: (tabId) =>
331	341	ipcRenderer.invoke(IpcChannels.PREVIEW_RECORDING_START_CHANNEL, { tabId }),
332	342	stopScreencast: (tabId) =>

apps/desktop/src/preview/GuestProtocol.ts+5

...	...	@@ -5,3 +5,8 @@ export const ANNOTATION_CAPTURED_CHANNEL = "preview:annotation-captured";
5	5	export const ANNOTATION_THEME_CHANNEL = "preview:annotation-theme";
6	6	export const HUMAN_INPUT_CHANNEL = "preview:human-input";
7	7	export const MOUSE_NAVIGATE_CHANNEL = "preview:mouse-navigate";
	8	export const RECORDING_CURSOR_CHANNEL = "preview:recording-cursor";
	9	export const RECORDING_POINTER_CHANNEL = "preview:recording-pointer";
	10	export const RECORDING_KEY_CHANNEL = "preview:recording-key";
	11	export const RECORDING_INPUT_CHANNEL = "preview:recording-input";
	12	export const RECORDING_CONTROLLER_CHANNEL = "preview:recording-controller";

apps/desktop/src/preview/Manager.test.ts+272-80

...	...	@@ -1,7 +1,10 @@
1	1	import * as NodeVM from "node:vm";
2	2	import { it as effectIt } from "@effect/vitest";
3	3	import { DESKTOP_PREVIEW_RECORDING_CAPTURE_TRIGGER } from "@t3tools/contracts";
4		import type { DesktopPreviewRecordingFrame } from "@t3tools/contracts";
	4	import type {
	5	DesktopPreviewRecordingFrame,
	6	DesktopPreviewRecordingInputEvent,
	7	} from "@t3tools/contracts";
5	8	import { HostProcessPlatform } from "@t3tools/shared/hostProcess";
6	9	import * as Cause from "effect/Cause";
7	10	import * as Deferred from "effect/Deferred";
...	...	@@ -2885,6 +2888,165 @@ describe("PreviewManager", () => {
2885	2888	),
2886	2889	);
2887	2890
	2891	effectIt.effect(
	2892	"restores the native cursor when recording startup fails, then allows a retry",
	2893	() =>
	2894	withManager((manager) =>
	2895	Effect.gen(function* () {
	2896	const host = makeTestHostWebContents();
	2897	host.executeJavaScript.mockResolvedValueOnce(false);
	2898	let cursorActive = false;
	2899	const cursorAtCapture: boolean[] = [];
	2900	const contents = Object.assign(
	2901	makeTestPreviewWebContents(
	2902	async () => {
	2903	cursorAtCapture.push(cursorActive);
	2904	return {
	2905	toJPEG: () => Buffer.from("frame"),
	2906	getSize: () => ({ width: 800, height: 600 }),
	2907	};
	2908	},
	2909	42,
	2910	host,
	2911	),
	2912	{
	2913	send: (channel: string, active: unknown) => {
	2914	if (channel === "preview:recording-cursor") cursorActive = active === true;
	2915	},
	2916	},
	2917	);
	2918	fromId.mockReturnValue(contents as never);
	2919	yield* manager.createTab("tab_cursor");
	2920	yield* manager.registerWebview("tab_cursor", 42);
	2921	const failed = yield* Effect.exit(manager.startRecording("tab_cursor"));
	2922	expect(Exit.isFailure(failed)).toBe(true);
	2923	expect(cursorAtCapture).toEqual([true]);
	2924	expect(cursorActive).toBe(false);
	2925
	2926	yield* manager.startRecording("tab_cursor");
	2927	expect(cursorAtCapture).toEqual([true, true]);
	2928	expect(cursorActive).toBe(true);
	2929	yield* manager.stopRecording("tab_cursor");
	2930	expect(cursorActive).toBe(false);
	2931	}),
	2932	),
	2933	);
	2934
	2935	effectIt.effect("restores the recording cursor after navigation only while recording", () =>
	2936	withManager((manager) =>
	2937	Effect.gen(function* () {
	2938	const listeners = new Map<string, () => void>();
	2939	let cursorActive = false;
	2940	let cursorUpdated: (() => void) \| undefined;
	2941	let inputOptions: unknown;
	2942	const options = { showKeyPresses: true, showMousePresses: false };
	2943	const contents = Object.assign(
	2944	makeTestPreviewWebContents(async () => ({
	2945	toJPEG: () => Buffer.from("frame"),
	2946	getSize: () => ({ width: 800, height: 600 }),
	2947	})),
	2948	{
	2949	on: (event: string, listener: () => void) => listeners.set(event, listener),
	2950	send: (channel: string, active: unknown, recordingOptions: unknown) => {
	2951	if (channel !== "preview:recording-cursor") return;
	2952	cursorActive = active === true;
	2953	inputOptions = recordingOptions;
	2954	cursorUpdated?.();
	2955	},
	2956	},
	2957	);
	2958	fromId.mockReturnValue(contents as never);
	2959	yield* manager.createTab("tab_cursor_reload");
	2960	yield* manager.registerWebview("tab_cursor_reload", 42);
	2961	yield* manager.startRecording("tab_cursor_reload", options);
	2962	for (const recording of [true, false]) {
	2963	if (!recording) yield* manager.stopRecording("tab_cursor_reload");
	2964	// A new document has lost the previous preload's cursor overlay.
	2965	cursorActive = false;
	2966	const restored = new Promise<void>((resolve) => {
	2967	cursorUpdated = resolve;
	2968	});
	2969	listeners.get("dom-ready")?.();
	2970	yield* Effect.promise(() => restored);
	2971	cursorUpdated = undefined;
	2972	expect(cursorActive).toBe(recording);
	2973	expect(inputOptions).toEqual(recording ? options : undefined);
	2974	}
	2975	}),
	2976	),
	2977	);
	2978
	2979	effectIt.effect("gates recording decorations and isolates failed subscribers", () =>
	2980	withManager((manager) =>
	2981	Effect.gen(function* () {
	2982	const callbacks = new Map<
	2983	string,
	2984	(event: unknown, input: unknown) => Fiber.Fiber<void, never> \| undefined
	2985	>();
	2986	const contents = Object.assign(
	2987	makeTestPreviewWebContents(async () => ({
	2988	toJPEG: () => Buffer.from("frame"),
	2989	getSize: () => ({ width: 800, height: 600 }),
	2990	})),
	2991	{
	2992	ipc: {
	2993	on: (
	2994	channel: string,
	2995	callback: (event: unknown, input: unknown) => Fiber.Fiber<void, never> \| undefined,
	2996	) => callbacks.set(channel, callback),
	2997	off: vi.fn(),
	2998	},
	2999	},
	3000	);
	3001	fromId.mockReturnValue(contents as never);
	3002	yield* manager.createTab("tab_recording_input");
	3003	yield* manager.registerWebview("tab_recording_input", 42);
	3004	const received: DesktopPreviewRecordingInputEvent[] = [];
	3005	yield* manager.subscribeRecordingInputs(() => Effect.die("renderer unavailable"));
	3006	yield* manager.subscribeRecordingInputs((event) =>
	3007	Effect.sync(() => {
	3008	received.push(event);
	3009	}),
	3010	);
	3011	const send = (input: unknown) =>
	3012	Effect.gen(function* () {
	3013	const fiber = callbacks.get("preview:recording-input")?.(null, input);
	3014	if (fiber) yield* Fiber.join(fiber);
	3015	});
	3016	const key = { type: "key", label: "⌘C", held: true, width: 800 };
	3017	const pointer = {
	3018	type: "pointer",
	3019	phase: "down",
	3020	x: 120,
	3021	y: 80,
	3022	width: 800,
	3023	height: 600,
	3024	};
	3025	yield* send(key);
	3026	expect(received).toEqual([]);
	3027	yield* manager.startRecording("tab_recording_input", {
	3028	showKeyPresses: true,
	3029	showMousePresses: false,
	3030	});
	3031	yield* send(key);
	3032	yield* send(pointer);
	3033	yield* send({ ...key, width: 0 });
	3034	expect(received).toEqual([{ tabId: "tab_recording_input", input: key }]);
	3035	yield* manager.stopRecording("tab_recording_input");
	3036	yield* send(key);
	3037	expect(received).toHaveLength(1);
	3038	yield* manager.startRecording("tab_recording_input", {
	3039	showKeyPresses: false,
	3040	showMousePresses: true,
	3041	});
	3042	yield* send(key);
	3043	yield* send(pointer);
	3044	expect(received.at(-1)).toEqual({ tabId: "tab_recording_input", input: pointer });
	3045	expect(received).toHaveLength(2);
	3046	}),
	3047	),
	3048	);
	3049
2888	3050	effectIt.effect("continues native recording when the source warmup fails", () =>
2889	3051	withManager((manager) =>
2890	3052	Effect.gen(function* () {
...	...	@@ -3853,88 +4015,108 @@ describe("PreviewManager", () => {
3853	4015	),
3854	4016	);
3855	4017
3856		effectIt.effect("emits the resolved pointer target before dispatching an automation click", () =>
3857		withManager((manager) =>
3858		Effect.gen(function* () {
3859		let humanInput: ((_event: unknown, signal: unknown) => void) \| undefined;
3860		const activity: string[] = [];
3861		const sendCommand = vi.fn(async (method: string, params?: Record<string, unknown>) => {
3862		if (method === "Runtime.evaluate") {
3863		return {
3864		result: {
3865		value: { width: 800, height: 600 },
3866		},
3867		};
3868		}
3869		if (method === "Input.dispatchMouseEvent" && params?.type === "mousePressed") {
3870		activity.push("mousePressed");
3871		humanInput?.({}, { kind: "pointer", x: params.x, y: params.y, button: 0 });
3872		}
3873		return undefined;
3874		});
3875		fromId.mockReturnValue({
3876		id: 42,
3877		isDestroyed: () => false,
3878		getType: () => "webview",
3879		getURL: () => "https://example.com",
3880		getTitle: () => "Example",
3881		isLoading: () => false,
3882		isDevToolsOpened: () => false,
3883		getZoomFactor: () => 1,
3884		setZoomFactor: vi.fn(),
3885		setAudioMuted: vi.fn(),
3886		isCurrentlyAudible: () => false,
3887		on: vi.fn(),
3888		off: vi.fn(),
3889		ipc: {
3890		on: vi.fn((channel: string, listener: typeof humanInput) => {
3891		if (channel === "preview:human-input") humanInput = listener;
3892		}),
3893		off: vi.fn(),
3894		},
3895		send: webviewSend,
3896		navigationHistory: { canGoBack: () => false, canGoForward: () => false },
3897		setIgnoreMenuShortcuts: vi.fn(),
3898		setWindowOpenHandler: vi.fn(),
3899		debugger: {
3900		isAttached: () => false,
3901		attach: vi.fn(),
3902		sendCommand,
	4018	effectIt.effect(
	4019	"records the resolved pointer target before dispatching an automation click",
	4020	() =>
	4021	withManager((manager) =>
	4022	Effect.gen(function* () {
	4023	let humanInput: ((_event: unknown, signal: unknown) => void) \| undefined;
	4024	const activity: string[] = [];
	4025	const sendCommand = vi.fn(async (method: string, params?: Record<string, unknown>) => {
	4026	if (method === "Runtime.evaluate") {
	4027	return {
	4028	result: {
	4029	value: { width: 800, height: 600 },
	4030	},
	4031	};
	4032	}
	4033	if (method === "Input.dispatchMouseEvent" && params?.type === "mousePressed") {
	4034	activity.push("mousePressed");
	4035	humanInput?.({}, { kind: "pointer", x: params.x, y: params.y, button: 0 });
	4036	}
	4037	return undefined;
	4038	});
	4039	fromId.mockReturnValue({
	4040	id: 42,
	4041	hostWebContents: makeTestHostWebContents(),
	4042	capturePage: vi.fn(async () => ({ toPNG: () => Buffer.from("frame") })),
	4043	setBackgroundThrottling: vi.fn(),
	4044	isDestroyed: () => false,
	4045	getType: () => "webview",
	4046	getURL: () => "https://example.com",
	4047	getTitle: () => "Example",
	4048	isLoading: () => false,
	4049	isDevToolsOpened: () => false,
	4050	getZoomFactor: () => 1,
	4051	setZoomFactor: vi.fn(),
	4052	setAudioMuted: vi.fn(),
	4053	isCurrentlyAudible: () => false,
3903	4054	on: vi.fn(),
3904	4055	off: vi.fn(),
3905		},
3906		} as never);
3907
3908		yield* manager.subscribePointerEvents((event) =>
3909		Effect.sync(() => {
3910		activity.push(event.phase);
3911		}),
3912		);
3913		yield* manager.createTab("tab_1");
3914		yield* manager.registerWebview("tab_1", 42);
3915		const click = yield* manager
3916		.automationClick("tab_1", { x: 120, y: 80 })
3917		.pipe(Effect.forkChild({ startImmediately: true }));
3918		yield* TestClock.adjust(200);
3919		yield* Fiber.join(click);
	4056	ipc: {
	4057	on: vi.fn((channel: string, listener: typeof humanInput) => {
	4058	if (channel === "preview:human-input") humanInput = listener;
	4059	}),
	4060	off: vi.fn(),
	4061	},
	4062	send: webviewSend,
	4063	navigationHistory: { canGoBack: () => false, canGoForward: () => false },
	4064	setIgnoreMenuShortcuts: vi.fn(),
	4065	setWindowOpenHandler: vi.fn(),
	4066	debugger: {
	4067	isAttached: () => false,
	4068	attach: vi.fn(),
	4069	sendCommand,
	4070	on: vi.fn(),
	4071	off: vi.fn(),
	4072	},
	4073	} as never);
3920	4074
3921		expect(activity).toEqual(["move", "click", "mousePressed"]);
3922		expect(sendCommand).toHaveBeenCalledWith("Input.dispatchMouseEvent", {
3923		type: "mousePressed",
3924		x: 120,
3925		y: 80,
3926		button: "left",
3927		clickCount: 1,
3928		});
3929		expect(sendCommand).toHaveBeenCalledWith("Input.dispatchMouseEvent", {
3930		type: "mouseReleased",
3931		x: 120,
3932		y: 80,
3933		button: "left",
3934		clickCount: 1,
3935		});
3936		}),
3937		),
	4075	yield* manager.subscribePointerEvents((event) =>
	4076	Effect.sync(() => {
	4077	activity.push(event.phase);
	4078	}),
	4079	);
	4080	yield* manager.createTab("tab_1");
	4081	yield* manager.registerWebview("tab_1", 42);
	4082	yield* manager.startRecording("tab_1");
	4083	const click = yield* manager
	4084	.automationClick("tab_1", { x: 120, y: 80 })
	4085	.pipe(Effect.forkChild({ startImmediately: true }));
	4086	yield* TestClock.adjust(200);
	4087	yield* Fiber.join(click);
	4088
	4089	expect(activity).toEqual(["move", "click", "mousePressed"]);
	4090	expect(
	4091	webviewSend.mock.calls
	4092	.filter(([channel]) => channel === "preview:recording-controller")
	4093	.map(([, controller]) => controller),
	4094	).toEqual(["agent", "none"]);
	4095
	4096	const recordedPointer = webviewSend.mock.calls
	4097	.filter(([channel]) => channel === "preview:recording-pointer")
	4098	.map(([, event]) => event);
	4099	expect(recordedPointer).toEqual([
	4100	expect.objectContaining({ phase: "move", x: 120, y: 80 }),
	4101	expect.objectContaining({ phase: "click", x: 120, y: 80 }),
	4102	]);
	4103	expect(sendCommand).toHaveBeenCalledWith("Input.dispatchMouseEvent", {
	4104	type: "mousePressed",
	4105	x: 120,
	4106	y: 80,
	4107	button: "left",
	4108	clickCount: 1,
	4109	});
	4110	yield* manager.stopRecording("tab_1");
	4111	expect(sendCommand).toHaveBeenCalledWith("Input.dispatchMouseEvent", {
	4112	type: "mouseReleased",
	4113	x: 120,
	4114	y: 80,
	4115	button: "left",
	4116	clickCount: 1,
	4117	});
	4118	}),
	4119	),
3938	4120	);
3939	4121
3940	4122	effectIt.effect("types in background webviews and enables native key input", () =>
...	...	@@ -4230,6 +4412,9 @@ describe("PreviewManager", () => {
4230	4412	});
4231	4413	fromId.mockReturnValue({
4232	4414	id: 42,
	4415	hostWebContents: makeTestHostWebContents(),
	4416	capturePage: vi.fn(async () => ({ toPNG: () => Buffer.from("frame") })),
	4417	setBackgroundThrottling: vi.fn(),
4233	4418	isDestroyed: () => false,
4234	4419	getType: () => "webview",
4235	4420	getURL: () => "https://example.com",
...	...	@@ -4263,6 +4448,7 @@ describe("PreviewManager", () => {
4263	4448
4264	4449	yield* manager.createTab("tab_1");
4265	4450	yield* manager.registerWebview("tab_1", 42);
	4451	yield* manager.startRecording("tab_1");
4266	4452
4267	4453	const click = yield* manager
4268	4454	.automationClick("tab_1", { x: 120, y: 80 })
...	...	@@ -4270,6 +4456,12 @@ describe("PreviewManager", () => {
4270	4456	yield* TestClock.adjust(200);
4271	4457	const exit = yield* Fiber.await(click);
4272	4458	expect(Exit.isFailure(exit)).toBe(true);
	4459	expect(
	4460	webviewSend.mock.calls
	4461	.filter(([channel]) => channel === "preview:recording-controller")
	4462	.map(([, controller]) => controller),
	4463	).toEqual(["agent", "human", "none"]);
	4464	yield* manager.stopRecording("tab_1");
4273	4465	if (Exit.isSuccess(exit)) return;
4274	4466	const error = Option.getOrThrow(Cause.findErrorOption(exit.cause));
4275	4467	expect(error).toMatchObject({

apps/desktop/src/preview/Manager.ts+148-6

...	...	@@ -6,7 +6,10 @@
6	6	* here). Single layer-scoped browser session partition.
7	7	*/
8	8	import * as NodeCrypto from "node:crypto";
9		import { DESKTOP_PREVIEW_RECORDING_CAPTURE_TRIGGER } from "@t3tools/contracts";
	9	import {
	10	DesktopPreviewRecordingInputSchema,
	11	DESKTOP_PREVIEW_RECORDING_CAPTURE_TRIGGER,
	12	} from "@t3tools/contracts";
10	13	import type {
11	14	DesktopPreviewAnnotationTheme,
12	15	DesktopPreviewAutomationStatus,
...	...	@@ -18,6 +21,7 @@ import type {
18	21	PreviewAnnotationSubmissionResult,
19	22	DesktopPreviewRecordingArtifact,
20	23	DesktopPreviewRecordingFrame,
	24	DesktopPreviewRecordingInputEvent,
21	25	DesktopPreviewScreenshotArtifact,
22	26	DesktopPreviewTabDefaults,
23	27	PreviewAutomationClickInput,
...	...	@@ -71,6 +75,11 @@ import {
71	75	ELEMENT_PICKED_CHANNEL,
72	76	HUMAN_INPUT_CHANNEL,
73	77	MOUSE_NAVIGATE_CHANNEL,
	78	RECORDING_CURSOR_CHANNEL,
	79	RECORDING_POINTER_CHANNEL,
	80	RECORDING_KEY_CHANNEL,
	81	RECORDING_INPUT_CHANNEL,
	82	RECORDING_CONTROLLER_CHANNEL,
74	83	START_PICK_CHANNEL,
75	84	} from "./GuestProtocol.ts";
76	85	import { isPreviewAnnotationPayload } from "./PickedElementPayload.ts";
...	...	@@ -81,6 +90,7 @@ import {
81	90	previewAutomationEditingCommandExpression,
82	91	} from "./PreviewKeyboard.ts";
83	92	import { captureFavicon, safeHttpOrigin, selectFaviconCandidates } from "./FaviconCapture.ts";
	93	import { DEFAULT_RECORDING_INPUT_OPTIONS, type RecordingInputOptions } from "./RecordingInput.ts";
84	94
85	95	export type PreviewNavStatus =
86	96	\| { kind: "Idle" }
...	...	@@ -437,6 +447,7 @@ interface ManagedListeners {
437	447	type FrameCaptureConsumer = "picture-in-picture" \| "recording";
438	448
439	449	interface FrameCaptureSession {
	450	readonly recordingInputOptions?: RecordingInputOptions;
440	451	readonly scope: Scope.Closeable \| null;
441	452	readonly consumers: ReadonlySet<FrameCaptureConsumer>;
442	453	readonly unthrottledWebContentsIds: ReadonlySet<number>;
...	...	@@ -485,6 +496,10 @@ interface BrowserDiagnostics {
485	496	readonly requests: ReadonlyMap<string, { url: string; method: string }>;
486	497	}
487	498
	499	const isRecordingInput = Schema.is(DesktopPreviewRecordingInputSchema);
	500
	501	type RecordingInputListener = (event: DesktopPreviewRecordingInputEvent) => Effect.Effect<void>;
	502
488	503	type PointerEventListener = (event: DesktopPreviewPointerEvent) => Effect.Effect<void>;
489	504
490	505	interface ExpectedAgentInput {
...	...	@@ -636,6 +651,9 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
636	651	const attachedRef = yield* Ref.make<ReadonlyMap<number, ManagedListeners>>(new Map());
637	652	const listenersRef = yield* Ref.make<ReadonlySet<Listener>>(new Set());
638	653	const pointerEventListenersRef = yield* Ref.make<ReadonlySet<PointerEventListener>>(new Set());
	654	const recordingInputListenersRef = yield* Ref.make<ReadonlySet<RecordingInputListener>>(
	655	new Set(),
	656	);
639	657	const recordingFrameListenersRef = yield* Ref.make<ReadonlySet<RecordingFrameListener>>(
640	658	new Set(),
641	659	);
...	...	@@ -821,6 +839,20 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
821	839	return Effect.succeed([undefined, sessions] as const);
822	840	}
823	841	return setFrameCaptureWebContentsBackgroundThrottling(wc, false).pipe(
	842	Effect.tap(() =>
	843	Effect.gen(function* () {
	844	if (!current.consumers.has("recording")) return;
	845	const tab = (yield* SynchronizedRef.get(tabsRef)).get(tabId);
	846	yield* attempt({ operation: "recording.cursor", tabId, webContentsId: wc.id }, () =>
	847	wc.send(
	848	RECORDING_CURSOR_CHANNEL,
	849	true,
	850	current.recordingInputOptions,
	851	tab?.controller,
	852	),
	853	);
	854	}),
	855	),
824	856	Effect.map(
825	857	() =>
826	858	[
...	...	@@ -846,6 +878,15 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
846	878	if (!current \|\| !current.consumers.has(consumer)) {
847	879	return [undefined, sessions] as const;
848	880	}
	881	if (consumer === "recording") {
	882	yield* Effect.forEach(current.unthrottledWebContentsIds, (id) =>
	883	attempt({ operation: "recording.cursor", tabId, webContentsId: id }, () => {
	884	const contents = webContents.fromId(id);
	885	if (contents && !contents.isDestroyed())
	886	contents.send(RECORDING_CURSOR_CHANNEL, false);
	887	}).pipe(Effect.ignore),
	888	);
	889	}
849	890	const consumers = new Set(current.consumers);
850	891	consumers.delete(consumer);
851	892	if (consumers.size > 0) {
...	...	@@ -896,7 +937,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
896	937	});
897	938
898	939	const deliverEvent = (
899		eventKind: "state-change" \| "recording-frame" \| "pointer-event",
	940	eventKind: "state-change" \| "recording-frame" \| "recording-input" \| "pointer-event",
900	941	tabId: string,
901	942	delivery: () => Effect.Effect<void>,
902	943	) =>
...	...	@@ -933,6 +974,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
933	974	const update = Effect.fn("PreviewManager.update")(function* (
934	975	tabId: string,
935	976	patch: Partial<PreviewTabState>,
	977	humanPoint?: { readonly x: number; readonly y: number },
936	978	) {
937	979	const updatedAt = yield* currentIso;
938	980	const next = yield* SynchronizedRef.modify(tabsRef, (tabs) => {
...	...	@@ -950,7 +992,20 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
950	992	// can commit between the modify above and here, and republishing this
951	993	// snapshot would roll the UI back to a value that writer will not send
952	994	// again because it suppresses unchanged audibility.
953		if (Option.isSome(next)) yield* emitIfCurrent(tabId, next.value);
	995	if (Option.isSome(next)) {
	996	if (patch.controller !== undefined && next.value.webContentsId != null) {
	997	const capture = (yield* SynchronizedRef.get(frameCaptureSessionsRef)).get(tabId);
	998	const webContentsId = next.value.webContentsId;
	999	if (capture?.consumers.has("recording")) {
	1000	yield* attempt({ operation: "recording.controller", tabId }, () => {
	1001	const contents = webContents.fromId(webContentsId);
	1002	if (contents && !contents.isDestroyed())
	1003	contents.send(RECORDING_CONTROLLER_CHANNEL, patch.controller, humanPoint);
	1004	}).pipe(Effect.ignore);
	1005	}
	1006	}
	1007	yield* emitIfCurrent(tabId, next.value);
	1008	}
954	1009	});
955	1010
956	1011	/**
...	...	@@ -1728,6 +1783,21 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
1728	1783	const sync = () => runFork(syncState(true));
1729	1784	const syncNavigation = () => runFork(syncState(false, true));
1730	1785	const syncInPageNavigation = () => runFork(syncState(false));
	1786	const restoreRecordingCursor = () =>
	1787	runFork(
	1788	Effect.gen(function* () {
	1789	const session = (yield* SynchronizedRef.get(frameCaptureSessionsRef)).get(tabId);
	1790	if (!wc.isDestroyed()) {
	1791	const tab = (yield* SynchronizedRef.get(tabsRef)).get(tabId);
	1792	wc.send(
	1793	RECORDING_CURSOR_CHANNEL,
	1794	session?.consumers.has("recording") ?? false,
	1795	session?.recordingInputOptions,
	1796	tab?.controller,
	1797	);
	1798	}
	1799	}),
	1800	);
1731	1801	const navigationStarted = (
1732	1802	event: Electron.Event<Electron.WebContentsDidStartNavigationEventParams>,
1733	1803	) => {
...	...	@@ -1860,13 +1930,37 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
1860	1930	copy.set(tabId, (epochs.get(tabId) ?? 0) + 1);
1861	1931	}),
1862	1932	);
1863		yield* update(tabId, { controller: "human" });
	1933	yield* update(
	1934	tabId,
	1935	{ controller: "human" },
	1936	isPreviewInputSignal(rawSignal) && rawSignal.kind === "pointer"
	1937	? { x: rawSignal.x, y: rawSignal.y }
	1938	: undefined,
	1939	);
1864	1940	yield* Effect.sleep(750);
1865	1941	const tabs = yield* SynchronizedRef.get(tabsRef);
1866	1942	if (tabs.get(tabId)?.controller === "human") {
1867	1943	yield* update(tabId, { controller: "none" });
1868	1944	}
1869	1945	});
	1946	const recordingInput = (_event: unknown, input: unknown) => {
	1947	if (!isRecordingInput(input)) return;
	1948	return runFork(
	1949	Effect.gen(function* () {
	1950	const tab = (yield* SynchronizedRef.get(tabsRef)).get(tabId);
	1951	const capture = (yield* SynchronizedRef.get(frameCaptureSessionsRef)).get(tabId);
	1952	if (tab?.webContentsId !== wc.id \|\| !capture?.consumers.has("recording")) return;
	1953	if (input.type === "key" && !capture.recordingInputOptions?.showKeyPresses) return;
	1954	if (input.type === "pointer" && !capture.recordingInputOptions?.showMousePresses) return;
	1955	const listeners = yield* Ref.get(recordingInputListenersRef);
	1956	yield* Effect.forEach(
	1957	listeners,
	1958	(listener) => deliverEvent("recording-input", tabId, () => listener({ tabId, input })),
	1959	{ discard: true },
	1960	);
	1961	}),
	1962	);
	1963	};
1870	1964	const humanInput = (_event: unknown, rawSignal?: unknown): void => {
1871	1965	runFork(handleHumanInput(rawSignal));
1872	1966	};
...	...	@@ -1928,11 +2022,13 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
1928	2022	wc.off("page-favicon-updated", faviconUpdated as never);
1929	2023	wc.off("did-start-loading", sync);
1930	2024	wc.off("did-stop-loading", sync);
	2025	wc.off("dom-ready", restoreRecordingCursor);
1931	2026	wc.off("did-fail-load", failed as never);
1932	2027	wc.off("audio-state-changed", audioStateChanged);
1933	2028	wc.off("did-create-window", windowCreated);
1934	2029	wc.off("before-input-event", beforeInput);
1935	2030	wc.ipc.off(HUMAN_INPUT_CHANNEL, humanInput);
	2031	wc.ipc.off(RECORDING_INPUT_CHANNEL, recordingInput);
1936	2032	wc.ipc.off(MOUSE_NAVIGATE_CHANNEL, mouseNavigate);
1937	2033	}).pipe(Effect.ignore),
1938	2034	);
...	...	@@ -1948,9 +2044,11 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
1948	2044	wc.on("page-favicon-updated", faviconUpdated as never);
1949	2045	wc.on("did-start-loading", sync);
1950	2046	wc.on("did-stop-loading", sync);
	2047	wc.on("dom-ready", restoreRecordingCursor);
1951	2048	wc.on("did-fail-load", failed as never);
1952	2049	wc.on("audio-state-changed", audioStateChanged);
1953	2050	wc.ipc.on(HUMAN_INPUT_CHANNEL, humanInput);
	2051	wc.ipc.on(RECORDING_INPUT_CHANNEL, recordingInput);
1954	2052	wc.ipc.on(MOUSE_NAVIGATE_CHANNEL, mouseNavigate);
1955	2053	wc.setWindowOpenHandler((details) => {
1956	2054	if (previewWindowOpenAction(details) === "popup") {
...	...	@@ -3405,7 +3503,10 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
3405	3503	});
3406	3504	};
3407	3505
3408		const startRecording = Effect.fn("PreviewManager.startRecording")(function* (tabId: string) {
	3506	const startRecording = Effect.fn("PreviewManager.startRecording")(function* (
	3507	tabId: string,
	3508	options: RecordingInputOptions = DEFAULT_RECORDING_INPUT_OPTIONS,
	3509	) {
3409	3510	if ((yield* Ref.get(closingTabIdsRef)).has(tabId)) {
3410	3511	return yield* new PreviewTabNotFoundError({ tabId });
3411	3512	}
...	...	@@ -3413,11 +3514,21 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
3413	3514	tabId,
3414	3515	Effect.gen(function* () {
3415	3516	yield* startFrameCapture(tabId, "recording");
	3517	yield* SynchronizedRef.update(frameCaptureSessionsRef, (sessions) =>
	3518	replaceMap(sessions, (copy) => {
	3519	const current = copy.get(tabId);
	3520	if (current) copy.set(tabId, { ...current, recordingInputOptions: options });
	3521	}),
	3522	);
3416	3523	const wc = yield* requireWebContents(tabId);
3417	3524	const requestWebContents = wc.hostWebContents;
3418	3525	if (requestWebContents === null) {
3419	3526	return yield* new PreviewMainWindowClosedError({ tabId });
3420	3527	}
	3528	const tab = (yield* SynchronizedRef.get(tabsRef)).get(tabId);
	3529	yield* attempt({ operation: "recording.cursor", tabId, webContentsId: wc.id }, () =>
	3530	wc.send(RECORDING_CURSOR_CHANNEL, true, options, tab?.controller),
	3531	);
3421	3532	yield* attemptPromise(
3422	3533	{
3423	3534	operation: "recording.warmSource",
...	...	@@ -3720,6 +3831,15 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
3720	3831	const emitPointerEvent = Effect.fn("PreviewManager.emitPointerEvent")(function* (
3721	3832	event: DesktopPreviewPointerEvent,
3722	3833	) {
	3834	const recording = (yield* SynchronizedRef.get(frameCaptureSessionsRef)).get(event.tabId);
	3835	const tab = (yield* SynchronizedRef.get(tabsRef)).get(event.tabId);
	3836	const webContentsId = tab?.webContentsId;
	3837	if (recording?.consumers.has("recording") && webContentsId != null) {
	3838	yield* attempt({ operation: "recording.pointer", tabId: event.tabId }, () => {
	3839	const contents = webContents.fromId(webContentsId);
	3840	if (contents && !contents.isDestroyed()) contents.send(RECORDING_POINTER_CHANNEL, event);
	3841	});
	3842	}
3723	3843	const listeners = yield* Ref.get(pointerEventListenersRef);
3724	3844	yield* Effect.forEach(
3725	3845	listeners,
...	...	@@ -4110,6 +4230,18 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
4110	4230	const keySequence = makePreviewAutomationNativeKeySequence(input, {
4111	4231	isMac: hostPlatform === "darwin",
4112	4232	});
	4233	const recording = (yield* SynchronizedRef.get(frameCaptureSessionsRef)).get(tabId);
	4234	if (recording?.consumers.has("recording") && recording.recordingInputOptions?.showKeyPresses) {
	4235	yield* attempt({ operation: "recording.key", tabId, webContentsId: wc.id }, () =>
	4236	wc.send(RECORDING_KEY_CHANNEL, {
	4237	key: keySequence.signal.key \|\| input.key,
	4238	metaKey: input.modifiers?.includes("Meta") ?? false,
	4239	ctrlKey: input.modifiers?.includes("Control") ?? false,
	4240	altKey: input.modifiers?.includes("Alt") ?? false,
	4241	shiftKey: input.modifiers?.includes("Shift") ?? false,
	4242	}),
	4243	);
	4244	}
4113	4245	// CDP keyboard dispatch follows the embedder's focused renderer, and
4114	4246	// WebContents.focus() is a no-op for webview guests. Native input targets
4115	4247	// this guest's widget directly, so Enter cannot submit the host composer.
...	...	@@ -4470,6 +4602,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
4470	4602	Ref.set(expectedAgentInputsRef, new Map()),
4471	4603	Ref.set(pointerEventListenersRef, new Set()),
4472	4604	Ref.set(recordingFrameListenersRef, new Set()),
	4605	Ref.set(recordingInputListenersRef, new Set()),
4473	4606	],
4474	4607	{ discard: true },
4475	4608	);
...	...	@@ -4513,6 +4646,8 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
4513	4646	stopRecording,
4514	4647	subscribePointerEvents: (listener: PointerEventListener) =>
4515	4648	subscribe(pointerEventListenersRef, listener),
	4649	subscribeRecordingInputs: (listener: RecordingInputListener) =>
	4650	subscribe(recordingInputListenersRef, listener),
4516	4651	subscribeRecordingFrames: (listener: RecordingFrameListener) =>
4517	4652	subscribe(recordingFrameListenersRef, listener),
4518	4653	subscribeStateChanges: (listener: Listener) => subscribe(listenersRef, listener),
...	...	@@ -4877,7 +5012,10 @@ export class PreviewManager extends Context.Service<
4877	5012	readonly copyArtifactToClipboard: (path: string) => Effect.Effect<void, PreviewManagerError>;
4878	5013	readonly openPictureInPicture: (tabId: string) => Effect.Effect<void, PreviewManagerError>;
4879	5014	readonly closePictureInPicture: (tabId: string) => Effect.Effect<void, PreviewManagerError>;
4880		readonly startRecording: (tabId: string) => Effect.Effect<void, PreviewManagerError>;
	5015	readonly startRecording: (
	5016	tabId: string,
	5017	options?: RecordingInputOptions,
	5018	) => Effect.Effect<void, PreviewManagerError>;
4881	5019	readonly stopRecording: (tabId: string) => Effect.Effect<void, PreviewManagerError>;
4882	5020	readonly saveRecording: (
4883	5021	tabId: string,
...	...	@@ -4918,6 +5056,9 @@ export class PreviewManager extends Context.Service<
4918	5056	readonly subscribePointerEvents: (
4919	5057	listener: PointerEventListener,
4920	5058	) => Effect.Effect<void, never, Scope.Scope>;
	5059	readonly subscribeRecordingInputs: (
	5060	listener: RecordingInputListener,
	5061	) => Effect.Effect<void, never, Scope.Scope>;
4921	5062	readonly subscribeRecordingFrames: (
4922	5063	listener: RecordingFrameListener,
4923	5064	) => Effect.Effect<void, never, Scope.Scope>;
...	...	@@ -5011,6 +5152,7 @@ export const make = Effect.gen(function* PreviewManagerMake() {
5011	5152	subscribeStateChanges: operations.subscribeStateChanges,
5012	5153	subscribePointerEvents: operations.subscribePointerEvents,
5013	5154	subscribeRecordingFrames: operations.subscribeRecordingFrames,
	5155	subscribeRecordingInputs: operations.subscribeRecordingInputs,
5014	5156	});
5015	5157	}).pipe(Effect.withSpan("PreviewManager.make"));
5016	5158

apps/desktop/src/preview/PickPreload.ts+82

...	...	@@ -16,6 +16,8 @@ import type {
16	16
17	17	import { resolveAnnotationSubmission } from "./AnnotationKeyboard.ts";
18	18	import { previewAnnotationStyles } from "./AnnotationStyles.generated.ts";
	19	import { installRecordingCursor } from "./RecordingCursor.ts";
	20	import { DEFAULT_RECORDING_INPUT_OPTIONS } from "./RecordingInput.ts";
19	21	import {
20	22	ANNOTATION_CAPTURED_CHANNEL,
21	23	ANNOTATION_THEME_CHANNEL,
...	...	@@ -23,6 +25,11 @@ import {
23	25	ELEMENT_PICKED_CHANNEL,
24	26	HUMAN_INPUT_CHANNEL,
25	27	MOUSE_NAVIGATE_CHANNEL,
	28	RECORDING_CURSOR_CHANNEL,
	29	RECORDING_POINTER_CHANNEL,
	30	RECORDING_KEY_CHANNEL,
	31	RECORDING_INPUT_CHANNEL,
	32	RECORDING_CONTROLLER_CHANNEL,
26	33	START_PICK_CHANNEL,
27	34	} from "./GuestProtocol.ts";
28	35	const OVERLAY_ATTRIBUTE = "data-t3code-annotation-ui";
...	...	@@ -35,6 +42,80 @@ const ELEMENT_CONTEXT_TIMEOUT_MS = 5_000;
35	42	const CONTENT_LAYER_Z_INDEX = 1;
36	43	const CHROME_LAYER_Z_INDEX = 10;
37	44
	45	let recordingCursor: ReturnType<typeof installRecordingCursor> \| null = null;
	46	ipcRenderer.on(
	47	RECORDING_CURSOR_CHANNEL,
	48	(_event, active: unknown, inputOptions: unknown, controller: unknown) => {
	49	if (active === true) {
	50	const options =
	51	typeof inputOptions === "object" && inputOptions !== null
	52	? {
	53	showKeyPresses:
	54	"showKeyPresses" in inputOptions && inputOptions.showKeyPresses === true,
	55	showMousePresses:
	56	"showMousePresses" in inputOptions && inputOptions.showMousePresses === true,
	57	}
	58	: DEFAULT_RECORDING_INPUT_OPTIONS;
	59	recordingCursor ??= installRecordingCursor(document, window, options, (input) =>
	60	ipcRenderer.send(RECORDING_INPUT_CHANNEL, input),
	61	);
	62	recordingCursor.setTheme(annotationTheme);
	63	if (controller === "agent" \|\| controller === "human" \|\| controller === "none")
	64	recordingCursor.setController(controller);
	65	} else {
	66	recordingCursor?.dispose();
	67	recordingCursor = null;
	68	}
	69	},
	70	);
	71	ipcRenderer.on(RECORDING_CONTROLLER_CHANNEL, (_event, controller: unknown, point: unknown) => {
	72	const humanPoint =
	73	typeof point === "object" &&
	74	point !== null &&
	75	"x" in point &&
	76	typeof point.x === "number" &&
	77	Number.isFinite(point.x) &&
	78	"y" in point &&
	79	typeof point.y === "number" &&
	80	Number.isFinite(point.y)
	81	? { x: point.x, y: point.y }
	82	: undefined;
	83	if (controller === "agent" \|\| controller === "human" \|\| controller === "none")
	84	recordingCursor?.setController(controller, humanPoint);
	85	});
	86	ipcRenderer.on(RECORDING_KEY_CHANNEL, (_event, input: unknown) => {
	87	if (
	88	typeof input !== "object" \|\|
	89	input === null \|\|
	90	!("key" in input) \|\|
	91	typeof input.key !== "string"
	92	)
	93	return;
	94	recordingCursor?.keyPress({
	95	key: input.key,
	96	metaKey: "metaKey" in input && input.metaKey === true,
	97	ctrlKey: "ctrlKey" in input && input.ctrlKey === true,
	98	altKey: "altKey" in input && input.altKey === true,
	99	shiftKey: "shiftKey" in input && input.shiftKey === true,
	100	});
	101	});
	102	ipcRenderer.on(RECORDING_POINTER_CHANNEL, (_event, point: unknown) => {
	103	if (
	104	typeof point === "object" &&
	105	point !== null &&
	106	"x" in point &&
	107	typeof point.x === "number" &&
	108	Number.isFinite(point.x) &&
	109	"y" in point &&
	110	typeof point.y === "number" &&
	111	Number.isFinite(point.y)
	112	)
	113	recordingCursor?.move(
	114	{ x: point.x, y: point.y },
	115	"phase" in point && point.phase === "click" ? "click" : "move",
	116	);
	117	});
	118
38	119	type AnnotationTool = "select" \| "marquee" \| "draw" \| "erase";
39	120
40	121	interface SelectedElement {
...	...	@@ -1361,6 +1442,7 @@ ipcRenderer.on(START_PICK_CHANNEL, (_event, theme: DesktopPreviewAnnotationTheme
1361	1442	});
1362	1443	ipcRenderer.on(ANNOTATION_THEME_CHANNEL, (_event, theme: DesktopPreviewAnnotationTheme) => {
1363	1444	annotationTheme = theme;
	1445	recordingCursor?.setTheme(theme);
1364	1446	activeSession?.applyTheme(theme);
1365	1447	});
1366	1448	ipcRenderer.on(CANCEL_PICK_CHANNEL, () => activeSession?.teardown(false));

apps/desktop/src/preview/RecordingCursor.ts created+197

...	...	@@ -0,0 +1,197 @@
	1	import type {
	2	DesktopPreviewAnnotationTheme,
	3	DesktopPreviewRecordingInput,
	4	} from "@t3tools/contracts";
	5
	6	import {
	7	DEFAULT_RECORDING_INPUT_OPTIONS,
	8	recordingKeyLabel,
	9	recordingKeysAreSensitive,
	10	type RecordingInputOptions,
	11	type RecordingKeyPress,
	12	} from "./RecordingInput.ts";
	13
	14	/**
	15	* Chromium's capture cursor uses native window bounds, which do not follow a
	16	* webview's CSS placement or scale. Draw it in the guest's coordinate space
	17	* while recording, and make the native cursor transparent to avoid two cursors.
	18	*/
	19	export function installRecordingCursor(
	20	document: Document,
	21	window: Window,
	22	options: RecordingInputOptions = DEFAULT_RECORDING_INPUT_OPTIONS,
	23	emit: (input: DesktopPreviewRecordingInput) => void = () => {},
	24	) {
	25	const style = document.createElement("style");
	26	style.textContent =
	27	"html, html * { cursor: none !important; } @media (prefers-reduced-motion: reduce) { [data-t3code-recording-agent-cursor] { transition: none !important; } }";
	28	const cursor = document.createElement("div");
	29	cursor.setAttribute("aria-hidden", "true");
	30	cursor.setAttribute("data-t3code-recording-cursor", "");
	31	cursor.style.cssText =
	32	"position:fixed;left:0;top:0;width:16px;height:24px;pointer-events:none;z-index:2147483647;display:none;";
	33	cursor.innerHTML =
	34	'<svg xmlns="http://www.w3.org/2000/svg" width="16" height="24" viewBox="0 0 16 24"><path d="M1 1v18l4-4 4 8 3-1.5-4-8H15Z" fill="black" stroke="white" stroke-width="1.5" stroke-linejoin="round"/></svg>';
	35	const agentCursor = document.createElement("div");
	36	agentCursor.setAttribute("aria-hidden", "true");
	37	agentCursor.setAttribute("data-t3code-recording-agent-cursor", "");
	38	agentCursor.style.cssText =
	39	"position:fixed;left:0;top:0;width:20px;height:20px;pointer-events:none;z-index:2147483647;display:none;filter:drop-shadow(0 1px 2px #0003);transition:transform 150ms ease-out,opacity 150ms ease-out;";
	40	// Match the MousePointer2 icon used by the live AgentBrowserCursor.
	41	agentCursor.innerHTML =
	42	'<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="var(--recording-cursor-background,white)" stroke="var(--recording-cursor-primary,#2563eb)" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="transform:translate(-2px,-2px)"><path d="M4.037 4.688a.495.495 0 0 1 .651-.651l16 6.5a.5.5 0 0 1-.063.947l-6.124 1.58a2 2 0 0 0-1.438 1.435l-1.579 6.126a.5.5 0 0 1-.947.063z"/></svg>';
	43	document.documentElement.append(style, cursor, agentCursor);
	44	let controller: "human" \| "agent" \| "none" = "none";
	45	let humanPoint: { readonly x: number; readonly y: number } \| null = null;
	46	const drawHuman = () => {
	47	if (!humanPoint) return;
	48	cursor.style.transform = `translate(${humanPoint.x}px, ${humanPoint.y}px)`;
	49	cursor.style.display = "block";
	50	};
	51	let agentActive = false;
	52	let agentTimer: number \| undefined;
	53	const setController = (
	54	next: typeof controller,
	55	point?: { readonly x: number; readonly y: number },
	56	) => {
	57	if (point) humanPoint = point;
	58	controller = next;
	59	if (next === "agent") cursor.style.display = "none";
	60	if (next === "human") drawHuman();
	61	if (!agentActive) agentCursor.style.opacity = next === "human" ? "0.18" : "0.35";
	62	};
	63	const setTheme = (
	64	theme: Pick<DesktopPreviewAnnotationTheme, "primary" \| "background"> \| null,
	65	) => {
	66	agentCursor.style.setProperty("--recording-cursor-primary", theme?.primary ?? "#2563eb");
	67	agentCursor.style.setProperty("--recording-cursor-background", theme?.background ?? "white");
	68	};
	69	let lastKeyLabel: string \| null = null;
	70	let pointerHeld = false;
	71	let pointerFrame: number \| undefined;
	72	let pendingPointer: DesktopPreviewRecordingInput \| undefined;
	73	const cancelPendingPointer = () => {
	74	if (pointerFrame !== undefined) window.cancelAnimationFrame(pointerFrame);
	75	pointerFrame = undefined;
	76	pendingPointer = undefined;
	77	};
	78	const keyPress = (input: RecordingKeyPress, held = false) => {
	79	if (!options.showKeyPresses) return;
	80	const label = recordingKeysAreSensitive(document)
	81	? null
	82	: recordingKeyLabel(input, /Mac/.test(window.navigator.platform));
	83	lastKeyLabel = label;
	84	emit({ type: "key", label, held, width: window.innerWidth });
	85	};
	86	const pointer = (
	87	point: { readonly x: number; readonly y: number },
	88	phase: "move" \| "down" \| "up" \| "click",
	89	) => {
	90	if (!options.showMousePresses \|\| (phase === "move" && !pointerHeld)) return;
	91	if (phase === "down") pointerHeld = true;
	92	if (phase === "up") pointerHeld = false;
	93	const input: DesktopPreviewRecordingInput = {
	94	type: "pointer",
	95	phase,
	96	...point,
	97	width: window.innerWidth,
	98	height: window.innerHeight,
	99	};
	100	if (phase === "move") {
	101	pendingPointer = input;
	102	pointerFrame ??= window.requestAnimationFrame(() => {
	103	pointerFrame = undefined;
	104	if (pendingPointer) emit(pendingPointer);
	105	pendingPointer = undefined;
	106	});
	107	} else {
	108	cancelPendingPointer();
	109	emit(input);
	110	}
	111	};
	112	const move = (
	113	point: { readonly x: number; readonly y: number },
	114	phase: "move" \| "click" = "move",
	115	) => {
	116	agentCursor.style.transform = `translate(${point.x}px, ${point.y}px)`;
	117	agentCursor.style.display = "block";
	118	agentCursor.style.opacity = "1";
	119	agentActive = true;
	120	window.clearTimeout(agentTimer);
	121	agentTimer = window.setTimeout(() => {
	122	agentActive = false;
	123	agentCursor.style.opacity = controller === "human" ? "0.18" : "0.35";
	124	}, 700);
	125	pointer(point, phase);
	126	};
	127	const moveHuman = (point: { readonly x: number; readonly y: number }) => {
	128	if (controller === "agent") return;
	129	humanPoint = point;
	130	drawHuman();
	131	};
	132	const pointerMove = (event: PointerEvent) => {
	133	if (event.pointerType === "touch") return;
	134	const point = { x: event.clientX, y: event.clientY };
	135	moveHuman(point);
	136	pointer(point, "move");
	137	};
	138	const pointerDown = (event: PointerEvent) => {
	139	if (event.pointerType === "touch") return;
	140	moveHuman({ x: event.clientX, y: event.clientY });
	141	pointer({ x: event.clientX, y: event.clientY }, "down");
	142	};
	143	const pointerUp = (event: PointerEvent) => {
	144	if (event.pointerType !== "touch") pointer({ x: event.clientX, y: event.clientY }, "up");
	145	};
	146	const keyDown = (event: KeyboardEvent) => {
	147	if (event.isComposing \|\| event.repeat) return;
	148	keyPress(event, true);
	149	};
	150	const keyUp = () => {
	151	if (!options.showKeyPresses) return;
	152	emit({
	153	type: "key",
	154	label: recordingKeysAreSensitive(document) ? null : lastKeyLabel,
	155	held: false,
	156	width: window.innerWidth,
	157	});
	158	};
	159	const hide = () => {
	160	cursor.style.display = "none";
	161	pointerHeld = false;
	162	cancelPendingPointer();
	163	if (options.showKeyPresses \|\| options.showMousePresses) emit({ type: "clear" });
	164	};
	165	const leave = (event: PointerEvent) => {
	166	if (event.relatedTarget === null) hide();
	167	};
	168	window.addEventListener("pointermove", pointerMove, true);
	169	window.addEventListener("pointerdown", pointerDown, true);
	170	window.addEventListener("pointerup", pointerUp, true);
	171	window.addEventListener("pointercancel", pointerUp, true);
	172	window.addEventListener("keydown", keyDown, true);
	173	window.addEventListener("keyup", keyUp, true);
	174	window.addEventListener("pointerout", leave, true);
	175	window.addEventListener("blur", hide);
	176	return {
	177	move,
	178	keyPress,
	179	setController,
	180	setTheme,
	181	dispose: () => {
	182	window.removeEventListener("pointermove", pointerMove, true);
	183	window.removeEventListener("pointerdown", pointerDown, true);
	184	window.removeEventListener("pointerup", pointerUp, true);
	185	window.removeEventListener("pointercancel", pointerUp, true);
	186	window.removeEventListener("keydown", keyDown, true);
	187	window.removeEventListener("keyup", keyUp, true);
	188	window.removeEventListener("pointerout", leave, true);
	189	window.removeEventListener("blur", hide);
	190	cancelPendingPointer();
	191	window.clearTimeout(agentTimer);
	192	cursor.remove();
	193	agentCursor.remove();
	194	style.remove();
	195	},
	196	};
	197	}

apps/desktop/src/preview/RecordingInput.test.ts created+51

...	...	@@ -0,0 +1,51 @@
	1	import { describe, expect, it } from "vite-plus/test";
	2	import { recordingKeyLabel, recordingKeysAreSensitive } from "./RecordingInput.ts";
	3
	4	const key = (
	5	value: string,
	6	modifiers: Partial<{
	7	metaKey: boolean;
	8	ctrlKey: boolean;
	9	altKey: boolean;
	10	shiftKey: boolean;
	11	}> = {},
	12	) => ({
	13	key: value,
	14	metaKey: false,
	15	ctrlKey: false,
	16	altKey: false,
	17	shiftKey: false,
	18	...modifiers,
	19	});
	20
	21	describe("recording key labels", () => {
	22	it("formats macOS and other-platform shortcuts", () => {
	23	expect(recordingKeyLabel(key("c", { metaKey: true }), true)).toBe("⌘C");
	24	expect(recordingKeyLabel(key("c", { ctrlKey: true }), false)).toBe("Ctrl + C");
	25	expect(recordingKeyLabel(key("Tab", { altKey: true, shiftKey: true }), true)).toBe("⌥⇧⇥");
	26	});
	27	it("shows held modifiers once and labels navigation keys", () => {
	28	expect(recordingKeyLabel(key("Meta", { metaKey: true }), true)).toBe("⌘");
	29	expect(recordingKeyLabel(key("Shift", { shiftKey: true }), false)).toBe("Shift");
	30	expect(recordingKeyLabel(key("ArrowLeft"), true)).toBe("←");
	31	expect(recordingKeyLabel(key(" "), false)).toBe("Space");
	32	});
	33	it.each(["Dead", "Unidentified", "Process", ""])("excludes composition key %s", (value) => {
	34	expect(recordingKeyLabel(key(value), true)).toBeNull();
	35	});
	36	});
	37
	38	describe("recording key privacy", () => {
	39	const field = (type: string) => ({ tagName: "INPUT", getAttribute: () => type });
	40	const sensitive = (activeElement: unknown) =>
	41	recordingKeysAreSensitive({ activeElement } as Document);
	42	it("excludes password fields and their shadow-root focus", () => {
	43	expect(sensitive(field("password"))).toBe(true);
	44	expect(sensitive({ shadowRoot: { activeElement: field("password") } })).toBe(true);
	45	expect(sensitive(field("text"))).toBe(false);
	46	});
	47	it("excludes iframe focus whose field cannot be inspected", () => {
	48	expect(sensitive({ tagName: "IFRAME" })).toBe(true);
	49	expect(sensitive({ tagName: "SECRET-FIELD" })).toBe(true);
	50	});
	51	});

apps/desktop/src/preview/RecordingInput.ts created+58

...	...	@@ -0,0 +1,58 @@
	1	export interface RecordingInputOptions {
	2	readonly showKeyPresses: boolean;
	3	readonly showMousePresses: boolean;
	4	}
	5
	6	export const DEFAULT_RECORDING_INPUT_OPTIONS: RecordingInputOptions = {
	7	showKeyPresses: false,
	8	showMousePresses: false,
	9	};
	10
	11	export interface RecordingKeyPress {
	12	readonly key: string;
	13	readonly metaKey: boolean;
	14	readonly ctrlKey: boolean;
	15	readonly altKey: boolean;
	16	readonly shiftKey: boolean;
	17	}
	18
	19	/** Formats a single chord without duplicating a modifier pressed on its own. */
	20	export function recordingKeyLabel(input: RecordingKeyPress, isMac: boolean): string \| null {
	21	if (["Dead", "Process", "Unidentified", ""].includes(input.key)) return null;
	22	const modifiers = [
	23	input.ctrlKey \|\| input.key === "Control" ? (isMac ? "⌃" : "Ctrl") : null,
	24	input.altKey \|\| input.key === "Alt" ? (isMac ? "⌥" : "Alt") : null,
	25	input.shiftKey \|\| input.key === "Shift" ? (isMac ? "⇧" : "Shift") : null,
	26	input.metaKey \|\| input.key === "Meta" ? (isMac ? "⌘" : "Win") : null,
	27	].filter((value) => value !== null);
	28	const labels: Record<string, string> = {
	29	Enter: "↵",
	30	Tab: "⇥",
	31	Backspace: "⌫",
	32	Delete: "⌦",
	33	Escape: "Esc",
	34	ArrowUp: "↑",
	35	ArrowDown: "↓",
	36	ArrowLeft: "←",
	37	ArrowRight: "→",
	38	" ": "Space",
	39	Space: "Space",
	40	};
	41	if (!["Control", "Alt", "Shift", "Meta"].includes(input.key)) {
	42	modifiers.push(
	43	labels[input.key] ?? (input.key.length === 1 ? input.key.toUpperCase() : input.key),
	44	);
	45	}
	46	return modifiers.join(isMac ? "" : " + ");
	47	}
	48
	49	/** Unknown iframe or closed-shadow focus is excluded because its field type cannot be checked. */
	50	export function recordingKeysAreSensitive(document: Document): boolean {
	51	let element = document.activeElement;
	52	while (element?.shadowRoot?.activeElement) element = element.shadowRoot.activeElement;
	53	return (
	54	element?.tagName === "IFRAME" \|\|
	55	element?.tagName.includes("-") === true \|\|
	56	element?.getAttribute("type")?.toLowerCase() === "password"
	57	);
	58	}

apps/web/src/browser/browserRecording.ts+23-2

...	...	@@ -8,6 +8,8 @@ import { previewBridge } from "~/components/preview/previewBridge";
8	8	import { ensureClientSettingsHydrated, getClientSettings } from "~/hooks/useSettings";
9	9	import { appAtomRegistry } from "~/rpc/atomRegistry";
10	10
	11	import { createRecordingCompositor } from "./recordingCompositor";
	12
11	13	import { acquireBrowserSurfaceActivity } from "./browserSurfaceStore";
12	14
13	15	export class BrowserRecordingUnavailableError extends Schema.TaggedError<BrowserRecordingUnavailableError>()(
...	...	@@ -123,6 +125,7 @@ interface ActiveRecording {
123	125	releaseSurfaceActivity: (() => void) \| null;
124	126	stream: MediaStream \| null;
125	127	recorder: MediaRecorder \| null;
	128	compositor: Awaited<ReturnType<typeof createRecordingCompositor>>;
126	129	savedBlob?: Blob;
127	130	uploadPromise?: Promise<string>;
128	131	lifecycle: BrowserRecordingLifecycle;
...	...	@@ -381,6 +384,8 @@ const captureTabMediaStreamWithTimeout = async (
381	384	};
382	385
383	386	const clearActiveRecording = (recording: ActiveRecording): void => {
	387	recording.compositor?.dispose();
	388	recording.compositor = null;
384	389	recording.releaseSurfaceActivity?.();
385	390	recording.releaseSurfaceActivity = null;
386	391	if (activeRecordings.get(recording.tabId) !== recording) return;
...	...	@@ -525,6 +530,7 @@ export async function startBrowserRecording(
525	530	releaseSurfaceActivity,
526	531	stream: null,
527	532	recorder: null,
	533	compositor: null,
528	534	lifecycle: startingLifecycle,
529	535	};
530	536	activeRecordings.set(tabId, recording);
...	...	@@ -534,7 +540,8 @@ export async function startBrowserRecording(
534	540	clearActiveRecording(recording);
535	541	throw cause;
536	542	});
537		const frameRate = getClientSettings().browserRecordingFrameRate;
	543	const settings = getClientSettings();
	544	const frameRate = settings.browserRecordingFrameRate;
538	545	await waitForBrowserRecordingPaint();
539	546	const throwIfStartupCancelled = async (): Promise<void> => {
540	547	// Once a grant starts, a stop lets startup finish so the caller receives an artifact.
...	...	@@ -614,7 +621,19 @@ export async function startBrowserRecording(
614	621
615	622	let recorder: MediaRecorder;
616	623	try {
617		recorder = createMediaRecorder(stream);
	624	recording.compositor = await createRecordingCompositor(
	625	stream,
	626	{
	627	showKeyPresses: settings.browserRecordingShowKeyPresses,
	628	showMousePresses: settings.browserRecordingShowMousePresses,
	629	frameRate,
	630	},
	631	(listener) =>
	632	bridge.recording.onInput((event) => {
	633	if (event.tabId === tabId) listener(event.input);
	634	}),
	635	);
	636	recorder = createMediaRecorder(recording.compositor?.stream ?? stream);
618	637	recording.recorder = recorder;
619	638	recorder.addEventListener("dataavailable", (event) => {
620	639	if (event.data.size > 0) chunks.push(event.data);
...	...	@@ -694,6 +713,8 @@ const finalizeBrowserRecording = async (
694	713	cause,
695	714	});
696	715	}
	716	recording.compositor?.dispose();
	717	recording.compositor = null;
697	718	// Encoding has flushed; release native capture before materializing and saving the file.
698	719	stopMediaStream(recording.stream);
699	720	recording.stream = null;

apps/web/src/browser/recordingCompositor.test.ts created+179

...	...	@@ -0,0 +1,179 @@
	1	import type { DesktopPreviewRecordingInput } from "@t3tools/contracts";
	2	import { afterEach, describe, expect, it, vi } from "vite-plus/test";
	3
	4	import { createRecordingCompositor, RecordingDecorations } from "./recordingCompositor";
	5
	6	const primaryColor = "oklch(0.65 0.2 310)";
	7	vi.mock("./annotationTheme", () => ({
	8	readPreviewAnnotationTheme: () => ({ primary: "oklch(0.65 0.2 310)" }),
	9	}));
	10
	11	const options = { showKeyPresses: true, showMousePresses: true, frameRate: 30 };
	12	const pointer = (
	13	phase: "move" \| "down" \| "up" \| "click",
	14	x = 100,
	15	): DesktopPreviewRecordingInput => ({
	16	type: "pointer",
	17	phase,
	18	x,
	19	y: 80,
	20	width: 800,
	21	height: 600,
	22	});
	23	const context = () => ({
	24	save: vi.fn(),
	25	restore: vi.fn(),
	26	beginPath: vi.fn(),
	27	fill: vi.fn(),
	28	stroke: vi.fn(),
	29	ellipse: vi.fn(),
	30	roundRect: vi.fn(),
	31	fillText: vi.fn(),
	32	drawImage: vi.fn(),
	33	measureText: () => ({ width: 40 }),
	34	globalAlpha: 1,
	35	strokeStyle: "",
	36	fillStyle: "",
	37	});
	38
	39	describe("recording decorations", () => {
	40	it("keeps rings aligned through dragging and stops following the cursor after release", () => {
	41	const decorations = new RecordingDecorations(options, primaryColor);
	42	const ctx = context();
	43	decorations.apply(pointer("down"), 0);
	44	decorations.apply(pointer("move", 120), 10);
	45	decorations.draw(ctx as unknown as CanvasRenderingContext2D, 1600, 1200, 10);
	46	expect(ctx.ellipse.mock.calls[0]?.slice(0, 4)).toEqual([240, 160, 40, 40]);
	47	expect(ctx.strokeStyle).toBe(primaryColor);
	48	expect(ctx.fillStyle).toBe(primaryColor);
	49	expect(decorations.nextRedraw(10)).toBeNull();
	50	decorations.apply(pointer("up", 130), 20);
	51	decorations.apply(pointer("move", 300), 30);
	52	decorations.draw(ctx as unknown as CanvasRenderingContext2D, 1600, 1200, 320);
	53	expect(ctx.ellipse.mock.calls[1]?.slice(0, 4)).toEqual([260, 160, 50, 50]);
	54	ctx.ellipse.mockClear();
	55	decorations.draw(ctx as unknown as CanvasRenderingContext2D, 1600, 1200, 620);
	56	expect(ctx.ellipse).not.toHaveBeenCalled();
	57	expect(decorations.nextRedraw(620)).toBeNull();
	58	});
	59
	60	it("pulses agent clicks and clears decorations on blur or navigation", () => {
	61	const decorations = new RecordingDecorations(options, primaryColor);
	62	const ctx = context();
	63	decorations.apply(pointer("click"), 0);
	64	decorations.draw(ctx as unknown as CanvasRenderingContext2D, 800, 600, 300);
	65	expect(ctx.ellipse).toHaveBeenCalledOnce();
	66	decorations.apply({ type: "clear" }, 301);
	67	ctx.ellipse.mockClear();
	68	decorations.draw(ctx as unknown as CanvasRenderingContext2D, 800, 600, 302);
	69	expect(ctx.ellipse).not.toHaveBeenCalled();
	70	expect(decorations.nextRedraw(302)).toBeNull();
	71	});
	72
	73	it("holds shortcut badges until release, then expires them even on a static page", () => {
	74	const decorations = new RecordingDecorations(options, primaryColor);
	75	const ctx = context();
	76	const key = { type: "key" as const, label: "⌘C", held: true, width: 800 };
	77	decorations.apply(key, 0);
	78	decorations.draw(ctx as unknown as CanvasRenderingContext2D, 1600, 1200, 5000);
	79	expect(ctx.fillText.mock.calls[0]?.slice(0, 3)).toEqual(["⌘C", 800, 1098]);
	80	decorations.apply({ ...key, held: false }, 5000);
	81	expect(decorations.nextRedraw(5500)).toBe(400);
	82	ctx.fillText.mockClear();
	83	decorations.draw(ctx as unknown as CanvasRenderingContext2D, 1600, 1200, 5900);
	84	expect(ctx.fillText).not.toHaveBeenCalled();
	85	});
	86
	87	it("removes the previous key badge on password focus", () => {
	88	const decorations = new RecordingDecorations(options, primaryColor);
	89	const ctx = context();
	90	decorations.apply({ type: "key", label: "A", held: true, width: 800 }, 0);
	91	decorations.apply({ type: "key", label: null, held: true, width: 800 }, 1);
	92	decorations.draw(ctx as unknown as CanvasRenderingContext2D, 800, 600, 2);
	93	expect(ctx.fillText).not.toHaveBeenCalled();
	94	});
	95
	96	it("honors independent opt-in flags", () => {
	97	const decorations = new RecordingDecorations(
	98	{ ...options, showMousePresses: false },
	99	primaryColor,
	100	);
	101	const ctx = context();
	102	decorations.apply(pointer("down"), 0);
	103	decorations.apply({ type: "key", label: "⌘C", held: true, width: 800 }, 0);
	104	decorations.draw(ctx as unknown as CanvasRenderingContext2D, 800, 600, 1);
	105	expect(ctx.ellipse).not.toHaveBeenCalled();
	106	expect(ctx.fillText).toHaveBeenCalledOnce();
	107	});
	108	});
	109
	110	describe("detached recording compositor", () => {
	111	afterEach(() => vi.unstubAllGlobals());
	112
	113	it("keeps native capture when both decorations are off", async () => {
	114	vi.stubGlobal("document", {
	115	createElement: () => {
	116	throw new Error("must not allocate");
	117	},
	118	});
	119	expect(
	120	await createRecordingCompositor(
	121	{} as MediaStream,
	122	{
	123	...options,
	124	showKeyPresses: false,
	125	showMousePresses: false,
	126	},
	127	() => {
	128	throw new Error("must not subscribe");
	129	},
	130	),
	131	).toBeNull();
	132	});
	133
	134	it.each([false, true])(
	135	"releases the detached output on disposal or playback failure (%s)",
	136	async (failPlayback) => {
	137	const ctx = context();
	138	const stop = vi.fn();
	139	const unsubscribe = vi.fn();
	140	const cancelFrame = vi.fn();
	141	const source = {
	142	getVideoTracks: () => [{ getSettings: () => ({ width: 800, height: 600 }) }],
	143	} as unknown as MediaStream;
	144	const stream = { getTracks: () => [{ stop }] } as unknown as MediaStream;
	145	const canvas = { width: 0, height: 0, getContext: () => ctx, captureStream: () => stream };
	146	const video = {
	147	muted: false,
	148	playsInline: false,
	149	srcObject: null as MediaStream \| null,
	150	readyState: 2,
	151	videoWidth: 800,
	152	videoHeight: 600,
	153	pause: vi.fn(),
	154	play: async () => {
	155	if (failPlayback) throw new Error("play failed");
	156	},
	157	requestVideoFrameCallback: () => 1,
	158	cancelVideoFrameCallback: cancelFrame,
	159	};
	160	vi.stubGlobal("document", {
	161	createElement: (tag: string) => (tag === "canvas" ? canvas : video),
	162	});
	163	vi.stubGlobal("window", { clearTimeout: vi.fn(), setTimeout: vi.fn() });
	164	const compositor = createRecordingCompositor(source, options, () => unsubscribe);
	165	if (failPlayback) await expect(compositor).rejects.toThrow("play failed");
	166	else {
	167	const result = await compositor;
	168	expect(result?.stream).toBe(stream);
	169	expect(ctx.drawImage).toHaveBeenCalledOnce();
	170	result?.dispose();
	171	result?.dispose();
	172	}
	173	expect(stop).toHaveBeenCalledOnce();
	174	expect(unsubscribe).toHaveBeenCalledOnce();
	175	expect(cancelFrame).toHaveBeenCalledWith(1);
	176	expect(video.srcObject).toBeNull();
	177	},
	178	);
	179	});

apps/web/src/browser/recordingCompositor.ts created+190

...	...	@@ -0,0 +1,190 @@
	1	import type { DesktopPreviewRecordingInput } from "@t3tools/contracts";
	2
	3	import { readPreviewAnnotationTheme } from "./annotationTheme";
	4
	5	interface RecordingDecorationOptions {
	6	readonly showKeyPresses: boolean;
	7	readonly showMousePresses: boolean;
	8	readonly frameRate: number;
	9	}
	10
	11	/** Decorates a detached canvas; no recording UI is inserted into the preview page. */
	12	export async function createRecordingCompositor(
	13	source: MediaStream,
	14	options: RecordingDecorationOptions,
	15	subscribe: (listener: (input: DesktopPreviewRecordingInput) => void) => () => void,
	16	) {
	17	if (!options.showKeyPresses && !options.showMousePresses) return null;
	18	const canvas = document.createElement("canvas");
	19	const context = canvas.getContext("2d", { alpha: false });
	20	if (!context) throw new Error("Recording canvas is unavailable.");
	21	const video = document.createElement("video");
	22	video.muted = true;
	23	video.playsInline = true;
	24	video.srcObject = source;
	25	const settings = source.getVideoTracks()[0]?.getSettings();
	26	canvas.width = settings?.width ?? 1920;
	27	canvas.height = settings?.height ?? 1080;
	28	const decorations = new RecordingDecorations(options, readPreviewAnnotationTheme().primary);
	29	let disposed = false;
	30	let frameId: number \| undefined;
	31	let timer: number \| undefined;
	32	const draw = () => {
	33	if (disposed \|\| video.readyState < 2) return;
	34	const width = video.videoWidth \|\| canvas.width;
	35	const height = video.videoHeight \|\| canvas.height;
	36	if (canvas.width !== width) canvas.width = width;
	37	if (canvas.height !== height) canvas.height = height;
	38	context.drawImage(video, 0, 0, width, height);
	39	const now = performance.now();
	40	decorations.draw(context, width, height, now);
	41	window.clearTimeout(timer);
	42	const next = decorations.nextRedraw(now);
	43	if (next !== null) timer = window.setTimeout(draw, next);
	44	};
	45	const frame = () => {
	46	if (disposed) return;
	47	draw();
	48	frameId = video.requestVideoFrameCallback(frame);
	49	};
	50	const output = canvas.captureStream(options.frameRate);
	51	let unsubscribe: (() => void) \| undefined;
	52	const dispose = () => {
	53	if (disposed) return;
	54	disposed = true;
	55	unsubscribe?.();
	56	window.clearTimeout(timer);
	57	if (frameId !== undefined) video.cancelVideoFrameCallback(frameId);
	58	video.pause();
	59	video.srcObject = null;
	60	for (const track of output.getTracks()) track.stop();
	61	};
	62	try {
	63	unsubscribe = subscribe((input) => {
	64	decorations.apply(input, performance.now());
	65	draw();
	66	});
	67	frameId = video.requestVideoFrameCallback(frame);
	68	await video.play();
	69	draw();
	70	return { stream: output, dispose };
	71	} catch (error) {
	72	dispose();
	73	throw error;
	74	}
	75	}
	76
	77	/** Keeps input timing and coordinates independent of native video frame delivery. */
	78	export class RecordingDecorations {
	79	private ring: {
	80	x: number;
	81	y: number;
	82	width: number;
	83	height: number;
	84	held: boolean;
	85	releasedAt: number \| null;
	86	} \| null = null;
	87	private key: { label: string; width: number; expiresAt: number \| null } \| null = null;
	88
	89	constructor(
	90	private readonly options: RecordingDecorationOptions,
	91	private readonly primaryColor: string,
	92	) {}
	93
	94	apply(input: DesktopPreviewRecordingInput, now: number) {
	95	if (input.type === "clear") {
	96	this.ring = null;
	97	this.key = null;
	98	} else if (input.type === "key" && this.options.showKeyPresses) {
	99	this.key = input.label
	100	? { label: input.label, width: input.width, expiresAt: input.held ? null : now + 900 }
	101	: null;
	102	} else if (input.type === "pointer" && this.options.showMousePresses) {
	103	if (input.phase === "down" \|\| input.phase === "click") {
	104	this.ring = {
	105	x: input.x,
	106	y: input.y,
	107	width: input.width,
	108	height: input.height,
	109	held: input.phase === "down",
	110	releasedAt: input.phase === "click" ? now : null,
	111	};
	112	} else if (this.ring?.held) {
	113	this.ring = {
	114	...this.ring,
	115	x: input.x,
	116	y: input.y,
	117	width: input.width,
	118	height: input.height,
	119	held: input.phase !== "up",
	120	releasedAt: input.phase === "up" ? now : null,
	121	};
	122	}
	123	}
	124	}
	125
	126	nextRedraw(now: number): number \| null {
	127	if (
	128	this.ring?.releasedAt !== null &&
	129	this.ring?.releasedAt !== undefined &&
	130	now < this.ring.releasedAt + 600
	131	) {
	132	return 1000 / this.options.frameRate;
	133	}
	134	return this.key?.expiresAt !== null &&
	135	this.key?.expiresAt !== undefined &&
	136	now < this.key.expiresAt
	137	? this.key.expiresAt - now
	138	: null;
	139	}
	140
	141	draw(context: CanvasRenderingContext2D, width: number, height: number, now: number) {
	142	// Guest coordinates are CSS pixels; native frames include zoom and display scale.
	143	const scale = width / (this.key?.width ?? this.ring?.width ?? 1280);
	144	const ring = this.ring;
	145	if (ring && (ring.held \|\| (ring.releasedAt !== null && now < ring.releasedAt + 600))) {
	146	const progress = ring.releasedAt === null ? 0 : Math.min(1, (now - ring.releasedAt) / 600);
	147	context.save();
	148	const opacity = 0.9 * (1 - progress);
	149	context.strokeStyle = this.primaryColor;
	150	context.fillStyle = this.primaryColor;
	151	context.lineWidth = 2 * scale;
	152	context.beginPath();
	153	context.ellipse(
	154	(ring.x * width) / ring.width,
	155	(ring.y * height) / ring.height,
	156	((20 * width) / ring.width) * (1 + progress * 0.5),
	157	((20 * height) / ring.height) * (1 + progress * 0.5),
	158	0,
	159	0,
	160	Math.PI * 2,
	161	);
	162	context.globalAlpha = opacity * 0.15;
	163	context.fill();
	164	context.globalAlpha = opacity;
	165	context.stroke();
	166	context.restore();
	167	}
	168	const key = this.key;
	169	if (key && (key.expiresAt === null \|\| now < key.expiresAt)) {
	170	context.save();
	171	context.font = `500 ${26 * scale}px system-ui, sans-serif`;
	172	const badgeWidth = Math.min(
	173	width - 32 * scale,
	174	context.measureText(key.label).width + 36 * scale,
	175	);
	176	const badgeHeight = 54 * scale;
	177	const left = (width - badgeWidth) / 2;
	178	const top = height - 24 * scale - badgeHeight;
	179	context.fillStyle = "rgba(32,32,34,.86)";
	180	context.beginPath();
	181	context.roundRect(left, top, badgeWidth, badgeHeight, 14 * scale);
	182	context.fill();
	183	context.fillStyle = "white";
	184	context.textAlign = "center";
	185	context.textBaseline = "middle";
	186	context.fillText(key.label, width / 2, top + badgeHeight / 2, badgeWidth - 24 * scale);
	187	context.restore();
	188	}
	189	}
	190	}

apps/web/src/components/preview/AgentBrowserCursor.tsx+4-4

...	...	@@ -24,7 +24,6 @@ export function AgentBrowserCursor(props: {
24	24
25	25	return (
26	26	<AgentBrowserCursorEvent
27		key={event.sequence}
28	27	event={event}
29	28	content={content}
30	29	zoomFactor={zoomFactor}
...	...	@@ -46,12 +45,13 @@ function AgentBrowserCursorEvent(props: {
46	45	readonly controller: BrowserController;
47	46	}) {
48	47	const { event, content, zoomFactor, controller } = props;
49		const [active, setActive] = useState(true);
	48	const [inactiveSequence, setInactiveSequence] = useState<number \| null>(null);
	49	const active = inactiveSequence !== event.sequence;
50	50
51	51	useEffect(() => {
52		const timeout = window.setTimeout(() => setActive(false), CURSOR_ACTIVE_MS);
	52	const timeout = window.setTimeout(() => setInactiveSequence(event.sequence), CURSOR_ACTIVE_MS);
53	53	return () => window.clearTimeout(timeout);
54		}, []);
	54	}, [event.sequence]);
55	55
56	56	return (
57	57	<div

apps/web/src/components/preview/PreviewView.test.tsx+39-4

...	...	@@ -7,7 +7,7 @@ import {
7	7	FILL_PREVIEW_VIEWPORT,
8	8	ThreadId,
9	9	} from "@t3tools/contracts";
10		import { act, Profiler } from "react";
	10	import { act, createElement, Profiler } from "react";
11	11	import { renderToStaticMarkup } from "react-dom/server";
12	12	import { beforeEach, describe, expect, it, vi } from "vite-plus/test";
13	13
...	...	@@ -33,6 +33,9 @@ const mocks = vi.hoisted(() => ({
33	33	pictureInPicture: false,
34	34	showEmptyState: false,
35	35	loading: false,
	36	serverEpoch: null as string \| null,
	37	recordingTabIds: new Set<string>(),
	38	recordingRuntimeTabId: null as string \| null,
36	39	recordVisitForThread: vi.fn(),
37	40	}));
38	41
...	...	@@ -98,6 +101,7 @@ vi.mock("~/previewStateStore", () => ({
98	101	updatePreviewServerSnapshot: vi.fn(),
99	102	useThreadPreviewState: () => ({
100	103	activeTabId: "tab-1",
	104	serverEpoch: mocks.serverEpoch,
101	105	desktopByTabId: {
102	106	"tab-1": {
103	107	hasWebContents: true,
...	...	@@ -146,11 +150,11 @@ vi.mock("~/state/use-atom-command", () => ({
146	150	}));
147	151
148	152	vi.mock("~/browser/browserRecording", () => ({
149		findActiveBrowserRecordingRuntimeTabId: vi.fn(() => null),
	153	findActiveBrowserRecordingRuntimeTabId: () => mocks.recordingRuntimeTabId,
150	154	isBrowserRecordingStartCancelledError: vi.fn(() => false),
151	155	startBrowserRecording: vi.fn(),
152	156	stopBrowserRecording: vi.fn(),
153		useActiveBrowserRecordingTabIds: () => new Set(),
	157	useActiveBrowserRecordingTabIds: () => mocks.recordingTabIds,
154	158	}));
155	159
156	160	vi.mock("~/browser/browserSurfaceStore", () => ({
...	...	@@ -244,7 +248,9 @@ vi.mock("./PreviewMoreMenu", () => ({
244	248	}));
245	249	vi.mock("./PreviewUnreachable", () => ({ PreviewUnreachable: () => null }));
246	250	vi.mock("./ZoomIndicator", () => ({ ZoomIndicator: () => null }));
247		vi.mock("./AgentBrowserCursor", () => ({ AgentBrowserCursor: () => null }));
	251	vi.mock("./AgentBrowserCursor", () => ({
	252	AgentBrowserCursor: () => createElement("agent-cursor"),
	253	}));
248	254	vi.mock("~/browser/BrowserSurfaceSlot", () => ({ BrowserSurfaceSlot: () => null }));
249	255	vi.mock("./usePreviewSession", () => ({ usePreviewSession: vi.fn() }));
250	256
...	...	@@ -344,9 +350,38 @@ describe("PreviewView navigation", () => {
344	350	mocks.pictureInPicture = false;
345	351	mocks.showEmptyState = false;
346	352	mocks.loading = false;
	353	mocks.serverEpoch = null;
	354	mocks.recordingTabIds = new Set();
	355	mocks.recordingRuntimeTabId = null;
347	356	mocks.recordVisitForThread.mockClear();
348	357	});
349	358
	359	it("shows the cursor in a replacement browser while the old instance still records", async () => {
	360	const document = installTestDom();
	361	const { createRoot } = await import("react-dom/client");
	362	const container = document.createElement("div");
	363	const root = createRoot(container as unknown as Element);
	364	const hasCursor = (node: TestNode): boolean =>
	365	node.nodeName === "AGENT-CURSOR" \|\| node.childNodes.some(hasCursor);
	366	mocks.recordingTabIds.add(TEST_RUNTIME_TAB_ID);
	367	mocks.recordingRuntimeTabId = TEST_RUNTIME_TAB_ID;
	368	try {
	369	await act(() => {
	370	root.render(<PreviewView threadRef={TEST_THREAD_REF} tabId="tab-1" visible />);
	371	});
	372	expect(hasCursor(container)).toBe(false);
	373	mocks.serverEpoch = "replacement-server";
	374	await act(() => {
	375	root.render(<PreviewView threadRef={TEST_THREAD_REF} tabId="tab-1" visible />);
	376	});
	377	expect(hasCursor(container)).toBe(true);
	378	expect(mocks.recordingTabIds.has(TEST_RUNTIME_TAB_ID)).toBe(true);
	379	} finally {
	380	await act(() => root.unmount());
	381	vi.unstubAllGlobals();
	382	}
	383	});
	384
350	385	it("does not rerender while loading time passes", async () => {
351	386	vi.useFakeTimers();
352	387	mocks.loading = true;

apps/web/src/components/preview/PreviewView.tsx+5-6

...	...	@@ -798,18 +798,17 @@ export function PreviewView({
798	798	{snapshot && desktopOverlay ? (
799	799	<ZoomIndicator zoomFactor={desktopOverlay.zoomFactor} />
800	800	) : null}
801		{runtimeTabId && desktopOverlay && !showEmptyState && !isUnreachable ? (
	801	{runtimeTabId &&
	802	desktopOverlay &&
	803	!showEmptyState &&
	804	!isUnreachable &&
	805	!activeRecordingTabIds.has(runtimeTabId) ? (
802	806	<AgentBrowserCursor
803	807	tabId={runtimeTabId}
804	808	zoomFactor={desktopOverlay.zoomFactor}
805	809	controller={controller}
806	810	/>
807	811	) : null}
808		{controller !== "none" ? (
809		<div className="pointer-events-none absolute left-3 top-3 z-40 rounded-full border border-border/70 bg-background/90 px-2.5 py-1 text-[11px] font-medium shadow-sm backdrop-blur">
810		{controller === "agent" ? "Agent controlling browser" : "Human control"}
811		</div>
812		) : null}
813	812	{navStatus._tag === "LoadFailed" ? (
814	813	<div className="absolute inset-0 z-10 bg-background">
815	814	<PreviewUnreachable

apps/web/src/components/settings/IntegrationsSettings.tsx+39

...	...	@@ -470,6 +470,44 @@ function BrowserAppearanceSetting({ disabled }: { readonly disabled: boolean })
470	470	);
471	471	}
472	472
	473	function BrowserRecordingInputSettings({ disabled }: { readonly disabled: boolean }) {
	474	const showKeys = useClientSettings((settings) => settings.browserRecordingShowKeyPresses);
	475	const showMouse = useClientSettings((settings) => settings.browserRecordingShowMousePresses);
	476	const updateSettings = useUpdatePrimarySettings();
	477	return (
	478	<>
	479	<SettingsRow
	480	{...searchableSetting("browser-recording-key-presses")}
	481	description="Show pressed keys and shortcuts in new recordings. Password fields are excluded."
	482	control={
	483	<Switch
	484	disabled={disabled}
	485	checked={showKeys}
	486	aria-label="Show key presses in recordings"
	487	onCheckedChange={(checked) =>
	488	updateSettings({ browserRecordingShowKeyPresses: Boolean(checked) })
	489	}
	490	/>
	491	}
	492	/>
	493	<SettingsRow
	494	{...searchableSetting("browser-recording-mouse-presses")}
	495	description="Highlight mouse presses and held buttons in new recordings."
	496	control={
	497	<Switch
	498	disabled={disabled}
	499	checked={showMouse}
	500	aria-label="Show mouse presses in recordings"
	501	onCheckedChange={(checked) =>
	502	updateSettings({ browserRecordingShowMousePresses: Boolean(checked) })
	503	}
	504	/>
	505	}
	506	/>
	507	</>
	508	);
	509	}
	510
473	511	function BrowserRecordingFrameRateSetting({ disabled }: { readonly disabled: boolean }) {
474	512	const frameRate = useClientSettings((settings) => settings.browserRecordingFrameRate);
475	513	const updateSettings = useUpdatePrimarySettings();
...	...	@@ -1326,6 +1364,7 @@ export function IntegrationsSettingsPanel() {
1326	1364	<BrowserZoomSetting disabled={previewDefaultsDisabled} />
1327	1365	<BrowserAppearanceSetting disabled={previewDefaultsDisabled} />
1328	1366	<BrowserRecordingFrameRateSetting disabled={previewDefaultsDisabled} />
	1367	<BrowserRecordingInputSettings disabled={previewDefaultsDisabled} />
1329	1368	<BrowserLinkTargetSetting disabled={previewDefaultsDisabled} />
1330	1369	<BrowserAutoShowFloatingPreviewSetting disabled={previewDefaultsDisabled} />
1331	1370	</>

apps/web/src/components/settings/SettingsPanels.logic.test.ts+4

...	...	@@ -268,6 +268,8 @@ describe("getChangedBrowserSettingLabels", () => {
268	268	browserDefaultZoomFactor: 1.5,
269	269	browserDefaultAppearance: "dark",
270	270	browserRecordingFrameRate: 60,
	271	browserRecordingShowKeyPresses: true,
	272	browserRecordingShowMousePresses: true,
271	273	browserLinkTarget: "app",
272	274	browserAutoShowFloatingPreview: !DEFAULT_UNIFIED_SETTINGS.browserAutoShowFloatingPreview,
273	275	}),
...	...	@@ -276,6 +278,8 @@ describe("getChangedBrowserSettingLabels", () => {
276	278	"Browser zoom",
277	279	"Browser appearance",
278	280	"Recording frame rate",
	281	"Recording key presses",
	282	"Recording mouse presses",
279	283	"Open links in",
280	284	"Floating preview",
281	285	]);

apps/web/src/components/settings/SettingsPanels.logic.ts+4

...	...	@@ -115,6 +115,8 @@ export type BrowserDefaultSettings = Pick<
115	115	\| "browserDefaultZoomFactor"
116	116	\| "browserDefaultAppearance"
117	117	\| "browserRecordingFrameRate"
	118	\| "browserRecordingShowKeyPresses"
	119	\| "browserRecordingShowMousePresses"
118	120	\| "browserLinkTarget"
119	121	\| "browserAutoShowFloatingPreview"
120	122	>;
...	...	@@ -156,6 +158,8 @@ export function getChangedBrowserSettingLabels(settings: BrowserDefaultSettings)
156	158	...(settings.browserRecordingFrameRate !== DEFAULT_UNIFIED_SETTINGS.browserRecordingFrameRate
157	159	? ["Recording frame rate"]
158	160	: []),
	161	...(settings.browserRecordingShowKeyPresses ? ["Recording key presses"] : []),
	162	...(settings.browserRecordingShowMousePresses ? ["Recording mouse presses"] : []),
159	163	...(settings.browserLinkTarget !== DEFAULT_UNIFIED_SETTINGS.browserLinkTarget
160	164	? ["Open links in"]
161	165	: []),

apps/web/src/components/settings/SettingsPanels.tsx+4

...	...	@@ -635,6 +635,8 @@ export function useSettingsRestore(onRestored?: () => void) {
635	635	settings.browserDefaultZoomFactor,
636	636	settings.browserDefaultAppearance,
637	637	settings.browserRecordingFrameRate,
	638	settings.browserRecordingShowKeyPresses,
	639	settings.browserRecordingShowMousePresses,
638	640	settings.browserLinkTarget,
639	641	settings.browserAutoShowFloatingPreview,
640	642	settings.appearanceContrast,
...	...	@@ -798,6 +800,8 @@ export function useSettingsRestore(onRestored?: () => void) {
798	800	browserDefaultZoomFactor: DEFAULT_UNIFIED_SETTINGS.browserDefaultZoomFactor,
799	801	browserDefaultAppearance: DEFAULT_UNIFIED_SETTINGS.browserDefaultAppearance,
800	802	browserRecordingFrameRate: DEFAULT_UNIFIED_SETTINGS.browserRecordingFrameRate,
	803	browserRecordingShowKeyPresses: DEFAULT_UNIFIED_SETTINGS.browserRecordingShowKeyPresses,
	804	browserRecordingShowMousePresses: DEFAULT_UNIFIED_SETTINGS.browserRecordingShowMousePresses,
801	805	browserLinkTarget: DEFAULT_UNIFIED_SETTINGS.browserLinkTarget,
802	806	browserAutoShowFloatingPreview: DEFAULT_UNIFIED_SETTINGS.browserAutoShowFloatingPreview,
803	807	// Re-granted like any other default. The confirmation dialog lists it by

apps/web/src/components/settings/settingsSearch.ts+12

...	...	@@ -615,6 +615,18 @@ export const SETTINGS_SEARCH_ITEMS = [
615	615	title: "Browser recording frame rate",
616	616	to: "/settings/integrations",
617	617	},
	618	{
	619	id: "browser-recording-key-presses",
	620	title: "Show key presses in recordings",
	621	to: "/settings/integrations",
	622	searchTerms: ["browser preview keyboard shortcuts keystrokes overlay capture"],
	623	},
	624	{
	625	id: "browser-recording-mouse-presses",
	626	title: "Show mouse presses in recordings",
	627	to: "/settings/integrations",
	628	searchTerms: ["browser preview clicks buttons drag overlay capture"],
	629	},
618	630	{
619	631	id: "browser-link-target",
620	632	title: "Open links in",

packages/contracts/src/ipc.ts+25

...	...	@@ -658,6 +658,30 @@ export interface DesktopPreviewPointerEvent {
658	658	createdAt: string;
659	659	}
660	660
	661	/** Recording decorations are forwarded separately from the captured page pixels. */
	662	export const DesktopPreviewRecordingInputSchema = Schema.Union([
	663	Schema.Struct({
	664	type: Schema.Literal("pointer"),
	665	phase: Schema.Literals(["move", "down", "up", "click"]),
	666	x: Schema.Finite,
	667	y: Schema.Finite,
	668	width: Schema.Finite.check(Schema.isGreaterThan(0)),
	669	height: Schema.Finite.check(Schema.isGreaterThan(0)),
	670	}),
	671	Schema.Struct({
	672	type: Schema.Literal("key"),
	673	label: Schema.NullOr(Schema.String.check(Schema.isMaxLength(100))),
	674	held: Schema.Boolean,
	675	width: Schema.Finite.check(Schema.isGreaterThan(0)),
	676	}),
	677	Schema.Struct({ type: Schema.Literal("clear") }),
	678	]);
	679	export type DesktopPreviewRecordingInput = typeof DesktopPreviewRecordingInputSchema.Type;
	680	export interface DesktopPreviewRecordingInputEvent {
	681	readonly tabId: string;
	682	readonly input: DesktopPreviewRecordingInput;
	683	}
	684
661	685	/**
662	686	* Static config a renderer needs to mount a preview `<webview>`. Returned
663	687	* atomically by `DesktopPreviewBridge.getPreviewConfig()` so the renderer
...	...	@@ -1295,6 +1319,7 @@ export interface DesktopPreviewBridge {
1295	1319	close: (tabId: string) => Promise<void>;
1296	1320	};
1297	1321	recording: {
	1322	onInput: (listener: (event: DesktopPreviewRecordingInputEvent) => void) => () => void;
1298	1323	startScreencast: (tabId: string) => Promise<void>;
1299	1324	stopScreencast: (tabId: string) => Promise<void>;
1300	1325	save: (

packages/contracts/src/settings.test.ts+17

...	...	@@ -471,6 +471,23 @@ describe("ClientSettings browser recording frame rate", () => {
471	471	});
472	472	});
473	473
	474	describe("ClientSettings recording input overlays", () => {
	475	it("defaults both overlays off and accepts independent opt-ins", () => {
	476	const settings = decodeClientSettings({});
	477	expect(settings.browserRecordingShowKeyPresses).toBe(false);
	478	expect(settings.browserRecordingShowMousePresses).toBe(false);
	479	expect(
	480	decodeClientSettingsPatch({
	481	browserRecordingShowKeyPresses: true,
	482	browserRecordingShowMousePresses: false,
	483	}),
	484	).toMatchObject({
	485	browserRecordingShowKeyPresses: true,
	486	browserRecordingShowMousePresses: false,
	487	});
	488	});
	489	});
	490
474	491	describe("ClientSettings glass opacity", () => {
475	492	it("defaults to a readable translucent surface", () => {
476	493	expect(decodeClientSettings({}).glassOpacity).toBe(80);

packages/contracts/src/settings.ts+8

...	...	@@ -317,6 +317,12 @@ export const ClientSettingsSchema = Schema.Struct({
317	317	browserRecordingFrameRate: BrowserRecordingFrameRate.pipe(
318	318	Schema.withDecodingDefault(Effect.succeed(DEFAULT_BROWSER_RECORDING_FRAME_RATE)),
319	319	),
	320	browserRecordingShowKeyPresses: Schema.Boolean.pipe(
	321	Schema.withDecodingDefault(Effect.succeed(false)),
	322	),
	323	browserRecordingShowMousePresses: Schema.Boolean.pipe(
	324	Schema.withDecodingDefault(Effect.succeed(false)),
	325	),
320	326	/**
321	327	* Where links clicked in a thread (chat markdown, terminal output) open.
322	328	* Only the desktop app has an in-app browser, so other clients ignore "app".
...	...	@@ -1525,6 +1531,8 @@ export const ClientSettingsPatch = Schema.Struct({
1525	1531	browserDefaultZoomFactor: Schema.optionalKey(PreviewZoomFactor),
1526	1532	browserDefaultAppearance: Schema.optionalKey(PreviewAppearancePreference),
1527	1533	browserRecordingFrameRate: Schema.optionalKey(BrowserRecordingFrameRate),
	1534	browserRecordingShowKeyPresses: Schema.optionalKey(Schema.Boolean),
	1535	browserRecordingShowMousePresses: Schema.optionalKey(Schema.Boolean),
1528	1536	browserLinkTarget: Schema.optionalKey(BrowserLinkTarget),
1529	1537	browserAutoShowFloatingPreview: Schema.optionalKey(Schema.Boolean),
1530	1538	browserProfiles: Schema.optionalKey(Schema.Array(BrowserProfile)),

author	Julius Marminge <julius0216@outlook.com> 2026-09-20 13:48:27-07:00
committer	GitHub <noreply@github.com> 2026-09-20 13:48:27-07:00
log	55c24273a025874cdb8a96a25ee0c59eeb4e3051
tree	701b8681bf46985699d73b5ea448e412e349fc0a
parent	6a699f0f2fbd8847d7ec2df8d9245ce8c2eb8707
signature	Signed by PGP key B5690EEEBB952194

fix(desktop): align preview recording cursors and show input feedback (#12779)