clover's git

the-knife

1	// @effect-diagnostics globalFetch:off globalTimers:off - This browser and WebView transport runs without an Effect runtime.
2	/* oxlint-disable unicorn/prefer-add-event-listener -- Each client owns its sockets and their handlers. */
3
4	/**
5	* Framework-free client for expo-device-hub's per-device streams, reached
6	* through the T3 proxy. One class handles both platforms because the hub
7	* vendors two servers with different wire formats:
8	*
9	* - iOS (serve-sim): video is an HTTP `stream.avcc` body of length-prefixed
10	* envelopes (`u32be length, u8 tag, payload`; tag 1 avcC description,
11	* 2 keyframe, 3 delta, 4 JPEG seed) decoded with WebCodecs; input goes over
12	* `helper/ws?device=<udid>` as `[tag][json]` packets. When WebCodecs is
13	* unavailable (plain-http remote origins) the MJPEG endpoint is used as an
14	* `<img>` source instead.
15	* - Android (serve-emu): one WebSocket at `ws?device=<serial>&frame-meta=1`
16	* carries H.264 access units prefixed with a 16-byte "SEMU" header
17	* (magic, version, key flag, pts) and accepts JSON gestures upstream.
18	*
19	* The decoder only runs while frames arrive and the viewer is attached; a
20	* hidden panel calls `stop()` so an idle device costs nothing on the GPU.
21	*/
22	import {
23	createDuoControl,
24	type DuoCommand,
25	type DuoControlState,
26	type DuoPose,
27	} from "./duoControl.ts";
28	import * as Schema from "effect/Schema";
29	import * as Option from "effect/Option";
30	import { createCanvasFrameSink, type DeviceFrameSink } from "./frame.ts";
31	import { type DeviceHubAccess, withDeviceHubQuery } from "./hubAccess.ts";
32	import type { DevicePlatform } from "@t3tools/contracts";
33
34	export type DeviceStreamStatus = "connecting" | "streaming" | "error";
35
36	export interface DeviceScreenSize {
37	readonly width: number;
38	readonly height: number;
39	readonly orientation: "portrait" | "portrait_upside_down" | "landscape_left" | "landscape_right";
40	readonly screenId?: number;
41	readonly supportsHingeAngle?: boolean;
42	readonly supportsPhysicalOrientation?: boolean;
43	readonly hingeAngle?: number;
44	readonly hingePose?: DuoPose | null;
45	readonly tableMode?: boolean;
46	readonly tableModeAvailable?: boolean;
47	}
48
49	const screenConfigSchema = Schema.Struct({
50	width: Schema.Finite.check(Schema.isGreaterThan(0)),
51	height: Schema.Finite.check(Schema.isGreaterThan(0)),
52	orientation: Schema.Literals([
53	"portrait",
54	"portrait_upside_down",
55	"landscape_left",
56	"landscape_right",
57	]),
58	inputUnavailable: Schema.optionalKey(Schema.Boolean),
59	screenId: Schema.optionalKey(Schema.Number),
60	supportsHingeAngle: Schema.optionalKey(Schema.Boolean),
61	supportsPhysicalOrientation: Schema.optionalKey(Schema.Boolean),
62	hingeAngle: Schema.optionalKey(
63	Schema.Finite.check(Schema.isBetween({ minimum: 0, maximum: 180 })),
64	),
65	hingePose: Schema.optionalKey(
66	Schema.NullOr(Schema.Literals(["closed", "book", "open", "laptop", "tent"])),
67	),
68	tableMode: Schema.optionalKey(Schema.Boolean),
69	tableModeAvailable: Schema.optionalKey(Schema.Boolean),
70	});
71	const controlReplySchema = Schema.Struct({
72	requestId: Schema.Int,
73	ok: Schema.Boolean,
74	error: Schema.optionalKey(Schema.String),
75	});
76	const decodeScreenConfig = Schema.decodeUnknownOption(screenConfigSchema);
77	const decodeControlReply = Schema.decodeUnknownOption(controlReplySchema);
78
79	export interface DuoPanelSinks {
80	readonly cover: DeviceFrameSink;
81	readonly inner: DeviceFrameSink;
82	/** Invalidate captured input synchronously, before React can commit the new layout. */
83	readonly onScreen?: (screen: DeviceScreenSize) => void;
84	}
85
86	export interface DeviceStreamEvents {
87	readonly onDuoControl?: (state: DuoControlState) => void;
88	/** A fixed panel cannot be decoded; the owner should return to the active flat feed. */
89	readonly onDuoUnavailable?: (detail?: string) => void;
90	readonly onStatus: (status: DeviceStreamStatus, detail?: string) => void;
91	readonly onScreen: (screen: DeviceScreenSize) => void;
92	/** The proxy rejected the credential; the owner should refresh access and reconnect. */
93	readonly onUnauthorized: () => void;
94	/**
95	* H.264 cannot be decoded here (no WebCodecs, or the simulator's profile is
96	* unsupported); the owner should show an `<img>` instead of the canvas
97	* and attach it with `setMjpegImage` so the client can observe real frames.
98	*/
99	readonly onMjpegFallback: (url: string) => void;
100	/** Whether touches and keys can currently reach the device. */
101	readonly onInputConnected: (connected: boolean, detail?: string) => void;
102	}
103
104	export interface DeviceStreamTarget {
105	readonly platform: DevicePlatform;
106	readonly deviceId: string;
107	readonly access: DeviceHubAccess;
108	/** Native iOS WebViews can use MJPEG without cross-origin fetch or secure-context support. */
109	readonly preferMjpeg?: boolean;
110	/** Internal fixed-panel feeds share their parent's input session. */
111	readonly panelId?: 1 | 3;
112	readonly videoOnly?: boolean;
113	}
114
115	export type DeviceHardwareButton = "home" | "back" | "recents" | "power" | "appSwitcher";
116
117	const RETRY_DELAY_MS = 1_000;
118	const FIRST_FRAME_TIMEOUT_MS = 15_000;
119	const MJPEG_FRAME_CHECK_MS = 250;
120	const FRAME_DURATION_US = 16_667;
121	const SEMU_MAGIC = 0x53454d55;
122	const SEMU_HEADER_BYTES = 16;
123	const SEMU_FLAG_KEY = 1;
124	const SOFT_DECODE_QUEUE = 8;
125
126	// serve-sim binary WS message tags (browser -> helper).
127	const IOS_MSG_TOUCH = 0x03;
128	const IOS_MSG_BUTTON = 0x04;
129	const IOS_MSG_KEY = 0x06;
130	const IOS_MSG_ORIENTATION = 0x07;
131	// helper -> browser.
132	const IOS_TAG_SCREEN_CONFIG = 0x82;
133	const IOS_TAG_INPUT_ADMITTED = 0x83;
134
135	const encoder = new TextEncoder();
136	const decoder = new TextDecoder();
137
138	const isWebCodecsSupported = (): boolean =>
139	typeof globalThis !== "undefined" &&
140	"VideoDecoder" in globalThis &&
141	"EncodedVideoChunk" in globalThis;
142
143	function taggedJson(tag: number, payload: unknown): Uint8Array<ArrayBuffer> {
144	const json = encoder.encode(JSON.stringify(payload));
145	const out = new Uint8Array(1 + json.length);
146	out[0] = tag;
147	out.set(json, 1);
148	return out;
149	}
150
151	/** Build the WebCodecs `avc1.PPCCLL` string from an avcC record or an SPS NAL. */
152	export function avcCodecString(bytes: Uint8Array): string {
153	if (bytes.length < 4) return "avc1.42E01E";
154	const hex = (byte: number) => byte.toString(16).padStart(2, "0");
155	return `avc1.${hex(bytes[1]!)}${hex(bytes[2]!)}${hex(bytes[3]!)}`;
156	}
157
158	/** Split serve-emu's SEMU-framed message into metadata and the Annex-B payload. */
159	export function parseSemuPacket(raw: ArrayBuffer): {
160	readonly data: Uint8Array;
161	readonly isKey: boolean | null;
162	readonly timestamp: number | null;
163	} {
164	const bytes = new Uint8Array(raw);
165	if (bytes.byteLength > SEMU_HEADER_BYTES) {
166	const view = new DataView(raw, 0, SEMU_HEADER_BYTES);
167	if (view.getUint32(0, false) === SEMU_MAGIC && view.getUint8(4) === 1) {
168	const pts = view.getBigUint64(8, false);
169	return {
170	data: bytes.subarray(SEMU_HEADER_BYTES),
171	isKey: (view.getUint8(5) & SEMU_FLAG_KEY) !== 0,
172	timestamp: pts <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(pts) : null,
173	};
174	}
175	}
176	return { data: bytes, isKey: null, timestamp: null };
177	}
178
179	const isVideoSessionMessage = (text: string) => {
180	try {
181	const message = JSON.parse(text) as { type?: unknown };
182	return message.type === "video-session";
183	} catch {
184	return false;
185	}
186	};
187
188	/** Walk an Annex-B access unit for its keyframe flag and SPS bytes. */
189	export function scanAccessUnit(buf: Uint8Array): { isKey: boolean; sps: Uint8Array | null } {
190	let isKey = false;
191	let sps: Uint8Array | null = null;
192	const len = buf.length;
193	let i = 0;
194	while (i + 2 < len) {
195	if (buf[i] === 0 && buf[i + 1] === 0) {
196	let codeLen = 0;
197	if (buf[i + 2] === 1) codeLen = 3;
198	else if (i + 3 < len && buf[i + 2] === 0 && buf[i + 3] === 1) codeLen = 4;
199	if (codeLen) {
200	const nalType = buf[i + codeLen]! & 0x1f;
201	if (nalType === 7 && !sps) sps = buf.subarray(i + codeLen);
202	if (nalType === 5) isKey = true;
203	i += codeLen + 1;
204	continue;
205	}
206	}
207	i++;
208	}
209	return { isKey, sps };
210	}
211
212	export type AvccChunk = {
213	readonly type: "description" | "keyframe" | "delta" | "seed";
214	readonly payload: Uint8Array;
215	};
216
217	const AVCC_TAGS: Record<number, AvccChunk["type"] | undefined> = {
218	1: "description",
219	2: "keyframe",
220	3: "delta",
221	4: "seed",
222	};
223
224	/** Turns a fragmented AVCC byte stream into complete envelopes. */
225	export class AvccDemuxer {
226	private buffer = new Uint8Array(64 * 1024);
227	private length = 0;
228
229	push(bytes: Uint8Array): AvccChunk[] {
230	if (this.length + bytes.length > this.buffer.length) {
231	let capacity = this.buffer.length;
232	while (capacity < this.length + bytes.length) capacity *= 2;
233	const grown = new Uint8Array(capacity);
234	grown.set(this.buffer.subarray(0, this.length));
235	this.buffer = grown;
236	}
237	this.buffer.set(bytes, this.length);
238	this.length += bytes.length;
239
240	const chunks: AvccChunk[] = [];
241	let offset = 0;
242	while (this.length - offset >= 4) {
243	const view = new DataView(this.buffer.buffer, this.buffer.byteOffset + offset, 4);
244	const frameLength = view.getUint32(0, false);
245	if (this.length - offset - 4 < frameLength) break;
246	if (frameLength >= 1) {
247	const type = AVCC_TAGS[this.buffer[offset + 4]!];
248	if (type) {
249	chunks.push({ type, payload: this.buffer.slice(offset + 5, offset + 4 + frameLength) });
250	}
251	}
252	offset += 4 + frameLength;
253	}
254	if (offset > 0) {
255	this.buffer.copyWithin(0, offset, this.length);
256	this.length -= offset;
257	}
258	return chunks;
259	}
260
261	reset(): void {
262	this.length = 0;
263	}
264	}
265
266	export interface DeviceStreamClient {
267	readonly start: () => void;
268	readonly stop: () => void;
269	/**
270	* Own the displayed MJPEG image's source and frame/error observation.
271	* `stop()` detaches it; attach a fresh image for each restart.
272	*/
273	readonly setMjpegImage: (image: HTMLImageElement | null) => void;
274	/** Normalized 0..1 coordinates in the displayed frame. */
275	readonly sendTouch: (phase: "begin" | "move" | "end", x: number, y: number) => void;
276	readonly sendKey: (event: KeyboardEvent, phase: "down" | "up") => void;
277	readonly pressButton: (button: DeviceHardwareButton) => void;
278	readonly rotate: () => void;
279	readonly setOrientation: (orientation: DeviceScreenSize["orientation"]) => void;
280	readonly controlDuo: (command: DuoCommand) => void;
281	/** Switch between one active feed and two fixed-panel feeds without replacing HID. */
282	readonly setDuoPanels: (panels: DuoPanelSinks | null) => void;
283	/** Model UVs already map to the hardware framebuffer. */
284	readonly sendRawTouch: (phase: "begin" | "move" | "end", x: number, y: number) => void;
285	}
286
287	const HID_USAGE_BY_CODE: Readonly<Record<string, number>> = {
288	Enter: 0x28,
289	Escape: 0x29,
290	Backspace: 0x2a,
291	Tab: 0x2b,
292	Space: 0x2c,
293	Minus: 0x2d,
294	Equal: 0x2e,
295	BracketLeft: 0x2f,
296	BracketRight: 0x30,
297	Backslash: 0x31,
298	Semicolon: 0x33,
299	Quote: 0x34,
300	Backquote: 0x35,
301	Comma: 0x36,
302	Period: 0x37,
303	Slash: 0x38,
304	Delete: 0x4c,
305	ArrowRight: 0x4f,
306	ArrowLeft: 0x50,
307	ArrowDown: 0x51,
308	ArrowUp: 0x52,
309	ControlLeft: 0xe0,
310	ShiftLeft: 0xe1,
311	AltLeft: 0xe2,
312	MetaLeft: 0xe3,
313	ControlRight: 0xe4,
314	ShiftRight: 0xe5,
315	AltRight: 0xe6,
316	MetaRight: 0xe7,
317	};
318
319	function hidUsageForCode(code: string): number | null {
320	if (/^Key[A-Z]$/.test(code)) return 0x04 + (code.charCodeAt(3) - 65);
321	if (/^Digit[1-9]$/.test(code)) return 0x1e + (code.charCodeAt(5) - 49);
322	if (code === "Digit0") return 0x27;
323	return HID_USAGE_BY_CODE[code] ?? null;
324	}
325
326	const ANDROID_KEYCODE_BY_KEY: Readonly<Record<string, number>> = {
327	ArrowUp: 19,
328	ArrowDown: 20,
329	ArrowLeft: 21,
330	ArrowRight: 22,
331	Tab: 61,
332	Enter: 66,
333	Backspace: 67,
334	Delete: 112,
335	Home: 122,
336	End: 123,
337	PageUp: 92,
338	PageDown: 93,
339	};
340
341	const IOS_ORIENTATIONS: ReadonlyArray<DeviceScreenSize["orientation"]> = [
342	"portrait",
343	"landscape_left",
344	"portrait_upside_down",
345	"landscape_right",
346	];
347
348	export function createDeviceStreamClient(
349	target: DeviceStreamTarget,
350	output: HTMLCanvasElement | DeviceFrameSink,
351	events: DeviceStreamEvents,
352	): DeviceStreamClient {
353	const { access, platform, deviceId } = target;
354	const sink = "present" in output ? output : createCanvasFrameSink(output);
355	const vendor = platform === "ios" ? "/vendor/serve-sim" : "/vendor/serve-emu";
356	const device = encodeURIComponent(deviceId);
357	const httpUrl = (path: string) =>
358	withDeviceHubQuery(`${access.httpBase}${vendor}${path}`, access);
359	const wsUrl = (path: string) => withDeviceHubQuery(`${access.wsBase}${vendor}${path}`, access);
360	const useWebCodecs = isWebCodecsSupported() && !(platform === "ios" && target.preferMjpeg);
361
362	let stopped = true;
363	let socket: WebSocket | null = null;
364	let controller: AbortController | null = null;
365	const retryTimers = new Map<"video" | "input", ReturnType<typeof setTimeout>>();
366	let primeController: AbortController | null = null;
367	let videoDecoder: VideoDecoder | null = null;
368	let decodedVideoGeneration: number | null = null;
369	let iosInputAdmitted = false;
370	let iosInputUnavailable = false;
371	let timestamp = 0;
372	let awaitingKeyframe = true;
373	let screen: DeviceScreenSize | null = null;
374	let firstFrame = false;
375	let configuring = false;
376	let mjpeg = false;
377	let generation = 0;
378	let decoderEpoch = 0;
379	let frameTimer: ReturnType<typeof setTimeout> | null = null;
380	let videoReadTimer: ReturnType<typeof setTimeout> | null = null;
381	let mjpegImage: HTMLImageElement | null = null;
382	let releaseImage: (() => void) | null = null;
383	let videoGeneration = 0;
384	let panelClients: DeviceStreamClient[] = [];
385	let panelSinks: DuoPanelSinks | null = null;
386	let rotationCursor: DeviceScreenSize["orientation"] | null = null;
387	let pendingOrientation: { requestId: number } | null = null;
388	const duoControl = createDuoControl({
389	send(request) {
390	if (
391	!iosInputAdmitted ||
392	iosInputUnavailable ||
393	socket?.readyState !== WebSocket.OPEN ||
394	!screen?.supportsHingeAngle
395	)
396	return false;
397	if (request.command.control === "physical" && !screen.supportsPhysicalOrientation)
398	return false;
399	try {
400	pendingOrientation = null;
401	if (request.command.control === "orientation") {
402	const value = request.command.value;
403	pendingOrientation = { requestId: request.requestId };
404	rotationCursor = value;
405	// Upstream serializes orientation with hinge commands, then broadcasts config.
406	// An orientation-locked app can keep its framebuffer orientation after the sensor rotates.
407	socket.send(taggedJson(IOS_MSG_ORIENTATION, { orientation: value }));
408	} else socket.send(taggedJson(0x10, request));
409	return true;
410	} catch {
411	return false;
412	}
413	},
414	onChange(state) {
415	if (!state.pending) pendingOrientation = null;
416	events.onDuoControl?.(state);
417	},
418	});
419	const videoPath = `/helper/${device}${target.panelId ? `/panel/${target.panelId}` : ""}/stream.avcc`;
420
421	const mjpegUrl = () => httpUrl(`/helper/${device}/stream.mjpeg`);
422
423	const setStatus = (status: DeviceStreamStatus, detail?: string) => {
424	if (!stopped) events.onStatus(status, detail);
425	};
426
427	const clearFrameTimer = () => {
428	if (frameTimer !== null) clearTimeout(frameTimer);
429	frameTimer = null;
430	};
431
432	const fail = (detail: string) => {
433	if (stopped) return;
434	stop();
435	events.onInputConnected(false, detail);
436	events.onStatus("error", detail);
437	};
438
439	const connecting = (detail?: string) => {
440	firstFrame = false;
441	if (frameTimer === null) {
442	frameTimer = setTimeout(
443	() => fail("No video received from the device. Reconnect to try again."),
444	FIRST_FRAME_TIMEOUT_MS,
445	);
446	}
447	setStatus("connecting", detail);
448	};
449
450	const frameReceived = () => {
451	if (firstFrame) return;
452	firstFrame = true;
453	clearFrameTimer();
454	setStatus("streaming");
455	};
456
457	const observeMjpegImage = () => {
458	releaseImage?.();
459	releaseImage = null;
460	const image = mjpegImage;
461	if (!image || stopped || !mjpeg) return;
462	let timer: ReturnType<typeof setTimeout> | null = null;
463	let released = false;
464	const check = () => {
465	if (released || stopped) return;
466	if (timer !== null) clearTimeout(timer);
467	timer = null;
468	if (image.naturalWidth > 0 && image.naturalHeight > 0) frameReceived();
469	else {
470	// Multipart images may not emit load until the response ends. Stop checking after the first frame.
471	timer = setTimeout(check, MJPEG_FRAME_CHECK_MS);
472	}
473	};
474	const error = () => {
475	if (!released) fail("Could not receive the device stream. Reconnect to try again.");
476	};
477	image.addEventListener("load", check);
478	image.addEventListener("error", error);
479	releaseImage = () => {
480	released = true;
481	if (timer !== null) clearTimeout(timer);
482	image.removeEventListener("load", check);
483	image.removeEventListener("error", error);
484	image.removeAttribute("src");
485	};
486	image.src = mjpegUrl();
487	check();
488	};
489
490	const setMjpegImage = (image: HTMLImageElement | null) => {
491	if (mjpegImage === image) return;
492	releaseImage?.();
493	releaseImage = null;
494	mjpegImage = image;
495	observeMjpegImage();
496	};
497
498	const fallBackToMjpeg = () => {
499	if (stopped || mjpeg) return;
500	mjpeg = true;
501	controller?.abort();
502	controller = null;
503	closeDecoder();
504	connecting();
505	events.onMjpegFallback(mjpegUrl());
506	if (!releaseImage) observeMjpegImage();
507	};
508
509	const paint = (source: CanvasImageSource, width: number, height: number) => {
510	if (stopped) return;
511	if (platform === "android" && (screen?.width !== width || screen.height !== height)) {
512	screen = { width, height, orientation: width > height ? "landscape_left" : "portrait" };
513	events.onScreen(screen);
514	}
515	if (!sink.present(source, width, height)) {
516	fail("Could not display the device stream. Reconnect to try again.");
517	return;
518	}
519	frameReceived();
520	};
521
522	const closeDecoder = () => {
523	decoderEpoch++;
524	try {
525	videoDecoder?.close();
526	} catch {
527	// Already closed.
528	}
529	videoDecoder = null;
530	awaitingKeyframe = true;
531	};
532
533	const recoverDecoder = () => {
534	if (platform === "ios") fallBackToMjpeg();
535	else {
536	closeDecoder();
537	connecting("Video decoder restarted.");
538	requestKeyframe();
539	}
540	};
541
542	const makeDecoder = () => {
543	const feedGeneration = videoGeneration;
544	const decoder = new VideoDecoder({
545	output: (frame) => {
546	try {
547	if (
548	videoDecoder === decoder &&
549	(platform !== "ios" || feedGeneration === videoGeneration)
550	) {
551	decodedVideoGeneration = feedGeneration;
552	paint(frame, frame.displayWidth, frame.displayHeight);
553	}
554	} finally {
555	frame.close();
556	}
557	},
558	error: () => {
559	if (
560	stopped ||
561	videoDecoder !== decoder ||
562	(platform === "ios" && feedGeneration !== videoGeneration)
563	)
564	return;
565	recoverDecoder();
566	},
567	});
568	return decoder;
569	};
570
571	/** iOS can fall back to MJPEG when the stream's H.264 profile is unsupported. */
572	const configureDecoder = async (
573	config: VideoDecoderConfig,
574	isCurrent = () => !stopped,
575	): Promise<boolean> => {
576	const epoch = decoderEpoch;
577	const full: VideoDecoderConfig = { ...config, optimizeForLatency: true };
578	const support = await VideoDecoder.isConfigSupported(full).catch(() => ({ supported: false }));
579	if (!isCurrent() || epoch !== decoderEpoch) return false;
580	if (!support.supported) {
581	if (platform === "android") fail(`This browser cannot decode ${config.codec}.`);
582	return false;
583	}
584	try {
585	if (!videoDecoder || videoDecoder.state === "closed") videoDecoder = makeDecoder();
586	videoDecoder.configure(full);
587	return true;
588	} catch (cause) {
589	if (platform === "android") fail(`Video decoder: ${(cause as Error).message}`);
590	return false;
591	}
592	};
593
594	const decode = (isKey: boolean, data: Uint8Array, pts?: number | null) => {
595	if (!videoDecoder || videoDecoder.state !== "configured") return;
596	if (awaitingKeyframe) {
597	if (!isKey) return;
598	awaitingKeyframe = false;
599	}
600	if (videoDecoder.decodeQueueSize > SOFT_DECODE_QUEUE) {
601	if (platform !== "ios") {
602	recoverDecoder();
603	return;
604	}
605	// Native IDRs are spaced by frame count, so an idle simulator may take
606	// minutes to produce one. A new AVCC subscription forces a fresh IDR.
607	controller?.abort();
608	closeDecoder();
609	return;
610	}
611	try {
612	videoDecoder.decode(
613	new EncodedVideoChunk({
614	type: isKey ? "key" : "delta",
615	timestamp: pts ?? timestamp,
616	data,
617	}),
618	);
619	timestamp += FRAME_DURATION_US;
620	} catch {
621	recoverDecoder();
622	}
623	};
624
625	const requestKeyframe = () => {
626	if (platform === "android" && socket?.readyState === WebSocket.OPEN) {
627	socket.send(JSON.stringify({ type: "reset-video", ack: false }));
628	}
629	};
630
631	const scheduleRetry = (channel: "video" | "input", run: () => void) => {
632	if (stopped || retryTimers.has(channel)) return;
633	retryTimers.set(
634	channel,
635	setTimeout(() => {
636	retryTimers.delete(channel);
637	run();
638	}, RETRY_DELAY_MS),
639	);
640	};
641
642	const handleUnauthorized = () => {
643	stop();
644	events.onInputConnected(false);
645	events.onUnauthorized();
646	};
647
648	// iOS video: fetch the AVCC body and demux into the decoder.
649	const readIosVideo = async () => {
650	const lifecycle = generation;
651	const feedGeneration = ++videoGeneration;
652	const demuxer = new AvccDemuxer();
653	const videoController = new AbortController();
654	controller = videoController;
655	const isCurrent = () =>
656	!stopped &&
657	generation === lifecycle &&
658	videoGeneration === feedGeneration &&
659	controller === videoController;
660	let retryDetail: string | undefined;
661	try {
662	const response = await fetch(httpUrl(videoPath), {
663	signal: videoController.signal,
664	credentials: access.credentials ? "include" : "same-origin",
665	});
666	if (!isCurrent()) {
667	await response.body?.cancel();
668	return;
669	}
670	if (response.status === 401 || response.status === 403) return handleUnauthorized();
671	if (target.panelId && [400, 404, 405, 410].includes(response.status)) {
672	await response.body?.cancel();
673	setStatus("error", "This Device Hub does not provide fixed Duo display feeds.");
674	return;
675	}
676	if (!response.ok || !response.body) throw new Error(`stream ${response.status}`);
677	const reader = response.body.getReader();
678	for (;;) {
679	// An AVCC body can stay open after its helper stops producing frames.
680	const timer = setTimeout(() => {
681	if (!isCurrent()) return;
682	retryDetail = "Device stream stopped receiving video. Reconnecting…";
683	videoController.abort();
684	}, FIRST_FRAME_TIMEOUT_MS);
685	videoReadTimer = timer;
686	let result: ReadableStreamReadResult<Uint8Array>;
687	try {
688	result = await reader.read();
689	} finally {
690	clearTimeout(timer);
691	if (videoReadTimer === timer) videoReadTimer = null;
692	}
693	const { done, value } = result;
694	if (!isCurrent()) return;
695	if (done) break;
696	for (const chunk of demuxer.push(value)) {
697	if (videoController.signal.aborted) break;
698	switch (chunk.type) {
699	case "seed":
700	void createImageBitmap(new Blob([chunk.payload as BlobPart], { type: "image/jpeg" }))
701	.then((bitmap) => {
702	try {
703	if (isCurrent() && decodedVideoGeneration !== feedGeneration)
704	paint(bitmap, bitmap.width, bitmap.height);
705	} finally {
706	bitmap.close();
707	}
708	})
709	.catch(() => {});
710	break;
711	case "description": {
712	awaitingKeyframe = true;
713	const configured = await configureDecoder(
714	{ codec: avcCodecString(chunk.payload), description: chunk.payload },
715	isCurrent,
716	);
717	if (!isCurrent()) return;
718	if (!configured) {
719	await reader.cancel().catch(() => {});
720	if (!isCurrent()) return;
721	if (target.videoOnly)
722	setStatus(
723	"error",
724	`This browser cannot decode the Duo panel's ${avcCodecString(chunk.payload)} stream.`,
725	);
726	else fallBackToMjpeg();
727	return;
728	}
729	break;
730	}
731	case "keyframe":
732	case "delta":
733	decode(chunk.type === "keyframe", chunk.payload);
734	break;
735	}
736	}
737	}
738	} catch (cause) {
739	if (!isCurrent()) return;
740	retryDetail ??= videoController.signal.aborted
741	? "Video decoder fell behind. Reconnecting…"
742	: (cause as Error).message;
743	}
744	if (isCurrent()) {
745	controller = null;
746	videoController.abort();
747	closeDecoder();
748	connecting(retryDetail);
749	scheduleRetry("video", () => void readIosVideo());
750	}
751	};
752
753	/**
754	* serve-sim's helper only accepts HID and pushes its screen config once
755	* screen capture is running, and the AVCC stream does not reliably start
756	* it. Touching the MJPEG endpoint does; one aborted request is enough.
757	*/
758	const primeIosHelper = async (session: number) => {
759	const controller = new AbortController();
760	primeController = controller;
761	const timeout = setTimeout(() => controller.abort(), 2_000);
762	try {
763	const response = await fetch(httpUrl(`/helper/${device}/stream.mjpeg`), {
764	signal: controller.signal,
765	credentials: access.credentials ? "include" : "same-origin",
766	});
767	if (stopped || generation !== session) return;
768	if (response.status === 401 || response.status === 403) return handleUnauthorized();
769	await response.body?.getReader().read();
770	} catch {
771	// A failed prime just means the socket may take a retry to come up.
772	} finally {
773	clearTimeout(timeout);
774	controller.abort();
775	if (primeController === controller) primeController = null;
776	}
777	};
778
779	const startDuoVideo = (panels: DuoPanelSinks) => {
780	for (const panel of panelClients) panel.stop();
781	// Physical handoff elects a native surface. Fixed-panel encoders can keep
782	// an inactive shutdown frame after election, so this build uses one active
783	// feed instead of decoding a third stream alongside the two fixed feeds.
784	const ids = screen?.supportsPhysicalOrientation ? ([null] as const) : ([1, 3] as const);
785	panelClients = ids.map((id) => {
786	const output = id === 1 ? panels.cover : panels.inner;
787	return createDeviceStreamClient(
788	{ ...target, ...(id === null ? {} : { panelId: id }), videoOnly: true },
789	{
790	present(source, width, height) {
791	if (id === null) {
792	return sink.present(source, width, height);
793	}
794	// An inactive native LCD can emit its shutdown black frame. Retain its last useful image.
795	if (screen?.screenId !== id) return true;
796	const retained = output.present(source, width, height);
797	const primary = sink.present(source, width, height);
798	return retained && primary;
799	},
800	},
801	{
802	onStatus: (status, detail) => {
803	if (status === "error") events.onDuoUnavailable?.(detail);
804	},
805	onScreen: () => {},
806	onInputConnected: () => {},
807	onMjpegFallback: () => {},
808	onUnauthorized: handleUnauthorized,
809	},
810	);
811	});
812	for (const panel of panelClients) panel.start();
813	};
814
815	// iOS input socket; also carries the screen config the helper pushes.
816	const connectIosInput = async () => {
817	if (stopped) return;
818	const session = generation;
819	await primeIosHelper(session);
820	if (stopped || generation !== session) return;
821	const ws = new WebSocket(wsUrl(`/helper/ws?device=${device}`));
822	ws.binaryType = "arraybuffer";
823	socket = ws;
824	iosInputAdmitted = false;
825	iosInputUnavailable = false;
826	ws.onmessage = (event) => {
827	if (stopped || socket !== ws) return;
828	if (!(event.data instanceof ArrayBuffer)) return;
829	const bytes = new Uint8Array(event.data);
830	if (socket !== ws || stopped || bytes.length < 1) return;
831	if (bytes[0] === IOS_TAG_INPUT_ADMITTED) {
832	iosInputAdmitted = true;
833	events.onInputConnected(!iosInputUnavailable);
834	return;
835	}
836	try {
837	const payload: unknown = JSON.parse(decoder.decode(bytes.subarray(1)));
838	if (bytes[0] === 0x90) {
839	const reply = decodeControlReply(payload);
840	if (Option.isSome(reply)) duoControl.receive(reply.value);
841	} else if (bytes[0] === IOS_TAG_SCREEN_CONFIG) {
842	const config = decodeScreenConfig(payload);
843	if (Option.isSome(config)) {
844	iosInputUnavailable = config.value.inputUnavailable === true;
845	events.onInputConnected(
846	iosInputAdmitted && !iosInputUnavailable,
847	iosInputUnavailable ? "Simulator input is unavailable" : undefined,
848	);
849	const previous = screen;
850	screen = config.value;
851	if (screen.hingePose && screen.hingePose !== previous?.hingePose)
852	rotationCursor = screen.hingePose === "laptop" ? "landscape_left" : "portrait";
853	else if (screen.orientation !== previous?.orientation)
854	rotationCursor = screen.orientation;
855	panelSinks?.onScreen?.(screen);
856	events.onScreen(screen);
857	// A surface election can leave an existing decoder on the former
858	// encoder description. Reopen only video to acquire the elected
859	// surface's seed and codec configuration; HID and the viewer stay.
860	if (
861	panelSinks &&
862	screen.supportsPhysicalOrientation &&
863	previous &&
864	screen.screenId !== previous.screenId
865	)
866	startDuoVideo(panelSinks);
867	if (pendingOrientation) {
868	const receipt = pendingOrientation;
869	pendingOrientation = null;
870	duoControl.receive({
871	requestId: receipt.requestId,
872	ok: true,
873	});
874	}
875	}
876	}
877	} catch {
878	// Ignore malformed config frames.
879	}
880	};
881	ws.onclose = (event) => {
882	if (socket !== ws) return;
883	socket = null;
884	iosInputAdmitted = false;
885	duoControl.clear();
886	rotationCursor = null;
887	if (stopped) return;
888	events.onInputConnected(
889	false,
890	event.reason || (event.code === 1006 ? "input socket refused" : `closed ${event.code}`),
891	);
892	// A rejected HTTP upgrade surfaces as 1006, including an expired stream ticket.
893	if (
894	event.code === 1008 ||
895	event.code === 4401 ||
896	(event.code === 1006 && access.query.wsTicket)
897	)
898	return handleUnauthorized();
899	scheduleRetry("input", () => void connectIosInput());
900	};
901	ws.onerror = () => ws.close();
902	};
903
904	// Android: one socket for video and input.
905	const connectAndroid = () => {
906	if (stopped) return;
907	const ws = new WebSocket(wsUrl(`/ws?device=${device}&frame-meta=1`));
908	ws.binaryType = "arraybuffer";
909	socket = ws;
910	ws.onopen = () => {
911	if (stopped || socket !== ws) return;
912	connecting();
913	events.onInputConnected(true);
914	};
915	ws.onmessage = (event) => {
916	if (stopped || socket !== ws) return;
917	if (typeof event.data === "string") {
918	// The encoder restarts at a new size when the device rotates; the
919	// next keyframe carries a fresh SPS, so the decoder is rebuilt from it.
920	if (isVideoSessionMessage(event.data)) {
921	closeDecoder();
922	configuring = false;
923	connecting();
924	requestKeyframe();
925	}
926	return;
927	}
928	if (!(event.data instanceof ArrayBuffer)) return;
929	const packet = parseSemuPacket(event.data);
930	const needsScan =
931	packet.isKey === null ||
932	(packet.isKey && (!videoDecoder || videoDecoder.state !== "configured"));
933	const scanned = needsScan ? scanAccessUnit(packet.data) : null;
934	const isKey = packet.isKey ?? scanned?.isKey ?? false;
935	if (scanned?.sps && (!videoDecoder || videoDecoder.state !== "configured")) {
936	if (configuring) return;
937	configuring = true;
938	const epoch = decoderEpoch;
939	const isCurrent = () => !stopped && socket === ws;
940	void configureDecoder({ codec: avcCodecString(scanned.sps) }, isCurrent).then(
941	(configured) => {
942	if (!isCurrent() || epoch !== decoderEpoch) return;
943	configuring = false;
944	awaitingKeyframe = true;
945	if (configured) requestKeyframe();
946	},
947	);
948	return;
949	}
950	if (!videoDecoder || videoDecoder.state !== "configured") {
951	if (!isKey) requestKeyframe();
952	return;
953	}
954	decode(isKey, packet.data, packet.timestamp);
955	};
956	ws.onclose = (event) => {
957	if (socket !== ws) return;
958	socket = null;
959	closeDecoder();
960	if (stopped) return;
961	events.onInputConnected(false, event.reason || `closed ${event.code}`);
962	if (
963	event.code === 1008 ||
964	event.code === 4401 ||
965	(event.code === 1006 && access.query.wsTicket)
966	)
967	return handleUnauthorized();
968	configuring = false;
969	connecting(event.reason || undefined);
970	scheduleRetry("input", connectAndroid);
971	};
972	ws.onerror = () => ws.close();
973	};
974
975	const start = () => {
976	if (!stopped) return;
977	stopped = false;
978	generation++;
979	configuring = false;
980	iosInputAdmitted = false;
981	iosInputUnavailable = false;
982	connecting();
983	if (platform === "ios") {
984	if (!target.videoOnly) void connectIosInput();
985	if (useWebCodecs) void readIosVideo();
986	else fallBackToMjpeg();
987	} else if (useWebCodecs) {
988	connectAndroid();
989	} else {
990	fail("This browser cannot decode the Android stream (WebCodecs unavailable).");
991	}
992	};
993
994	const stop = () => {
995	if (stopped) return;
996	stopped = true;
997	generation++;
998	videoGeneration++;
999	duoControl.clear();
1000	rotationCursor = null;
1001	for (const panel of panelClients) panel.stop();
1002	panelClients = [];
1003	panelSinks = null;
1004	mjpeg = false;
1005	clearFrameTimer();
1006	if (videoReadTimer !== null) clearTimeout(videoReadTimer);
1007	videoReadTimer = null;
1008	releaseImage?.();
1009	releaseImage = null;
1010	mjpegImage = null;
1011	for (const timer of retryTimers.values()) clearTimeout(timer);
1012	retryTimers.clear();
1013	primeController?.abort();
1014	primeController = null;
1015	controller?.abort();
1016	controller = null;
1017	const discarded = socket;
1018	socket = null;
1019	discarded?.close();
1020	closeDecoder();
1021	};
1022
1023	const send = (payload: Uint8Array<ArrayBuffer> | string) => {
1024	if (
1025	!stopped &&
1026	(platform !== "ios" || (iosInputAdmitted && !iosInputUnavailable)) &&
1027	socket?.readyState === WebSocket.OPEN
1028	)
1029	socket.send(payload);
1030	};
1031
1032	const rawPoint = (x: number, y: number) => {
1033	// serve-sim streams the raw framebuffer; rotated devices need input
1034	// remapped into that raw space.
1035	if (platform !== "ios" || !screen || screen.width > screen.height) return { x, y };
1036	switch (screen.orientation) {
1037	case "landscape_left":
1038	return { x: y, y: 1 - x };
1039	case "landscape_right":
1040	return { x: 1 - y, y: x };
1041	case "portrait_upside_down":
1042	return { x: 1 - x, y: 1 - y };
1043	default:
1044	return { x, y };
1045	}
1046	};
1047
1048	return {
1049	start,
1050	stop,
1051	setMjpegImage,
1052	controlDuo: duoControl.enqueue,
1053	sendRawTouch: (phase, x, y) => {
1054	if (platform === "ios") send(taggedJson(IOS_MSG_TOUCH, { type: phase, x, y }));
1055	},
1056	setDuoPanels(panels) {
1057	if (platform !== "ios" || target.videoOnly || stopped || panelSinks === panels) return;
1058	if (panels && !screen?.supportsHingeAngle) return;
1059	panelSinks = panels;
1060	videoGeneration++;
1061	controller?.abort();
1062	controller = null;
1063	closeDecoder();
1064	const retry = retryTimers.get("video");
1065	if (retry) clearTimeout(retry);
1066	retryTimers.delete("video");
1067	for (const panel of panelClients) panel.stop();
1068	panelClients = [];
1069	if (!panels) {
1070	if (useWebCodecs) void readIosVideo();
1071	return;
1072	}
1073	startDuoVideo(panels);
1074	},
1075	sendTouch: (phase, x, y) => {
1076	if (platform === "ios") {
1077	send(taggedJson(IOS_MSG_TOUCH, { type: phase, ...rawPoint(x, y) }));
1078	return;
1079	}
1080	const action = phase === "begin" ? "down" : phase === "move" ? "move" : "up";
1081	send(JSON.stringify({ type: "touch", action, x, y }));
1082	},
1083	sendKey: (event, phase) => {
1084	if (platform === "ios") {
1085	const usage = hidUsageForCode(event.code);
1086	if (usage !== null) send(taggedJson(IOS_MSG_KEY, { type: phase, usage }));
1087	return;
1088	}
1089	if (phase !== "down") return;
1090	if (event.key === "Escape") return send(JSON.stringify({ type: "back" }));
1091	const keycode = ANDROID_KEYCODE_BY_KEY[event.key];
1092	if (keycode !== undefined) return send(JSON.stringify({ type: "key", keycode }));
1093	if (event.key.length === 1 && !event.metaKey && !event.ctrlKey) {
1094	send(JSON.stringify({ type: "text", text: event.key }));
1095	}
1096	},
1097	pressButton: (button) => {
1098	if (platform === "ios") {
1099	const name =
1100	button === "home"
1101	? "home"
1102	: button === "appSwitcher"
1103	? "app_switcher"
1104	: button === "power"
1105	? "lock"
1106	: null;
1107	if (name) send(taggedJson(IOS_MSG_BUTTON, { button: name }));
1108	return;
1109	}
1110	const type = button === "appSwitcher" ? "recents" : button;
1111	if (type === "home" || type === "back" || type === "recents" || type === "power") {
1112	send(JSON.stringify({ type }));
1113	}
1114	},
1115	rotate: () => {
1116	if (platform !== "ios") return;
1117	const current = screen?.supportsHingeAngle
1118	? (rotationCursor ?? screen.orientation)
1119	: (screen?.orientation ?? "portrait");
1120	const next =
1121	IOS_ORIENTATIONS[(IOS_ORIENTATIONS.indexOf(current) + 1) % IOS_ORIENTATIONS.length]!;
1122	if (screen?.supportsHingeAngle) {
1123	rotationCursor = next;
1124	duoControl.enqueue({ control: "orientation", value: next });
1125	} else send(taggedJson(IOS_MSG_ORIENTATION, { orientation: next }));
1126	},
1127	setOrientation: (orientation) => {
1128	if (platform === "ios") send(taggedJson(IOS_MSG_ORIENTATION, { orientation }));
1129	},
1130	};
1131	}