Commit 9f73ca3677faa975be00f0ac55460279671df726 | the-knife

author	Julius Marminge <julius0216@outlook.com> 2026-09-20 16:44:32-07:00
committer	GitHub <noreply@github.com> 2026-09-20 16:44:32-07:00
log	9f73ca3677faa975be00f0ac55460279671df726
tree	262f08d8c5f36d76214b1a79dfbe8d0302266d62
parent	f4d979a6c6139ab10aa1013335bd94a313a5b56e
signature	Signed by PGP key B5690EEEBB952194

fix(devices): recover simulator streams after failures (#12639)

14 files changed, 1457 insertions(+), 503 deletions(-)

apps
- mobile
  - src
    features
    devices
    DevicePreviewRouteScreen.tsx
    DeviceStreamWebView.tsx
    device-stream-document.test.ts
    device-stream-document.ts
    device-stream.browser.test.ts
    device-stream.browser.ts
- server
  - src
    device
    DeviceService.test.ts
    DeviceService.ts
- web
  - src
    components
    device
    DeviceLoadingView.tsx
    DeviceStreamView.test.tsx
    DeviceStreamView.tsx
    deviceStream.test.ts
packages
- client-runtime
  - src
    device
    stream.test.ts
    stream.ts

apps/mobile/src/features/devices/DevicePreviewRouteScreen.tsx+2-2

...	...	@@ -68,7 +68,7 @@ function DevicePreviewScreen({
68	68	const insets = useSafeAreaInsets();
69	69	const { themeVariables } = useAppearancePreferences();
70	70	const focused = useIsFocused();
71		const [foreground, setForeground] = useState(AppState.currentState === "active");
	71	const [foreground, setForeground] = useState(AppState.currentState !== "background");
72	72	const [selectedKey, setSelectedKey] = useState<string \| null>(null);
73	73	const [inputConnected, setInputConnected] = useState(false);
74	74	const [streamAttempt, setStreamAttempt] = useState(0);
...	...	@@ -87,7 +87,7 @@ function DevicePreviewScreen({
87	87	);
88	88	useEffect(() => {
89	89	const subscription = AppState.addEventListener("change", (state) =>
90		setForeground(state === "active"),
	90	setForeground(state !== "background"),
91	91	);
92	92	return () => subscription.remove();
93	93	}, []);

apps/mobile/src/features/devices/DeviceStreamWebView.tsx+127-26

...	...	@@ -1,7 +1,19 @@
1	1	import deviceStreamScript from "@t3tools/mobile-device-stream";
2		import { useImperativeHandle, useLayoutEffect, useMemo, useRef, useState, type Ref } from "react";
3		import { Platform } from "react-native";
	2	import {
	3	useEffect,
	4	useEffectEvent,
	5	useImperativeHandle,
	6	useLayoutEffect,
	7	useMemo,
	8	useRef,
	9	useState,
	10	type Ref,
	11	} from "react";
	12	import { ActivityIndicator, Platform, Pressable, View } from "react-native";
4	13	import { WebView } from "react-native-webview";
	14	import type { DeviceStreamStatus } from "@t3tools/client-runtime/device/stream";
	15
	16	import { AppText } from "../../components/AppText";
5	17
6	18	import {
7	19	deviceStreamDocument,
...	...	@@ -27,6 +39,7 @@ export function DeviceStreamWebView({
27	39	...props
28	40	}: DeviceStreamConfiguration & NativeStreamBridge) {
29	41	const [attempt, setAttempt] = useState(0);
	42	const processRetried = useRef(false);
30	43	const configuration = JSON.stringify({
31	44	access: props.access,
32	45	platform: props.platform,
...	...	@@ -41,7 +54,21 @@ export function DeviceStreamWebView({
41	54	background={props.colors.background}
42	55	onUnauthorized={props.onUnauthorized}
43	56	onInputConnected={props.onInputConnected}
44		onRetry={() => setAttempt((attempt) => attempt + 1)}
	57	onRetry={() => {
	58	processRetried.current = false;
	59	setAttempt((attempt) => attempt + 1);
	60	void props.onUnauthorized();
	61	}}
	62	onStreaming={() => {
	63	processRetried.current = false;
	64	}}
	65	onRecoverProcess={() => {
	66	if (processRetried.current) return false;
	67	processRetried.current = true;
	68	setAttempt((attempt) => attempt + 1);
	69	void props.onUnauthorized();
	70	return true;
	71	}}
45	72	/>
46	73	);
47	74	}
...	...	@@ -53,12 +80,38 @@ function DeviceStreamDocumentView({
53	80	onUnauthorized,
54	81	onInputConnected,
55	82	onRetry,
	83	onStreaming,
	84	onRecoverProcess,
56	85	}: NativeStreamBridge & {
57	86	readonly configuration: string;
58	87	readonly background: string;
59	88	readonly onRetry: () => void;
	89	readonly onStreaming: () => void;
	90	readonly onRecoverProcess: () => boolean;
60	91	}) {
61	92	const webView = useRef<WebView>(null);
	93	const active = useRef(true);
	94	const failed = useRef(false);
	95	const [status, setStatus] = useState<DeviceStreamStatus>("connecting");
	96	const [error, setError] = useState<string \| null>(null);
	97	const [started, setStarted] = useState(false);
	98	const fail = (message: string) => {
	99	if (!active.current \|\| failed.current) return;
	100	failed.current = true;
	101	void onInputConnected(false);
	102	webView.current?.injectJavaScript("window.T3DeviceStream?.stop(); true;");
	103	setError(message);
	104	setStatus("error");
	105	};
	106	// The shared transport owns video timeouts once the document acknowledges startup.
	107	const bootstrapTimedOut = useEffectEvent(() =>
	108	fail("Device viewer could not start. Reconnect to try again."),
	109	);
	110	useEffect(() => {
	111	if (started) return;
	112	const timer = setTimeout(bootstrapTimedOut, 15_000);
	113	return () => clearTimeout(timer);
	114	}, [started]);
62	115	const source = useMemo(
63	116	() => ({
64	117	html: deviceStreamDocument(configuration, deviceStreamScript),
...	...	@@ -78,32 +131,80 @@ function DeviceStreamDocumentView({
78	131	appSwitcher: () => command("appSwitcher"),
79	132	rotate: () => command("rotate"),
80	133	}));
	134	const resetInput = useEffectEvent(() => void onInputConnected(false));
81	135	useLayoutEffect(() => {
	136	active.current = true;
	137	resetInput();
82	138	const view = webView.current;
83		return () => view?.injectJavaScript("window.T3DeviceStream?.stop(); true;");
	139	return () => {
	140	active.current = false;
	141	view?.injectJavaScript("window.T3DeviceStream?.stop(); true;");
	142	};
84	143	}, []);
	144	const processTerminated = () => {
	145	if (!active.current \|\| failed.current) return;
	146	void onInputConnected(false);
	147	if (!onRecoverProcess()) fail("Device viewer stopped. Reconnect to try again.");
	148	};
85	149	return (
86		<WebView
87		ref={webView}
88		source={source}
89		originWhitelist={["*"]}
90		scrollEnabled={false}
91		bounces={false}
92		mixedContentMode="always"
93		allowUniversalAccessFromFileURLs
94		contentInsetAdjustmentBehavior="never"
95		setSupportMultipleWindows={false}
96		style={{ flex: 1, backgroundColor: background }}
97		onLoadStart={() => void onInputConnected(false)}
98		onShouldStartLoadWithRequest={(request) =>
99		request.url === "about:blank" \|\| request.url === source.baseUrl
100		}
101		onMessage={(event) => {
102		const message = deviceStreamMessage(event.nativeEvent.data);
103		if (message?.type === "unauthorized") void onUnauthorized();
104		else if (message?.type === "input") void onInputConnected(message.connected);
105		else if (message?.type === "retry") onRetry();
106		}}
107		/>
	150	<View className="flex-1" style={{ backgroundColor: background }}>
	151	<WebView
	152	ref={webView}
	153	source={source}
	154	originWhitelist={["*"]}
	155	scrollEnabled={false}
	156	bounces={false}
	157	mixedContentMode="always"
	158	allowUniversalAccessFromFileURLs
	159	contentInsetAdjustmentBehavior="never"
	160	setSupportMultipleWindows={false}
	161	style={{ flex: 1, backgroundColor: background }}
	162	onError={() => fail("Device viewer could not load. Reconnect to try again.")}
	163	onHttpError={() => fail("Device viewer could not load. Reconnect to try again.")}
	164	onContentProcessDidTerminate={processTerminated}
	165	onRenderProcessGone={processTerminated}
	166	onShouldStartLoadWithRequest={(request) =>
	167	request.url === "about:blank" \|\| request.url === source.baseUrl
	168	}
	169	onMessage={(event) => {
	170	if (!active.current \|\| failed.current) return;
	171	const message = deviceStreamMessage(event.nativeEvent.data);
	172	if (message?.type === "unauthorized") void onUnauthorized();
	173	else if (message?.type === "input") void onInputConnected(message.connected);
	174	else if (message?.type === "retry") onRetry();
	175	else if (message?.type === "status") {
	176	setStarted(true);
	177	if (message.status === "error") fail(message.detail ?? "Device stream failed.");
	178	else {
	179	setStatus(message.status);
	180	if (message.status === "streaming") onStreaming();
	181	}
	182	}
	183	}}
	184	/>
	185	{status !== "streaming" ? (
	186	<View
	187	className="absolute inset-0 items-center justify-center gap-4 px-6"
	188	style={{ backgroundColor: background }}
	189	>
	190	{status === "connecting" ? <ActivityIndicator colorClassName="accent-icon" /> : null}
	191	<AppText
	192	accessibilityLiveRegion="polite"
	193	className="text-center text-sm text-foreground-muted"
	194	>
	195	{status === "error" ? error : "Connecting to device..."}
	196	</AppText>
	197	{status === "error" ? (
	198	<Pressable
	199	accessibilityRole="button"
	200	className="rounded-full border border-secondary-border bg-secondary px-6 py-3"
	201	onPress={onRetry}
	202	>
	203	<AppText className="text-secondary-foreground">Reconnect</AppText>
	204	</Pressable>
	205	) : null}
	206	</View>
	207	) : null}
	208	</View>
108	209	);
109	210	}

apps/mobile/src/features/devices/device-stream-document.test.ts+44-2

...	...	@@ -1,5 +1,5 @@
1	1	import * as NodeVM from "node:vm";
2		import { describe, expect, it } from "vite-plus/test";
	2	import { describe, expect, it, vi } from "vite-plus/test";
3	3
4	4	import { deviceStreamDocument, deviceStreamMessage } from "./device-stream-document";
5	5
...	...	@@ -13,7 +13,7 @@ describe("native device stream document", () => {
13	13	);
14	14	const script = html.match(/<script>([\s\S]*)<\/script>/)?.[1];
15	15	expect(html.match(/<script>/g)).toHaveLength(1);
16		const input: unknown = NodeVM.runInNewContext(script!);
	16	const input: unknown = NodeVM.runInNewContext(script!, { window: { addEventListener() {} } });
17	17	expect(input).toEqual({ deviceId, ticket: "<ticket>" });
18	18	});
19	19
...	...	@@ -22,6 +22,39 @@ describe("native device stream document", () => {
22	22	expect(html.match(/<\/script>/g)).toHaveLength(1);
23	23	expect(html).toContain('var text="<\\/script>";');
24	24	});
	25
	26	it("reports a bootstrap exception to the native recovery UI", () => {
	27	const postMessage = vi.fn();
	28	const html = deviceStreamDocument(
	29	"{}",
	30	'var T3DeviceStream={start(){throw new Error("startup")}};',
	31	);
	32	NodeVM.runInNewContext(html.match(/<script>([\s\S]*)<\/script>/)![1]!, {
	33	window: { addEventListener() {}, ReactNativeWebView: { postMessage } },
	34	});
	35	expect(JSON.parse(postMessage.mock.calls[0]![0] as string)).toEqual({
	36	type: "status",
	37	status: "error",
	38	detail: "Device viewer stopped unexpectedly.",
	39	});
	40	});
	41
	42	it.each(["error", "unhandledrejection"])("reports later %s failures to native", (event) => {
	43	const postMessage = vi.fn();
	44	const listeners = new Map<string, () => void>();
	45	const html = deviceStreamDocument("{}", "var T3DeviceStream={start(){}};");
	46	NodeVM.runInNewContext(html.match(/<script>([\s\S]*)<\/script>/)![1]!, {
	47	window: {
	48	addEventListener: (name: string, callback: () => void) => listeners.set(name, callback),
	49	ReactNativeWebView: { postMessage },
	50	},
	51	});
	52	listeners.get(event)!();
	53	expect(JSON.parse(postMessage.mock.calls[0]![0] as string)).toMatchObject({
	54	type: "status",
	55	status: "error",
	56	});
	57	});
25	58	});
26	59
27	60	describe("native device stream messages", () => {
...	...	@@ -38,6 +71,12 @@ describe("native device stream messages", () => {
38	71	expect(deviceStreamMessage('{"type":"retry"}')).toEqual({ type: "retry" });
39	72	});
40	73
	74	it.each(["connecting", "streaming", "error"])("accepts stream %s feedback", (status) => {
	75	expect(
	76	deviceStreamMessage(JSON.stringify({ type: "status", status, detail: "Stream feedback" })),
	77	).toEqual({ type: "status", status, detail: "Stream feedback" });
	78	});
	79
41	80	it.each([
42	81	"invalid JSON",
43	82	"null",
...	...	@@ -45,5 +84,8 @@ describe("native device stream messages", () => {
45	84	"{}",
46	85	'{"type":"input","connected":"yes"}',
47	86	'{"type":"unknown"}',
	87	'{"type":"status","status":"unknown"}',
	88	'{"type":"status","status":"error","detail":42}',
	89	'{"type":"status"}',
48	90	])("ignores invalid bridge messages: %s", (data) => expect(deviceStreamMessage(data)).toBeNull());
49	91	});

apps/mobile/src/features/devices/device-stream-document.ts+17-1

...	...	@@ -19,7 +19,8 @@ export function deviceStreamDocument(configuration: string, script: string) {
19	19	// Tickets and device names are data, including any HTML delimiter characters.
20	20	const safeConfiguration = configuration.replace(/</g, "\\u003c");
21	21	const safeScript = script.replace(/<\/script/gi, "<\\/script");
22		return `<!doctype html><html><head><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=1,user-scalable=no"></head><body><script>${safeScript}\nT3DeviceStream.start(${safeConfiguration});</script></body></html>`;
	22	const failure = `window.ReactNativeWebView.postMessage(JSON.stringify({type:"status",status:"error",detail:"Device viewer stopped unexpectedly."}));`;
	23	return `<!doctype html><html><head><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=1,user-scalable=no"></head><body><script>window.addEventListener("error",function(){${failure}});window.addEventListener("unhandledrejection",function(){${failure}});\n${safeScript}\ntry{T3DeviceStream.start(${safeConfiguration});}catch{${failure}}</script></body></html>`;
23	24	}
24	25
25	26	export function deviceStreamMessage(data: string) {
...	...	@@ -36,6 +37,21 @@ export function deviceStreamMessage(data: string) {
36	37	) {
37	38	return { type: message.type, connected: message.connected } as const;
38	39	}
	40	if (
	41	message.type === "status" &&
	42	"status" in message &&
	43	(message.status === "connecting" \|\|
	44	message.status === "streaming" \|\|
	45	message.status === "error") &&
	46	(!("detail" in message) \|\| typeof message.detail === "string")
	47	) {
	48	return {
	49	type: message.type,
	50	status: message.status,
	51	detail:
	52	"detail" in message && typeof message.detail === "string" ? message.detail : undefined,
	53	} as const;
	54	}
39	55	} catch {
40	56	// Ignore messages that are not part of the stream bridge.
41	57	}

apps/mobile/src/features/devices/device-stream.browser.test.ts created+117

...	...	@@ -0,0 +1,117 @@
	1	import { afterEach, expect, it, vi } from "vite-plus/test";
	2	import { start, stop } from "./device-stream.browser";
	3
	4	class Element extends EventTarget {
	5	readonly style = {};
	6	naturalWidth = 0;
	7	naturalHeight = 0;
	8	src = "";
	9	readonly tag: string;
	10	constructor(tag: string) {
	11	super();
	12	this.tag = tag;
	13	}
	14	setAttribute() {}
	15	removeAttribute(name: string) {
	16	if (name === "src") this.src = "";
	17	}
	18	append() {}
	19	}
	20
	21	async function setup() {
	22	vi.useFakeTimers();
	23	const elements: Element[] = [];
	24	vi.stubGlobal("document", {
	25	documentElement: { style: {} },
	26	body: { style: {}, replaceChildren() {} },
	27	createElement: (tag: string) => {
	28	const element = new Element(tag);
	29	elements.push(element);
	30	return element;
	31	},
	32	});
	33	const postMessage = vi.fn();
	34	vi.stubGlobal("window", { ReactNativeWebView: { postMessage }, addEventListener() {} });
	35	vi.stubGlobal("fetch", () => Promise.resolve(new Response("prime")));
	36	const sockets: Socket[] = [];
	37	class Socket {
	38	static OPEN = 1;
	39	readyState = 1;
	40	onopen: (() => void) \| null = null;
	41	close = vi.fn();
	42	send = vi.fn();
	43	constructor() {
	44	sockets.push(this);
	45	}
	46	}
	47	vi.stubGlobal("WebSocket", Socket);
	48	const configuration = {
	49	platform: "ios" as const,
	50	deviceId: "fixture-device",
	51	access: {
	52	httpBase: "https://device.test",
	53	wsBase: "wss://device.test",
	54	credentials: false,
	55	query: {},
	56	},
	57	colors: {
	58	background: "white",
	59	foreground: "black",
	60	muted: "gray",
	61	buttonBackground: "gray",
	62	buttonForeground: "black",
	63	buttonBorder: "gray",
	64	},
	65	};
	66	start(configuration);
	67	await vi.advanceTimersByTimeAsync(0);
	68	return {
	69	configuration,
	70	elements,
	71	sockets,
	72	messages: () =>
	73	postMessage.mock.calls.map(
	74	([message]) => JSON.parse(message as string) as { type: string; status?: string },
	75	),
	76	};
	77	}
	78
	79	afterEach(() => {
	80	stop();
	81	vi.useRealTimers();
	82	vi.unstubAllGlobals();
	83	});
	84
	85	it("bridges shared first-frame readiness, image failure, and a successful fresh attempt to native", async () => {
	86	const { elements, messages, sockets, configuration } = await setup();
	87	sockets[0]!.onopen?.();
	88	expect(messages()).not.toContainEqual({ type: "status", status: "streaming" });
	89	expect(messages()).toContainEqual({ type: "input", connected: true });
	90	const image = elements.find((element) => element.tag === "img")!;
	91	image.naturalWidth = 400;
	92	image.naturalHeight = 800;
	93	await vi.advanceTimersByTimeAsync(250);
	94	expect(messages()).toContainEqual({ type: "status", status: "streaming" });
	95	image.dispatchEvent(new Event("error"));
	96	expect(messages()).toContainEqual({
	97	type: "status",
	98	status: "error",
	99	detail: "Could not receive the device stream. Reconnect to try again.",
	100	});
	101	expect(messages()).toContainEqual({ type: "input", connected: false });
	102	expect(messages()).not.toContainEqual({ type: "unauthorized" });
	103	expect(image.src).toBe("");
	104	expect(sockets[0]!.close).toHaveBeenCalledOnce();
	105	start(configuration);
	106	await vi.advanceTimersByTimeAsync(0);
	107	const replacement = elements.findLast((element) => element.tag === "img")!;
	108	replacement.naturalWidth = 400;
	109	replacement.naturalHeight = 800;
	110	replacement.dispatchEvent(new Event("load"));
	111	expect(messages().at(-1)).toEqual({ type: "status", status: "streaming" });
	112	image.dispatchEvent(new Event("error"));
	113	expect(sockets[1]!.close).not.toHaveBeenCalled();
	114	stop();
	115	expect(replacement.src).toBe("");
	116	expect(vi.getTimerCount()).toBe(0);
	117	});

apps/mobile/src/features/devices/device-stream.browser.ts+10-44

...	...	@@ -12,13 +12,10 @@ declare global {
12	12	}
13	13
14	14	let activeClient: ReturnType<typeof createDeviceStreamClient> \| null = null;
15		let activeImage: HTMLImageElement \| null = null;
16	15
17	16	export function stop() {
18	17	activeClient?.stop();
19	18	activeClient = null;
20		activeImage?.removeAttribute("src");
21		activeImage = null;
22	19	}
23	20
24	21	export function command(button: "home" \| "back" \| "appSwitcher" \| "rotate") {
...	...	@@ -70,34 +67,6 @@ export function start(configuration: DeviceStreamConfiguration) {
70	67	image.alt = "";
71	68	image.draggable = false;
72	69	image.style.display = "none";
73		const overlay = document.createElement("div");
74		overlay.setAttribute("role", "status");
75		Object.assign(overlay.style, {
76		position: "fixed",
77		inset: "0",
78		display: "flex",
79		flexDirection: "column",
80		alignItems: "center",
81		justifyContent: "center",
82		gap: "16px",
83		padding: "24px",
84		textAlign: "center",
85		background: colors.background,
86		});
87		const detail = document.createElement("span");
88		const retry = document.createElement("button");
89		retry.textContent = "Retry";
90		Object.assign(retry.style, {
91		padding: "12px 24px",
92		borderRadius: "20px",
93		border: `1px solid ${colors.buttonBorder}`,
94		background: colors.buttonBackground,
95		color: colors.buttonForeground,
96		font: "inherit",
97		display: "none",
98		});
99		retry.addEventListener("click", () => post({ type: "retry" }));
100		overlay.append(detail, retry);
101	70	const inputStatus = document.createElement("div");
102	71	inputStatus.setAttribute("role", "status");
103	72	inputStatus.textContent = "Reconnecting device controls...";
...	...	@@ -114,11 +83,17 @@ export function start(configuration: DeviceStreamConfiguration) {
114	83	});
115	84	frame.append(canvas, image);
116	85	container.append(frame);
117		document.body.replaceChildren(container, overlay, inputStatus);
	86	document.body.replaceChildren(container, inputStatus);
118	87
119	88	let pointerId: number \| null = null;
120	89	let inputConnected = false;
121	90	let streaming = false;
	91	const reportStatus = (status: "connecting" \| "streaming" \| "error", detail?: string) => {
	92	if (activeClient !== client) return;
	93	streaming = status === "streaming";
	94	inputStatus.style.display = streaming && !inputConnected ? "block" : "none";
	95	post({ type: "status", status, detail });
	96	};
122	97	const layout = (screen: DeviceScreenSize \| null) => {
123	98	const landscape =
124	99	screen?.orientation === "landscape_left" \|\| screen?.orientation === "landscape_right";
...	...	@@ -160,19 +135,11 @@ export function start(configuration: DeviceStreamConfiguration) {
160	135	{ ...configuration, preferMjpeg: platform === "ios" },
161	136	canvas,
162	137	{
163		onStatus: (status, message) => {
164		streaming = status === "streaming";
165		overlay.style.display = streaming ? "none" : "flex";
166		inputStatus.style.display = streaming && !inputConnected ? "block" : "none";
167		detail.textContent =
168		status === "error" ? (message ?? "Device stream failed.") : "Connecting to device...";
169		retry.style.display = status === "error" ? "block" : "none";
170		},
	138	onStatus: reportStatus,
171	139	onScreen: layout,
172		onMjpegFallback: (url) => {
	140	onMjpegFallback: () => {
173	141	canvas.style.display = "none";
174	142	image.style.display = "block";
175		image.src = url;
176	143	},
177	144	onUnauthorized: unauthorized,
178	145	onInputConnected: (connected) => {
...	...	@@ -183,8 +150,7 @@ export function start(configuration: DeviceStreamConfiguration) {
183	150	},
184	151	);
185	152	activeClient = client;
186		activeImage = image;
187		image.addEventListener("error", unauthorized);
	153	client.setMjpegImage(image);
188	154	const touch = (event: PointerEvent, phase: "begin" \| "move" \| "end") => {
189	155	const rect = frame.getBoundingClientRect();
190	156	client.sendTouch(

apps/server/src/device/DeviceService.test.ts+91

...	...	@@ -11,6 +11,7 @@ import * as Deferred from "effect/Deferred";
11	11	import * as Fiber from "effect/Fiber";
12	12	import * as PubSub from "effect/PubSub";
13	13	import * as Ref from "effect/Ref";
	14	import * as Schema from "effect/Schema";
14	15	import * as Stream from "effect/Stream";
15	16	import { HttpClient, HttpClientResponse } from "effect/unstable/http";
16	17	import { ServerSettingsService } from "../serverSettings.ts";
...	...	@@ -19,6 +20,8 @@ import { NodeRuntimeUnavailableError } from "@t3tools/shared/nodeRuntime";
19	20
20	21	import { type DeviceService, makeWithHosts, stateStream } from "./DeviceService.ts";
21	22
	23	const decodeJson = Schema.decodeUnknownSync(Schema.fromJsonString(Schema.Unknown));
	24
22	25	const baseState: DeviceServiceState = {
23	26	hosts: [],
24	27	hostStatus: "idle",
...	...	@@ -378,3 +381,91 @@ it.effect("keeps shutdown successful when subsequent discovery fails", () =>
378	381	expect(state.devices.find((device) => device.id === session.deviceId)?.booted).toBe(false);
379	382	}).pipe(Effect.scoped),
380	383	);
	384
	385	it.effect.each(["shutdown", "close"] as const)(
	386	"%s releases iOS capture so reopening uses a fresh session",
	387	(operation) =>
	388	Effect.gen(function* () {
	389	const deviceId = DeviceId.make("11111111-1111-1111-1111-111111111111");
	390	const threadId = ThreadId.make("capture-recovery");
	391	let booted = true;
	392	let capture: number \| null = null;
	393	let generation = 0;
	394	const ready: DeviceHost.DeviceHostReady = {
	395	nodePath: process.execPath,
	396	hub: { origin: "http://device.test" },
	397	helpers: { serveSimAxSettings: null, serveSimCli: null },
	398	run: () => Effect.succeed({ code: 0, stdout: "", stderr: "" }),
	399	};
	400	const host: DeviceHost.DeviceHost["Service"] = {
	401	id: LOCAL_DEVICE_HOST_ID,
	402	summary: Effect.succeed({
	403	id: LOCAL_DEVICE_HOST_ID,
	404	kind: "local",
	405	label: "Simulator host",
	406	platforms: [{ platform: "ios", available: true }],
	407	hubInstalled: true,
	408	agentDeviceInstalled: false,
	409	}),
	410	platformAvailability: (platform) => Effect.succeed({ platform, available: true }),
	411	ensureReady: () => Effect.succeed(ready),
	412	ensureAgentReady: () => Effect.die("Agent access is not used in this test"),
	413	current: Effect.succeed(ready),
	414	stopAgent: Effect.void,
	415	stop: Effect.void,
	416	};
	417	const http = HttpClient.make((request) =>
	418	Effect.sync(() => {
	419	const path = new URL(request.url).pathname;
	420	if (path === "/api/devices") {
	421	return HttpClientResponse.fromWeb(
	422	request,
	423	Response.json({
	424	emulators: [],
	425	simulators: [
	426	{
	427	id: deviceId,
	428	name: "iPhone",
	429	platform: "ios",
	430	version: "26",
	431	physical: false,
	432	booted,
	433	},
	434	],
	435	}),
	436	);
	437	}
	438	if (path === "/vendor/serve-sim/grid/api/start") capture ??= ++generation;
	439	else if (path === "/vendor/serve-sim/grid/api/shutdown") {
	440	if (request.body._tag !== "Uint8Array") throw new Error("Missing shutdown body");
	441	expect(decodeJson(new TextDecoder().decode(request.body.body))).toEqual({
	442	udid: deviceId,
	443	});
	444	capture = null;
	445	booted = false;
	446	} else if (path === "/api/devices/shutdown") {
	447	// This route powers off without releasing serve-sim's cached capture.
	448	booted = false;
	449	} else if (path === "/api/devices/boot") booted = true;
	450	else throw new Error(`Unexpected hub path: ${path}`);
	451	return HttpClientResponse.fromWeb(request, Response.json({ ok: true, id: deviceId }));
	452	}),
	453	);
	454	const service = yield* makeWithHosts(new Map([[host.id, host]])).pipe(
	455	Effect.provideService(HttpClient.HttpClient, http),
	456	);
	457	const input = { threadId, deviceId, platform: "ios" as const };
	458	yield* service.open(input);
	459	expect(capture).toBe(1);
	460	if (operation === "shutdown") yield* service.shutdown(input);
	461	else yield* service.close({ threadId, deviceId, shutdown: true });
	462	expect(capture).toBeNull();
	463	expect((yield* service.state).sessions).toEqual([]);
	464	yield* service.open(input);
	465	expect(capture).toBe(2);
	466	expect((yield* service.state).sessions).toHaveLength(1);
	467	}).pipe(
	468	Effect.provide(ServerSettingsService.layerTest({ enableDeviceSupport: true })),
	469	Effect.scoped,
	470	),
	471	);

apps/server/src/device/DeviceService.ts+8-2

...	...	@@ -670,8 +670,14 @@ export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function*
670	670	platform: DevicePlatform,
671	671	) {
672	672	const ready = yield* readiness(hostId);
673		yield* HttpClientRequest.post(`${ready.hub.origin}/api/devices/shutdown`).pipe(
674		HttpClientRequest.bodyJson({ platform, id: deviceId }),
	673	// serve-sim's shutdown closes its in-process capture session before powering off.
	674	// The hub's generic shutdown can leave that session cached across a reboot.
	675	const path =
	676	platform === "ios" ? `${vendorPrefix("ios")}/grid/api/shutdown` : "/api/devices/shutdown";
	677	yield* HttpClientRequest.post(`${ready.hub.origin}${path}`).pipe(
	678	HttpClientRequest.bodyJson(
	679	platform === "ios" ? { udid: deviceId } : { platform, id: deviceId },
	680	),
675	681	Effect.mapError(
676	682	(cause) =>
677	683	new DeviceOperationError({ operation: "shutdown", reason: "invalid_payload", cause }),

apps/web/src/components/device/DeviceLoadingView.tsx+3

...	...	@@ -1,3 +1,4 @@
	1	import type { ReactNode } from "react";
1	2	import { Smartphone } from "lucide-react";
2	3
3	4	import { Spinner } from "~/components/ui/spinner";
...	...	@@ -8,6 +9,7 @@ export function DeviceLoadingView(props: {
8	9	readonly stage: "opening" \| "stream";
9	10	readonly message: string;
10	11	readonly error?: boolean;
	12	readonly children?: ReactNode;
11	13	}) {
12	14	return (
13	15	<div
...	...	@@ -28,6 +30,7 @@ export function DeviceLoadingView(props: {
28	30	{!props.error ? <Spinner className="size-3" /> : null}
29	31	<span>{props.message}</span>
30	32	</div>
	33	{props.children}
31	34	{!props.error ? (
32	35	<div
33	36	className="flex w-24 gap-1"

apps/web/src/components/device/DeviceStreamView.test.tsx+66-17

...	...	@@ -8,25 +8,37 @@ vi.mock("~/state/device", () => ({
8	8	refreshDeviceHubAccess: vi.fn(),
9	9	}));
10	10	const access = { httpBase: "http://test", wsBase: "ws://test", query: {}, credentials: true };
11		vi.mock("@t3tools/client-runtime/device/stream", () => ({
12		createDeviceStreamClient: (
13		_target: unknown,
14		_canvas: unknown,
15		events: { onMjpegFallback: (url: string) => void },
16		) => ({
17		start: () => events.onMjpegFallback("http://test/stream.mjpeg"),
18		stop: vi.fn(),
19		}),
20		}));
21	11	import { DeviceStreamView } from "./DeviceStreamView";
	12
	13	class Image extends EventTarget {
	14	src = "";
	15	naturalWidth = 0;
	16	naturalHeight = 0;
	17	removeAttribute(name: string) {
	18	if (name === "src") this.src = "";
	19	}
	20	}
22	21	let renderer: ReactTestRenderer \| undefined;
23	22	afterEach(async () => {
24	23	await act(async () => renderer?.unmount());
	24	renderer = undefined;
	25	vi.useRealTimers();
25	26	vi.unstubAllGlobals();
26	27	});
27	28
28		it("removes MJPEG requests while hidden and reconnects when shown", async () => {
	29	async function setup() {
	30	vi.useFakeTimers();
29	31	vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true);
	32	vi.stubGlobal("fetch", () => Promise.resolve(new Response("prime")));
	33	vi.stubGlobal(
	34	"WebSocket",
	35	class {
	36	static OPEN = 1;
	37	readyState = 1;
	38	send() {}
	39	close() {}
	40	},
	41	);
30	42	vi.stubGlobal(
31	43	"ResizeObserver",
32	44	class {
...	...	@@ -34,6 +46,7 @@ it("removes MJPEG requests while hidden and reconnects when shown", async () =>
34	46	disconnect() {}
35	47	},
36	48	);
	49	const images: Image[] = [];
37	50	const view = (visible: boolean) => (
38	51	<DeviceStreamView
39	52	hostId="local"
...	...	@@ -45,15 +58,51 @@ it("removes MJPEG requests while hidden and reconnects when shown", async () =>
45	58	);
46	59	await act(async () => {
47	60	renderer = create(view(true), {
48		createNodeMock: () => ({
49		style: { setProperty() {} },
50		getBoundingClientRect: () => ({ width: 400, height: 800 }),
51		}),
	61	createNodeMock: (node) => {
	62	if (node.type === "img") {
	63	const image = new Image();
	64	images.push(image);
	65	return image;
	66	}
	67	return {
	68	style: { setProperty() {} },
	69	getBoundingClientRect: () => ({ width: 400, height: 800 }),
	70	};
	71	},
52	72	});
53	73	});
54		expect(renderer!.root.findAllByType("img")).toHaveLength(1);
	74	return { images, view };
	75	}
	76
	77	it("removes MJPEG requests while hidden and reconnects when shown", async () => {
	78	const { images, view } = await setup();
	79	expect(images[0]!.src).toContain("stream.mjpeg");
55	80	await act(async () => renderer!.update(view(false)));
56	81	expect(renderer!.root.findAllByType("img")).toHaveLength(0);
	82	expect(images[0]!.src).toBe("");
	83	expect(vi.getTimerCount()).toBe(0);
57	84	await act(async () => renderer!.update(view(true)));
58		expect(renderer!.root.findAllByType("img")).toHaveLength(1);
	85	expect(images.at(-1)!.src).toContain("stream.mjpeg");
	86	});
	87
	88	it("offers Reconnect after the shared timeout and receives a frame after retry with unchanged access", async () => {
	89	const { images } = await setup();
	90	await act(async () => {
	91	await vi.advanceTimersByTimeAsync(15_000);
	92	});
	93	expect(
	94	renderer!.root.findByProps({ role: "alert" }).findByType("span").children.join(""),
	95	).toContain("No video");
	96	expect(images[0]!.src).toBe("");
	97	await act(async () => renderer!.root.findByType("button").props.onClick());
	98	expect(renderer!.root.findAllByProps({ role: "alert" })).toHaveLength(0);
	99	expect(images[1]!.src).toContain("stream.mjpeg");
	100	await act(async () => {
	101	images[1]!.naturalWidth = 400;
	102	images[1]!.naturalHeight = 800;
	103	images[1]!.dispatchEvent(new Event("load"));
	104	});
	105	expect(renderer!.root.findAllByProps({ role: "status" })).toHaveLength(0);
	106	expect(renderer!.root.findAllByType("button")).toHaveLength(0);
	107	expect(vi.getTimerCount()).toBe(0);
59	108	});

apps/web/src/components/device/DeviceStreamView.tsx+24-4

...	...	@@ -1,7 +1,8 @@
1	1	import type { DevicePlatform, EnvironmentId } from "@t3tools/contracts";
2		import { useEffect, useMemo, useRef, useState } from "react";
	2	import { useCallback, useEffect, useMemo, useRef, useState } from "react";
3	3
4	4	import { cn } from "~/lib/utils";
	5	import { Button } from "~/components/ui/button";
5	6	import { refreshDeviceHubAccess, useDeviceHubAccess } from "~/state/device";
6	7	import { DeviceLoadingView } from "./DeviceLoadingView";
7	8	import { type DeviceAxElement, fetchDeviceAxTree } from "./deviceHubApi";
...	...	@@ -49,6 +50,9 @@ export function DeviceStreamView(props: {
49	50	const [screen, setScreen] = useState<DeviceScreenSize \| null>(null);
50	51	const [mjpegUrl, setMjpegUrl] = useState<string \| null>(null);
51	52	const [mjpegGeneration, setMjpegGeneration] = useState(0);
	53	const attachMjpegImage = useCallback((image: HTMLImageElement \| null) => {
	54	clientRef.current?.setMjpegImage(image);
	55	}, []);
52	56	const [inputState, setInputState] = useState<{ connected: boolean; detail?: string }>({
53	57	connected: false,
54	58	});
...	...	@@ -235,11 +239,13 @@ export function DeviceStreamView(props: {
235	239	role="application"
236	240	aria-label={`${props.platform === "ios" ? "iOS Simulator" : "Android Emulator"} screen`}
237	241	onKeyDown={(event) => {
	242	if (event.target !== event.currentTarget) return;
238	243	if (event.metaKey && !["r", "R"].includes(event.key)) return;
239	244	event.preventDefault();
240	245	clientRef.current?.sendKey(event.nativeEvent, "down");
241	246	}}
242	247	onKeyUp={(event) => {
	248	if (event.target !== event.currentTarget) return;
243	249	clientRef.current?.sendKey(event.nativeEvent, "up");
244	250	}}
245	251	>
...	...	@@ -279,7 +285,7 @@ export function DeviceStreamView(props: {
279	285	{props.visible && access && mjpegUrl ? (
280	286	<img
281	287	key={mjpegGeneration}
282		src={mjpegUrl}
	288	ref={attachMjpegImage}
283	289	alt=""
284	290	draggable={false}
285	291	className="absolute top-0 left-0 object-contain"
...	...	@@ -317,14 +323,28 @@ export function DeviceStreamView(props: {
317	323	</div>
318	324	) : null}
319	325	{status !== "streaming" ? (
320		<div className="pointer-events-none absolute inset-0">
	326	<div className="absolute inset-0">
321	327	<DeviceLoadingView
322	328	name={props.deviceName ?? "Device"}
323	329	description={props.deviceDescription ?? ""}
324	330	stage="stream"
325	331	message={status === "error" ? (detail ?? "Stream failed.") : "Connecting video…"}
326	332	error={status === "error"}
327		/>
	333	>
	334	{status === "error" ? (
	335	<Button
	336	size="sm"
	337	variant="outline"
	338	onClick={() => {
	339	refreshDeviceHubAccess(props.environmentId);
	340	clientRef.current?.stop();
	341	clientRef.current?.start();
	342	}}
	343	>
	344	Reconnect
	345	</Button>
	346	) : null}
	347	</DeviceLoadingView>
328	348	</div>
329	349	) : null}
330	350	</div>

apps/web/src/components/device/deviceStream.test.ts deleted-345

...	...	@@ -1,345 +0,0 @@
1		import { afterEach, describe, expect, it, vi } from "vite-plus/test";
2
3		import {
4		createDeviceStreamClient,
5		AvccDemuxer,
6		avcCodecString,
7		parseSemuPacket,
8		scanAccessUnit,
9		} from "@t3tools/client-runtime/device/stream";
10
11		const envelope = (tag: number, payload: number[]) => {
12		const length = 1 + payload.length;
13		return [
14		(length >>> 24) & 0xff,
15		(length >>> 16) & 0xff,
16		(length >>> 8) & 0xff,
17		length & 0xff,
18		tag,
19		...payload,
20		];
21		};
22
23		describe("AvccDemuxer", () => {
24		it("reassembles envelopes split across reads", () => {
25		const demuxer = new AvccDemuxer();
26		const bytes = new Uint8Array([
27		...envelope(1, [1, 0x64, 0x00, 0x1f]),
28		...envelope(2, [9, 9, 9]),
29		...envelope(0x7f, [0]),
30		...envelope(3, [4]),
31		]);
32		const first = demuxer.push(bytes.subarray(0, 7));
33		const rest = demuxer.push(bytes.subarray(7));
34		const chunks = [...first, ...rest];
35		expect(chunks.map((chunk) => chunk.type)).toEqual(["description", "keyframe", "delta"]);
36		expect(Array.from(chunks[0]!.payload)).toEqual([1, 0x64, 0x00, 0x1f]);
37		expect(Array.from(chunks[1]!.payload)).toEqual([9, 9, 9]);
38		});
39
40		it("derives the WebCodecs codec string from the avcC record", () => {
41		expect(avcCodecString(new Uint8Array([1, 0x64, 0x00, 0x1f]))).toBe("avc1.64001f");
42		expect(avcCodecString(new Uint8Array([1]))).toBe("avc1.42E01E");
43		});
44		});
45
46		describe("native device stream transport", () => {
47		afterEach(() => {
48		vi.useRealTimers();
49		vi.unstubAllGlobals();
50		});
51
52		function setup(platform: "ios" \| "android") {
53		vi.useFakeTimers();
54		vi.stubGlobal("VideoDecoder", vi.fn());
55		vi.stubGlobal("EncodedVideoChunk", vi.fn());
56		let resolveOpened!: (socket: FakeSocket) => void;
57		const waitForSocket = () =>
58		new Promise<FakeSocket>((resolve) => {
59		resolveOpened = resolve;
60		});
61		const opened = waitForSocket();
62		class FakeSocket {
63		static OPEN = 1;
64		readyState = 1;
65		binaryType = "";
66		onopen: (() => void) \| null = null;
67		onclose: ((event: { code: number; reason: string }) => void) \| null = null;
68		send = vi.fn();
69		close = vi.fn();
70		readonly url: string;
71		constructor(url: string) {
72		this.url = url;
73		resolveOpened(this);
74		}
75		}
76		vi.stubGlobal("WebSocket", FakeSocket);
77		const fetch = vi.fn(() => Promise.resolve(new Response("frame")));
78		vi.stubGlobal("fetch", fetch);
79		const events = {
80		onStatus: vi.fn(),
81		onScreen: vi.fn(),
82		onUnauthorized: vi.fn(),
83		onMjpegFallback: vi.fn(),
84		onInputConnected: vi.fn(),
85		};
86		const client = createDeviceStreamClient(
87		{
88		platform,
89		deviceId: "test device",
90		preferMjpeg: platform === "ios",
91		access: {
92		httpBase: "https://environment.test/api/device-hub",
93		wsBase: "wss://environment.test/api/device-hub",
94		credentials: false,
95		query: { wsTicket: "stream-ticket", hostId: "ssh-host" },
96		},
97		},
98		{ getContext: () => null } as unknown as HTMLCanvasElement,
99		events,
100		);
101		return { client, opened, waitForSocket, events, fetch };
102		}
103
104		it("uses authenticated iOS MJPEG and forwards controls without a cross-origin video fetch", async () => {
105		const { client, opened, events, fetch } = setup("ios");
106		client.start();
107		const socket = await opened;
108		const imageUrl = new URL(events.onMjpegFallback.mock.calls[0]![0] as string);
109		expect(imageUrl.pathname).toContain("test%20device/stream.mjpeg");
110		expect(imageUrl.searchParams.get("wsTicket")).toBe("stream-ticket");
111		expect(imageUrl.searchParams.get("hostId")).toBe("ssh-host");
112		expect(fetch).toHaveBeenCalledTimes(1);
113		socket.onopen?.();
114		client.pressButton("home");
115		client.sendTouch("begin", 0.25, 0.75);
116		const messages = socket.send.mock.calls.slice(1).map(([bytes]) => {
117		const packet = bytes as Uint8Array;
118		return {
119		tag: packet[0],
120		body: JSON.parse(new TextDecoder().decode(packet.subarray(1))) as unknown,
121		};
122		});
123		expect(messages).toEqual([
124		{ tag: 0x04, body: { button: "home" } },
125		{ tag: 0x03, body: { type: "begin", x: 0.25, y: 0.75 } },
126		]);
127		client.stop();
128		expect(socket.close).toHaveBeenCalledTimes(1);
129		expect(vi.getTimerCount()).toBe(0);
130		});
131
132		it("forwards Android gestures and device controls through the ticketed host socket", async () => {
133		const { client, opened } = setup("android");
134		client.start();
135		const socket = await opened;
136		const url = new URL(socket.url);
137		expect(url.searchParams.get("device")).toBe("test device");
138		expect(url.searchParams.get("wsTicket")).toBe("stream-ticket");
139		expect(url.searchParams.get("hostId")).toBe("ssh-host");
140		client.sendTouch("begin", 0.2, 0.8);
141		client.sendTouch("move", 0.2, 0.4);
142		client.sendTouch("end", 0.2, 0.1);
143		client.pressButton("home");
144		client.pressButton("appSwitcher");
145		expect(
146		socket.send.mock.calls.map(([message]) => JSON.parse(message as string) as unknown),
147		).toEqual([
148		{ type: "touch", action: "down", x: 0.2, y: 0.8 },
149		{ type: "touch", action: "move", x: 0.2, y: 0.4 },
150		{ type: "touch", action: "up", x: 0.2, y: 0.1 },
151		{ type: "home" },
152		{ type: "recents" },
153		]);
154		client.stop();
155		expect(socket.close).toHaveBeenCalledTimes(1);
156		});
157
158		it("renews an expired Android ticket when the HTTP upgrade is rejected instead of retrying it forever", async () => {
159		const { client, opened, events } = setup("android");
160		client.start();
161		const socket = await opened;
162		socket.onclose?.({ code: 1006, reason: "" });
163		expect(events.onUnauthorized).toHaveBeenCalledTimes(1);
164		expect(vi.getTimerCount()).toBe(0);
165		client.stop();
166		});
167
168		it.each(["ios", "android"] as const)(
169		"keeps a restarted %s stream connected when the discarded socket closes late",
170		async (platform) => {
171		const { client, opened, waitForSocket, events } = setup(platform);
172		client.start();
173		const discarded = await opened;
174		client.stop();
175		const reopened = waitForSocket();
176		client.start();
177		const replacement = await reopened;
178		replacement.onopen?.();
179		replacement.send.mockClear();
180		events.onInputConnected.mockClear();
181
182		discarded.onclose?.({ code: 1006, reason: "" });
183
184		expect(events.onUnauthorized).not.toHaveBeenCalled();
185		expect(events.onInputConnected).not.toHaveBeenCalled();
186		expect(replacement.close).not.toHaveBeenCalled();
187		client.pressButton("home");
188		expect(replacement.send).toHaveBeenCalledTimes(1);
189		client.stop();
190		expect(replacement.close).toHaveBeenCalledTimes(1);
191		expect(vi.getTimerCount()).toBe(0);
192		},
193		);
194
195		it.each(["ios", "android"] as const)(
196		"does not renew credentials or reconnect a stopped %s stream",
197		async (platform) => {
198		const { client, opened, events } = setup(platform);
199		client.start();
200		const socket = await opened;
201		events.onInputConnected.mockClear();
202		client.stop();
203		socket.onclose?.({ code: 1006, reason: "" });
204		expect(events.onUnauthorized).not.toHaveBeenCalled();
205		expect(events.onInputConnected).not.toHaveBeenCalled();
206		expect(vi.getTimerCount()).toBe(0);
207		},
208		);
209		});
210
211		describe("serve-emu frames", () => {
212		it("strips the SEMU header and reads the keyframe flag and timestamp", () => {
213		const buffer = new ArrayBuffer(16 + 3);
214		const view = new DataView(buffer);
215		view.setUint32(0, 0x53454d55);
216		view.setUint8(4, 1);
217		view.setUint8(5, 1);
218		view.setBigUint64(8, 123456n);
219		new Uint8Array(buffer).set([7, 8, 9], 16);
220		const packet = parseSemuPacket(buffer);
221		expect(packet.isKey).toBe(true);
222		expect(packet.timestamp).toBe(123456);
223		expect(Array.from(packet.data)).toEqual([7, 8, 9]);
224		});
225
226		it("treats a frame without the header as raw data", () => {
227		const packet = parseSemuPacket(new Uint8Array([0, 0, 1, 0x65]).buffer);
228		expect(packet.isKey).toBeNull();
229		expect(packet.data.length).toBe(4);
230		});
231
232		it("finds the SPS and IDR NAL units in an Annex-B access unit", () => {
233		const unit = new Uint8Array([0, 0, 0, 1, 0x67, 0x64, 0x00, 0x1f, 0, 0, 1, 0x65, 0xaa]);
234		const scanned = scanAccessUnit(unit);
235		expect(scanned.isKey).toBe(true);
236		expect(scanned.sps && avcCodecString(scanned.sps)).toBe("avc1.64001f");
237		expect(scanAccessUnit(new Uint8Array([0, 0, 1, 0x41, 0x00])).isKey).toBe(false);
238		});
239		});
240
241		describe("iOS input startup", () => {
242		afterEach(() => {
243		vi.useRealTimers();
244		vi.unstubAllGlobals();
245		});
246
247		const setup = () => {
248		vi.useFakeTimers();
249		const sockets: FakeSocket[] = [];
250		class FakeSocket {
251		static OPEN = 1;
252		readyState = 1;
253		binaryType = "";
254		onopen: (() => void) \| null = null;
255		onmessage: ((event: { data: ArrayBuffer }) => void) \| null = null;
256		onclose: ((event: { code: number; reason: string }) => void) \| null = null;
257		send = vi.fn();
258		close = vi.fn();
259		constructor() {
260		sockets.push(this);
261		}
262		}
263		vi.stubGlobal("WebSocket", FakeSocket);
264		const signals: AbortSignal[] = [];
265		vi.stubGlobal(
266		"fetch",
267		vi.fn((_url, init: RequestInit) => {
268		const signal = init.signal!;
269		signals.push(signal);
270		return Promise.resolve(
271		new Response(
272		new ReadableStream({
273		start(controller) {
274		signal.addEventListener("abort", () => controller.error(new Error("aborted")));
275		},
276		}),
277		),
278		);
279		}),
280		);
281		const client = createDeviceStreamClient(
282		{
283		platform: "ios",
284		deviceId: "test-device",
285		access: {
286		httpBase: "http://test/api/device-hub",
287		wsBase: "ws://test/api/device-hub",
288		credentials: true,
289		query: {},
290		},
291		},
292		{ getContext: () => null } as unknown as HTMLCanvasElement,
293		{
294		onStatus: vi.fn(),
295		onScreen: vi.fn(),
296		onUnauthorized: vi.fn(),
297		onMjpegFallback: vi.fn(),
298		onInputConnected: vi.fn(),
299		},
300		);
301		return { client, sockets, signals };
302		};
303
304		it("connects input when the MJPEG prime never produces a frame", async () => {
305		const { client, sockets, signals } = setup();
306		client.start();
307		await vi.advanceTimersByTimeAsync(2_000);
308		expect(signals[0]?.aborted).toBe(true);
309		expect(sockets).toHaveLength(1);
310		client.stop();
311		});
312
313		it("aborts priming immediately when hidden without opening a socket later", async () => {
314		const { client, sockets, signals } = setup();
315		client.start();
316		await vi.advanceTimersByTimeAsync(0);
317		client.stop();
318		expect(signals[0]?.aborted).toBe(true);
319		await vi.advanceTimersByTimeAsync(2_000);
320		expect(sockets).toHaveLength(0);
321		});
322
323		it.each([
324		["landscape_left", 0.7, 1 - 0.2],
325		["landscape_right", 1 - 0.7, 0.2],
326		])("maps %s touches back to the raw iOS framebuffer", async (orientation, x, y) => {
327		const { client, sockets } = setup();
328		client.start();
329		await vi.advanceTimersByTimeAsync(2_000);
330		const socket = sockets[0]!;
331		const json = new TextEncoder().encode(JSON.stringify({ width: 400, height: 800, orientation }));
332		const packet = new Uint8Array(1 + json.length);
333		packet[0] = 0x82;
334		packet.set(json, 1);
335		socket.onmessage?.({ data: packet.buffer });
336		client.sendTouch("begin", 0.2, 0.7);
337		const sent = socket.send.mock.calls[0]![0] as Uint8Array;
338		expect(JSON.parse(new TextDecoder().decode(sent.subarray(1)))).toEqual({
339		type: "begin",
340		x,
341		y,
342		});
343		client.stop();
344		});
345		});

packages/client-runtime/src/device/stream.test.ts created+735

...	...	@@ -0,0 +1,735 @@
	1	import { afterEach, describe, expect, it, vi } from "vite-plus/test";
	2
	3	import {
	4	createDeviceStreamClient,
	5	AvccDemuxer,
	6	avcCodecString,
	7	parseSemuPacket,
	8	scanAccessUnit,
	9	} from "./stream.ts";
	10
	11	const envelope = (tag: number, payload: number[]) => {
	12	const length = 1 + payload.length;
	13	return [
	14	(length >>> 24) & 0xff,
	15	(length >>> 16) & 0xff,
	16	(length >>> 8) & 0xff,
	17	length & 0xff,
	18	tag,
	19	...payload,
	20	];
	21	};
	22
	23	describe("AvccDemuxer", () => {
	24	it("reassembles envelopes split across reads", () => {
	25	const demuxer = new AvccDemuxer();
	26	const bytes = new Uint8Array([
	27	...envelope(1, [1, 0x64, 0x00, 0x1f]),
	28	...envelope(2, [9, 9, 9]),
	29	...envelope(0x7f, [0]),
	30	...envelope(3, [4]),
	31	]);
	32	const first = demuxer.push(bytes.subarray(0, 7));
	33	const rest = demuxer.push(bytes.subarray(7));
	34	const chunks = [...first, ...rest];
	35	expect(chunks.map((chunk) => chunk.type)).toEqual(["description", "keyframe", "delta"]);
	36	expect(Array.from(chunks[0]!.payload)).toEqual([1, 0x64, 0x00, 0x1f]);
	37	expect(Array.from(chunks[1]!.payload)).toEqual([9, 9, 9]);
	38	});
	39
	40	it("derives the WebCodecs codec string from the avcC record", () => {
	41	expect(avcCodecString(new Uint8Array([1, 0x64, 0x00, 0x1f]))).toBe("avc1.64001f");
	42	expect(avcCodecString(new Uint8Array([1]))).toBe("avc1.42E01E");
	43	});
	44	});
	45
	46	describe("native device stream transport", () => {
	47	afterEach(() => {
	48	vi.useRealTimers();
	49	vi.unstubAllGlobals();
	50	});
	51
	52	function setup(platform: "ios" \| "android") {
	53	vi.useFakeTimers();
	54	vi.stubGlobal("VideoDecoder", vi.fn());
	55	vi.stubGlobal("EncodedVideoChunk", vi.fn());
	56	let resolveOpened!: (socket: FakeSocket) => void;
	57	const waitForSocket = () =>
	58	new Promise<FakeSocket>((resolve) => {
	59	resolveOpened = resolve;
	60	});
	61	const opened = waitForSocket();
	62	class FakeSocket {
	63	static OPEN = 1;
	64	readyState = 1;
	65	binaryType = "";
	66	onopen: (() => void) \| null = null;
	67	onclose: ((event: { code: number; reason: string }) => void) \| null = null;
	68	send = vi.fn();
	69	close = vi.fn();
	70	readonly url: string;
	71	constructor(url: string) {
	72	this.url = url;
	73	resolveOpened(this);
	74	}
	75	}
	76	vi.stubGlobal("WebSocket", FakeSocket);
	77	const fetch = vi.fn(() => Promise.resolve(new Response("frame")));
	78	vi.stubGlobal("fetch", fetch);
	79	const events = {
	80	onStatus: vi.fn(),
	81	onScreen: vi.fn(),
	82	onUnauthorized: vi.fn(),
	83	onMjpegFallback: vi.fn(),
	84	onInputConnected: vi.fn(),
	85	};
	86	const client = createDeviceStreamClient(
	87	{
	88	platform,
	89	deviceId: "test device",
	90	preferMjpeg: platform === "ios",
	91	access: {
	92	httpBase: "https://environment.test/api/device-hub",
	93	wsBase: "wss://environment.test/api/device-hub",
	94	credentials: false,
	95	query: { wsTicket: "stream-ticket", hostId: "ssh-host" },
	96	},
	97	},
	98	{ getContext: () => null } as unknown as HTMLCanvasElement,
	99	events,
	100	);
	101	return { client, opened, waitForSocket, events, fetch };
	102	}
	103
	104	it("uses authenticated iOS MJPEG and forwards controls without a cross-origin video fetch", async () => {
	105	const { client, opened, events, fetch } = setup("ios");
	106	client.start();
	107	const socket = await opened;
	108	const imageUrl = new URL(events.onMjpegFallback.mock.calls[0]![0] as string);
	109	expect(imageUrl.pathname).toContain("test%20device/stream.mjpeg");
	110	expect(imageUrl.searchParams.get("wsTicket")).toBe("stream-ticket");
	111	expect(imageUrl.searchParams.get("hostId")).toBe("ssh-host");
	112	expect(fetch).toHaveBeenCalledTimes(1);
	113	socket.onopen?.();
	114	client.pressButton("home");
	115	client.sendTouch("begin", 0.25, 0.75);
	116	const messages = socket.send.mock.calls.slice(1).map(([bytes]) => {
	117	const packet = bytes as Uint8Array;
	118	return {
	119	tag: packet[0],
	120	body: JSON.parse(new TextDecoder().decode(packet.subarray(1))) as unknown,
	121	};
	122	});
	123	expect(messages).toEqual([
	124	{ tag: 0x04, body: { button: "home" } },
	125	{ tag: 0x03, body: { type: "begin", x: 0.25, y: 0.75 } },
	126	]);
	127	client.stop();
	128	expect(socket.close).toHaveBeenCalledTimes(1);
	129	expect(vi.getTimerCount()).toBe(0);
	130	});
	131
	132	it("forwards Android gestures and device controls through the ticketed host socket", async () => {
	133	const { client, opened } = setup("android");
	134	client.start();
	135	const socket = await opened;
	136	const url = new URL(socket.url);
	137	expect(url.searchParams.get("device")).toBe("test device");
	138	expect(url.searchParams.get("wsTicket")).toBe("stream-ticket");
	139	expect(url.searchParams.get("hostId")).toBe("ssh-host");
	140	client.sendTouch("begin", 0.2, 0.8);
	141	client.sendTouch("move", 0.2, 0.4);
	142	client.sendTouch("end", 0.2, 0.1);
	143	client.pressButton("home");
	144	client.pressButton("appSwitcher");
	145	expect(
	146	socket.send.mock.calls.map(([message]) => JSON.parse(message as string) as unknown),
	147	).toEqual([
	148	{ type: "touch", action: "down", x: 0.2, y: 0.8 },
	149	{ type: "touch", action: "move", x: 0.2, y: 0.4 },
	150	{ type: "touch", action: "up", x: 0.2, y: 0.1 },
	151	{ type: "home" },
	152	{ type: "recents" },
	153	]);
	154	client.stop();
	155	expect(socket.close).toHaveBeenCalledTimes(1);
	156	});
	157
	158	it("renews an expired Android ticket when the HTTP upgrade is rejected instead of retrying it forever", async () => {
	159	const { client, opened, events } = setup("android");
	160	client.start();
	161	const socket = await opened;
	162	socket.onclose?.({ code: 1006, reason: "" });
	163	expect(events.onUnauthorized).toHaveBeenCalledTimes(1);
	164	expect(vi.getTimerCount()).toBe(0);
	165	client.stop();
	166	});
	167
	168	it.each(["ios", "android"] as const)(
	169	"keeps a restarted %s stream connected when the discarded socket closes late",
	170	async (platform) => {
	171	const { client, opened, waitForSocket, events } = setup(platform);
	172	client.start();
	173	const discarded = await opened;
	174	client.stop();
	175	const reopened = waitForSocket();
	176	client.start();
	177	const replacement = await reopened;
	178	replacement.onopen?.();
	179	replacement.send.mockClear();
	180	events.onInputConnected.mockClear();
	181
	182	discarded.onclose?.({ code: 1006, reason: "" });
	183
	184	expect(events.onUnauthorized).not.toHaveBeenCalled();
	185	expect(events.onInputConnected).not.toHaveBeenCalled();
	186	expect(replacement.close).not.toHaveBeenCalled();
	187	client.pressButton("home");
	188	expect(replacement.send).toHaveBeenCalledTimes(1);
	189	client.stop();
	190	expect(replacement.close).toHaveBeenCalledTimes(1);
	191	expect(vi.getTimerCount()).toBe(0);
	192	},
	193	);
	194
	195	it.each(["ios", "android"] as const)(
	196	"does not renew credentials or reconnect a stopped %s stream",
	197	async (platform) => {
	198	const { client, opened, events } = setup(platform);
	199	client.start();
	200	const socket = await opened;
	201	events.onInputConnected.mockClear();
	202	client.stop();
	203	socket.onclose?.({ code: 1006, reason: "" });
	204	expect(events.onUnauthorized).not.toHaveBeenCalled();
	205	expect(events.onInputConnected).not.toHaveBeenCalled();
	206	expect(vi.getTimerCount()).toBe(0);
	207	},
	208	);
	209	});
	210
	211	describe("serve-emu frames", () => {
	212	it("strips the SEMU header and reads the keyframe flag and timestamp", () => {
	213	const buffer = new ArrayBuffer(16 + 3);
	214	const view = new DataView(buffer);
	215	view.setUint32(0, 0x53454d55);
	216	view.setUint8(4, 1);
	217	view.setUint8(5, 1);
	218	view.setBigUint64(8, 123456n);
	219	new Uint8Array(buffer).set([7, 8, 9], 16);
	220	const packet = parseSemuPacket(buffer);
	221	expect(packet.isKey).toBe(true);
	222	expect(packet.timestamp).toBe(123456);
	223	expect(Array.from(packet.data)).toEqual([7, 8, 9]);
	224	});
	225
	226	it("treats a frame without the header as raw data", () => {
	227	const packet = parseSemuPacket(new Uint8Array([0, 0, 1, 0x65]).buffer);
	228	expect(packet.isKey).toBeNull();
	229	expect(packet.data.length).toBe(4);
	230	});
	231
	232	it("finds the SPS and IDR NAL units in an Annex-B access unit", () => {
	233	const unit = new Uint8Array([0, 0, 0, 1, 0x67, 0x64, 0x00, 0x1f, 0, 0, 1, 0x65, 0xaa]);
	234	const scanned = scanAccessUnit(unit);
	235	expect(scanned.isKey).toBe(true);
	236	expect(scanned.sps && avcCodecString(scanned.sps)).toBe("avc1.64001f");
	237	expect(scanAccessUnit(new Uint8Array([0, 0, 1, 0x41, 0x00])).isKey).toBe(false);
	238	});
	239	});
	240
	241	describe("iOS input startup", () => {
	242	afterEach(() => {
	243	vi.useRealTimers();
	244	vi.unstubAllGlobals();
	245	});
	246
	247	const setup = () => {
	248	vi.useFakeTimers();
	249	const sockets: FakeSocket[] = [];
	250	class FakeSocket {
	251	static OPEN = 1;
	252	readyState = 1;
	253	binaryType = "";
	254	onopen: (() => void) \| null = null;
	255	onmessage: ((event: { data: ArrayBuffer }) => void) \| null = null;
	256	onclose: ((event: { code: number; reason: string }) => void) \| null = null;
	257	send = vi.fn();
	258	close = vi.fn();
	259	constructor() {
	260	sockets.push(this);
	261	}
	262	}
	263	vi.stubGlobal("WebSocket", FakeSocket);
	264	const signals: AbortSignal[] = [];
	265	vi.stubGlobal(
	266	"fetch",
	267	vi.fn((_url, init: RequestInit) => {
	268	const signal = init.signal!;
	269	signals.push(signal);
	270	return Promise.resolve(
	271	new Response(
	272	new ReadableStream({
	273	start(controller) {
	274	signal.addEventListener("abort", () => controller.error(new Error("aborted")));
	275	},
	276	}),
	277	),
	278	);
	279	}),
	280	);
	281	const client = createDeviceStreamClient(
	282	{
	283	platform: "ios",
	284	deviceId: "test-device",
	285	access: {
	286	httpBase: "http://test/api/device-hub",
	287	wsBase: "ws://test/api/device-hub",
	288	credentials: true,
	289	query: {},
	290	},
	291	},
	292	{ getContext: () => null } as unknown as HTMLCanvasElement,
	293	{
	294	onStatus: vi.fn(),
	295	onScreen: vi.fn(),
	296	onUnauthorized: vi.fn(),
	297	onMjpegFallback: vi.fn(),
	298	onInputConnected: vi.fn(),
	299	},
	300	);
	301	return { client, sockets, signals };
	302	};
	303
	304	it("connects input when the MJPEG prime never produces a frame", async () => {
	305	const { client, sockets, signals } = setup();
	306	client.start();
	307	await vi.advanceTimersByTimeAsync(2_000);
	308	expect(signals[0]?.aborted).toBe(true);
	309	expect(sockets).toHaveLength(1);
	310	client.stop();
	311	});
	312
	313	it("aborts priming immediately when hidden without opening a socket later", async () => {
	314	const { client, sockets, signals } = setup();
	315	client.start();
	316	await vi.advanceTimersByTimeAsync(0);
	317	client.stop();
	318	expect(signals[0]?.aborted).toBe(true);
	319	await vi.advanceTimersByTimeAsync(2_000);
	320	expect(sockets).toHaveLength(0);
	321	});
	322
	323	it.each([
	324	["landscape_left", 0.7, 1 - 0.2],
	325	["landscape_right", 1 - 0.7, 0.2],
	326	])("maps %s touches back to the raw iOS framebuffer", async (orientation, x, y) => {
	327	const { client, sockets } = setup();
	328	client.start();
	329	await vi.advanceTimersByTimeAsync(2_000);
	330	const socket = sockets[0]!;
	331	const json = new TextEncoder().encode(JSON.stringify({ width: 400, height: 800, orientation }));
	332	const packet = new Uint8Array(1 + json.length);
	333	packet[0] = 0x82;
	334	packet.set(json, 1);
	335	socket.onmessage?.({ data: packet.buffer });
	336	client.sendTouch("begin", 0.2, 0.7);
	337	const sent = socket.send.mock.calls[0]![0] as Uint8Array;
	338	expect(JSON.parse(new TextDecoder().decode(sent.subarray(1)))).toEqual({
	339	type: "begin",
	340	x,
	341	y,
	342	});
	343	client.stop();
	344	});
	345	});
	346
	347	class MjpegImage extends EventTarget {
	348	naturalWidth = 0;
	349	naturalHeight = 0;
	350	src = "";
	351	removeAttribute(name: string) {
	352	if (name === "src") {
	353	this.src = "";
	354	this.naturalWidth = 0;
	355	this.naturalHeight = 0;
	356	}
	357	}
	358	}
	359
	360	function recoveryFixture(platform: "ios" \| "android" = "ios", preferMjpeg = true) {
	361	vi.useFakeTimers();
	362	const sockets: Socket[] = [];
	363	class Socket {
	364	static OPEN = 1;
	365	readyState = 1;
	366	binaryType = "";
	367	onopen: (() => void) \| null = null;
	368	onmessage: ((event: { data: ArrayBuffer \| string }) => void) \| null = null;
	369	onclose: ((event: { code: number; reason: string }) => void) \| null = null;
	370	send = vi.fn();
	371	close = vi.fn();
	372	constructor() {
	373	sockets.push(this);
	374	}
	375	}
	376	vi.stubGlobal("WebSocket", Socket);
	377	const decoders: Decoder[] = [];
	378	class Decoder {
	379	static isConfigSupported = vi.fn(async () => ({ supported: true }));
	380	state = "unconfigured";
	381	decodeQueueSize = 0;
	382	configure() {
	383	this.state = "configured";
	384	}
	385	close() {
	386	this.state = "closed";
	387	}
	388	decode() {}
	389	readonly callbacks: { output: (frame: VideoFrame) => void; error: () => void };
	390	constructor(callbacks: { output: (frame: VideoFrame) => void; error: () => void }) {
	391	this.callbacks = callbacks;
	392	decoders.push(this);
	393	}
	394	}
	395	vi.stubGlobal("VideoDecoder", Decoder);
	396	vi.stubGlobal("EncodedVideoChunk", vi.fn());
	397	const videoBodies: ReadableStreamDefaultController<Uint8Array>[] = [];
	398	const signals: AbortSignal[] = [];
	399	const fetch = vi.fn((url: string, init: RequestInit) => {
	400	const signal = init.signal!;
	401	signals.push(signal);
	402	if (!url.includes("stream.avcc")) return Promise.resolve(new Response("prime"));
	403	return Promise.resolve(
	404	new Response(
	405	new ReadableStream<Uint8Array>({
	406	start(body) {
	407	videoBodies.push(body);
	408	signal.addEventListener("abort", () => body.error(new Error("aborted")), {
	409	once: true,
	410	});
	411	},
	412	}),
	413	),
	414	);
	415	});
	416	vi.stubGlobal("fetch", fetch);
	417	const events = {
	418	onStatus: vi.fn(),
	419	onScreen: vi.fn(),
	420	onUnauthorized: vi.fn(),
	421	onMjpegFallback: vi.fn(),
	422	onInputConnected: vi.fn(),
	423	};
	424	const drawImage = vi.fn();
	425	const image = new MjpegImage();
	426	const client = createDeviceStreamClient(
	427	{
	428	platform,
	429	preferMjpeg,
	430	deviceId: "test-device",
	431	access: {
	432	httpBase: "https://test/api/device-hub",
	433	wsBase: "wss://test/api/device-hub",
	434	credentials: true,
	435	query: {},
	436	},
	437	},
	438	{ getContext: () => ({ drawImage }) } as unknown as HTMLCanvasElement,
	439	events,
	440	);
	441	client.setMjpegImage(image as unknown as HTMLImageElement);
	442	const imageReady = () => {
	443	image.naturalWidth = 400;
	444	image.naturalHeight = 800;
	445	image.dispatchEvent(new Event("load"));
	446	};
	447	const decodedFrame = (index = decoders.length - 1) => {
	448	const close = vi.fn();
	449	decoders[index]!.callbacks.output({
	450	displayWidth: 400,
	451	displayHeight: 800,
	452	close,
	453	} as unknown as VideoFrame);
	454	return close;
	455	};
	456	const sps = new Uint8Array([0, 0, 0, 1, 0x67, 0x64, 0, 0x1f, 0, 0, 1, 0x65]).buffer;
	457	return {
	458	client,
	459	events,
	460	image,
	461	imageReady,
	462	sockets,
	463	signals,
	464	fetch,
	465	videoBodies,
	466	decoders,
	467	Decoder,
	468	drawImage,
	469	decodedFrame,
	470	sps,
	471	};
	472	}
	473
	474	describe("shared device stream readiness and recovery", () => {
	475	afterEach(() => {
	476	vi.useRealTimers();
	477	vi.unstubAllGlobals();
	478	});
	479
	480	it("waits for actual MJPEG dimensions without requiring a multipart load event", async () => {
	481	const { client, events, image } = recoveryFixture();
	482	client.start();
	483	await vi.advanceTimersByTimeAsync(0);
	484	expect(image.src).toContain("stream.mjpeg");
	485	expect(events.onStatus).not.toHaveBeenCalledWith("streaming", undefined);
	486	image.naturalWidth = 400;
	487	image.naturalHeight = 800;
	488	await vi.advanceTimersByTimeAsync(250);
	489	expect(events.onStatus).toHaveBeenLastCalledWith("streaming", undefined);
	490	expect(vi.getTimerCount()).toBe(0);
	491	await vi.advanceTimersByTimeAsync(20_000);
	492	expect(events.onStatus).toHaveBeenLastCalledWith("streaming", undefined);
	493	client.stop();
	494	});
	495
	496	it("observes a web image attached after MJPEG fallback has already begun", async () => {
	497	const { client, image, imageReady, events } = recoveryFixture();
	498	client.setMjpegImage(null);
	499	client.start();
	500	expect(image.src).toBe("");
	501	await vi.advanceTimersByTimeAsync(1000);
	502	client.setMjpegImage(image as unknown as HTMLImageElement);
	503	imageReady();
	504	expect(events.onStatus).toHaveBeenLastCalledWith("streaming", undefined);
	505	client.stop();
	506	expect(image.src).toBe("");
	507	expect(vi.getTimerCount()).toBe(0);
	508	});
	509
	510	it("stops input and video on an image error without treating it as expired credentials", async () => {
	511	const { client, image, imageReady, sockets, events } = recoveryFixture();
	512	client.start();
	513	await vi.advanceTimersByTimeAsync(0);
	514	sockets[0]!.onopen?.();
	515	imageReady();
	516	image.dispatchEvent(new Event("error"));
	517	expect(events.onStatus).toHaveBeenLastCalledWith("error", expect.stringContaining("Reconnect"));
	518	expect(events.onInputConnected).toHaveBeenLastCalledWith(false, expect.any(String));
	519	expect(events.onUnauthorized).not.toHaveBeenCalled();
	520	expect(sockets[0]!.close).toHaveBeenCalledOnce();
	521	expect(image.src).toBe("");
	522	client.pressButton("home");
	523	expect(sockets[0]!.send).toHaveBeenCalledTimes(1);
	524	expect(vi.getTimerCount()).toBe(0);
	525	});
	526
	527	it.each([
	528	["ios", true],
	529	["ios", false],
	530	["android", false],
	531	] as const)(
	532	"bounds missing first frames for %s with preferMjpeg=%s",
	533	async (platform, preferMjpeg) => {
	534	const { client, events, signals, sockets, image } = recoveryFixture(platform, preferMjpeg);
	535	client.start();
	536	await vi.advanceTimersByTimeAsync(14_999);
	537	expect(events.onStatus.mock.calls.some(([status]) => status === "error")).toBe(false);
	538	await vi.advanceTimersByTimeAsync(1);
	539	expect(events.onStatus).toHaveBeenLastCalledWith(
	540	"error",
	541	expect.stringContaining("No video"),
	542	);
	543	expect(events.onInputConnected).toHaveBeenLastCalledWith(false, expect.any(String));
	544	expect(signals.every((signal) => signal.aborted)).toBe(true);
	545	expect(sockets[0]!.close).toHaveBeenCalledOnce();
	546	expect(image.src).toBe("");
	547	expect(vi.getTimerCount()).toBe(0);
	548	},
	549	);
	550
	551	it("does not extend the first-frame deadline each time a socket reconnects", async () => {
	552	const { client, sockets, events } = recoveryFixture("android");
	553	client.start();
	554	sockets[0]!.onopen?.();
	555	sockets[0]!.onclose?.({ code: 1000, reason: "restart" });
	556	await vi.advanceTimersByTimeAsync(14_999);
	557	sockets[1]!.onopen?.();
	558	await vi.advanceTimersByTimeAsync(1);
	559	expect(events.onStatus).toHaveBeenLastCalledWith("error", expect.stringContaining("No video"));
	560	expect(sockets[1]!.close).toHaveBeenCalledOnce();
	561	expect(vi.getTimerCount()).toBe(0);
	562	});
	563
	564	it("gives a manual restart a fresh deadline and ignores queued image events from its old binding", async () => {
	565	const { client, image, events } = recoveryFixture();
	566	const listeners: EventListener[] = [];
	567	const add = image.addEventListener.bind(image);
	568	vi.spyOn(image, "addEventListener").mockImplementation((name, callback, options) => {
	569	if (typeof callback === "function") listeners.push(callback);
	570	add(name, callback, options);
	571	});
	572	client.start();
	573	await vi.advanceTimersByTimeAsync(10_000);
	574	client.stop();
	575	client.start();
	576	events.onStatus.mockClear();
	577	for (const listener of listeners.slice(0, 2)) listener(new Event("error"));
	578	await vi.advanceTimersByTimeAsync(10_000);
	579	expect(events.onStatus).not.toHaveBeenCalled();
	580	const replacement = new MjpegImage();
	581	client.setMjpegImage(replacement as unknown as HTMLImageElement);
	582	replacement.naturalWidth = 400;
	583	replacement.naturalHeight = 800;
	584	replacement.dispatchEvent(new Event("load"));
	585	expect(events.onStatus).toHaveBeenLastCalledWith("streaming", undefined);
	586	client.stop();
	587	expect(vi.getTimerCount()).toBe(0);
	588	});
	589
	590	it("falls back from unsupported iOS H.264 without publishing a terminal error", async () => {
	591	const { client, videoBodies, Decoder, events, imageReady } = recoveryFixture("ios", false);
	592	Decoder.isConfigSupported.mockResolvedValue({ supported: false });
	593	client.start();
	594	await vi.advanceTimersByTimeAsync(0);
	595	videoBodies[0]!.enqueue(new Uint8Array(envelope(1, [1, 0x64, 0, 0x1f])));
	596	await vi.advanceTimersByTimeAsync(0);
	597	expect(events.onMjpegFallback).toHaveBeenCalledOnce();
	598	expect(events.onStatus.mock.calls.some(([status]) => status === "error")).toBe(false);
	599	imageReady();
	600	expect(events.onStatus).toHaveBeenLastCalledWith("streaming", undefined);
	601	client.stop();
	602	});
	603
	604	it("publishes a terminal unsupported-profile error for Android and closes its socket", async () => {
	605	const { client, sockets, sps, Decoder, events } = recoveryFixture("android");
	606	Decoder.isConfigSupported.mockResolvedValue({ supported: false });
	607	client.start();
	608	sockets[0]!.onmessage?.({ data: sps });
	609	await vi.advanceTimersByTimeAsync(0);
	610	expect(events.onStatus).toHaveBeenLastCalledWith(
	611	"error",
	612	expect.stringContaining("cannot decode"),
	613	);
	614	expect(sockets[0]!.close).toHaveBeenCalledOnce();
	615	expect(events.onMjpegFallback).not.toHaveBeenCalled();
	616	expect(vi.getTimerCount()).toBe(0);
	617	});
	618
	619	it("waits for decoded Android output and bounds a rotation that never produces another frame", async () => {
	620	const { client, sockets, sps, events, decodedFrame, drawImage } = recoveryFixture("android");
	621	client.start();
	622	sockets[0]!.onmessage?.({ data: sps });
	623	await vi.advanceTimersByTimeAsync(0);
	624	expect(events.onStatus.mock.calls.some(([status]) => status === "streaming")).toBe(false);
	625	expect(decodedFrame()).toHaveBeenCalledOnce();
	626	expect(drawImage).toHaveBeenCalledOnce();
	627	expect(events.onStatus).toHaveBeenLastCalledWith("streaming", undefined);
	628	sockets[0]!.onmessage?.({ data: JSON.stringify({ type: "video-session" }) });
	629	await vi.advanceTimersByTimeAsync(15_000);
	630	expect(events.onStatus).toHaveBeenLastCalledWith("error", expect.stringContaining("No video"));
	631	expect(vi.getTimerCount()).toBe(0);
	632	});
	633
	634	it("discards decoder support results and output from a previous Android socket", async () => {
	635	const { client, sockets, sps, events, Decoder, decoders, decodedFrame, drawImage } =
	636	recoveryFixture("android");
	637	let resolveSupport!: (support: { supported: boolean }) => void;
	638	Decoder.isConfigSupported.mockImplementationOnce(
	639	() =>
	640	new Promise((resolve) => {
	641	resolveSupport = resolve;
	642	}),
	643	);
	644	client.start();
	645	sockets[0]!.onmessage?.({ data: sps });
	646	client.stop();
	647	client.start();
	648	resolveSupport({ supported: true });
	649	await vi.advanceTimersByTimeAsync(0);
	650	expect(decoders).toHaveLength(0);
	651	sockets[1]!.onmessage?.({ data: sps });
	652	await vi.advanceTimersByTimeAsync(0);
	653	client.stop();
	654	client.start();
	655	events.onStatus.mockClear();
	656	expect(decodedFrame()).toHaveBeenCalledOnce();
	657	expect(drawImage).not.toHaveBeenCalled();
	658	expect(events.onStatus).not.toHaveBeenCalled();
	659	client.stop();
	660	});
	661
	662	it("ignores an old AVCC response rejected after a new attempt is connected", async () => {
	663	const { client, fetch, events } = recoveryFixture("ios", false);
	664	let resolveResponse!: (response: Response) => void;
	665	fetch.mockImplementationOnce(() => Promise.resolve(new Response("prime")));
	666	fetch.mockImplementationOnce(
	667	() =>
	668	new Promise((resolve) => {
	669	resolveResponse = resolve;
	670	}),
	671	);
	672	client.start();
	673	await vi.advanceTimersByTimeAsync(0);
	674	client.stop();
	675	client.start();
	676	await vi.advanceTimersByTimeAsync(0);
	677	events.onStatus.mockClear();
	678	resolveResponse(new Response(null, { status: 401 }));
	679	await vi.advanceTimersByTimeAsync(0);
	680	expect(events.onUnauthorized).not.toHaveBeenCalled();
	681	expect(events.onStatus).not.toHaveBeenCalled();
	682	client.stop();
	683	expect(vi.getTimerCount()).toBe(0);
	684	});
	685
	686	it("closes a delayed JPEG seed from a stopped AVCC attempt without painting it", async () => {
	687	const { client, videoBodies, drawImage, events } = recoveryFixture("ios", false);
	688	let resolveBitmap!: (bitmap: ImageBitmap) => void;
	689	vi.stubGlobal(
	690	"createImageBitmap",
	691	() =>
	692	new Promise((resolve) => {
	693	resolveBitmap = resolve;
	694	}),
	695	);
	696	client.start();
	697	await vi.advanceTimersByTimeAsync(0);
	698	videoBodies[0]!.enqueue(new Uint8Array(envelope(4, [1])));
	699	await vi.advanceTimersByTimeAsync(0);
	700	client.stop();
	701	client.start();
	702	events.onStatus.mockClear();
	703	const close = vi.fn();
	704	resolveBitmap({ width: 400, height: 800, close } as unknown as ImageBitmap);
	705	await vi.advanceTimersByTimeAsync(0);
	706	expect(close).toHaveBeenCalledOnce();
	707	expect(drawImage).not.toHaveBeenCalled();
	708	expect(events.onStatus).not.toHaveBeenCalled();
	709	client.stop();
	710	});
	711	});
	712
	713	it("reports a stalled AVCC body after its initial image instead of leaving a frozen streaming state", async () => {
	714	const { client, videoBodies, events, signals } = recoveryFixture("ios", false);
	715	const close = vi.fn();
	716	vi.stubGlobal("createImageBitmap", () => Promise.resolve({ width: 400, height: 800, close }));
	717	try {
	718	client.start();
	719	await vi.advanceTimersByTimeAsync(0);
	720	videoBodies[0]!.enqueue(new Uint8Array(envelope(4, [1])));
	721	await vi.advanceTimersByTimeAsync(0);
	722	expect(events.onStatus).toHaveBeenLastCalledWith("streaming", undefined);
	723	await vi.advanceTimersByTimeAsync(15_000);
	724	expect(events.onStatus).toHaveBeenLastCalledWith(
	725	"error",
	726	expect.stringContaining("stopped receiving video"),
	727	);
	728	expect(signals.every((signal) => signal.aborted)).toBe(true);
	729	expect(vi.getTimerCount()).toBe(0);
	730	} finally {
	731	client.stop();
	732	vi.useRealTimers();
	733	vi.unstubAllGlobals();
	734	}
	735	});

packages/client-runtime/src/device/stream.ts+213-60

...	...	@@ -37,8 +37,8 @@ export interface DeviceStreamEvents {
37	37	readonly onUnauthorized: () => void;
38	38	/**
39	39	* H.264 cannot be decoded here (no WebCodecs, or the simulator's profile is
40		* unsupported); the owner should show this MJPEG URL in an `<img>` instead of
41		* the canvas.
	40	* unsupported); the owner should show an `<img>` instead of the canvas
	41	* and attach it with `setMjpegImage` so the client can observe real frames.
42	42	*/
43	43	readonly onMjpegFallback: (url: string) => void;
44	44	/** Whether touches and keys can currently reach the device. */
...	...	@@ -56,6 +56,8 @@ export interface DeviceStreamTarget {
56	56	export type DeviceHardwareButton = "home" \| "back" \| "recents" \| "power" \| "appSwitcher";
57	57
58	58	const RETRY_DELAY_MS = 1_000;
	59	const FIRST_FRAME_TIMEOUT_MS = 15_000;
	60	const MJPEG_FRAME_CHECK_MS = 250;
59	61	const FRAME_DURATION_US = 16_667;
60	62	const SEMU_MAGIC = 0x53454d55;
61	63	const SEMU_HEADER_BYTES = 16;
...	...	@@ -205,6 +207,11 @@ export class AvccDemuxer {
205	207	export interface DeviceStreamClient {
206	208	readonly start: () => void;
207	209	readonly stop: () => void;
	210	/**
	211	* Own the displayed MJPEG image's source and frame/error observation.
	212	* `stop()` detaches it; attach a fresh image for each restart.
	213	*/
	214	readonly setMjpegImage: (image: HTMLImageElement \| null) => void;
208	215	/** Normalized 0..1 coordinates in the displayed frame. */
209	216	readonly sendTouch: (phase: "begin" \| "move" \| "end", x: number, y: number) => void;
210	217	readonly sendKey: (event: KeyboardEvent, phase: "down" \| "up") => void;
...	...	@@ -298,19 +305,99 @@ export function createDeviceStreamClient(
298	305	let firstFrame = false;
299	306	let configuring = false;
300	307	let mjpeg = false;
	308	let generation = 0;
	309	let decoderEpoch = 0;
	310	let frameTimer: ReturnType<typeof setTimeout> \| null = null;
	311	let videoReadTimer: ReturnType<typeof setTimeout> \| null = null;
	312	let mjpegImage: HTMLImageElement \| null = null;
	313	let releaseImage: (() => void) \| null = null;
301	314
302	315	const mjpegUrl = () => httpUrl(`/helper/${device}/stream.mjpeg`);
303	316
	317	const setStatus = (status: DeviceStreamStatus, detail?: string) => {
	318	if (!stopped) events.onStatus(status, detail);
	319	};
	320
	321	const clearFrameTimer = () => {
	322	if (frameTimer !== null) clearTimeout(frameTimer);
	323	frameTimer = null;
	324	};
	325
	326	const fail = (detail: string) => {
	327	if (stopped) return;
	328	stop();
	329	events.onInputConnected(false, detail);
	330	events.onStatus("error", detail);
	331	};
	332
	333	const connecting = (detail?: string) => {
	334	firstFrame = false;
	335	if (frameTimer === null) {
	336	frameTimer = setTimeout(
	337	() => fail("No video received from the device. Reconnect to try again."),
	338	FIRST_FRAME_TIMEOUT_MS,
	339	);
	340	}
	341	setStatus("connecting", detail);
	342	};
	343
	344	const frameReceived = () => {
	345	if (firstFrame) return;
	346	firstFrame = true;
	347	clearFrameTimer();
	348	setStatus("streaming");
	349	};
	350
	351	const observeMjpegImage = () => {
	352	releaseImage?.();
	353	releaseImage = null;
	354	const image = mjpegImage;
	355	if (!image \|\| stopped \|\| !mjpeg) return;
	356	let timer: ReturnType<typeof setTimeout> \| null = null;
	357	let released = false;
	358	const check = () => {
	359	if (released \|\| stopped) return;
	360	if (timer !== null) clearTimeout(timer);
	361	timer = null;
	362	if (image.naturalWidth > 0 && image.naturalHeight > 0) frameReceived();
	363	else {
	364	// Multipart images may not emit load until the response ends. Stop checking after the first frame.
	365	timer = setTimeout(check, MJPEG_FRAME_CHECK_MS);
	366	}
	367	};
	368	const error = () => {
	369	if (!released) fail("Could not receive the device stream. Reconnect to try again.");
	370	};
	371	image.addEventListener("load", check);
	372	image.addEventListener("error", error);
	373	releaseImage = () => {
	374	released = true;
	375	if (timer !== null) clearTimeout(timer);
	376	image.removeEventListener("load", check);
	377	image.removeEventListener("error", error);
	378	image.removeAttribute("src");
	379	};
	380	image.src = mjpegUrl();
	381	check();
	382	};
	383
	384	const setMjpegImage = (image: HTMLImageElement \| null) => {
	385	if (mjpegImage === image) return;
	386	releaseImage?.();
	387	releaseImage = null;
	388	mjpegImage = image;
	389	observeMjpegImage();
	390	};
	391
304	392	const fallBackToMjpeg = () => {
305	393	if (stopped \|\| mjpeg) return;
306	394	mjpeg = true;
	395	controller?.abort();
	396	controller = null;
307	397	closeDecoder();
	398	connecting();
308	399	events.onMjpegFallback(mjpegUrl());
309		setStatus("streaming");
310		};
311
312		const setStatus = (status: DeviceStreamStatus, detail?: string) => {
313		if (!stopped) events.onStatus(status, detail);
	400	if (!releaseImage) observeMjpegImage();
314	401	};
315	402
316	403	const paint = (source: CanvasImageSource, width: number, height: number) => {
...	...	@@ -323,14 +410,14 @@ export function createDeviceStreamClient(
323	410	events.onScreen(screen);
324	411	}
325	412	}
326		canvas.getContext("2d")?.drawImage(source, 0, 0, width, height);
327		if (!firstFrame) {
328		firstFrame = true;
329		setStatus("streaming");
330		}
	413	const context = canvas.getContext("2d");
	414	if (!context) return fail("Could not display the device stream. Reconnect to try again.");
	415	context.drawImage(source, 0, 0, width, height);
	416	frameReceived();
331	417	};
332	418
333	419	const closeDecoder = () => {
	420	decoderEpoch++;
334	421	try {
335	422	videoDecoder?.close();
336	423	} catch {
...	...	@@ -340,40 +427,51 @@ export function createDeviceStreamClient(
340	427	awaitingKeyframe = true;
341	428	};
342	429
343		const makeDecoder = () =>
344		new VideoDecoder({
	430	const recoverDecoder = () => {
	431	if (platform === "ios") fallBackToMjpeg();
	432	else {
	433	closeDecoder();
	434	connecting("Video decoder restarted.");
	435	requestKeyframe();
	436	}
	437	};
	438
	439	const makeDecoder = () => {
	440	const decoder = new VideoDecoder({
345	441	output: (frame) => {
346	442	try {
347		paint(frame, frame.displayWidth, frame.displayHeight);
	443	if (videoDecoder === decoder) paint(frame, frame.displayWidth, frame.displayHeight);
348	444	} finally {
349	445	frame.close();
350	446	}
351	447	},
352	448	error: () => {
353		closeDecoder();
354		requestKeyframe();
	449	if (stopped \|\| videoDecoder !== decoder) return;
	450	recoverDecoder();
355	451	},
356	452	});
	453	return decoder;
	454	};
357	455
358		/**
359		* Resolves false when this browser cannot decode the stream's profile
360		* (simulators encode High 5.1, which headless and some hardware decoders
361		* reject). iOS then falls back to MJPEG; Android has no MJPEG.
362		*/
363		const configureDecoder = async (config: VideoDecoderConfig): Promise<boolean> => {
	456	/** iOS can fall back to MJPEG when the stream's H.264 profile is unsupported. */
	457	const configureDecoder = async (
	458	config: VideoDecoderConfig,
	459	isCurrent: () => boolean,
	460	): Promise<boolean> => {
	461	const epoch = decoderEpoch;
364	462	const full: VideoDecoderConfig = { ...config, optimizeForLatency: true };
365	463	const support = await VideoDecoder.isConfigSupported(full).catch(() => ({ supported: false }));
366		if (stopped) return false;
	464	if (!isCurrent() \|\| epoch !== decoderEpoch) return false;
367	465	if (!support.supported) {
368		setStatus("error", `This browser cannot decode ${config.codec}.`);
	466	if (platform === "android") fail(`This browser cannot decode ${config.codec}.`);
369	467	return false;
370	468	}
371		if (!videoDecoder \|\| videoDecoder.state === "closed") videoDecoder = makeDecoder();
372	469	try {
	470	if (!videoDecoder \|\| videoDecoder.state === "closed") videoDecoder = makeDecoder();
373	471	videoDecoder.configure(full);
374	472	return true;
375	473	} catch (cause) {
376		setStatus("error", `Video decoder: ${(cause as Error).message}`);
	474	if (platform === "android") fail(`Video decoder: ${(cause as Error).message}`);
377	475	return false;
378	476	}
379	477	};
...	...	@@ -385,8 +483,7 @@ export function createDeviceStreamClient(
385	483	awaitingKeyframe = false;
386	484	}
387	485	if (videoDecoder.decodeQueueSize > SOFT_DECODE_QUEUE) {
388		closeDecoder();
389		requestKeyframe();
	486	recoverDecoder();
390	487	return;
391	488	}
392	489	try {
...	...	@@ -399,8 +496,7 @@ export function createDeviceStreamClient(
399	496	);
400	497	timestamp += FRAME_DURATION_US;
401	498	} catch {
402		closeDecoder();
403		requestKeyframe();
	499	recoverDecoder();
404	500	}
405	501	};
406	502
...	...	@@ -423,42 +519,66 @@ export function createDeviceStreamClient(
423	519
424	520	const handleUnauthorized = () => {
425	521	stop();
	522	events.onInputConnected(false);
426	523	events.onUnauthorized();
427	524	};
428	525
429	526	// iOS video: fetch the AVCC body and demux into the decoder.
430	527	const readIosVideo = async () => {
431	528	const demuxer = new AvccDemuxer();
432		controller = new AbortController();
	529	const session = generation;
	530	const videoController = new AbortController();
	531	controller = videoController;
	532	const isCurrent = () => !stopped && generation === session && controller === videoController;
	533	let retryDetail: string \| undefined;
433	534	try {
434	535	const response = await fetch(httpUrl(`/helper/${device}/stream.avcc`), {
435		signal: controller.signal,
	536	signal: videoController.signal,
436	537	credentials: access.credentials ? "include" : "same-origin",
437	538	});
	539	if (!isCurrent()) return;
438	540	if (response.status === 401 \|\| response.status === 403) return handleUnauthorized();
439	541	if (!response.ok \|\| !response.body) throw new Error(`stream ${response.status}`);
440	542	const reader = response.body.getReader();
441	543	for (;;) {
442		const { done, value } = await reader.read();
443		if (done \|\| stopped) break;
	544	// An AVCC body can stay open after its helper stops producing frames.
	545	const timer = setTimeout(() => {
	546	if (isCurrent()) fail("Device stream stopped receiving video. Reconnect to try again.");
	547	}, FIRST_FRAME_TIMEOUT_MS);
	548	videoReadTimer = timer;
	549	let result: ReadableStreamReadResult<Uint8Array>;
	550	try {
	551	result = await reader.read();
	552	} finally {
	553	clearTimeout(timer);
	554	if (videoReadTimer === timer) videoReadTimer = null;
	555	}
	556	const { done, value } = result;
	557	if (!isCurrent()) return;
	558	if (done) break;
444	559	for (const chunk of demuxer.push(value)) {
445	560	switch (chunk.type) {
446	561	case "seed":
447	562	void createImageBitmap(new Blob([chunk.payload as BlobPart], { type: "image/jpeg" }))
448	563	.then((bitmap) => {
449		paint(bitmap, bitmap.width, bitmap.height);
450		bitmap.close();
	564	try {
	565	if (isCurrent()) paint(bitmap, bitmap.width, bitmap.height);
	566	} finally {
	567	bitmap.close();
	568	}
451	569	})
452	570	.catch(() => {});
453	571	break;
454	572	case "description": {
455	573	awaitingKeyframe = true;
456		const configured = await configureDecoder({
457		codec: avcCodecString(chunk.payload),
458		description: chunk.payload,
459		});
	574	const configured = await configureDecoder(
	575	{ codec: avcCodecString(chunk.payload), description: chunk.payload },
	576	isCurrent,
	577	);
	578	if (!isCurrent()) return;
460	579	if (!configured) {
461	580	await reader.cancel().catch(() => {});
	581	if (!isCurrent()) return;
462	582	fallBackToMjpeg();
463	583	return;
464	584	}
...	...	@@ -472,10 +592,16 @@ export function createDeviceStreamClient(
472	592	}
473	593	}
474	594	} catch (cause) {
475		if (stopped) return;
476		setStatus("connecting", (cause as Error).message);
	595	if (!isCurrent()) return;
	596	retryDetail = (cause as Error).message;
	597	}
	598	if (isCurrent()) {
	599	controller = null;
	600	videoController.abort();
	601	closeDecoder();
	602	connecting(retryDetail);
	603	scheduleRetry("video", () => void readIosVideo());
477	604	}
478		if (!stopped) scheduleRetry("video", () => void readIosVideo());
479	605	};
480	606
481	607	/**
...	...	@@ -483,7 +609,7 @@ export function createDeviceStreamClient(
483	609	* screen capture is running, and the AVCC stream does not reliably start
484	610	* it. Touching the MJPEG endpoint does; one aborted request is enough.
485	611	*/
486		const primeIosHelper = async () => {
	612	const primeIosHelper = async (session: number) => {
487	613	const controller = new AbortController();
488	614	primeController = controller;
489	615	const timeout = setTimeout(() => controller.abort(), 2_000);
...	...	@@ -492,6 +618,7 @@ export function createDeviceStreamClient(
492	618	signal: controller.signal,
493	619	credentials: access.credentials ? "include" : "same-origin",
494	620	});
	621	if (stopped \|\| generation !== session) return;
495	622	if (response.status === 401 \|\| response.status === 403) return handleUnauthorized();
496	623	await response.body?.getReader().read();
497	624	} catch {
...	...	@@ -506,16 +633,19 @@ export function createDeviceStreamClient(
506	633	// iOS input socket; also carries the screen config the helper pushes.
507	634	const connectIosInput = async () => {
508	635	if (stopped) return;
509		await primeIosHelper();
510		if (stopped) return;
	636	const session = generation;
	637	await primeIosHelper(session);
	638	if (stopped \|\| generation !== session) return;
511	639	const ws = new WebSocket(wsUrl(`/helper/ws?device=${device}`));
512	640	ws.binaryType = "arraybuffer";
513	641	socket = ws;
514	642	ws.onopen = () => {
	643	if (stopped \|\| socket !== ws) return;
515	644	ws.send(taggedJson(IOS_MSG_HARDWARE_KEYBOARD, { enabled: false }));
516	645	events.onInputConnected(true);
517	646	};
518	647	ws.onmessage = (event) => {
	648	if (stopped \|\| socket !== ws) return;
519	649	if (!(event.data instanceof ArrayBuffer)) return;
520	650	const bytes = new Uint8Array(event.data);
521	651	if (bytes.length < 1 \|\| bytes[0] !== IOS_TAG_SCREEN_CONFIG) return;
...	...	@@ -556,14 +686,21 @@ export function createDeviceStreamClient(
556	686	ws.binaryType = "arraybuffer";
557	687	socket = ws;
558	688	ws.onopen = () => {
559		setStatus("connecting");
	689	if (stopped \|\| socket !== ws) return;
	690	connecting();
560	691	events.onInputConnected(true);
561	692	};
562	693	ws.onmessage = (event) => {
	694	if (stopped \|\| socket !== ws) return;
563	695	if (typeof event.data === "string") {
564	696	// The encoder restarts at a new size when the device rotates; the
565	697	// next keyframe carries a fresh SPS, so the decoder is rebuilt from it.
566		if (isVideoSessionMessage(event.data)) closeDecoder();
	698	if (isVideoSessionMessage(event.data)) {
	699	closeDecoder();
	700	configuring = false;
	701	connecting();
	702	requestKeyframe();
	703	}
567	704	return;
568	705	}
569	706	if (!(event.data instanceof ArrayBuffer)) return;
...	...	@@ -576,11 +713,16 @@ export function createDeviceStreamClient(
576	713	if (scanned?.sps && (!videoDecoder \|\| videoDecoder.state !== "configured")) {
577	714	if (configuring) return;
578	715	configuring = true;
579		void configureDecoder({ codec: avcCodecString(scanned.sps) }).then((configured) => {
580		configuring = false;
581		awaitingKeyframe = true;
582		if (configured) requestKeyframe();
583		});
	716	const epoch = decoderEpoch;
	717	const isCurrent = () => !stopped && socket === ws;
	718	void configureDecoder({ codec: avcCodecString(scanned.sps) }, isCurrent).then(
	719	(configured) => {
	720	if (!isCurrent() \|\| epoch !== decoderEpoch) return;
	721	configuring = false;
	722	awaitingKeyframe = true;
	723	if (configured) requestKeyframe();
	724	},
	725	);
584	726	return;
585	727	}
586	728	if (!videoDecoder \|\| videoDecoder.state !== "configured") {
...	...	@@ -601,7 +743,8 @@ export function createDeviceStreamClient(
601	743	(event.code === 1006 && access.query.wsTicket)
602	744	)
603	745	return handleUnauthorized();
604		setStatus("connecting", event.reason \|\| undefined);
	746	configuring = false;
	747	connecting(event.reason \|\| undefined);
605	748	scheduleRetry("input", connectAndroid);
606	749	};
607	750	ws.onerror = () => ws.close();
...	...	@@ -610,8 +753,9 @@ export function createDeviceStreamClient(
610	753	const start = () => {
611	754	if (!stopped) return;
612	755	stopped = false;
613		firstFrame = false;
614		events.onStatus("connecting");
	756	generation++;
	757	configuring = false;
	758	connecting();
615	759	if (platform === "ios") {
616	760	void connectIosInput();
617	761	if (useWebCodecs) void readIosVideo();
...	...	@@ -619,27 +763,35 @@ export function createDeviceStreamClient(
619	763	} else if (useWebCodecs) {
620	764	connectAndroid();
621	765	} else {
622		setStatus("error", "This browser cannot decode the Android stream (WebCodecs unavailable).");
	766	fail("This browser cannot decode the Android stream (WebCodecs unavailable).");
623	767	}
624	768	};
625	769
626	770	const stop = () => {
627	771	if (stopped) return;
628	772	stopped = true;
	773	generation++;
629	774	mjpeg = false;
	775	clearFrameTimer();
	776	if (videoReadTimer !== null) clearTimeout(videoReadTimer);
	777	videoReadTimer = null;
	778	releaseImage?.();
	779	releaseImage = null;
	780	mjpegImage = null;
630	781	for (const timer of retryTimers.values()) clearTimeout(timer);
631	782	retryTimers.clear();
632	783	primeController?.abort();
633	784	primeController = null;
634	785	controller?.abort();
635	786	controller = null;
636		socket?.close();
	787	const discarded = socket;
637	788	socket = null;
	789	discarded?.close();
638	790	closeDecoder();
639	791	};
640	792
641	793	const send = (payload: Uint8Array<ArrayBuffer> \| string) => {
642		if (socket?.readyState === WebSocket.OPEN) socket.send(payload);
	794	if (!stopped && socket?.readyState === WebSocket.OPEN) socket.send(payload);
643	795	};
644	796
645	797	const rawPoint = (x: number, y: number) => {
...	...	@@ -661,6 +813,7 @@ export function createDeviceStreamClient(
661	813	return {
662	814	start,
663	815	stop,
	816	setMjpegImage,
664	817	sendTouch: (phase, x, y) => {
665	818	if (platform === "ios") {
666	819	send(taggedJson(IOS_MSG_TOUCH, { type: phase, ...rawPoint(x, y) }));