Commit 46f3c2ca2c99ceab0547452ac863232944621fce | the-knife

apps/server/src/device/DeviceHubProxy.test.ts+26

...	@@ -139,3 +139,29 @@ it.each([	...	@@ -139,3 +139,29 @@ it.each([
139	expect(await response.text()).not.toContain("private credential diagnostic");	139	expect(await response.text()).not.toContain("private credential diagnostic");
140	expect(requests).toEqual([]);	140	expect(requests).toEqual([]);
141	});	141	});
		142
		143	it.each([1, 3])(
		144	"forwards fixed Duo display %s through the authenticated read proxy",
		145	async (panel) => {
		146	const { handler, requests } = fixture([AuthOrchestrationReadScope]);
		147	const route = `/vendor/serve-sim/helper/duo/panel/${panel}/stream.avcc`;
		148	const response = await handler(
		149	new Request(`http://t3.test/api/device-hub${route}?wsTicket=secret`),
		150	);
		151	expect(response.status).toBe(200);
		152	await response.text();
		153	expect(requests).toEqual([`http://hub.test${route}`]);
		154	},
		155	);
		156
		157	it.each(["/panel/2/stream.avcc", "/panel/1/webrtc/offer", "/panel/3/exec"])(
		158	"rejects unsupported Duo route %s",
		159	async (route) => {
		160	const { handler, requests } = fixture([AuthOrchestrationReadScope]);
		161	const response = await handler(
		162	new Request(`http://t3.test/api/device-hub/vendor/serve-sim/helper/duo${route}`),
		163	);
		164	expect(response.status).toBe(404);
		165	expect(requests).toEqual([]);
		166	},
		167	);

apps/server/src/device/DeviceHubProxy.ts+1

...	@@ -42,6 +42,7 @@ const ALLOWED_PATHS: ReadonlyArray<RegExp> = [	...	@@ -42,6 +42,7 @@ const ALLOWED_PATHS: ReadonlyArray<RegExp> = [
42	/^\/vendor\/serve-sim\/api\/screenshot$/,	42	/^\/vendor\/serve-sim\/api\/screenshot$/,
43	/^\/vendor\/serve-sim\/api\/event-log(\/events)?$/,	43	/^\/vendor\/serve-sim\/api\/event-log(\/events)?$/,
44	/^\/vendor\/serve-sim\/helper\/[^/]+\/(stream\.mjpeg\|stream\.avcc\|config\|health\|ax\|foreground)$/,	44	/^\/vendor\/serve-sim\/helper\/[^/]+\/(stream\.mjpeg\|stream\.avcc\|config\|health\|ax\|foreground)$/,
		45	/^\/vendor\/serve-sim\/helper\/[^/]+\/panel\/(1\|3)\/stream\.avcc$/,
45	/^\/vendor\/serve-sim\/appstate$/,	46	/^\/vendor\/serve-sim\/appstate$/,
46	/^\/vendor\/serve-emu\/api\/(devices\|screenshot\|stream-mode\|stream-settings\|accessibility)$/,	47	/^\/vendor\/serve-emu\/api\/(devices\|screenshot\|stream-mode\|stream-settings\|accessibility)$/,
47	/^\/vendor\/serve-emu\/health$/,	48	/^\/vendor\/serve-emu\/health$/,

apps/server/src/device/DeviceToolchain.ts+1-1

...	@@ -26,7 +26,7 @@ import * as Semaphore from "effect/Semaphore";	...	@@ -26,7 +26,7 @@ import * as Semaphore from "effect/Semaphore";
26	import * as ProcessRunner from "../processRunner.ts";	26	import * as ProcessRunner from "../processRunner.ts";
27		27
28	const DEVICE_HUB_PACKAGE = "expo-device-hub";	28	const DEVICE_HUB_PACKAGE = "expo-device-hub";
29	export const DEVICE_HUB_VERSION = "0.10.1";	29	export const DEVICE_HUB_VERSION = "0.12.0";
30	const AGENT_DEVICE_PACKAGE = "agent-device";	30	const AGENT_DEVICE_PACKAGE = "agent-device";
31	export const AGENT_DEVICE_VERSION = "0.21.12";	31	export const AGENT_DEVICE_VERSION = "0.21.12";
32		32

apps/web/src/components/device/DeviceControlsRail.tsx+4-3

 @@ -57,9 +57,9 @@ export function DeviceControlsRail(props: {
     <aside
       aria-label="Device controls"
       data-layout="rail"
-      className="pointer-events-none absolute inset-y-0 right-0 z-10 flex w-14 flex-col items-center justify-center py-3 pr-2"
+      className="pointer-events-none absolute inset-y-0 right-0 z-10 flex w-14 flex-col items-center gap-2 overflow-y-auto [justify-content:safe_center] py-3 pr-2 [scrollbar-width:none]"
     >
-      <div className="pointer-events-auto flex max-h-full shrink-0 flex-col items-center gap-1 overflow-y-auto rounded-full border border-border/50 bg-background/80 p-2 shadow-sm [scrollbar-width:none]">
+      <div className="pointer-events-auto flex shrink-0 flex-col items-center gap-1 overflow-y-auto rounded-full border border-border/50 bg-background/80 p-2 shadow-sm [scrollbar-width:none]">
         <RailButton
           tooltipSide={popupSide}
           label="Home"
 @@ -255,11 +255,12 @@ export function DeviceControlsRail(props: {
           </RailButton>
         ) : null}
         {view.phone ? (
-          <RailButton tooltipSide={popupSide} label="Reset phone view" onClick={view.resetView}>
+          <RailButton tooltipSide={popupSide} label="Restore 3D view" onClick={view.resetView}>
             <Maximize />
           </RailButton>
         ) : null}
       </div>
+      {view.foldingControls}
     </aside>
   );
 }

apps/web/src/components/device/DeviceDuoControls.tsx created+75

...	@@ -0,0 +1,75 @@
1	import {
2	DUO_POSES,
3	type DuoCommand,
4	type DuoControlState,
5	} from "@t3tools/client-runtime/device/duo-control";
6	import type { DeviceScreenSize } from "@t3tools/client-runtime/device/stream";
7	import { DeviceDuoGlyph } from "./DeviceDuoGlyph";
8	import { Button } from "~/components/ui/button";
9	import { Tooltip, TooltipPopup, TooltipTrigger } from "~/components/ui/tooltip";
10
11	/** Physical presets live beside the device. Pinching supplies continuous hinge control. */
12	export function DeviceDuoControls(props: {
13	screen: DeviceScreenSize;
14	state: DuoControlState;
15	enabled: boolean;
16	onCommand: (command: DuoCommand) => void;
17	}) {
18	const angle = props.screen.hingeAngle;
19	const fold = angle == null ? null : angle === 0 ? "closed" : angle === 180 ? "open" : "book";
20	const selected = (id: (typeof DUO_POSES)[number]["id"]) =>
21	id === "laptop" \|\| id === "tent" ? props.screen.hingePose === id : fold === id;
22	return (
23	<div aria-label="iPhone Duo stands" className="flex flex-col items-center gap-2">
24	{([DUO_POSES.slice(0, 3), DUO_POSES.slice(3)] as const).map((poses, index) => (
25	<div
26	key={poses[0]?.id}
27	role="group"
28	aria-label={index === 0 ? "Fold shape" : "Device stance"}
29	className="pointer-events-auto flex shrink-0 flex-col items-center gap-1 rounded-full border border-border/50 bg-background/80 p-1 shadow-sm"
30	>
31	{poses.map((pose) => (
32	<Tooltip key={pose.id}>
33	<TooltipTrigger
34	render={
35	<Button
36	size="icon"
37	variant={selected(pose.id) ? "secondary" : "ghost"}
38	disabled={!props.enabled}
39	aria-label={`${pose.label} stand`}
40	aria-pressed={selected(pose.id)}
41	data-pressed={selected(pose.id) ? "" : undefined}
42	onClick={() => props.onCommand({ control: "pose", value: pose.id })}
43	/>
44	}
45	>
46	<DeviceDuoGlyph pose={pose.id} />
47	</TooltipTrigger>
48	<TooltipPopup side="left">
49	{pose.label}
50	{pose.id === "book" ? " / bookshelf" : ""}
51	</TooltipPopup>
52	</Tooltip>
53	))}
54	</div>
55	))}
56	{props.state.error ? (
57	<Tooltip>
58	<TooltipTrigger
59	render={
60	<span
61	tabIndex={0}
62	role="alert"
63	aria-label={props.state.error}
64	className="pointer-events-auto text-xs text-destructive"
65	>
66	!
67	</span>
68	}
69	/>
70	<TooltipPopup side="left">{props.state.error}</TooltipPopup>
71	</Tooltip>
72	) : null}
73	</div>
74	);
75	}

apps/web/src/components/device/DeviceDuoGlyph.tsx created+43

...	@@ -0,0 +1,43 @@
1	import type { DuoPose } from "@t3tools/client-runtime/device/duo-control";
2
3	// Bitrig 0.25's toolbar glyphs. The fold outlines follow the native toolbar;
4	// the rounded stance contours are ported from SimulatorFoldingPoseGlyph's SwiftUI paths.
5	const stancePaths = {
6	laptop:
7	"M2.7351 12.9857 L15.8128 11.4159 L14.4114 3.3787 C14.1943 2.134 13.1275 1.2319 12.0283 1.3639 L2.9308 2.4559 C1.8317 2.5879 1.1167 3.7038 1.3337 4.9485 L2.7351 12.9857 Z M2.7351 12.9857 L15.8128 11.4159 L19.9103 12.8914 C20.5449 13.1199 20.1683 13.4121 19.0692 13.5441 L9.9717 14.6361 C8.8725 14.7681 7.4672 14.6898 6.8326 14.4612 L2.7351 12.9857 Z",
8	tent: "M3.2522 2.7267 L18.7478 1.4816 L16.696 9.7017 C16.3782 10.9748 15.0649 12.0915 13.7626 12.1962 L2.983 13.0623 C1.6807 13.167 0.8826 12.2199 1.2004 10.9468 L3.2522 2.7267 Z M3.2522 2.7267 L18.7478 1.4816 L20.7996 10.6909 C21.1174 12.1172 20.3193 13.3581 19.017 13.4628 L8.2374 14.3289 C6.9351 14.4336 5.6218 13.3623 5.304 11.936 L3.2522 2.7267 Z",
9	} as const;
10
11	export function DeviceDuoGlyph({ pose }: { pose: DuoPose }) {
12	const stance = pose === "laptop" \|\| pose === "tent";
13	return (
14	<svg
15	viewBox={stance ? "0 0 22 16" : "0 0 24 32"}
16	fill="none"
17	stroke="currentColor"
18	strokeWidth={stance ? 1.25 : 1.8}
19	strokeLinecap="round"
20	strokeLinejoin="round"
21	className="size-7 shrink-0"
22	aria-hidden
23	>
24	{stance ? <path d={stancePaths[pose]} /> : null}
25	{pose === "closed" ? (
26	<>
27	<rect x="2" y="9" width="20" height="13" rx="2.2" />
28	<path d="M19.3 12.5v6" />
29	<circle cx="5" cy="12" r=".5" fill="currentColor" stroke="none" />
30	</>
31	) : null}
32	{pose === "book" ? (
33	<path d="M4.2 6.5h15.6c1 0 1.6.8 1.2 1.7L17.7 16l3.3 7.8c.4.9-.2 1.7-1.2 1.7H4.2c-1 0-1.6-.8-1.2-1.7L6.3 16 3 8.2c-.4-.9.2-1.7 1.2-1.7Z" />
34	) : null}
35	{pose === "open" ? (
36	<>
37	<rect x="2" y="2" width="20" height="28" rx="2.4" />
38	<path d="M10 27h4" />
39	</>
40	) : null}
41	</svg>
42	);
43	}

apps/web/src/components/device/DeviceDuoViewport.tsx created+205

...	@@ -0,0 +1,205 @@
1	import { useEffect, useRef, type RefObject } from "react";
2	import type { DuoViewer } from "@t3tools/client-runtime/device/duo-viewer";
3	import type { DeviceModelSource } from "@t3tools/client-runtime/device/model";
4	import { createPhoneInteraction } from "@t3tools/client-runtime/device/phone-interaction";
5	import { createCanvasFrameSink } from "@t3tools/client-runtime/device/frame";
6	import type { DeviceScreenSize, DeviceStreamClient } from "@t3tools/client-runtime/device/stream";
7	import { createDuoPinch } from "@t3tools/client-runtime/device/duo-control";
8	import { bindPhoneTrackpad } from "./phoneTrackpad";
9
10	const loadDuoViewer = () => import("@t3tools/client-runtime/device/duo-viewer");
11
12	/** Web shell for the framework-independent viewer. The decoded screen and input connection remain owned by DeviceStreamView. */
13	export function DeviceDuoViewport(props: {
14	readonly model: DeviceModelSource;
15	readonly source: RefObject<HTMLCanvasElement \| null>;
16	readonly onFrameListener: (listener: (() => void) \| null) => void;
17	readonly onResetReady: (reset: (() => void) \| null) => void;
18	readonly onInputCancel: (cancel: (() => void) \| null) => void;
19	readonly client: RefObject<DeviceStreamClient \| null>;
20	readonly screen: DeviceScreenSize \| null;
21	readonly hingePreview: number \| null;
22	readonly controlError: string \| null;
23	readonly onUnavailable: () => void;
24	}) {
25	const hostRef = useRef<HTMLDivElement \| null>(null);
26	const canvasRef = useRef<HTMLCanvasElement \| null>(null);
27	const viewerRef = useRef<DuoViewer \| null>(null);
28	const interactionRef = useRef<ReturnType<typeof createPhoneInteraction> \| null>(null);
29	const pinchRef = useRef<ReturnType<typeof createDuoPinch> \| null>(null);
30	const trackpadRef = useRef<ReturnType<typeof bindPhoneTrackpad> \| null>(null);
31	const screenRef = useRef(props.screen);
32	const modelRef = useRef(props.model);
33	const previewRef = useRef(props.hingePreview);
34	const { source, client, onInputCancel, onResetReady, onUnavailable, onFrameListener } = props;
35
36	useEffect(() => {
37	screenRef.current = props.screen;
38	modelRef.current = props.model;
39	interactionRef.current?.end();
40	viewerRef.current?.setScreen(props.screen);
41	}, [props.screen, props.model]);
42
43	useEffect(() => {
44	if (props.controlError \|\| !props.screen) {
45	trackpadRef.current?.cancel();
46	viewerRef.current?.rejectOrientation();
47	}
48	}, [props.controlError, props.screen]);
49
50	useEffect(() => {
51	previewRef.current = props.hingePreview;
52	if (!pinchRef.current?.active) viewerRef.current?.setHingePreview(props.hingePreview);
53	}, [props.hingePreview]);
54
55	useEffect(() => {
56	const host = hostRef.current;
57	const canvas = canvasRef.current;
58	const decoded = source.current;
59	if (!host \|\| !canvas \|\| !decoded) return;
60	let disposed = false;
61	let trackpad: ReturnType<typeof bindPhoneTrackpad> \| null = null;
62	let stopTrackpadEnd: (() => void) \| undefined;
63	const resize = () => {
64	const { width, height } = host.getBoundingClientRect();
65	viewerRef.current?.resize(width, height, window.devicePixelRatio);
66	};
67	const observer = new ResizeObserver(resize);
68	observer.observe(host);
69	const blur = () => {
70	interactionRef.current?.end();
71	trackpad?.cancel();
72	viewerRef.current?.cancelInput();
73	};
74	onInputCancel(blur);
75	window.addEventListener("blur", blur);
76	void loadDuoViewer()
77	.then(({ createDuoViewer }) => {
78	if (disposed) return;
79	const sources = {
80	1: document.createElement("canvas"),
81	3: document.createElement("canvas"),
82	};
83	const viewer = createDuoViewer({
84	canvas,
85	sources,
86	onUnavailable,
87	onPanelRequested: (panel) =>
88	client.current?.controlDuo({
89	control: "physical",
90	value: panel === 1 ? "facedown" : "faceup",
91	}),
92	onOrientationRequested: (value) =>
93	client.current?.controlDuo({ control: "orientation", value }),
94	onModelError: (cause) => console.warn("Device 3D asset could not load", cause),
95	model: modelRef.current,
96	});
97	viewerRef.current = viewer;
98	onResetReady(viewer.resetPose);
99	viewer.setScreen(screenRef.current);
100	const primaryFrame = () => {
101	const panel = screenRef.current?.screenId;
102	if (panel === 1 \|\| panel === 3) viewer.frameUpdated(panel, decoded);
103	};
104	onFrameListener(primaryFrame);
105	primaryFrame();
106	viewer.setHingePreview(previewRef.current);
107	client.current?.setDuoPanels({
108	onScreen(next) {
109	screenRef.current = next;
110	interactionRef.current?.end();
111	viewer.setScreen(next);
112	},
113	cover: createCanvasFrameSink(sources[1], () => viewer.frameUpdated(1)),
114	inner: createCanvasFrameSink(sources[3], () => viewer.frameUpdated(3)),
115	});
116	interactionRef.current = createPhoneInteraction({
117	screenPoint: (point, captured) => viewer.screenPoint(point.x, point.y, captured),
118	touch: (phase, point) => client.current?.sendRawTouch(phase, point.x, point.y),
119	orbit: viewer.orbit,
120	zoomBy: () => {},
121	onInteractionActive: viewer.setInteractionActive,
122	});
123	const pinch = createDuoPinch({
124	angle: () =>
125	previewRef.current ??
126	screenRef.current?.hingeAngle ??
127	(screenRef.current?.screenId === 1 ? 0 : 180),
128	contains: (x, y) => !!screenRef.current && viewer.beginHinge(x, y),
129	change: (angle) => {
130	viewer.setHingePreview(angle ?? previewRef.current);
131	if (angle !== null) client.current?.controlDuo({ control: "angle", value: angle });
132	},
133	});
134	pinchRef.current = pinch;
135	trackpad = bindPhoneTrackpad(canvas, interactionRef.current, {
136	begin(x, y) {
137	interactionRef.current?.end();
138	return pinch.begin(x, y);
139	},
140	move: pinch.move,
141	end: pinch.end,
142	});
143	trackpadRef.current = trackpad;
144	stopTrackpadEnd = window.desktopBridge?.onTrackpadScrollEnd?.(() => trackpad?.endOrbit());
145	resize();
146	})
147	.catch(() => {
148	if (!disposed) onUnavailable();
149	});
150	return () => {
151	disposed = true;
152	stopTrackpadEnd?.();
153	trackpad?.dispose();
154	trackpadRef.current = null;
155	pinchRef.current = null;
156	interactionRef.current?.end();
157	interactionRef.current = null;
158	onInputCancel(null);
159	onResetReady(null);
160	onFrameListener(null);
161	client.current?.setDuoPanels(null);
162	observer.disconnect();
163	window.removeEventListener("blur", blur);
164	viewerRef.current?.dispose();
165	viewerRef.current = null;
166	};
167	}, [onInputCancel, onResetReady, client, onUnavailable, source, onFrameListener]);
168
169	const point = (event: React.PointerEvent<HTMLCanvasElement>) => {
170	const rect = event.currentTarget.getBoundingClientRect();
171	return {
172	x: (event.clientX - rect.left) / rect.width,
173	y: (event.clientY - rect.top) / rect.height,
174	};
175	};
176	return (
177	<div className="absolute inset-0">
178	<div ref={hostRef} className="absolute inset-0">
179	<div
180	aria-hidden
181	className="pointer-events-none absolute bottom-[6%] left-1/2 h-5 w-2/5 -translate-x-1/2 rounded-full bg-foreground/10 blur-xl"
182	/>
183	<canvas
184	ref={canvasRef}
185	aria-label="Interactive 3D iPhone Duo. Drag the screen to interact. Drag outside it or swipe with two fingers to turn. Pinch over the device to open or close its hinge."
186	className="size-full touch-none"
187	onPointerDown={(event) => {
188	if (event.button !== 0) return;
189	if (!interactionRef.current?.begin(event.pointerId, point(event), event.altKey)) return;
190	event.preventDefault();
191	event.currentTarget.setPointerCapture(event.pointerId);
192	(event.currentTarget.closest('[role="application"]') as HTMLElement \| null)?.focus();
193	}}
194	onPointerMove={(event) => interactionRef.current?.move(event.pointerId, point(event))}
195	onPointerUp={(event) => {
196	interactionRef.current?.move(event.pointerId, point(event));
197	interactionRef.current?.end(event.pointerId);
198	}}
199	onPointerCancel={(event) => interactionRef.current?.end(event.pointerId)}
200	onLostPointerCapture={(event) => interactionRef.current?.end(event.pointerId)}
201	/>
202	</div>
203	</div>
204	);
205	}

apps/web/src/components/device/DeviceStreamView.tsx+61-22

 @@ -8,6 +8,9 @@ import { createCanvasFrameSink } from "@t3tools/client-runtime/device/frame";
 import { resolveDeviceShape } from "@t3tools/client-runtime/device/shape-profile";
 import { deviceKeyboard, deviceModel } from "./deviceModels";
 import { fitDeviceFrame } from "./deviceFrameLayout";
+import { DeviceDuoViewport } from "./DeviceDuoViewport";
+import { DeviceDuoControls } from "./DeviceDuoControls";
+import type { DuoControlState } from "@t3tools/client-runtime/device/duo-control";
 import { DevicePhoneViewport } from "./DevicePhoneViewport";
 import { DeviceLoadingView } from "./DeviceLoadingView";
 import { type DeviceAxElement, fetchDeviceAxTree } from "./deviceHubApi";
 @@ -29,6 +32,7 @@ export interface DeviceViewControls {
   readonly showPhone: () => void;
   readonly showFlat: () => void;
   readonly resetView: () => void;
+  readonly foldingControls?: ReactNode;
   readonly keyboard: { readonly attached: boolean; readonly toggle: () => void } | null;
 }
 @@ -61,6 +65,13 @@ export function DeviceStreamView(props: {
   readonly onHandle?: (handle: DeviceStreamHandle | null) => void;
   readonly onScreen?: (screen: DeviceScreenSize | null) => void;
 }) {
+  const [duoControl, setDuoControl] = useState<DuoControlState>({
+    pending: false,
+    requested: null,
+    error: null,
+  });
+  const model = deviceModel(props.platform, props.deviceName ?? "");
+  const isDuo = model?.id === "iphone-duo";
   const [presentation, setPresentation] = useState<"phone" | "flat">("phone");
   const [keyboardAttached, setKeyboardAttached] = useState(false);
   const [phoneUnavailable, setPhoneUnavailable] = useState(false);
 @@ -105,6 +116,8 @@ export function DeviceStreamView(props: {
       { platform: props.platform, deviceId: props.deviceId, access },
       createCanvasFrameSink(canvas, () => frameListenerRef.current?.()),
       {
+        onDuoControl: setDuoControl,
+        onDuoUnavailable: onPhoneUnavailable,
         onStatus: (next, nextDetail) => {
           setStatus(next);
           setDetail(nextDetail);
 @@ -148,6 +161,7 @@ export function DeviceStreamView(props: {
     access,
     cancelPhoneInput,
     onHandle,
+    onPhoneUnavailable,
     onScreen,
     props.deviceId,
     props.environmentId,
 @@ -176,7 +190,8 @@ export function DeviceStreamView(props: {
     presentation === "phone" &&
     !phoneUnavailable &&
     !mjpegUrl &&
-    !props.axOverlay;
+    !props.axOverlay &&
+    (!isDuo || screen?.supportsHingeAngle === true);
   const controlsInset = props.renderControls && !showPhone ? CONTROLS_RAIL_WIDTH : 0;
   // The frame is the largest box at `aspect` that fits the container, so a
 @@ -275,20 +290,25 @@ export function DeviceStreamView(props: {
     return { x: Math.min(1, Math.max(0, x)), y: Math.min(1, Math.max(0, y)) };
   };
-  const phoneUnavailableReason = phoneUnavailable
+  const phoneUnavailableReason =
-    ? "3D is unavailable on this browser"
+    isDuo && !screen?.supportsHingeAngle
-    : mjpegUrl
+      ? "iPhone Duo 3D requires Device Hub 0.11.0 or newer"
-      ? "3D requires the H.264 stream"
+      : phoneUnavailable
-      : props.axOverlay
+        ? "3D is unavailable on this browser"
-        ? "Turn off accessibility frames to use 3D"
+        : mjpegUrl
-        : null;
+          ? "3D requires the H.264 stream"
+          : props.axOverlay
+            ? "Turn off accessibility frames to use 3D"
+            : null;
   const keyboardSource = deviceKeyboard(props.platform, props.deviceName ?? "");
   const resetView = useCallback(() => {
-    const orientation = keyboardAttached ? "landscape_right" : "portrait";
+    if (!isDuo) {
-    if (screen?.orientation !== orientation) clientRef.current?.setOrientation(orientation);
+      const orientation = keyboardAttached ? "landscape_right" : "portrait";
+      if (screen?.orientation !== orientation) clientRef.current?.setOrientation(orientation);
+    }
     resetViewRef.current?.();
-  }, [keyboardAttached, screen?.orientation]);
+  }, [isDuo, keyboardAttached, screen?.orientation]);
   const profile = resolveDeviceShape({
     platform: props.platform,
     name: props.deviceName ?? "",
 @@ -309,6 +329,18 @@ export function DeviceStreamView(props: {
             phone: !!showPhone,
             streaming: status === "streaming",
             phoneUnavailableReason,
+            foldingControls:
+              showPhone && isDuo && screen?.supportsHingeAngle ? (
+                <DeviceDuoControls
+                  screen={screen}
+                  state={duoControl}
+                  enabled={inputState.connected}
+                  onCommand={(command) => {
+                    cancelPhoneInput();
+                    clientRef.current?.controlDuo(command);
+                  }}
+                />
+              ) : null,
             keyboard:
               showPhone && keyboardSource
                 ? {
 @@ -413,7 +445,22 @@ export function DeviceStreamView(props: {
             </div>
           ) : null}
         </div>
-        {showPhone ? (
+        {showPhone && isDuo && model ? (
+          <DeviceDuoViewport
+            onFrameListener={onFrameListener}
+            model={model}
+            controlError={duoControl.error}
+            hingePreview={
+              duoControl.requested?.control === "angle" ? duoControl.requested.value : null
+            }
+            source={canvasRef}
+            client={clientRef}
+            onInputCancel={onInputCancel}
+            onResetReady={onResetReady}
+            screen={screen}
+            onUnavailable={onPhoneUnavailable}
+          />
+        ) : showPhone ? (
           <DevicePhoneViewport
             profile={profile}
             model={deviceModel(props.platform, props.deviceName ?? "")}
 @@ -433,16 +480,8 @@ export function DeviceStreamView(props: {
               variant={showPhone ? "secondary" : "ghost"}
               size="xs"
               aria-pressed={!!showPhone}
-              disabled={phoneUnavailable || !!mjpegUrl || !!props.axOverlay}
+              disabled={!!phoneUnavailableReason}
-              title={
+              title={phoneUnavailableReason ?? "Show 3D phone"}
-                phoneUnavailable
-                  ? "3D is unavailable on this browser"
-                  : mjpegUrl
-                    ? "3D requires the H.264 stream"
-                    : props.axOverlay
-                      ? "Turn off accessibility frames to use 3D"
-                      : "Show 3D phone"
-              }
               onClick={() => setPresentation("phone")}
             >
               3D

apps/web/src/components/device/deviceModels.ts+2

apps/web/src/components/device/models/iphone-duo.glb created

Binary files /dev/null and b/apps/web/src/components/device/models/iphone-duo.glb differ

apps/web/src/components/device/models/sources.json+9-1

 @@ -6,7 +6,7 @@
     "gltfTransform": "4.2.1",
     "changes": [
       "Extracted device bodies and a separate Magic Keyboard accessory; excluded unrelated parts and presentation duplicates.",
-      "Normalized portrait axes, centered the screen and uniformly scaled its height to 2.2 scene units.",
+      "Normalized conventional device portrait axes, centered their screen and uniformly scaled its height to 2.2 scene units. The Duo retains its upstream centimeter hinge rig.",
       "Replaced the baked marketing screen with a named placeholder for the live framebuffer.",
       "Resized textures to at most 1024 pixels and encoded them as WebP; retained body geometry."
     ]
 @@ -44,6 +44,14 @@
       "bodyNode": "zRrSLDpdYmKeRJQ",
       "screenNode": "lsDiIbtoSGSmWWZ",
       "accessoryNode": "PoBqSMmyhhcJsBX"
+    },
+    {
+      "id": "iphone-duo",
+      "file": "iphone-duo.glb",
+      "sourceUrl": "https://www.apple.com/105/media/us/iphone-duo/2026/9305e4b9-72d9-4c05-9381-b572adadd5e5/ar/iPhone_Duo_e-sim_Star-White_Variant.usdz",
+      "sourceSha256": "5cab2ea636da0bc0b06c8abf4809843498f7c1d680042b4f95e383b5c6a718b2",
+      "upstreamConversionCommit": "bb265b11c13b395e5302d121458e2d42224a2e9f",
+      "conversion": "Preserved the centimeter split hinge rig and three display meshes from serve-sim; resized textures to 1024 pixels and encoded them as WebP without simplifying or flattening the rig."
     }
   ]
 }

docs/user/devices.md+6

...	@@ -39,6 +39,12 @@ iOS, and power off. Close the tab to stop watching; the device keeps running	...	@@ -39,6 +39,12 @@ iOS, and power off. Close the tab to stop watching; the device keeps running
39	unless you power it off. Closed tabs stay closed after a reload. To watch the	39	unless you power it off. Closed tabs stay closed after a reload. To watch the
40	device again, choose it from + → Device.	40	device again, choose it from + → Device.
41		41
		42	Choose 3D view to inspect supported devices while the live screen stays
		43	interactive. On iPhone Duo, use the fold and stance controls to change its
		44	physical pose, or pinch over the device to adjust the hinge. Turning the model
		45	to the other screen switches the live display and touch input to that screen.
		46	Restore 3D view returns the device to a screen-facing position.
		47
42	## Tools	48	## Tools
43		49
44	The toolbar's Tools button opens a drawer for the open device. It shows the	50	The toolbar's Tools button opens a drawer for the open device. It shows the

packages/client-runtime/package.json+8

...	@@ -290,6 +290,14 @@	...	@@ -290,6 +290,14 @@
290	"./device/model": {	290	"./device/model": {
291	"types": "./src/device/model.ts",	291	"types": "./src/device/model.ts",
292	"default": "./src/device/model.ts"	292	"default": "./src/device/model.ts"
		293	},
		294	"./device/duo-viewer": {
		295	"types": "./src/device/duoViewer.ts",
		296	"default": "./src/device/duoViewer.ts"
		297	},
		298	"./device/duo-control": {
		299	"types": "./src/device/duoControl.ts",
		300	"default": "./src/device/duoControl.ts"
293	}	301	}
294	},	302	},
295	"scripts": {	303	"scripts": {

packages/client-runtime/src/device/duoControl.test.ts created+109

...	@@ -0,0 +1,109 @@
1	import { afterEach, expect, it, vi } from "vite-plus/test";
2	import { createDuoControl, createDuoPinch, type DuoCommand } from "./duoControl.ts";
3	afterEach(() => vi.useRealTimers());
4
5	it("keeps a failed send visible, including a disconnect while draining queued motion", () => {
6	const send = vi.fn((_request: { requestId: number; command: DuoCommand }) => false);
7	const onChange = vi.fn();
8	const queue = createDuoControl({ send, onChange });
9	queue.enqueue({ control: "angle", value: 40 });
10	expect(onChange).toHaveBeenLastCalledWith({
11	pending: false,
12	requested: null,
13	error: "Device is disconnected.",
14	});
15	send.mockReturnValueOnce(true);
16	queue.enqueue({ control: "pose", value: "book" });
17	queue.enqueue({ control: "angle", value: 60 });
18	queue.receive({ requestId: 2, ok: true });
19	expect(onChange).toHaveBeenLastCalledWith({
20	pending: false,
21	requested: null,
22	error: "Device is disconnected.",
23	});
24	});
25
26	it("coalesces hinge edits behind acknowledgements and lets a preset replace queued edits", () => {
27	const send = vi.fn((_request: { requestId: number; command: DuoCommand }) => true);
28	const onChange = vi.fn();
29	const queue = createDuoControl({ send, onChange });
30	queue.enqueue({ control: "angle", value: 40 });
31	queue.enqueue({ control: "angle", value: 50 });
32	queue.enqueue({ control: "angle", value: 60 });
33	expect(send).toHaveBeenCalledTimes(1);
34	queue.receive({ requestId: 1, ok: true });
35	expect(send.mock.calls[1]?.[0]).toEqual({
36	requestId: 2,
37	command: { control: "angle", value: 60 },
38	});
39	queue.enqueue({ control: "angle", value: 100 });
40	queue.enqueue({ control: "pose", value: "tent" });
41	queue.receive({ requestId: 2, ok: true });
42	expect(send.mock.calls[2]?.[0]).toEqual({
43	requestId: 3,
44	command: { control: "pose", value: "tent" },
45	});
46	queue.receive({ requestId: 3, ok: true });
47	expect(onChange).toHaveBeenLastCalledWith({ pending: false, requested: null, error: null });
48	queue.clear();
49	});
50
51	it("drops queued commands on failure, timeout and disconnect; late replies cannot acknowledge later work", () => {
52	vi.useFakeTimers();
53	const send = vi.fn((_request: { requestId: number; command: DuoCommand }) => true);
54	const onChange = vi.fn();
55	const queue = createDuoControl({ send, onChange, timeoutMs: 100 });
56	queue.enqueue({ control: "angle", value: 90 });
57	queue.enqueue({ control: "angle", value: 100 });
58	vi.advanceTimersByTime(100);
59	expect(onChange.mock.lastCall?.[0].error).toContain("timed out");
60	queue.enqueue({ control: "pose", value: "open" });
61	queue.receive({ requestId: 1, ok: true });
62	expect(onChange.mock.lastCall?.[0].pending).toBe(true);
63	queue.enqueue({ control: "angle", value: 130 });
64	queue.receive({ requestId: 2, ok: false, error: "native refused" });
65	expect(onChange.mock.lastCall?.[0]).toEqual({
66	pending: false,
67	requested: null,
68	error: "native refused",
69	});
70	queue.enqueue({ control: "pose", value: "book" });
71	queue.enqueue({ control: "pose", value: "closed" });
72	queue.clear();
73	queue.receive({ requestId: 3, ok: true });
74	expect(send).toHaveBeenCalledTimes(3);
75	queue.enqueue({ control: "angle", value: Infinity });
76	expect(send).toHaveBeenCalledTimes(3);
77	queue.clear();
78	});
79
80	it("pinches only a hit device, accumulates independently of native readback, clamps and cancels", () => {
81	let confirmed = 90;
82	const change = vi.fn();
83	const pinch = createDuoPinch({
84	angle: () => confirmed,
85	contains: (x, y) => x > 0.2 && y > 0.2,
86	change,
87	});
88	expect(pinch.begin(0.1, 0.5)).toBe(false);
89	pinch.move(1);
90	expect(change).not.toHaveBeenCalled();
91	expect(pinch.begin(0.5, 0.5)).toBe(true);
92	pinch.move(0.25);
93	expect(change).toHaveBeenLastCalledWith(120);
94	confirmed = 100;
95	pinch.move(0.25);
96	expect(change).toHaveBeenLastCalledWith(150);
97	pinch.move(2);
98	expect(change).toHaveBeenLastCalledWith(180);
99	pinch.move(-3);
100	expect(change).toHaveBeenLastCalledWith(0);
101	pinch.move(NaN);
102	pinch.end();
103	expect(change).toHaveBeenLastCalledWith(null);
104	const count = change.mock.calls.length;
105	pinch.move(1);
106	pinch.end();
107	expect(change).toHaveBeenCalledTimes(count);
108	expect(pinch.active).toBe(false);
109	});

packages/client-runtime/src/device/duoControl.ts created+113

...	@@ -0,0 +1,113 @@
1	// @effect-diagnostics globalTimers:off - The stream owns this browser control queue and its timeout.
2	export const DUO_POSES = [
3	{ id: "closed", label: "Closed", angle: 0 },
4	{ id: "book", label: "Book", angle: 90 },
5	{ id: "open", label: "Open", angle: 180 },
6	{ id: "laptop", label: "Laptop", angle: 90 },
7	{ id: "tent", label: "Tent", angle: 80 },
8	] as const;
9	export type DuoPose = (typeof DUO_POSES)[number]["id"];
10	export type DuoOrientation =
11	\| "portrait"
12	\| "landscape_left"
13	\| "portrait_upside_down"
14	\| "landscape_right";
15	export type DuoCommand =
16	\| { control: "angle"; value: number }
17	\| { control: "pose"; value: DuoPose }
18	\| { control: "table"; value: boolean }
19	\| { control: "physical"; value: "faceup" \| "facedown" }
20	\| { control: "orientation"; value: DuoOrientation };
21	export type DuoControlState = {
22	pending: boolean;
23	requested: DuoCommand \| null;
24	error: string \| null;
25	};
26
27	/** One in-flight native transaction. Hinge motion coalesces; presets replace queued motion. Nothing replays after reconnect. */
28	export function createDuoControl(options: {
29	send: (request: { requestId: number; command: DuoCommand }) => boolean;
30	onChange: (state: DuoControlState) => void;
31	timeoutMs?: number;
32	}) {
33	let nextId = 1;
34	let active: { requestId: number; command: DuoCommand } \| null = null;
35	let queued: DuoCommand \| null = null;
36	let timer: ReturnType<typeof setTimeout> \| null = null;
37	const publish = (error: string \| null = null) =>
38	options.onChange({ pending: !!active, requested: queued ?? active?.command ?? null, error });
39	const clear = (error: string \| null = null) => {
40	if (timer) clearTimeout(timer);
41	timer = null;
42	active = null;
43	queued = null;
44	publish(error);
45	};
46	const drain = () => {
47	if (active \|\| !queued) return;
48	active = { requestId: nextId++, command: queued };
49	queued = null;
50	const id = active.requestId;
51	timer = setTimeout(() => {
52	if (active?.requestId === id) clear("Device control timed out. Its position is unknown.");
53	}, options.timeoutMs ?? 5_000);
54	publish();
55	if (!options.send(active)) clear("Device is disconnected.");
56	};
57	return {
58	enqueue(command: DuoCommand) {
59	if (
60	command.control === "angle" &&
61	(!Number.isFinite(command.value) \|\| command.value < 0 \|\| command.value > 180)
62	)
63	return;
64	queued = command;
65	if (active) publish();
66	else drain();
67	},
68	receive(reply: { requestId: number; ok: boolean; error?: string }) {
69	if (!active \|\| reply.requestId !== active.requestId) return;
70	if (timer) clearTimeout(timer);
71	timer = null;
72	active = null;
73	if (!reply.ok) {
74	clear(reply.error ?? "Device control failed. Its position is unknown.");
75	return;
76	}
77	if (queued) drain();
78	else publish();
79	},
80	clear,
81	};
82	}
83
84	/** A pinch keeps its own accumulator across asynchronous native acknowledgements. */
85	export function createDuoPinch(options: {
86	angle: () => number;
87	contains: (x: number, y: number) => boolean;
88	change: (angle: number \| null) => void;
89	}) {
90	let angle: number \| null = null;
91	return {
92	begin(x: number, y: number) {
93	if (!options.contains(x, y)) return false;
94	angle = Math.max(0, Math.min(180, options.angle()));
95	return true;
96	},
97	move(logScale: number) {
98	if (angle === null \|\| !Number.isFinite(logScale)) return;
99	const next = Math.max(0, Math.min(180, angle + logScale * 120));
100	if (next === angle) return;
101	angle = next;
102	options.change(next);
103	},
104	end() {
105	if (angle === null) return;
106	angle = null;
107	options.change(null);
108	},
109	get active() {
110	return angle !== null;
111	},
112	};
113	}

packages/client-runtime/src/device/duoScene.test.ts created+108

...	@@ -0,0 +1,108 @@
1	import {
2	BoxGeometry,
3	Group,
4	Mesh,
5	MeshBasicMaterial,
6	PerspectiveCamera,
7	PlaneGeometry,
8	Texture,
9	Vector3,
10	} from "three";
11	import { expect, it } from "vite-plus/test";
12	import { createDuoScene, duoDisplayKey, duoRawPoint } from "./duoScene.ts";
13
14	function fixture() {
15	const asset = new Group();
16	const left = new Group();
17	left.name = "left-half";
18	const right = new Group();
19	right.name = "right-half";
20	const screen = (name: string, x: number, z: number, rear = false) => {
21	const mesh = new Mesh(new PlaneGeometry(1, 2), new MeshBasicMaterial());
22	if (rear) mesh.geometry.rotateY(Math.PI);
23	mesh.geometry.translate(x, 0, z);
24	mesh.name = name;
25	return mesh;
26	};
27	const innerLeft = screen("inner-display-left", -0.5, 0.06);
28	const innerRight = screen("inner-display-right", 0.5, 0.06);
29	const cover = screen("cover-display", -0.5, -0.06, true);
30	left.add(innerLeft, cover);
31	right.add(innerRight);
32	for (const [group, x] of [
33	[left, -0.5],
34	[right, 0.5],
35	] as const) {
36	const body = new Mesh(new BoxGeometry(1, 2, 0.1), new MeshBasicMaterial());
37	body.geometry.translate(x, 0, 0);
38	group.add(body);
39	}
40	asset.add(left, right);
41	const scene = createDuoScene(asset, { 1: new Texture(), 3: new Texture() });
42	const camera = new PerspectiveCamera(36, 1, 0.1, 50);
43	camera.position.z = 6;
44	camera.updateMatrixWorld();
45	return { scene, camera, left, right, innerLeft, innerRight };
46	}
47
48	it("keeps one continuous inner UV map across the hinge and folds opposite leaves", () => {
49	const { scene, left, right, innerLeft, innerRight } = fixture();
50	scene.setAngle(90);
51	expect(left.rotation.y).toBeCloseTo(Math.PI / 4);
52	expect(right.rotation.y).toBeCloseTo(-Math.PI / 4);
53	const values = (mesh: Mesh) =>
54	Array.from({ length: mesh.geometry.getAttribute("uv").count }, (_, i) =>
55	mesh.geometry.getAttribute("uv").getX(i),
56	);
57	expect(Math.max(...values(innerLeft))).toBe(0.5);
58	expect(Math.min(...values(innerRight))).toBe(0.5);
59	scene.dispose();
60	});
61
62	it("derives resting views from the hinged display planes independently of the inspection orbit", () => {
63	const { scene } = fixture();
64	scene.setAngle(90);
65	const frames = scene.restFrames(3);
66	expect(frames.map((frame) => frame.face)).toEqual(["inside", "left", "right"]);
67	expect(frames[1]!.normal.x).toBeGreaterThan(0.5);
68	expect(frames[2]!.normal.x).toBeLessThan(-0.5);
69	scene.root.rotation.set(0.5, 1.2, -0.7);
70	scene.root.position.set(3, -2, 1);
71	const rotated = scene.restFrames(3);
72	for (let index = 0; index < frames.length; index++) {
73	expect(rotated[index]!.normal.distanceTo(frames[index]!.normal)).toBeLessThan(1e-6);
74	expect(rotated[index]!.center.distanceTo(frames[index]!.center)).toBeLessThan(1e-6);
75	}
76	scene.setAngle(180);
77	expect(scene.restFrames(3).map((frame) => frame.face)).toEqual(["inside"]);
78	expect(scene.restFrames(1).map((frame) => frame.face)).toEqual(["cover"]);
79	scene.dispose();
80	});
81
82	it("maps active display input through hardware mounting and blocks rear, inactive and stale surfaces", () => {
83	const { scene, camera, innerLeft } = fixture();
84	scene.setAngle(180);
85	const screen = { width: 2007, height: 2853, orientation: "portrait" as const, screenId: 3 };
86	const key = duoDisplayKey(screen);
87	scene.root.updateMatrixWorld();
88	const world = innerLeft.localToWorld(new Vector3(-0.5, -0.5, 0.06)).project(camera);
89	const x = (world.x + 1) / 2,
90	y = (1 - world.y) / 2;
91	const hit = scene.screenPoint(x, y, camera, screen, key);
92	expect(hit?.x).toBeCloseTo(0.75);
93	expect(hit?.y).toBeCloseTo(0.75);
94	expect(
95	scene.screenPoint(
96	x,
97	y,
98	camera,
99	{ ...screen, screenId: 1 },
100	duoDisplayKey({ ...screen, screenId: 1 }),
101	),
102	).toBeNull();
103	expect(scene.screenPoint(x, y, camera, screen, "old")).toBeNull();
104	expect(scene.screenPoint(x, y, camera, screen, key)).not.toBeNull();
105	expect(scene.screenPoint(x, 1.1, camera, screen, key, true)?.x).toBeGreaterThan(1);
106	expect(duoRawPoint(1, 0.2, 0.7)).toEqual({ x: 0.2, y: 0.7 });
107	scene.dispose();
108	});

packages/client-runtime/src/device/duoScene.ts created+244

...	@@ -0,0 +1,244 @@
1	import {
2	Box3,
3	BufferAttribute,
4	Group,
5	Mesh,
6	MeshBasicMaterial,
7	Matrix4,
8	Quaternion,
9	Plane,
10	Raycaster,
11	Triangle,
12	Vector2,
13	Vector3,
14	type PerspectiveCamera,
15	type Texture,
16	} from "three";
17	import type { DuoRestFrame } from "./duoSnap.ts";
18	import type { DeviceScreenSize } from "./stream.ts";
19
20	export type DuoPanelId = 1 \| 3;
21	export type DuoHingeLeaf = "left" \| "right";
22	export type DuoFrameLayout = { width: number; height: number };
23
24	/** Hardware mounting is independent of app orientation and the model's orbit. The inverse is shared by taps and drags. */
25	export function duoRawPoint(panel: DuoPanelId, x: number, y: number) {
26	return panel === 1 ? { x, y } : { x: y, y: 1 - x };
27	}
28	export function duoFrameMatches(frame: DuoFrameLayout, screen: DeviceScreenSize) {
29	return (
30	frame.width > 0 &&
31	frame.height > 0 &&
32	Math.abs(frame.width / frame.height - screen.width / screen.height) <=
33	1 / frame.height + 1 / screen.height
34	);
35	}
36	export function duoDisplayKey(screen: DeviceScreenSize \| null) {
37	return screen ? `${screen.screenId}:${screen.width}:${screen.height}:${screen.orientation}` : "";
38	}
39
40	/** Asset resources belong to its model slot; this scene owns only live-screen replacement materials. */
41	export function createDuoScene(asset: Group, textures: Record<DuoPanelId, Texture>) {
42	const left = asset.getObjectByName("left-half");
43	const right = asset.getObjectByName("right-half");
44	const cover = asset.getObjectByName("cover-display");
45	const innerLeft = asset.getObjectByName("inner-display-left");
46	const innerRight = asset.getObjectByName("inner-display-right");
47	if (
48	!left \|\|
49	!right \|\|
50	!(cover instanceof Mesh) \|\|
51	!(innerLeft instanceof Mesh) \|\|
52	!(innerRight instanceof Mesh)
53	)
54	throw new Error("Duo model must have two hinge groups and all three display meshes");
55	const surfaces = { 1: [cover], 3: [innerLeft, innerRight] };
56	const boundsByPanel = ([1, 3] as const).map((id) => {
57	const bounds = new Box3();
58	for (const mesh of surfaces[id]) {
59	mesh.geometry.computeBoundingBox();
60	bounds.union(mesh.geometry.boundingBox!);
61	}
62	const size = bounds.getSize(new Vector3());
63	if (![size.x, size.y].every((value) => Number.isFinite(value) && value > 0))
64	throw new Error("Invalid Duo display bounds");
65	return { bounds, size };
66	});
67	const root = new Group();
68	const content = new Group();
69	content.add(asset);
70	root.add(content);
71	let currentAngle = Number.NaN;
72	const localBounds = new Box3();
73	const relative = new Matrix4();
74	const inverse = new Matrix4();
75	const materials = {
76	1: new MeshBasicMaterial({ map: textures[1], toneMapped: false }),
77	3: new MeshBasicMaterial({ map: textures[3], toneMapped: false }),
78	};
79	const originals = new Map<Mesh, Mesh["material"]>();
80	// The two inner meshes sample halves of one framebuffer. Cover UVs face outward on the rear leaf.
81	for (const id of [1, 3] as const) {
82	const { bounds, size } = boundsByPanel[id === 1 ? 0 : 1]!;
83	for (const mesh of surfaces[id]) {
84	const position = mesh.geometry.getAttribute("position");
85	const uv = new Float32Array(position.count * 2);
86	for (let i = 0; i < position.count; i++) {
87	const x = (position.getX(i) - bounds.min.x) / size.x;
88	uv[i * 2] = id === 1 ? 1 - x : x;
89	uv[i * 2 + 1] = (position.getY(i) - bounds.min.y) / size.y;
90	}
91	mesh.geometry.setAttribute("uv", new BufferAttribute(uv, 2));
92	originals.set(mesh, mesh.material);
93	mesh.material = materials[id];
94	}
95	}
96	const ray = new Raycaster();
97	let captured: {
98	panel: DuoPanelId;
99	plane: Plane;
100	triangle: Triangle;
101	uv: [Vector2, Vector2, Vector2];
102	key: string;
103	} \| null = null;
104	return {
105	root,
106	setAngle(angle: number) {
107	if (angle === currentAngle) return;
108	currentAngle = angle;
109	const radians = ((180 - Math.min(180, Math.max(0, angle))) * Math.PI) / 360;
110	left.rotation.y = radians;
111	right.rotation.y = -radians;
112	// Orbit the folded body's center, while retaining the authored hinge pivots.
113	content.position.set(0, 0, 0);
114	root.updateWorldMatrix(true, true);
115	inverse.copy(root.matrixWorld).invert();
116	localBounds.makeEmpty();
117	asset.traverse((object) => {
118	if (!(object instanceof Mesh)) return;
119	if (!object.geometry.boundingBox) object.geometry.computeBoundingBox();
120	if (!object.geometry.boundingBox) return;
121	relative.multiplyMatrices(inverse, object.matrixWorld);
122	localBounds.union(object.geometry.boundingBox.clone().applyMatrix4(relative));
123	});
124	content.position.copy(localBounds.getCenter(new Vector3())).negate();
125	},
126	restFrames(panel: DuoPanelId): DuoRestFrame[] {
127	root.updateWorldMatrix(true, true);
128	inverse.copy(root.matrixWorld).invert();
129	const frame = (meshes: Mesh[], face: DuoRestFrame["face"]): DuoRestFrame => {
130	const bounds = new Box3();
131	const normal = new Vector3();
132	const up = new Vector3();
133	for (const mesh of meshes) {
134	if (!mesh.geometry.boundingBox) mesh.geometry.computeBoundingBox();
135	relative.multiplyMatrices(inverse, mesh.matrixWorld);
136	bounds.union(mesh.geometry.boundingBox!.clone().applyMatrix4(relative));
137	normal.add(
138	new Vector3()
139	.fromBufferAttribute(mesh.geometry.getAttribute("normal"), 0)
140	.transformDirection(relative),
141	);
142	up.add(new Vector3(0, 1, 0).transformDirection(relative));
143	}
144	return {
145	face,
146	center: bounds.getCenter(new Vector3()),
147	normal: normal.normalize(),
148	up: up.normalize(),
149	};
150	};
151	if (panel === 1) return [frame([cover], "cover")];
152	const inside = frame([innerLeft, innerRight], "inside");
153	// Leaf views only make sense when both displays form a useful open fold.
154	return currentAngle > 20 && currentAngle < 165
155	? [inside, frame([innerLeft], "left"), frame([innerRight], "right")]
156	: [inside];
157	},
158	leafRotation(leaf: DuoHingeLeaf) {
159	root.updateWorldMatrix(true, true);
160	relative.multiplyMatrices(
161	root.matrixWorld.clone().invert(),
162	(leaf === "left" ? left : right).matrixWorld,
163	);
164	return new Quaternion().setFromRotationMatrix(relative.extractRotation(relative));
165	},
166	hingeLeafAt(x: number, y: number, camera: PerspectiveCamera): DuoHingeLeaf \| null {
167	if (!Number.isFinite(x) \|\| !Number.isFinite(y)) return null;
168	root.updateMatrixWorld(true);
169	camera.updateMatrixWorld(true);
170	ray.setFromCamera(new Vector2(x * 2 - 1, 1 - y * 2), camera);
171	let object = ray.intersectObject(root, true)[0]?.object;
172	while (object) {
173	if (object === left) return "left";
174	if (object === right) return "right";
175	object = object.parent ?? undefined;
176	}
177	return null;
178	},
179	cancelInput() {
180	captured = null;
181	},
182	screenPoint(
183	x: number,
184	y: number,
185	camera: PerspectiveCamera,
186	screen: DeviceScreenSize \| null,
187	readyKey: string,
188	extend = false,
189	) {
190	const key = duoDisplayKey(screen);
191	if (!screen \|\| !readyKey \|\| key !== readyKey) {
192	captured = null;
193	return null;
194	}
195	root.updateMatrixWorld(true);
196	camera.updateMatrixWorld(true);
197	ray.setFromCamera(new Vector2(x * 2 - 1, 1 - y * 2), camera);
198	if (extend && captured?.key === key) {
199	const point = ray.ray.intersectPlane(captured.plane, new Vector3());
200	const bary = point ? captured.triangle.getBarycoord(point, new Vector3()) : null;
201	if (!bary) return null;
202	const uv = captured.uv[0]
203	.clone()
204	.multiplyScalar(bary.x)
205	.addScaledVector(captured.uv[1], bary.y)
206	.addScaledVector(captured.uv[2], bary.z);
207	return duoRawPoint(captured.panel, uv.x, 1 - uv.y);
208	}
209	// The chassis occludes rear displays. Only the first visible hit can own a contact.
210	const hit = ray.intersectObject(root, true)[0];
211	if (!hit?.uv \|\| !hit.face \|\| !(hit.object instanceof Mesh)) return null;
212	const panel: DuoPanelId \| null = surfaces[1].includes(hit.object)
213	? 1
214	: surfaces[3].includes(hit.object)
215	? 3
216	: null;
217	if (!panel \|\| panel !== screen.screenId) return null;
218	const indices = [hit.face.a, hit.face.b, hit.face.c] as const;
219	const positions = hit.object.geometry.getAttribute("position");
220	const triangle = new Triangle(
221	...(indices.map((index) =>
222	new Vector3().fromBufferAttribute(positions, index).applyMatrix4(hit.object.matrixWorld),
223	) as [Vector3, Vector3, Vector3]),
224	);
225	const hitUvs = hit.object.geometry.getAttribute("uv");
226	const uv = indices.map((index) => new Vector2().fromBufferAttribute(hitUvs, index)) as [
227	Vector2,
228	Vector2,
229	Vector2,
230	];
231	captured = { panel, triangle, uv, plane: triangle.getPlane(new Plane()), key };
232	return duoRawPoint(panel, hit.uv.x, 1 - hit.uv.y);
233	},
234	dispose() {
235	for (const [mesh, material] of originals) mesh.material = material;
236	for (const material of Object.values(materials)) {
237	material.map = null;
238	material.dispose();
239	}
240	content.remove(asset);
241	captured = null;
242	},
243	};
244	}

packages/client-runtime/src/device/duoSnap.test.ts created+54

...	@@ -0,0 +1,54 @@
1	import { expect, it } from "vite-plus/test";
2	import { Quaternion, Vector3 } from "three";
3	import { duoViewSnaps, nearestDuoView } from "./duoSnap.ts";
4
5	const frames = [
6	{
7	face: "inside" as const,
8	normal: new Vector3(0, 0, 1),
9	up: new Vector3(0, 1, 0),
10	center: new Vector3(),
11	},
12	];
13	const rotation = (x: number, y: number, z = 0) =>
14	new Quaternion().setFromAxisAngle(new Vector3(x, y, z).normalize(), Math.hypot(x, y, z));
15
16	it("chooses a leaf-focused seated view when it is closer than the middle of the fold", () => {
17	const leaf = {
18	face: "right" as const,
19	normal: new Vector3(0.6, 0, 0.8),
20	up: new Vector3(0, 1, 0),
21	center: new Vector3(1, 0, 0),
22	};
23	const candidates = duoViewSnaps([...frames, leaf], 3);
24	const seat = candidates.find(
25	(candidate) => candidate.face === "right" && candidate.orientation === "portrait",
26	)!;
27	const released = seat.rotation.clone().premultiply(rotation(0.04, 0.05));
28	const chosen = nearestDuoView(released, candidates)!;
29	expect(chosen.face).toBe("right");
30	expect(chosen.orientation).toBe("portrait");
31	expect(chosen.center.x).toBe(1);
32	});
33
34	it("a seated view exposes the base, and upright yaw keeps both inner displays facing the camera", () => {
35	const folded = [
36	frames[0]!,
37	...(["left", "right"] as const).map((face) => ({
38	face,
39	normal: new Vector3(face === "left" ? Math.SQRT1_2 : -Math.SQRT1_2, 0, Math.SQRT1_2),
40	up: new Vector3(0, 1, 0),
41	center: new Vector3(face === "left" ? -1 : 1, 0, 0),
42	})),
43	];
44	const candidates = duoViewSnaps(folded, 3);
45	expect(candidates.filter((candidate) => candidate.face === "right")).toHaveLength(1);
46	for (const candidate of candidates) {
47	for (const frame of folded.slice(1))
48	expect(frame.normal.clone().applyQuaternion(candidate.rotation).z).toBeGreaterThan(0.2);
49	const side = candidate.rotation.clone().premultiply(rotation(0, 1.5));
50	const chosen = nearestDuoView(side, [candidate])!;
51	for (const frame of folded.slice(1))
52	expect(frame.normal.clone().applyQuaternion(chosen.rotation).z).toBeGreaterThan(0.02);
53	}
54	});

packages/client-runtime/src/device/duoSnap.ts created+86

...	@@ -0,0 +1,86 @@
1	import { Matrix4, Quaternion, Vector3 } from "three";
2	import { nearestDeviceView } from "./deviceViewSnap.ts";
3	import type { DeviceScreenSize } from "./stream.ts";
4
5	export type DuoRestFace = "cover" \| "inside" \| "left" \| "right";
6	export type DuoRestFrame = { face: DuoRestFace; normal: Vector3; up: Vector3; center: Vector3 };
7	export type DuoViewSnap = {
8	rotation: Quaternion;
9	face: DuoRestFace;
10	orientation: DeviceScreenSize["orientation"];
11	center: Vector3;
12	yawLimit: number;
13	};
14	const orientations = [
15	"portrait",
16	"landscape_left",
17	"portrait_upside_down",
18	"landscape_right",
19	] as const;
20	const z = new Vector3(0, 0, 1);
21	const y = new Vector3(0, 1, 0);
22
23	/** Build views from the actual hinged display planes, rather than model-specific Euler offsets. */
24	export function duoViewSnaps(frames: readonly DuoRestFrame[], panel: 1 \| 3) {
25	const snaps: DuoViewSnap[] = [];
26	for (const frame of frames) {
27	const normal = frame.normal.clone().normalize();
28	if (normal.lengthSq() < 0.5) continue;
29	const right = frame.up.clone().cross(normal).normalize();
30	const up = normal.clone().cross(right).normalize();
31	const faceRotation = new Quaternion()
32	.setFromRotationMatrix(new Matrix4().makeBasis(right, up, normal))
33	.invert();
34	for (let index = 0; index < orientations.length; index++) {
35	// Only these rolls put the partner below the focused lid. Other quarter
36	// turns belong to the upright, whole-display family.
37	if (frame.face === "right" && index !== 0) continue;
38	if (frame.face === "left" && index !== 2) continue;
39	const roll = (panel === 3 ? Math.PI / 2 : 0) - (index * Math.PI) / 2;
40	const rotation = faceRotation.clone().premultiply(new Quaternion().setFromAxisAngle(z, roll));
41	// A leaf-focused view looks slightly down onto its partner, as in a seated laptop.
42	if (frame.face === "left" \|\| frame.face === "right")
43	rotation.premultiply(new Quaternion().setFromAxisAngle(new Vector3(1, 0, 0), Math.PI / 12));
44	if (
45	panel === 3 &&
46	frames.some(
47	(leaf) =>
48	(leaf.face === "left" \|\| leaf.face === "right") &&
49	leaf.normal.clone().applyQuaternion(rotation).z < 0.04,
50	)
51	)
52	continue;
53	// Bound yaw using both real leaf normals. A narrow fold cannot tolerate
54	// the same side view as a fully open display.
55	let yawLimit = frame.face === "cover" ? Math.PI / 9 : Math.PI / 3;
56	if (panel === 3) {
57	for (const sign of [-1, 1]) {
58	for (let step = 1; step <= 60; step++) {
59	const turn = new Quaternion().setFromAxisAngle(y, (sign * step * Math.PI) / 180);
60	if (
61	frames.some(
62	(leaf) =>
63	(leaf.face === "left" \|\| leaf.face === "right") &&
64	leaf.normal.clone().applyQuaternion(rotation).applyQuaternion(turn).z < 0.04,
65	)
66	) {
67	yawLimit = Math.min(yawLimit, ((step - 1) * Math.PI) / 180);
68	break;
69	}
70	}
71	}
72	}
73	snaps.push({
74	rotation,
75	face: frame.face,
76	orientation: orientations[index]!,
77	center: frame.center.clone(),
78	yawLimit,
79	});
80	}
81	}
82	return snaps;
83	}
84
85	/** Fold-specific candidates use the base viewer's nearest-family selection. */
86	export const nearestDuoView = nearestDeviceView<DuoViewSnap>;

packages/client-runtime/src/device/duoStream.test.ts created+285

...	@@ -0,0 +1,285 @@
1	import { afterEach, expect, it, vi } from "vite-plus/test";
2	import { createDeviceStreamClient, type DeviceScreenSize } from "./stream.ts";
3	afterEach(() => vi.unstubAllGlobals());
4
5	it("switches to fixed authenticated feeds while retaining one HID socket, routes the active panel and rejects superseded decoded output", async () => {
6	const feeds: {
7	url: string;
8	signal: AbortSignal;
9	controller: ReadableStreamDefaultController<Uint8Array>;
10	}[] = [];
11	const outputs: VideoFrameOutputCallback[] = [];
12	const errors: VideoDecoderInit["error"][] = [];
13	let supported = true;
14	let panelHttpStatus = 200;
15	let expected = 0;
16	let decoderReady = () => {};
17	const waitDecoders = (count: number) => {
18	expected = count;
19	return new Promise<void>((resolve) => {
20	decoderReady = resolve;
21	});
22	};
23	class Decoder {
24	static isConfigSupported = async () => ({ supported });
25	static instances: Decoder[] = [];
26	state = "unconfigured";
27	constructor(options: VideoDecoderInit) {
28	outputs.push(options.output);
29	errors.push(options.error);
30	Decoder.instances.push(this);
31	if (outputs.length === expected) decoderReady();
32	}
33	configure() {
34	this.state = "configured";
35	}
36	close() {
37	this.state = "closed";
38	}
39	}
40	let socketReady = () => {};
41	const socketConstructed = new Promise<void>((resolve) => {
42	socketReady = resolve;
43	});
44	class Socket {
45	static OPEN = 1;
46	static instances: Socket[] = [];
47	readyState = 1;
48	binaryType = "arraybuffer";
49	onopen?: () => void;
50	onmessage?: (event: { data: ArrayBuffer }) => void;
51	onclose?: (event: { code: number; reason: string }) => void;
52	send = vi.fn();
53	close = vi.fn();
54	constructor() {
55	Socket.instances.push(this);
56	socketReady();
57	}
58	}
59	vi.stubGlobal("VideoDecoder", Decoder);
60	vi.stubGlobal("EncodedVideoChunk", vi.fn());
61	vi.stubGlobal("WebSocket", Socket);
62	vi.stubGlobal(
63	"fetch",
64	vi.fn(async (url: string, options: { signal: AbortSignal }) => {
65	if (!url.includes("avcc")) return new Response("prime");
66	if (url.includes("/panel/") && panelHttpStatus !== 200)
67	return new Response("unsupported panel", { status: panelHttpStatus });
68	const stream = new ReadableStream<Uint8Array>({
69	start(controller) {
70	feeds.push({ url, signal: options.signal, controller });
71	options.signal.addEventListener("abort", () =>
72	controller.error(new DOMException("Aborted", "AbortError")),
73	);
74	},
75	});
76	return new Response(stream);
77	}),
78	);
79	const present = vi.fn((_source: CanvasImageSource, _width: number, _height: number) => true),
80	cover = vi.fn((_source: CanvasImageSource, _width: number, _height: number) => true),
81	inner = vi.fn((_source: CanvasImageSource, _width: number, _height: number) => true);
82	let panelFailed = () => {};
83	const panelFailure = new Promise<void>((resolve) => {
84	panelFailed = resolve;
85	});
86	const onStatus = vi.fn();
87	const onDuoControl = vi.fn();
88	const onDuoUnavailable = vi.fn((_detail?: string) => panelFailed());
89	const client = createDeviceStreamClient(
90	{
91	platform: "ios",
92	deviceId: "duo",
93	access: {
94	httpBase: "https://t3.test/api/device-hub",
95	wsBase: "wss://t3.test/api/device-hub",
96	credentials: true,
97	query: { hostId: "remote", wsTicket: "ticket" },
98	},
99	},
100	{ present },
101	{
102	onStatus,
103	onDuoControl,
104	onDuoUnavailable,
105	onScreen: vi.fn(),
106	onInputConnected: vi.fn(),
107	onUnauthorized: vi.fn(),
108	onMjpegFallback: vi.fn(),
109	},
110	);
111	client.start();
112	await socketConstructed;
113	const ws = Socket.instances[0]!;
114	const config = (
115	id: number,
116	orientation: DeviceScreenSize["orientation"] = "portrait",
117	physical = false,
118	) => {
119	const json = new TextEncoder().encode(
120	JSON.stringify({
121	width: id === 1 ? 1398 : 2007,
122	height: id === 1 ? 2034 : 2853,
123	orientation,
124	screenId: id,
125	supportsHingeAngle: true,
126	supportsPhysicalOrientation: physical,
127	hingePose: "open",
128	}),
129	);
130	const packet = new Uint8Array(json.length + 1);
131	packet[0] = 0x82;
132	packet.set(json, 1);
133	ws.onmessage?.({ data: packet.buffer });
134	};
135	config(3);
136	const requestedOrientation = () =>
137	JSON.parse(new TextDecoder().decode(ws.send.mock.lastCall?.[0].subarray(1))).orientation;
138	client.rotate();
139	expect(requestedOrientation()).toBe("landscape_left");
140	config(3); // An orientation-locked app keeps its framebuffer orientation after the sensor rotates.
141	expect(onDuoControl.mock.lastCall?.[0].error).toBeNull();
142	client.rotate();
143	expect(requestedOrientation()).toBe("portrait_upside_down");
144	config(3, "portrait_upside_down");
145	client.rotate();
146	expect(requestedOrientation()).toBe("landscape_right");
147	config(3); // An external native orientation change becomes authoritative again.
148	client.rotate();
149	expect(requestedOrientation()).toBe("landscape_left");
150	config(3, "landscape_left");
151	expect(onDuoControl.mock.lastCall?.[0].pending).toBe(false);
152	client.controlDuo({ control: "angle", value: 40 });
153	const angleRequest = JSON.parse(new TextDecoder().decode(ws.send.mock.lastCall?.[0].subarray(1)));
154	const before = ws.send.mock.calls.length;
155	client.controlDuo({ control: "orientation", value: "portrait" });
156	expect(ws.send.mock.calls.length).toBe(before);
157	config(3, "landscape_left"); // The angle's config precedes its receipt; it cannot acknowledge a queued rotation.
158	const reply = new TextEncoder().encode(
159	JSON.stringify({ requestId: angleRequest.requestId, ok: true }),
160	);
161	const receipt = new Uint8Array(reply.length + 1);
162	receipt[0] = 0x90;
163	receipt.set(reply, 1);
164	ws.onmessage?.({ data: receipt.buffer });
165	expect(requestedOrientation()).toBe("portrait");
166	expect(ws.send.mock.lastCall?.[0][0]).toBe(0x07);
167	client.controlDuo({ control: "angle", value: 55 });
168	const orientationSends = ws.send.mock.calls.length;
169	expect(onDuoControl.mock.lastCall?.[0].pending).toBe(true);
170	config(3);
171	expect(ws.send.mock.calls.length).toBe(orientationSends + 1);
172	const after = JSON.parse(new TextDecoder().decode(ws.send.mock.lastCall?.[0].subarray(1)));
173	expect(after.command).toEqual({ control: "angle", value: 55 });
174	const finalReply = new TextEncoder().encode(
175	JSON.stringify({ requestId: after.requestId, ok: true }),
176	);
177	const finalReceipt = new Uint8Array(finalReply.length + 1);
178	finalReceipt[0] = 0x90;
179	finalReceipt.set(finalReply, 1);
180	ws.onmessage?.({ data: finalReceipt.buffer });
181	expect(onDuoControl.mock.lastCall?.[0].pending).toBe(false);
182	const description = new Uint8Array([0, 0, 0, 5, 1, 1, 0x64, 0, 0x1f]);
183	let ready = waitDecoders(1);
184	feeds[0]!.controller.enqueue(description);
185	await ready;
186	client.setDuoPanels({ cover: { present: cover }, inner: { present: inner } });
187	expect(feeds[0]!.signal.aborted).toBe(true);
188	expect(Socket.instances).toHaveLength(1);
189	for (const [index, id] of [
190	[1, 1],
191	[2, 3],
192	]) {
193	const url = new URL(feeds[index!]!.url);
194	expect(url.pathname.endsWith(`/panel/${id}/stream.avcc`)).toBe(true);
195	expect(url.searchParams.get("hostId")).toBe("remote");
196	expect(url.searchParams.get("wsTicket")).toBe("ticket");
197	}
198	ready = waitDecoders(3);
199	feeds[1]!.controller.enqueue(description);
200	feeds[2]!.controller.enqueue(description);
201	await ready;
202	const frame = {
203	displayWidth: 2007,
204	displayHeight: 2853,
205	close: vi.fn(),
206	} as unknown as VideoFrame;
207	outputs[0]!(frame);
208	expect(present).not.toHaveBeenCalled();
209	outputs[1]!(frame);
210	expect(cover).not.toHaveBeenCalled();
211	outputs[2]!(frame);
212	expect(inner).toHaveBeenCalledOnce();
213	expect(present).toHaveBeenCalledOnce();
214	config(1);
215	outputs[2]!(frame);
216	expect(inner).toHaveBeenCalledOnce();
217	outputs[1]!(frame);
218	expect(cover).toHaveBeenCalledOnce();
219	client.setDuoPanels(null);
220	expect(feeds[1]!.signal.aborted).toBe(true);
221	expect(feeds[2]!.signal.aborted).toBe(true);
222	expect(feeds[3]!.url).not.toContain("/panel/");
223	outputs[1]!(frame);
224	expect(cover).toHaveBeenCalledOnce();
225	ready = waitDecoders(4);
226	feeds[3]!.controller.enqueue(description);
227	await ready;
228	errors[0]!(new DOMException("Late decoder failure"));
229	expect(Decoder.instances[3]!.state).toBe("configured");
230	config(1, "portrait", true);
231	client.setDuoPanels({ cover: { present: cover }, inner: { present: inner } });
232	expect(feeds).toHaveLength(5); // One active feed replaces the two fixed feeds.
233	expect(feeds[4]!.url).not.toContain("/panel/");
234	expect(new URL(feeds[4]!.url).searchParams.get("hostId")).toBe("remote");
235	ready = waitDecoders(5);
236	feeds[4]!.controller.enqueue(description);
237	await ready;
238	const primary = { ...frame, displayWidth: 1398, displayHeight: 2034 } as VideoFrame;
239	const painted = present.mock.calls.length;
240	outputs[4]!(primary);
241	expect(present).toHaveBeenCalledTimes(painted + 1);
242	expect(present.mock.lastCall?.[0]).toBe(primary);
243	expect(cover).toHaveBeenCalledOnce(); // The main sink owns primary texture delivery.
244	config(3, "portrait", true);
245	expect(feeds[4]!.signal.aborted).toBe(true);
246	expect(feeds).toHaveLength(6);
247	ready = waitDecoders(6);
248	feeds[5]!.controller.enqueue(description);
249	await ready;
250	outputs[4]!(primary); // The old elected display cannot paint after handoff.
251	expect(present).toHaveBeenCalledTimes(painted + 1);
252	outputs[5]!(frame);
253	expect(present).toHaveBeenCalledTimes(painted + 2);
254	expect(present.mock.lastCall?.[0]).toBe(frame);
255	config(3, "portrait", true);
256	expect(feeds).toHaveLength(6); // Duplicate native readback does not reconnect.
257	supported = false;
258	client.setDuoPanels(null);
259	client.setDuoPanels({ cover: { present: cover }, inner: { present: inner } });
260	expect(feeds[7]!.url).not.toContain("/panel/");
261	feeds[7]!.controller.enqueue(description);
262	await panelFailure;
263	expect(onDuoUnavailable.mock.lastCall?.[0]).toContain("cannot decode");
264	config(3);
265	const fixedPanelFailure = new Promise<void>((resolve) => {
266	panelFailed = resolve;
267	});
268	client.setDuoPanels({ cover: { present: cover }, inner: { present: inner } });
269	feeds[8]!.controller.enqueue(description);
270	await fixedPanelFailure;
271	expect(onDuoUnavailable).toHaveBeenCalled();
272	expect(onStatus.mock.calls.some(([status]) => status === "error")).toBe(false);
273	panelHttpStatus = 404;
274	const missingPanel = new Promise<void>((resolve) => {
275	panelFailed = resolve;
276	});
277	client.setDuoPanels({ cover: { present: cover }, inner: { present: inner } });
278	await missingPanel;
279	expect(onDuoUnavailable.mock.lastCall?.[0]).toContain("does not provide fixed Duo");
280	expect(Socket.instances).toHaveLength(1);
281	client.stop();
282	expect(feeds[3]!.signal.aborted).toBe(true);
283	expect(ws.close).toHaveBeenCalledOnce();
284	expect(frame.close).toHaveBeenCalledTimes(9);
285	});

packages/client-runtime/src/device/duoViewer.test.ts created+703

...	@@ -0,0 +1,703 @@
1	import { afterEach, expect, it, vi } from "vite-plus/test";
2	import type { Scene, PerspectiveCamera, Box3, Quaternion } from "three";
3
4	const gpu = vi.hoisted(() => ({
5	instances: [] as {
6	blank: boolean;
7	allocations: number;
8	frames: Scene[];
9	views: { camera: PerspectiveCamera; bounds: Box3; quaternion: Quaternion }[];
10	dispose: ReturnType<typeof vi.fn>;
11	forceContextLoss: ReturnType<typeof vi.fn>;
12	}[],
13	environmentDispose: vi.fn(),
14	}));
15	vi.mock("three", async () => {
16	const actual = await vi.importActual<typeof import("three")>("three");
17	return {
18	...actual,
19	WebGLRenderer: class {
20	state = {
21	blank: true,
22	allocations: 0,
23	frames: [] as Scene[],
24	views: [] as { camera: PerspectiveCamera; bounds: Box3; quaternion: Quaternion }[],
25	dispose: vi.fn(),
26	forceContextLoss: vi.fn(),
27	};
28	constructor() {
29	gpu.instances.push(this.state);
30	}
31	setDrawingBufferSize() {
32	this.state.blank = true;
33	this.state.allocations++;
34	}
35	render(scene: Scene, camera: PerspectiveCamera) {
36	this.state.blank = false;
37	this.state.frames.push(scene);
38	const root = scene.children.find((child) => child instanceof actual.Group)!;
39	root.updateMatrixWorld(true);
40	camera.updateMatrixWorld(true);
41	this.state.views.push({
42	camera: camera.clone(),
43	bounds: new actual.Box3().setFromObject(root),
44	quaternion: root.quaternion.clone(),
45	});
46	}
47	dispose() {
48	this.state.dispose();
49	}
50	forceContextLoss() {
51	this.state.forceContextLoss();
52	}
53	},
54	PMREMGenerator: class {
55	fromScene() {
56	return { texture: new actual.Texture(), dispose: gpu.environmentDispose };
57	}
58	dispose() {}
59	},
60	};
61	});
62	const models = vi.hoisted(() => ({
63	resolve: (_model: { asset: import("three").Group; dispose: () => void }) => {},
64	signal: null as AbortSignal \| null,
65	}));
66	vi.mock("./modelScene.ts", () => ({
67	loadDeviceModel: (_source: unknown, signal: AbortSignal) => {
68	models.signal = signal;
69	return new Promise((resolve) => {
70	models.resolve = resolve;
71	});
72	},
73	}));
74	import {
75	Group,
76	Mesh,
77	MeshBasicMaterial,
78	PlaneGeometry,
79	Vector3,
80	Euler,
81	Quaternion as Rotation,
82	} from "three";
83	import { createDuoViewer } from "./duoViewer.ts";
84
85	afterEach(() => {
86	vi.useRealTimers();
87	vi.unstubAllGlobals();
88	gpu.instances.length = 0;
89	gpu.environmentDispose.mockClear();
90	});
91
92	function fixture(
93	reduced = true,
94	onOrientationRequested = vi.fn(),
95	onPanelRequested?: (panel: 1 \| 3) => void,
96	) {
97	const pending = new Map<number, FrameRequestCallback>();
98	let id = 0;
99	vi.stubGlobal("requestAnimationFrame", (callback: FrameRequestCallback) => {
100	pending.set(++id, callback);
101	return id;
102	});
103	vi.stubGlobal("cancelAnimationFrame", (id: number) => pending.delete(id));
104	vi.stubGlobal("matchMedia", () => ({ matches: reduced }));
105	const drawImage = vi.fn();
106	vi.stubGlobal("document", {
107	createElement: () => ({
108	width: 0,
109	height: 0,
110	getContext: () => ({
111	fillRect() {},
112	save() {},
113	restore() {},
114	translate() {},
115	rotate() {},
116	drawImage,
117	getImageData: () => ({ data: new Uint8ClampedArray(8 * 8 * 4).fill(255) }),
118	}),
119	}),
120	});
121	const canvas = new EventTarget() as HTMLCanvasElement;
122	const viewer = createDuoViewer({
123	canvas,
124	sources: {
125	1: { width: 1398, height: 2034 } as HTMLCanvasElement,
126	3: { width: 2007, height: 2853 } as HTMLCanvasElement,
127	},
128	model: { id: "iphone-duo", url: "/duo.glb" },
129	onUnavailable: vi.fn(),
130	onOrientationRequested,
131	...(onPanelRequested ? { onPanelRequested } : {}),
132	});
133	const draw = () => {
134	const callbacks = [...pending.values()];
135	pending.clear();
136	callbacks.forEach((callback) => callback(0));
137	};
138	return { viewer, draw, pending, state: gpu.instances[0]!, onOrientationRequested, drawImage };
139	}
140
141	function asset() {
142	const group = new Group();
143	for (const [name, names] of [
144	["left-half", ["cover-display", "inner-display-left"]],
145	["right-half", ["inner-display-right"]],
146	] as const) {
147	const leaf = new Group();
148	leaf.name = name;
149	for (const name of names) {
150	const mesh = new Mesh(new PlaneGeometry(1, 2), new MeshBasicMaterial());
151	if (name === "cover-display") mesh.geometry.rotateY(Math.PI);
152	mesh.geometry.translate(
153	name === "inner-display-right" ? 0.5 : -0.5,
154	0,
155	name === "cover-display" ? -0.05 : 0.05,
156	);
157	mesh.name = name;
158	leaf.add(mesh);
159	}
160	group.add(leaf);
161	}
162	return group;
163	}
164
165	it("coalesces resize with redraw, retains the renderer and scene, and settles without an idle animation loop", async () => {
166	let now = 0;
167	vi.stubGlobal("performance", { now: () => now });
168	const { viewer, draw, pending, state } = fixture();
169	const dispose = vi.fn();
170	models.resolve({ asset: asset(), dispose });
171	await Promise.resolve();
172	viewer.setScreen({
173	width: 1398,
174	height: 2034,
175	orientation: "portrait",
176	screenId: 1,
177	hingeAngle: 0,
178	});
179	viewer.resize(500, 700, 2);
180	draw();
181	const scene = state.frames.at(-1);
182	viewer.resize(450, 700, 2);
183	viewer.resize(400, 700, 2);
184	viewer.orbit(0.1, 0.04);
185	expect(state.blank).toBe(false);
186	expect(state.allocations).toBe(1);
187	expect(pending.size).toBe(1);
188	draw();
189	expect(state.blank).toBe(false);
190	expect(state.allocations).toBe(2);
191	expect(state.frames.at(-1)).toBe(scene);
192	expect(gpu.instances).toHaveLength(1);
193	now += 1000;
194	draw();
195	draw();
196	expect(pending.size).toBe(0);
197	viewer.dispose();
198	viewer.dispose();
199	expect(dispose).toHaveBeenCalledOnce();
200	expect(state.dispose).toHaveBeenCalledOnce();
201	expect(gpu.environmentDispose).toHaveBeenCalledOnce();
202	expect(pending.size).toBe(0);
203	});
204
205	it("cancels loading and disposes a late model after unmount without installing or rendering it", async () => {
206	const { viewer, draw, state } = fixture();
207	viewer.resize(400, 700, 2);
208	viewer.dispose();
209	expect(models.signal?.aborted).toBe(true);
210	const dispose = vi.fn();
211	models.resolve({ asset: asset(), dispose });
212	await Promise.resolve();
213	draw();
214	expect(dispose).toHaveBeenCalledOnce();
215	expect(state.frames).toHaveLength(0);
216	});
217
218	it("keeps the chosen release view when an app locks orientation, freezes it during contact, and stops drawing after release", async () => {
219	let now = 0;
220	vi.stubGlobal("performance", { now: () => now });
221	const { viewer, draw, state, pending, onOrientationRequested } = fixture();
222	models.resolve({ asset: asset(), dispose: vi.fn() });
223	await Promise.resolve();
224	const screen = {
225	width: 1398,
226	height: 2034,
227	orientation: "portrait" as const,
228	screenId: 1,
229	hingeAngle: 0,
230	};
231	viewer.setScreen(screen);
232	viewer.resize(500, 700, 2);
233	draw();
234	const initial = state.views.at(-1)!.quaternion.clone();
235	viewer.setInteractionActive(true, "orbit");
236	viewer.orbit(0, Math.PI / 3);
237	draw();
238	const held = state.views.at(-1)!.quaternion.clone();
239	now = 1000;
240	viewer.frameUpdated(1);
241	draw();
242	expect(state.views.at(-1)!.quaternion.angleTo(held)).toBeLessThan(1e-6);
243	expect(onOrientationRequested).not.toHaveBeenCalled();
244	viewer.setInteractionActive(false, "orbit");
245	draw();
246	expect(onOrientationRequested).toHaveBeenCalledOnce();
247	const chosen = state.views.at(-1)!.quaternion.clone();
248	expect(chosen.angleTo(initial)).toBeGreaterThan(1);
249	viewer.setScreen({ ...screen }); // Native config confirms the sensor command, even if the app stays portrait.
250	draw();
251	expect(state.views.at(-1)!.quaternion.angleTo(chosen)).toBeLessThan(1e-6);
252	expect(pending.size).toBe(0);
253	viewer.resetPose();
254	draw();
255	expect(onOrientationRequested).toHaveBeenCalledTimes(2);
256	expect(onOrientationRequested.mock.lastCall?.[0]).toBe("portrait");
257	viewer.setScreen({ ...screen });
258	draw();
259	expect(pending.size).toBe(0);
260	viewer.dispose();
261	});
262
263	it("settles a hinge transition after a throttled frame instead of stretching time", async () => {
264	let now = 100;
265	vi.stubGlobal("performance", { now: () => now });
266	const { viewer, draw, pending } = fixture(false);
267	models.resolve({ asset: asset(), dispose: vi.fn() });
268	await Promise.resolve();
269	viewer.resize(400, 700, 2);
270	viewer.setScreen({
271	width: 1398,
272	height: 2034,
273	orientation: "portrait",
274	screenId: 1,
275	hingeAngle: 0,
276	});
277	draw();
278	viewer.setScreen({
279	width: 2007,
280	height: 2853,
281	orientation: "portrait",
282	screenId: 3,
283	hingeAngle: 180,
284	});
285	now += 1_000;
286	draw();
287	draw();
288	expect(pending.size).toBe(0);
289	viewer.dispose();
290	});
291
292	it("faces a display handed off by native rotation without requesting another sensor rotation", async () => {
293	let now = 0;
294	vi.stubGlobal("performance", { now: () => now });
295	const { viewer, draw, pending, onOrientationRequested } = fixture();
296	const loaded = asset();
297	models.resolve({ asset: loaded, dispose: vi.fn() });
298	await Promise.resolve();
299	viewer.resize(500, 700, 2);
300	viewer.setScreen({
301	width: 2007,
302	height: 2853,
303	orientation: "portrait",
304	screenId: 3,
305	hingeAngle: 90,
306	});
307	draw();
308	viewer.setInteractionActive(true, "orbit");
309	viewer.orbit(0, Math.PI / 6);
310	viewer.setInteractionActive(false, "orbit");
311	now = 1000;
312	draw();
313	const requests = onOrientationRequested.mock.calls.length;
314	expect(requests).toBe(1);
315	viewer.setScreen({
316	width: 1398,
317	height: 2034,
318	orientation: "landscape_left",
319	screenId: 1,
320	hingeAngle: 90,
321	});
322	draw();
323	draw();
324	const cover = loaded.getObjectByName("cover-display") as Mesh;
325	const normal = new Vector3()
326	.fromBufferAttribute(cover.geometry.getAttribute("normal"), 0)
327	.transformDirection(cover.matrixWorld);
328	expect(normal.z).toBeGreaterThan(0.45);
329	expect(onOrientationRequested).toHaveBeenCalledTimes(requests);
330	expect(pending.size).toBe(0);
331	viewer.dispose();
332	});
333
334	it.each(["book", "laptop", "open"] as const)(
335	"faces native cover readback for %s even when configuration arrives before the model",
336	async (hingePose) => {
337	const { viewer, draw } = fixture();
338	viewer.resize(500, 700, 2);
339	viewer.setScreen({
340	width: 1398,
341	height: 2034,
342	orientation: "landscape_left",
343	screenId: 1,
344	hingeAngle: hingePose === "open" ? 180 : 90,
345	hingePose,
346	});
347	const loaded = asset();
348	models.resolve({ asset: loaded, dispose: vi.fn() });
349	await Promise.resolve();
350	draw();
351	draw();
352	const cover = loaded.getObjectByName("cover-display") as Mesh;
353	const normal = new Vector3()
354	.fromBufferAttribute(cover.geometry.getAttribute("normal"), 0)
355	.transformDirection(cover.matrixWorld);
356	expect(normal.z).toBeGreaterThan(0.45);
357	viewer.dispose();
358	},
359	);
360
361	it("keeps default-zoom orbits framed across folds and fits the current assembly", async () => {
362	const { viewer, draw, state } = fixture();
363	models.resolve({ asset: asset(), dispose: vi.fn() });
364	await Promise.resolve();
365	viewer.resize(380, 620, 2);
366	viewer.resetPose();
367	viewer.setInteractionActive(true, "orbit");
368	let minimumDistance = Infinity;
369	let maximumDistance = 0;
370	for (const hingeAngle of [0, 30, 90, 127, 180]) {
371	viewer.setScreen({
372	width: 2007,
373	height: 2853,
374	orientation: "portrait",
375	screenId: 3,
376	hingeAngle,
377	});
378	for (let turn = 0; turn < 16; turn++) {
379	viewer.orbit(0.1, turn % 2 ? -0.08 : 0.08);
380	draw();
381	const { camera, bounds } = state.views.at(-1)!;
382	const center = bounds.getCenter(new Vector3()).project(camera);
383	expect(Math.abs(center.x)).toBeLessThan(0.18);
384	expect(Math.abs(center.y)).toBeLessThan(0.18);
385	state.frames.at(-1)!.traverse((object) => {
386	if (!(object instanceof Mesh)) return;
387	const positions = object.geometry.getAttribute("position");
388	for (let index = 0; index < positions.count; index++) {
389	const point = new Vector3()
390	.fromBufferAttribute(positions, index)
391	.applyMatrix4(object.matrixWorld)
392	.project(camera);
393	expect(Math.abs(point.x)).toBeLessThan(1);
394	expect(Math.abs(point.y)).toBeLessThan(1);
395	}
396	});
397	minimumDistance = Math.min(minimumDistance, camera.position.z);
398	maximumDistance = Math.max(maximumDistance, camera.position.z);
399	}
400	}
401	expect(maximumDistance).toBeGreaterThan(minimumDistance);
402	viewer.dispose();
403	});
404
405	it("previews hinge articulation while preserving the laptop lid orientation and blocking unconfirmed input", async () => {
406	const { viewer, draw, pending } = fixture();
407	const loaded = asset();
408	models.resolve({ asset: loaded, dispose: vi.fn() });
409	await Promise.resolve();
410	viewer.resize(500, 700, 2);
411	viewer.setScreen({
412	width: 2853,
413	height: 2007,
414	orientation: "landscape_left",
415	screenId: 3,
416	hingeAngle: 90,
417	hingePose: "laptop",
418	});
419	draw();
420	const lid = loaded.getObjectByName("right-half")!;
421	const presentation = lid.getWorldQuaternion(new Rotation());
422	for (const value of [127, 50, 170]) {
423	viewer.setHingePreview(value);
424	draw();
425	expect(loaded.getObjectByName("left-half")!.rotation.y).toBeCloseTo(
426	((180 - value) * Math.PI) / 360,
427	);
428	expect(lid.getWorldQuaternion(new Rotation()).angleTo(presentation)).toBeLessThan(0.00001);
429	expect(viewer.screenPoint(0.5, 0.5)).toBeNull();
430	viewer.setScreen({
431	width: 2007,
432	height: 2853,
433	orientation: "portrait",
434	screenId: 3,
435	hingeAngle: value,
436	hingePose: null,
437	});
438	draw();
439	expect(lid.getWorldQuaternion(new Rotation()).angleTo(presentation)).toBeLessThan(0.00001);
440	}
441	viewer.setHingePreview(null);
442	draw();
443	expect(pending.size).toBe(0);
444	expect(lid.getWorldQuaternion(new Rotation()).angleTo(presentation)).toBeLessThan(0.00001);
445	viewer.dispose();
446	});
447
448	it("does not restart an animated preset on duplicate native configurations", async () => {
449	let now = 100;
450	vi.stubGlobal("performance", { now: () => now });
451	const { viewer, draw, pending } = fixture(false);
452	models.resolve({ asset: asset(), dispose: vi.fn() });
453	await Promise.resolve();
454	viewer.resize(400, 700, 2);
455	viewer.setScreen({
456	width: 1398,
457	height: 2034,
458	orientation: "portrait",
459	screenId: 1,
460	hingeAngle: 0,
461	hingePose: "closed",
462	});
463	draw();
464	const next = {
465	width: 2007,
466	height: 2853,
467	orientation: "portrait" as const,
468	screenId: 3,
469	hingeAngle: 180,
470	hingePose: "open" as const,
471	};
472	viewer.setScreen(next);
473	now += 1_000;
474	viewer.setScreen(next);
475	draw();
476	now += 1000;
477	draw();
478	draw();
479	expect(pending.size).toBe(0);
480	viewer.dispose();
481	});
482
483	it("lets standalone rotation leave Laptop and Tent and stand a closed device upright after hinge edits", async () => {
484	const { viewer, draw, state } = fixture();
485	models.resolve({ asset: asset(), dispose: vi.fn() });
486	await Promise.resolve();
487	viewer.resize(500, 700, 2);
488	viewer.resetPose();
489	for (const hingePose of ["laptop", "tent"] as const) {
490	viewer.setScreen({
491	width: 1398,
492	height: 2034,
493	orientation: "landscape_left",
494	screenId: 1,
495	hingeAngle: 90,
496	hingePose,
497	});
498	draw();
499	viewer.setHingePreview(0);
500	viewer.setScreen({
501	width: 1398,
502	height: 2034,
503	orientation: "landscape_left",
504	screenId: 1,
505	hingeAngle: 0,
506	hingePose: null,
507	});
508	viewer.setHingePreview(null);
509	draw();
510	viewer.cancelInput(); // Explicit toolbar rotation ends folding ownership.
511	viewer.setScreen({
512	width: 1398,
513	height: 2034,
514	orientation: "portrait",
515	screenId: 1,
516	hingeAngle: 0,
517	hingePose: null,
518	});
519	draw();
520	const upright = new Rotation().setFromEuler(new Euler(0, Math.PI / 2, 0, "YXZ"));
521	expect(state.views.at(-1)!.quaternion.angleTo(upright)).toBeLessThan(0.00001);
522	const bounds = state.views.at(-1)!.bounds;
523	expect(bounds.max.y - bounds.min.y).toBeGreaterThan(bounds.max.x - bounds.min.x);
524	}
525	viewer.dispose();
526	});
527
528	it("snaps onto the opposite screen, requests native handoff once and retains the chosen view on readback", async () => {
529	vi.useFakeTimers();
530	let now = 0;
531	vi.stubGlobal("performance", { now: () => now });
532	const onPanelRequested = vi.fn();
533	const { viewer, draw, state, pending, onOrientationRequested } = fixture(
534	true,
535	vi.fn(),
536	onPanelRequested,
537	);
538	models.resolve({ asset: asset(), dispose: vi.fn() });
539	await Promise.resolve();
540	viewer.resize(500, 700, 2);
541	const inner = {
542	width: 2007,
543	height: 2853,
544	orientation: "portrait" as const,
545	screenId: 3,
546	hingeAngle: 90,
547	supportsPhysicalOrientation: true,
548	};
549	viewer.setScreen(inner);
550	draw();
551	viewer.setInteractionActive(true, "orbit");
552	viewer.orbit(Math.PI / 3, 0);
553	draw();
554	viewer.setInteractionActive(false, "orbit");
555	draw();
556	expect(onPanelRequested).toHaveBeenCalledExactlyOnceWith(1);
557	expect(onOrientationRequested).not.toHaveBeenCalled();
558	expect(viewer.screenPoint(0.5, 0.5)).toBeNull();
559	const chosen = state.views.at(-1)!.quaternion.clone();
560	viewer.setScreen({ ...inner, hingePose: null }); // Command acknowledgement precedes sensor readback.
561	draw();
562	expect(state.views.at(-1)!.quaternion.angleTo(chosen)).toBeLessThan(1e-6);
563	viewer.setScreen({ ...inner, width: 1398, height: 2034, screenId: 1 });
564	draw();
565	expect(state.views.at(-1)!.quaternion.angleTo(chosen)).toBeLessThan(1e-6);
566	vi.advanceTimersByTime(5000);
567	expect(onPanelRequested).toHaveBeenCalledTimes(1);
568	expect(pending.size).toBe(0);
569	viewer.setInteractionActive(true, "orbit");
570	viewer.orbit(-Math.PI / 3, 0);
571	draw();
572	viewer.setInteractionActive(false, "orbit");
573	draw();
574	expect(onPanelRequested).toHaveBeenLastCalledWith(3);
575	const reverse = state.views.at(-1)!.quaternion.clone();
576	viewer.setScreen(inner);
577	draw();
578	expect(state.views.at(-1)!.quaternion.angleTo(reverse)).toBeLessThan(1e-6);
579	vi.advanceTimersByTime(5000);
580	draw();
581	expect(pending.size).toBe(0);
582	expect(onPanelRequested).toHaveBeenCalledTimes(2);
583	viewer.dispose();
584	vi.useRealTimers();
585	});
586
587	it("rolls an unconfirmed opposite-screen snap back to the native active screen after timeout", async () => {
588	vi.useFakeTimers();
589	const request = vi.fn();
590	const { viewer, draw, state, pending } = fixture(true, vi.fn(), request);
591	models.resolve({ asset: asset(), dispose: vi.fn() });
592	await Promise.resolve();
593	viewer.resize(500, 700, 2);
594	viewer.setScreen({
595	width: 2007,
596	height: 2853,
597	orientation: "portrait",
598	screenId: 3,
599	hingeAngle: 90,
600	supportsPhysicalOrientation: true,
601	});
602	draw();
603	viewer.setInteractionActive(true, "orbit");
604	viewer.orbit(Math.PI / 3, 0);
605	draw();
606	viewer.setInteractionActive(false, "orbit");
607	draw();
608	const unconfirmed = state.views.at(-1)!.quaternion.clone();
609	expect(request).toHaveBeenCalledExactlyOnceWith(1);
610	vi.advanceTimersByTime(5000);
611	draw();
612	expect(state.views.at(-1)!.quaternion.angleTo(unconfirmed)).toBeGreaterThan(0.5);
613	expect(request).toHaveBeenCalledOnce();
614	draw();
615	expect(pending.size).toBe(0);
616	viewer.dispose();
617	});
618
619	it("uses the elected primary frame during handoff and never lets a stale fixed-panel feed overwrite it", async () => {
620	const { viewer, drawImage } = fixture();
621	const cover = {
622	width: 1398,
623	height: 2034,
624	orientation: "portrait" as const,
625	screenId: 1,
626	hingeAngle: 90,
627	};
628	viewer.setScreen(cover);
629	const primary = { width: 1398, height: 2034 } as HTMLCanvasElement;
630	viewer.frameUpdated(1, primary);
631	expect(drawImage.mock.calls.at(-1)?.[0]).toBe(primary);
632	const uploads = drawImage.mock.calls.length;
633	viewer.frameUpdated(1);
634	expect(drawImage).toHaveBeenCalledTimes(uploads);
635	viewer.frameUpdated(3, primary);
636	expect(drawImage).toHaveBeenCalledTimes(uploads);
637	viewer.setScreen({ ...cover, width: 2007, height: 2853, screenId: 3 });
638	const inner = { width: 2007, height: 2853 } as HTMLCanvasElement;
639	viewer.frameUpdated(3, inner);
640	expect(drawImage.mock.calls.at(-1)?.[0]).toBe(inner);
641	viewer.setScreen(cover);
642	viewer.frameUpdated(1);
643	expect(drawImage.mock.calls.length).toBeGreaterThan(uploads + 1);
644	viewer.dispose();
645	});
646
647	it.each(["left", "right"] as const)(
648	"keeps the primary surface oriented when pinching over the %s half through closure and reopening",
649	async (face) => {
650	const { viewer, draw, state, pending, onOrientationRequested } = fixture();
651	const loaded = asset();
652	models.resolve({ asset: loaded, dispose: vi.fn() });
653	await Promise.resolve();
654	viewer.resize(500, 700, 2);
655	const inner = {
656	width: 2007,
657	height: 2853,
658	orientation: "portrait" as const,
659	screenId: 3,
660	hingeAngle: 180,
661	hingePose: "open" as const,
662	};
663	viewer.setScreen(inner);
664	draw();
665	expect(viewer.beginHinge(-1, -1)).toBe(false);
666	const leaf = loaded.getObjectByName("right-half")!;
667	const mesh = loaded.getObjectByName(`inner-display-${face}`) as Mesh;
668	const center = mesh.geometry.boundingBox!.getCenter(new Vector3());
669	const point = mesh.localToWorld(center).project(state.views.at(-1)!.camera);
670	expect(viewer.beginHinge((point.x + 1) / 2, (1 - point.y) / 2)).toBe(true);
671	const orientation = leaf.getWorldQuaternion(new Rotation());
672	const partner = loaded.getObjectByName("left-half")!;
673	const partnerStart = partner.getWorldQuaternion(new Rotation());
674	for (const value of [150, 90, 30, 0, 30, 90, 180]) {
675	viewer.setHingePreview(value);
676	draw();
677	expect(leaf.getWorldQuaternion(new Rotation()).angleTo(orientation)).toBeLessThan(1e-6);
678	if (value === 90)
679	expect(partner.getWorldQuaternion(new Rotation()).angleTo(partnerStart)).toBeGreaterThan(1);
680	const next =
681	value === 0
682	? { ...inner, width: 1398, height: 2034, screenId: 1, hingeAngle: value, hingePose: null }
683	: { ...inner, hingeAngle: value, hingePose: null };
684	viewer.setScreen(next);
685	viewer.setHingePreview(null);
686	draw();
687	viewer.setScreen({ ...next }); // Late native readback cannot change the view.
688	draw();
689	expect(leaf.getWorldQuaternion(new Rotation()).angleTo(orientation)).toBeLessThan(1e-6);
690	expect(viewer.screenPoint(0.5, 0.5)).toBeNull(); // No fresh native frame yet.
691	if (value === 0) {
692	const cover = loaded.getObjectByName("cover-display") as Mesh;
693	const center = cover.geometry.boundingBox!.getCenter(new Vector3());
694	const hit = cover.localToWorld(center).project(state.views.at(-1)!.camera);
695	expect(viewer.beginHinge((hit.x + 1) / 2, (1 - hit.y) / 2)).toBe(true);
696	}
697	}
698	expect(onOrientationRequested).not.toHaveBeenCalled();
699	draw();
700	expect(pending.size).toBe(0);
701	viewer.dispose();
702	},
703	);

packages/client-runtime/src/device/duoViewer.ts created+566

...	@@ -0,0 +1,566 @@
1	// @effect-diagnostics globalTimers:off - Native display handoff has a bounded acknowledgement window.
2	import {
3	AmbientLight,
4	Box3,
5	CanvasTexture,
6	DirectionalLight,
7	Euler,
8	LinearFilter,
9	PerspectiveCamera,
10	PMREMGenerator,
11	Quaternion,
12	Scene,
13	SRGBColorSpace,
14	Vector3,
15	WebGLRenderer,
16	} from "three";
17	import { RoomEnvironment } from "three/addons/environments/RoomEnvironment.js";
18	import {
19	createDuoScene,
20	duoDisplayKey,
21	duoFrameMatches,
22	type DuoPanelId,
23	type DuoHingeLeaf,
24	} from "./duoScene.ts";
25	import { loadDeviceModel } from "./modelScene.ts";
26	import { createDeviceModelSlot, type DeviceModelSource } from "./model.ts";
27	import { createDeviceFraming } from "./deviceFraming.ts";
28	import { createDeviceMotion } from "./deviceMotion.ts";
29	import { duoViewSnaps, nearestDuoView, type DuoRestFace } from "./duoSnap.ts";
30	import { createRenderScheduler } from "./renderScheduler.ts";
31	import type { DeviceScreenSize } from "./stream.ts";
32	import type { DuoPose } from "./duoControl.ts";
33
34	export interface DuoViewer {
35	readonly setScreen: (screen: DeviceScreenSize \| null) => void;
36	readonly setHingePreview: (angle: number \| null) => void;
37	readonly setInteractionActive: (active: boolean, mode?: "touch" \| "orbit") => void;
38	readonly rejectOrientation: () => void;
39	readonly frameUpdated: (panel: DuoPanelId, primary?: HTMLCanvasElement) => void;
40	readonly resize: (width: number, height: number, pixelRatio: number) => void;
41	readonly screenPoint: (
42	x: number,
43	y: number,
44	captured?: boolean,
45	) => { x: number; y: number } \| null;
46	readonly orbit: (x: number, y: number) => void;
47	readonly beginHinge: (x: number, y: number) => boolean;
48	readonly resetPose: () => void;
49	readonly cancelInput: () => void;
50	readonly dispose: () => void;
51	}
52
53	/** On-demand renderer for the articulated body. One inner framebuffer spans both leaves; HID belongs to the stream. */
54	export function createDuoViewer(options: {
55	canvas: HTMLCanvasElement;
56	sources: Record<DuoPanelId, HTMLCanvasElement>;
57	model: DeviceModelSource;
58	onUnavailable: () => void;
59	onModelError?: (cause: unknown) => void;
60	onPanelRequested?: (panel: DuoPanelId) => void;
61	onOrientationRequested?: (orientation: DeviceScreenSize["orientation"]) => void;
62	}): DuoViewer {
63	const surfaces = ([1, 3] as const)
64	.map((id) => {
65	const canvas = document.createElement("canvas");
66	canvas.width = id === 1 ? 784 : 1600;
67	canvas.height = id === 1 ? 1140 : 1125;
68	const context = canvas.getContext("2d", { alpha: false });
69	if (!context) throw new Error("Duo display canvas is unavailable");
70	context.fillStyle = "#080a10";
71	context.fillRect(0, 0, canvas.width, canvas.height);
72	return { id, canvas, context };
73	})
74	.map(({ id, canvas, context }) => {
75	const texture = new CanvasTexture(canvas);
76	texture.colorSpace = SRGBColorSpace;
77	texture.minFilter = texture.magFilter = LinearFilter;
78	texture.generateMipmaps = false;
79	return { id, canvas, context, texture };
80	});
81	const renderer = new WebGLRenderer({
82	canvas: options.canvas,
83	alpha: true,
84	antialias: true,
85	powerPreference: "low-power",
86	});
87	renderer.outputColorSpace = SRGBColorSpace;
88	const scene = new Scene();
89	const environment = (() => {
90	const generator = new PMREMGenerator(renderer);
91	const room = new RoomEnvironment();
92	try {
93	return generator.fromScene(room, 0.04);
94	} catch (cause) {
95	for (const surface of surfaces) surface.texture.dispose();
96	renderer.dispose();
97	renderer.forceContextLoss();
98	throw cause;
99	} finally {
100	room.dispose();
101	generator.dispose();
102	}
103	})();
104	scene.environment = environment.texture;
105	const camera = new PerspectiveCamera(36, 1, 0.1, 150);
106	const key = new DirectionalLight(0xffffff, 4);
107	key.position.set(-6, 10, 14);
108	const fill = new DirectionalLight(0xc7dcff, 2);
109	fill.position.set(8, -3, -5);
110	scene.add(new AmbientLight(0xffffff, 2.4), key, fill);
111	let model: ReturnType<typeof createDuoScene> \| null = null;
112	let screen: DeviceScreenSize \| null = null;
113	let readyKey = "";
114	let primaryKey = "";
115	let activationAt = 0;
116	let disposed = false;
117	let viewport = { width: 0, height: 0, ratio: 1 };
118	let buffer = { width: 0, height: 0, ratio: 0 };
119	let restFace: DuoRestFace = "inside";
120	let requestedOrientation: DeviceScreenSize["orientation"] \| null = null;
121	let viewOrientation: DeviceScreenSize["orientation"] \| null = null;
122	const pivot = new Vector3();
123	const targetPivot = new Vector3();
124	let requestedPanel: DuoPanelId \| null = null;
125	let handoffTimer: ReturnType<typeof setTimeout> \| null = null;
126	const clearHandoff = () => {
127	if (handoffTimer) clearTimeout(handoffTimer);
128	handoffTimer = null;
129	requestedPanel = null;
130	};
131	const activeSnaps = () =>
132	model
133	? duoViewSnaps(
134	model.restFrames(screen?.screenId === 1 ? 1 : 3),
135	screen?.screenId === 1 ? 1 : 3,
136	)
137	: [];
138	const snaps = () => {
139	if (!model \|\| !screen?.supportsPhysicalOrientation \|\| !options.onPanelRequested)
140	return activeSnaps();
141	const cover = duoViewSnaps(model.restFrames(1), 1);
142	// A shut inner display is occluded and cannot become a useful rest view.
143	return angle > 20 && angle < 180
144	? [...duoViewSnaps(model.restFrames(3), 3), ...cover]
145	: activeSnaps();
146	};
147	const orbit = createDeviceMotion({
148	choose(rotation) {
149	const snap = nearestDuoView(rotation, snaps());
150	if (!snap) return rotation;
151	restFace = snap.face;
152	const panel = snap.face === "cover" ? 1 : 3;
153	if (screen && panel !== (requestedPanel ?? screen.screenId) && options.onPanelRequested) {
154	clearHandoff();
155	requestedPanel = panel;
156	model?.cancelInput();
157	options.onPanelRequested(panel);
158	handoffTimer = setTimeout(() => {
159	clearHandoff();
160	const confirmed = nearestDuoView(orbit.rotation, activeSnaps());
161	if (confirmed) {
162	restFace = confirmed.face;
163	orbit.setPose(confirmed.rotation, performance.now());
164	}
165	scheduler.invalidate();
166	}, 5000);
167	} else if (
168	screen &&
169	requestedPanel === null &&
170	snap.orientation !== viewOrientation &&
171	options.onOrientationRequested
172	) {
173	viewOrientation = snap.orientation;
174	requestedOrientation = snap.orientation;
175	options.onOrientationRequested(snap.orientation);
176	}
177	return snap.rotation;
178	},
179	});
180	let angle = 180;
181	let targetAngle = 180;
182	let previewAngle: number \| null = null;
183	let hingeLeaf: DuoHingeLeaf \| null = null;
184	let appliedAngle = Number.NaN;
185	let interactionActive = false;
186	const framing = createDeviceFraming();
187	const framingBounds = new Box3();
188	let firstPose = true;
189	let physicalPose: DuoPose = "open";
190	let presentationAngle = 180;
191	let targetPresentation = new Quaternion();
192	let lastTime = 0;
193	const reduced =
194	typeof matchMedia === "function" && matchMedia("(prefers-reduced-motion: reduce)").matches;
195	const applyPose = () => {
196	if (!model) return;
197	const before = hingeLeaf && appliedAngle !== angle ? model.leafRotation(hingeLeaf) : null;
198	model.setAngle(angle);
199	appliedAngle = angle;
200	if (before && hingeLeaf) {
201	// The primary surface stays in camera space; its partner supplies the fold.
202	// Rebase both the displayed rotation and its rest target so releasing a
203	// pinch cannot resume the pre-fold body rotation.
204	const correction = before.multiply(model.leafRotation(hingeLeaf).invert());
205	if (correction.angleTo(new Quaternion()) > 1e-8)
206	orbit.setPose(orbit.rotation.clone().multiply(correction), performance.now(), true);
207	}
208	model.root.quaternion.copy(orbit.rotation);
209	model.root.position.set(0, 0, 0);
210	const frames = model.restFrames(restFace === "cover" ? 1 : 3);
211	const frame = frames.find((frame) => frame.face === restFace) ?? frames[0];
212	targetPivot.copy(frame?.center ?? new Vector3());
213	model.root.position.copy(pivot).applyQuaternion(model.root.quaternion).negate();
214	};
215	const updateCamera = () => {
216	camera.position.set(framing.center.x, framing.center.y, framing.distance());
217	camera.lookAt(framing.center.x, framing.center.y, 0);
218	camera.updateProjectionMatrix();
219	};
220	const fit = (immediate = false) => {
221	if (!model \|\| !viewport.width \|\| !viewport.height) return;
222	camera.aspect = viewport.width / viewport.height;
223	framing.setBounds(
224	framingBounds.setFromObject(model.root),
225	(camera.fov * Math.PI) / 360,
226	camera.aspect,
227	performance.now(),
228	immediate,
229	);
230	updateCamera();
231	};
232	const moving = () =>
233	Math.abs(angle - targetAngle) > 0.01 \|\| pivot.distanceTo(targetPivot) > 0.001;
234	const scheduler = createRenderScheduler(() => {
235	if (disposed \|\| !model \|\| !viewport.width \|\| !viewport.height) return;
236	try {
237	const now = performance.now();
238	const elapsed = Math.max(0, (now - lastTime) / 1000);
239	const amount = reduced
240	? 1
241	: 1 - Math.exp(-14 * Math.max(0, (now - lastTime) / 1000 \|\| 0.016));
242	lastTime = now;
243	const inMotion = moving();
244	const orbitChanged = orbit.advance(now, reduced);
245	if (inMotion) {
246	angle += (targetAngle - angle) * amount;
247
248	if (Math.abs(angle - targetAngle) <= 0.01) angle = targetAngle;
249	applyPose();
250	pivot.lerp(targetPivot, amount);
251	if (pivot.distanceTo(targetPivot) <= 0.001) pivot.copy(targetPivot);
252	applyPose();
253	fit(reduced \|\| elapsed > 0.5);
254	}
255	if (orbitChanged) {
256	applyPose();
257	fit(reduced \|\| elapsed > 0.5);
258	}
259	if (framing.advance(now, reduced)) updateCamera();
260	if (
261	buffer.width !== viewport.width \|\|
262	buffer.height !== viewport.height \|\|
263	buffer.ratio !== viewport.ratio
264	) {
265	renderer.setDrawingBufferSize(viewport.width, viewport.height, viewport.ratio);
266	buffer = viewport;
267	}
268	renderer.render(scene, camera);
269	if (inMotion \|\| moving() \|\| orbit.needsFrame() \|\| framing.needsFrame())
270	scheduler.invalidate();
271	} catch {
272	options.onUnavailable();
273	}
274	});
275	const slot = createDeviceModelSlot({
276	load: loadDeviceModel,
277	onError(cause) {
278	options.onModelError?.(cause);
279	options.onUnavailable();
280	},
281	install(loaded) {
282	const next = loaded
283	? createDuoScene(loaded.asset, { 1: surfaces[0]!.texture, 3: surfaces[1]!.texture })
284	: null;
285	if (model) {
286	scene.remove(model.root);
287	model.dispose();
288	}
289	model = next;
290	appliedAngle = Number.NaN;
291	if (!model \|\| disposed) return;
292	scene.add(model.root);
293	applyPose();
294	if (screen?.screenId === 1) {
295	const snap = nearestDuoView(orbit.rotation, activeSnaps());
296	if (snap) {
297	restFace = snap.face;
298	orbit.setPose(snap.rotation, performance.now(), true);
299	applyPose();
300	}
301	}
302	pivot.copy(targetPivot);
303	applyPose();
304	fit();
305	scheduler.invalidate();
306	},
307	});
308	const lost = (event: Event) => {
309	event.preventDefault();
310	options.onUnavailable();
311	};
312	options.canvas.addEventListener("webglcontextlost", lost);
313	slot.set(options.model);
314	return {
315	setScreen(next) {
316	if (disposed) return;
317	const wasMoving = moving() \|\| orbit.needsFrame();
318	if (duoDisplayKey(screen) !== duoDisplayKey(next)) {
319	readyKey = "";
320	activationAt = performance.now();
321	model?.cancelInput();
322	}
323	const previous = screen;
324	const ownedHandoff = requestedPanel !== null;
325	if (!next \|\| next.screenId === requestedPanel) clearHandoff();
326	const ownedRotation = requestedOrientation !== null && next?.screenId === previous?.screenId;
327	const changedDisplay = next?.screenId !== previous?.screenId;
328	requestedOrientation = null;
329	screen = next;
330	const changedPose = next?.hingePose && next.hingePose !== previous?.hingePose;
331	const folding = hingeLeaf !== null && !changedPose;
332	const rotated =
333	next?.screenId === previous?.screenId &&
334	next?.hingeAngle === previous?.hingeAngle &&
335	next?.orientation !== previous?.orientation;
336	const leftPhysicalPose =
337	rotated &&
338	!ownedRotation &&
339	!ownedHandoff &&
340	!folding &&
341	!next?.hingePose &&
342	previewAngle === null;
343	if (firstPose \|\| changedPose \|\| leftPhysicalPose) {
344	physicalPose = next?.hingePose ?? (next?.screenId === 1 ? "closed" : "open");
345	presentationAngle = next?.hingeAngle ?? (next?.screenId === 1 ? 0 : 180);
346	}
347	// A command reply/config confirms hinge state. Display identity, never an angle heuristic, owns input.
348	targetAngle = previewAngle ?? next?.hingeAngle ?? (next?.screenId === 1 ? 0 : 180);
349	// Native angle commands clear hingePose. They change articulation only;
350	// preserve the viewing pose, including laptop/tent and user orbit. A
351	// separate rotation clears the physical preset and follows panel orientation.
352	if (firstPose \|\| changedPose \|\| (rotated && !ownedRotation && !ownedHandoff && !folding)) {
353	hingeLeaf = null;
354	viewOrientation = next?.orientation ?? null;
355	const poseAngle = presentationAngle;
356	const fold = ((180 - poseAngle) * Math.PI) / 360;
357	let roll = physicalPose === "closed" ? 0 : Math.PI / 2;
358	if (next && next.width < next.height) {
359	if (next.orientation === "landscape_left") roll -= Math.PI / 2;
360	if (next.orientation === "landscape_right") roll += Math.PI / 2;
361	}
362	if (next?.orientation === "portrait_upside_down") roll -= Math.PI;
363	const physical =
364	physicalPose === "laptop"
365	? new Euler(-fold + Math.PI / 9, -Math.PI / 9, Math.PI / 2, "YXZ")
366	: physicalPose === "tent"
367	? new Euler(Math.PI / 2 + Math.PI / 18, -Math.PI / 9, -Math.PI / 2, "YXZ")
368	: new Euler(0, (Math.PI / 2) * Math.pow(1 - poseAngle / 180, 3), 0, "YXZ");
369	targetPresentation = new Quaternion().setFromEuler(physical);
370	if (physicalPose !== "laptop" && physicalPose !== "tent")
371	targetPresentation.premultiply(
372	new Quaternion().setFromAxisAngle(new Vector3(0, 0, 1), roll),
373	);
374	if (next?.screenId === 1) {
375	// Native readback can retain the cover even for an open physical preset.
376	// A screen-facing rest must use that display's actual hinged normal.
377	const snap = nearestDuoView(targetPresentation, activeSnaps());
378	if (snap) targetPresentation.copy(snap.rotation);
379	}
380	if (firstPose) angle = targetAngle;
381	restFace = next?.screenId === 1 ? "cover" : physicalPose === "laptop" ? "right" : "inside";
382	orbit.setPose(targetPresentation, performance.now(), firstPose);
383	firstPose = false;
384	} else if (next && changedDisplay && !ownedHandoff && !folding) {
385	// A sensor rotation can hand ownership to the other native display.
386	// Face that display without sending another rotation and creating a feedback loop.
387	const snap = nearestDuoView(orbit.rotation, activeSnaps());
388	if (snap) {
389	restFace = snap.face;
390	orbit.setPose(snap.rotation, performance.now());
391	}
392	}
393	if (!wasMoving) lastTime = performance.now();
394	applyPose();
395	fit();
396	scheduler.invalidate();
397	},
398	setHingePreview(next) {
399	if (disposed \|\| (next !== null && (!Number.isFinite(next) \|\| next < 0 \|\| next > 180))) return;
400	if (!moving()) lastTime = performance.now();
401	previewAngle = next;
402	if (next !== null && !hingeLeaf && model) {
403	clearHandoff();
404	requestedOrientation = null;
405	hingeLeaf = screen?.screenId === 1 && angle > 20 ? "left" : "right";
406	orbit.setPose(orbit.rotation.clone(), performance.now(), true);
407	}
408	orbit.hold(interactionActive \|\| next !== null, performance.now());
409	targetAngle = next ?? screen?.hingeAngle ?? (screen?.screenId === 1 ? 0 : 180);
410	if (next !== null) {
411	angle = next;
412	applyPose();
413	fit();
414	}
415	model?.cancelInput();
416	scheduler.invalidate();
417	},
418	setInteractionActive(active, mode = "touch") {
419	if (disposed) return;
420	if (mode === "orbit") {
421	if (active) hingeLeaf = null;
422	orbit.dragActive(active, performance.now());
423	} else {
424	interactionActive = active;
425	orbit.hold(active \|\| previewAngle !== null, performance.now());
426	framing.hold(active, performance.now());
427	}
428	scheduler.invalidate();
429	},
430	rejectOrientation() {
431	if (disposed \|\| (requestedOrientation === null && requestedPanel === null)) return;
432	clearHandoff();
433	requestedOrientation = null;
434	viewOrientation = screen?.orientation ?? null;
435	const confirmed = activeSnaps().filter((snap) => snap.orientation === screen?.orientation);
436	const snap = nearestDuoView(orbit.rotation, confirmed);
437	if (snap) {
438	restFace = snap.face;
439	orbit.setPose(snap.rotation, performance.now());
440	}
441	scheduler.invalidate();
442	},
443	frameUpdated(id, primary) {
444	if (disposed \|\| screen?.screenId !== id) return;
445	const key = duoDisplayKey(screen);
446	// The elected native feed is authoritative during handoff. Fixed-panel
447	// encoders can retain an inactive blank until that surface changes again.
448	if (!primary && primaryKey === key) return;
449	const source = primary ?? options.sources[id];
450	if (!duoFrameMatches(source, screen)) return;
451	const surface = surfaces[id === 1 ? 0 : 1]!;
452	const { context, canvas } = surface;
453	// Ignore native shutdown blanks only while waiting for an activation. Steady black application content remains valid.
454	if (!primary && !readyKey && performance.now() - activationAt < 1500) {
455	const probe = document.createElement("canvas");
456	probe.width = probe.height = 8;
457	const probeContext = probe.getContext("2d", { willReadFrequently: true });
458	if (probeContext) {
459	probeContext.drawImage(source, 0, 0, 8, 8);
460	if (
461	!probeContext
462	.getImageData(0, 0, 8, 8)
463	.data.some((value, index) => index % 4 !== 3 && value > 3)
464	)
465	return;
466	}
467	}
468	context.save();
469	context.fillStyle = "#080a10";
470	context.fillRect(0, 0, canvas.width, canvas.height);
471	context.translate(canvas.width / 2, canvas.height / 2);
472	// The inner panel is mounted a quarter turn from the native framebuffer.
473	const rotate = id === 3;
474	if (rotate) context.rotate(Math.PI / 2);
475	const scale = Math.min(
476	canvas.width / (rotate ? source.height : source.width),
477	canvas.height / (rotate ? source.width : source.height),
478	);
479	context.drawImage(
480	source,
481	(-source.width * scale) / 2,
482	(-source.height * scale) / 2,
483	source.width * scale,
484	source.height * scale,
485	);
486	context.restore();
487	surface.texture.needsUpdate = true;
488	readyKey = duoDisplayKey(screen);
489	if (primary) primaryKey = readyKey;
490	scheduler.invalidate();
491	},
492	resize(width, height, ratio) {
493	if (disposed \|\| ![width, height, ratio].every(Number.isFinite) \|\| width <= 0 \|\| height <= 0)
494	return;
495	viewport = { width, height, ratio: Math.min(2, Math.max(1, ratio)) };
496	fit(true);
497	scheduler.invalidate();
498	},
499	screenPoint(x, y, captured = false) {
500	if (
501	disposed \|\|
502	moving() \|\|
503	previewAngle !== null \|\|
504	requestedOrientation !== null \|\|
505	requestedPanel !== null
506	)
507	return null;
508	applyPose();
509	return model?.screenPoint(x, y, camera, screen, readyKey, captured) ?? null;
510	},
511	cancelInput() {
512	hingeLeaf = null;
513	model?.cancelInput();
514	},
515	orbit(x, y) {
516	if (disposed) return;
517	hingeLeaf = null;
518	orbit.orbit(x * viewport.width, y * viewport.height, performance.now());
519	scheduler.invalidate();
520	},
521	beginHinge(x, y) {
522	if (disposed \|\| !model \|\| !screen) return false;
523	applyPose();
524	const leaf = model.hingeLeafAt(x, y, camera);
525	if (!leaf) return false;
526	clearHandoff();
527	requestedOrientation = null;
528	// The right inner leaf and the shut cover share the front-facing plane.
529	// Holding that leaf lets the cover replace the inner image without a
530	// half turn, even when the pinch starts over the moving partner.
531	hingeLeaf = screen.screenId === 1 && angle > 20 ? "left" : "right";
532	orbit.setPose(orbit.rotation.clone(), performance.now(), true);
533	model.cancelInput();
534	return true;
535	},
536	resetPose() {
537	if (disposed) return;
538	hingeLeaf = null;
539	requestedOrientation = null;
540	clearHandoff();
541	const snap = activeSnaps().find((snap) => snap.orientation === screen?.orientation);
542	if (snap && snap.orientation !== viewOrientation && options.onOrientationRequested) {
543	viewOrientation = snap.orientation;
544	requestedOrientation = snap.orientation;
545	options.onOrientationRequested(snap.orientation);
546	}
547	orbit.reset(snap?.rotation ?? targetPresentation, performance.now());
548	applyPose();
549	fit();
550	scheduler.invalidate();
551	},
552	dispose() {
553	clearHandoff();
554	if (disposed) return;
555	disposed = true;
556	scheduler.dispose();
557	slot.dispose();
558	options.canvas.removeEventListener("webglcontextlost", lost);
559	for (const surface of surfaces) surface.texture.dispose();
560	scene.environment = null;
561	environment.dispose();
562	renderer.dispose();
563	renderer.forceContextLoss();
564	},
565	};
566	}

packages/client-runtime/src/device/model.test.ts+1

...	@@ -24,6 +24,7 @@ function fixture() {	...	@@ -24,6 +24,7 @@ function fixture() {
24	it("matches only exact supported hardware, including iPad size and generation", () => {	24	it("matches only exact supported hardware, including iPad size and generation", () => {
25	expect(resolveDeviceModelId("ios", "iPhone 18 Pro")).toBe(pro.id);	25	expect(resolveDeviceModelId("ios", "iPhone 18 Pro")).toBe(pro.id);
26	expect(resolveDeviceModelId("ios", "iPhone 18 Pro Max")).toBe(max.id);	26	expect(resolveDeviceModelId("ios", "iPhone 18 Pro Max")).toBe(max.id);
		27	expect(resolveDeviceModelId("ios", "iPhone Duo")).toBe("iphone-duo");
27	expect(resolveDeviceModelId("ios", "iPad Pro 13-inch (M5)")).toBe("ipad-pro-13-m5");	28	expect(resolveDeviceModelId("ios", "iPad Pro 13-inch (M5)")).toBe("ipad-pro-13-m5");
28	for (const name of [	29	for (const name of [
29	"iPhone 17 Pro",	30	"iPhone 17 Pro",

packages/client-runtime/src/device/model.ts+2-1

...	@@ -1,6 +1,6 @@	...	@@ -1,6 +1,6 @@
1	import type { DevicePlatform } from "@t3tools/contracts";	1	import type { DevicePlatform } from "@t3tools/contracts";
2		2
3	export type DeviceModelId = "iphone-18-pro" \| "iphone-18-pro-max" \| "ipad-pro-13-m5";	3	export type DeviceModelId = "iphone-18-pro" \| "iphone-18-pro-max" \| "ipad-pro-13-m5" \| "iphone-duo";
4		4
5	export interface DeviceModelSource {	5	export interface DeviceModelSource {
6	readonly id: DeviceModelId;	6	readonly id: DeviceModelId;
...	@@ -18,6 +18,7 @@ export type DeviceAssetSource = DeviceModelSource \| DeviceAccessorySource;	...	@@ -18,6 +18,7 @@ export type DeviceAssetSource = DeviceModelSource \| DeviceAccessorySource;
18	/** Match actual hardware, never stretch an available model to impersonate another device. */	18	/** Match actual hardware, never stretch an available model to impersonate another device. */
19	export function resolveDeviceModelId(platform: DevicePlatform, name: string): DeviceModelId \| null {	19	export function resolveDeviceModelId(platform: DevicePlatform, name: string): DeviceModelId \| null {
20	if (platform !== "ios") return null;	20	if (platform !== "ios") return null;
		21	if (/^iPhone Duo$/i.test(name)) return "iphone-duo";
21	if (/^iPhone 18 Pro Max$/i.test(name)) return "iphone-18-pro-max";	22	if (/^iPhone 18 Pro Max$/i.test(name)) return "iphone-18-pro-max";
22	if (/^iPhone 18 Pro$/i.test(name)) return "iphone-18-pro";	23	if (/^iPhone 18 Pro$/i.test(name)) return "iphone-18-pro";
23	if (/^iPad Pro 13-inch $M5$$/i.test(name)) return "ipad-pro-13-m5";	24	if (/^iPad Pro 13-inch $M5$$/i.test(name)) return "ipad-pro-13-m5";

packages/client-runtime/src/device/stream.ts+239-16

...	@@ -19,6 +19,14 @@	...	@@ -19,6 +19,14 @@
19	* The decoder only runs while frames arrive and the viewer is attached; a	19	* The decoder only runs while frames arrive and the viewer is attached; a
20	* hidden panel calls `stop()` so an idle device costs nothing on the GPU.	20	* hidden panel calls `stop()` so an idle device costs nothing on the GPU.
21	*/	21	*/
		22	import {
		23	createDuoControl,
		24	type DuoCommand,
		25	type DuoControlState,
		26	type DuoPose,
		27	} from "./duoControl.ts";
		28	import * as Schema from "effect/Schema";
		29	import * as Option from "effect/Option";
22	import { createCanvasFrameSink, type DeviceFrameSink } from "./frame.ts";	30	import { createCanvasFrameSink, type DeviceFrameSink } from "./frame.ts";
23	import { type DeviceHubAccess, withDeviceHubQuery } from "./hubAccess.ts";	31	import { type DeviceHubAccess, withDeviceHubQuery } from "./hubAccess.ts";
24	import type { DevicePlatform } from "@t3tools/contracts";	32	import type { DevicePlatform } from "@t3tools/contracts";
...	@@ -29,9 +37,55 @@ export interface DeviceScreenSize {	...	@@ -29,9 +37,55 @@ export interface DeviceScreenSize {
29	readonly width: number;	37	readonly width: number;
30	readonly height: number;	38	readonly height: number;
31	readonly orientation: "portrait" \| "portrait_upside_down" \| "landscape_left" \| "landscape_right";	39	readonly orientation: "portrait" \| "portrait_upside_down" \| "landscape_left" \| "landscape_right";
		40	readonly screenId?: number;
		41	readonly supportsHingeAngle?: boolean;
		42	readonly supportsPhysicalOrientation?: boolean;
		43	readonly hingeAngle?: number;
		44	readonly hingePose?: DuoPose \| null;
		45	readonly tableMode?: boolean;
		46	readonly tableModeAvailable?: boolean;
		47	}
		48
		49	const screenConfigSchema = Schema.Struct({
		50	width: Schema.Finite.check(Schema.isGreaterThan(0)),
		51	height: Schema.Finite.check(Schema.isGreaterThan(0)),
		52	orientation: Schema.Literals([
		53	"portrait",
		54	"portrait_upside_down",
		55	"landscape_left",
		56	"landscape_right",
		57	]),
		58	screenId: Schema.optionalKey(Schema.Number),
		59	supportsHingeAngle: Schema.optionalKey(Schema.Boolean),
		60	supportsPhysicalOrientation: Schema.optionalKey(Schema.Boolean),
		61	hingeAngle: Schema.optionalKey(
		62	Schema.Finite.check(Schema.isBetween({ minimum: 0, maximum: 180 })),
		63	),
		64	hingePose: Schema.optionalKey(
		65	Schema.NullOr(Schema.Literals(["closed", "book", "open", "laptop", "tent"])),
		66	),
		67	tableMode: Schema.optionalKey(Schema.Boolean),
		68	tableModeAvailable: Schema.optionalKey(Schema.Boolean),
		69	});
		70	const controlReplySchema = Schema.Struct({
		71	requestId: Schema.Int,
		72	ok: Schema.Boolean,
		73	error: Schema.optionalKey(Schema.String),
		74	});
		75	const decodeScreenConfig = Schema.decodeUnknownOption(screenConfigSchema);
		76	const decodeControlReply = Schema.decodeUnknownOption(controlReplySchema);
		77
		78	export interface DuoPanelSinks {
		79	readonly cover: DeviceFrameSink;
		80	readonly inner: DeviceFrameSink;
		81	/** Invalidate captured input synchronously, before React can commit the new layout. */
		82	readonly onScreen?: (screen: DeviceScreenSize) => void;
32	}	83	}
33		84
34	export interface DeviceStreamEvents {	85	export interface DeviceStreamEvents {
		86	readonly onDuoControl?: (state: DuoControlState) => void;
		87	/** A fixed panel cannot be decoded; the owner should return to the active flat feed. */
		88	readonly onDuoUnavailable?: (detail?: string) => void;
35	readonly onStatus: (status: DeviceStreamStatus, detail?: string) => void;	89	readonly onStatus: (status: DeviceStreamStatus, detail?: string) => void;
36	readonly onScreen: (screen: DeviceScreenSize) => void;	90	readonly onScreen: (screen: DeviceScreenSize) => void;
37	/** The proxy rejected the credential; the owner should refresh access and reconnect. */	91	/** The proxy rejected the credential; the owner should refresh access and reconnect. */
...	@@ -52,6 +106,9 @@ export interface DeviceStreamTarget {	...	@@ -52,6 +106,9 @@ export interface DeviceStreamTarget {
52	readonly access: DeviceHubAccess;	106	readonly access: DeviceHubAccess;
53	/** Native iOS WebViews can use MJPEG without cross-origin fetch or secure-context support. */	107	/** Native iOS WebViews can use MJPEG without cross-origin fetch or secure-context support. */
54	readonly preferMjpeg?: boolean;	108	readonly preferMjpeg?: boolean;
		109	/** Internal fixed-panel feeds share their parent's input session. */
		110	readonly panelId?: 1 \| 3;
		111	readonly videoOnly?: boolean;
55	}	112	}
56		113
57	export type DeviceHardwareButton = "home" \| "back" \| "recents" \| "power" \| "appSwitcher";	114	export type DeviceHardwareButton = "home" \| "back" \| "recents" \| "power" \| "appSwitcher";
...	@@ -219,6 +276,11 @@ export interface DeviceStreamClient {	...	@@ -219,6 +276,11 @@ export interface DeviceStreamClient {
219	readonly pressButton: (button: DeviceHardwareButton) => void;	276	readonly pressButton: (button: DeviceHardwareButton) => void;
220	readonly rotate: () => void;	277	readonly rotate: () => void;
221	readonly setOrientation: (orientation: DeviceScreenSize["orientation"]) => void;	278	readonly setOrientation: (orientation: DeviceScreenSize["orientation"]) => void;
		279	readonly controlDuo: (command: DuoCommand) => void;
		280	/** Switch between one active feed and two fixed-panel feeds without replacing HID. */
		281	readonly setDuoPanels: (panels: DuoPanelSinks \| null) => void;
		282	/** Model UVs already map to the hardware framebuffer. */
		283	readonly sendRawTouch: (phase: "begin" \| "move" \| "end", x: number, y: number) => void;
222	}	284	}
223		285
224	const HID_USAGE_BY_CODE: Readonly<Record<string, number>> = {	286	const HID_USAGE_BY_CODE: Readonly<Record<string, number>> = {
...	@@ -314,6 +376,37 @@ export function createDeviceStreamClient(	...	@@ -314,6 +376,37 @@ export function createDeviceStreamClient(
314	let videoReadTimer: ReturnType<typeof setTimeout> \| null = null;	376	let videoReadTimer: ReturnType<typeof setTimeout> \| null = null;
315	let mjpegImage: HTMLImageElement \| null = null;	377	let mjpegImage: HTMLImageElement \| null = null;
316	let releaseImage: (() => void) \| null = null;	378	let releaseImage: (() => void) \| null = null;
		379	let videoGeneration = 0;
		380	let panelClients: DeviceStreamClient[] = [];
		381	let panelSinks: DuoPanelSinks \| null = null;
		382	let rotationCursor: DeviceScreenSize["orientation"] \| null = null;
		383	let pendingOrientation: { requestId: number } \| null = null;
		384	const duoControl = createDuoControl({
		385	send(request) {
		386	if (socket?.readyState !== WebSocket.OPEN \|\| !screen?.supportsHingeAngle) return false;
		387	if (request.command.control === "physical" && !screen.supportsPhysicalOrientation)
		388	return false;
		389	try {
		390	pendingOrientation = null;
		391	if (request.command.control === "orientation") {
		392	const value = request.command.value;
		393	pendingOrientation = { requestId: request.requestId };
		394	rotationCursor = value;
		395	// Upstream serializes orientation with hinge commands, then broadcasts config.
		396	// An orientation-locked app can keep its framebuffer orientation after the sensor rotates.
		397	socket.send(taggedJson(IOS_MSG_ORIENTATION, { orientation: value }));
		398	} else socket.send(taggedJson(0x10, request));
		399	return true;
		400	} catch {
		401	return false;
		402	}
		403	},
		404	onChange(state) {
		405	if (!state.pending) pendingOrientation = null;
		406	events.onDuoControl?.(state);
		407	},
		408	});
		409	const videoPath = `/helper/${device}${target.panelId ? `/panel/${target.panelId}` : ""}/stream.avcc`;
317		410
318	const mjpegUrl = () => httpUrl(`/helper/${device}/stream.mjpeg`);	411	const mjpegUrl = () => httpUrl(`/helper/${device}/stream.mjpeg`);
319		412
...	@@ -437,16 +530,26 @@ export function createDeviceStreamClient(	...	@@ -437,16 +530,26 @@ export function createDeviceStreamClient(
437	};	530	};
438		531
439	const makeDecoder = () => {	532	const makeDecoder = () => {
		533	const feedGeneration = videoGeneration;
440	const decoder = new VideoDecoder({	534	const decoder = new VideoDecoder({
441	output: (frame) => {	535	output: (frame) => {
442	try {	536	try {
443	if (videoDecoder === decoder) paint(frame, frame.displayWidth, frame.displayHeight);	537	if (
		538	videoDecoder === decoder &&
		539	(platform !== "ios" \|\| feedGeneration === videoGeneration)
		540	)
		541	paint(frame, frame.displayWidth, frame.displayHeight);
444	} finally {	542	} finally {
445	frame.close();	543	frame.close();
446	}	544	}
447	},	545	},
448	error: () => {	546	error: () => {
449	if (stopped \|\| videoDecoder !== decoder) return;	547	if (
		548	stopped \|\|
		549	videoDecoder !== decoder \|\|
		550	(platform === "ios" && feedGeneration !== videoGeneration)
		551	)
		552	return;
450	recoverDecoder();	553	recoverDecoder();
451	},	554	},
452	});	555	});
...	@@ -456,7 +559,7 @@ export function createDeviceStreamClient(	...	@@ -456,7 +559,7 @@ export function createDeviceStreamClient(
456	/** iOS can fall back to MJPEG when the stream's H.264 profile is unsupported. */	559	/** iOS can fall back to MJPEG when the stream's H.264 profile is unsupported. */
457	const configureDecoder = async (	560	const configureDecoder = async (
458	config: VideoDecoderConfig,	561	config: VideoDecoderConfig,
459	isCurrent: () => boolean,	562	isCurrent = () => !stopped,
460	): Promise<boolean> => {	563	): Promise<boolean> => {
461	const epoch = decoderEpoch;	564	const epoch = decoderEpoch;
462	const full: VideoDecoderConfig = { ...config, optimizeForLatency: true };	565	const full: VideoDecoderConfig = { ...config, optimizeForLatency: true };
...	@@ -525,19 +628,32 @@ export function createDeviceStreamClient(	...	@@ -525,19 +628,32 @@ export function createDeviceStreamClient(
525		628
526	// iOS video: fetch the AVCC body and demux into the decoder.	629	// iOS video: fetch the AVCC body and demux into the decoder.
527	const readIosVideo = async () => {	630	const readIosVideo = async () => {
		631	const lifecycle = generation;
		632	const feedGeneration = ++videoGeneration;
528	const demuxer = new AvccDemuxer();	633	const demuxer = new AvccDemuxer();
529	const session = generation;
530	const videoController = new AbortController();	634	const videoController = new AbortController();
531	controller = videoController;	635	controller = videoController;
532	const isCurrent = () => !stopped && generation === session && controller === videoController;	636	const isCurrent = () =>
		637	!stopped &&
		638	generation === lifecycle &&
		639	videoGeneration === feedGeneration &&
		640	controller === videoController;
533	let retryDetail: string \| undefined;	641	let retryDetail: string \| undefined;
534	try {	642	try {
535	const response = await fetch(httpUrl(`/helper/${device}/stream.avcc`), {	643	const response = await fetch(httpUrl(videoPath), {
536	signal: videoController.signal,	644	signal: videoController.signal,
537	credentials: access.credentials ? "include" : "same-origin",	645	credentials: access.credentials ? "include" : "same-origin",
538	});	646	});
539	if (!isCurrent()) return;	647	if (!isCurrent()) {
		648	await response.body?.cancel();
		649	return;
		650	}
540	if (response.status === 401 \|\| response.status === 403) return handleUnauthorized();	651	if (response.status === 401 \|\| response.status === 403) return handleUnauthorized();
		652	if (target.panelId && [400, 404, 405, 410].includes(response.status)) {
		653	await response.body?.cancel();
		654	setStatus("error", "This Device Hub does not provide fixed Duo display feeds.");
		655	return;
		656	}
541	if (!response.ok \|\| !response.body) throw new Error(`stream ${response.status}`);	657	if (!response.ok \|\| !response.body) throw new Error(`stream ${response.status}`);
542	const reader = response.body.getReader();	658	const reader = response.body.getReader();
543	for (;;) {	659	for (;;) {
...	@@ -579,7 +695,12 @@ export function createDeviceStreamClient(	...	@@ -579,7 +695,12 @@ export function createDeviceStreamClient(
579	if (!configured) {	695	if (!configured) {
580	await reader.cancel().catch(() => {});	696	await reader.cancel().catch(() => {});
581	if (!isCurrent()) return;	697	if (!isCurrent()) return;
582	fallBackToMjpeg();	698	if (target.videoOnly)
		699	setStatus(
		700	"error",
		701	`This browser cannot decode the Duo panel's ${avcCodecString(chunk.payload)} stream.`,
		702	);
		703	else fallBackToMjpeg();
583	return;	704	return;
584	}	705	}
585	break;	706	break;
...	@@ -630,6 +751,42 @@ export function createDeviceStreamClient(	...	@@ -630,6 +751,42 @@ export function createDeviceStreamClient(
630	}	751	}
631	};	752	};
632		753
		754	const startDuoVideo = (panels: DuoPanelSinks) => {
		755	for (const panel of panelClients) panel.stop();
		756	// Physical handoff elects a native surface. Fixed-panel encoders can keep
		757	// an inactive shutdown frame after election, so this build uses one active
		758	// feed instead of decoding a third stream alongside the two fixed feeds.
		759	const ids = screen?.supportsPhysicalOrientation ? ([null] as const) : ([1, 3] as const);
		760	panelClients = ids.map((id) => {
		761	const output = id === 1 ? panels.cover : panels.inner;
		762	return createDeviceStreamClient(
		763	{ ...target, ...(id === null ? {} : { panelId: id }), videoOnly: true },
		764	{
		765	present(source, width, height) {
		766	if (id === null) {
		767	return sink.present(source, width, height);
		768	}
		769	// An inactive native LCD can emit its shutdown black frame. Retain its last useful image.
		770	if (screen?.screenId !== id) return true;
		771	const retained = output.present(source, width, height);
		772	const primary = sink.present(source, width, height);
		773	return retained && primary;
		774	},
		775	},
		776	{
		777	onStatus: (status, detail) => {
		778	if (status === "error") events.onDuoUnavailable?.(detail);
		779	},
		780	onScreen: () => {},
		781	onInputConnected: () => {},
		782	onMjpegFallback: () => {},
		783	onUnauthorized: handleUnauthorized,
		784	},
		785	);
		786	});
		787	for (const panel of panelClients) panel.start();
		788	};
		789
633	// iOS input socket; also carries the screen config the helper pushes.	790	// iOS input socket; also carries the screen config the helper pushes.
634	const connectIosInput = async () => {	791	const connectIosInput = async () => {
635	if (stopped) return;	792	if (stopped) return;
...	@@ -648,12 +805,42 @@ export function createDeviceStreamClient(	...	@@ -648,12 +805,42 @@ export function createDeviceStreamClient(
648	if (stopped \|\| socket !== ws) return;	805	if (stopped \|\| socket !== ws) return;
649	if (!(event.data instanceof ArrayBuffer)) return;	806	if (!(event.data instanceof ArrayBuffer)) return;
650	const bytes = new Uint8Array(event.data);	807	const bytes = new Uint8Array(event.data);
651	if (bytes.length < 1 \|\| bytes[0] !== IOS_TAG_SCREEN_CONFIG) return;	808	if (socket !== ws \|\| stopped \|\| bytes.length < 1) return;
652	try {	809	try {
653	const config = JSON.parse(decoder.decode(bytes.subarray(1))) as DeviceScreenSize;	810	const payload: unknown = JSON.parse(decoder.decode(bytes.subarray(1)));
654	if (config.width > 0 && config.height > 0) {	811	if (bytes[0] === 0x90) {
655	screen = config;	812	const reply = decodeControlReply(payload);
656	events.onScreen(config);	813	if (Option.isSome(reply)) duoControl.receive(reply.value);
		814	} else if (bytes[0] === IOS_TAG_SCREEN_CONFIG) {
		815	const config = decodeScreenConfig(payload);
		816	if (Option.isSome(config)) {
		817	const previous = screen;
		818	screen = config.value;
		819	if (screen.hingePose && screen.hingePose !== previous?.hingePose)
		820	rotationCursor = screen.hingePose === "laptop" ? "landscape_left" : "portrait";
		821	else if (screen.orientation !== previous?.orientation)
		822	rotationCursor = screen.orientation;
		823	panelSinks?.onScreen?.(screen);
		824	events.onScreen(screen);
		825	// A surface election can leave an existing decoder on the former
		826	// encoder description. Reopen only video to acquire the elected
		827	// surface's seed and codec configuration; HID and the viewer stay.
		828	if (
		829	panelSinks &&
		830	screen.supportsPhysicalOrientation &&
		831	previous &&
		832	screen.screenId !== previous.screenId
		833	)
		834	startDuoVideo(panelSinks);
		835	if (pendingOrientation) {
		836	const receipt = pendingOrientation;
		837	pendingOrientation = null;
		838	duoControl.receive({
		839	requestId: receipt.requestId,
		840	ok: true,
		841	});
		842	}
		843	}
657	}	844	}
658	} catch {	845	} catch {
659	// Ignore malformed config frames.	846	// Ignore malformed config frames.
...	@@ -662,6 +849,8 @@ export function createDeviceStreamClient(	...	@@ -662,6 +849,8 @@ export function createDeviceStreamClient(
662	ws.onclose = (event) => {	849	ws.onclose = (event) => {
663	if (socket !== ws) return;	850	if (socket !== ws) return;
664	socket = null;	851	socket = null;
		852	duoControl.clear();
		853	rotationCursor = null;
665	if (stopped) return;	854	if (stopped) return;
666	events.onInputConnected(	855	events.onInputConnected(
667	false,	856	false,
...	@@ -757,7 +946,7 @@ export function createDeviceStreamClient(	...	@@ -757,7 +946,7 @@ export function createDeviceStreamClient(
757	configuring = false;	946	configuring = false;
758	connecting();	947	connecting();
759	if (platform === "ios") {	948	if (platform === "ios") {
760	void connectIosInput();	949	if (!target.videoOnly) void connectIosInput();
761	if (useWebCodecs) void readIosVideo();	950	if (useWebCodecs) void readIosVideo();
762	else fallBackToMjpeg();	951	else fallBackToMjpeg();
763	} else if (useWebCodecs) {	952	} else if (useWebCodecs) {
...	@@ -771,6 +960,12 @@ export function createDeviceStreamClient(	...	@@ -771,6 +960,12 @@ export function createDeviceStreamClient(
771	if (stopped) return;	960	if (stopped) return;
772	stopped = true;	961	stopped = true;
773	generation++;	962	generation++;
		963	videoGeneration++;
		964	duoControl.clear();
		965	rotationCursor = null;
		966	for (const panel of panelClients) panel.stop();
		967	panelClients = [];
		968	panelSinks = null;
774	mjpeg = false;	969	mjpeg = false;
775	clearFrameTimer();	970	clearFrameTimer();
776	if (videoReadTimer !== null) clearTimeout(videoReadTimer);	971	if (videoReadTimer !== null) clearTimeout(videoReadTimer);
...	@@ -814,6 +1009,29 @@ export function createDeviceStreamClient(	...	@@ -814,6 +1009,29 @@ export function createDeviceStreamClient(
814	start,	1009	start,
815	stop,	1010	stop,
816	setMjpegImage,	1011	setMjpegImage,
		1012	controlDuo: duoControl.enqueue,
		1013	sendRawTouch: (phase, x, y) => {
		1014	if (platform === "ios") send(taggedJson(IOS_MSG_TOUCH, { type: phase, x, y }));
		1015	},
		1016	setDuoPanels(panels) {
		1017	if (platform !== "ios" \|\| target.videoOnly \|\| stopped \|\| panelSinks === panels) return;
		1018	if (panels && !screen?.supportsHingeAngle) return;
		1019	panelSinks = panels;
		1020	videoGeneration++;
		1021	controller?.abort();
		1022	controller = null;
		1023	closeDecoder();
		1024	const retry = retryTimers.get("video");
		1025	if (retry) clearTimeout(retry);
		1026	retryTimers.delete("video");
		1027	for (const panel of panelClients) panel.stop();
		1028	panelClients = [];
		1029	if (!panels) {
		1030	if (useWebCodecs) void readIosVideo();
		1031	return;
		1032	}
		1033	startDuoVideo(panels);
		1034	},
817	sendTouch: (phase, x, y) => {	1035	sendTouch: (phase, x, y) => {
818	if (platform === "ios") {	1036	if (platform === "ios") {
819	send(taggedJson(IOS_MSG_TOUCH, { type: phase, ...rawPoint(x, y) }));	1037	send(taggedJson(IOS_MSG_TOUCH, { type: phase, ...rawPoint(x, y) }));
...	@@ -856,10 +1074,15 @@ export function createDeviceStreamClient(	...	@@ -856,10 +1074,15 @@ export function createDeviceStreamClient(
856	},	1074	},
857	rotate: () => {	1075	rotate: () => {
858	if (platform !== "ios") return;	1076	if (platform !== "ios") return;
859	const current = screen?.orientation ?? "portrait";	1077	const current = screen?.supportsHingeAngle
		1078	? (rotationCursor ?? screen.orientation)
		1079	: (screen?.orientation ?? "portrait");
860	const next =	1080	const next =
861	IOS_ORIENTATIONS[(IOS_ORIENTATIONS.indexOf(current) + 1) % IOS_ORIENTATIONS.length]!;	1081	IOS_ORIENTATIONS[(IOS_ORIENTATIONS.indexOf(current) + 1) % IOS_ORIENTATIONS.length]!;
862	send(taggedJson(IOS_MSG_ORIENTATION, { orientation: next }));	1082	if (screen?.supportsHingeAngle) {
		1083	rotationCursor = next;
		1084	duoControl.enqueue({ control: "orientation", value: next });
		1085	} else send(taggedJson(IOS_MSG_ORIENTATION, { orientation: next }));
863	},	1086	},
864	setOrientation: (orientation) => {	1087	setOrientation: (orientation) => {
865	if (platform === "ios") send(taggedJson(IOS_MSG_ORIENTATION, { orientation }));	1088	if (platform === "ios") send(taggedJson(IOS_MSG_ORIENTATION, { orientation }));

author	Julius Marminge <julius0216@outlook.com> 2026-09-24 18:07:33-07:00
committer	GitHub <noreply@github.com> 2026-09-24 18:07:33-07:00
log	46f3c2ca2c99ceab0547452ac863232944621fce
tree	c91f46d82e70aef3f58aa7482f9d5a09e3dfe49f
parent	fd465100581055383126aab922a8da2c5d7952fd
signature	Signed by PGP key B5690EEEBB952194

...	@@ -4,6 +4,7 @@ import {	...	@@ -4,6 +4,7 @@ import {
4	type DeviceModelSource,	4	type DeviceModelSource,
5	} from "@t3tools/client-runtime/device/model";	5	} from "@t3tools/client-runtime/device/model";
6	import type { DevicePlatform } from "@t3tools/contracts";	6	import type { DevicePlatform } from "@t3tools/contracts";
		7	import iphoneDuo from "./models/iphone-duo.glb?url";
7	import iphone18Pro from "./models/iphone-18-pro.glb?url";	8	import iphone18Pro from "./models/iphone-18-pro.glb?url";
8	import iphone18ProMax from "./models/iphone-18-pro-max.glb?url";	9	import iphone18ProMax from "./models/iphone-18-pro-max.glb?url";
9	import magicKeyboard from "./models/ipad-pro-13-m5-magic-keyboard.glb?url";	10	import magicKeyboard from "./models/ipad-pro-13-m5-magic-keyboard.glb?url";
...	@@ -11,6 +12,7 @@ import ipadPro13M5 from "./models/ipad-pro-13-m5.glb?url";	...	@@ -11,6 +12,7 @@ import ipadPro13M5 from "./models/ipad-pro-13-m5.glb?url";
11		12
12	// Bundled URLs follow the client origin in local, desktop, hosted and remote sessions.	13	// Bundled URLs follow the client origin in local, desktop, hosted and remote sessions.
13	const models: Record<DeviceModelSource["id"], DeviceModelSource> = {	14	const models: Record<DeviceModelSource["id"], DeviceModelSource> = {
		15	"iphone-duo": { id: "iphone-duo", url: iphoneDuo },
14	"iphone-18-pro": { id: "iphone-18-pro", url: iphone18Pro },	16	"iphone-18-pro": { id: "iphone-18-pro", url: iphone18Pro },
15	"iphone-18-pro-max": { id: "iphone-18-pro-max", url: iphone18ProMax },	17	"iphone-18-pro-max": { id: "iphone-18-pro-max", url: iphone18ProMax },
16	"ipad-pro-13-m5": { id: "ipad-pro-13-m5", url: ipadPro13M5 },	18	"ipad-pro-13-m5": { id: "ipad-pro-13-m5", url: ipadPro13M5 },

feat(web): add iPhone Duo 3D controls (#12813)