authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-09-24 21:50:58-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-09-24 21:50:58-07:00
log9957349c0a066a9efc657600fcb901c9b9cca34b
treea14db3e7576ca423c16415f1acee9bbd3c3dc5e2
parent6391be2729566a1fe1045ff8617f003580b37bfd
signature Signed by PGP key B5690EEEBB952194

feat(web): control Android foldables in the Device panel (#13534)

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

11 files changed, 985 insertions(+), 15 deletions(-)

apps/server/src/device/DeviceHubProxy.test.ts+26
...@@ -111,6 +111,32 @@ describe("device hub proxy", () => {...@@ -111,6 +111,32 @@ describe("device hub proxy", () => {
111 await response.text();111 await response.text();
112 });112 });
113113
114 it("reads Android fold state but requires operate scope to change it", async () => {
115 const path = "http://t3.test/api/device-hub/vendor/serve-emu/api/fold?device=emulator-5554";
116 const reader = fixture([AuthOrchestrationReadScope]);
117 const read = await reader.handler(new Request(path));
118 expect(read.status).toBe(200);
119 await read.text();
120 expect(reader.requests).toEqual([
121 "http://hub.test/vendor/serve-emu/api/fold?device=emulator-5554",
122 ]);
123 const denied = await reader.handler(
124 new Request(path, { method: "POST", body: '{"posture":"closed"}' }),
125 );
126 expect(denied.status).toBe(403);
127 expect(reader.requests).toHaveLength(1);
128
129 const operator = fixture([AuthOrchestrationOperateScope]);
130 const changed = await operator.handler(
131 new Request(path, { method: "POST", body: '{"posture":"closed"}' }),
132 );
133 expect(changed.status).toBe(200);
134 await changed.text();
135 expect(operator.requests).toEqual([
136 "http://hub.test/vendor/serve-emu/api/fold?device=emulator-5554",
137 ]);
138 });
139
114 it("never forwards the vendor shell endpoint", async () => {140 it("never forwards the vendor shell endpoint", async () => {
115 const { handler, requests } = fixture([AuthOrchestrationOperateScope]);141 const { handler, requests } = fixture([AuthOrchestrationOperateScope]);
116 expect(142 expect(
apps/server/src/device/DeviceHubProxy.ts+3-2
...@@ -44,7 +44,7 @@ const ALLOWED_PATHS: ReadonlyArray<RegExp> = [...@@ -44,7 +44,7 @@ const ALLOWED_PATHS: ReadonlyArray<RegExp> = [
44 /^\/vendor\/serve-sim\/helper\/[^/]+\/(stream\.mjpeg|stream\.avcc|config|health|ax|foreground)$/,44 /^\/vendor\/serve-sim\/helper\/[^/]+\/(stream\.mjpeg|stream\.avcc|config|health|ax|foreground)$/,
45 /^\/vendor\/serve-sim\/helper\/[^/]+\/panel\/(1|3)\/stream\.avcc$/,45 /^\/vendor\/serve-sim\/helper\/[^/]+\/panel\/(1|3)\/stream\.avcc$/,
46 /^\/vendor\/serve-sim\/appstate$/,46 /^\/vendor\/serve-sim\/appstate$/,
47 /^\/vendor\/serve-emu\/api\/(devices|screenshot|stream-mode|stream-settings|accessibility)$/,47 /^\/vendor\/serve-emu\/api\/(devices|screenshot|stream-mode|stream-settings|accessibility|fold)$/,
48 /^\/vendor\/serve-emu\/health$/,48 /^\/vendor\/serve-emu\/health$/,
49];49];
5050
...@@ -52,6 +52,7 @@ const ALLOWED_PATHS: ReadonlyArray<RegExp> = [...@@ -52,6 +52,7 @@ const ALLOWED_PATHS: ReadonlyArray<RegExp> = [
52const MUTABLE_PATHS: ReadonlyArray<RegExp> = [52const MUTABLE_PATHS: ReadonlyArray<RegExp> = [
53 /^\/vendor\/serve-sim\/api\/screenshot$/,53 /^\/vendor\/serve-sim\/api\/screenshot$/,
54 /^\/vendor\/serve-emu\/api\/(screenshot|stream-mode|stream-settings)$/,54 /^\/vendor\/serve-emu\/api\/(screenshot|stream-mode|stream-settings)$/,
55 /^\/vendor\/serve-emu\/api\/fold$/,
55];56];
5657
57const ALLOWED_WS_PATHS: ReadonlyArray<RegExp> = [58const ALLOWED_WS_PATHS: ReadonlyArray<RegExp> = [
...@@ -204,7 +205,7 @@ const handler = Effect.gen(function* () {...@@ -204,7 +205,7 @@ const handler = Effect.gen(function* () {
204 }205 }
205 const controlsDevice =206 const controlsDevice =
206 (upgrade && hubPath !== "/api/devices/ws") ||207 (upgrade && hubPath !== "/api/devices/ws") ||
207 (!readOnly && /\/api\/stream-(mode|settings)$/.test(hubPath));208 (!readOnly && /\/api\/(stream-(mode|settings)|fold)$/.test(hubPath));
208 yield* authenticate(controlsDevice ? AuthOrchestrationOperateScope : AuthOrchestrationReadScope);209 yield* authenticate(controlsDevice ? AuthOrchestrationOperateScope : AuthOrchestrationReadScope);
209 const devices = yield* DeviceService.DeviceService;210 const devices = yield* DeviceService.DeviceService;
210 const ready = yield* devices.currentReadiness(url.value.searchParams.get("hostId") ?? undefined);211 const ready = yield* devices.currentReadiness(url.value.searchParams.get("hostId") ?? undefined);
apps/web/src/components/device/DeviceAndroidFoldControls.tsx created+155
...@@ -0,0 +1,155 @@
1import type { DeviceHubAccess } from "@t3tools/client-runtime/state/deviceHubAccess";
2import { useEffect, useRef, useState } from "react";
3import { Button } from "~/components/ui/button";
4import { Tooltip, TooltipPopup, TooltipTrigger } from "~/components/ui/tooltip";
5import {
6 readAndroidFold,
7 setAndroidFold,
8 type AndroidFoldPosture,
9 type AndroidFoldState,
10} from "./deviceFold";
11import { DeviceDuoGlyph } from "./DeviceDuoGlyph";
12
13/** Capability comes from the emulator, not its AVD name or screen dimensions. */
14export function DeviceAndroidFoldControls(props: {
15 readonly access: DeviceHubAccess;
16 readonly deviceId: string;
17 readonly visible: boolean;
18 readonly enabled: boolean;
19 readonly screenWidth: number | undefined;
20 readonly screenHeight: number | undefined;
21 readonly onFoldAngle: (angle: number | null) => void;
22}) {
23 const [fold, setFold] = useState<AndroidFoldState | null>(null);
24 const [pending, setPending] = useState(false);
25 const [error, setError] = useState<string | null>(null);
26 const busy = useRef(false);
27 const mounted = useRef(true);
28 const { access, deviceId, visible, enabled, screenWidth, screenHeight, onFoldAngle } = props;
29
30 useEffect(() => {
31 mounted.current = true;
32 return () => {
33 mounted.current = false;
34 };
35 }, []);
36
37 useEffect(() => {
38 if (!visible || !enabled || pending || !screenWidth || !screenHeight) return;
39 const controller = new AbortController();
40 let retry: number | undefined;
41 const read = () => {
42 void readAndroidFold(access, deviceId, controller.signal)
43 .then((next) => {
44 if (!controller.signal.aborted) {
45 setFold(next);
46 onFoldAngle(
47 next.supported ? (next.hingeAngle ?? (next.posture === "closed" ? 0 : 180)) : null,
48 );
49 }
50 })
51 .catch(() => {
52 if (!controller.signal.aborted) retry = window.setTimeout(read, 3_000);
53 });
54 };
55 read();
56 return () => {
57 controller.abort();
58 window.clearTimeout(retry);
59 };
60 }, [access, deviceId, visible, enabled, screenWidth, screenHeight, pending, onFoldAngle]);
61
62 if (!fold?.supported || !props.visible) return null;
63
64 const change = (posture: AndroidFoldPosture) => {
65 if (busy.current || !props.enabled) return;
66 busy.current = true;
67 setPending(true);
68 setError(null);
69 props.onFoldAngle(posture === "closed" ? 0 : 180);
70 const controller = new AbortController();
71 const timeout = window.setTimeout(() => controller.abort(), 12_000);
72 void setAndroidFold(props.access, props.deviceId, posture, controller.signal)
73 .then((next) => {
74 if (mounted.current) {
75 setFold(next);
76 props.onFoldAngle(next.hingeAngle ?? (next.posture === "closed" ? 0 : 180));
77 }
78 })
79 .catch((cause: unknown) => {
80 if (mounted.current) {
81 props.onFoldAngle(fold.hingeAngle ?? (fold.posture === "closed" ? 0 : 180));
82 setError(
83 controller.signal.aborted
84 ? "Fold command timed out."
85 : cause instanceof Error
86 ? cause.message
87 : "Could not change fold posture.",
88 );
89 }
90 })
91 .finally(() => {
92 window.clearTimeout(timeout);
93 busy.current = false;
94 if (mounted.current) setPending(false);
95 });
96 };
97
98 return (
99 <div aria-label="Android fold controls" className="flex flex-col items-center gap-2">
100 <div className="pointer-events-auto flex shrink-0 flex-col items-center gap-1 rounded-full border border-border/50 bg-background/80 p-1 shadow-sm">
101 <Tooltip>
102 <TooltipTrigger
103 render={
104 <Button
105 size="icon"
106 variant={fold.posture === "closed" ? "secondary" : "ghost"}
107 aria-label="Fold device"
108 aria-pressed={fold.posture === "closed"}
109 disabled={pending || !props.enabled}
110 onClick={() => change("closed")}
111 />
112 }
113 >
114 <DeviceDuoGlyph pose="closed" />
115 </TooltipTrigger>
116 <TooltipPopup side="left">Fold device</TooltipPopup>
117 </Tooltip>
118 <Tooltip>
119 <TooltipTrigger
120 render={
121 <Button
122 size="icon"
123 variant={fold.posture === "opened" ? "secondary" : "ghost"}
124 aria-label="Unfold device"
125 aria-pressed={fold.posture === "opened"}
126 disabled={pending || !props.enabled}
127 onClick={() => change("opened")}
128 />
129 }
130 >
131 <DeviceDuoGlyph pose="open" />
132 </TooltipTrigger>
133 <TooltipPopup side="left">Unfold device</TooltipPopup>
134 </Tooltip>
135 </div>
136 {error ? (
137 <Tooltip>
138 <TooltipTrigger
139 render={
140 <span
141 tabIndex={0}
142 role="alert"
143 aria-label={error}
144 className="pointer-events-auto text-xs text-destructive"
145 >
146 !
147 </span>
148 }
149 />
150 <TooltipPopup side="left">{error}</TooltipPopup>
151 </Tooltip>
152 ) : null}
153 </div>
154 );
155}
apps/web/src/components/device/DevicePhoneViewport.tsx+8
...@@ -22,6 +22,7 @@ export function DevicePhoneViewport(props: {...@@ -22,6 +22,7 @@ export function DevicePhoneViewport(props: {
22 readonly onInputCancel: (cancel: (() => void) | null) => void;22 readonly onInputCancel: (cancel: (() => void) | null) => void;
23 readonly client: RefObject<DeviceStreamClient | null>;23 readonly client: RefObject<DeviceStreamClient | null>;
24 readonly screen: DeviceScreenSize | null;24 readonly screen: DeviceScreenSize | null;
25 readonly foldAngle: number | null;
25 readonly onUnavailable: () => void;26 readonly onUnavailable: () => void;
26}) {27}) {
27 const hostRef = useRef<HTMLDivElement | null>(null);28 const hostRef = useRef<HTMLDivElement | null>(null);
...@@ -29,11 +30,17 @@ export function DevicePhoneViewport(props: {...@@ -29,11 +30,17 @@ export function DevicePhoneViewport(props: {
29 const viewerRef = useRef<PhoneViewer | null>(null);30 const viewerRef = useRef<PhoneViewer | null>(null);
30 const interactionRef = useRef<ReturnType<typeof createPhoneInteraction> | null>(null);31 const interactionRef = useRef<ReturnType<typeof createPhoneInteraction> | null>(null);
31 const screenRef = useRef(props.screen);32 const screenRef = useRef(props.screen);
33 const foldAngleRef = useRef(props.foldAngle);
32 const profileRef = useRef(props.profile);34 const profileRef = useRef(props.profile);
33 const modelRef = useRef(props.model);35 const modelRef = useRef(props.model);
34 const accessoryRef = useRef(props.accessory);36 const accessoryRef = useRef(props.accessory);
35 const { source, onFrameListener, client, onInputCancel, onResetReady, onUnavailable } = props;37 const { source, onFrameListener, client, onInputCancel, onResetReady, onUnavailable } = props;
3638
39 useEffect(() => {
40 foldAngleRef.current = props.foldAngle;
41 viewerRef.current?.setFoldAngle(props.foldAngle);
42 }, [props.foldAngle]);
43
37 useEffect(() => {44 useEffect(() => {
38 screenRef.current = props.screen;45 screenRef.current = props.screen;
39 profileRef.current = props.profile;46 profileRef.current = props.profile;
...@@ -76,6 +83,7 @@ export function DevicePhoneViewport(props: {...@@ -76,6 +83,7 @@ export function DevicePhoneViewport(props: {
76 profile: profileRef.current,83 profile: profileRef.current,
77 model: modelRef.current,84 model: modelRef.current,
78 accessory: accessoryRef.current,85 accessory: accessoryRef.current,
86 foldAngle: foldAngleRef.current,
79 });87 });
80 viewerRef.current = viewer;88 viewerRef.current = viewer;
81 onResetReady(viewer.resetPose);89 onResetReady(viewer.resetPose);
apps/web/src/components/device/DeviceStreamView.tsx+39-3
...@@ -10,6 +10,7 @@ import { deviceKeyboard, deviceModel } from "./deviceModels";...@@ -10,6 +10,7 @@ import { deviceKeyboard, deviceModel } from "./deviceModels";
10import { fitDeviceFrame } from "./deviceFrameLayout";10import { fitDeviceFrame } from "./deviceFrameLayout";
11import { DeviceDuoViewport } from "./DeviceDuoViewport";11import { DeviceDuoViewport } from "./DeviceDuoViewport";
12import { DeviceDuoControls } from "./DeviceDuoControls";12import { DeviceDuoControls } from "./DeviceDuoControls";
13import { DeviceAndroidFoldControls } from "./DeviceAndroidFoldControls";
13import type { DuoControlState } from "@t3tools/client-runtime/device/duo-control";14import type { DuoControlState } from "@t3tools/client-runtime/device/duo-control";
14import { DevicePhoneViewport } from "./DevicePhoneViewport";15import { DevicePhoneViewport } from "./DevicePhoneViewport";
15import { DeviceLoadingView } from "./DeviceLoadingView";16import { DeviceLoadingView } from "./DeviceLoadingView";
...@@ -94,7 +95,9 @@ export function DeviceStreamView(props: {...@@ -94,7 +95,9 @@ export function DeviceStreamView(props: {
94 const clientRef = useRef<DeviceStreamClient | null>(null);95 const clientRef = useRef<DeviceStreamClient | null>(null);
95 const [status, setStatus] = useState<DeviceStreamStatus>("connecting");96 const [status, setStatus] = useState<DeviceStreamStatus>("connecting");
96 const [detail, setDetail] = useState<string | undefined>(undefined);97 const [detail, setDetail] = useState<string | undefined>(undefined);
98 const [showRestartNotice, setShowRestartNotice] = useState(false);
97 const [screen, setScreen] = useState<DeviceScreenSize | null>(null);99 const [screen, setScreen] = useState<DeviceScreenSize | null>(null);
100 const [foldAngle, setFoldAngle] = useState<number | null>(null);
98 const [mjpegUrl, setMjpegUrl] = useState<string | null>(null);101 const [mjpegUrl, setMjpegUrl] = useState<string | null>(null);
99 const [mjpegGeneration, setMjpegGeneration] = useState(0);102 const [mjpegGeneration, setMjpegGeneration] = useState(0);
100 const attachMjpegImage = useCallback((image: HTMLImageElement | null) => {103 const attachMjpegImage = useCallback((image: HTMLImageElement | null) => {
...@@ -121,6 +124,7 @@ export function DeviceStreamView(props: {...@@ -121,6 +124,7 @@ export function DeviceStreamView(props: {
121 onStatus: (next, nextDetail) => {124 onStatus: (next, nextDetail) => {
122 setStatus(next);125 setStatus(next);
123 setDetail(nextDetail);126 setDetail(nextDetail);
127 if (next !== "connecting") setShowRestartNotice(false);
124 },128 },
125 onScreen: (next) => {129 onScreen: (next) => {
126 setScreen(next);130 setScreen(next);
...@@ -136,6 +140,7 @@ export function DeviceStreamView(props: {...@@ -136,6 +140,7 @@ export function DeviceStreamView(props: {
136 },140 },
137 onInputConnected: (connected, detail) => {141 onInputConnected: (connected, detail) => {
138 setInputState({ connected, ...(detail ? { detail } : {}) });142 setInputState({ connected, ...(detail ? { detail } : {}) });
143 if (!connected) setShowRestartNotice(false);
139 onHandle?.({144 onHandle?.({
140 pressButton: client.pressButton,145 pressButton: client.pressButton,
141 rotate: client.rotate,146 rotate: client.rotate,
...@@ -183,15 +188,27 @@ export function DeviceStreamView(props: {...@@ -183,15 +188,27 @@ export function DeviceStreamView(props: {
183 return w / h;188 return w / h;
184 }, [props.platform, screen]);189 }, [props.platform, screen]);
185190
191 // Android restarts its encoder when a fold changes the framebuffer size.
192 // Keep the last decoded frame and viewer mounted while the next keyframe arrives.
193 const retainingAndroidFrame =
194 props.platform === "android" &&
195 status === "connecting" &&
196 inputState.connected &&
197 screen !== null;
186 const showPhone =198 const showPhone =
187 props.allowPhoneView &&199 props.allowPhoneView &&
188 status === "streaming" &&200 (status === "streaming" || retainingAndroidFrame) &&
189 props.visible &&201 props.visible &&
190 presentation === "phone" &&202 presentation === "phone" &&
191 !phoneUnavailable &&203 !phoneUnavailable &&
192 !mjpegUrl &&204 !mjpegUrl &&
193 !props.axOverlay &&205 !props.axOverlay &&
194 (!isDuo || screen?.supportsHingeAngle === true);206 (!isDuo || screen?.supportsHingeAngle === true);
207 useEffect(() => {
208 if (!retainingAndroidFrame || !showPhone) return;
209 const timeout = window.setTimeout(() => setShowRestartNotice(true), 2_000);
210 return () => window.clearTimeout(timeout);
211 }, [retainingAndroidFrame, showPhone]);
195 const controlsInset = props.renderControls && !showPhone ? CONTROLS_RAIL_WIDTH : 0;212 const controlsInset = props.renderControls && !showPhone ? CONTROLS_RAIL_WIDTH : 0;
196213
197 // The frame is the largest box at `aspect` that fits the container, so a214 // The frame is the largest box at `aspect` that fits the container, so a
...@@ -330,7 +347,18 @@ export function DeviceStreamView(props: {...@@ -330,7 +347,18 @@ export function DeviceStreamView(props: {
330 streaming: status === "streaming",347 streaming: status === "streaming",
331 phoneUnavailableReason,348 phoneUnavailableReason,
332 foldingControls:349 foldingControls:
333 showPhone && isDuo && screen?.supportsHingeAngle ? (350 props.platform === "android" && access ? (
351 <DeviceAndroidFoldControls
352 key={`${props.hostId}:${props.deviceId}`}
353 access={access}
354 deviceId={props.deviceId}
355 visible={props.visible}
356 enabled={status === "streaming"}
357 screenWidth={screen?.width}
358 screenHeight={screen?.height}
359 onFoldAngle={setFoldAngle}
360 />
361 ) : showPhone && isDuo && screen?.supportsHingeAngle ? (
334 <DeviceDuoControls362 <DeviceDuoControls
335 screen={screen}363 screen={screen}
336 state={duoControl}364 state={duoControl}
...@@ -471,6 +499,7 @@ export function DeviceStreamView(props: {...@@ -471,6 +499,7 @@ export function DeviceStreamView(props: {
471 onInputCancel={onInputCancel}499 onInputCancel={onInputCancel}
472 onResetReady={onResetReady}500 onResetReady={onResetReady}
473 screen={screen}501 screen={screen}
502 foldAngle={foldAngle}
474 onUnavailable={onPhoneUnavailable}503 onUnavailable={onPhoneUnavailable}
475 />504 />
476 ) : null}505 ) : null}
...@@ -503,7 +532,14 @@ export function DeviceStreamView(props: {...@@ -503,7 +532,14 @@ export function DeviceStreamView(props: {
503 </span>532 </span>
504 </div>533 </div>
505 ) : null}534 ) : null}
506 {status !== "streaming" ? (535 {retainingAndroidFrame && showPhone && showRestartNotice ? (
536 <div className="pointer-events-none absolute inset-x-0 bottom-0 flex justify-center p-2">
537 <span className="rounded-md bg-background/85 px-2 py-1 text-xs text-muted-foreground">
538 Waiting for device video…
539 </span>
540 </div>
541 ) : null}
542 {status !== "streaming" && !(retainingAndroidFrame && showPhone) ? (
507 <div className="absolute inset-0">543 <div className="absolute inset-0">
508 <DeviceLoadingView544 <DeviceLoadingView
509 name={props.deviceName ?? "Device"}545 name={props.deviceName ?? "Device"}
apps/web/src/components/device/deviceFold.ts created+72
...@@ -0,0 +1,72 @@
1import type { DeviceHubAccess } from "@t3tools/client-runtime/state/deviceHubAccess";
2import { withDeviceHubQuery } from "@t3tools/client-runtime/state/deviceHubAccess";
3
4export type AndroidFoldPosture = "closed" | "opened";
5
6export interface AndroidFoldState {
7 readonly supported: boolean;
8 readonly posture: "closed" | "half_opened" | "opened" | "flipped" | "tent" | null;
9 readonly hingeAngle: number | null;
10}
11
12const isRecord = (value: unknown): value is Record<string, unknown> =>
13 typeof value === "object" && value !== null && !Array.isArray(value);
14
15const parseFold = (payload: unknown): AndroidFoldState => {
16 if (!isRecord(payload) || payload.ok !== true || !isRecord(payload.fold)) {
17 throw new Error("Unexpected Android fold response.");
18 }
19 const { supported, posture, hingeAngle } = payload.fold;
20 if (
21 typeof supported !== "boolean" ||
22 (posture !== null &&
23 posture !== "closed" &&
24 posture !== "half_opened" &&
25 posture !== "opened" &&
26 posture !== "flipped" &&
27 posture !== "tent") ||
28 (hingeAngle !== null && (typeof hingeAngle !== "number" || !Number.isFinite(hingeAngle)))
29 ) {
30 throw new Error("Unexpected Android fold response.");
31 }
32 return { supported, posture, hingeAngle };
33};
34
35const foldRequest = async (
36 access: DeviceHubAccess,
37 deviceId: string,
38 posture?: AndroidFoldPosture,
39 signal?: AbortSignal,
40): Promise<AndroidFoldState> => {
41 const url = withDeviceHubQuery(
42 `${access.httpBase}/vendor/serve-emu/api/fold?${new URLSearchParams({ device: deviceId })}`,
43 access,
44 );
45 const response = await fetch(url, {
46 method: posture ? "POST" : "GET",
47 cache: "no-store",
48 credentials: access.credentials ? "include" : "same-origin",
49 ...(posture
50 ? { headers: { "content-type": "application/json" }, body: JSON.stringify({ posture }) }
51 : {}),
52 ...(signal ? { signal } : {}),
53 });
54 if (!response.ok) {
55 const payload: unknown = await response.json().catch(() => null);
56 const error = isRecord(payload) ? payload.error : null;
57 throw new Error(
58 typeof error === "string" ? error : `Fold command failed (${response.status}).`,
59 );
60 }
61 return parseFold(await response.json());
62};
63
64export const readAndroidFold = (access: DeviceHubAccess, deviceId: string, signal?: AbortSignal) =>
65 foldRequest(access, deviceId, undefined, signal);
66
67export const setAndroidFold = (
68 access: DeviceHubAccess,
69 deviceId: string,
70 posture: AndroidFoldPosture,
71 signal?: AbortSignal,
72) => foldRequest(access, deviceId, posture, signal);
docs/user/devices.md+2
...@@ -44,6 +44,8 @@ interactive. On iPhone Duo, use the fold and stance controls to change its...@@ -44,6 +44,8 @@ interactive. On iPhone Duo, use the fold and stance controls to change its
44physical pose, or pinch over the device to adjust the hinge. Turning the model44physical pose, or pinch over the device to adjust the hinge. Turning the model
45to the other screen switches the live display and touch input to that screen.45to the other screen switches the live display and touch input to that screen.
46**Restore 3D view** returns the device to a screen-facing position.46**Restore 3D view** returns the device to a screen-facing position.
47On supported Android foldables, use **Fold device** and **Unfold device** beside
48the screen to change its posture in either view.
4749
48## Tools50## Tools
4951
packages/client-runtime/src/device/androidFoldScene.test.ts created+57
...@@ -0,0 +1,57 @@
1import { Box3, Mesh, PerspectiveCamera, Texture, Vector3 } from "three";
2import { describe, expect, it } from "vite-plus/test";
3import { createAndroidFoldScene } from "./androidFoldScene.ts";
4import { phoneDisplayLayout } from "./phoneScene.ts";
5
6describe("Android fold scene", () => {
7 it("moves one physical half around the hinge while preserving both screen halves", () => {
8 const texture = new Texture();
9 const scene = createAndroidFoldScene(
10 texture,
11 phoneDisplayLayout({ width: 2200, height: 1840, orientation: "landscape_left" }, 2200, 1840),
12 180,
13 );
14 const moving = scene.orientation.children[0]!;
15 const fixed = scene.orientation.children[1]!;
16 const openWidth = new Box3().setFromObject(scene.root).getSize(new Vector3()).x;
17 const leftScreen = scene.root.getObjectByName("left-inner-screen") as Mesh;
18 const rightScreen = scene.root.getObjectByName("right-inner-screen") as Mesh;
19 leftScreen.geometry.computeBoundingBox();
20 rightScreen.geometry.computeBoundingBox();
21 const creaseWidth =
22 rightScreen.geometry.boundingBox!.min.x - leftScreen.geometry.boundingBox!.max.x;
23 expect(creaseWidth).toBeLessThan(0.01);
24 const continuousScreen = scene.root.getObjectByName("continuous-inner-screen") as Mesh;
25 const positions = continuousScreen.geometry.getAttribute("position");
26 expect(continuousScreen.geometry.index).not.toBeNull();
27 expect(Array.from({ length: positions.count }, (_, i) => positions.getX(i))).toContain(0);
28 scene.setAngle(90);
29 expect(moving.rotation.y).toBeCloseTo(Math.PI / 2);
30 expect(fixed.rotation.y).toBe(0);
31 scene.setAngle(0);
32 const closedWidth = new Box3().setFromObject(scene.root).getSize(new Vector3()).x;
33 expect(closedWidth).toBeLessThan(openWidth * 0.7);
34 expect(scene.root.getObjectByName("cover-screen")?.visible).toBe(true);
35 scene.dispose();
36 texture.dispose();
37 });
38
39 it("maps touches on each open half and the closed cover to the live frame", () => {
40 const texture = new Texture();
41 const scene = createAndroidFoldScene(texture, phoneDisplayLayout(null, 2200, 1840), 180);
42 const camera = new PerspectiveCamera(32, 1, 0.1, 30);
43 camera.position.z = 6;
44 camera.updateMatrixWorld(true);
45 const project = (x: number) => {
46 const point = new Vector3(x, 0, 0.041).project(camera);
47 return scene.screenPoint((point.x + 1) / 2, (1 - point.y) / 2, camera);
48 };
49 expect(project(-0.52)?.x).toBeCloseTo(0.25, 1);
50 expect(project(0.52)?.x).toBeCloseTo(0.75, 1);
51 expect(scene.screenPoint(0.99, 0.5, camera, true)?.x).toBe(1);
52 scene.setAngle(0);
53 expect(project(0.52)?.x).toBeCloseTo(0.5, 1);
54 scene.dispose();
55 texture.dispose();
56 });
57});
packages/client-runtime/src/device/androidFoldScene.ts created+395
...@@ -0,0 +1,395 @@
1import {
2 BoxGeometry,
3 CircleGeometry,
4 CylinderGeometry,
5 ExtrudeGeometry,
6 Group,
7 Mesh,
8 MeshBasicMaterial,
9 MeshPhysicalMaterial,
10 MeshStandardMaterial,
11 PlaneGeometry,
12 Raycaster,
13 Shape,
14 ShapeGeometry,
15 Vector2,
16 Vector3,
17 type Camera,
18 type Texture,
19} from "three";
20import type { PhoneDisplayLayout } from "./phoneScene.ts";
21
22const HALF_WIDTH = 1.04;
23const HEIGHT = 2.2;
24const DEPTH = 0.075;
25const INSET = 0.026;
26const CREASE = 0.004;
27const BEVEL = 0.008;
28// The hinge axis sits just above the inner screens so closed halves meet face to face.
29const PIVOT_Z = DEPTH / 2 + 0.005;
30const SPINE_RADIUS = PIVOT_Z + DEPTH / 2 - 0.002;
31
32function panelPath(side: "left" | "right", inset: number, radius: number, hingeInset = 0) {
33 const left = side === "left" ? -HALF_WIDTH + inset : CREASE / 2 + hingeInset;
34 const right = side === "left" ? -CREASE / 2 - hingeInset : HALF_WIDTH - inset;
35 const bottom = -HEIGHT / 2 + inset;
36 const top = HEIGHT / 2 - inset;
37 const path = new Shape();
38 if (side === "left") {
39 path.moveTo(left + radius, bottom);
40 path.lineTo(right, bottom);
41 path.lineTo(right, top);
42 path.lineTo(left + radius, top);
43 path.quadraticCurveTo(left, top, left, top - radius);
44 path.lineTo(left, bottom + radius);
45 path.quadraticCurveTo(left, bottom, left + radius, bottom);
46 } else {
47 path.moveTo(left, bottom);
48 path.lineTo(right - radius, bottom);
49 path.quadraticCurveTo(right, bottom, right, bottom + radius);
50 path.lineTo(right, top - radius);
51 path.quadraticCurveTo(right, top, right - radius, top);
52 path.lineTo(left, top);
53 }
54 path.closePath();
55 return path;
56}
57
58function roundedRectPath(width: number, height: number, radius: number) {
59 const path = new Shape();
60 path.moveTo(-width / 2 + radius, -height / 2);
61 path.lineTo(width / 2 - radius, -height / 2);
62 path.quadraticCurveTo(width / 2, -height / 2, width / 2, -height / 2 + radius);
63 path.lineTo(width / 2, height / 2 - radius);
64 path.quadraticCurveTo(width / 2, height / 2, width / 2 - radius, height / 2);
65 path.lineTo(-width / 2 + radius, height / 2);
66 path.quadraticCurveTo(-width / 2, height / 2, -width / 2, height / 2 - radius);
67 path.lineTo(-width / 2, -height / 2 + radius);
68 path.quadraticCurveTo(-width / 2, -height / 2, -width / 2 + radius, -height / 2);
69 path.closePath();
70 return path;
71}
72
73function coverPath() {
74 return roundedRectPath(HALF_WIDTH - INSET * 2 - 0.02, HEIGHT - INSET * 2 - 0.04, 0.07);
75}
76
77/** A procedural book-style foldable: one fixed half, one half rotating around a shared hinge. */
78export function createAndroidFoldScene(
79 texture: Texture,
80 layout: PhoneDisplayLayout,
81 initialAngle: number,
82) {
83 const root = new Group();
84 const orientation = new Group();
85 root.add(orientation);
86 const left = new Group();
87 const right = new Group();
88 orientation.add(left, right);
89 const frameMetal = new MeshStandardMaterial({
90 color: 0xa3abb2,
91 metalness: 0.9,
92 roughness: 0.28,
93 });
94 const polishedMetal = new MeshStandardMaterial({
95 color: 0xc4cad0,
96 metalness: 0.95,
97 roughness: 0.16,
98 });
99 const bezel = new MeshPhysicalMaterial({
100 color: 0x0b0d10,
101 metalness: 0.1,
102 roughness: 0.2,
103 clearcoat: 1,
104 });
105 const backGlass = new MeshPhysicalMaterial({
106 color: 0x2c3237,
107 metalness: 0.35,
108 roughness: 0.52,
109 clearcoat: 0.4,
110 clearcoatRoughness: 0.6,
111 });
112 const island = new MeshPhysicalMaterial({
113 color: 0x1a1e22,
114 metalness: 0.55,
115 roughness: 0.3,
116 clearcoat: 1,
117 });
118 const lensMaterial = new MeshPhysicalMaterial({
119 color: 0x061022,
120 metalness: 0.6,
121 roughness: 0.1,
122 clearcoat: 1,
123 });
124 const flashMaterial = new MeshBasicMaterial({ color: 0xf2ead6 });
125 const displayMaterial = new MeshBasicMaterial({ map: texture, toneMapped: false });
126 const hitMaterial = new MeshBasicMaterial({ colorWrite: false, depthWrite: false });
127 const coverMaterial = new MeshBasicMaterial({ map: texture, toneMapped: false });
128 const materials = [
129 frameMetal,
130 polishedMetal,
131 bezel,
132 backGlass,
133 island,
134 lensMaterial,
135 flashMaterial,
136 displayMaterial,
137 hitMaterial,
138 coverMaterial,
139 ];
140
141 // Each half's meshes live in body coordinates; `left` pivots them around the hinge axis.
142 left.position.z = PIVOT_Z;
143 const leftBody = new Group();
144 leftBody.position.z = -PIVOT_Z;
145 left.add(leftBody);
146
147 function half(group: Group, side: "left" | "right", back: MeshPhysicalMaterial) {
148 const body = new Mesh(
149 new ExtrudeGeometry(panelPath(side, BEVEL, 0.1, BEVEL), {
150 depth: DEPTH - BEVEL * 2,
151 bevelEnabled: true,
152 bevelSize: BEVEL,
153 bevelThickness: BEVEL,
154 bevelSegments: 3,
155 curveSegments: 12,
156 }),
157 frameMetal,
158 );
159 body.position.z = -DEPTH / 2 + BEVEL;
160 group.add(body);
161 const frame = new Mesh(new ShapeGeometry(panelPath(side, 0.01, 0.095, 0.002), 12), bezel);
162 frame.position.z = DEPTH / 2 + 0.001;
163 group.add(frame);
164 // A back-facing shape mirrors X, so it is drawn from the opposite side's outline.
165 const rear = new Mesh(
166 new ShapeGeometry(panelPath(side === "left" ? "right" : "left", 0.01, 0.095, 0.002), 12),
167 back,
168 );
169 rear.name = `${side}-back`;
170 rear.rotation.y = Math.PI;
171 rear.position.set(0, 0, -DEPTH / 2 - 0.001);
172 group.add(rear);
173 const geometry = new ShapeGeometry(panelPath(side, INSET, 0.076));
174 geometry.computeBoundingBox();
175 const display = new Mesh(geometry, hitMaterial);
176 display.name = `${side}-inner-screen`;
177 display.position.z = DEPTH / 2 + 0.003;
178 group.add(display);
179 return display;
180 }
181
182 const innerLeft = half(leftBody, "left", bezel);
183 const innerRight = half(right, "right", backGlass);
184 // One indexed surface keeps adjacent pixels joined at the crease. The
185 // physical halves move separately underneath it.
186 const screenWidth = 2 * (HALF_WIDTH - INSET);
187 const screenHeight = HEIGHT - 2 * INSET;
188 const screenGeometry = new PlaneGeometry(screenWidth, screenHeight, 40, 48);
189 const screenPositions = screenGeometry.getAttribute("position");
190 const screenUvs = screenGeometry.getAttribute("uv");
191 const baseX = new Float32Array(screenPositions.count);
192 for (let i = 0; i < screenPositions.count; i++) {
193 const y = screenPositions.getY(i);
194 const outerX = screenWidth / 2;
195 const outerY = screenHeight / 2;
196 const radius = 0.076;
197 const cornerY = Math.max(0, Math.abs(y) - (outerY - radius));
198 const limit = outerX - radius + Math.sqrt(Math.max(0, radius * radius - cornerY * cornerY));
199 const x = Math.max(-limit, Math.min(limit, screenPositions.getX(i)));
200 baseX[i] = x;
201 screenUvs.setXY(i, x / screenWidth + 0.5, y / screenHeight + 0.5);
202 }
203 screenUvs.needsUpdate = true;
204 const innerSurface = new Mesh(screenGeometry, displayMaterial);
205 innerSurface.name = "continuous-inner-screen";
206 orientation.add(innerSurface);
207 // The outer half of the hinge housing. It tucks behind the back glass when
208 // open and becomes the rounded spine when closed.
209 const spineSlack = 0.15;
210 const spine = new Mesh(
211 new CylinderGeometry(
212 SPINE_RADIUS,
213 SPINE_RADIUS,
214 HEIGHT - 0.012,
215 32,
216 1,
217 false,
218 Math.PI / 2 + spineSlack,
219 Math.PI - spineSlack * 2,
220 ),
221 polishedMetal,
222 );
223 spine.name = "hinge-spine";
224 spine.position.z = PIVOT_Z;
225 orientation.add(spine);
226
227 const cover = new Mesh(new ShapeGeometry(coverPath()), coverMaterial);
228 cover.name = "cover-screen";
229 cover.position.set(-HALF_WIDTH / 2, 0, -DEPTH / 2 - 0.003);
230 cover.rotation.y = Math.PI;
231 leftBody.add(cover);
232
233 // Rear components use back-surface coordinates, with outward positive Z.
234 const rearCamera = new Group();
235 rearCamera.name = "rear-camera";
236 const islandWidth = 0.46;
237 const islandHeight = 0.2;
238 rearCamera.position.set(
239 HALF_WIDTH - 0.07 - islandWidth / 2,
240 HEIGHT / 2 - 0.08 - islandHeight / 2,
241 -DEPTH / 2 - 0.002,
242 );
243 rearCamera.rotation.y = Math.PI;
244 right.add(rearCamera);
245 const plateDepth = 0.02;
246 const plate = new Mesh(
247 new ExtrudeGeometry(roundedRectPath(islandWidth, islandHeight, 0.07), {
248 depth: plateDepth,
249 bevelEnabled: true,
250 bevelSize: 0.008,
251 bevelThickness: 0.006,
252 bevelSegments: 3,
253 curveSegments: 12,
254 }),
255 island,
256 );
257 plate.name = "camera-plate";
258 rearCamera.add(plate);
259 const plateFront = plateDepth + 0.006;
260 for (const [x, radius] of [
261 [-0.14, 0.05],
262 [-0.01, 0.05],
263 [0.105, 0.036],
264 ] as const) {
265 const ring = new Mesh(
266 new CylinderGeometry(radius + 0.012, radius + 0.012, 0.012, 32),
267 frameMetal,
268 );
269 ring.rotation.x = Math.PI / 2;
270 ring.position.set(x, 0, plateFront + 0.004);
271 rearCamera.add(ring);
272 const lens = new Mesh(new CircleGeometry(radius, 32), lensMaterial);
273 lens.name = "camera-lens";
274 lens.position.set(x, 0, plateFront + 0.0105);
275 rearCamera.add(lens);
276 }
277 const flash = new Mesh(new CircleGeometry(0.018, 20), flashMaterial);
278 flash.position.set(0.185, 0.045, plateFront + 0.0005);
279 rearCamera.add(flash);
280
281 // Power and volume keys sit on the fixed half's outer edge.
282 for (const [y, length] of [
283 [0.52, 0.16],
284 [0.2, 0.3],
285 ] as const) {
286 const key = new Mesh(new BoxGeometry(0.02, length, DEPTH * 0.45), frameMetal);
287 key.position.set(HALF_WIDTH + 0.008, y, 0);
288 right.add(key);
289 }
290
291 const raycaster = new Raycaster();
292 const pointer = new Vector2();
293 const local = new Vector3();
294 let capturedDisplay: Mesh | null = null;
295 let activeLayout = layout;
296 let angle = initialAngle;
297 const updateVisibleScreen = () => {
298 const innerActive = angle >= 90;
299 innerSurface.visible = innerActive;
300 innerLeft.visible = innerActive;
301 innerRight.visible = innerActive;
302 cover.visible = !innerActive;
303 };
304 const setAngle = (next: number) => {
305 angle = Math.max(0, Math.min(180, next));
306 left.rotation.y = Math.PI * (1 - angle / 180);
307 spine.rotation.y = left.rotation.y / 2;
308 const radians = left.rotation.y;
309 const cosine = Math.cos(radians);
310 const sine = Math.sin(radians);
311 const frontZ = DEPTH / 2 + 0.004 - PIVOT_Z;
312 for (let i = 0; i < screenPositions.count; i++) {
313 const x = baseX[i]!;
314 if (x < 0) {
315 screenPositions.setXYZ(
316 i,
317 x * cosine + frontZ * sine,
318 screenPositions.getY(i),
319 -x * sine + frontZ * cosine + PIVOT_Z,
320 );
321 } else {
322 screenPositions.setXYZ(i, x, screenPositions.getY(i), frontZ + PIVOT_Z);
323 }
324 }
325 screenPositions.needsUpdate = true;
326 screenGeometry.computeBoundingBox();
327 screenGeometry.computeBoundingSphere();
328 updateVisibleScreen();
329 };
330 const setDisplay = (nextTexture: Texture, nextLayout: PhoneDisplayLayout) => {
331 activeLayout = nextLayout;
332 displayMaterial.map = nextTexture;
333 coverMaterial.map = nextTexture;
334 cover.geometry.computeBoundingBox();
335 const bounds = cover.geometry.boundingBox!;
336 const uv = cover.geometry.getAttribute("uv");
337 const position = cover.geometry.getAttribute("position");
338 for (let i = 0; i < uv.count; i++) {
339 const u = (position.getX(i) - bounds.min.x) / (bounds.max.x - bounds.min.x);
340 const v = (position.getY(i) - bounds.min.y) / (bounds.max.y - bounds.min.y);
341 uv.setXY(i, u, v);
342 }
343 uv.needsUpdate = true;
344 updateVisibleScreen();
345 };
346 setAngle(initialAngle);
347 setDisplay(texture, layout);
348
349 return {
350 root,
351 orientation,
352 width: HALF_WIDTH * 2,
353 height: HEIGHT,
354 setAngle,
355 setDisplay,
356 screenPoint(x: number, y: number, camera: Camera, captured = false) {
357 orientation.updateWorldMatrix(true, true);
358 camera.updateMatrixWorld(true);
359 pointer.set(x * 2 - 1, 1 - y * 2);
360 raycaster.setFromCamera(pointer, camera);
361 const screens = cover.visible ? [cover] : [innerLeft, innerRight];
362 const hit = raycaster.intersectObjects(screens, false)[0];
363 if (!hit && !captured) {
364 capturedDisplay = null;
365 return null;
366 }
367 const display =
368 (hit?.object as Mesh | undefined) ??
369 (capturedDisplay?.visible ? capturedDisplay : screens[0]);
370 if (!display) return null;
371 if (hit) capturedDisplay = display;
372 if (hit) local.copy(hit.point);
373 else {
374 const plane = new Vector3(0, 0, 1).transformDirection(display.matrixWorld);
375 const point = new Vector3().setFromMatrixPosition(display.matrixWorld);
376 const distance =
377 plane.dot(point.clone().sub(raycaster.ray.origin)) / plane.dot(raycaster.ray.direction);
378 if (!Number.isFinite(distance)) return null;
379 local.copy(raycaster.ray.direction).multiplyScalar(distance).add(raycaster.ray.origin);
380 }
381 display.worldToLocal(local);
382 const bounds = display.geometry.boundingBox!;
383 const u = Math.max(0, Math.min(1, (local.x - bounds.min.x) / (bounds.max.x - bounds.min.x)));
384 const v = Math.max(0, Math.min(1, (bounds.max.y - local.y) / (bounds.max.y - bounds.min.y)));
385 const across = display === cover ? u : (display === innerLeft ? u : 1 + u) / 2;
386 return activeLayout.rotation === Math.PI ? { x: 1 - across, y: 1 - v } : { x: across, y: v };
387 },
388 dispose() {
389 root.traverse((object) => {
390 if (object instanceof Mesh) object.geometry.dispose();
391 });
392 for (const material of materials) material.dispose();
393 },
394 };
395}
packages/client-runtime/src/device/phoneViewer.test.ts+127-3
...@@ -11,6 +11,7 @@ const gpu = vi.hoisted(() => ({...@@ -11,6 +11,7 @@ const gpu = vi.hoisted(() => ({
11 scene: Scene;11 scene: Scene;
12 phone: Object3D | undefined;12 phone: Object3D | undefined;
13 rotation: Quaternion | undefined;13 rotation: Quaternion | undefined;
14 displayAngle: number | undefined;
14 yaw: number | undefined;15 yaw: number | undefined;
15 cameraZ: number;16 cameraZ: number;
16 }[];17 }[];
...@@ -50,6 +51,7 @@ vi.mock("three", async () => {...@@ -50,6 +51,7 @@ vi.mock("three", async () => {
50 scene,51 scene,
51 phone,52 phone,
52 rotation: phone?.quaternion.clone(),53 rotation: phone?.quaternion.clone(),
54 displayAngle: phone?.children[0]?.rotation.z,
53 yaw: phone?.rotation.y,55 yaw: phone?.rotation.y,
54 cameraZ: camera.position.z,56 cameraZ: camera.position.z,
55 });57 });
...@@ -81,7 +83,12 @@ vi.mock("./modelScene.ts", async () => {...@@ -81,7 +83,12 @@ vi.mock("./modelScene.ts", async () => {
81import { BoxGeometry, Group, Mesh, MeshBasicMaterial, PlaneGeometry, Quaternion } from "three";83import { BoxGeometry, Group, Mesh, MeshBasicMaterial, PlaneGeometry, Quaternion } from "three";
82import { disposeDeviceModel } from "./modelScene.ts";84import { disposeDeviceModel } from "./modelScene.ts";
83import { createPhoneViewer } from "./phoneViewer.ts";85import { createPhoneViewer } from "./phoneViewer.ts";
84import { IOS_TABLET_SHAPE } from "./shapeProfile.ts";86import {
87 ANDROID_PHONE_SHAPE,
88 IOS_TABLET_SHAPE,
89 resolveDeviceShape,
90 type DeviceShapeProfile,
91} from "./shapeProfile.ts";
8592
86afterEach(() => {93afterEach(() => {
87 vi.restoreAllMocks();94 vi.restoreAllMocks();
...@@ -90,7 +97,7 @@ afterEach(() => {...@@ -90,7 +97,7 @@ afterEach(() => {
90 models.pending.length = 0;97 models.pending.length = 0;
91});98});
9299
93function fixture() {100function fixture(profile?: DeviceShapeProfile) {
94 const pending = new Map<number, FrameRequestCallback>();101 const pending = new Map<number, FrameRequestCallback>();
95 let id = 0;102 let id = 0;
96 let now = 0;103 let now = 0;
...@@ -104,7 +111,13 @@ function fixture() {...@@ -104,7 +111,13 @@ function fixture() {
104 const source = { width: 1206, height: 2622 } as HTMLCanvasElement;111 const source = { width: 1206, height: 2622 } as HTMLCanvasElement;
105 const onUnavailable = vi.fn();112 const onUnavailable = vi.fn();
106 const onFramingAspect = vi.fn();113 const onFramingAspect = vi.fn();
107 const viewer = createPhoneViewer({ canvas, source, onUnavailable, onFramingAspect });114 const viewer = createPhoneViewer({
115 canvas,
116 source,
117 onUnavailable,
118 onFramingAspect,
119 ...(profile ? { profile } : {}),
120 });
108 const draw = (time = now) => {121 const draw = (time = now) => {
109 now = time;122 now = time;
110 const callbacks = [...pending.values()];123 const callbacks = [...pending.values()];
...@@ -194,6 +207,117 @@ it("changes device shape without replacing the renderer, decoded source or pose"...@@ -194,6 +207,117 @@ it("changes device shape without replacing the renderer, decoded source or pose"
194 viewer.dispose();207 viewer.dispose();
195});208});
196209
210it("keeps the Android viewer while the resized framebuffer turns between fold postures", () => {
211 const openProfile = resolveDeviceShape({ platform: "android", portraitAspect: 0.96 });
212 const { viewer, draw, source, state } = fixture(openProfile);
213 const scene = state.frames.at(-1)!.scene;
214 source.width = 2076;
215 source.height = 2152;
216 viewer.setScreen({ width: 2076, height: 2152, orientation: "landscape_left" });
217 viewer.frameUpdated();
218 draw(0);
219 expect(state.frames.at(-1)!.displayAngle).toBeCloseTo(0);
220 draw(225);
221 expect(state.frames.at(-1)!.displayAngle).toBeCloseTo(-Math.PI / 4);
222 draw(450);
223 expect(state.frames.at(-1)!.displayAngle).toBeCloseTo(-Math.PI / 2);
224 expect(state.frames.at(-1)!.scene).toBe(scene);
225 expect(gpu.instances).toHaveLength(1);
226
227 source.width = 1080;
228 source.height = 2424;
229 viewer.setScreen({ width: 1080, height: 2424, orientation: "portrait" }, ANDROID_PHONE_SHAPE);
230 viewer.frameUpdated();
231 draw(450);
232 expect(state.frames.at(-1)!.displayAngle).toBeCloseTo(-Math.PI / 2);
233 draw(675);
234 expect(state.frames.at(-1)!.displayAngle).toBeCloseTo(-Math.PI / 4);
235 draw(900);
236 expect(state.frames.at(-1)!.displayAngle).toBeCloseTo(0);
237 expect(state.frames.at(-1)!.scene).toBe(scene);
238 viewer.dispose();
239});
240
241it("animates the Android hinge on the same scene through an encoder resize", () => {
242 const openProfile = resolveDeviceShape({ platform: "android", portraitAspect: 0.96 });
243 const { viewer, draw, source, state } = fixture(openProfile);
244 viewer.setFoldAngle(180);
245 draw(0);
246 const shell = state.frames.at(-1)!.phone!;
247 const moving = shell.children[0]!.children[0]!;
248 viewer.setFoldAngle(0);
249 draw(425);
250 expect(moving.rotation.y).toBeCloseTo(Math.PI / 2);
251 source.width = 1080;
252 source.height = 2424;
253 viewer.setScreen({ width: 1080, height: 2424, orientation: "portrait" }, ANDROID_PHONE_SHAPE);
254 viewer.frameUpdated();
255 draw(850);
256 expect(moving.rotation.y).toBeCloseTo(Math.PI);
257 expect(state.frames.at(-1)!.phone).toBe(shell);
258 expect(gpu.instances).toHaveLength(1);
259 viewer.dispose();
260});
261
262it("retargets an unfinished hinge turn from its visible angle", () => {
263 const { viewer, draw, state } = fixture(ANDROID_PHONE_SHAPE);
264 viewer.setFoldAngle(180);
265 draw(0);
266 const moving = state.frames.at(-1)!.phone!.children[0]!.children[0]!;
267 viewer.setFoldAngle(0);
268 draw(200);
269 const visibleAngle = moving.rotation.y;
270 viewer.setFoldAngle(180);
271 draw(200);
272 expect(moving.rotation.y).toBeCloseTo(visibleAngle);
273 draw(1050);
274 expect(moving.rotation.y).toBeCloseTo(0);
275 viewer.dispose();
276});
277
278it("stops a hinge turn when a loaded model replaces the fold scene", async () => {
279 const { viewer, draw, pending } = fixture(ANDROID_PHONE_SHAPE);
280 viewer.setFoldAngle(180);
281 draw(0);
282 viewer.setFoldAngle(0);
283 viewer.setModel({ id: "iphone-18-pro", url: "/fold.glb" });
284 const asset = new Group();
285 const body = new Mesh(new BoxGeometry(1, 2, 0.1), new MeshBasicMaterial());
286 const display = new Mesh(new PlaneGeometry(0.9, 1.9), new MeshBasicMaterial());
287 display.name = "device-screen";
288 asset.add(body, display);
289 models.pending[0]!.resolve({ asset, dispose: () => disposeDeviceModel(asset) });
290 await Promise.resolve();
291 draw(200);
292 draw(1050);
293 expect(pending.size).toBe(0);
294 viewer.dispose();
295});
296
297it("keeps a loaded model when the fold angle changes and releases it once", async () => {
298 const { viewer, draw, state } = fixture(ANDROID_PHONE_SHAPE);
299 viewer.setModel({ id: "iphone-18-pro", url: "/pro.glb" });
300 const asset = new Group();
301 const body = new Mesh(new BoxGeometry(1.15, 2.3, 0.1), new MeshBasicMaterial());
302 body.position.z = -0.02;
303 const display = new Mesh(new PlaneGeometry(1, 2.2), new MeshBasicMaterial());
304 display.geometry.translate(0, 0, 0.043);
305 display.name = "device-screen";
306 asset.add(body, display);
307 const dispose = vi.fn(() => disposeDeviceModel(asset));
308 models.pending[0]!.resolve({ asset, dispose });
309 await Promise.resolve();
310 draw();
311 const loaded = state.frames.at(-1)!.phone;
312 expect(loaded?.getObjectByName("device-screen")).toBe(display);
313 viewer.setFoldAngle(180);
314 viewer.setFoldAngle(0);
315 draw();
316 expect(state.frames.at(-1)!.phone).toBe(loaded);
317 viewer.dispose();
318 expect(dispose).toHaveBeenCalledOnce();
319});
320
197it("retains the loaded model and pose through rotation and framebuffer resolution changes, then releases it once", async () => {321it("retains the loaded model and pose through rotation and framebuffer resolution changes, then releases it once", async () => {
198 const { viewer, draw, source, state } = fixture();322 const { viewer, draw, source, state } = fixture();
199 viewer.orbit(0.08, 0.04);323 viewer.orbit(0.08, 0.04);
packages/client-runtime/src/device/phoneViewer.ts+101-7
...@@ -19,6 +19,7 @@ import {...@@ -19,6 +19,7 @@ import {
19} from "./model.ts";19} from "./model.ts";
20import { createImportedPhoneScene, loadDeviceModel } from "./modelScene.ts";20import { createImportedPhoneScene, loadDeviceModel } from "./modelScene.ts";
21import { createPhoneScene, phoneDisplayLayout } from "./phoneScene.ts";21import { createPhoneScene, phoneDisplayLayout } from "./phoneScene.ts";
22import { createAndroidFoldScene } from "./androidFoldScene.ts";
22import { createRenderScheduler } from "./renderScheduler.ts";23import { createRenderScheduler } from "./renderScheduler.ts";
23import { createDeviceMotion } from "./deviceMotion.ts";24import { createDeviceMotion } from "./deviceMotion.ts";
24import { createDeviceFraming } from "./deviceFraming.ts";25import { createDeviceFraming } from "./deviceFraming.ts";
...@@ -31,6 +32,7 @@ export interface PhoneViewer {...@@ -31,6 +32,7 @@ export interface PhoneViewer {
31 readonly setAccessory: (source: DeviceAccessorySource | null) => void;32 readonly setAccessory: (source: DeviceAccessorySource | null) => void;
32 readonly frameUpdated: () => void;33 readonly frameUpdated: () => void;
33 readonly setScreen: (screen: DeviceScreenSize | null, profile?: DeviceShapeProfile) => void;34 readonly setScreen: (screen: DeviceScreenSize | null, profile?: DeviceShapeProfile) => void;
35 readonly setFoldAngle: (angle: number | null) => void;
34 readonly resize: (width: number, height: number, pixelRatio: number) => void;36 readonly resize: (width: number, height: number, pixelRatio: number) => void;
35 readonly screenPoint: (37 readonly screenPoint: (
36 x: number,38 x: number,
...@@ -43,6 +45,9 @@ export interface PhoneViewer {...@@ -43,6 +45,9 @@ export interface PhoneViewer {
43 readonly dispose: () => void;45 readonly dispose: () => void;
44}46}
4547
48const ANDROID_ORIENTATION_TURN_MS = 450;
49const ANDROID_FOLD_TURN_MS = 850;
50
46/** Owns only presentation resources. The caller retains the decoded canvas and the stream connection. */51/** Owns only presentation resources. The caller retains the decoded canvas and the stream connection. */
47export function createPhoneViewer(options: {52export function createPhoneViewer(options: {
48 readonly canvas: HTMLCanvasElement;53 readonly canvas: HTMLCanvasElement;
...@@ -53,6 +58,7 @@ export function createPhoneViewer(options: {...@@ -53,6 +58,7 @@ export function createPhoneViewer(options: {
53 readonly profile?: DeviceShapeProfile;58 readonly profile?: DeviceShapeProfile;
54 readonly model?: DeviceModelSource | null;59 readonly model?: DeviceModelSource | null;
55 readonly accessory?: DeviceAccessorySource | null;60 readonly accessory?: DeviceAccessorySource | null;
61 readonly foldAngle?: number | null;
56}): PhoneViewer {62}): PhoneViewer {
57 const renderer = new WebGLRenderer({63 const renderer = new WebGLRenderer({
58 canvas: options.canvas,64 canvas: options.canvas,
...@@ -87,11 +93,19 @@ export function createPhoneViewer(options: {...@@ -87,11 +93,19 @@ export function createPhoneViewer(options: {
87 let screen: DeviceScreenSize | null = null;93 let screen: DeviceScreenSize | null = null;
88 let layout = phoneDisplayLayout(screen, options.source.width, options.source.height);94 let layout = phoneDisplayLayout(screen, options.source.width, options.source.height);
89 let profile = options.profile ?? IOS_PHONE_SHAPE;95 let profile = options.profile ?? IOS_PHONE_SHAPE;
96 let foldAngle = options.foldAngle ?? null;
97 let orientationAngle =
98 foldAngle !== null && profile.id.startsWith("android") ? 0 : layout.rotation;
99 let orientationTurn: { from: number; to: number; startedAt: number } | null = null;
90 let imported: Awaited<ReturnType<typeof loadDeviceModel>> | null = null;100 let imported: Awaited<ReturnType<typeof loadDeviceModel>> | null = null;
91 let modelSource = options.model ?? null;101 let modelSource = options.model ?? null;
92 let accessory: Awaited<ReturnType<typeof loadDeviceModel>> | null = null;102 let accessory: Awaited<ReturnType<typeof loadDeviceModel>> | null = null;
93 let accessoryBounds: Box3 | null = null;103 let accessoryBounds: Box3 | null = null;
94 let phone = createPhoneScene(texture, layout, profile);104 let foldTurn: { from: number; to: number; startedAt: number } | null = null;
105 let phone: ReturnType<typeof createPhoneScene> | ReturnType<typeof createAndroidFoldScene> =
106 foldAngle !== null && profile.id.startsWith("android")
107 ? createAndroidFoldScene(texture, layout, foldAngle)
108 : createPhoneScene(texture, layout, profile);
95 scene.add(phone.root);109 scene.add(phone.root);
96 let disposed = false;110 let disposed = false;
97 const rest = new Quaternion();111 const rest = new Quaternion();
...@@ -116,7 +130,7 @@ export function createPhoneViewer(options: {...@@ -116,7 +130,7 @@ export function createPhoneViewer(options: {
116 new Vector3(phone.width / 2, phone.height / 2, 0),130 new Vector3(phone.width / 2, phone.height / 2, 0),
117 );131 );
118 if (imported && accessoryBounds) bounds.union(accessoryBounds);132 if (imported && accessoryBounds) bounds.union(accessoryBounds);
119 bounds.applyMatrix4(new Matrix4().makeRotationZ(layout.rotation));133 bounds.applyMatrix4(new Matrix4().makeRotationZ(orientationAngle));
120 const size = bounds.getSize(new Vector3());134 const size = bounds.getSize(new Vector3());
121 const aspect = size.x / size.y;135 const aspect = size.x / size.y;
122 if (aspect !== framingAspect) {136 if (aspect !== framingAspect) {
...@@ -140,7 +154,7 @@ export function createPhoneViewer(options: {...@@ -140,7 +154,7 @@ export function createPhoneViewer(options: {
140 };154 };
141 const applyPose = () => {155 const applyPose = () => {
142 phone.root.quaternion.copy(motion.rotation);156 phone.root.quaternion.copy(motion.rotation);
143 phone.orientation.rotation.z = layout.rotation;157 phone.orientation.rotation.z = orientationAngle;
144 };158 };
145 const scheduler = createRenderScheduler(() => {159 const scheduler = createRenderScheduler(() => {
146 if (disposed || !viewport.width || !viewport.height) return;160 if (disposed || !viewport.width || !viewport.height) return;
...@@ -160,10 +174,30 @@ export function createPhoneViewer(options: {...@@ -160,10 +174,30 @@ export function createPhoneViewer(options: {
160 applyPose();174 applyPose();
161 fit(reducedMotion());175 fit(reducedMotion());
162 }176 }
177 if (orientationTurn) {
178 const progress = Math.min(
179 1,
180 (now - orientationTurn.startedAt) / ANDROID_ORIENTATION_TURN_MS,
181 );
182 const eased = progress * progress * (3 - 2 * progress);
183 orientationAngle =
184 orientationTurn.from + (orientationTurn.to - orientationTurn.from) * eased;
185 if (progress === 1) orientationTurn = null;
186 applyPose();
187 fit(reducedMotion());
188 }
189 if (foldTurn && "setAngle" in phone) {
190 const progress = Math.min(1, (now - foldTurn.startedAt) / ANDROID_FOLD_TURN_MS);
191 const eased = progress * progress * (3 - 2 * progress);
192 phone.setAngle(foldTurn.from + (foldTurn.to - foldTurn.from) * eased);
193 if (progress === 1) foldTurn = null;
194 fit(reducedMotion());
195 }
163 framing.advance(now, reducedMotion());196 framing.advance(now, reducedMotion());
164 applyCamera();197 applyCamera();
165 renderer.render(scene, camera);198 renderer.render(scene, camera);
166 if (motion.needsFrame() || framing.needsFrame()) scheduler.invalidate();199 if (motion.needsFrame() || framing.needsFrame() || orientationTurn || foldTurn)
200 scheduler.invalidate();
167 } catch {201 } catch {
168 options.onUnavailable();202 options.onUnavailable();
169 }203 }
...@@ -188,7 +222,11 @@ export function createPhoneViewer(options: {...@@ -188,7 +222,11 @@ export function createPhoneViewer(options: {
188 phone.setDisplay(texture, next);222 phone.setDisplay(texture, next);
189 previous.dispose();223 previous.dispose();
190 }224 }
191 if (!imported && (nextProfile !== profile || next.aspect !== layout.aspect)) {225 if (
226 !imported &&
227 !("setAngle" in phone) &&
228 (nextProfile !== profile || next.aspect !== layout.aspect)
229 ) {
192 scene.remove(phone.root);230 scene.remove(phone.root);
193 phone.dispose();231 phone.dispose();
194 phone = createPhoneScene(texture, next, nextProfile);232 phone = createPhoneScene(texture, next, nextProfile);
...@@ -196,10 +234,26 @@ export function createPhoneViewer(options: {...@@ -196,10 +234,26 @@ export function createPhoneViewer(options: {
196 } else {234 } else {
197 phone.setDisplay(texture, next);235 phone.setDisplay(texture, next);
198 }236 }
237 if (next.rotation !== layout.rotation) {
238 if (nextProfile.id.startsWith("android") && !("setAngle" in phone) && !reducedMotion()) {
239 const difference = Math.atan2(
240 Math.sin(next.rotation - orientationAngle),
241 Math.cos(next.rotation - orientationAngle),
242 );
243 orientationTurn = {
244 from: orientationAngle,
245 to: orientationAngle + difference,
246 startedAt: performance.now(),
247 };
248 } else {
249 orientationTurn = null;
250 orientationAngle = "setAngle" in phone ? 0 : next.rotation;
251 }
252 }
199 layout = next;253 layout = next;
200 profile = nextProfile;254 profile = nextProfile;
201 applyPose();255 applyPose();
202 fit(true);256 fit(!orientationTurn);
203 }257 }
204 applyPose();258 applyPose();
205 };259 };
...@@ -212,7 +266,10 @@ export function createPhoneViewer(options: {...@@ -212,7 +266,10 @@ export function createPhoneViewer(options: {
212 ? null266 ? null
213 : model267 : model
214 ? createImportedPhoneScene(model.asset, texture, layout)268 ? createImportedPhoneScene(model.asset, texture, layout)
215 : createPhoneScene(texture, layout, profile);269 : foldAngle !== null && profile.id.startsWith("android")
270 ? createAndroidFoldScene(texture, layout, foldAngle)
271 : createPhoneScene(texture, layout, profile);
272 foldTurn = null;
216 scene.remove(phone.root);273 scene.remove(phone.root);
217 phone.dispose();274 phone.dispose();
218 imported = model;275 imported = model;
...@@ -267,6 +324,43 @@ export function createPhoneViewer(options: {...@@ -267,6 +324,43 @@ export function createPhoneViewer(options: {
267 modelSlot.set(source);324 modelSlot.set(source);
268 },325 },
269 setAccessory,326 setAccessory,
327 setFoldAngle(next) {
328 if (disposed || next === foldAngle) return;
329 const previous = foldAngle;
330 foldAngle = next;
331 // A loaded model owns the scene; install() reads foldAngle if it is removed.
332 if (imported) return;
333 if (next === null || !("setAngle" in phone)) {
334 scene.remove(phone.root);
335 phone.dispose();
336 phone =
337 next === null
338 ? createPhoneScene(texture, layout, profile)
339 : createAndroidFoldScene(texture, layout, next);
340 scene.add(phone.root);
341 orientationTurn = null;
342 orientationAngle = next === null ? layout.rotation : 0;
343 foldTurn = null;
344 applyPose();
345 fit(true);
346 } else {
347 const progress = foldTurn
348 ? Math.min(1, (performance.now() - foldTurn.startedAt) / ANDROID_FOLD_TURN_MS)
349 : 1;
350 const eased = progress * progress * (3 - 2 * progress);
351 const from = foldTurn
352 ? foldTurn.from + (foldTurn.to - foldTurn.from) * eased
353 : (previous ?? next);
354 if (reducedMotion()) {
355 foldTurn = null;
356 phone.setAngle(next);
357 fit(true);
358 } else {
359 foldTurn = { from, to: next, startedAt: performance.now() };
360 }
361 }
362 scheduler.invalidate();
363 },
270 frameUpdated() {364 frameUpdated() {
271 if (disposed) return;365 if (disposed) return;
272 updateLayout();366 updateLayout();