1import {
2 AmbientLight,
3 Box3,
4 CanvasTexture,
5 DirectionalLight,
6 LinearFilter,
7 Matrix4,
8 PerspectiveCamera,
9 Quaternion,
10 Scene,
11 SRGBColorSpace,
12 Vector3,
13 WebGLRenderer,
14} from "three";
15import {
16 createDeviceModelSlot,
17 type DeviceModelSource,
18 type DeviceAccessorySource,
19} from "./model.ts";
20import { createImportedPhoneScene, loadDeviceModel } from "./modelScene.ts";
21import { createPhoneScene, phoneDisplayLayout } from "./phoneScene.ts";
22import {
23 createAndroidFoldScene,
24 DEFAULT_FOLD_INNER_ASPECT,
25 isFoldInnerAspect,
26} from "./androidFoldScene.ts";
27import { createRenderScheduler } from "./renderScheduler.ts";
28import { createDeviceMotion } from "./deviceMotion.ts";
29import { createDeviceFraming } from "./deviceFraming.ts";
30import { nearestDeviceView } from "./deviceViewSnap.ts";
31import type { DeviceScreenSize } from "./stream.ts";
32import { IOS_PHONE_SHAPE, type DeviceShapeProfile } from "./shapeProfile.ts";
33
34export interface PhoneViewer {
35 readonly setModel: (source: DeviceModelSource | null) => void;
36 readonly setAccessory: (source: DeviceAccessorySource | null) => void;
37 readonly frameUpdated: () => void;
38 readonly setScreen: (screen: DeviceScreenSize | null, profile?: DeviceShapeProfile) => void;
39 readonly setFoldAngle: (angle: number | null) => void;
40 readonly resize: (width: number, height: number, pixelRatio: number) => void;
41 readonly screenPoint: (
42 x: number,
43 y: number,
44 captured?: boolean,
45 ) => { x: number; y: number } | null;
46 readonly orbit: (deltaX: number, deltaY: number) => void;
47 readonly setInteractionActive: (active: boolean, mode: "touch" | "orbit") => void;
48 readonly resetPose: () => void;
49 readonly dispose: () => void;
50}
51
52const ANDROID_ORIENTATION_TURN_MS = 450;
53const ANDROID_FOLD_TURN_MS = 850;
54
55/** Owns only presentation resources. The caller retains the decoded canvas and the stream connection. */
56export function createPhoneViewer(options: {
57 readonly canvas: HTMLCanvasElement;
58 readonly source: HTMLCanvasElement;
59 readonly onUnavailable: () => void;
60 readonly onModelError?: (cause: unknown) => void;
61 readonly onFramingAspect?: (aspect: number) => void;
62 readonly profile?: DeviceShapeProfile;
63 readonly model?: DeviceModelSource | null;
64 readonly accessory?: DeviceAccessorySource | null;
65 readonly foldAngle?: number | null;
66}): PhoneViewer {
67 const renderer = new WebGLRenderer({
68 canvas: options.canvas,
69 alpha: true,
70 antialias: true,
71 powerPreference: "low-power",
72 });
73 renderer.outputColorSpace = SRGBColorSpace;
74 const makeTexture = () => {
75 const next = new CanvasTexture(options.source);
76 next.colorSpace = SRGBColorSpace;
77 next.minFilter = LinearFilter;
78 next.magFilter = LinearFilter;
79 next.generateMipmaps = false;
80 return next;
81 };
82 let texture = makeTexture();
83 let textureWidth = options.source.width;
84 let textureHeight = options.source.height;
85 const scene = new Scene();
86 const camera = new PerspectiveCamera(32, 1, 0.1, 30);
87 camera.position.z = 5.5;
88 const ambient = new AmbientLight(0xffffff, 2.4);
89 const key = new DirectionalLight(0xe4edff, 5);
90 key.position.set(-3, 4, 5);
91 const rim = new DirectionalLight(0xffffff, 4);
92 rim.position.set(3, 1, -3);
93 const fill = new DirectionalLight(0x9facd4, 2);
94 fill.position.set(-2, -2, -4);
95 scene.add(ambient, key, rim, fill);
96
97 let screen: DeviceScreenSize | null = null;
98 let layout = phoneDisplayLayout(screen, options.source.width, options.source.height);
99 let profile = options.profile ?? IOS_PHONE_SHAPE;
100 let foldAngle = options.foldAngle ?? null;
101 let orientationAngle =
102 foldAngle !== null && profile.id.startsWith("android") ? 0 : layout.rotation;
103 let orientationTurn: { from: number; to: number; startedAt: number } | null = null;
104 let imported: Awaited<ReturnType<typeof loadDeviceModel>> | null = null;
105 let modelSource = options.model ?? null;
106 let accessory: Awaited<ReturnType<typeof loadDeviceModel>> | null = null;
107 let accessoryBounds: Box3 | null = null;
108 let foldTurn: { from: number; to: number; startedAt: number } | null = null;
109 // The inner display's raw width over height. Cover frames leave the last unfolded shape.
110 const rawAspect = () => options.source.width / options.source.height;
111 let foldAspect = isFoldInnerAspect(rawAspect()) ? rawAspect() : DEFAULT_FOLD_INNER_ASPECT;
112 const createFoldScene = (angle: number, displayLayout = layout) =>
113 createAndroidFoldScene(texture, displayLayout, angle, foldAspect);
114 /** The hinge angle currently on screen, including an unfinished turn. */
115 const visibleFoldAngle = (fallback: number) => {
116 if (!foldTurn) return fallback;
117 const progress = Math.min(1, (performance.now() - foldTurn.startedAt) / ANDROID_FOLD_TURN_MS);
118 const eased = progress * progress * (3 - 2 * progress);
119 return foldTurn.from + (foldTurn.to - foldTurn.from) * eased;
120 };
121 let phone: ReturnType<typeof createPhoneScene> | ReturnType<typeof createAndroidFoldScene> =
122 foldAngle !== null && profile.id.startsWith("android")
123 ? createFoldScene(foldAngle)
124 : createPhoneScene(texture, layout, profile);
125 scene.add(phone.root);
126 let disposed = false;
127 const rest = new Quaternion();
128 const motion = createDeviceMotion({
129 choose: (rotation) =>
130 nearestDeviceView(rotation, [{ rotation: new Quaternion(), yawLimit: Math.PI / 3 }])!
131 .rotation,
132 });
133 motion.setPose(rest, performance.now(), true);
134 const framing = createDeviceFraming();
135 const reducedMotion = () =>
136 globalThis.matchMedia?.("(prefers-reduced-motion: reduce)").matches ?? false;
137 let viewport = { width: 0, height: 0, pixelRatio: 1 };
138 let drawingBuffer = { width: 0, height: 0, pixelRatio: 0 };
139
140 let framingAspect: number | null = null;
141 const fit = (immediate = false) => {
142 if (!viewport.width || !viewport.height) return;
143 camera.aspect = viewport.width / viewport.height;
144 const bounds = new Box3(
145 new Vector3(-phone.width / 2, -phone.height / 2, 0),
146 new Vector3(phone.width / 2, phone.height / 2, 0),
147 );
148 if (imported && accessoryBounds) bounds.union(accessoryBounds);
149 bounds.applyMatrix4(new Matrix4().makeRotationZ(orientationAngle));
150 const size = bounds.getSize(new Vector3());
151 const aspect = size.x / size.y;
152 if (aspect !== framingAspect) {
153 framingAspect = aspect;
154 options.onFramingAspect?.(aspect);
155 }
156 phone.root.updateMatrixWorld(true);
157 framing.setBounds(
158 new Box3().setFromObject(phone.root),
159 (camera.fov * Math.PI) / 360,
160 camera.aspect,
161 performance.now(),
162 immediate,
163 );
164 applyCamera();
165 };
166 const applyCamera = () => {
167 camera.position.set(framing.center.x, framing.center.y, framing.distance());
168 camera.lookAt(framing.center.x, framing.center.y, 0);
169 camera.updateProjectionMatrix();
170 };
171 const applyPose = () => {
172 phone.root.quaternion.copy(motion.rotation);
173 phone.orientation.rotation.z = orientationAngle;
174 };
175 const scheduler = createRenderScheduler(() => {
176 if (disposed || !viewport.width || !viewport.height) return;
177 try {
178 if (
179 drawingBuffer.width !== viewport.width ||
180 drawingBuffer.height !== viewport.height ||
181 drawingBuffer.pixelRatio !== viewport.pixelRatio
182 ) {
183 // Canvas allocation clears the previous image. Commit it with the redraw,
184 // rather than exposing an empty buffer between ResizeObserver and the next frame.
185 renderer.setDrawingBufferSize(viewport.width, viewport.height, viewport.pixelRatio);
186 drawingBuffer = viewport;
187 }
188 const now = performance.now();
189 if (motion.advance(now, reducedMotion())) {
190 applyPose();
191 fit(reducedMotion());
192 }
193 if (orientationTurn) {
194 const progress = Math.min(
195 1,
196 (now - orientationTurn.startedAt) / ANDROID_ORIENTATION_TURN_MS,
197 );
198 const eased = progress * progress * (3 - 2 * progress);
199 orientationAngle =
200 orientationTurn.from + (orientationTurn.to - orientationTurn.from) * eased;
201 if (progress === 1) orientationTurn = null;
202 applyPose();
203 fit(reducedMotion());
204 }
205 if (foldTurn && "setAngle" in phone) {
206 const progress = Math.min(1, (now - foldTurn.startedAt) / ANDROID_FOLD_TURN_MS);
207 const eased = progress * progress * (3 - 2 * progress);
208 phone.setAngle(foldTurn.from + (foldTurn.to - foldTurn.from) * eased);
209 if (progress === 1) foldTurn = null;
210 fit(reducedMotion());
211 }
212 framing.advance(now, reducedMotion());
213 applyCamera();
214 renderer.render(scene, camera);
215 if (motion.needsFrame() || framing.needsFrame() || orientationTurn || foldTurn)
216 scheduler.invalidate();
217 } catch {
218 options.onUnavailable();
219 }
220 });
221 const updateLayout = (nextProfile = profile) => {
222 const next = phoneDisplayLayout(screen, options.source.width, options.source.height);
223 const resized =
224 textureWidth !== options.source.width || textureHeight !== options.source.height;
225 if (
226 resized ||
227 nextProfile !== profile ||
228 next.aspect !== layout.aspect ||
229 next.rawLandscape !== layout.rawLandscape ||
230 next.rotation !== layout.rotation
231 ) {
232 // The model and renderer survive framebuffer rotation and native resolution changes.
233 if (resized) {
234 const previous = texture;
235 texture = makeTexture();
236 textureWidth = options.source.width;
237 textureHeight = options.source.height;
238 phone.setDisplay(texture, next);
239 previous.dispose();
240 }
241 // Learn the inner display shape from any unfolded frame, including before fold mode.
242 const frameAspect = rawAspect();
243 const innerChanged = isFoldInnerAspect(frameAspect) && frameAspect !== foldAspect;
244 if (innerChanged) foldAspect = frameAspect;
245 if (!imported && "setAngle" in phone && innerChanged) {
246 // A new inner display shape resizes the body; the hinge keeps its visible angle.
247 const angle = visibleFoldAngle(foldAngle ?? 180);
248 scene.remove(phone.root);
249 phone.dispose();
250 phone = createFoldScene(angle, next);
251 scene.add(phone.root);
252 } else if (
253 !imported &&
254 !("setAngle" in phone) &&
255 (nextProfile !== profile || next.aspect !== layout.aspect)
256 ) {
257 scene.remove(phone.root);
258 phone.dispose();
259 phone = createPhoneScene(texture, next, nextProfile);
260 scene.add(phone.root);
261 } else {
262 phone.setDisplay(texture, next);
263 }
264 if (next.rotation !== layout.rotation) {
265 if (nextProfile.id.startsWith("android") && !("setAngle" in phone) && !reducedMotion()) {
266 const difference = Math.atan2(
267 Math.sin(next.rotation - orientationAngle),
268 Math.cos(next.rotation - orientationAngle),
269 );
270 orientationTurn = {
271 from: orientationAngle,
272 to: orientationAngle + difference,
273 startedAt: performance.now(),
274 };
275 } else {
276 orientationTurn = null;
277 orientationAngle = "setAngle" in phone ? 0 : next.rotation;
278 }
279 }
280 layout = next;
281 profile = nextProfile;
282 applyPose();
283 fit(!orientationTurn);
284 }
285 applyPose();
286 };
287 const modelSlot = createDeviceModelSlot({
288 load: loadDeviceModel,
289 onError: options.onModelError,
290 install(model) {
291 // Validate and prepare the next scene before releasing the visible one.
292 const next = disposed
293 ? null
294 : model
295 ? createImportedPhoneScene(model.asset, texture, layout)
296 : foldAngle !== null && profile.id.startsWith("android")
297 ? createFoldScene(foldAngle)
298 : createPhoneScene(texture, layout, profile);
299 foldTurn = null;
300 scene.remove(phone.root);
301 phone.dispose();
302 imported = model;
303 if (!next) return;
304 phone = next;
305 scene.add(phone.root);
306 if (accessory) {
307 if (imported) phone.orientation.add(accessory.asset);
308 else accessory.asset.removeFromParent();
309 }
310 applyPose();
311 fit(true);
312 scheduler.invalidate();
313 },
314 });
315 const accessorySlot = createDeviceModelSlot({
316 load: loadDeviceModel,
317 onError: options.onModelError,
318 install(model) {
319 const bounds = model ? new Box3().setFromObject(model.asset) : null;
320 if (
321 bounds &&
322 (bounds.isEmpty() ||
323 ![...bounds.min.toArray(), ...bounds.max.toArray()].every(Number.isFinite))
324 )
325 throw new Error("Device accessory has invalid bounds");
326 accessory?.asset.removeFromParent();
327 accessory = model;
328 accessoryBounds = bounds;
329 if (disposed) return;
330 if (imported && accessory) phone.orientation.add(accessory.asset);
331 fit(true);
332 scheduler.invalidate();
333 },
334 });
335 const setAccessory = (source: DeviceAccessorySource | null) => {
336 accessorySlot.set(source?.modelId === modelSource?.id ? source : null);
337 };
338 const contextLost = (event: Event) => {
339 event.preventDefault();
340 options.onUnavailable();
341 };
342 options.canvas.addEventListener("webglcontextlost", contextLost);
343 applyPose();
344 modelSlot.set(options.model ?? null);
345 setAccessory(options.accessory ?? null);
346 return {
347 setModel(source) {
348 if (source?.id !== modelSource?.id || source?.url !== modelSource?.url)
349 accessorySlot.set(null);
350 modelSource = source;
351 modelSlot.set(source);
352 },
353 setAccessory,
354 setFoldAngle(next) {
355 if (disposed || next === foldAngle) return;
356 const previous = foldAngle;
357 foldAngle = next;
358 // A loaded model owns the scene; install() reads foldAngle if it is removed.
359 if (imported) return;
360 if (next === null || !("setAngle" in phone)) {
361 scene.remove(phone.root);
362 phone.dispose();
363 phone = next === null ? createPhoneScene(texture, layout, profile) : createFoldScene(next);
364 scene.add(phone.root);
365 orientationTurn = null;
366 orientationAngle = next === null ? layout.rotation : 0;
367 foldTurn = null;
368 applyPose();
369 fit(true);
370 } else {
371 const from = visibleFoldAngle(previous ?? next);
372 if (reducedMotion()) {
373 foldTurn = null;
374 phone.setAngle(next);
375 fit(true);
376 } else {
377 foldTurn = { from, to: next, startedAt: performance.now() };
378 }
379 }
380 scheduler.invalidate();
381 },
382 frameUpdated() {
383 if (disposed) return;
384 updateLayout();
385 texture.needsUpdate = true;
386 scheduler.invalidate();
387 },
388 setScreen(next, nextProfile = profile) {
389 if (disposed) return;
390 screen = next;
391 updateLayout(nextProfile);
392 scheduler.invalidate();
393 },
394 resize(width, height, pixelRatio) {
395 if (disposed) return;
396 if (![width, height, pixelRatio].every(Number.isFinite) || width <= 0 || height <= 0) return;
397 const ratio = Math.min(2, Math.max(1, pixelRatio));
398 if (viewport.width === width && viewport.height === height && viewport.pixelRatio === ratio)
399 return;
400 viewport = { width, height, pixelRatio: ratio };
401 fit(true);
402 scheduler.invalidate();
403 },
404 screenPoint(x, y, captured = false) {
405 if (disposed) return null;
406 applyPose();
407 return phone.screenPoint(x, y, camera, captured);
408 },
409 orbit(deltaX, deltaY) {
410 if (disposed) return;
411 motion.orbit(deltaX * viewport.width, deltaY * viewport.height, performance.now());
412 scheduler.invalidate();
413 },
414 setInteractionActive(active, mode) {
415 if (disposed) return;
416 const now = performance.now();
417 if (mode === "orbit") motion.dragActive(active, now);
418 else {
419 motion.hold(active, now);
420 framing.hold(active, now);
421 }
422 scheduler.invalidate();
423 },
424 resetPose() {
425 if (disposed) return;
426 motion.reset(rest, performance.now());
427 scheduler.invalidate();
428 },
429 dispose() {
430 if (disposed) return;
431 disposed = true;
432 scheduler.dispose();
433 const hadImported = imported !== null;
434 accessorySlot.dispose();
435 modelSlot.dispose();
436 options.canvas.removeEventListener("webglcontextlost", contextLost);
437 scene.remove(phone.root);
438 if (!hadImported) phone.dispose();
439 texture.dispose();
440 renderer.dispose();
441 renderer.forceContextLoss();
442 },
443 };
444}