diff --git a/apps/web/src/components/device/DeviceDuoControls.tsx b/apps/web/src/components/device/DeviceDuoControls.tsx index 57922a0243d35b8065a190a772ceac1d15707515..824af5a644fcc56ff13bcedd05898e66f7b442be 100644 --- a/apps/web/src/components/device/DeviceDuoControls.tsx +++ b/apps/web/src/components/device/DeviceDuoControls.tsx @@ -1,5 +1,7 @@ +import { useState } from "react"; import { duoFoldState, + duoHoldOrientation, type DuoCommand, type DuoControlState, } from "@t3tools/client-runtime/device/duo-control"; @@ -31,7 +33,18 @@ export function DeviceDuoControls(props: { onCommand: (command: DuoCommand) => void; }) { const { screen } = props; - const { fold, stand, phoneVertical } = duoFoldState(screen); + const { fold, stand, phoneVertical: reportedVertical, settled } = duoFoldState(screen); + // A fold never changes how the phone is held. Keep the last settled reading + // through display handoffs, whose interim orientation belongs to the other display. + const [settledVertical, setSettledVertical] = useState(reportedVertical); + if (settled && settledVertical !== reportedVertical) setSettledVertical(reportedVertical); + // Stands rotate the device. Folding out of one returns it to how it was held before. + const [standVertical, setStandVertical] = useState(settledVertical); + const phoneVertical = stand ? standVertical : settledVertical; + // Folding out of a stand sends the rotation back, then the fold, and only one command can + // wait. Hold the fold buttons until a stand, or anything queued on one, has landed; the + // rotation must also be read in the frame of the display the stand settles on. + const foldWaits = props.state.pending && (stand || props.state.requested?.control === "pose"); const foldLabels = { closed: "Closed", half: phoneVertical ? "Book" : "Laptop", @@ -43,6 +56,7 @@ export function DeviceDuoControls(props: { pressed: boolean, onClick: () => void, glyph: React.ReactNode, + waits = false, ) => ( props.onCommand({ control: "angle", value }), + () => { + if (stand) + props.onCommand({ + control: "orientation", + value: duoHoldOrientation(standVertical, screen.screenId), + }); + props.onCommand({ control: "angle", value }); + }, , + foldWaits, ), )} @@ -84,7 +106,10 @@ export function DeviceDuoControls(props: { id, label, screen.hingePose === id, - () => props.onCommand({ control: "pose", value: id }), + () => { + if (!stand) setStandVertical(settledVertical); + props.onCommand({ control: "pose", value: id }); + }, , ), )} diff --git a/apps/web/src/components/device/DeviceDuoViewport.tsx b/apps/web/src/components/device/DeviceDuoViewport.tsx index f168ceccc30d7187d8f5442b5acc6c6fa9bd9725..125ff9a03cfcd83529107466d221cd82b135495a 100644 --- a/apps/web/src/components/device/DeviceDuoViewport.tsx +++ b/apps/web/src/components/device/DeviceDuoViewport.tsx @@ -127,7 +127,7 @@ export function DeviceDuoViewport(props: { (screenRef.current?.screenId === 1 ? 0 : 180), contains: (x, y) => !!screenRef.current && viewer.beginHinge(x, y), change: (angle) => { - viewer.setHingePreview(angle ?? previewRef.current); + viewer.setHingePreview(angle ?? previewRef.current, true); if (angle !== null) client.current?.controlDuo({ control: "angle", value: angle }); }, }); diff --git a/packages/client-runtime/src/device/duoControl.test.ts b/packages/client-runtime/src/device/duoControl.test.ts index f325d3a25de74c2a094306dde1776d2f4597be45..2755e19759394be2a3c7023bc9802af5217eb416 100644 --- a/packages/client-runtime/src/device/duoControl.test.ts +++ b/packages/client-runtime/src/device/duoControl.test.ts @@ -1,5 +1,11 @@ import { afterEach, expect, it, vi } from "vite-plus/test"; -import { createDuoControl, createDuoPinch, duoFoldState, type DuoCommand } from "./duoControl.ts"; +import { + createDuoControl, + createDuoPinch, + duoFoldState, + duoHoldOrientation, + type DuoCommand, +} from "./duoControl.ts"; afterEach(() => vi.useRealTimers()); it("keeps a failed send visible, including a disconnect while draining queued motion", () => { @@ -136,13 +142,27 @@ it.each([ ], ["half-open book", { screenId: 3, orientation: "landscape_left", hingeAngle: 90 }, "half", true], ] as const)("reads a %s", (_name, screen, fold, phoneVertical) => { - expect(duoFoldState(screen)).toEqual({ fold, stand: false, phoneVertical }); + expect(duoFoldState(screen)).toEqual({ fold, stand: false, phoneVertical, settled: true }); +}); + +it("marks a display handoff unsettled while the cover reports the inner display's orientation", () => { + // Recorded from the iPhone Duo simulator opening a closed vertical phone to a book. + const handoff = [ + { screenId: 1, orientation: "portrait", hingeAngle: 90 }, + { screenId: 1, orientation: "landscape_left", hingeAngle: 90 }, + { screenId: 3, orientation: "landscape_left", hingeAngle: 90 }, + ] as const; + expect(handoff.map((screen) => duoFoldState(screen).settled)).toEqual([false, false, true]); + // Closing hands back to the cover the same way. + expect(duoFoldState({ screenId: 3, orientation: "landscape_left", hingeAngle: 0 }).settled).toBe( + false, + ); }); it("marks native stands so the fold group does not also claim them", () => { expect( duoFoldState({ screenId: 3, orientation: "portrait", hingeAngle: 90, hingePose: "laptop" }), - ).toEqual({ fold: "half", stand: true, phoneVertical: false }); + ).toEqual({ fold: "half", stand: true, phoneVertical: false, settled: true }); }); it("falls back like the 3D view when hinge fields are missing", () => { @@ -156,3 +176,20 @@ it("falls back like the 3D view when hinge fields are missing", () => { phoneVertical: true, }); }); + +it("rotates a stand back to how the phone was held, in the frame of the display receiving it", () => { + // Recorded from the iPhone Duo simulator: Tent can rest on either display. + // Rotating the inner display to landscape_left, or the cover to portrait, + // leaves a phone that opens and closes vertical. + expect(duoHoldOrientation(true, 3)).toBe("landscape_left"); + expect(duoHoldOrientation(true, 1)).toBe("portrait"); + // A quarter turn apart, these describe the same horizontal phone. + expect(duoHoldOrientation(false, 1)).toBe("landscape_left"); + expect(duoHoldOrientation(false, 3)).toBe("portrait_upside_down"); + for (const screenId of [1, 3]) + for (const vertical of [true, false]) + expect( + duoFoldState({ screenId, orientation: duoHoldOrientation(vertical, screenId) }) + .phoneVertical, + ).toBe(vertical); +}); diff --git a/packages/client-runtime/src/device/duoControl.ts b/packages/client-runtime/src/device/duoControl.ts index e5be353ca625b2452566753a3f7020b450ba8fbf..3a4026b976505c4256ec0b5114686e2956ddd3b1 100644 --- a/packages/client-runtime/src/device/duoControl.ts +++ b/packages/client-runtime/src/device/duoControl.ts @@ -20,6 +20,17 @@ export type DuoControlState = { error: string | null; }; +/** + * Whether the reporting display is the one the fold shows: the cover when closed, the inner + * display when open. A fold across the closed position hands off in steps, and the outgoing + * display first reports the incoming one's orientation, so an unsettled screen's orientation + * does not say how the phone is held. Stands such as Tent can also rest unsettled on the cover. + */ +export function duoScreenSettled(screen: Pick) { + const angle = screen.hingeAngle ?? (screen.screenId === 1 ? 0 : 180); + return (screen.screenId === 1) === (angle === 0); +} + /** * The fold the device is in and the way it is held. Missing hinge fields fall back the same way * the 3D view does, so controls never disagree with what is drawn. The inner panel is mounted a @@ -34,9 +45,23 @@ export function duoFoldState( fold: angle === 0 ? "closed" : angle === 180 ? "open" : "half", stand: screen.hingePose === "laptop" || screen.hingePose === "tent", phoneVertical: screen.screenId === 1 ? !landscape : landscape, + settled: duoScreenSettled(screen), } as const; } +/** + * The rotation that holds the phone vertical or horizontal. Native rotation is + * read in the frame of the display active when it arrives, and the inner panel + * is mounted a quarter turn from the cover. + */ +export function duoHoldOrientation( + vertical: boolean, + screenId: number | undefined, +): DuoOrientation { + if (screenId === 1) return vertical ? "portrait" : "landscape_left"; + return vertical ? "landscape_left" : "portrait_upside_down"; +} + /** One in-flight native transaction. Hinge motion coalesces; presets replace queued motion. Nothing replays after reconnect. */ export function createDuoControl(options: { send: (request: { requestId: number; command: DuoCommand }) => boolean; diff --git a/packages/client-runtime/src/device/duoViewer.test.ts b/packages/client-runtime/src/device/duoViewer.test.ts index b85d306bcfd6db401c8a3c49fc9a599d8bb69839..abe3da6506503f558b2bcbbe32821a94f1c9b237 100644 --- a/packages/client-runtime/src/device/duoViewer.test.ts +++ b/packages/client-runtime/src/device/duoViewer.test.ts @@ -331,6 +331,108 @@ it("faces a display handed off by native rotation without requesting another sen viewer.dispose(); }); +// Recorded from the iPhone Duo simulator: a fold button across the closed +// position lets the cover report the inner display's orientation before handing +// off. A button's pending angle reaches the viewer as a hinge preview first. +it("presents fold buttons centered on the hinge: book, tablet, and an upright closed phone", async () => { + // Configurations arrive while the view is still easing; time only advances to settle. + let now = 0; + vi.stubGlobal("performance", { now: () => now }); + const { viewer, draw, state, onOrientationRequested } = fixture(false); + const loaded = asset(); + models.resolve({ asset: loaded, dispose: vi.fn() }); + await Promise.resolve(); + viewer.resize(500, 700, 2); + const cover = { width: 1398, height: 2034, screenId: 1, hingePose: null } as const; + const inner = { width: 2007, height: 2853, screenId: 3, hingePose: null } as const; + const press = ( + angle: number, + configs: ReadonlyArray<{ orientation: "portrait" | "landscape_left"; screenId: 1 | 3 }>, + ) => { + viewer.setHingePreview(angle); + configs.forEach((config, index) => { + viewer.setScreen({ + ...(config.screenId === 1 ? cover : inner), + ...config, + hingeAngle: angle, + }); + // The command reply follows the first configuration. + if (index === 0) viewer.setHingePreview(null); + draw(); + }); + now += 5_000; + draw(); + draw(); + }; + const expectInnerFacing = (minimum: number) => { + for (const name of ["inner-display-left", "inner-display-right"]) { + const mesh = loaded.getObjectByName(name) as Mesh; + const normal = new Vector3() + .fromBufferAttribute(mesh.geometry.getAttribute("normal"), 0) + .transformDirection(mesh.matrixWorld); + expect(normal.z).toBeGreaterThan(minimum); + } + }; + const upright = new Rotation().setFromEuler(new Euler(0, Math.PI / 2, 0, "YXZ")); + const closeAndExpectUpright = () => { + press(0, [ + { screenId: 3, orientation: "landscape_left" }, + { screenId: 1, orientation: "portrait" }, + ]); + expect(state.views.at(-1)!.quaternion.angleTo(upright)).toBeLessThan(0.00001); + }; + viewer.setScreen({ ...inner, orientation: "landscape_left", hingeAngle: 180 }); + draw(); + closeAndExpectUpright(); + press(90, [ + { screenId: 1, orientation: "portrait" }, + { screenId: 1, orientation: "landscape_left" }, + { screenId: 3, orientation: "landscape_left" }, + // Election can flap back to the cover once before settling. + { screenId: 1, orientation: "landscape_left" }, + { screenId: 3, orientation: "landscape_left" }, + ]); + expectInnerFacing(0.45); + // Opening the book flat faces the whole tablet; folding back centers the book again. + press(180, [{ screenId: 3, orientation: "landscape_left" }]); + expectInnerFacing(0.99); + press(90, [{ screenId: 3, orientation: "landscape_left" }]); + expectInnerFacing(0.45); + closeAndExpectUpright(); + expect(onOrientationRequested).not.toHaveBeenCalled(); + viewer.dispose(); +}); + +it("eases a fold button's hinge while a pinch follows the fingers", async () => { + let now = 0; + vi.stubGlobal("performance", { now: () => now }); + const { viewer, draw } = fixture(false); + const loaded = asset(); + models.resolve({ asset: loaded, dispose: vi.fn() }); + await Promise.resolve(); + viewer.resize(500, 700, 2); + viewer.setScreen({ + width: 2007, + height: 2853, + orientation: "landscape_left", + screenId: 3, + hingeAngle: 180, + }); + draw(); + const leaf = loaded.getObjectByName("left-half")!; + const open = ((180 - 90) * Math.PI) / 360; + viewer.setHingePreview(90); + draw(); + expect(leaf.rotation.y).toBeGreaterThan(0); + expect(leaf.rotation.y).toBeLessThan(open * 0.9); + now += 1_000; + draw(); + expect(leaf.rotation.y).toBeCloseTo(open); + viewer.setHingePreview(30, true); + expect(leaf.rotation.y).toBeCloseTo(((180 - 30) * Math.PI) / 360); + viewer.dispose(); +}); + it.each(["book", "laptop", "open"] as const)( "faces native cover readback for %s even when configuration arrives before the model", async (hingePose) => { @@ -402,7 +504,7 @@ it("keeps default-zoom orbits framed across folds and fits the current assembly" viewer.dispose(); }); -it("previews hinge articulation while preserving the laptop lid orientation and blocking unconfirmed input", async () => { +it("pinches the hinge while preserving the laptop lid orientation and blocking unconfirmed input", async () => { const { viewer, draw, pending } = fixture(); const loaded = asset(); models.resolve({ asset: loaded, dispose: vi.fn() }); @@ -420,7 +522,7 @@ it("previews hinge articulation while preserving the laptop lid orientation and const lid = loaded.getObjectByName("right-half")!; const presentation = lid.getWorldQuaternion(new Rotation()); for (const value of [127, 50, 170]) { - viewer.setHingePreview(value); + viewer.setHingePreview(value, true); draw(); expect(loaded.getObjectByName("left-half")!.rotation.y).toBeCloseTo( ((180 - value) * Math.PI) / 360, @@ -438,7 +540,7 @@ it("previews hinge articulation while preserving the laptop lid orientation and draw(); expect(lid.getWorldQuaternion(new Rotation()).angleTo(presentation)).toBeLessThan(0.00001); } - viewer.setHingePreview(null); + viewer.setHingePreview(null, true); draw(); expect(pending.size).toBe(0); expect(lid.getWorldQuaternion(new Rotation()).angleTo(presentation)).toBeLessThan(0.00001); @@ -480,6 +582,42 @@ it("does not restart an animated preset on duplicate native configurations", asy viewer.dispose(); }); +it.each([true, false])( + "stands Tent on both edges when the cover reads it back (configuration before model: %s)", + async (configurationFirst) => { + // Recorded from the iPhone Duo simulator: Tent can rest with the cover active. + const { viewer, draw } = fixture(); + viewer.resize(500, 700, 2); + const tent = { + width: 1398, + height: 2034, + orientation: "portrait", + screenId: 1, + hingeAngle: 80, + hingePose: "tent", + } as const; + if (configurationFirst) viewer.setScreen(tent); + const loaded = asset(); + models.resolve({ asset: loaded, dispose: vi.fn() }); + await Promise.resolve(); + if (!configurationFirst) viewer.setScreen(tent); + draw(); + draw(); + // Each leaf runs from the hinge down to the surface it rests on. + for (const [name, outward] of [ + ["left-half", -1], + ["right-half", 1], + ] as const) { + const leaf = loaded.getObjectByName(name)!; + const down = new Vector3(outward, 0, 0).applyQuaternion( + leaf.getWorldQuaternion(new Rotation()), + ); + expect(down.y).toBeLessThan(-0.4); + } + viewer.dispose(); + }, +); + it("lets standalone rotation leave Laptop and Tent and stand a closed device upright after hinge edits", async () => { const { viewer, draw, state } = fixture(); models.resolve({ asset: asset(), dispose: vi.fn() }); @@ -672,7 +810,7 @@ it.each(["left", "right"] as const)( const partner = loaded.getObjectByName("left-half")!; const partnerStart = partner.getWorldQuaternion(new Rotation()); for (const value of [150, 90, 30, 0, 30, 90, 180]) { - viewer.setHingePreview(value); + viewer.setHingePreview(value, true); draw(); expect(leaf.getWorldQuaternion(new Rotation()).angleTo(orientation)).toBeLessThan(1e-6); if (value === 90) @@ -682,7 +820,7 @@ it.each(["left", "right"] as const)( ? { ...inner, width: 1398, height: 2034, screenId: 1, hingeAngle: value, hingePose: null } : { ...inner, hingeAngle: value, hingePose: null }; viewer.setScreen(next); - viewer.setHingePreview(null); + viewer.setHingePreview(null, true); draw(); viewer.setScreen({ ...next }); // Late native readback cannot change the view. draw(); diff --git a/packages/client-runtime/src/device/duoViewer.ts b/packages/client-runtime/src/device/duoViewer.ts index 7feca7452e0d40b14160fb00d434f747a2f72a3e..6e4b443da3175f77f7850ff72ffe73129404974e 100644 --- a/packages/client-runtime/src/device/duoViewer.ts +++ b/packages/client-runtime/src/device/duoViewer.ts @@ -29,11 +29,12 @@ import { createDeviceMotion } from "./deviceMotion.ts"; import { duoViewSnaps, nearestDuoView, type DuoRestFace } from "./duoSnap.ts"; import { createRenderScheduler } from "./renderScheduler.ts"; import type { DeviceScreenSize } from "./stream.ts"; -import type { DuoPose } from "./duoControl.ts"; +import { duoScreenSettled, type DuoPose } from "./duoControl.ts"; export interface DuoViewer { readonly setScreen: (screen: DeviceScreenSize | null) => void; - readonly setHingePreview: (angle: number | null) => void; + /** A pinch holds the leaf under it and follows the fingers; a fold button eases the hinge and re-presents the device. */ + readonly setHingePreview: (angle: number | null, pinch?: boolean) => void; readonly setInteractionActive: (active: boolean, mode?: "touch" | "orbit") => void; readonly rejectOrientation: () => void; readonly frameUpdated: (panel: DuoPanelId, primary?: HTMLCanvasElement) => void; @@ -186,6 +187,7 @@ export function createDuoViewer(options: { const framing = createDeviceFraming(); const framingBounds = new Box3(); let firstPose = true; + let foldPending = false; let physicalPose: DuoPose = "open"; let presentationAngle = 180; let targetPresentation = new Quaternion(); @@ -291,7 +293,7 @@ export function createDuoViewer(options: { if (!model || disposed) return; scene.add(model.root); applyPose(); - if (screen?.screenId === 1) { + if (screen?.screenId === 1 && physicalPose !== "tent") { const snap = nearestDuoView(orbit.rotation, activeSnaps()); if (snap) { restFace = snap.face; @@ -323,13 +325,25 @@ export function createDuoViewer(options: { const previous = screen; const ownedHandoff = requestedPanel !== null; if (!next || next.screenId === requestedPanel) clearHandoff(); - const ownedRotation = requestedOrientation !== null && next?.screenId === previous?.screenId; + const requestedRotation = requestedOrientation !== null; + const ownedRotation = requestedRotation && next?.screenId === previous?.screenId; const changedDisplay = next?.screenId !== previous?.screenId; requestedOrientation = null; screen = next; const changedPose = next?.hingePose && next.hingePose !== previous?.hingePose; const folding = hingeLeaf !== null && !changedPose; + // A hinge change no held leaf owns re-presents the device, centered on the + // hinge, once the display that shows the new fold reports. Across the closed + // position that takes steps: the outgoing display first reports the incoming + // one's orientation. + const angleBefore = previous?.hingeAngle ?? (previous?.screenId === 1 ? 0 : 180); + const angleAfter = next?.hingeAngle ?? (next?.screenId === 1 ? 0 : 180); + if (!next || folding || changedPose) foldPending = false; + else if (previous && angleBefore !== angleAfter) foldPending = true; + const landedFold = foldPending && !!next && duoScreenSettled(next) && !ownedHandoff; + if (next && duoScreenSettled(next)) foldPending = false; const rotated = + !foldPending && next?.screenId === previous?.screenId && next?.hingeAngle === previous?.hingeAngle && next?.orientation !== previous?.orientation; @@ -340,16 +354,21 @@ export function createDuoViewer(options: { !folding && !next?.hingePose && previewAngle === null; - if (firstPose || changedPose || leftPhysicalPose) { + if (firstPose || changedPose || leftPhysicalPose || landedFold) { physicalPose = next?.hingePose ?? (next?.screenId === 1 ? "closed" : "open"); presentationAngle = next?.hingeAngle ?? (next?.screenId === 1 ? 0 : 180); } // A command reply/config confirms hinge state. Display identity, never an angle heuristic, owns input. targetAngle = previewAngle ?? next?.hingeAngle ?? (next?.screenId === 1 ? 0 : 180); - // Native angle commands clear hingePose. They change articulation only; - // preserve the viewing pose, including laptop/tent and user orbit. A - // separate rotation clears the physical preset and follows panel orientation. - if (firstPose || changedPose || (rotated && !ownedRotation && !ownedHandoff && !folding)) { + // Native angle commands clear hingePose. While a leaf is held (a pinch, or a + // fold on a stand) they change articulation only and preserve the viewing + // pose. A separate rotation clears the physical preset and follows panel orientation. + if ( + firstPose || + changedPose || + landedFold || + (rotated && !ownedRotation && !ownedHandoff && !folding) + ) { hingeLeaf = null; viewOrientation = next?.orientation ?? null; const poseAngle = presentationAngle; @@ -371,9 +390,12 @@ export function createDuoViewer(options: { targetPresentation.premultiply( new Quaternion().setFromAxisAngle(new Vector3(0, 0, 1), roll), ); - if (next?.screenId === 1) { + if (next?.screenId === 1 && !landedFold && physicalPose !== "tent") { // Native readback can retain the cover even for an open physical preset. // A screen-facing rest must use that display's actual hinged normal. + // A landed fold is shut, so its pose already faces the cover; the + // hinge is still moving and its current normal would be stale. + // Tent stands on both edges with the cover outward; facing it flat would tip it over. const snap = nearestDuoView(targetPresentation, activeSnaps()); if (snap) targetPresentation.copy(snap.rotation); } @@ -381,9 +403,18 @@ export function createDuoViewer(options: { restFace = next?.screenId === 1 ? "cover" : physicalPose === "laptop" ? "right" : "inside"; orbit.setPose(targetPresentation, performance.now(), firstPose); firstPose = false; - } else if (next && changedDisplay && !ownedHandoff && !folding) { + } else if ( + next && + changedDisplay && + !ownedHandoff && + !folding && + (restFace === "cover") !== (next.screenId === 1) && + (requestedRotation || duoScreenSettled(next)) + ) { // A sensor rotation can hand ownership to the other native display. // Face that display without sending another rotation and creating a feedback loop. + // Native election can briefly flap to the display a fold does not show and + // back; neither is a handoff, and the view may still be easing into place. const snap = nearestDuoView(orbit.rotation, activeSnaps()); if (snap) { restFace = snap.face; @@ -395,19 +426,24 @@ export function createDuoViewer(options: { fit(); scheduler.invalidate(); }, - setHingePreview(next) { + setHingePreview(next, pinch = false) { if (disposed || (next !== null && (!Number.isFinite(next) || next < 0 || next > 180))) return; if (!moving()) lastTime = performance.now(); previewAngle = next; - if (next !== null && !hingeLeaf && model) { + if (next !== null && !pinch) { + // A fold button holds no leaf, so the device is presented, centered on + // the hinge, once the fold lands. + hingeLeaf = null; + } else if (next !== null && !hingeLeaf && model) { clearHandoff(); requestedOrientation = null; hingeLeaf = screen?.screenId === 1 && angle > 20 ? "left" : "right"; orbit.setPose(orbit.rotation.clone(), performance.now(), true); } - orbit.hold(interactionActive || next !== null, performance.now()); + orbit.hold(interactionActive || (pinch && next !== null), performance.now()); targetAngle = next ?? screen?.hingeAngle ?? (screen?.screenId === 1 ? 0 : 180); - if (next !== null) { + // A pinch follows the fingers; anything else eases toward the target. + if (next !== null && pinch) { angle = next; applyPose(); fit();