From d60a71ef6e371d8b345ecb1f7575117d675187c7 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sun, 4 Oct 2026 00:48:28 -0700 Subject: [PATCH] feat(mobile): full-screen simulator viewer with on-demand controls (#15551) Co-authored-by: Claude Opus 5.5 (1M context) --- apps/mobile/app.config.ts | 5 + apps/mobile/package.json | 1 + apps/mobile/src/Stack.tsx | 2 + .../devices/DevicePreviewRouteScreen.tsx | 227 +++++++++++------- .../features/devices/shakeDetector.test.ts | 26 ++ .../src/features/devices/shakeDetector.ts | 30 +++ pnpm-lock.yaml | 15 ++ 7 files changed, 225 insertions(+), 81 deletions(-) create mode 100644 apps/mobile/src/features/devices/shakeDetector.test.ts create mode 100644 apps/mobile/src/features/devices/shakeDetector.ts diff --git a/apps/mobile/app.config.ts b/apps/mobile/app.config.ts index cf009b3bc37a7f78052302b220070fc428a4c4c2..4eb77db7b651849650682f7a3d61dd6359248056 100644 --- a/apps/mobile/app.config.ts +++ b/apps/mobile/app.config.ts @@ -306,6 +306,8 @@ const config: ExpoConfig = { // JS back handling survives it via react-native's Android 16 shim plus // withAndroidPredictiveBackCompat on Android 13-15. predictiveBackGestureEnabled: true, + // expo-sensors declares this for its pedometer, which the app does not use. + blockedPermissions: ["android.permission.ACTIVITY_RECOGNITION"], }, web: { favicon: variant.assets.appIcon, @@ -440,6 +442,9 @@ const config: ExpoConfig = { }, ], "./plugins/withIosCocoaPodsUuidCache.cjs", + // Only the accelerometer is used (device viewer shake). Compile out the + // pedometer so iOS needs no motion purpose string. + ["expo-sensors", { motionPermission: false }], // Must be listed BEFORE expo-widgets: same-type mods run last-registered- // first, so registering earlier makes this plugin's mods run AFTER // expo-widgets' — its dangerous mod wipes ios/ExpoWidgetsTarget/ (which diff --git a/apps/mobile/package.json b/apps/mobile/package.json index b5f0d49d2bc1b0d766c75071ed1963bb6e074a55..df0d8aea4c52d2d5378f642d36c71ddc6b19ae52 100644 --- a/apps/mobile/package.json +++ b/apps/mobile/package.json @@ -101,6 +101,7 @@ "expo-paste-input": "^0.1.15", "expo-quick-actions": "^6.0.2", "expo-secure-store": "~58.0.2", + "expo-sensors": "~58.0.1", "expo-sharing": "58.0.13", "expo-splash-screen": "~58.0.4", "expo-sqlite": "~58.0.8", diff --git a/apps/mobile/src/Stack.tsx b/apps/mobile/src/Stack.tsx index 3fd6844ccc60ca10a6151fff7032b36a5a151a2f..452d1a4dbed536149adc2647d0d6b624db4d2829 100644 --- a/apps/mobile/src/Stack.tsx +++ b/apps/mobile/src/Stack.tsx @@ -701,6 +701,8 @@ const RootStackConfig = createNativeStackNavigator({ presentation: "fullScreenModal", headerShown: false, gestureEnabled: false, + autoHideHomeIndicator: true, + navigationBarHidden: true, }, }), ThreadReview: createNativeStackScreen({ diff --git a/apps/mobile/src/features/devices/DevicePreviewRouteScreen.tsx b/apps/mobile/src/features/devices/DevicePreviewRouteScreen.tsx index 8d2a6700005077097d2abe5ef267ba8e8ecbe84d..d3b960844efb85b013f3aa392e9c9c73e7b57b86 100644 --- a/apps/mobile/src/features/devices/DevicePreviewRouteScreen.tsx +++ b/apps/mobile/src/features/devices/DevicePreviewRouteScreen.tsx @@ -4,16 +4,32 @@ import { deviceToolUpdatePolicy, } from "@t3tools/client-runtime/state/device"; import { useIsFocused, useNavigation, type StaticScreenProps } from "@react-navigation/native"; -import { createNativeStackNavigator } from "@react-navigation/native-stack"; import { EnvironmentId, ThreadId } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; import { useCallback, useEffect, useMemo, useRef, useState, type RefObject } from "react"; -import { ActivityIndicator, Alert, AppState, Platform, Pressable, View } from "react-native"; +import * as Haptics from "expo-haptics"; +import { Accelerometer } from "expo-sensors"; +import { + ActivityIndicator, + Alert, + AppState, + BackHandler, + Pressable, + StatusBar, + View, +} from "react-native"; +import Animated, { FadeIn, FadeOut, ReduceMotion } from "react-native-reanimated"; import { useSafeAreaInsets } from "react-native-safe-area-context"; +import { SymbolView } from "../../components/AppSymbol"; import { AppText } from "../../components/AppText"; -import { ScreenHeader, type ScreenHeaderMenuItem } from "../../components/ScreenHeader"; -import { NativeHeaderToolbar } from "../../native/StackHeader"; +import { ControlPill, ControlPillMenu } from "../../components/ControlPill"; +import { GlassSurface } from "../../components/GlassSurface"; +import { + androidHeaderMenuActions, + findHeaderMenuAction, +} from "../../components/headerMenu.android"; +import type { ScreenHeaderMenuItem } from "../../components/ScreenHeader.types"; import { deviceEnvironment, refreshDeviceHubAccess, useDeviceHubAccess } from "../../state/device"; import { useEnvironmentQuery } from "../../state/query"; import { useAtomCommand } from "../../state/use-atom-command"; @@ -24,40 +40,26 @@ import { threadDevicePreviews, type ThreadDevicePreview, } from "./threadDevicePreviews"; +import { createShakeDetector } from "./shakeDetector"; -const DevicePreviewStack = createNativeStackNavigator<{ DevicePreview: undefined }>(); +const OVERLAY_ENTERING = FadeIn.duration(160).reduceMotion(ReduceMotion.System); +const OVERLAY_EXITING = FadeOut.duration(120).reduceMotion(ReduceMotion.System); type DevicePreviewRouteScreenProps = StaticScreenProps<{ readonly environmentId: string; readonly threadId: string; }>; -/** The nested native stack supplies the navigation bar inside the modal. */ +/** Full-screen viewer; controls live in an overlay toggled by the handle or a shake. */ export function DevicePreviewRouteScreen({ route }: DevicePreviewRouteScreenProps) { const navigation = useNavigation(); const onClose = useCallback(() => navigation.goBack(), [navigation]); return ( - - - - {() => ( - - )} - - - + ); } @@ -78,6 +80,7 @@ function DevicePreviewScreen({ const [inputConnected, setInputConnected] = useState(false); const [streamAttempt, setStreamAttempt] = useState(0); const [shuttingDown, setShuttingDown] = useState(false); + const [controlsVisible, setControlsVisible] = useState(true); const retryHost = useAtomCommand(deviceEnvironment.list); const shutdown = useAtomCommand(deviceEnvironment.shutdown, { reportFailure: false }); const streamRef = useRef(null); @@ -87,10 +90,11 @@ function DevicePreviewScreen({ [state.data, threadId], ); const preview = selectedThreadDevicePreview(previews, selectedKey); - const onInputConnected = useCallback( - async (connected: boolean) => setInputConnected(connected), - [], - ); + const onInputConnected = useCallback(async (connected: boolean) => { + setInputConnected(connected); + // Controls greet the user while connecting, then get out of the stream's way. + if (connected) setControlsVisible(false); + }, []); useEffect(() => { const subscription = AppState.addEventListener("change", (state) => setForeground(state !== "background"), @@ -100,6 +104,26 @@ function DevicePreviewScreen({ useEffect(() => { if (focused && state.data !== null && previews.length === 0) onClose(); }, [focused, state.data, previews.length, onClose]); + useEffect(() => { + if (!focused || !foreground) return; + const shaken = createShakeDetector(); + Accelerometer.setUpdateInterval(50); + const subscription = Accelerometer.addListener((sample) => { + if (!shaken({ ...sample, timestamp: sample.timestamp * 1000 })) return; + void Haptics.selectionAsync(); + setControlsVisible((visible) => !visible); + }); + return () => subscription.remove(); + }, [focused, foreground]); + // Android back reveals the controls first, so leaving takes a deliberate second press. + useEffect(() => { + if (controlsVisible) return; + const subscription = BackHandler.addEventListener("hardwareBackPress", () => { + setControlsVisible(true); + return true; + }); + return () => subscription.remove(); + }, [controlsVisible]); const shutDownDevice = async () => { if (!preview || shuttingDown) return; @@ -217,57 +241,29 @@ function DevicePreviewScreen({ onPress: () => void shutDownDevice(), }, ]; + const pressHome = () => streamRef.current?.home(); + const menuItems: ScreenHeaderMenuItem[] = [ + ...(previews.length > 1 + ? [ + { + id: "devices", + title: "Devices", + inline: true, + items: previews.map((device) => ({ + id: device.key, + title: device.name, + subtitle: device.description, + selected: device.key === preview?.key, + onPress: () => setSelectedKey(device.key), + })), + }, + ] + : []), + ...controls, + ]; return ( - - streamRef.current?.home(), - }, - ]} - menus={[ - { - title: "Device options", - icon: "ellipsis", - items: [ - ...(previews.length > 1 - ? [ - { - id: "devices", - title: "Devices", - inline: true, - items: previews.map((device) => ({ - id: device.key, - title: device.name, - subtitle: device.description, - selected: device.key === preview?.key, - onPress: () => setSelectedKey(device.key), - })), - }, - ] - : []), - ...controls, - ], - }, - ]} - /> - {Platform.OS === "ios" ? ( - - - - ) : null} + + ); } +function DeviceOptionsMenu({ items }: { readonly items: ReadonlyArray }) { + return ( + { + const action = findHeaderMenuAction(items, nativeEvent.event); + if (action && !action.disabled) action.onPress(); + }} + > + + + + + ); +} + function OpenDevicePreview({ environmentId, preview, diff --git a/apps/mobile/src/features/devices/shakeDetector.test.ts b/apps/mobile/src/features/devices/shakeDetector.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..362d0b639516914e90158ba62262bf4bd57b3bae --- /dev/null +++ b/apps/mobile/src/features/devices/shakeDetector.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { createShakeDetector } from "./shakeDetector"; + +const resting = (timestamp: number) => ({ x: 0, y: 0, z: -1, timestamp }); +const jolt = (timestamp: number) => ({ x: 2.2, y: 0.4, z: -1, timestamp }); + +describe("shake detector", () => { + it("ignores gravity and a single bump", () => { + const detect = createShakeDetector(); + expect(detect(resting(0))).toBe(false); + expect(detect(jolt(100))).toBe(false); + expect(detect(resting(200))).toBe(false); + expect(detect(jolt(900))).toBe(false); + }); + + it("reports two jolts within the window once, then cools down", () => { + const detect = createShakeDetector(); + expect(detect(jolt(0))).toBe(false); + expect(detect(jolt(300))).toBe(true); + expect(detect(jolt(400))).toBe(false); + expect(detect(jolt(600))).toBe(false); + expect(detect(jolt(1_400))).toBe(false); + expect(detect(jolt(1_500))).toBe(true); + }); +}); diff --git a/apps/mobile/src/features/devices/shakeDetector.ts b/apps/mobile/src/features/devices/shakeDetector.ts new file mode 100644 index 0000000000000000000000000000000000000000..8c47eb95fe12bb573adf5f9bcd3e17390921b9a1 --- /dev/null +++ b/apps/mobile/src/features/devices/shakeDetector.ts @@ -0,0 +1,30 @@ +export interface AccelerationSample { + /** Acceleration per axis in g, gravity included. */ + readonly x: number; + readonly y: number; + readonly z: number; + /** Milliseconds. */ + readonly timestamp: number; +} + +/** + * Reports a shake after two strong jolts within a short window. A single bump, + * setting the phone down, or walking stays below the threshold or the count. + */ +export function createShakeDetector({ + threshold = 1.8, + window = 600, + cooldown = 1_000, +}: { threshold?: number; window?: number; cooldown?: number } = {}) { + let jolts: number[] = []; + let lastShake = Number.NEGATIVE_INFINITY; + return (sample: AccelerationSample) => { + if (sample.timestamp - lastShake < cooldown) return false; + if (Math.hypot(sample.x, sample.y, sample.z) < threshold) return false; + jolts = [...jolts.filter((time) => sample.timestamp - time <= window), sample.timestamp]; + if (jolts.length < 2) return false; + jolts = []; + lastShake = sample.timestamp; + return true; + }; +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 632485b2ee228a5d7316dee6ffe2acf1410a86cd..c45ca5713bd361a4664192bcafac6a24a15f1b32 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -408,6 +408,9 @@ importers: expo-secure-store: specifier: ~58.0.2 version: 58.0.2(expo@58.0.2) + expo-sensors: + specifier: ~58.0.1 + version: 58.0.1(expo@58.0.2)(react-native@0.88.0-rc.3(@babel/core@7.29.7)(@react-native/metro-config@0.88.0-rc.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(utf-8-validate@6.0.6)) expo-sharing: specifier: 58.0.13 version: 58.0.13(patch_hash=fe40d857765bcd6a77aac1fed275e5b8f9415c29b972decdce05fb96de5665e6)(expo@58.0.2)(react-native@0.88.0-rc.3(@babel/core@7.29.7)(@react-native/metro-config@0.88.0-rc.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(utf-8-validate@6.0.6))(react@19.3.0)(typescript@7.0.2) @@ -7842,6 +7845,12 @@ packages: peerDependencies: expo: '*' + expo-sensors@58.0.1: + resolution: {integrity: sha512-hXS10z/wu55sYv++yPstmSjaA7pTGHV1Rc3kAhThf/hZbJZXtbIaFMGsgujIH6RFqi5pcUho0aMWHjvdCewH8g==} + peerDependencies: + expo: '*' + react-native: '*' + expo-server@58.0.2: resolution: {integrity: sha512-QOdIBRZ7hEhlKzDud99/4YPggg+rASCD901PuuDy/oHzSP+eQKs2B4PDwzwfYvtelDA45nOwJsyCKnBvwpuJZA==} engines: {node: ^22.13.0 || ^24.3.0 || ^26.0.0 || >=27.0.0} @@ -18221,6 +18230,12 @@ snapshots: dependencies: expo: 58.0.2(56fc8ddb8129740412739f0f0a6ee025) + expo-sensors@58.0.1(expo@58.0.2)(react-native@0.88.0-rc.3(@babel/core@7.29.7)(@react-native/metro-config@0.88.0-rc.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(utf-8-validate@6.0.6)): + dependencies: + expo: 58.0.2(56fc8ddb8129740412739f0f0a6ee025) + invariant: 2.2.4 + react-native: 0.88.0-rc.3(@babel/core@7.29.7)(@react-native/metro-config@0.88.0-rc.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(utf-8-validate@6.0.6) + expo-server@58.0.2: {} expo-sharing@58.0.13(patch_hash=fe40d857765bcd6a77aac1fed275e5b8f9415c29b972decdce05fb96de5665e6)(expo@58.0.2)(react-native@0.88.0-rc.3(@babel/core@7.29.7)(@react-native/metro-config@0.88.0-rc.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(utf-8-validate@6.0.6))(react@19.3.0)(typescript@7.0.2): -- 2.54.0