authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-09-23 15:01:31-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-09-23 15:01:31-07:00
log11e91f126405883c0194da5b1e3ad39d81c48966
treeb47da3da9012340dd5d99dbd0c38706a82faaef7
parentf5ef0ddb90a8c36584e181b1913e7b8a5df30ffc
signature Signed by PGP key B5690EEEBB952194

feat(mobile): capture Live Activities and agent notifications in showcase screenshots (#13316)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

13 files changed, 609 insertions(+), 38 deletions(-)

.github/workflows/mobile-showcase-screenshots.yml+11
......@@ -76,6 +76,17 @@ jobs:
7676 echo "$vp_pnpm_bin" >> "$GITHUB_PATH"
7777 "$vp_pnpm_bin/pnpm" --version
7878
79 - name: Install AXe
80 # Locks the simulator and answers the notification prompt for the
81 # agent-activity scene.
82 # Homebrew refuses third-party tap formulae until they are trusted;
83 # older releases have no trust command and install them as is.
84 run: |
85 brew tap cameroncooke/axe
86 if brew commands | grep -qx trust; then brew trust --formula cameroncooke/axe/axe; fi
87 brew install cameroncooke/axe/axe
88 axe --version
89
7990 - name: Capture iOS showcase
8091 run: pnpm screenshots:mobile --platform ios --appearance "${{ inputs.appearance }}" --theme "${{ inputs.theme }}"
8192
apps/mobile/modules/t3-agent-notifications/android/src/main/java/expo/modules/t3agentnotifications/AgentNotifications.kt+31-11
......@@ -138,17 +138,7 @@ object AgentNotifications {
138138 // Match iOS foreground presentation. Consume suppressed alerts as well,
139139 // so a delivery retry cannot surface them after the app backgrounds.
140140 if (!ProcessLifecycleOwner.get().lifecycle.currentState.isAtLeast(Lifecycle.State.RESUMED)) {
141 val title = data["alert_title"].orEmpty().take(120)
142 // Grouped alerts list up to five 120-character thread titles.
143 val body = data["alert_body"].orEmpty().take(608)
144 val id = alertId.hashCode()
145 val notification = base(context, ALERT_CHANNEL)
146 .setContentTitle(title).setContentText(body)
147 .setStyle(NotificationCompat.BigTextStyle().bigText(body))
148 .setAutoCancel(true)
149 .setContentIntent(contentIntent(context, scheme, data["alert_path"], id))
150 .build()
151 manager(context).notify(ALERT_TAG, id, notification)
141 postAlert(context, scheme, data, alertId)
152142 }
153143 prefs.edit().remove("seenAlerts").putString(
154144 "seenAlertsOrdered",
......@@ -157,6 +147,36 @@ object AgentNotifications {
157147 }
158148 }
159149
150 private fun postAlert(
151 context: Context,
152 scheme: String,
153 data: Map<String, String>,
154 alertId: String
155 ) {
156 val title = data["alert_title"].orEmpty().take(120)
157 // Grouped alerts list up to five 120-character thread titles.
158 val body = data["alert_body"].orEmpty().take(608)
159 val id = alertId.hashCode()
160 val notification = base(context, ALERT_CHANNEL)
161 .setContentTitle(title).setContentText(body)
162 .setStyle(NotificationCompat.BigTextStyle().bigText(body))
163 .setAutoCancel(true)
164 .setContentIntent(contentIntent(context, scheme, data["alert_path"], id))
165 .build()
166 manager(context).notify(ALERT_TAG, id, notification)
167 }
168
169 /**
170 * Renders a relay-shaped payload without the registration, freshness and
171 * foreground checks, for the showcase capture's staged notifications.
172 */
173 @Synchronized
174 fun showcase(context: Context, scheme: String, data: Map<String, String>) {
175 channels(context)
176 data["alert_id"]?.let { postAlert(context, scheme, data, it) }
177 showActivity(context, scheme, data, data["active"] == "true", RUNNING_LIFETIME_MS)
178 }
179
160180 private fun updateActivity(
161181 context: Context,
162182 prefs: SharedPreferences,
apps/mobile/modules/t3-agent-notifications/android/src/main/java/expo/modules/t3agentnotifications/T3AgentNotificationsModule.kt+4
......@@ -26,6 +26,10 @@ class T3AgentNotificationsModule : Module() {
2626 appContext.reactContext?.let { AgentNotifications.clear(it) }
2727 }
2828
29 Function("showShowcaseActivity") { scheme: String, data: Map<String, String> ->
30 appContext.reactContext?.let { AgentNotifications.showcase(it, scheme, data) }
31 }
32
2933 Function("openLiveUpdateSettings") {
3034 val context = appContext.reactContext
3135 if (context == null || Build.VERSION.SDK_INT < 36) {
apps/mobile/src/features/agent-awareness/androidNotifications.ts+14-7
......@@ -6,6 +6,7 @@ interface AndroidAgentNotifications {
66 configure(deviceId: string, userId: string, scheme: string, ongoingEnabled: boolean): void;
77 clear(): void;
88 openLiveUpdateSettings?(): boolean;
9 showShowcaseActivity?(scheme: string, data: Record<string, string>): void;
910}
1011
1112const native =
......@@ -17,18 +18,24 @@ export function supportsAndroidAgentNotifications(): boolean {
1718 return typeof native?.configure === "function" && typeof native?.clear === "function";
1819}
1920
21function appScheme(): string {
22 const scheme = Constants.expoConfig?.scheme;
23 return (Array.isArray(scheme) ? scheme[0] : scheme) ?? "t3code";
24}
25
2026export function configureAndroidAgentNotifications(
2127 deviceId: string,
2228 userId: string,
2329 ongoingEnabled: boolean,
2430): void {
25 const scheme = Constants.expoConfig?.scheme;
26 native?.configure?.(
27 deviceId,
28 userId,
29 (Array.isArray(scheme) ? scheme[0] : scheme) ?? "t3code",
30 ongoingEnabled,
31 );
31 native?.configure?.(deviceId, userId, appScheme(), ongoingEnabled);
32}
33
34/** Posts a staged relay payload for the showcase capture; false when unsupported. */
35export function showAndroidShowcaseAgentActivity(data: Record<string, string>): boolean {
36 if (!native?.showShowcaseActivity) return false;
37 native.showShowcaseActivity(appScheme(), data);
38 return true;
3239}
3340
3441export function clearAndroidAgentNotifications(): void {
apps/mobile/src/features/showcase/ShowcaseCaptureCoordinator.tsx+65-2
......@@ -29,6 +29,7 @@ import {
2929 buildShowcasePendingTasks,
3030 SHOWCASE_PENDING_TASK_DEFINITIONS,
3131} from "./showcasePendingTasks";
32import { buildShowcaseAgentActivity } from "./showcaseAgentActivity";
3233import { retryShowcaseOperation } from "./showcaseRetry";
3334import {
3435 clearShowcaseRenderSignal,
......@@ -36,6 +37,7 @@ import {
3637 isShowcaseNativeContentReady,
3738 subscribeToShowcaseRenderSignal,
3839} from "./showcaseRenderSignal";
40import { stageShowcaseAgentActivity } from "./stageShowcaseAgentActivity";
3941
4042const SHOWCASE_ENABLED = process.env.EXPO_PUBLIC_SHOWCASE === "1";
4143const SHOWCASE_THREAD_ID = "remote-command-center";
......@@ -75,7 +77,14 @@ export function ShowcaseCaptureCoordinator(props: { readonly pathname: string })
7577 const [themeRequestSettled, setThemeRequestSettled] = useState(false);
7678 const [readyScene, setReadyScene] = useState<ShowcaseScene | null>(null);
7779 const [orientationSettled, setOrientationSettled] = useState(false);
80 const [agentActivityStaged, setAgentActivityStaged] = useState(false);
7881 const requestedSceneRef = useRef<ShowcaseScene | null>(null);
82 // Staging reads the latest entities without restarting on every shell
83 // update, which would re-enter a permission prompt that is still open.
84 const entitiesRef = useRef({ threads, projects });
85 useEffect(() => {
86 entitiesRef.current = { threads, projects };
87 }, [projects, threads]);
7988 const renderSignal = useSyncExternalStore(
8089 subscribeToShowcaseRenderSignal,
8190 getShowcaseRenderSignal,
......@@ -109,6 +118,7 @@ export function ShowcaseCaptureCoordinator(props: { readonly pathname: string })
109118 }
110119
111120 let cancelled = false;
121 let lastOutcome: string | null = null;
112122 void retryShowcaseOperation(async () => applyNativeShowcaseOrientation(orientation), {
113123 isCancelled: () => cancelled,
114124 }).then((applied) => {
......@@ -129,6 +139,7 @@ export function ShowcaseCaptureCoordinator(props: { readonly pathname: string })
129139 // A native draw belongs only to the scene request that produced it. In
130140 // particular, revisiting review must wait for its newly mounted surface.
131141 clearShowcaseRenderSignal();
142 setAgentActivityStaged(false);
132143 setRequestedScene(value);
133144 };
134145 readRequestedScene();
......@@ -176,7 +187,11 @@ export function ShowcaseCaptureCoordinator(props: { readonly pathname: string })
176187 };
177188 }, [connectPairingUrl, pairingUrls]);
178189
179 const scene = sceneFromPathname(props.pathname);
190 const routeScene = sceneFromPathname(props.pathname);
191 // Agent activity is captured over the thread list: the runner locks the
192 // simulator or opens the notification shade on top of it.
193 const scene =
194 requestedScene === "agent-activity" && routeScene === "threads" ? "agent-activity" : routeScene;
180195 const hasServerFixture =
181196 workspace.state.hasReadyEnvironment &&
182197 workspace.environments.length >= 3 &&
......@@ -228,6 +243,11 @@ export function ShowcaseCaptureCoordinator(props: { readonly pathname: string })
228243 navigation.dispatch(StackActions.popToTop());
229244 return;
230245 }
246 // Follows the environments scene, whose settings sheet popToTop leaves open.
247 if (requestedScene === "agent-activity") {
248 navigation.dispatch(CommonActions.reset({ index: 0, routes: [{ name: "Home" }] }));
249 return;
250 }
231251 const routes: ShowcaseResetRoute[] = [{ name: "Home" }];
232252 if (requestedScene === "environments") {
233253 routes.push({
......@@ -264,6 +284,39 @@ export function ShowcaseCaptureCoordinator(props: { readonly pathname: string })
264284 );
265285 }, [hasFixture, navigation, requestedScene, scene, showcaseThread]);
266286
287 useEffect(() => {
288 if (!SHOWCASE_ENABLED || scene !== "agent-activity" || !hasFixture || agentActivityStaged) {
289 return;
290 }
291 let cancelled = false;
292 let lastOutcome: string | null = null;
293 void retryShowcaseOperation(
294 async () => {
295 const now = Date.now();
296 const { threads: latestThreads, projects: latestProjects } = entitiesRef.current;
297 const activity = buildShowcaseAgentActivity(latestThreads, latestProjects, now);
298 const outcome =
299 activity === null
300 ? "fixture threads not loaded"
301 : await stageShowcaseAgentActivity(activity, now);
302 if (outcome === true) return true;
303 // Surfaces in the runner's Metro output when the scene never turns ready.
304 if (outcome !== lastOutcome)
305 console.warn(`[showcase] agent activity not staged: ${outcome}`);
306 lastOutcome = outcome;
307 return false;
308 },
309 // The first attempt waits on the notification permission prompt until
310 // the runner answers it.
311 { isCancelled: () => cancelled, attemptTimeoutMs: 60_000 },
312 ).then((staged) => {
313 if (!cancelled && staged) setAgentActivityStaged(true);
314 });
315 return () => {
316 cancelled = true;
317 };
318 }, [agentActivityStaged, hasFixture, scene]);
319
267320 useEffect(() => {
268321 if (
269322 !SHOWCASE_ENABLED ||
......@@ -276,6 +329,7 @@ export function ShowcaseCaptureCoordinator(props: { readonly pathname: string })
276329 !orientationSettled ||
277330 // Likewise for the palette: an early screenshot shows the default theme.
278331 !themeApplied ||
332 (scene === "agent-activity" && !agentActivityStaged) ||
279333 !isShowcaseNativeContentReady({ scene, themeId, renderSignal })
280334 ) {
281335 setReadyScene(null);
......@@ -298,7 +352,16 @@ export function ShowcaseCaptureCoordinator(props: { readonly pathname: string })
298352 if (renderFrame !== null) cancelAnimationFrame(renderFrame);
299353 if (readyFrame !== null) cancelAnimationFrame(readyFrame);
300354 };
301 }, [hasFixture, orientationSettled, renderSignal, requestedScene, scene, themeApplied, themeId]);
355 }, [
356 agentActivityStaged,
357 hasFixture,
358 orientationSettled,
359 renderSignal,
360 requestedScene,
361 scene,
362 themeApplied,
363 themeId,
364 ]);
302365
303366 if (!SHOWCASE_ENABLED || readyScene === null) return null;
304367
apps/mobile/src/features/showcase/nativeShowcaseScene.ts+8-1
......@@ -2,7 +2,14 @@ import { requireOptionalNativeModule } from "expo";
22
33import { MOBILE_THEME_IDS, type MobileThemeId } from "../../lib/mobileTheme";
44
5export const SHOWCASE_SCENES = ["threads", "thread", "terminal", "review", "environments"] as const;
5export const SHOWCASE_SCENES = [
6 "threads",
7 "thread",
8 "terminal",
9 "review",
10 "environments",
11 "agent-activity",
12] as const;
613export type ShowcaseScene = (typeof SHOWCASE_SCENES)[number];
714
815export type ShowcaseOrientation = "portrait" | "landscape";
apps/mobile/src/features/showcase/showcaseAgentActivity.test.ts created+93
......@@ -0,0 +1,93 @@
1import type {
2 EnvironmentProject,
3 EnvironmentThreadShell,
4} from "@t3tools/client-runtime/state/shell";
5import { EnvironmentId, ProjectId, ThreadId } from "@t3tools/contracts";
6import { assert, it } from "@effect/vitest";
7
8import {
9 buildShowcaseAgentActivity,
10 SHOWCASE_AGENT_ACTIVITY_ROWS,
11 showcaseAndroidActivityData,
12} from "./showcaseAgentActivity";
13
14const NOW = Date.parse("2026-07-16T09:00:00.000Z");
15
16const project = (environmentId: string, id: string, title: string) =>
17 ({
18 environmentId: EnvironmentId.make(environmentId),
19 id: ProjectId.make(id),
20 title,
21 }) as EnvironmentProject;
22
23const thread = (environmentId: string, id: string, projectId: string, title: string) =>
24 ({
25 environmentId: EnvironmentId.make(environmentId),
26 id: ThreadId.make(id),
27 projectId: ProjectId.make(projectId),
28 title,
29 }) as EnvironmentThreadShell;
30
31const projects = [
32 project("moonbase-terminal", "t3code", "T3 Code"),
33 project("suspense-station", "react", "React"),
34 project("kernel-cabin", "linux", "Linux"),
35];
36
37const threads = [
38 thread("moonbase-terminal", "remote-command-center", "t3code", "Make remote coding feel local"),
39 thread(
40 "moonbase-terminal",
41 "pocket-command-center",
42 "t3code",
43 "Put the command center in your pocket",
44 ),
45 thread("suspense-station", "buttery-suspense", "react", "Make Suspense transitions buttery"),
46 thread("kernel-cabin", "beautiful-boot", "linux", "Make boot logs oddly beautiful"),
47];
48
49it("waits until every staged thread and its project have loaded", () => {
50 assert.isNull(buildShowcaseAgentActivity(threads.slice(1), projects, NOW));
51 assert.isNull(buildShowcaseAgentActivity(threads, projects.slice(0, 2), NOW));
52});
53
54it("stages relay-shaped rows against the seeded threads", () => {
55 const activity = buildShowcaseAgentActivity(threads, projects, NOW);
56 assert.isNotNull(activity);
57 if (!activity) return;
58
59 assert.strictEqual(activity.activeCount, 3);
60 assert.deepStrictEqual(
61 activity.activities.map((row) => [row.threadId, row.status, row.projectTitle, row.updatedAt]),
62 [
63 ["pocket-command-center", "Approval", "T3 Code", "2026-07-16T08:59:00.000Z"],
64 ["beautiful-boot", "Input", "Linux", "2026-07-16T08:56:00.000Z"],
65 ["buttery-suspense", "Working", "React", "2026-07-16T08:58:00.000Z"],
66 ["remote-command-center", "Done", "T3 Code", "2026-07-16T08:57:00.000Z"],
67 ],
68 );
69 assert.strictEqual(
70 activity.activities[0]?.deepLink,
71 "/threads/moonbase-terminal/pocket-command-center",
72 );
73 assert.strictEqual(activity.activities.length, SHOWCASE_AGENT_ACTIVITY_ROWS.length);
74});
75
76it("encodes the Android Live Update and its alert like a relay push", () => {
77 const activity = buildShowcaseAgentActivity(threads, projects, NOW);
78 assert.isNotNull(activity);
79 if (!activity) return;
80 const data = showcaseAndroidActivityData(activity, NOW);
81
82 assert.strictEqual(data.active, "true");
83 assert.strictEqual(data.activity_chip, "Review");
84 assert.strictEqual(data.activity_title, "3 active agents · 2 need attention");
85 assert.strictEqual(
86 data.activity_line_0,
87 "Approval\tPut the command center in your pocket\tT3 Code",
88 );
89 assert.strictEqual(data.activity_line_3, "Done\tMake remote coding feel local\tT3 Code");
90 assert.strictEqual(data.alert_title, "Put the command center in your pocket");
91 assert.strictEqual(data.alert_body, "Approval: T3 Code");
92 assert.strictEqual(data.alert_path, "/threads/moonbase-terminal/pocket-command-center");
93});
apps/mobile/src/features/showcase/showcaseAgentActivity.ts created+138
......@@ -0,0 +1,138 @@
1import type {
2 EnvironmentProject,
3 EnvironmentThreadShell,
4} from "@t3tools/client-runtime/state/shell";
5
6import type {
7 AgentActivityPhase,
8 AgentActivityProps,
9 AgentActivityRowProps,
10} from "../../widgets/AgentActivity";
11
12/**
13 * Agent work shown by the agent-activity scene. The rows point at
14 * seeded showcase threads so the copy matches the thread list, but the phases
15 * are staged: the relay that normally derives them is not part of the capture.
16 * Rows follow the relay's order: attention first, then running, then finished.
17 */
18export const SHOWCASE_AGENT_ACTIVITY_ROWS = [
19 { threadId: "pocket-command-center", phase: "waiting_for_approval", minutesAgo: 1 },
20 { threadId: "beautiful-boot", phase: "waiting_for_input", minutesAgo: 4 },
21 { threadId: "buttery-suspense", phase: "running", minutesAgo: 2 },
22 { threadId: "remote-command-center", phase: "completed", minutesAgo: 3 },
23] as const satisfies ReadonlyArray<{
24 readonly threadId: string;
25 readonly phase: AgentActivityPhase;
26 readonly minutesAgo: number;
27}>;
28
29// Matches the relay's row wording (AgentActivityPublisher.statusForPhase).
30const STATUS_BY_PHASE: Record<AgentActivityPhase, string> = {
31 starting: "Connecting",
32 running: "Working",
33 waiting_for_approval: "Approval",
34 waiting_for_input: "Input",
35 stale: "Waiting",
36 completed: "Done",
37 failed: "Failed",
38};
39
40const ACTIVE_PHASES: ReadonlySet<AgentActivityPhase> = new Set([
41 "starting",
42 "running",
43 "waiting_for_approval",
44 "waiting_for_input",
45]);
46
47/** Returns null until every staged thread and its project have loaded. */
48export function buildShowcaseAgentActivity(
49 threads: ReadonlyArray<EnvironmentThreadShell>,
50 projects: ReadonlyArray<EnvironmentProject>,
51 now: number,
52): AgentActivityProps | null {
53 const rows: AgentActivityRowProps[] = [];
54 for (const definition of SHOWCASE_AGENT_ACTIVITY_ROWS) {
55 const thread = threads.find((candidate) => String(candidate.id) === definition.threadId);
56 const project = thread
57 ? projects.find(
58 (candidate) =>
59 candidate.environmentId === thread.environmentId && candidate.id === thread.projectId,
60 )
61 : undefined;
62 if (!thread || !project) return null;
63 const environmentId = String(thread.environmentId);
64 rows.push({
65 environmentId,
66 threadId: definition.threadId,
67 projectTitle: project.title,
68 threadTitle: thread.title,
69 modelTitle: "",
70 phase: definition.phase,
71 status: STATUS_BY_PHASE[definition.phase],
72 updatedAt: new Date(now - definition.minutesAgo * 60_000).toISOString(),
73 deepLink: `/threads/${encodeURIComponent(environmentId)}/${encodeURIComponent(definition.threadId)}`,
74 });
75 }
76 return {
77 title: "T3 Code",
78 subtitle: "Agent work in progress",
79 activeCount: rows.filter((row) => ACTIVE_PHASES.has(row.phase)).length,
80 updatedAt: new Date(now).toISOString(),
81 activities: rows,
82 };
83}
84
85/** The alert the relay sends when the hero row starts waiting on the user. */
86export function showcaseAgentAlert(activity: AgentActivityProps) {
87 const row = activity.activities[0];
88 if (!row) return null;
89 return {
90 title: row.threadTitle,
91 body: `${row.status}: ${row.projectTitle}`,
92 path: row.deepLink,
93 environmentId: row.environmentId,
94 threadId: row.threadId,
95 };
96}
97
98/**
99 * The FCM data map Android's native receiver renders, mirroring the relay's
100 * androidActivityData so the shade shows exactly what a real push produces.
101 */
102export function showcaseAndroidActivityData(
103 activity: AgentActivityProps,
104 now: number,
105): Record<string, string> {
106 const attentionCount = activity.activities.filter(
107 (row) => row.phase === "waiting_for_approval" || row.phase === "waiting_for_input",
108 ).length;
109 const hero = activity.activities[0];
110 const alert = showcaseAgentAlert(activity);
111 return {
112 t3_kind: "agent_activity",
113 updated_at: String(now),
114 active: String(activity.activeCount > 0),
115 activity_chip: attentionCount > 0 ? "Review" : "Active",
116 activity_title: `${activity.activeCount} active agents · ${attentionCount} need attention`,
117 activity_phase: hero?.phase ?? "",
118 activity_active_count: String(activity.activeCount),
119 activity_attention_count: String(attentionCount),
120 activity_body: hero ? `${hero.status}: ${hero.threadTitle} · ${hero.projectTitle}` : "",
121 ...Object.fromEntries(
122 activity.activities.map((row, index) => [
123 `activity_line_${index}`,
124 [row.status, row.threadTitle, row.projectTitle].join("\t"),
125 ]),
126 ),
127 activity_path: hero?.deepLink ?? "/",
128 activity_expires_at: String(now + 2 * 60 * 60_000),
129 ...(alert
130 ? {
131 alert_id: "showcase-alert",
132 alert_title: alert.title,
133 alert_body: alert.body,
134 alert_path: alert.path,
135 }
136 : {}),
137 };
138}
apps/mobile/src/features/showcase/stageShowcaseAgentActivity.ts created+42
......@@ -0,0 +1,42 @@
1import * as Notifications from "expo-notifications";
2import { Platform } from "react-native";
3
4import type { AgentActivityProps } from "../../widgets/AgentActivity";
5import {
6 getAgentLiveActivities,
7 startAgentLiveActivity,
8} from "../agent-awareness/agentLiveActivity";
9import { showAndroidShowcaseAgentActivity } from "../agent-awareness/androidNotifications";
10import { showcaseAndroidActivityData } from "./showcaseAgentActivity";
11
12/**
13 * Puts the staged agent activity on screen for the capture runner, which then
14 * locks the simulator (iOS) or opens the notification shade (Android).
15 * Resolves true once shown, otherwise the reason it could not be, so the
16 * caller can retry and report.
17 */
18export async function stageShowcaseAgentActivity(
19 activity: AgentActivityProps,
20 now: number,
21): Promise<true | string> {
22 // The runner answers the iOS prompt and pre-grants Android's, so this only
23 // settles the permission the runner's alert delivery depends on.
24 const permission = await Notifications.requestPermissionsAsync({
25 ios: { allowAlert: true, allowBadge: true, allowSound: true },
26 });
27 if (!permission.granted) return `notification permission ${permission.status}`;
28
29 if (Platform.OS === "android") {
30 return (
31 showAndroidShowcaseAgentActivity(showcaseAndroidActivityData(activity, now)) ||
32 "native showShowcaseActivity missing"
33 );
34 }
35 if (Platform.OS !== "ios") return `unsupported platform ${Platform.OS}`;
36
37 // A retried or revisited scene must not stack a second card.
38 await Promise.all(getAgentLiveActivities().map((existing) => existing.end("immediate")));
39 // ActivityKit only starts activities while the app is foreground, which
40 // holds here: the runner locks the device after the scene reports ready.
41 return startAgentLiveActivity(activity) !== null || "Live Activity did not start";
42}
docs/operations/mobile-app-store-screenshots.md+18-9
......@@ -52,8 +52,8 @@ verify process ownership. Concurrent screenshot harnesses in different worktrees
5252collide or attach to the wrong Metro process.
5353
5454Every configured device defaults to dark appearance and the `t3-code` palette, so plain
55`pnpm screenshots:mobile` produces 30 dark PNGs. Pass `--appearance light`, `--appearance dark`, or
56`--appearance both` to override the configured appearance; `both` produces 60 PNGs.
55`pnpm screenshots:mobile` produces 35 dark PNGs. Pass `--appearance light`, `--appearance dark`, or
56`--appearance both` to override the configured appearance; `both` produces 70 PNGs.
5757
5858Pass `--theme <id>` (repeatable) or `--theme all` to capture the app's other palettes: `t3-code`,
5959`t3-chat`, `grove`, `ocean`, `ember`, and `iris`. The runner hands the palette to the app as a launch
......@@ -72,23 +72,32 @@ The default matrix is:
7272| `google-play/tablet-7/dark/t3-code/` | Pixel AVD at 600dp width | 1080×1920 | Google Play 7-inch tablet, portrait 9:16 |
7373| `google-play/tablet-10/dark/t3-code/` | Pixel AVD at 800dp width | 1440×2560 | Google Play 10-inch tablet, portrait 9:16 |
7474
75Each target captures thread, terminal, review, thread list, and environments. Each palette folder's
76five screenshots satisfy the configured Apple limit of 1–10, Google
75Each target captures thread, terminal, review, thread list, and environments, and every target but
76the iPad also captures agent activity. Each palette folder's five or six screenshots satisfy the configured Apple limit of 1–10, Google
7777phone requirement of 2–8, and Google tablet recommendation/slot minimum of 4 with a maximum of 8.
7878Every palette gets its own leaf folder so one upload slot never mixes themes and each folder keeps a
7979store-legal screenshot count.
8080
81The agent-activity scene shows what a user sees away from the app. The app stages the same Live
82Activity (iOS) or ongoing Live Update (Android) the relay would publish for four seeded threads. On
83iOS the runner then locks the simulator and pushes the matching approval alert with `simctl push`;
84on Android the staged update carries the alert and the runner opens the notification shade. Locking
85the simulator and answering the notification permission prompt use
86[AXe](https://github.com/cameroncooke/AXe), so install it (`brew tap cameroncooke/axe && brew
87install axe`) or set `AXE_PATH` before capturing iOS. The iPad skips the scene because the lock
88screen does not follow the app's landscape self-rotation.
89
8190The generated tree is deliberately aligned with the store upload fields:
8291
8392 artifacts/app-store/screenshots/
8493 ├── apple/
85 │ ├── iphone-6.9/dark/t3-code/{thread,terminal,review,threads,environments}.png
86 │ ├── iphone-6.5/dark/t3-code/{thread,terminal,review,threads,environments}.png
94 │ ├── iphone-6.9/dark/t3-code/{thread,terminal,review,threads,environments,agent-activity}.png
95 │ ├── iphone-6.5/dark/t3-code/{thread,terminal,review,threads,environments,agent-activity}.png
8796 │ └── ipad-13/dark/t3-code/{thread,terminal,review,threads,environments}.png
8897 └── google-play/
89 ├── phone/dark/t3-code/{thread,terminal,review,threads,environments}.png
90 ├── tablet-7/dark/t3-code/{thread,terminal,review,threads,environments}.png
91 └── tablet-10/dark/t3-code/{thread,terminal,review,threads,environments}.png
98 ├── phone/dark/t3-code/{thread,terminal,review,threads,environments,agent-activity}.png
99 ├── tablet-7/dark/t3-code/{thread,terminal,review,threads,environments,agent-activity}.png
100 └── tablet-10/dark/t3-code/{thread,terminal,review,threads,environments,agent-activity}.png
92101
93102A light-only run writes the same tree under `light/`; `--appearance both` writes both appearance
94103folders, and each requested theme adds a sibling folder next to `t3-code/`.
scripts/mobile-showcase-environment.ts+8-1
......@@ -11,7 +11,14 @@ export const SHOWCASE_PROJECT_ID = "t3code";
1111export const SHOWCASE_THREAD_ID = "remote-command-center";
1212export const SHOWCASE_TERMINAL_ID = "term-1";
1313
14export const SHOWCASE_SCENES = ["threads", "thread", "terminal", "review", "environments"] as const;
14export const SHOWCASE_SCENES = [
15 "threads",
16 "thread",
17 "terminal",
18 "review",
19 "environments",
20 "agent-activity",
21] as const;
1522export type ShowcaseScene = (typeof SHOWCASE_SCENES)[number];
1623
1724const PROJECTOR_NAMES = [
scripts/mobile-showcase.config.ts+7-5
......@@ -108,7 +108,7 @@ const config: ShowcaseConfig = {
108108 simulatorDeviceType: "com.apple.CoreSimulator.SimDeviceType.iPhone-17-Pro-Max",
109109 appearance: "dark",
110110 theme: DEFAULT_SHOWCASE_THEME,
111 scenes: ["thread", "terminal", "review", "threads", "environments"],
111 scenes: ["thread", "terminal", "review", "threads", "environments", "agent-activity"],
112112 storeAsset: {
113113 store: "apple",
114114 directory: "apple/iphone-6.9",
......@@ -125,7 +125,7 @@ const config: ShowcaseConfig = {
125125 simulatorDeviceType: "com.apple.CoreSimulator.SimDeviceType.iPhone-14-Plus",
126126 appearance: "dark",
127127 theme: DEFAULT_SHOWCASE_THEME,
128 scenes: ["thread", "terminal", "review", "threads", "environments"],
128 scenes: ["thread", "terminal", "review", "threads", "environments", "agent-activity"],
129129 storeAsset: {
130130 store: "apple",
131131 directory: "apple/iphone-6.5",
......@@ -143,6 +143,8 @@ const config: ShowcaseConfig = {
143143 appearance: "dark",
144144 theme: DEFAULT_SHOWCASE_THEME,
145145 orientation: "landscape",
146 // The lock screen does not follow the app's self-rotation, so a
147 // headless landscape capture of it would come out sideways.
146148 scenes: ["thread", "terminal", "review", "threads", "environments"],
147149 storeAsset: {
148150 store: "apple",
......@@ -167,7 +169,7 @@ const config: ShowcaseConfig = {
167169 height: 1920,
168170 density: 420,
169171 },
170 scenes: ["thread", "terminal", "review", "threads", "environments"],
172 scenes: ["thread", "terminal", "review", "threads", "environments", "agent-activity"],
171173 storeAsset: {
172174 store: "google-play",
173175 directory: "google-play/phone",
......@@ -190,7 +192,7 @@ const config: ShowcaseConfig = {
190192 height: 1920,
191193 density: 288,
192194 },
193 scenes: ["thread", "terminal", "review", "threads", "environments"],
195 scenes: ["thread", "terminal", "review", "threads", "environments", "agent-activity"],
194196 storeAsset: {
195197 store: "google-play",
196198 directory: "google-play/tablet-7",
......@@ -213,7 +215,7 @@ const config: ShowcaseConfig = {
213215 height: 2560,
214216 density: 288,
215217 },
216 scenes: ["thread", "terminal", "review", "threads", "environments"],
218 scenes: ["thread", "terminal", "review", "threads", "environments", "agent-activity"],
217219 storeAsset: {
218220 store: "google-play",
219221 directory: "google-play/tablet-10",
scripts/mobile-showcase.ts+170-2
......@@ -26,6 +26,7 @@ import showcaseConfig, {
2626import {
2727 SHOWCASE_ENVIRONMENTS,
2828 SHOWCASE_PROJECTS,
29 SHOWCASE_THREADS,
2930 seedShowcaseEnvironment,
3031} from "./mobile-showcase-environment.ts";
3132
......@@ -934,6 +935,133 @@ async function waitForIosShowcaseScene(
934935 throw new Error(`iOS showcase scene '${scene}' did not render within ${timeoutMs}ms.`);
935936}
936937
938function iosAxe(): string {
939 return NodeProcess.env.AXE_PATH ?? "axe";
940}
941
942async function runAxe(udid: string, args: ReadonlyArray<string>): Promise<void> {
943 // HID events sent right after the simulator settles are dropped without it.
944 await runCommand(iosAxe(), [...args, "--udid", udid], {
945 env: { ...NodeProcess.env, AXE_HID_STABILIZATION_MS: "3000" },
946 }).catch((error: unknown) => {
947 if ((error as NodeJS.ErrnoException).code === "ENOENT") {
948 throw new Error(
949 "The agent-activity scene drives the simulator with AXe. Install it with `brew tap cameroncooke/axe && brew install axe`, or set AXE_PATH.",
950 );
951 }
952 throw error;
953 });
954}
955
956async function iosSimulatorLocked(udid: string): Promise<boolean> {
957 const state = await commandOutput("xcrun", [
958 "simctl",
959 "spawn",
960 udid,
961 "notifyutil",
962 "-g",
963 "com.apple.springboard.lockstate",
964 ]).catch(() => "");
965 return state.trim().endsWith(" 1");
966}
967
968/**
969 * The staging app asks for notification permission; simctl has no privacy
970 * service for it, so answer the prompt until the scene reports ready.
971 */
972async function allowIosNotificationsUntilReady(udid: string, ready: Promise<void>): Promise<void> {
973 const state = { settled: false };
974 const tapping = (async () => {
975 while (!state.settled) {
976 await runAxe(udid, ["tap", "--label", "Allow"]).catch(() => undefined);
977 if (!state.settled) await delay(1_000);
978 }
979 })();
980 try {
981 await ready;
982 } finally {
983 state.settled = true;
984 await tapping;
985 }
986}
987
988/**
989 * Locks the simulator over the staged Live Activity and delivers the alert a
990 * relay push would, so the lock screen shows both.
991 */
992async function presentIosLockScreen(udid: string): Promise<void> {
993 await runAxe(udid, ["button", "lock"]);
994 const deadline = Date.now() + 15_000;
995 while (!(await iosSimulatorLocked(udid))) {
996 if (Date.now() > deadline) throw new Error(`Simulator ${udid} did not lock.`);
997 await delay(500);
998 }
999 const alert = showcaseAgentAlert();
1000 await new Promise<void>((resolve, reject) => {
1001 const child = NodeChildProcess.execFile(
1002 "xcrun",
1003 ["simctl", "push", udid, ANDROID_PACKAGE, "-"],
1004 { cwd: REPO_ROOT },
1005 (error) => (error ? reject(error) : resolve()),
1006 );
1007 child.stdin?.end(
1008 JSON.stringify({
1009 aps: { alert: { title: alert.title, body: alert.body }, sound: "default" },
1010 }),
1011 );
1012 });
1013 await wakeIosLockScreen(udid);
1014 // The first Live Activity on the lock screen asks to keep allowing them.
1015 await delay(2_000);
1016 await runAxe(udid, ["tap", "--label", "Allow"]).catch(() => undefined);
1017}
1018
1019/**
1020 * The alert usually wakes the display, but not always. A home press on a dark
1021 * display only wakes it, so press only after a screenshot proves it is dark;
1022 * pressing on a lit lock screen would unlock the device instead.
1023 */
1024async function wakeIosLockScreen(udid: string): Promise<void> {
1025 const probe = NodePath.join(NodeOS.tmpdir(), `t3-showcase-wake-${udid}.png`);
1026 try {
1027 for (let attempt = 0; attempt < 5; attempt += 1) {
1028 await delay(2_000);
1029 await runCommand("xcrun", ["simctl", "io", udid, "screenshot", probe]);
1030 if (!pngIsBlack(await NodeFSP.readFile(probe))) return;
1031 await runAxe(udid, ["button", "home"]);
1032 }
1033 throw new Error(`Simulator ${udid} lock screen stayed dark.`);
1034 } finally {
1035 await NodeFSP.rm(probe, { force: true });
1036 }
1037}
1038
1039function pngIsBlack(bytes: Uint8Array): boolean {
1040 const { data } = PNG.sync.read(Buffer.from(bytes));
1041 // Sample a sparse grid; a sleeping display is uniformly black.
1042 for (let offset = 0; offset < data.length; offset += 4 * 997) {
1043 if ((data[offset] ?? 0) > 8 || (data[offset + 1] ?? 0) > 8 || (data[offset + 2] ?? 0) > 8) {
1044 return false;
1045 }
1046 }
1047 return true;
1048}
1049
1050async function unlockIosSimulator(udid: string): Promise<void> {
1051 // The first press wakes the display, the second dismisses the lock screen.
1052 await runAxe(udid, ["button", "home"]);
1053 await delay(2_000);
1054 await runAxe(udid, ["button", "home"]);
1055}
1056
1057/** Mirrors the app's staged hero row (showcaseAgentActivity.ts). */
1058function showcaseAgentAlert(): { readonly title: string; readonly body: string } {
1059 const thread = SHOWCASE_THREADS.find((candidate) => candidate.id === "pocket-command-center");
1060 const project = SHOWCASE_PROJECTS.find((candidate) => candidate.id === thread?.projectId);
1061 if (!thread || !project) throw new Error("The showcase fixture lost its agent-activity thread.");
1062 return { title: thread.title, body: `Approval: ${project.title}` };
1063}
1064
9371065async function captureIos(
9381066 capture: ShowcaseCapture & { readonly device: ShowcaseIosDevice },
9391067 appPath: string | null,
......@@ -1021,6 +1149,12 @@ async function captureIos(
10211149 for (const [sceneIndex, scene] of capture.scenes.entries()) {
10221150 if (sceneIndex > 0) await NodeFSP.rm(readyPath, { force: true });
10231151 await NodeFSP.writeFile(scenePath, scene);
1152 const waitForScene = (timeoutMs?: number) => {
1153 const ready = waitForIosShowcaseScene(simulator.udid, scene, timeoutMs);
1154 return scene === "agent-activity"
1155 ? allowIosNotificationsUntilReady(simulator.udid, ready)
1156 : ready;
1157 };
10241158 if (sceneIndex === 0) {
10251159 for (let attempt = 0; attempt < 2; attempt += 1) {
10261160 const isLastAttempt = attempt === 1;
......@@ -1028,7 +1162,7 @@ async function captureIos(
10281162 // A freshly installed Expo development build can spend well over 30s
10291163 // applying an already-bundled update after it reaches 100%. Killing it
10301164 // at that point sends the next capture back to the dev launcher.
1031 await waitForIosShowcaseScene(simulator.udid, scene, 120_000);
1165 await waitForScene(120_000);
10321166 break;
10331167 } catch (error) {
10341168 if (isLastAttempt) throw error;
......@@ -1036,8 +1170,9 @@ async function captureIos(
10361170 }
10371171 }
10381172 } else {
1039 await waitForIosShowcaseScene(simulator.udid, scene);
1173 await waitForScene();
10401174 }
1175 if (scene === "agent-activity") await presentIosLockScreen(simulator.udid);
10411176 await delay(scene === "review" ? Math.max(config.settleDelayMs, 8_000) : config.settleDelayMs);
10421177 const destination = NodePath.join(
10431178 showcaseCaptureDirectory(outputDirectory, capture),
......@@ -1053,6 +1188,7 @@ async function captureIos(
10531188 await runCommand("sips", ["--rotate", "270", destination]);
10541189 }
10551190 }
1191 if (scene === "agent-activity") await unlockIosSimulator(simulator.udid);
10561192 await finalizeCapture(destination, capture.device);
10571193 }
10581194}
......@@ -1069,6 +1205,22 @@ async function runAdb(serial: string, args: ReadonlyArray<string>): Promise<void
10691205 await runCommand(androidSdkTool("platform-tools/adb"), ["-s", serial, ...args]);
10701206}
10711207
1208/**
1209 * Emulator images post their own ongoing notices (keyboard configured, serial
1210 * console enabled) that would share the shade with the app's. Snoozing hides
1211 * them for the capture; they come back on their own an hour later.
1212 */
1213async function snoozeAndroidSystemNotifications(serial: string): Promise<void> {
1214 const keys = (await adbOutput(serial, ["shell", "cmd", "notification", "list"]))
1215 .split("\n")
1216 .map((line) => line.trim())
1217 .filter((key) => key.includes("|") && !key.includes(`|${ANDROID_PACKAGE}|`));
1218 for (const key of keys) {
1219 // adb joins shell args with spaces, so the key needs quoting for the pipes.
1220 await runAdb(serial, ["shell", `cmd notification snooze --for 3600000 '${key}'`]);
1221 }
1222}
1223
10721224async function runningAndroidAvds(): Promise<ReadonlyMap<string, string>> {
10731225 const adb = androidSdkTool("platform-tools/adb");
10741226 const devices = (await commandOutput(adb, ["devices"]))
......@@ -1248,6 +1400,15 @@ async function captureAndroid(
12481400 await runAdb(serial, ["install", "-r", apkPath]);
12491401 }
12501402 await runAdb(serial, ["shell", "pm", "clear", ANDROID_PACKAGE]);
1403 // The agent-activity scene posts notifications; granting up front keeps the
1404 // runtime prompt off every capture.
1405 await runAdb(serial, [
1406 "shell",
1407 "pm",
1408 "grant",
1409 ANDROID_PACKAGE,
1410 "android.permission.POST_NOTIFICATIONS",
1411 ]);
12511412 await prepareAndroidShowcaseApp(serial);
12521413 await runAdb(serial, ["reverse", `tcp:${config.metroPort}`, `tcp:${config.metroPort}`]);
12531414 const metroUrl = encodeURIComponent(`http://127.0.0.1:${config.metroPort}?disableOnboarding=1`);
......@@ -1275,6 +1436,10 @@ async function captureAndroid(
12751436 for (const [sceneIndex, scene] of capture.scenes.entries()) {
12761437 if (sceneIndex > 0) await writeAndroidShowcaseScene(serial, scene);
12771438 await waitForAndroidShowcaseScene(serial, scene);
1439 if (scene === "agent-activity") {
1440 await snoozeAndroidSystemNotifications(serial);
1441 await runAdb(serial, ["shell", "cmd", "statusbar", "expand-notifications"]);
1442 }
12781443 await delay(Math.max(config.settleDelayMs, scene === "review" ? 8_000 : 5_000));
12791444 const destination = NodePath.join(
12801445 showcaseCaptureDirectory(outputDirectory, capture),
......@@ -1292,6 +1457,9 @@ async function captureAndroid(
12921457 );
12931458 });
12941459 await NodeFSP.writeFile(destination, png);
1460 if (scene === "agent-activity") {
1461 await runAdb(serial, ["shell", "cmd", "statusbar", "collapse"]);
1462 }
12951463 await finalizeCapture(destination, capture.device);
12961464 }
12971465}