authorgravatar for me@t3.ggTheo Browne <me@t3.gg> 2026-09-03 03:08:39-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-09-03 03:08:39-07:00
log19c97ea56d30b3a2de31a060f8f47d6b7404b78f
tree4915d07aca5d8acc8ade88e8a003c72e632781ce
parent2b745efe57cddc1753d3007d081c64bd20e3ab79
signature Signed by PGP key B5690EEEBB952194

fix(web): unlock the composer when preview capture fails (#9127)

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>

8 files changed, 495 insertions(+), 83 deletions(-)

apps/desktop/src/preview/Manager.test.ts+158
......@@ -3137,6 +3137,164 @@ describe("PreviewManager", () => {
31373137 ),
31383138 );
31393139
3140 effectIt.effect("settles the pick when the annotation screenshot never arrives", () =>
3141 withManager((manager) =>
3142 Effect.gen(function* () {
3143 let onPicked: ((event: unknown, ...args: unknown[]) => void) | undefined;
3144 fromId.mockReturnValue({
3145 id: 42,
3146 isDestroyed: () => false,
3147 getType: () => "webview",
3148 getURL: () => "https://example.com",
3149 getTitle: () => "Example",
3150 isLoading: () => false,
3151 isFocused: () => true,
3152 getZoomFactor: () => 1,
3153 setZoomFactor: vi.fn(),
3154 setAudioMuted: vi.fn(),
3155 isCurrentlyAudible: () => false,
3156 on: vi.fn(),
3157 once: vi.fn(),
3158 off: vi.fn(),
3159 // A wedged compositor leaves `capturePage` pending forever.
3160 capturePage: vi.fn(() => new Promise(() => {})),
3161 ipc: {
3162 on: vi.fn((channel: string, listener: typeof onPicked) => {
3163 if (channel === "preview:element-picked") onPicked = listener;
3164 }),
3165 off: vi.fn(),
3166 removeListener: vi.fn(),
3167 },
3168 send: webviewSend,
3169 navigationHistory: { canGoBack: () => false, canGoForward: () => false },
3170 setWindowOpenHandler: vi.fn(),
3171 debugger: {
3172 isAttached: () => false,
3173 attach: vi.fn(),
3174 sendCommand: vi.fn(async () => undefined),
3175 on: vi.fn(),
3176 off: vi.fn(),
3177 },
3178 } as never);
3179
3180 yield* manager.createTab("tab_1");
3181 yield* manager.registerWebview("tab_1", 42);
3182 const pick = yield* manager.pickElement("tab_1").pipe(Effect.forkChild);
3183 yield* Effect.yieldNow;
3184
3185 onPicked?.(
3186 {},
3187 {
3188 id: "annotation_1",
3189 pageUrl: "https://example.com",
3190 pageTitle: "Example",
3191 comment: "Tighten this spacing",
3192 elements: [],
3193 regions: [{ id: "region_1", rect: { x: 5, y: 6, width: 20, height: 30 } }],
3194 strokes: [],
3195 styleChanges: [],
3196 screenshot: null,
3197 createdAt: "2026-06-11T00:00:00.000Z",
3198 },
3199 null,
3200 "send",
3201 );
3202 yield* Effect.yieldNow;
3203 expect(pick.pollUnsafe()).toBeUndefined();
3204
3205 yield* TestClock.adjust("6 seconds");
3206 // The pick has to give up on the crop rather than strand the renderer,
3207 // which would leave the composer stuck on "Capturing…".
3208 const result = yield* Fiber.join(pick);
3209 expect(result?.annotation.screenshot).toBeNull();
3210 expect(result?.screenshotFailed).toBe(true);
3211 expect(result?.submission).toBe("send");
3212 expect(webviewSend).toHaveBeenCalledWith("preview:annotation-captured");
3213 }),
3214 ),
3215 );
3216
3217 effectIt.effect("a stale capture from a replaced pick never touches the next pick", () =>
3218 withManager((manager) =>
3219 Effect.gen(function* () {
3220 let onPicked: ((event: unknown, ...args: unknown[]) => void) | undefined;
3221 fromId.mockReturnValue({
3222 id: 42,
3223 isDestroyed: () => false,
3224 getType: () => "webview",
3225 getURL: () => "https://example.com",
3226 getTitle: () => "Example",
3227 isLoading: () => false,
3228 isFocused: () => true,
3229 getZoomFactor: () => 1,
3230 setZoomFactor: vi.fn(),
3231 setAudioMuted: vi.fn(),
3232 isCurrentlyAudible: () => false,
3233 on: vi.fn(),
3234 once: vi.fn(),
3235 off: vi.fn(),
3236 capturePage: vi.fn(() => new Promise(() => {})),
3237 ipc: {
3238 on: vi.fn((channel: string, listener: typeof onPicked) => {
3239 if (channel === "preview:element-picked") onPicked = listener;
3240 }),
3241 off: vi.fn(),
3242 removeListener: vi.fn(),
3243 },
3244 send: webviewSend,
3245 navigationHistory: { canGoBack: () => false, canGoForward: () => false },
3246 setWindowOpenHandler: vi.fn(),
3247 debugger: {
3248 isAttached: () => false,
3249 attach: vi.fn(),
3250 sendCommand: vi.fn(async () => undefined),
3251 on: vi.fn(),
3252 off: vi.fn(),
3253 },
3254 } as never);
3255 const annotation = {
3256 id: "annotation_1",
3257 pageUrl: "https://example.com",
3258 pageTitle: "Example",
3259 comment: "Tighten this spacing",
3260 elements: [],
3261 regions: [{ id: "region_1", rect: { x: 5, y: 6, width: 20, height: 30 } }],
3262 strokes: [],
3263 styleChanges: [],
3264 screenshot: null,
3265 createdAt: "2026-06-11T00:00:00.000Z",
3266 };
3267
3268 yield* manager.createTab("tab_1");
3269 yield* manager.registerWebview("tab_1", 42);
3270 const firstPick = yield* manager.pickElement("tab_1").pipe(Effect.forkChild);
3271 yield* Effect.yieldNow;
3272 // The first pick submits and its crop hangs.
3273 onPicked?.({}, annotation, null, "send");
3274 yield* Effect.yieldNow;
3275
3276 // A second pick on the same tab replaces the first, which resumes null.
3277 const secondPick = yield* manager.pickElement("tab_1").pipe(Effect.forkChild);
3278 yield* Effect.yieldNow;
3279 expect(yield* Fiber.join(firstPick)).toBeNull();
3280 webviewSend.mockClear();
3281
3282 // The first pick's crop times out while the second pick is live. It
3283 // must not signal the overlay, which would tear down the second pick.
3284 yield* TestClock.adjust("6 seconds");
3285 yield* Effect.yieldNow;
3286 expect(webviewSend).not.toHaveBeenCalledWith("preview:annotation-captured");
3287 expect(secondPick.pollUnsafe()).toBeUndefined();
3288
3289 onPicked?.({}, { ...annotation, id: "annotation_2" }, null, "attach");
3290 yield* TestClock.adjust("6 seconds");
3291 const result = yield* Fiber.join(secondPick);
3292 expect(result?.annotation.id).toBe("annotation_2");
3293 expect(result?.submission).toBe("attach");
3294 }),
3295 ),
3296 );
3297
31403298 effectIt.effect("navigates the guest history when the thumb-button ipc fires", () =>
31413299 withManager((manager) =>
31423300 Effect.gen(function* () {
apps/desktop/src/preview/Manager.ts+88-23
......@@ -306,13 +306,24 @@ const normalizeCaptureRect = (value: unknown): PreviewAnnotationRect | null => {
306306 };
307307};
308308
309/** `capturePage` never settles when the guest's compositor is wedged. */
310const ANNOTATION_SCREENSHOT_TIMEOUT = "5 seconds";
311
312/**
313 * Crops the guest for a picked annotation. A stalled `capturePage` resolves to
314 * `null` after the timeout: the annotation is still sendable without its
315 * screenshot, and the pick session must settle either way.
316 */
309317const captureAnnotationScreenshot = (
310318 tabId: string,
311319 wc: Electron.WebContents,
312320 cropRect: PreviewAnnotationRect | null,
313321): Effect.Effect<PreviewAnnotationPayload["screenshot"], PreviewManagerError> =>
314322 Effect.tryPromise({
315 try: () =>
323 // The unused abort signal is what makes this interruptible, and therefore
324 // what lets the timeout below fire. Drop the parameter and a stalled
325 // capture strands the pick session again.
326 try: (_signal) =>
316327 wc.capturePage(
317328 cropRect
318329 ? {
......@@ -331,7 +342,7 @@ const captureAnnotationScreenshot = (
331342 cause,
332343 }),
333344 }).pipe(
334 Effect.map((image) => {
345 Effect.map((image): PreviewAnnotationPayload["screenshot"] => {
335346 const size = image.getSize();
336347 return {
337348 dataUrl: image.toDataURL(),
......@@ -340,6 +351,15 @@ const captureAnnotationScreenshot = (
340351 cropRect: cropRect ?? { x: 0, y: 0, width: size.width, height: size.height },
341352 };
342353 }),
354 Effect.timeoutOption(ANNOTATION_SCREENSHOT_TIMEOUT),
355 Effect.flatMap((screenshot) =>
356 Option.isSome(screenshot)
357 ? Effect.succeed(screenshot.value)
358 : Effect.logWarning("preview annotation screenshot timed out").pipe(
359 Effect.annotateLogs({ tabId, webContentsId: wc.id }),
360 Effect.as(null),
361 ),
362 ),
343363 );
344364
345365const findZoomStep = (current: number): number => {
......@@ -2370,30 +2390,52 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
23702390 const annotationTheme = yield* Ref.get(annotationThemeRef);
23712391 return yield* Effect.callback<PreviewAnnotationSubmissionResult | null, PreviewManagerError>(
23722392 (resume) => {
2393 // Declared first so cleanup can check slot ownership by identity
2394 // without a type cycle through the cancel effect it builds.
2395 const session: PickSession = { cancel: Effect.suspend(() => cancelPickSession()) };
23732396 const cleanup = Effect.fn("PreviewManager.cleanupPickElement")(function* () {
23742397 yield* attempt({ operation: "pickElement.cleanup", tabId, webContentsId: wc.id }, () => {
23752398 wc.ipc.removeListener(ELEMENT_PICKED_CHANNEL, onMessage);
23762399 wc.off("destroyed", onDestroyed);
23772400 wc.off("did-start-navigation", onNavigated);
23782401 }).pipe(Effect.ignore);
2402 // Only drop the slot while it is still ours. A newer session may
2403 // already have swapped itself in before cancelling this one.
23792404 yield* Ref.update(pickSessionsRef, (sessions) =>
2380 replaceMap(sessions, (copy) => {
2381 copy.delete(tabId);
2382 }),
2405 sessions.get(tabId) === session
2406 ? replaceMap(sessions, (copy) => {
2407 copy.delete(tabId);
2408 })
2409 : sessions,
23832410 );
23842411 });
2385 const settlePick = Effect.fn("PreviewManager.settlePickElement")(function* (
2412 // Every exit from this session runs through `claimSettle`, so the
2413 // renderer's `pickElement` promise resolves exactly once. The previous
2414 // identity check let a cancelled or replaced session return without
2415 // resuming, which left the composer waiting forever.
2416 let settled = false;
2417 const claimSettle = (): boolean => {
2418 if (settled) return false;
2419 settled = true;
2420 return true;
2421 };
2422 const finishPick = Effect.fn("PreviewManager.finishPickElement")(function* (
23862423 payload: PreviewAnnotationSubmissionResult | null,
23872424 ) {
2388 const active = (yield* Ref.get(pickSessionsRef)).get(tabId);
2389 if (!active || active.cancel !== cancel) return;
23902425 yield* cleanup();
23912426 resume(Effect.succeed(payload));
23922427 });
2428 const settlePick = Effect.fn("PreviewManager.settlePickElement")(function* (
2429 payload: PreviewAnnotationSubmissionResult | null,
2430 ) {
2431 if (!claimSettle()) return;
2432 yield* finishPick(payload);
2433 });
23932434 const settle = (payload: PreviewAnnotationSubmissionResult | null) => {
23942435 runFork(settlePick(payload));
23952436 };
23962437 const cancelPickSession = Effect.fn("PreviewManager.cancelPickSession")(function* () {
2438 if (!claimSettle()) return;
23972439 yield* cleanup();
23982440 const tabs = yield* SynchronizedRef.get(tabsRef);
23992441 const activeTab = tabs.get(tabId);
......@@ -2412,7 +2454,6 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
24122454 }
24132455 resume(Effect.succeed(null));
24142456 });
2415 const cancel = cancelPickSession();
24162457 const onMessage = (_event: Electron.IpcMainEvent, ...args: unknown[]): void => {
24172458 const payload = args[0];
24182459 if (!isPreviewAnnotationPayload(payload)) {
......@@ -2423,19 +2464,32 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
24232464 const submission = args[2] === "send" ? "send" : "attach";
24242465 runFork(
24252466 captureAnnotationScreenshot(tabId, wc, cropRect).pipe(
2426 Effect.matchEffect({
2427 onFailure: () => Effect.sync(() => settle({ annotation: payload, submission })),
2428 onSuccess: (screenshot) =>
2429 Effect.sync(() => settle({ annotation: { ...payload, screenshot }, submission })),
2467 // The renderer cannot tell a dropped crop from a comment-only
2468 // pick by the null alone, so a failed or timed-out capture is
2469 // flagged on the result.
2470 Effect.match({
2471 onFailure: (): PreviewAnnotationSubmissionResult => ({
2472 annotation: payload,
2473 submission,
2474 screenshotFailed: true,
2475 }),
2476 onSuccess: (screenshot): PreviewAnnotationSubmissionResult =>
2477 screenshot === null
2478 ? { annotation: payload, submission, screenshotFailed: true }
2479 : { annotation: { ...payload, screenshot }, submission },
24302480 }),
2431 Effect.ensuring(
2432 attempt(
2481 Effect.flatMap((result) => {
2482 // A capture that outlives its session must not touch the
2483 // overlay: the preload tears down on the captured signal, and
2484 // by now it may be running a newer pick.
2485 if (!claimSettle()) return Effect.void;
2486 return attempt(
24332487 { operation: "pickElement.captureComplete", tabId, webContentsId: wc.id },
24342488 () => {
24352489 if (!wc.isDestroyed()) wc.send(ANNOTATION_CAPTURED_CHANNEL);
24362490 },
2437 ).pipe(Effect.ignore),
2438 ),
2491 ).pipe(Effect.ignore, Effect.andThen(finishPick(result)));
2492 }),
24392493 ),
24402494 );
24412495 };
......@@ -2449,6 +2503,21 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
24492503 if (isMainFrame) settle(null);
24502504 };
24512505 const registerPickElement = Effect.fn("PreviewManager.registerPickElement")(function* () {
2506 // Two picks on one tab can overlap. Swap this session in and cancel
2507 // the previous holder in one step, so no third pick can slip into an
2508 // empty slot in between and the session we push out still resumes
2509 // its renderer.
2510 const replaced = yield* Ref.modify(pickSessionsRef, (sessions) => [
2511 sessions.get(tabId) ?? null,
2512 replaceMap(sessions, (copy) => {
2513 copy.set(tabId, session);
2514 }),
2515 ]);
2516 if (replaced) yield* replaced.cancel;
2517 // A newer pick may have cancelled this session while the previous
2518 // one was torn down. Cleanup already ran, so attaching listeners now
2519 // would leak them and start an overlay nobody is waiting on.
2520 if (settled) return;
24522521 yield* attempt({ operation: "pickElement.register", tabId, webContentsId: wc.id }, () => {
24532522 wc.ipc.on(ELEMENT_PICKED_CHANNEL, onMessage);
24542523 wc.once("destroyed", onDestroyed);
......@@ -2456,21 +2525,17 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
24562525 if (!wc.isFocused()) wc.focus();
24572526 wc.send(START_PICK_CHANNEL, annotationTheme);
24582527 });
2459 yield* Ref.update(pickSessionsRef, (sessions) =>
2460 replaceMap(sessions, (copy) => {
2461 copy.set(tabId, { cancel });
2462 }),
2463 );
24642528 });
24652529 runFork(
24662530 registerPickElement().pipe(
24672531 Effect.catch((error: PreviewManagerError) => {
2532 if (!claimSettle()) return Effect.void;
24682533 resume(Effect.fail(error));
24692534 return cleanup();
24702535 }),
24712536 ),
24722537 );
2473 return cancel;
2538 return session.cancel;
24742539 },
24752540 );
24762541 });
apps/desktop/src/preview/PickPreload.ts+105-44
......@@ -1,4 +1,4 @@
1// @effect-diagnostics globalDate:off - This isolated Electron preload does not run inside an Effect runtime.
1// @effect-diagnostics globalDate:off globalTimers:off - This isolated Electron preload does not run inside an Effect runtime.
22import { ipcRenderer } from "electron";
33import { getElementContext } from "react-grab/primitives";
44import type {
......@@ -30,6 +30,8 @@ const Z_INDEX_OVERLAY = 2147483646;
3030const PRIMARY = "var(--t3-primary)";
3131const PRIMARY_FILL = "color-mix(in srgb, var(--t3-primary) 10%, transparent)";
3232const MAX_MARQUEE_ELEMENTS = 20;
33/** Upper bound on one element's React context lookup during submit. */
34const ELEMENT_CONTEXT_TIMEOUT_MS = 5_000;
3335const CONTENT_LAYER_Z_INDEX = 1;
3436const CHROME_LAYER_Z_INDEX = 10;
3537
......@@ -279,25 +281,67 @@ function toStackFrame(frame: {
279281 };
280282}
281283
282async function captureElement(element: Element): Promise<PickedElementPayload | null> {
284/**
285 * Resolves to `null` instead of hanging when `promise` outlives `millis`.
286 * `getElementContext` walks the inspected page's React internals, and some
287 * pages leave it pending forever. Without a bound, the whole submit chain
288 * stalls and the overlay sits on "Capturing…".
289 */
290function withCaptureTimeout<A>(promise: Promise<A>, millis: number): Promise<A | null> {
291 let timer: ReturnType<typeof setTimeout> | undefined;
292 return Promise.race([
293 promise,
294 new Promise<null>((resolve) => {
295 timer = setTimeout(() => resolve(null), millis);
296 }),
297 ]).finally(() => clearTimeout(timer));
298}
299
300/** Truncation for the DOM-only preview used when React context is unavailable. */
301const HTML_PREVIEW_MAX_CHARS = 500;
302
303/**
304 * Describes a picked element. The React context lookup can stall or throw on
305 * some pages, so the element is never dropped: without context it still
306 * carries its tag, a short HTML preview, and its rect so the crop stays on the
307 * pick instead of falling back to the whole viewport.
308 */
309async function captureElement(element: Element): Promise<PickedElementPayload> {
310 const base = {
311 pageUrl: location.href,
312 pageTitle: document.title?.trim() || null,
313 tagName: element.tagName.toLowerCase(),
314 pickedAt: new Date().toISOString(),
315 };
283316 try {
284 const context = await getElementContext(element);
285 const stack = (context.stack ?? []).map(toStackFrame);
286 return {
287 pageUrl: location.href,
288 pageTitle: document.title?.trim() || null,
289 tagName: element.tagName.toLowerCase(),
290 selector: context.selector,
291 htmlPreview: context.htmlPreview ?? "",
292 componentName: context.componentName,
293 source: stack[0] ?? null,
294 stack,
295 styles: context.styles ?? "",
296 pickedAt: new Date().toISOString(),
297 };
317 const context = await withCaptureTimeout(
318 Promise.resolve(getElementContext(element)),
319 ELEMENT_CONTEXT_TIMEOUT_MS,
320 );
321 if (context) {
322 const stack = (context.stack ?? []).map(toStackFrame);
323 return {
324 ...base,
325 selector: context.selector,
326 htmlPreview: context.htmlPreview ?? "",
327 componentName: context.componentName,
328 source: stack[0] ?? null,
329 stack,
330 styles: context.styles ?? "",
331 };
332 }
298333 } catch {
299 return null;
334 // Fall through to the DOM-only payload.
300335 }
336 return {
337 ...base,
338 selector: null,
339 htmlPreview: element.outerHTML.slice(0, HTML_PREVIEW_MAX_CHARS),
340 componentName: null,
341 source: null,
342 stack: [],
343 styles: "",
344 };
301345}
302346
303347function createButton(label: string, title: string): HTMLButtonElement {
......@@ -1225,12 +1269,20 @@ function startAnnotation(): void {
12251269 pendingCapture = true;
12261270 submit.disabled = true;
12271271 submit.textContent = "Capturing…";
1272 // Snapshot everything the annotation will carry before the capture runs.
1273 // The element context lookup can take up to its timeout, and the user can
1274 // keep editing meanwhile; the annotation must describe what they submitted.
1275 const submittedComment = comment.value.trim();
1276 const submittedRegions = [...regions];
1277 const submittedStrokes = [...strokes];
1278 const submittedStyleChanges = Array.from(styleChanges.values(), (change) => ({ ...change }));
12281279 void Promise.all(
12291280 Array.from(selected.values()).map(async (target) => {
12301281 const element = await captureElement(target.element);
1231 if (!element) return null;
1232 for (const change of styleChanges.values()) {
1233 if (change.targetId === target.id) change.selector = element.selector;
1282 for (const change of submittedStyleChanges) {
1283 if (change.targetId === target.id && element.selector !== null) {
1284 change.selector = element.selector;
1285 }
12341286 }
12351287 return {
12361288 id: target.id,
......@@ -1238,30 +1290,39 @@ function startAnnotation(): void {
12381290 rect: rectFromDomRect(target.element.getBoundingClientRect()),
12391291 };
12401292 }),
1241 ).then((captured) => {
1242 const elements = captured.filter((target) => target !== null);
1243 const annotation: PreviewAnnotationPayload = {
1244 id: nextId("annotation"),
1245 pageUrl: location.href,
1246 pageTitle: document.title?.trim() || null,
1247 comment: comment.value.trim(),
1248 elements,
1249 regions: [...regions],
1250 strokes: [...strokes],
1251 styleChanges: Array.from(styleChanges.values()),
1252 screenshot: null,
1253 createdAt: new Date().toISOString(),
1254 };
1255 editor.style.display = "none";
1256 toolbar.style.display = "none";
1257 hoverOutline.style.display = "none";
1258 const screenshotRect = unionRects([
1259 ...elements.map((target) => target.rect),
1260 ...regions.map((region) => region.rect),
1261 ...strokes.map((stroke) => stroke.bounds),
1262 ]);
1263 ipcRenderer.send(ELEMENT_PICKED_CHANNEL, annotation, screenshotRect, submission);
1264 });
1293 )
1294 .then((elements) => {
1295 // The overlay may have been cancelled or replaced while the capture
1296 // ran. A late submit must not deliver into the next pick's listener.
1297 if (finished) return;
1298 const annotation: PreviewAnnotationPayload = {
1299 id: nextId("annotation"),
1300 pageUrl: location.href,
1301 pageTitle: document.title?.trim() || null,
1302 comment: submittedComment,
1303 elements,
1304 regions: submittedRegions,
1305 strokes: submittedStrokes,
1306 styleChanges: submittedStyleChanges,
1307 screenshot: null,
1308 createdAt: new Date().toISOString(),
1309 };
1310 editor.style.display = "none";
1311 toolbar.style.display = "none";
1312 hoverOutline.style.display = "none";
1313 const screenshotRect = unionRects([
1314 ...elements.map((target) => target.rect),
1315 ...submittedRegions.map((region) => region.rect),
1316 ...submittedStrokes.map((stroke) => stroke.bounds),
1317 ]);
1318 ipcRenderer.send(ELEMENT_PICKED_CHANNEL, annotation, screenshotRect, submission);
1319 })
1320 .catch(() => {
1321 // Last resort. Main is waiting on this message, so hand it an empty
1322 // pick rather than leaving the button stuck on "Capturing…" and the
1323 // renderer's pick promise pending. teardown is a no-op once finished.
1324 teardown(true);
1325 });
12651326 };
12661327 submit.addEventListener("click", () => submitAnnotation("attach"));
12671328 root.addEventListener("keydown", (event) => {
apps/web/src/components/preview/PreviewView.test.tsx+45-6
......@@ -27,7 +27,7 @@ const mocks = vi.hoisted(() => ({
2727 openPictureInPicture: vi.fn(async (_tabId: string): Promise<void> => undefined),
2828 closePictureInPicture: vi.fn(async (_tabId: string): Promise<void> => undefined),
2929 pickElement: vi.fn(),
30 previewAnnotationScreenshotFile: vi.fn(),
30 capturePreviewAnnotationScreenshot: vi.fn(),
3131 addPreviewAnnotation: vi.fn(),
3232 addImage: vi.fn(),
3333 toggleAnnotation: null as (() => void) | null,
......@@ -90,7 +90,7 @@ vi.mock("~/composerDraftStore", () => ({
9090}));
9191
9292vi.mock("~/lib/previewAnnotation", () => ({
93 previewAnnotationScreenshotFile: mocks.previewAnnotationScreenshotFile,
93 capturePreviewAnnotationScreenshot: mocks.capturePreviewAnnotationScreenshot,
9494}));
9595
9696vi.mock("~/localApi", () => ({
......@@ -252,6 +252,7 @@ vi.mock("~/browser/BrowserSurfaceSlot", () => ({ BrowserSurfaceSlot: () => null
252252vi.mock("./usePreviewSession", () => ({ usePreviewSession: vi.fn() }));
253253
254254import { PreviewView, previewProfileName } from "./PreviewView";
255import { toastManager } from "~/components/ui/toast";
255256import { previewRuntimeTabId } from "~/browser/previewRuntimeTabId";
256257
257258const TEST_THREAD_REF = {
......@@ -339,8 +340,10 @@ describe("PreviewView navigation", () => {
339340 mocks.openPictureInPicture.mockClear();
340341 mocks.closePictureInPicture.mockClear();
341342 mocks.pickElement.mockReset();
342 mocks.previewAnnotationScreenshotFile.mockReset();
343 mocks.capturePreviewAnnotationScreenshot.mockReset();
344 mocks.capturePreviewAnnotationScreenshot.mockResolvedValue({ status: "none" });
343345 mocks.addPreviewAnnotation.mockClear();
346 vi.mocked(toastManager.add).mockClear();
344347 mocks.addImage.mockClear();
345348 mocks.toggleAnnotation = null;
346349 mocks.pictureInPicture = false;
......@@ -551,7 +554,39 @@ describe("PreviewView navigation", () => {
551554 expect(mocks.addPreviewAnnotation).toHaveBeenCalledWith(TEST_THREAD_REF, annotation);
552555 });
553556
554 it("still sends when screenshot attachment conversion fails", async () => {
557 it("warns when main dropped the crop before handing over the pick", async () => {
558 const annotation = {
559 id: "annotation-3",
560 pageUrl: "https://example.com/dashboard",
561 pageTitle: "Dashboard",
562 comment: "Tighten this spacing",
563 elements: [],
564 regions: [],
565 strokes: [],
566 styleChanges: [],
567 screenshot: null,
568 createdAt: "2026-07-27T00:00:00.000Z",
569 };
570 const onSendAnnotation = vi.fn();
571 mocks.pickElement.mockResolvedValue({ annotation, submission: "send", screenshotFailed: true });
572
573 renderToStaticMarkup(
574 <PreviewView
575 threadRef={TEST_THREAD_REF}
576 tabId="tab-1"
577 visible
578 onSendAnnotation={onSendAnnotation}
579 />,
580 );
581 mocks.toggleAnnotation?.();
582
583 await vi.waitFor(() => expect(onSendAnnotation).toHaveBeenCalledWith(annotation, null));
584 // A null screenshot alone looks like a comment-only pick; the flag is what
585 // separates "no crop requested" from "crop lost to a timeout".
586 expect(toastManager.add).toHaveBeenCalledTimes(1);
587 });
588
589 it("still sends when the picked element's crop cannot be captured", async () => {
555590 const annotation = {
556591 id: "annotation-2",
557592 pageUrl: "https://example.com/dashboard",
......@@ -571,7 +606,7 @@ describe("PreviewView navigation", () => {
571606 };
572607 const onSendAnnotation = vi.fn();
573608 mocks.pickElement.mockResolvedValue({ annotation, submission: "send" });
574 mocks.previewAnnotationScreenshotFile.mockRejectedValue(new Error("conversion failed"));
609 mocks.capturePreviewAnnotationScreenshot.mockResolvedValue({ status: "failed" });
575610
576611 renderToStaticMarkup(
577612 <PreviewView
......@@ -583,7 +618,11 @@ describe("PreviewView navigation", () => {
583618 );
584619 mocks.toggleAnnotation?.();
585620
586 await vi.waitFor(() => expect(onSendAnnotation).toHaveBeenCalledWith(annotation, null));
621 // The forwarded and stored annotation both drop the screenshot, so the
622 // prompt does not claim a crop that was never attached.
623 const sent = { ...annotation, screenshot: null };
624 await vi.waitFor(() => expect(onSendAnnotation).toHaveBeenCalledWith(sent, null));
625 expect(mocks.addPreviewAnnotation).toHaveBeenCalledWith(TEST_THREAD_REF, sent);
587626 expect(mocks.addImage).not.toHaveBeenCalled();
588627 });
589628});
apps/web/src/components/preview/PreviewView.tsx+23-8
......@@ -20,7 +20,7 @@ import {
2020 useThreadRecentHistory,
2121} from "~/browserHistoryStore";
2222import { type ComposerImageAttachment, useComposerDraftStore } from "~/composerDraftStore";
23import { previewAnnotationScreenshotFile } from "~/lib/previewAnnotation";
23import { capturePreviewAnnotationScreenshot } from "~/lib/previewAnnotation";
2424import { ensureLocalApi } from "~/localApi";
2525import {
2626 rememberPreviewUrl,
......@@ -579,15 +579,30 @@ export function PreviewView({
579579 try {
580580 const result = await previewBridge.pickElement(runtimeTabId);
581581 if (!result) return;
582 const { annotation, submission } = result;
582 const { annotation: picked, submission, screenshotFailed = false } = result;
583 // The structured annotation is still sendable when its optional crop
584 // stalls or fails, so tell the user what they lost and keep going
585 // instead of holding the composer for an attachment that never lands.
586 // The stored copy drops the screenshot on failure, otherwise the prompt
587 // would tell the agent a crop is attached when none was sent.
588 const capture = await capturePreviewAnnotationScreenshot(picked);
589 // Main reports a crop that failed or timed out on its side; the local
590 // conversion can fail too. Either way the user should hear about it.
591 const cropDropped = screenshotFailed || capture.status === "failed";
592 const annotation = capture.status === "failed" ? { ...picked, screenshot: null } : picked;
583593 addPreviewAnnotation(threadRef, annotation);
584 let screenshotFile: File | null = null;
585 try {
586 screenshotFile = await previewAnnotationScreenshotFile(annotation);
587 } catch {
588 // The structured annotation is still sendable when converting its
589 // optional screenshot into a composer attachment fails.
594 if (cropDropped) {
595 toastManager.add(
596 stackedThreadToast({
597 type: "error",
598 title: "Could not capture the picked element",
599 // The send path reports its own outcome, so only say what this
600 // handler knows: the crop was dropped.
601 description: "The annotation was kept without the screenshot.",
602 }),
603 );
590604 }
605 const screenshotFile = capture.status === "captured" ? capture.file : null;
591606 const image =
592607 screenshotFile && annotation.screenshot
593608 ? ({
apps/web/src/lib/previewAnnotation.test.ts+35-1
......@@ -1,9 +1,10 @@
11import type { PreviewAnnotationPayload } from "@t3tools/contracts";
2import { describe, expect, it } from "vite-plus/test";
2import { afterEach, describe, expect, it, vi } from "vite-plus/test";
33
44import {
55 appendPreviewAnnotationPrompt,
66 buildPreviewAnnotationPrompt,
7 capturePreviewAnnotationScreenshot,
78 extractTrailingPreviewAnnotation,
89} from "./previewAnnotation";
910
......@@ -85,3 +86,36 @@ describe("preview annotations", () => {
8586 expect(extractedFirst.promptText).toBe("Fix this");
8687 });
8788});
89
90describe("preview annotation capture", () => {
91 afterEach(() => {
92 vi.useRealTimers();
93 vi.unstubAllGlobals();
94 });
95
96 it("returns the crop when the fetch resolves", async () => {
97 vi.stubGlobal("fetch", async () => new Response(new Blob(["png"], { type: "image/png" })));
98 const capture = await capturePreviewAnnotationScreenshot(annotation);
99 expect(capture.status).toBe("captured");
100 });
101
102 it("reports none when the annotation carries no crop", async () => {
103 const capture = await capturePreviewAnnotationScreenshot({ ...annotation, screenshot: null });
104 expect(capture).toEqual({ status: "none" });
105 });
106
107 it("fails instead of hanging when the crop never arrives", async () => {
108 vi.useFakeTimers();
109 vi.stubGlobal("fetch", () => new Promise<Response>(() => {}));
110 const capturePromise = capturePreviewAnnotationScreenshot(annotation, 1_000);
111 await vi.advanceTimersByTimeAsync(1_000);
112 expect(await capturePromise).toEqual({ status: "failed" });
113 });
114
115 it("fails when the crop fetch throws", async () => {
116 vi.stubGlobal("fetch", async () => {
117 throw new Error("data url unreadable");
118 });
119 expect(await capturePreviewAnnotationScreenshot(annotation)).toEqual({ status: "failed" });
120 });
121});
apps/web/src/lib/previewAnnotation.ts+38-1
......@@ -99,7 +99,7 @@ export function extractTrailingPreviewAnnotation(prompt: string): ExtractedPrevi
9999 };
100100}
101101
102export async function previewAnnotationScreenshotFile(
102async function previewAnnotationScreenshotFile(
103103 annotation: PreviewAnnotationPayload,
104104): Promise<File | null> {
105105 if (!annotation.screenshot) return null;
......@@ -109,3 +109,40 @@ export async function previewAnnotationScreenshotFile(
109109 type: blob.type || "image/png",
110110 });
111111}
112
113/** Upper bound on turning a picked element's crop into a composer attachment. */
114export const PREVIEW_ANNOTATION_CAPTURE_TIMEOUT_MS = 5_000;
115
116export type PreviewAnnotationCapture =
117 /** The crop is ready to attach. */
118 | { readonly status: "captured"; readonly file: File }
119 /** The pick carried no crop, which is normal for comment-only annotations. */
120 | { readonly status: "none" }
121 /** The crop stalled or threw. Send the annotation without it. */
122 | { readonly status: "failed" };
123
124/**
125 * Bounded wrapper around `previewAnnotationScreenshotFile`. The picker holds the
126 * composer while this runs, so it must always settle: a stalled crop resolves as
127 * `failed` instead of leaving the caller waiting.
128 */
129export async function capturePreviewAnnotationScreenshot(
130 annotation: PreviewAnnotationPayload,
131 timeoutMs: number = PREVIEW_ANNOTATION_CAPTURE_TIMEOUT_MS,
132): Promise<PreviewAnnotationCapture> {
133 if (!annotation.screenshot) return { status: "none" };
134 let timer: ReturnType<typeof setTimeout> | undefined;
135 try {
136 const file = await Promise.race([
137 previewAnnotationScreenshotFile(annotation),
138 new Promise<null>((resolve) => {
139 timer = setTimeout(() => resolve(null), timeoutMs);
140 }),
141 ]);
142 return file ? { status: "captured", file } : { status: "failed" };
143 } catch {
144 return { status: "failed" };
145 } finally {
146 clearTimeout(timer);
147 }
148}
packages/contracts/src/ipc.ts+3
......@@ -934,11 +934,14 @@ export const PreviewAnnotationSubmissionSchema: Schema.Codec<PreviewAnnotationSu
934934export interface PreviewAnnotationSubmissionResult {
935935 annotation: PreviewAnnotationPayload;
936936 submission: PreviewAnnotationSubmission;
937 /** The crop was requested but failed or timed out, so `annotation.screenshot` is null. */
938 screenshotFailed?: boolean;
937939}
938940export const PreviewAnnotationSubmissionResultSchema: Schema.Codec<PreviewAnnotationSubmissionResult> =
939941 Schema.Struct({
940942 annotation: PreviewAnnotationPayloadSchema,
941943 submission: PreviewAnnotationSubmissionSchema,
944 screenshotFailed: Schema.optionalKey(Schema.Boolean),
942945 });
943946
944947export const DesktopPreviewTabInputSchema = Schema.Struct({