authorgravatar for yashafaid@gmail.comCestercian <yashafaid@gmail.com> 2026-09-20 03:38:53+05:30
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-09-19 15:08:53-07:00
logdcf8942304cfef382af55c0068cb2f04c86757fc
treeb920978d181fdc59df0368f473ade848081f2efe
parentf9e8f578fa0fb1ea25e458334a28bbb4fe22302d
signature Signed by PGP key B5690EEEBB952194

fix(web): keep desktop annotation screenshots under CSP (#12636)

4 files changed, 44 insertions(+), 89 deletions(-)

apps/web/src/components/preview/PreviewView.test.tsx+2-10
...@@ -27,7 +27,6 @@ const mocks = vi.hoisted(() => ({...@@ -27,7 +27,6 @@ const mocks = vi.hoisted(() => ({
27 openPictureInPicture: vi.fn(async (_tabId: string): Promise<void> => undefined),27 openPictureInPicture: vi.fn(async (_tabId: string): Promise<void> => undefined),
28 closePictureInPicture: vi.fn(async (_tabId: string): Promise<void> => undefined),28 closePictureInPicture: vi.fn(async (_tabId: string): Promise<void> => undefined),
29 pickElement: vi.fn(),29 pickElement: vi.fn(),
30 capturePreviewAnnotationScreenshot: vi.fn(),
31 addPreviewAnnotation: vi.fn(),30 addPreviewAnnotation: vi.fn(),
32 addImage: vi.fn(),31 addImage: vi.fn(),
33 toggleAnnotation: null as (() => void) | null,32 toggleAnnotation: null as (() => void) | null,
...@@ -90,10 +89,6 @@ vi.mock("~/composerDraftStore", () => ({...@@ -90,10 +89,6 @@ vi.mock("~/composerDraftStore", () => ({
90 }),89 }),
91}));90}));
9291
93vi.mock("~/lib/previewAnnotation", () => ({
94 capturePreviewAnnotationScreenshot: mocks.capturePreviewAnnotationScreenshot,
95}));
96
97vi.mock("~/localApi", () => ({92vi.mock("~/localApi", () => ({
98 ensureLocalApi: vi.fn(),93 ensureLocalApi: vi.fn(),
99}));94}));
...@@ -342,8 +337,6 @@ describe("PreviewView navigation", () => {...@@ -342,8 +337,6 @@ describe("PreviewView navigation", () => {
342 mocks.openPictureInPicture.mockClear();337 mocks.openPictureInPicture.mockClear();
343 mocks.closePictureInPicture.mockClear();338 mocks.closePictureInPicture.mockClear();
344 mocks.pickElement.mockReset();339 mocks.pickElement.mockReset();
345 mocks.capturePreviewAnnotationScreenshot.mockReset();
346 mocks.capturePreviewAnnotationScreenshot.mockResolvedValue({ status: "none" });
347 mocks.addPreviewAnnotation.mockClear();340 mocks.addPreviewAnnotation.mockClear();
348 vi.mocked(toastManager.add).mockClear();341 vi.mocked(toastManager.add).mockClear();
349 mocks.addImage.mockClear();342 mocks.addImage.mockClear();
...@@ -585,7 +578,7 @@ describe("PreviewView navigation", () => {...@@ -585,7 +578,7 @@ describe("PreviewView navigation", () => {
585 expect(toastManager.add).toHaveBeenCalledTimes(1);578 expect(toastManager.add).toHaveBeenCalledTimes(1);
586 });579 });
587580
588 it("still sends when the picked element's crop cannot be captured", async () => {581 it("still sends annotation text when the picked element's crop is malformed", async () => {
589 const annotation = {582 const annotation = {
590 id: "annotation-2",583 id: "annotation-2",
591 pageUrl: "https://example.com/dashboard",584 pageUrl: "https://example.com/dashboard",
...@@ -596,7 +589,7 @@ describe("PreviewView navigation", () => {...@@ -596,7 +589,7 @@ describe("PreviewView navigation", () => {
596 strokes: [],589 strokes: [],
597 styleChanges: [],590 styleChanges: [],
598 screenshot: {591 screenshot: {
599 dataUrl: "data:image/png;base64,c2NyZWVuc2hvdA==",592 dataUrl: "data:image/png;base64,%%%",
600 width: 10,593 width: 10,
601 height: 10,594 height: 10,
602 cropRect: { x: 0, y: 0, width: 10, height: 10 },595 cropRect: { x: 0, y: 0, width: 10, height: 10 },
...@@ -605,7 +598,6 @@ describe("PreviewView navigation", () => {...@@ -605,7 +598,6 @@ describe("PreviewView navigation", () => {
605 };598 };
606 const onSendAnnotation = vi.fn();599 const onSendAnnotation = vi.fn();
607 mocks.pickElement.mockResolvedValue({ annotation, submission: "send" });600 mocks.pickElement.mockResolvedValue({ annotation, submission: "send" });
608 mocks.capturePreviewAnnotationScreenshot.mockResolvedValue({ status: "failed" });
609601
610 renderToStaticMarkup(602 renderToStaticMarkup(
611 <PreviewView603 <PreviewView
apps/web/src/components/preview/PreviewView.tsx+1-1
...@@ -603,7 +603,7 @@ export function PreviewView({...@@ -603,7 +603,7 @@ export function PreviewView({
603 // instead of holding the composer for an attachment that never lands.603 // instead of holding the composer for an attachment that never lands.
604 // The stored copy drops the screenshot on failure, otherwise the prompt604 // The stored copy drops the screenshot on failure, otherwise the prompt
605 // would tell the agent a crop is attached when none was sent.605 // would tell the agent a crop is attached when none was sent.
606 const capture = await capturePreviewAnnotationScreenshot(picked);606 const capture = capturePreviewAnnotationScreenshot(picked);
607 // Main reports a crop that failed or timed out on its side; the local607 // Main reports a crop that failed or timed out on its side; the local
608 // conversion can fail too. Either way the user should hear about it.608 // conversion can fail too. Either way the user should hear about it.
609 const cropDropped = screenshotFailed || capture.status === "failed";609 const cropDropped = screenshotFailed || capture.status === "failed";
apps/web/src/lib/previewAnnotation.test.ts+28-46
...@@ -9,66 +9,48 @@ const annotation: PreviewAnnotationPayload = {...@@ -9,66 +9,48 @@ const annotation: PreviewAnnotationPayload = {
9 pageTitle: "Example",9 pageTitle: "Example",
10 comment: "Make these cards feel related.",10 comment: "Make these cards feel related.",
11 elements: [],11 elements: [],
12 regions: [{ id: "region_1", rect: { x: 10, y: 20, width: 100, height: 80 } }],12 regions: [],
13 strokes: [13 strokes: [],
14 {14 styleChanges: [],
15 id: "stroke_1",
16 color: "#7c3aed",
17 width: 4,
18 points: [
19 { x: 10, y: 10 },
20 { x: 20, y: 20 },
21 ],
22 bounds: { x: 6, y: 6, width: 18, height: 18 },
23 },
24 ],
25 styleChanges: [
26 {
27 targetId: "element_1",
28 selector: ".card",
29 property: "border-radius",
30 previousValue: "4px",
31 value: "16px",
32 },
33 ],
34 screenshot: {15 screenshot: {
35 dataUrl: "data:image/png;base64,AA==",16 dataUrl:
36 width: 100,17 "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+ip1sAAAAASUVORK5CYII=",
37 height: 80,18 width: 1,
38 cropRect: { x: 10, y: 20, width: 100, height: 80 },19 height: 1,
20 cropRect: { x: 0, y: 0, width: 1, height: 1 },
39 },21 },
40 createdAt: "2026-06-11T00:00:00.000Z",22 createdAt: "2026-06-11T00:00:00.000Z",
41};23};
4224
43describe("preview annotation capture", () => {25describe("preview annotation capture", () => {
44 afterEach(() => {26 afterEach(() => {
45 vi.useRealTimers();
46 vi.unstubAllGlobals();27 vi.unstubAllGlobals();
47 });28 });
4829
49 it("returns the crop when the fetch resolves", async () => {30 it("preserves the PNG bytes, MIME type, and filename without fetching", async () => {
50 vi.stubGlobal("fetch", async () => new Response(new Blob(["png"], { type: "image/png" })));31 const fetch = vi.fn().mockRejectedValue(new TypeError("Blocked by connect-src"));
51 const capture = await capturePreviewAnnotationScreenshot(annotation);32 vi.stubGlobal("fetch", fetch);
33 const capture = capturePreviewAnnotationScreenshot(annotation);
52 expect(capture.status).toBe("captured");34 expect(capture.status).toBe("captured");
35 if (capture.status !== "captured") throw new Error("Expected a screenshot file");
36 expect(capture.file.name).toBe("preview-annotation-annotation_1.png");
37 expect(capture.file.type).toBe("image/png");
38 expect(Buffer.from(await capture.file.arrayBuffer()).toString("hex")).toBe(
39 "89504e470d0a1a0a0000000d4948445200000001000000010804000000b51c0c020000000b4944415478da63fcff1f0003030200efa2a75b0000000049454e44ae426082",
40 );
41 expect(fetch).not.toHaveBeenCalled();
53 });42 });
5443
55 it("reports none when the annotation carries no crop", async () => {44 it("reports none when the annotation carries no crop", () => {
56 const capture = await capturePreviewAnnotationScreenshot({ ...annotation, screenshot: null });45 const capture = capturePreviewAnnotationScreenshot({ ...annotation, screenshot: null });
57 expect(capture).toEqual({ status: "none" });46 expect(capture).toEqual({ status: "none" });
58 });47 });
5948
60 it("fails instead of hanging when the crop never arrives", async () => {49 it.each(["data:image/jpeg;base64,AA==", "data:image/png;base64,", "data:image/png;base64,%%%"])(
61 vi.useFakeTimers();50 "reports a malformed screenshot as failed: %s",
62 vi.stubGlobal("fetch", () => new Promise<Response>(() => {}));51 (dataUrl) => {
63 const capturePromise = capturePreviewAnnotationScreenshot(annotation, 1_000);52 const picked = { ...annotation, screenshot: { ...annotation.screenshot!, dataUrl } };
64 await vi.advanceTimersByTimeAsync(1_000);53 expect(capturePreviewAnnotationScreenshot(picked)).toEqual({ status: "failed" });
65 expect(await capturePromise).toEqual({ status: "failed" });54 },
66 });55 );
67
68 it("fails when the crop fetch throws", async () => {
69 vi.stubGlobal("fetch", async () => {
70 throw new Error("data url unreadable");
71 });
72 expect(await capturePreviewAnnotationScreenshot(annotation)).toEqual({ status: "failed" });
73 });
74});56});
apps/web/src/lib/previewAnnotation.ts+13-32
...@@ -1,49 +1,30 @@...@@ -1,49 +1,30 @@
1import type { PreviewAnnotationPayload } from "@t3tools/contracts";1import type { PreviewAnnotationPayload } from "@t3tools/contracts";
22
3async function previewAnnotationScreenshotFile(3import { dataUrlToFile } from "./imageCompression";
4 annotation: PreviewAnnotationPayload,
5): Promise<File | null> {
6 if (!annotation.screenshot) return null;
7 const response = await fetch(annotation.screenshot.dataUrl);
8 const blob = await response.blob();
9 return new File([blob], `preview-annotation-${annotation.id}.png`, {
10 type: blob.type || "image/png",
11 });
12}
13
14/** Upper bound on turning a picked element's crop into a composer attachment. */
15const PREVIEW_ANNOTATION_CAPTURE_TIMEOUT_MS = 5_000;
164
17export type PreviewAnnotationCapture =5export type PreviewAnnotationCapture =
18 /** The crop is ready to attach. */6 /** The crop is ready to attach. */
19 | { readonly status: "captured"; readonly file: File }7 | { readonly status: "captured"; readonly file: File }
20 /** The pick carried no crop, which is normal for comment-only annotations. */8 /** The pick carried no crop, which is normal for comment-only annotations. */
21 | { readonly status: "none" }9 | { readonly status: "none" }
22 /** The crop stalled or threw. Send the annotation without it. */10 /** The crop could not be decoded. Send the annotation without it. */
23 | { readonly status: "failed" };11 | { readonly status: "failed" };
2412
25/**13const PNG_DATA_URL_PREFIX = "data:image/png;base64,";
26 * Bounded wrapper around `previewAnnotationScreenshotFile`. The picker holds the14
27 * composer while this runs, so it must always settle: a stalled crop resolves as15/** Decode Electron's PNG crop locally; fetching a data URL violates desktop connect-src. */
28 * `failed` instead of leaving the caller waiting.16export function capturePreviewAnnotationScreenshot(
29 */
30export async function capturePreviewAnnotationScreenshot(
31 annotation: PreviewAnnotationPayload,17 annotation: PreviewAnnotationPayload,
32 timeoutMs: number = PREVIEW_ANNOTATION_CAPTURE_TIMEOUT_MS,18): PreviewAnnotationCapture {
33): Promise<PreviewAnnotationCapture> {
34 if (!annotation.screenshot) return { status: "none" };19 if (!annotation.screenshot) return { status: "none" };
35 let timer: ReturnType<typeof setTimeout> | undefined;
36 try {20 try {
37 const file = await Promise.race([21 const { dataUrl } = annotation.screenshot;
38 previewAnnotationScreenshotFile(annotation),22 if (!dataUrl.startsWith(PNG_DATA_URL_PREFIX)) {
39 new Promise<null>((resolve) => {23 return { status: "failed" };
40 timer = setTimeout(() => resolve(null), timeoutMs);24 }
41 }),25 const file = dataUrlToFile(dataUrl, `preview-annotation-${annotation.id}.png`, "image/png");
42 ]);26 return file.size > 0 ? { status: "captured", file } : { status: "failed" };
43 return file ? { status: "captured", file } : { status: "failed" };
44 } catch {27 } catch {
45 return { status: "failed" };28 return { status: "failed" };
46 } finally {
47 clearTimeout(timer);
48 }29 }
49}30}