authorgravatar for 62337003+Bil0000@users.noreply.github.comBilal Bakr <62337003+Bil0000@users.noreply.github.com> 2026-09-12 10:18:24+03:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-09-12 00:18:24-07:00
log57b23a09f9dff15d57e3fa288c111381259a9a12
treeff125b6b21476a2783c858280cd6fc45662c0b57
parent4fed6cfb3517ef9af9b2a361142152c079504530
signature Signed by PGP key B5690EEEBB952194

fix(desktop): preserve long offscreen text in SnapShots (#11250)

4 files changed, 180 insertions(+), 13 deletions(-)

apps/desktop/src/snapShot/DesktopSnapShot.test.ts+138-2
...@@ -2484,7 +2484,7 @@ it.each(["client", "frame"] as const)(...@@ -2484,7 +2484,7 @@ it.each(["client", "frame"] as const)(
2484 },2484 },
2485);2485);
24862486
2487it.each(["darwin", "win32"] as const)(2487it.each(["darwin", "win32", "linux"] as const)(
2488 "extracts the same structured accessibility tree on %s",2488 "extracts the same structured accessibility tree on %s",
2489 async (platform) => {2489 async (platform) => {
2490 const bounds = { x: 100, y: 200, width: 800, height: 600 };2490 const bounds = { x: 100, y: 200, width: 800, height: 600 };
...@@ -2492,7 +2492,13 @@ it.each(["darwin", "win32"] as const)(...@@ -2492,7 +2492,13 @@ it.each(["darwin", "win32"] as const)(
2492 role: "window",2492 role: "window",
2493 name: "Editor",2493 name: "Editor",
2494 bounds,2494 bounds,
2495 tree: async () => ({ name: "Editor", children: [{ name: "Save", children: [] }] }),2495 tree: async () => ({
2496 name: "Editor",
2497 children: [
2498 { name: "Save", children: [] },
2499 { name: "Below scroll view", children: [] },
2500 ],
2501 }),
2496 children: async () => [2502 children: async () => [
2497 {2503 {
2498 role: "button",2504 role: "button",
...@@ -2500,6 +2506,13 @@ it.each(["darwin", "win32"] as const)(...@@ -2500,6 +2506,13 @@ it.each(["darwin", "win32"] as const)(
2500 bounds: { x: 300, y: 350, width: 100, height: 50 },2506 bounds: { x: 300, y: 350, width: 100, height: 50 },
2501 children: async () => [],2507 children: async () => [],
2502 },2508 },
2509 {
2510 role: "static_text",
2511 name: "Below scroll view",
2512 bounds: { x: 300, y: 1_000, width: 100, height: 50 },
2513 visible: false,
2514 children: async () => [],
2515 },
2503 ],2516 ],
2504 };2517 };
2505 accessibilityByPidMock.mockReset().mockResolvedValue({ children: async () => [window] });2518 accessibilityByPidMock.mockReset().mockResolvedValue({ children: async () => [window] });
...@@ -2521,6 +2534,12 @@ it.each(["darwin", "win32"] as const)(...@@ -2521,6 +2534,12 @@ it.each(["darwin", "win32"] as const)(
2521 : undefined,2534 : undefined,
2522 "Save",2535 "Save",
2523 );2536 );
2537 assert.deepInclude(
2538 result?.accessibility?.format === "element-tree"
2539 ? result.accessibility.root.children[1]
2540 : undefined,
2541 { name: "Below scroll view", bounds: null, state: { visible: false } },
2542 );
2524 assert.lengthOf(accessibilityByPidMock.mock.calls, platform === "win32" ? 0 : 1);2543 assert.lengthOf(accessibilityByPidMock.mock.calls, platform === "win32" ? 0 : 1);
2525 assert.lengthOf(accessibilityForegroundMock.mock.calls, platform === "win32" ? 1 : 0);2544 assert.lengthOf(accessibilityForegroundMock.mock.calls, platform === "win32" ? 1 : 0);
2526 },2545 },
...@@ -2559,6 +2578,48 @@ it.each(["darwin", "win32"] as const)(...@@ -2559,6 +2578,48 @@ it.each(["darwin", "win32"] as const)(
2559 },2578 },
2560);2579);
25612580
2581it.each([
2582 ["darwin", "", "value", 650],
2583 ["win32", "", "value", 650],
2584 ["linux", "x11", "value", 650],
2585 ["linux", "wayland", "value", 650],
2586 ["darwin", "", "name", 100],
2587] as const)(
2588 "preserves long text outside the scroll view on %s %s (%s)",
2589 async (platform, session, field, lines) => {
2590 const text = `${"Document line\n".repeat(lines)}End of document`;
2591 vi.stubEnv("XDG_SESSION_TYPE", session);
2592 const bounds = { x: 0, y: 0, width: 800, height: 600 };
2593 const window = {
2594 role: "window",
2595 name: "Editor",
2596 bounds,
2597 tree: async () => ({ name: "Editor", children: [{ [field]: text, children: [] }] }),
2598 children: async () => [
2599 { role: "text_area", [field]: text, bounds, children: async () => [] },
2600 ],
2601 };
2602 accessibilityByPidMock.mockReset().mockResolvedValue({ children: async () => [window] });
2603 accessibilityForegroundMock
2604 .mockReset()
2605 .mockResolvedValue({ pid: 123, asElement: () => window });
2606 try {
2607 const result = await readAccessibleWindowContext(
2608 { title: "Editor", bounds, owner: { processId: 123 } },
2609 platform,
2610 "Editor",
2611 );
2612 assert.deepEqual(result?.accessibility, {
2613 format: "flat-text",
2614 text: `Editor\n${text}`,
2615 truncated: false,
2616 });
2617 } finally {
2618 vi.unstubAllEnvs();
2619 }
2620 },
2621);
2622
2562it.each([2623it.each([
2563 { names: ["⠙ t3code"], expected: "Verified text" },2624 { names: ["⠙ t3code"], expected: "Verified text" },
2564 { names: ["⠋ t3code", "⠙ t3code"], expected: undefined },2625 { names: ["⠋ t3code", "⠙ t3code"], expected: undefined },
...@@ -2686,6 +2747,81 @@ it("falls back to completed flat text when rich traversal reaches the deadline",...@@ -2686,6 +2747,81 @@ it("falls back to completed flat text when rich traversal reaches the deadline",
2686 }2747 }
2687});2748});
26882749
2750it("keeps a truncated element tree when the flat text read fails", async () => {
2751 const bounds = { x: 0, y: 0, width: 800, height: 600 };
2752 accessibilityByPidMock.mockReset().mockResolvedValue({
2753 children: async () => [
2754 {
2755 role: "window",
2756 name: "Editor",
2757 bounds,
2758 tree: async () => {
2759 throw new Error("Text read failed");
2760 },
2761 children: async () => [
2762 {
2763 role: "text_area",
2764 value: "x".repeat(8_001),
2765 bounds: { x: 10, y: 50, width: 700, height: 500 },
2766 children: async () => [],
2767 },
2768 ],
2769 },
2770 ],
2771 });
2772
2773 const result = await readAccessibleWindowContext(
2774 { title: "Editor", bounds, owner: { processId: 123 } },
2775 "darwin",
2776 "Editor",
2777 );
2778 assert.equal(result?.accessibility?.format, "element-tree");
2779 assert.isTrue(result?.accessibility?.truncated);
2780 assert.deepEqual(
2781 result?.accessibility?.format === "element-tree"
2782 ? result.accessibility.root.children[0]?.bounds
2783 : undefined,
2784 { x: 10, y: 50, width: 700, height: 500 },
2785 );
2786});
2787
2788it("keeps a truncated tree when flat text would not recover any text", async () => {
2789 const bounds = { x: 0, y: 0, width: 800, height: 600 };
2790 accessibilityByPidMock.mockReset().mockResolvedValue({
2791 children: async () => [
2792 {
2793 role: "window",
2794 name: "Editor",
2795 bounds,
2796 tree: async () => ({ name: "Editor", children: [{ name: "Help", children: [] }] }),
2797 children: async () => [
2798 {
2799 role: "button",
2800 name: "Help",
2801 description: "Help text ".repeat(250),
2802 bounds,
2803 children: async () => [],
2804 },
2805 ],
2806 },
2807 ],
2808 });
2809
2810 const result = await readAccessibleWindowContext(
2811 { title: "Editor", bounds, owner: { processId: 123 } },
2812 "darwin",
2813 "Editor",
2814 );
2815 assert.equal(result?.accessibility?.format, "element-tree");
2816 assert.isTrue(result?.accessibility?.truncated);
2817 assert.include(
2818 result?.accessibility?.format === "element-tree"
2819 ? result.accessibility.root.children[0]?.description
2820 : undefined,
2821 "Help text",
2822 );
2823});
2824
2689it("times out after three seconds without overlapping the outstanding accessibility read", async () => {2825it("times out after three seconds without overlapping the outstanding accessibility read", async () => {
2690 vi.useFakeTimers();2826 vi.useFakeTimers();
2691 accessibilityByPidMock.mockReset();2827 accessibilityByPidMock.mockReset();
apps/desktop/src/snapShot/SnapShotAccessibility.ts+12-10
...@@ -57,25 +57,27 @@ function accessibilityReadSnapshot(...@@ -57,25 +57,27 @@ function accessibilityReadSnapshot(
57 descendantLocationsReliable: progress.richLocationsReliable,57 descendantLocationsReliable: progress.richLocationsReliable,
58 })58 })
59 : undefined;59 : undefined;
60 const accessibleText =60 const richText = richTree
61 progress.accessibleText ??61 ? accessibleWindowText(richTree.root, SNAP_SHOT_ACCESSIBLE_TEXT_MAX_CHARS)
62 (richTree62 : undefined;
63 ? accessibleWindowText(richTree.root, SNAP_SHOT_ACCESSIBLE_TEXT_MAX_CHARS)63 const accessibleText = progress.accessibleText ?? richText;
64 : undefined);64 const richTruncated = progress.richTruncated || richTree?.truncated === true;
65 const accessibility: SnapShotAccessibility | undefined =65 const accessibility: SnapShotAccessibility | undefined =
66 progress.richComplete && richTree66 progress.richComplete &&
67 richTree &&
68 (!richTruncated || !progress.accessibleText || progress.accessibleText === richText)
67 ? {69 ? {
68 format: "element-tree",70 format: "element-tree",
69 coordinateSpace: "captured-image",71 coordinateSpace: "captured-image",
70 imageSize,72 imageSize,
71 truncated: progress.richTruncated || richTree.truncated,73 truncated: richTruncated,
72 root: richTree.root,74 root: richTree.root,
73 }75 }
74 : progress.flatComplete && accessibleText76 : progress.flatComplete && progress.accessibleText
75 ? {77 ? {
76 format: "flat-text",78 format: "flat-text",
77 text: accessibleText,79 text: progress.accessibleText,
78 truncated: accessibleText.length >= SNAP_SHOT_ACCESSIBLE_TEXT_MAX_CHARS,80 truncated: progress.accessibleText.length >= SNAP_SHOT_ACCESSIBLE_TEXT_MAX_CHARS,
79 }81 }
80 : richTree82 : richTree
81 ? {83 ? {
apps/desktop/src/snapShot/snapShot.test.ts+17
...@@ -263,6 +263,23 @@ describe("accessibleWindowElementTree", () => {...@@ -263,6 +263,23 @@ describe("accessibleWindowElementTree", () => {
263 expect(tree?.root.children.length).toBeLessThan(10);263 expect(tree?.root.children.length).toBeLessThan(10);
264 expect(JSON.stringify(tree).length).toBeLessThanOrEqual(32_000);264 expect(JSON.stringify(tree).length).toBeLessThanOrEqual(32_000);
265 });265 });
266
267 it.each(["name", "value", "description"] as const)(
268 "reports clipped %s text in partial trees",
269 async (field) => {
270 const tree = await accessibleWindowElementTree(
271 {
272 role: "window",
273 children: async () => [{ role: "text_area", [field]: "x".repeat(8_001) }],
274 },
275 { x: 0, y: 0, width: 800, height: 600 },
276 { width: 800, height: 600 },
277 );
278
279 expect(tree?.truncated).toBe(true);
280 expect(tree?.root.children[0]?.[field]?.length).toBeLessThan(8_001);
281 },
282 );
266});283});
267284
268describe("compactAccessibilityTree", () => {285describe("compactAccessibilityTree", () => {
apps/desktop/src/snapShot/snapShot.ts+13-1
...@@ -155,11 +155,16 @@ function safeProperty<T>(read: () => T): T | undefined {...@@ -155,11 +155,16 @@ function safeProperty<T>(read: () => T): T | undefined {
155 }155 }
156}156}
157157
158export function boundedSnapShotString(value: unknown, maxChars: number): string | undefined {158export function boundedSnapShotString(
159 value: unknown,
160 maxChars: number,
161 onTruncated?: () => void,
162): string | undefined {
159 if (typeof value !== "string") return undefined;163 if (typeof value !== "string") return undefined;
160 const candidate = value.replaceAll("\0", "").trim();164 const candidate = value.replaceAll("\0", "").trim();
161 if (!candidate) return undefined;165 if (!candidate) return undefined;
162 if (candidate.length <= maxChars) return candidate;166 if (candidate.length <= maxChars) return candidate;
167 onTruncated?.();
163 const end = /[\uD800-\uDBFF]/.test(candidate[maxChars - 1] ?? "") ? maxChars - 1 : maxChars;168 const end = /[\uD800-\uDBFF]/.test(candidate[maxChars - 1] ?? "") ? maxChars - 1 : maxChars;
164 return candidate.slice(0, end).trimEnd();169 return candidate.slice(0, end).trimEnd();
165}170}
...@@ -204,18 +209,22 @@ function accessibilityNode(...@@ -204,18 +209,22 @@ function accessibilityNode(
204 imageSize: CapturedImageSize,209 imageSize: CapturedImageSize,
205 isRoot: boolean,210 isRoot: boolean,
206 locationsReliable: boolean,211 locationsReliable: boolean,
212 onTruncated: () => void,
207): MutableAccessibilityNode {213): MutableAccessibilityNode {
208 const name = boundedSnapShotString(214 const name = boundedSnapShotString(
209 safeProperty(() => element.name),215 safeProperty(() => element.name),
210 1_000,216 1_000,
217 onTruncated,
211 );218 );
212 const value = boundedSnapShotString(219 const value = boundedSnapShotString(
213 safeProperty(() => element.value),220 safeProperty(() => element.value),
214 8_000,221 8_000,
222 onTruncated,
215 );223 );
216 const description = boundedSnapShotString(224 const description = boundedSnapShotString(
217 safeProperty(() => element.description),225 safeProperty(() => element.description),
218 2_000,226 2_000,
227 onTruncated,
219 );228 );
220 const checked = safeProperty(() => element.checked);229 const checked = safeProperty(() => element.checked);
221 const expanded = safeProperty(() => element.expanded);230 const expanded = safeProperty(() => element.expanded);
...@@ -389,6 +398,9 @@ export async function accessibleWindowElementTree(...@@ -389,6 +398,9 @@ export async function accessibleWindowElementTree(
389 imageSize,398 imageSize,
390 required,399 required,
391 options.locationsReliable !== false,400 options.locationsReliable !== false,
401 () => {
402 truncated = true;
403 },
392 );404 );
393 nodes += 1;405 nodes += 1;
394 root ??= node;406 root ??= node;