authorgravatar for 62337003+Bil0000@users.noreply.github.comBilal Bakr <62337003+Bil0000@users.noreply.github.com> 2026-09-12 10:18:24+03:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-09-12 00:18:24-07:00
log57b23a09f9dff15d57e3fa288c111381259a9a12
treeff125b6b21476a2783c858280cd6fc45662c0b57
parent4fed6cfb3517ef9af9b2a361142152c079504530
signature Signed by PGP key B5690EEEBB952194

fix(desktop): preserve long offscreen text in SnapShots (#11250)

4 files changed, 180 insertions(+), 13 deletions(-)

apps/desktop/src/snapShot/DesktopSnapShot.test.ts+138-2
......@@ -2484,7 +2484,7 @@ it.each(["client", "frame"] as const)(
24842484 },
24852485);
24862486
2487it.each(["darwin", "win32"] as const)(
2487it.each(["darwin", "win32", "linux"] as const)(
24882488 "extracts the same structured accessibility tree on %s",
24892489 async (platform) => {
24902490 const bounds = { x: 100, y: 200, width: 800, height: 600 };
......@@ -2492,7 +2492,13 @@ it.each(["darwin", "win32"] as const)(
24922492 role: "window",
24932493 name: "Editor",
24942494 bounds,
2495 tree: async () => ({ name: "Editor", children: [{ name: "Save", children: [] }] }),
2495 tree: async () => ({
2496 name: "Editor",
2497 children: [
2498 { name: "Save", children: [] },
2499 { name: "Below scroll view", children: [] },
2500 ],
2501 }),
24962502 children: async () => [
24972503 {
24982504 role: "button",
......@@ -2500,6 +2506,13 @@ it.each(["darwin", "win32"] as const)(
25002506 bounds: { x: 300, y: 350, width: 100, height: 50 },
25012507 children: async () => [],
25022508 },
2509 {
2510 role: "static_text",
2511 name: "Below scroll view",
2512 bounds: { x: 300, y: 1_000, width: 100, height: 50 },
2513 visible: false,
2514 children: async () => [],
2515 },
25032516 ],
25042517 };
25052518 accessibilityByPidMock.mockReset().mockResolvedValue({ children: async () => [window] });
......@@ -2521,6 +2534,12 @@ it.each(["darwin", "win32"] as const)(
25212534 : undefined,
25222535 "Save",
25232536 );
2537 assert.deepInclude(
2538 result?.accessibility?.format === "element-tree"
2539 ? result.accessibility.root.children[1]
2540 : undefined,
2541 { name: "Below scroll view", bounds: null, state: { visible: false } },
2542 );
25242543 assert.lengthOf(accessibilityByPidMock.mock.calls, platform === "win32" ? 0 : 1);
25252544 assert.lengthOf(accessibilityForegroundMock.mock.calls, platform === "win32" ? 1 : 0);
25262545 },
......@@ -2559,6 +2578,48 @@ it.each(["darwin", "win32"] as const)(
25592578 },
25602579);
25612580
2581it.each([
2582 ["darwin", "", "value", 650],
2583 ["win32", "", "value", 650],
2584 ["linux", "x11", "value", 650],
2585 ["linux", "wayland", "value", 650],
2586 ["darwin", "", "name", 100],
2587] as const)(
2588 "preserves long text outside the scroll view on %s %s (%s)",
2589 async (platform, session, field, lines) => {
2590 const text = `${"Document line\n".repeat(lines)}End of document`;
2591 vi.stubEnv("XDG_SESSION_TYPE", session);
2592 const bounds = { x: 0, y: 0, width: 800, height: 600 };
2593 const window = {
2594 role: "window",
2595 name: "Editor",
2596 bounds,
2597 tree: async () => ({ name: "Editor", children: [{ [field]: text, children: [] }] }),
2598 children: async () => [
2599 { role: "text_area", [field]: text, bounds, children: async () => [] },
2600 ],
2601 };
2602 accessibilityByPidMock.mockReset().mockResolvedValue({ children: async () => [window] });
2603 accessibilityForegroundMock
2604 .mockReset()
2605 .mockResolvedValue({ pid: 123, asElement: () => window });
2606 try {
2607 const result = await readAccessibleWindowContext(
2608 { title: "Editor", bounds, owner: { processId: 123 } },
2609 platform,
2610 "Editor",
2611 );
2612 assert.deepEqual(result?.accessibility, {
2613 format: "flat-text",
2614 text: `Editor\n${text}`,
2615 truncated: false,
2616 });
2617 } finally {
2618 vi.unstubAllEnvs();
2619 }
2620 },
2621);
2622
25622623it.each([
25632624 { names: ["⠙ t3code"], expected: "Verified text" },
25642625 { names: ["⠋ t3code", "⠙ t3code"], expected: undefined },
......@@ -2686,6 +2747,81 @@ it("falls back to completed flat text when rich traversal reaches the deadline",
26862747 }
26872748});
26882749
2750it("keeps a truncated element tree when the flat text read fails", async () => {
2751 const bounds = { x: 0, y: 0, width: 800, height: 600 };
2752 accessibilityByPidMock.mockReset().mockResolvedValue({
2753 children: async () => [
2754 {
2755 role: "window",
2756 name: "Editor",
2757 bounds,
2758 tree: async () => {
2759 throw new Error("Text read failed");
2760 },
2761 children: async () => [
2762 {
2763 role: "text_area",
2764 value: "x".repeat(8_001),
2765 bounds: { x: 10, y: 50, width: 700, height: 500 },
2766 children: async () => [],
2767 },
2768 ],
2769 },
2770 ],
2771 });
2772
2773 const result = await readAccessibleWindowContext(
2774 { title: "Editor", bounds, owner: { processId: 123 } },
2775 "darwin",
2776 "Editor",
2777 );
2778 assert.equal(result?.accessibility?.format, "element-tree");
2779 assert.isTrue(result?.accessibility?.truncated);
2780 assert.deepEqual(
2781 result?.accessibility?.format === "element-tree"
2782 ? result.accessibility.root.children[0]?.bounds
2783 : undefined,
2784 { x: 10, y: 50, width: 700, height: 500 },
2785 );
2786});
2787
2788it("keeps a truncated tree when flat text would not recover any text", async () => {
2789 const bounds = { x: 0, y: 0, width: 800, height: 600 };
2790 accessibilityByPidMock.mockReset().mockResolvedValue({
2791 children: async () => [
2792 {
2793 role: "window",
2794 name: "Editor",
2795 bounds,
2796 tree: async () => ({ name: "Editor", children: [{ name: "Help", children: [] }] }),
2797 children: async () => [
2798 {
2799 role: "button",
2800 name: "Help",
2801 description: "Help text ".repeat(250),
2802 bounds,
2803 children: async () => [],
2804 },
2805 ],
2806 },
2807 ],
2808 });
2809
2810 const result = await readAccessibleWindowContext(
2811 { title: "Editor", bounds, owner: { processId: 123 } },
2812 "darwin",
2813 "Editor",
2814 );
2815 assert.equal(result?.accessibility?.format, "element-tree");
2816 assert.isTrue(result?.accessibility?.truncated);
2817 assert.include(
2818 result?.accessibility?.format === "element-tree"
2819 ? result.accessibility.root.children[0]?.description
2820 : undefined,
2821 "Help text",
2822 );
2823});
2824
26892825it("times out after three seconds without overlapping the outstanding accessibility read", async () => {
26902826 vi.useFakeTimers();
26912827 accessibilityByPidMock.mockReset();
apps/desktop/src/snapShot/SnapShotAccessibility.ts+12-10
......@@ -57,25 +57,27 @@ function accessibilityReadSnapshot(
5757 descendantLocationsReliable: progress.richLocationsReliable,
5858 })
5959 : undefined;
60 const accessibleText =
61 progress.accessibleText ??
62 (richTree
63 ? accessibleWindowText(richTree.root, SNAP_SHOT_ACCESSIBLE_TEXT_MAX_CHARS)
64 : undefined);
60 const richText = richTree
61 ? accessibleWindowText(richTree.root, SNAP_SHOT_ACCESSIBLE_TEXT_MAX_CHARS)
62 : undefined;
63 const accessibleText = progress.accessibleText ?? richText;
64 const richTruncated = progress.richTruncated || richTree?.truncated === true;
6565 const accessibility: SnapShotAccessibility | undefined =
66 progress.richComplete && richTree
66 progress.richComplete &&
67 richTree &&
68 (!richTruncated || !progress.accessibleText || progress.accessibleText === richText)
6769 ? {
6870 format: "element-tree",
6971 coordinateSpace: "captured-image",
7072 imageSize,
71 truncated: progress.richTruncated || richTree.truncated,
73 truncated: richTruncated,
7274 root: richTree.root,
7375 }
74 : progress.flatComplete && accessibleText
76 : progress.flatComplete && progress.accessibleText
7577 ? {
7678 format: "flat-text",
77 text: accessibleText,
78 truncated: accessibleText.length >= SNAP_SHOT_ACCESSIBLE_TEXT_MAX_CHARS,
79 text: progress.accessibleText,
80 truncated: progress.accessibleText.length >= SNAP_SHOT_ACCESSIBLE_TEXT_MAX_CHARS,
7981 }
8082 : richTree
8183 ? {
apps/desktop/src/snapShot/snapShot.test.ts+17
......@@ -263,6 +263,23 @@ describe("accessibleWindowElementTree", () => {
263263 expect(tree?.root.children.length).toBeLessThan(10);
264264 expect(JSON.stringify(tree).length).toBeLessThanOrEqual(32_000);
265265 });
266
267 it.each(["name", "value", "description"] as const)(
268 "reports clipped %s text in partial trees",
269 async (field) => {
270 const tree = await accessibleWindowElementTree(
271 {
272 role: "window",
273 children: async () => [{ role: "text_area", [field]: "x".repeat(8_001) }],
274 },
275 { x: 0, y: 0, width: 800, height: 600 },
276 { width: 800, height: 600 },
277 );
278
279 expect(tree?.truncated).toBe(true);
280 expect(tree?.root.children[0]?.[field]?.length).toBeLessThan(8_001);
281 },
282 );
266283});
267284
268285describe("compactAccessibilityTree", () => {
apps/desktop/src/snapShot/snapShot.ts+13-1
......@@ -155,11 +155,16 @@ function safeProperty<T>(read: () => T): T | undefined {
155155 }
156156}
157157
158export function boundedSnapShotString(value: unknown, maxChars: number): string | undefined {
158export function boundedSnapShotString(
159 value: unknown,
160 maxChars: number,
161 onTruncated?: () => void,
162): string | undefined {
159163 if (typeof value !== "string") return undefined;
160164 const candidate = value.replaceAll("\0", "").trim();
161165 if (!candidate) return undefined;
162166 if (candidate.length <= maxChars) return candidate;
167 onTruncated?.();
163168 const end = /[\uD800-\uDBFF]/.test(candidate[maxChars - 1] ?? "") ? maxChars - 1 : maxChars;
164169 return candidate.slice(0, end).trimEnd();
165170}
......@@ -204,18 +209,22 @@ function accessibilityNode(
204209 imageSize: CapturedImageSize,
205210 isRoot: boolean,
206211 locationsReliable: boolean,
212 onTruncated: () => void,
207213): MutableAccessibilityNode {
208214 const name = boundedSnapShotString(
209215 safeProperty(() => element.name),
210216 1_000,
217 onTruncated,
211218 );
212219 const value = boundedSnapShotString(
213220 safeProperty(() => element.value),
214221 8_000,
222 onTruncated,
215223 );
216224 const description = boundedSnapShotString(
217225 safeProperty(() => element.description),
218226 2_000,
227 onTruncated,
219228 );
220229 const checked = safeProperty(() => element.checked);
221230 const expanded = safeProperty(() => element.expanded);
......@@ -389,6 +398,9 @@ export async function accessibleWindowElementTree(
389398 imageSize,
390399 required,
391400 options.locationsReliable !== false,
401 () => {
402 truncated = true;
403 },
392404 );
393405 nodes += 1;
394406 root ??= node;