authorgravatar for yashafaid@gmail.comCestercian <yashafaid@gmail.com> 2026-09-20 03:39:10+05:30
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-09-19 15:09:10-07:00
log52d08a14b9e475371658fabbb12c056c3d3dad67
treefe4eb466d801801aba3f44c9d0b0b3941be913d6
parent63ff33756c47f127391962458d8388a647e8ed44
signature Signed by PGP key B5690EEEBB952194

fix(desktop): include SnapShot app text for Flatpak and GTK4 (#12635)

Co-authored-by: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

4 files changed, 224 insertions(+), 11 deletions(-)

apps/desktop/src/snapShot/DesktopSnapShot.test.ts+138
......@@ -29,6 +29,7 @@ beforeEach(() => {
2929 transitionCapturePageMock.mockReset().mockResolvedValue(undefined);
3030 transitionSnapshotMock.mockReset().mockResolvedValue(undefined);
3131 prepareCaptureRevealMock.mockReset();
32 accessibilityListMock.mockReset().mockResolvedValue([]);
3233});
3334
3435const {
......@@ -39,6 +40,7 @@ const {
3940 accessibilityProcessCoolMock,
4041 accessibilityProcessReadMock,
4142 accessibilityByPidMock,
43 accessibilityListMock,
4244 accessibilityForegroundMock,
4345 accessibilityTrustedMock,
4446 allWindowsMock,
......@@ -87,6 +89,7 @@ const {
8789 }
8890 >(),
8991 accessibilityByPidMock: vi.fn(),
92 accessibilityListMock: vi.fn(),
9093 accessibilityForegroundMock: vi.fn(),
9194 accessibilityTrustedMock: vi.fn((_prompt = false) => true),
9295 allWindowsMock: vi.fn(
......@@ -430,6 +433,7 @@ import * as SnapShotAccessibility from "./SnapShotAccessibility.ts";
430433// Tests hand it this stand-in so the mocks above drive window lookups.
431434const accessibilityApp = {
432435 byPid: accessibilityByPidMock,
436 list: accessibilityListMock,
433437 foreground: accessibilityForegroundMock,
434438} as unknown as Parameters<typeof SnapShotAccessibility.readAccessibleWindowContextWithApp>[0];
435439const readAccessibleWindowContext = (
......@@ -2654,6 +2658,140 @@ it.each([
26542658 }
26552659});
26562660
2661it("reads Flatpak app text from the AT-SPI proxy when the compositor PID misses", async () => {
2662 vi.stubEnv("XDG_SESSION_TYPE", "wayland");
2663 const bounds = { x: 0, y: 0, width: 1_920, height: 1_048 };
2664 const tree = vi.fn(async () => ({
2665 name: "Issue — Zen Browser",
2666 children: [{ name: "New Tab", children: [] }],
2667 }));
2668 const proxyWindow = {
2669 role: "window",
2670 name: "Issue — Zen Browser",
2671 bounds,
2672 tree,
2673 children: async () => [],
2674 };
2675 accessibilityByPidMock
2676 .mockReset()
2677 .mockRejectedValue(
2678 new Error("XA11Y_SELECTOR_NOT_MATCHED: No element matched selector: application[pid=207651]"),
2679 );
2680 accessibilityListMock.mockReset().mockResolvedValue([
2681 {
2682 pid: 1,
2683 children: async () => [
2684 {
2685 role: "window",
2686 name: "Files",
2687 bounds,
2688 tree: async () => ({ name: "Files", children: [] }),
2689 },
2690 ],
2691 },
2692 { pid: 207646, children: async () => [proxyWindow] },
2693 ]);
2694 try {
2695 assert.strictEqual(
2696 await readAccessibleWindowText(
2697 {
2698 title: "Issue — Zen Browser",
2699 bounds: { x: 0, y: 0, width: 1_920, height: 1_048 },
2700 owner: { processId: 207651 },
2701 },
2702 "linux",
2703 "Issue — Zen Browser",
2704 ),
2705 "Issue — Zen Browser\nNew Tab",
2706 );
2707 assert.deepEqual(accessibilityByPidMock.mock.calls, [[207651, { timeout: 0 }]]);
2708 assert.lengthOf(accessibilityListMock.mock.calls, 1);
2709 assert.lengthOf(tree.mock.calls, 1);
2710 } finally {
2711 accessibilityListMock.mockReset().mockResolvedValue([]);
2712 vi.unstubAllEnvs();
2713 }
2714});
2715
2716it("reads GTK4 app text from one unnamed PID-scoped window of the captured size", async () => {
2717 vi.stubEnv("XDG_SESSION_TYPE", "wayland");
2718 const bounds = { x: 0, y: 0, width: 1_920, height: 1_048 };
2719 const tree = vi.fn(async () => ({
2720 name: "System Monitor",
2721 children: [{ name: "CPU", children: [] }],
2722 }));
2723 accessibilityByPidMock.mockReset().mockResolvedValue({
2724 children: async () => [
2725 {
2726 role: "group",
2727 name: null,
2728 bounds,
2729 tree,
2730 children: async () => [
2731 { role: "page_tab_list", name: "Processes", bounds, children: async () => [] },
2732 ],
2733 },
2734 ],
2735 });
2736 accessibilityListMock.mockReset().mockResolvedValue([]);
2737 try {
2738 assert.strictEqual(
2739 await readAccessibleWindowText(
2740 {
2741 title: "System Monitor",
2742 bounds: { x: 12, y: 48, width: 1_920, height: 1_048 },
2743 owner: { processId: 210600 },
2744 },
2745 "linux",
2746 "System Monitor",
2747 ),
2748 "System Monitor\nCPU",
2749 );
2750 assert.deepEqual(accessibilityByPidMock.mock.calls, [[210600, { timeout: 0 }]]);
2751 assert.lengthOf(accessibilityListMock.mock.calls, 0);
2752 assert.lengthOf(tree.mock.calls, 1);
2753 } finally {
2754 accessibilityListMock.mockReset().mockResolvedValue([]);
2755 vi.unstubAllEnvs();
2756 }
2757});
2758
2759it("does not guess GTK4 app text when two unnamed PID windows share the captured size", async () => {
2760 vi.stubEnv("XDG_SESSION_TYPE", "wayland");
2761 const bounds = { x: 0, y: 0, width: 1_920, height: 1_048 };
2762 const tree = vi.fn(async () => ({ value: "Wrong window", children: [] }));
2763 accessibilityByPidMock.mockReset().mockResolvedValue({
2764 children: async () => [
2765 { role: "group", name: null, bounds, tree },
2766 { role: "group", name: null, bounds: { ...bounds, x: 12 }, tree },
2767 ],
2768 });
2769 accessibilityListMock.mockReset().mockResolvedValue([
2770 {
2771 pid: 99,
2772 children: async () => [{ role: "window", name: "System Monitor", bounds, tree }],
2773 },
2774 ]);
2775 try {
2776 assert.isUndefined(
2777 await readAccessibleWindowText(
2778 {
2779 title: "System Monitor",
2780 bounds: { x: 12, y: 48, width: 1_920, height: 1_048 },
2781 owner: { processId: 210600 },
2782 },
2783 "linux",
2784 "System Monitor",
2785 ),
2786 );
2787 assert.lengthOf(accessibilityListMock.mock.calls, 0);
2788 assert.lengthOf(tree.mock.calls, 0);
2789 } finally {
2790 accessibilityListMock.mockReset().mockResolvedValue([]);
2791 vi.unstubAllEnvs();
2792 }
2793});
2794
26572795it.each([20, 1_350, 2_999])(
26582796 "includes accessibility text as soon as a %d ms read completes",
26592797 async (duration) => {
apps/desktop/src/snapShot/SnapShotAccessibility.ts+28-4
......@@ -97,6 +97,23 @@ function accessibilityReadSnapshot(
9797 ) as CapturedWindowAccessibilityContext;
9898}
9999
100type AccessibilityElement = Awaited<ReturnType<InstanceType<AccessibilityApp>["children"]>>[number];
101
102async function windowsForPid(
103 App: AccessibilityApp,
104 processId: number,
105): Promise<readonly AccessibilityElement[]> {
106 return await App.byPid(processId, { timeout: 0 })
107 .then((app) => app.children())
108 .catch(() => []);
109}
110
111/** Flatpak apps reach AT-SPI through xdg-dbus-proxy, so the compositor PID misses. */
112async function windowsFromAppList(App: AccessibilityApp): Promise<readonly AccessibilityElement[]> {
113 const apps = await App.list().catch(() => []);
114 return (await Promise.all(apps.map((app) => app.children().catch(() => [])))).flat();
115}
116
100117async function readCapturedWindowAccessibility(
101118 App: AccessibilityApp,
102119 request: SnapShotAccessibilityRequest,
......@@ -105,17 +122,24 @@ async function readCapturedWindowAccessibility(
105122): Promise<CapturedWindowAccessibilityContext | undefined> {
106123 const { active, platform, sourceTitle, imageSize } = request;
107124 const foreground = platform === "win32" ? await App.foreground({ timeout: 0 }) : undefined;
108 const windows =
125 const pidWindows =
109126 foreground !== undefined
110127 ? foreground.pid === active.owner.processId
111128 ? [foreground.asElement()]
112129 : []
113 : await (await App.byPid(active.owner.processId, { timeout: 0 })).children();
130 : await windowsForPid(App, active.owner.processId);
114131 const matchMode = isWaylandSession(platform, process.env) ? "wayland" : "screen-bounds";
132 const captured = {
133 title: active.title,
134 sourceTitle,
135 bounds: active.bounds,
136 clientBounds: active.clientBounds,
137 };
115138 const window = findAccessibleWindow(
116 windows,
117 { title: active.title, sourceTitle, bounds: active.bounds, clientBounds: active.clientBounds },
139 pidWindows.length > 0 || foreground !== undefined ? pidWindows : await windowsFromAppList(App),
140 captured,
118141 matchMode,
142 { allowUntitledUniqueBounds: foreground === undefined && pidWindows.length > 0 },
119143 );
120144 if (!window) {
121145 onStarted();
apps/desktop/src/snapShot/snapShot.test.ts+39
......@@ -464,6 +464,45 @@ describe("findAccessibleWindow", () => {
464464 expect(findAccessibleWindow(windows, { ...captured, title: "" }, "wayland")).toBeUndefined();
465465 });
466466
467 it("accepts one PID-scoped untitled window whose bounds match", () => {
468 const windows = [{ name: null, bounds: captured.bounds }];
469
470 expect(
471 findAccessibleWindow(windows, captured, "wayland", { allowUntitledUniqueBounds: true }),
472 ).toBe(windows[0]);
473 expect(findAccessibleWindow(windows, captured, "wayland")).toBeUndefined();
474 });
475
476 it("rejects ambiguous PID-scoped untitled windows even when bounds match", () => {
477 const windows = [
478 { name: null, bounds: captured.bounds },
479 { name: "", bounds: { ...captured.bounds, x: 0, y: 0 } },
480 ];
481
482 expect(
483 findAccessibleWindow(windows, captured, "wayland", { allowUntitledUniqueBounds: true }),
484 ).toBeUndefined();
485 });
486
487 it("does not fall back to a differently titled PID-scoped window with matching bounds", () => {
488 const windows = [{ name: "Preferences", bounds: captured.bounds }];
489
490 expect(
491 findAccessibleWindow(windows, captured, "wayland", { allowUntitledUniqueBounds: true }),
492 ).toBeUndefined();
493 });
494
495 it("does not use unique bounds when a titled match is already ambiguous", () => {
496 const windows = [
497 { name: "Editor", bounds: captured.bounds },
498 { name: "Editor", bounds: { ...captured.bounds, x: 0, y: 0 } },
499 ];
500
501 expect(
502 findAccessibleWindow(windows, captured, "wayland", { allowUntitledUniqueBounds: true }),
503 ).toBeUndefined();
504 });
505
467506 it.each(["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"])(
468507 "ignores a leading Wayland title spinner frame %s",
469508 (frame) => {
apps/desktop/src/snapShot/snapShot.ts+19-7
......@@ -480,6 +480,7 @@ export function findAccessibleWindow<
480480 readonly clientBounds?: WindowBounds | undefined;
481481 },
482482 matchMode: "screen-bounds" | "wayland" = "screen-bounds",
483 options: { readonly allowUntitledUniqueBounds?: boolean } = {},
483484): T | undefined {
484485 const normalizeTitle = (value: string) => {
485486 const title = value.trim();
......@@ -489,7 +490,6 @@ export function findAccessibleWindow<
489490 const titles = new Set(
490491 [captured.title, captured.sourceTitle ?? ""].map(normalizeTitle).filter(Boolean),
491492 );
492 if (titles.size === 0) return undefined;
493493 // Wayland accessibility providers can expose window size without a screen position.
494494 const boundsKeys =
495495 matchMode === "wayland"
......@@ -499,19 +499,31 @@ export function findAccessibleWindow<
499499 matchMode === "wayland" && captured.clientBounds
500500 ? [captured.bounds, captured.clientBounds]
501501 : [captured.bounds];
502 const matches = windows.filter((window) => {
502 const matchesBounds = (window: T) => {
503503 const bounds = window.bounds;
504504 return (
505 titles.has(normalizeTitle(window.name ?? "")) &&
506505 bounds !== null &&
507506 candidateBounds.some((candidate) =>
508507 boundsKeys.every((key) => Math.abs(bounds[key] - candidate[key]) <= 2),
509508 )
510509 );
511 });
512 if (matches.length === 1) return matches[0];
513 const activeMatches = matches.filter((window) => safeProperty(() => window.active) === true);
514 return activeMatches.length === 1 ? activeMatches[0] : undefined;
510 };
511 if (titles.size > 0) {
512 const matches = windows.filter(
513 (window) => titles.has(normalizeTitle(window.name ?? "")) && matchesBounds(window),
514 );
515 if (matches.length === 1) return matches[0];
516 const activeMatches = matches.filter((window) => safeProperty(() => window.active) === true);
517 if (activeMatches.length === 1) return activeMatches[0];
518 if (matches.length > 1) return undefined;
519 }
520 // GTK4/libadwaita often exposes the frame as an unnamed group. A PID-scoped
521 // lookup can accept the one window whose bounds match; size-only guesses cannot.
522 if (!options.allowUntitledUniqueBounds) return undefined;
523 const boundsMatches = windows.filter(
524 (window) => normalizeTitle(window.name ?? "") === "" && matchesBounds(window),
525 );
526 return boundsMatches.length === 1 ? boundsMatches[0] : undefined;
515527}
516528
517529const ELECTRON_KEY_NAMES: Readonly<Record<string, string>> = {