1// @effect-diagnostics globalTimers:off -- Accessibility timeouts run outside Effect fibers.
2
3import {
4 SNAP_SHOT_ACCESSIBLE_TEXT_MAX_CHARS,
5 type SnapShotAccessibility,
6} from "@t3tools/contracts";
7import type * as Electron from "electron";
8
9import {
10 accessibleWindowElementTree,
11 accessibleWindowText,
12 compactAccessibilityTree,
13 findAccessibleWindow,
14 isWaylandSession,
15} from "./snapShot.ts";
16
17const ACCESSIBILITY_TIMEOUT_MS = 3_000;
18
19export type AccessibleWindowIdentity = {
20 readonly title: string;
21 readonly bounds: Electron.Rectangle;
22 readonly clientBounds?: Electron.Rectangle;
23 readonly owner: { readonly processId: number };
24 readonly accessibilityBoundsReliable?: boolean;
25};
26
27export type CapturedWindowAccessibilityContext = {
28 readonly accessibleText?: string;
29 readonly accessibility?: SnapShotAccessibility;
30};
31
32export type SnapShotAccessibilityRequest = {
33 readonly active: AccessibleWindowIdentity;
34 readonly platform: NodeJS.Platform;
35 readonly sourceTitle: string;
36 readonly imageSize: Electron.Size;
37};
38
39type AccessibilityApp = (typeof import("@crowecawcaw/xa11y"))["App"];
40
41type AccessibilityReadProgress = {
42 accessibleText: string | undefined;
43 flatComplete: boolean;
44 richComplete: boolean;
45 richLocationsReliable: boolean;
46 richRoot?: Extract<SnapShotAccessibility, { format: "element-tree" }>["root"];
47 richTruncated: boolean;
48 timedOut: boolean;
49};
50
51function accessibilityReadSnapshot(
52 progress: AccessibilityReadProgress,
53 imageSize: Electron.Size,
54): CapturedWindowAccessibilityContext | undefined {
55 const richTree = progress.richRoot
56 ? compactAccessibilityTree(progress.richRoot, {
57 descendantLocationsReliable: progress.richLocationsReliable,
58 })
59 : undefined;
60 const richText = richTree
61 ? accessibleWindowText(richTree.root, SNAP_SHOT_ACCESSIBLE_TEXT_MAX_CHARS)
62 : undefined;
63 const accessibleText = progress.accessibleText ?? richText;
64 const richTruncated = progress.richTruncated || richTree?.truncated === true;
65 const accessibility: SnapShotAccessibility | undefined =
66 progress.richComplete &&
67 richTree &&
68 (!richTruncated || !progress.accessibleText || progress.accessibleText === richText)
69 ? {
70 format: "element-tree",
71 coordinateSpace: "captured-image",
72 imageSize,
73 truncated: richTruncated,
74 root: richTree.root,
75 }
76 : progress.flatComplete && progress.accessibleText
77 ? {
78 format: "flat-text",
79 text: progress.accessibleText,
80 truncated: progress.accessibleText.length >= SNAP_SHOT_ACCESSIBLE_TEXT_MAX_CHARS,
81 }
82 : richTree
83 ? {
84 format: "element-tree",
85 coordinateSpace: "captured-image",
86 imageSize,
87 truncated: true,
88 root: richTree.root,
89 }
90 : undefined;
91 if (!accessibleText && !accessibility) return undefined;
92 return JSON.parse(
93 JSON.stringify({
94 ...(accessibleText ? { accessibleText } : {}),
95 ...(accessibility ? { accessibility } : {}),
96 }),
97 ) as CapturedWindowAccessibilityContext;
98}
99
100type AccessibilityElement = Awaited<ReturnType<InstanceType<AccessibilityApp>["children"]>>[number];
101
102async function windowsForPid(
103 App: AccessibilityApp,
104 processId: number,
105): Promise<readonly AccessibilityElement[]> {
106 return await App.byPid(processId, { timeout: 0 })
107 .then((app) => app.children())
108 .catch(() => []);
109}
110
111/** Flatpak apps reach AT-SPI through xdg-dbus-proxy, so the compositor PID misses. */
112async function windowsFromAppList(App: AccessibilityApp): Promise<readonly AccessibilityElement[]> {
113 const apps = await App.list().catch(() => []);
114 return (await Promise.all(apps.map((app) => app.children().catch(() => [])))).flat();
115}
116
117async function readCapturedWindowAccessibility(
118 App: AccessibilityApp,
119 request: SnapShotAccessibilityRequest,
120 progress: AccessibilityReadProgress,
121 onStarted: () => void,
122): Promise<CapturedWindowAccessibilityContext | undefined> {
123 const { active, platform, sourceTitle, imageSize } = request;
124 const foreground = platform === "win32" ? await App.foreground({ timeout: 0 }) : undefined;
125 const pidWindows =
126 foreground !== undefined
127 ? foreground.pid === active.owner.processId
128 ? [foreground.asElement()]
129 : []
130 : await windowsForPid(App, active.owner.processId);
131 const matchMode = isWaylandSession(platform, process.env) ? "wayland" : "screen-bounds";
132 const captured = {
133 title: active.title,
134 sourceTitle,
135 bounds: active.bounds,
136 clientBounds: active.clientBounds,
137 };
138 const window = findAccessibleWindow(
139 pidWindows.length > 0 || foreground !== undefined ? pidWindows : await windowsFromAppList(App),
140 captured,
141 matchMode,
142 { allowUntitledUniqueBounds: foreground === undefined && pidWindows.length > 0 },
143 );
144 if (!window) {
145 onStarted();
146 return undefined;
147 }
148 const accessibleBounds = window.bounds;
149 const matchingBounds =
150 matchMode === "wayland" &&
151 active.clientBounds &&
152 accessibleBounds &&
153 Math.abs(accessibleBounds.width - active.clientBounds.width) <= 2 &&
154 Math.abs(accessibleBounds.height - active.clientBounds.height) <= 2
155 ? active.clientBounds
156 : active.bounds;
157 const locationsReliable =
158 active.accessibilityBoundsReliable !== false &&
159 (matchMode === "screen-bounds" ||
160 (accessibleBounds !== null &&
161 Math.abs(accessibleBounds.x - matchingBounds.x) <= 2 &&
162 Math.abs(accessibleBounds.y - matchingBounds.y) <= 2));
163 const flatRead = window
164 .tree()
165 .then((tree) => {
166 progress.accessibleText =
167 accessibleWindowText(tree, SNAP_SHOT_ACCESSIBLE_TEXT_MAX_CHARS) || undefined;
168 progress.flatComplete = true;
169 })
170 .catch(() => {
171 progress.flatComplete = true;
172 });
173 // A decorated screenshot contains more than the accessibility client area. Keep its
174 // frame origin/scale so element coordinates include the actual decoration offset.
175 const sourceBounds =
176 matchMode === "wayland" && active.clientBounds
177 ? active.bounds
178 : (accessibleBounds ?? active.bounds);
179 const richRead = accessibleWindowElementTree(window, sourceBounds, imageSize, {
180 locationsReliable,
181 onProgress: (root, truncated, descendantLocationsReliable) => {
182 progress.richLocationsReliable = descendantLocationsReliable;
183 progress.richRoot = root;
184 progress.richTruncated = truncated;
185 },
186 shouldContinue: () => !progress.timedOut,
187 verifyDescendantLocations: matchMode === "wayland",
188 })
189 .then((rich) => {
190 progress.richComplete = true;
191 if (rich) {
192 progress.richRoot = rich.root;
193 progress.richTruncated = rich.truncated;
194 }
195 })
196 .catch(() => {
197 progress.richComplete = true;
198 });
199 onStarted();
200 await Promise.all([flatRead, richRead]);
201 return accessibilityReadSnapshot(progress, imageSize);
202}
203
204let activeAccessibilityRead: Promise<unknown> | undefined;
205
206async function raceAccessibleRead<T>(
207 run: () => Promise<T>,
208 timeoutValue: () => T | undefined,
209): Promise<T | undefined> {
210 if (activeAccessibilityRead) return undefined;
211 const read = run().catch(() => undefined);
212 activeAccessibilityRead = read;
213 void read.finally(() => {
214 if (activeAccessibilityRead === read) activeAccessibilityRead = undefined;
215 });
216 let timeout: ReturnType<typeof setTimeout> | undefined;
217 try {
218 return await Promise.race([
219 read,
220 new Promise<T | undefined>((resolve) => {
221 timeout = setTimeout(() => resolve(timeoutValue()), ACCESSIBILITY_TIMEOUT_MS);
222 }),
223 ]);
224 } finally {
225 if (timeout) clearTimeout(timeout);
226 }
227}
228
229export function readAccessibleWindowContextWithApp(
230 App: AccessibilityApp,
231 request: SnapShotAccessibilityRequest,
232 onStarted: () => void = () => undefined,
233): Promise<CapturedWindowAccessibilityContext | undefined> {
234 const progress: AccessibilityReadProgress = {
235 accessibleText: undefined,
236 flatComplete: false,
237 richComplete: false,
238 richLocationsReliable: false,
239 richTruncated: false,
240 timedOut: false,
241 };
242 return raceAccessibleRead(
243 () => readCapturedWindowAccessibility(App, request, progress, onStarted),
244 () => {
245 progress.timedOut = true;
246 return accessibilityReadSnapshot(progress, request.imageSize);
247 },
248 );
249}