authorgravatar for maria@kuuro.netmaria <maria@kuuro.net> 2026-10-14 22:25:50+00:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-10-14 22:25:50+00:00
log78f4b69a1fc6c37e2bdbb3f8d02717a49d4ccacd
treeec18181ea085e1a6c6aa83fd9661ff40aa677d30
parentaa81d1203ea43c3b6cc09c5442559fa0cf18db63
signature Signed by PGP key B5690EEEBB952194

fix(server): preview copy only forwards text from the viewer's copy keys (#19561)

2 files changed, 227 insertions(+), 61 deletions(-)

apps/server/src/preview/ServerBrowser.test.ts+96-16
......@@ -60,14 +60,43 @@ vi.mock("./ServerBrowserContexts.ts", () => ({
6060 },
6161}));
6262
63function makeSession() {
63/** A frame in its own process. Only its own session reaches the copy it holds. */
64function makeFrame() {
65 const frame = {
66 copy: "",
67 detached: false,
68 isDetached: () => frame.detached,
69 recording: Promise.withResolvers<void>(),
70 };
71 return frame;
72}
73
74function makeSession(frame?: ReturnType<typeof makeFrame>) {
75 const events = new NodeEvents.EventEmitter();
6476 return {
65 on: vi.fn(),
77 on: vi.fn((name: string, listener: () => void) => events.on(name, listener)),
78 emit: (name: string) => events.emit(name),
6679 detach: vi.fn(async () => {}),
67 send: vi.fn(async (method: string, _input?: unknown): Promise<Record<string, unknown>> => {
80 send: vi.fn(async (method: string, input?: unknown): Promise<Record<string, unknown>> => {
6881 if (method === "Page.getNavigationHistory") return { currentIndex: 0, entries: [{}] };
6982 if (method === "Page.getLayoutMetrics") return { cssVisualViewport: { pageX: 0, pageY: 0 } };
7083 if (method === "Page.captureScreenshot") return { data: "ZnJhbWU=" };
84 if (method === "Page.getFrameTree") return { frameTree: { frame: { id: "main" } } };
85 if (method === "Page.createIsolatedWorld") return { executionContextId: 7 };
86 if (method === "Page.addScriptToEvaluateOnNewDocument") frame?.recording.resolve();
87 // A copy or cut key copies keyCopy in the focused frame, or the page's process if none;
88 // one that copies nothing leaves the frames as they were.
89 if (method === "Input.dispatchKeyEvent" && (input as { commands?: unknown }).commands) {
90 if (focusedFrame) focusedFrame.copy = keyCopy || focusedFrame.copy;
91 else pageCopy = keyCopy || pageCopy;
92 }
93 // The clipboard world hands over what the session's frames copied, once.
94 if (method === "Runtime.evaluate" && (input as { contextId?: number }).contextId === 7) {
95 const value = frame ? frame.copy : pageCopy;
96 if (frame) frame.copy = "";
97 else pageCopy = "";
98 return { result: { value } };
99 }
71100 return { result: { value: "evaluated" } };
72101 }),
73102 };
......@@ -91,6 +120,7 @@ function makeContext(onClose?: (context: BrowserContext) => void) {
91120 setDefaultNavigationTimeout: vi.fn(),
92121 context: () => context,
93122 mainFrame: () => page,
123 frames: () => [page],
94124 url: () => url,
95125 title: vi.fn(async () => "test page"),
96126 viewportSize: () => viewport,
......@@ -130,12 +160,10 @@ function makeContext(onClose?: (context: BrowserContext) => void) {
130160 sessions,
131161 newPage: async () => page as unknown as Page,
132162 grantPermissions: vi.fn(async () => {}),
133 exposeBinding: vi.fn(async (_name: string, binding: ClipboardBinding) => {
134 clipboardBinding = binding;
135 }),
136 addInitScript: vi.fn(async () => {}),
137 newCDPSession: async () => {
138 const session = makeSession();
163 newCDPSession: async (target: unknown) => {
164 const session = makeSession(
165 target === page ? undefined : (target as ReturnType<typeof makeFrame>),
166 );
139167 sessions.push(session);
140168 return session;
141169 },
......@@ -153,8 +181,12 @@ const contexts: ReturnType<typeof makeContext>[] = [];
153181/** The profile and isolation each headless tab asked its context for. */
154182const contextRequests: Array<{ profileId: string; isolated: boolean }> = [];
155183let contextGate: PromiseWithResolvers<void> | null = null;
156type ClipboardBinding = (source: { page: unknown }, text: unknown) => void;
157let clipboardBinding: ClipboardBinding | null = null;
184/** The text the page's frames hold from their latest copy. */
185let pageCopy = "";
186/** The text the next copy or cut key copies. */
187let keyCopy = "";
188/** The frame in its own process that has focus; otherwise keys copy in the page's process. */
189let focusedFrame: ReturnType<typeof makeFrame> | null = null;
158190let contextFailure: Error | null = null;
159191/** Server tabs the fake desktop renders, and the endpoints the server connected to. */
160192let desktopRendersNext = false;
......@@ -272,6 +304,9 @@ beforeEach(() => {
272304 contextRequests.length = 0;
273305 contextGate = null;
274306 contextFailure = null;
307 pageCopy = "";
308 keyCopy = "";
309 focusedFrame = null;
275310 desktopTabs.clear();
276311 desktopRendersNext = false;
277312 releasedDesktopTabs.length = 0;
......@@ -1007,11 +1042,10 @@ it.live("closing a tab while a viewer is still opening it does not leave its pag
10071042 ).pipe(Effect.provide(layer)),
10081043);
10091044
1010it.live("page copies reach only the controlling viewer right after its input", () =>
1045it.live("only the controlling viewer's copy and cut keys reach its clipboard", () =>
10111046 Effect.scoped(
10121047 Effect.gen(function* () {
10131048 const { browser, tabId } = yield* ready;
1014 const page = contexts[0]!.page;
10151049 const watcher = yield* browser.attachViewer(viewerInput(tabId, false));
10161050 const viewer = yield* browser.attachViewer(viewerInput(tabId, true));
10171051 yield* viewer.input({ type: "takeControl" });
......@@ -1019,22 +1053,68 @@ it.live("page copies reach only the controlling viewer right after its input", (
10191053 Queue.clear(queue).pipe(
10201054 Effect.map((items) => items.filter((item) => item._tag === "clipboard")),
10211055 );
1022 // A page writing on its own, without a recent gesture, stays on the server.
1023 clipboardBinding!({ page }, "unprompted");
1056 // What the page copied on its own stays on the server through other input.
1057 pageCopy = "unprompted";
1058 yield* viewer.input({ type: "mouse", action: "move", x: 10, y: 10 });
1059 yield* viewer.input({ type: "wheel", x: 10, y: 10, deltaY: 40 });
1060 yield* viewer.input({ type: "key", action: "down", key: "v", code: "KeyV", modifiers: 4 });
1061 yield* viewer.input({ type: "unknown" });
10241062 expect(yield* clipboard(viewer.output)).toEqual([]);
1063 keyCopy = "copied";
10251064 yield* viewer.input({ type: "key", action: "down", key: "c", code: "KeyC", modifiers: 4 });
10261065 const cdp = contexts[0]!.sessions.at(-1)!;
10271066 expect(cdp.send).toHaveBeenCalledWith(
10281067 "Input.dispatchKeyEvent",
10291068 expect.objectContaining({ key: "c", commands: ["copy"] }),
10301069 );
1031 clipboardBinding!({ page }, "copied");
10321070 expect(yield* clipboard(viewer.output)).toEqual([{ _tag: "clipboard", text: "copied" }]);
1071 keyCopy = "cut";
1072 yield* viewer.input({ type: "key", action: "down", key: "x", code: "KeyX", modifiers: 2 });
1073 expect(yield* clipboard(viewer.output)).toEqual([{ _tag: "clipboard", text: "cut" }]);
1074 // A copy key with nothing copied sends nothing, not what a frame kept from before.
1075 pageCopy = "left over";
1076 keyCopy = "";
1077 yield* viewer.input({ type: "key", action: "down", key: "c", code: "KeyC", modifiers: 4 });
1078 expect(yield* clipboard(viewer.output)).toEqual([]);
10331079 expect(yield* clipboard(watcher.output)).toEqual([]);
10341080 }),
10351081 ).pipe(Effect.provide(layer)),
10361082);
10371083
1084it.live("copy keys reach the viewer from a frame in its own process", () =>
1085 Effect.scoped(
1086 Effect.gen(function* () {
1087 const { browser, tabId } = yield* ready;
1088 const viewer = yield* browser.attachViewer(viewerInput(tabId, true));
1089 yield* viewer.input({ type: "takeControl" });
1090 const copy = () =>
1091 viewer.input({ type: "key", action: "down", key: "c", code: "KeyC", modifiers: 4 }).pipe(
1092 Effect.andThen(Queue.clear(viewer.output)),
1093 Effect.map((items) => items.filter((item) => item._tag === "clipboard")),
1094 );
1095 const frame = makeFrame();
1096 contexts[0]!.page.emit("framenavigated", frame);
1097 yield* Effect.promise(() => frame.recording.promise);
1098 const frameSession = contexts[0]!.sessions.at(-1)!;
1099 focusedFrame = frame;
1100 keyCopy = "framed";
1101 expect(yield* copy()).toEqual([{ _tag: "clipboard", text: "framed" }]);
1102 // What the frame kept from before is cleared, not sent by a key elsewhere.
1103 frame.copy = "left over";
1104 focusedFrame = null;
1105 keyCopy = "";
1106 expect(yield* copy()).toEqual([]);
1107 expect(frame.copy).toBe("");
1108 // A frame that went away is no longer read.
1109 frame.detached = true;
1110 frame.copy = "gone";
1111 frameSession.emit("close");
1112 expect(yield* copy()).toEqual([]);
1113 expect(frame.copy).toBe("gone");
1114 }),
1115 ).pipe(Effect.provide(layer)),
1116);
1117
10381118it.live("a page download is saved, offered to the controller, and listed for the agent", () =>
10391119 Effect.scoped(
10401120 Effect.gen(function* () {
apps/server/src/preview/ServerBrowser.ts+131-45
......@@ -66,6 +66,7 @@ import type {
6666 Dialog,
6767 Download,
6868 FileChooser,
69 Frame,
6970 Page,
7071} from "playwright-core";
7172
......@@ -262,8 +263,6 @@ interface ViewerState {
262263 readonly pause: () => Promise<void>;
263264 readonly resume: () => Promise<void>;
264265 scrolledAt: number;
265 /** Last input from this viewer; page copies reach its clipboard only right after. */
266 inputAt: number;
267266 /** Panel bounds, retained in fixed mode; passive viewers never request a size. */
268267 requestedSize: { width: number; height: number; order: number } | null;
269268}
......@@ -290,6 +289,8 @@ interface ServerTab {
290289 readonly tabId: string;
291290 readonly page: Page;
292291 readonly cdp: CDPSession;
292 /** Sessions of the page's frames that run in their own process. */
293 readonly frameSessions: Map<Frame, CDPSession>;
293294 readonly createdAt: number;
294295 /** The agent's latest request on this tab; idle agent tabs close. */
295296 usedAt: number;
......@@ -434,43 +435,50 @@ const editingCommand = (key: string, modifiers: number) => {
434435};
435436
436437const CLIPBOARD_TEXT_LIMIT = 1024 * 1024;
437/** Copies reach the viewer only this soon after it last touched the page. */
438const CLIPBOARD_GESTURE_MS = 5_000;
439const CLIPBOARD_BINDING = "__t3PreviewClipboard";
440// Headless Chromium shares one clipboard between every context, so pages
441// report their own copies instead of the clipboard being read back.
438const CLIPBOARD_WORLD = "t3-preview-clipboard";
439// Headless Chromium shares one clipboard between every context, so the copy
440// a viewer's copy or cut key makes is read from this world instead of the
441// clipboard. Page scripts cannot see the world, and only a copy made while a
442// real key is handled counts.
442443const CLIPBOARD_SCRIPT = `(() => {
443 const send = (text) => {
444 if (typeof text !== "string" || text.length === 0) return;
445 try { globalThis.${CLIPBOARD_BINDING}?.(text.slice(0, ${CLIPBOARD_TEXT_LIMIT})); } catch {}
446 };
444 let keying = false;
445 let copied = "";
446 // The same Ctrl or Cmd + C or X that editingCommand sends as copy or cut.
447 addEventListener("keydown", (event) => {
448 if (!event.isTrusted || event.altKey || event.ctrlKey === event.metaKey) return;
449 const key = event.key.toLowerCase();
450 if (key !== "c" && key !== "x") return;
451 keying = true;
452 copied = "";
453 setTimeout(() => { keying = false; });
454 }, true);
447455 const selection = () => {
448 const field = document.activeElement;
456 let field = document.activeElement;
457 while (field?.shadowRoot?.activeElement) field = field.shadowRoot.activeElement;
449458 if ((field instanceof HTMLInputElement || field instanceof HTMLTextAreaElement) && field.selectionStart !== null)
450459 return field.value.slice(field.selectionStart, field.selectionEnd ?? field.selectionStart);
451460 return String(document.getSelection() ?? "");
452461 };
453 // Bubbling to window runs after page handlers, before cut removes the selection.
454 const copied = (event) =>
455 send(event.defaultPrevented ? event.clipboardData?.getData("text/plain") : selection());
456 addEventListener("copy", copied);
457 addEventListener("cut", copied);
458 const proto = globalThis.Clipboard?.prototype;
459 if (!proto) return;
460 const writeText = proto.writeText;
461 proto.writeText = function (text) {
462 const result = writeText.call(this, text);
463 result.then(() => send(String(text)), () => {});
464 return result;
462 const record = (text) => {
463 copied = String(text ?? "").slice(0, ${CLIPBOARD_TEXT_LIMIT});
464 };
465 // Capturing at window runs before page handlers can stop the event, and
466 // before cut removes the selection.
467 const snapshot = (event) => {
468 if (keying && event.isTrusted) record(selection());
469 };
470 // Bubbling back to window picks up text a page handler set instead.
471 const replace = (event) => {
472 if (keying && event.isTrusted && event.defaultPrevented) record(event.clipboardData?.getData("text/plain"));
465473 };
466 const write = proto.write;
467 proto.write = function (items) {
468 const result = write.call(this, items);
469 result.then(async () => {
470 const item = [...items].find((candidate) => candidate.types.includes("text/plain"));
471 if (item) send(await (await item.getType("text/plain")).text());
472 }, () => {});
473 return result;
474 for (const type of ["copy", "cut"]) {
475 addEventListener(type, snapshot, true);
476 addEventListener(type, replace);
477 }
478 globalThis.takeCopy = () => {
479 const text = copied;
480 copied = "";
481 return text;
474482 };
475483})();`;
476484
......@@ -763,13 +771,14 @@ const make = Effect.gen(function* () {
763771 isolatedContext ? tabKey(snapshot.threadId, snapshot.tabId) : undefined,
764772 ));
765773 if (adopted?.page.isClosed()) throw new Error("The popup closed before it opened.");
766 // The desktop page already has its own clipboard; the bridge script is for headless tabs.
774 // The desktop page already has its own clipboard.
767775 if (!desktop) await prepareContext(context);
768776 const page = adopted?.page ?? desktop?.page ?? (await context.newPage());
769777 const cdp = await context.newCDPSession(page);
770778 // The desktop's page is already a normal browser. A popup's first request
771779 // went out before it became a tab; everything after presents as Chrome.
772780 if (!desktop) await presentAsChrome(cdp, host);
781 if (!desktop) await recordCopies(cdp);
773782 page.setDefaultTimeout(NAVIGATION_TIMEOUT_MS);
774783 page.setDefaultNavigationTimeout(NAVIGATION_TIMEOUT_MS);
775784 const control = new SessionControl(snapshot.automationOwner ?? null, () =>
......@@ -780,6 +789,7 @@ const make = Effect.gen(function* () {
780789 tabId: snapshot.tabId,
781790 page,
782791 cdp,
792 frameSessions: new Map(),
783793 createdAt: Date.now(),
784794 usedAt: Date.now(),
785795 viewers: new Set(),
......@@ -828,7 +838,10 @@ const make = Effect.gen(function* () {
828838 page.on("framenavigated", (frame) => {
829839 // Same-document navigations (SPA routes) fire no load event.
830840 if (frame === page.mainFrame() && !tab.loading) void reportLoaded(tab);
841 if (!desktop) void recordFrameCopies(tab, frame);
831842 });
843 // An adopted popup's frames may have loaded before the tab listened.
844 if (adopted) for (const frame of page.frames()) void recordFrameCopies(tab, frame);
832845 page.on("requestfailed", (request) => {
833846 const errorText = request.failure()?.errorText ?? "";
834847 pushBounded(tab.networkEntries, {
......@@ -1129,16 +1142,81 @@ const make = Effect.gen(function* () {
11291142 if (preparedContexts.has(context)) return;
11301143 preparedContexts.add(context);
11311144 await context.grantPermissions(["clipboard-write"]);
1132 await context.exposeBinding(CLIPBOARD_BINDING, ({ page }, text: unknown) => {
1133 if (typeof text !== "string") return;
1134 const tab = [...tabs.values()].find((candidate) => candidate.page === page);
1135 const controller = tab
1136 ? [...tab.viewers].find((viewer) => viewer.id === tab.control.controller)
1137 : undefined;
1138 if (!controller || Date.now() - controller.inputAt > CLIPBOARD_GESTURE_MS) return;
1139 controller.push({ _tag: "clipboard", text: text.slice(0, CLIPBOARD_TEXT_LIMIT) });
1145 };
1146
1147 /** Records the copies made in each frame a session reaches. */
1148 const recordCopies = async (cdp: CDPSession) => {
1149 // Scripts for new documents run only while this session's Page domain is on.
1150 await cdp.send("Page.enable");
1151 await cdp.send("Page.addScriptToEvaluateOnNewDocument", {
1152 source: CLIPBOARD_SCRIPT,
1153 worldName: CLIPBOARD_WORLD,
1154 runImmediately: true,
1155 });
1156 };
1157
1158 /**
1159 * A frame in its own process, such as one with an `Origin-Agent-Cluster`
1160 * header, is out of the page session's reach, so it records copies through
1161 * its own session until that target closes.
1162 */
1163 const recordFrameCopies = async (tab: ServerTab, frame: Frame) => {
1164 if (frame === tab.page.mainFrame() || tab.frameSessions.has(frame)) return;
1165 // A frame in its parent's process has no session of its own.
1166 const cdp = await tab.page
1167 .context()
1168 .newCDPSession(frame)
1169 .catch(() => null);
1170 if (!cdp) return;
1171 if (tab.frameSessions.has(frame) || frame.isDetached()) {
1172 await cdp.detach().catch(constVoid);
1173 return;
1174 }
1175 tab.frameSessions.set(frame, cdp);
1176 cdp.on("close", () => {
1177 if (tab.frameSessions.get(frame) !== cdp) return;
1178 tab.frameSessions.delete(frame);
1179 // A frame that moved to another process records through its new target.
1180 if (!frame.isDetached()) void recordFrameCopies(tab, frame);
11401181 });
1141 await context.addInitScript(CLIPBOARD_SCRIPT);
1182 await recordCopies(cdp).catch(constVoid);
1183 };
1184
1185 /** Clears and returns the copy of each frame a session reaches. */
1186 const takeSessionCopies = async (cdp: CDPSession) => {
1187 // An out-of-process frame that closed meanwhile copied nothing.
1188 const tree = await cdp.send("Page.getFrameTree").catch(() => null);
1189 if (!tree) return [];
1190 const frameIds = (node: typeof tree.frameTree): Array<string> => [
1191 node.frame.id,
1192 ...(node.childFrames ?? []).flatMap(frameIds),
1193 ];
1194 return Promise.all(
1195 frameIds(tree.frameTree).map((frameId) =>
1196 cdp
1197 .send("Page.createIsolatedWorld", { frameId, worldName: CLIPBOARD_WORLD })
1198 .then(({ executionContextId }) =>
1199 cdp.send("Runtime.evaluate", {
1200 contextId: executionContextId,
1201 expression: "globalThis.takeCopy?.()",
1202 returnByValue: true,
1203 }),
1204 )
1205 .then(
1206 ({ result }) => (typeof result.value === "string" ? result.value : ""),
1207 // A frame that detached meanwhile copied nothing.
1208 () => "",
1209 ),
1210 ),
1211 );
1212 };
1213
1214 /** Clears every frame's copy and returns what the key just handled copied or cut. */
1215 const takeCopiedText = async (tab: ServerTab) => {
1216 const copies = await Promise.all(
1217 [tab.cdp, ...tab.frameSessions.values()].map(takeSessionCopies),
1218 );
1219 return copies.flat().find((text) => text.length > 0);
11421220 };
11431221
11441222 const adoptPopup = async (opener: ServerTab, popup: Page) => {
......@@ -2058,7 +2136,6 @@ const make = Effect.gen(function* () {
20582136 ) => {
20592137 const message = asRecord(raw);
20602138 if (!message) return;
2061 viewer.inputAt = Date.now();
20622139 const modifiers = modifiersOf(message);
20632140 switch (message.type) {
20642141 case "mouse": {
......@@ -2104,19 +2181,29 @@ const make = Effect.gen(function* () {
21042181 viewer.pressedKeys.delete(code || key);
21052182 return;
21062183 }
2184 const command = editingCommand(key, modifiers);
2185 // Only the controlling viewer's own copy and cut reach its clipboard.
2186 const copying =
2187 !tab.desktop && (command?.commands[0] === "copy" || command?.commands[0] === "cut");
2188 // Clear what any frame kept from an earlier key, so only this key's copy is read.
2189 if (copying) await takeCopiedText(tab);
21072190 await session.send("Input.dispatchKeyEvent", {
21082191 type: text ? "keyDown" : "rawKeyDown",
21092192 key,
21102193 code,
21112194 modifiers,
21122195 ...(text ? { text, unmodifiedText: text } : {}),
2113 ...editingCommand(key, modifiers),
2196 ...command,
21142197 windowsVirtualKeyCode: num(
21152198 message.keyCode,
21162199 key.length === 1 ? key.toUpperCase().charCodeAt(0) : 0,
21172200 ),
21182201 });
21192202 viewer.pressedKeys.set(code || key, { key, code });
2203 if (copying) {
2204 const copied = await takeCopiedText(tab);
2205 if (copied) viewer.push({ _tag: "clipboard", text: copied });
2206 }
21202207 return;
21212208 }
21222209 case "text":
......@@ -2251,7 +2338,6 @@ const make = Effect.gen(function* () {
22512338 },
22522339 resume: () => startScreencast(screencastScale),
22532340 scrolledAt: 0,
2254 inputAt: 0,
22552341 requestedSize: null,
22562342 };
22572343 yield* Effect.acquireRelease(