From 78f4b69a1fc6c37e2bdbb3f8d02717a49d4ccacd Mon Sep 17 00:00:00 2001 From: maria Date: Wed, 14 Oct 2026 22:25:50 +0000 Subject: [PATCH] fix(server): preview copy only forwards text from the viewer's copy keys (#19561) --- apps/server/src/preview/ServerBrowser.test.ts | 112 +++++++++-- apps/server/src/preview/ServerBrowser.ts | 176 +++++++++++++----- 2 files changed, 227 insertions(+), 61 deletions(-) diff --git a/apps/server/src/preview/ServerBrowser.test.ts b/apps/server/src/preview/ServerBrowser.test.ts index 385223a42d397034d4eddde2b35afbe2f4219a83..dec1e52d4c9c4feacff5738bd1408bc48a325402 100644 --- a/apps/server/src/preview/ServerBrowser.test.ts +++ b/apps/server/src/preview/ServerBrowser.test.ts @@ -60,14 +60,43 @@ vi.mock("./ServerBrowserContexts.ts", () => ({ }, })); -function makeSession() { +/** A frame in its own process. Only its own session reaches the copy it holds. */ +function makeFrame() { + const frame = { + copy: "", + detached: false, + isDetached: () => frame.detached, + recording: Promise.withResolvers(), + }; + return frame; +} + +function makeSession(frame?: ReturnType) { + const events = new NodeEvents.EventEmitter(); return { - on: vi.fn(), + on: vi.fn((name: string, listener: () => void) => events.on(name, listener)), + emit: (name: string) => events.emit(name), detach: vi.fn(async () => {}), - send: vi.fn(async (method: string, _input?: unknown): Promise> => { + send: vi.fn(async (method: string, input?: unknown): Promise> => { if (method === "Page.getNavigationHistory") return { currentIndex: 0, entries: [{}] }; if (method === "Page.getLayoutMetrics") return { cssVisualViewport: { pageX: 0, pageY: 0 } }; if (method === "Page.captureScreenshot") return { data: "ZnJhbWU=" }; + if (method === "Page.getFrameTree") return { frameTree: { frame: { id: "main" } } }; + if (method === "Page.createIsolatedWorld") return { executionContextId: 7 }; + if (method === "Page.addScriptToEvaluateOnNewDocument") frame?.recording.resolve(); + // A copy or cut key copies keyCopy in the focused frame, or the page's process if none; + // one that copies nothing leaves the frames as they were. + if (method === "Input.dispatchKeyEvent" && (input as { commands?: unknown }).commands) { + if (focusedFrame) focusedFrame.copy = keyCopy || focusedFrame.copy; + else pageCopy = keyCopy || pageCopy; + } + // The clipboard world hands over what the session's frames copied, once. + if (method === "Runtime.evaluate" && (input as { contextId?: number }).contextId === 7) { + const value = frame ? frame.copy : pageCopy; + if (frame) frame.copy = ""; + else pageCopy = ""; + return { result: { value } }; + } return { result: { value: "evaluated" } }; }), }; @@ -91,6 +120,7 @@ function makeContext(onClose?: (context: BrowserContext) => void) { setDefaultNavigationTimeout: vi.fn(), context: () => context, mainFrame: () => page, + frames: () => [page], url: () => url, title: vi.fn(async () => "test page"), viewportSize: () => viewport, @@ -130,12 +160,10 @@ function makeContext(onClose?: (context: BrowserContext) => void) { sessions, newPage: async () => page as unknown as Page, grantPermissions: vi.fn(async () => {}), - exposeBinding: vi.fn(async (_name: string, binding: ClipboardBinding) => { - clipboardBinding = binding; - }), - addInitScript: vi.fn(async () => {}), - newCDPSession: async () => { - const session = makeSession(); + newCDPSession: async (target: unknown) => { + const session = makeSession( + target === page ? undefined : (target as ReturnType), + ); sessions.push(session); return session; }, @@ -153,8 +181,12 @@ const contexts: ReturnType[] = []; /** The profile and isolation each headless tab asked its context for. */ const contextRequests: Array<{ profileId: string; isolated: boolean }> = []; let contextGate: PromiseWithResolvers | null = null; -type ClipboardBinding = (source: { page: unknown }, text: unknown) => void; -let clipboardBinding: ClipboardBinding | null = null; +/** The text the page's frames hold from their latest copy. */ +let pageCopy = ""; +/** The text the next copy or cut key copies. */ +let keyCopy = ""; +/** The frame in its own process that has focus; otherwise keys copy in the page's process. */ +let focusedFrame: ReturnType | null = null; let contextFailure: Error | null = null; /** Server tabs the fake desktop renders, and the endpoints the server connected to. */ let desktopRendersNext = false; @@ -272,6 +304,9 @@ beforeEach(() => { contextRequests.length = 0; contextGate = null; contextFailure = null; + pageCopy = ""; + keyCopy = ""; + focusedFrame = null; desktopTabs.clear(); desktopRendersNext = false; releasedDesktopTabs.length = 0; @@ -1007,11 +1042,10 @@ it.live("closing a tab while a viewer is still opening it does not leave its pag ).pipe(Effect.provide(layer)), ); -it.live("page copies reach only the controlling viewer right after its input", () => +it.live("only the controlling viewer's copy and cut keys reach its clipboard", () => Effect.scoped( Effect.gen(function* () { const { browser, tabId } = yield* ready; - const page = contexts[0]!.page; const watcher = yield* browser.attachViewer(viewerInput(tabId, false)); const viewer = yield* browser.attachViewer(viewerInput(tabId, true)); yield* viewer.input({ type: "takeControl" }); @@ -1019,22 +1053,68 @@ it.live("page copies reach only the controlling viewer right after its input", ( Queue.clear(queue).pipe( Effect.map((items) => items.filter((item) => item._tag === "clipboard")), ); - // A page writing on its own, without a recent gesture, stays on the server. - clipboardBinding!({ page }, "unprompted"); + // What the page copied on its own stays on the server through other input. + pageCopy = "unprompted"; + yield* viewer.input({ type: "mouse", action: "move", x: 10, y: 10 }); + yield* viewer.input({ type: "wheel", x: 10, y: 10, deltaY: 40 }); + yield* viewer.input({ type: "key", action: "down", key: "v", code: "KeyV", modifiers: 4 }); + yield* viewer.input({ type: "unknown" }); expect(yield* clipboard(viewer.output)).toEqual([]); + keyCopy = "copied"; yield* viewer.input({ type: "key", action: "down", key: "c", code: "KeyC", modifiers: 4 }); const cdp = contexts[0]!.sessions.at(-1)!; expect(cdp.send).toHaveBeenCalledWith( "Input.dispatchKeyEvent", expect.objectContaining({ key: "c", commands: ["copy"] }), ); - clipboardBinding!({ page }, "copied"); expect(yield* clipboard(viewer.output)).toEqual([{ _tag: "clipboard", text: "copied" }]); + keyCopy = "cut"; + yield* viewer.input({ type: "key", action: "down", key: "x", code: "KeyX", modifiers: 2 }); + expect(yield* clipboard(viewer.output)).toEqual([{ _tag: "clipboard", text: "cut" }]); + // A copy key with nothing copied sends nothing, not what a frame kept from before. + pageCopy = "left over"; + keyCopy = ""; + yield* viewer.input({ type: "key", action: "down", key: "c", code: "KeyC", modifiers: 4 }); + expect(yield* clipboard(viewer.output)).toEqual([]); expect(yield* clipboard(watcher.output)).toEqual([]); }), ).pipe(Effect.provide(layer)), ); +it.live("copy keys reach the viewer from a frame in its own process", () => + Effect.scoped( + Effect.gen(function* () { + const { browser, tabId } = yield* ready; + const viewer = yield* browser.attachViewer(viewerInput(tabId, true)); + yield* viewer.input({ type: "takeControl" }); + const copy = () => + viewer.input({ type: "key", action: "down", key: "c", code: "KeyC", modifiers: 4 }).pipe( + Effect.andThen(Queue.clear(viewer.output)), + Effect.map((items) => items.filter((item) => item._tag === "clipboard")), + ); + const frame = makeFrame(); + contexts[0]!.page.emit("framenavigated", frame); + yield* Effect.promise(() => frame.recording.promise); + const frameSession = contexts[0]!.sessions.at(-1)!; + focusedFrame = frame; + keyCopy = "framed"; + expect(yield* copy()).toEqual([{ _tag: "clipboard", text: "framed" }]); + // What the frame kept from before is cleared, not sent by a key elsewhere. + frame.copy = "left over"; + focusedFrame = null; + keyCopy = ""; + expect(yield* copy()).toEqual([]); + expect(frame.copy).toBe(""); + // A frame that went away is no longer read. + frame.detached = true; + frame.copy = "gone"; + frameSession.emit("close"); + expect(yield* copy()).toEqual([]); + expect(frame.copy).toBe("gone"); + }), + ).pipe(Effect.provide(layer)), +); + it.live("a page download is saved, offered to the controller, and listed for the agent", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/preview/ServerBrowser.ts b/apps/server/src/preview/ServerBrowser.ts index 939d112c4908105863cf38e3603a1240576dfd5e..87789692eb3e831f9ea6231ad14ff22144ea97c4 100644 --- a/apps/server/src/preview/ServerBrowser.ts +++ b/apps/server/src/preview/ServerBrowser.ts @@ -66,6 +66,7 @@ import type { Dialog, Download, FileChooser, + Frame, Page, } from "playwright-core"; @@ -262,8 +263,6 @@ interface ViewerState { readonly pause: () => Promise; readonly resume: () => Promise; scrolledAt: number; - /** Last input from this viewer; page copies reach its clipboard only right after. */ - inputAt: number; /** Panel bounds, retained in fixed mode; passive viewers never request a size. */ requestedSize: { width: number; height: number; order: number } | null; } @@ -290,6 +289,8 @@ interface ServerTab { readonly tabId: string; readonly page: Page; readonly cdp: CDPSession; + /** Sessions of the page's frames that run in their own process. */ + readonly frameSessions: Map; readonly createdAt: number; /** The agent's latest request on this tab; idle agent tabs close. */ usedAt: number; @@ -434,43 +435,50 @@ const editingCommand = (key: string, modifiers: number) => { }; const CLIPBOARD_TEXT_LIMIT = 1024 * 1024; -/** Copies reach the viewer only this soon after it last touched the page. */ -const CLIPBOARD_GESTURE_MS = 5_000; -const CLIPBOARD_BINDING = "__t3PreviewClipboard"; -// Headless Chromium shares one clipboard between every context, so pages -// report their own copies instead of the clipboard being read back. +const CLIPBOARD_WORLD = "t3-preview-clipboard"; +// Headless Chromium shares one clipboard between every context, so the copy +// a viewer's copy or cut key makes is read from this world instead of the +// clipboard. Page scripts cannot see the world, and only a copy made while a +// real key is handled counts. const CLIPBOARD_SCRIPT = `(() => { - const send = (text) => { - if (typeof text !== "string" || text.length === 0) return; - try { globalThis.${CLIPBOARD_BINDING}?.(text.slice(0, ${CLIPBOARD_TEXT_LIMIT})); } catch {} - }; + let keying = false; + let copied = ""; + // The same Ctrl or Cmd + C or X that editingCommand sends as copy or cut. + addEventListener("keydown", (event) => { + if (!event.isTrusted || event.altKey || event.ctrlKey === event.metaKey) return; + const key = event.key.toLowerCase(); + if (key !== "c" && key !== "x") return; + keying = true; + copied = ""; + setTimeout(() => { keying = false; }); + }, true); const selection = () => { - const field = document.activeElement; + let field = document.activeElement; + while (field?.shadowRoot?.activeElement) field = field.shadowRoot.activeElement; if ((field instanceof HTMLInputElement || field instanceof HTMLTextAreaElement) && field.selectionStart !== null) return field.value.slice(field.selectionStart, field.selectionEnd ?? field.selectionStart); return String(document.getSelection() ?? ""); }; - // Bubbling to window runs after page handlers, before cut removes the selection. - const copied = (event) => - send(event.defaultPrevented ? event.clipboardData?.getData("text/plain") : selection()); - addEventListener("copy", copied); - addEventListener("cut", copied); - const proto = globalThis.Clipboard?.prototype; - if (!proto) return; - const writeText = proto.writeText; - proto.writeText = function (text) { - const result = writeText.call(this, text); - result.then(() => send(String(text)), () => {}); - return result; + const record = (text) => { + copied = String(text ?? "").slice(0, ${CLIPBOARD_TEXT_LIMIT}); }; - const write = proto.write; - proto.write = function (items) { - const result = write.call(this, items); - result.then(async () => { - const item = [...items].find((candidate) => candidate.types.includes("text/plain")); - if (item) send(await (await item.getType("text/plain")).text()); - }, () => {}); - return result; + // Capturing at window runs before page handlers can stop the event, and + // before cut removes the selection. + const snapshot = (event) => { + if (keying && event.isTrusted) record(selection()); + }; + // Bubbling back to window picks up text a page handler set instead. + const replace = (event) => { + if (keying && event.isTrusted && event.defaultPrevented) record(event.clipboardData?.getData("text/plain")); + }; + for (const type of ["copy", "cut"]) { + addEventListener(type, snapshot, true); + addEventListener(type, replace); + } + globalThis.takeCopy = () => { + const text = copied; + copied = ""; + return text; }; })();`; @@ -763,13 +771,14 @@ const make = Effect.gen(function* () { isolatedContext ? tabKey(snapshot.threadId, snapshot.tabId) : undefined, )); if (adopted?.page.isClosed()) throw new Error("The popup closed before it opened."); - // The desktop page already has its own clipboard; the bridge script is for headless tabs. + // The desktop page already has its own clipboard. if (!desktop) await prepareContext(context); const page = adopted?.page ?? desktop?.page ?? (await context.newPage()); const cdp = await context.newCDPSession(page); // The desktop's page is already a normal browser. A popup's first request // went out before it became a tab; everything after presents as Chrome. if (!desktop) await presentAsChrome(cdp, host); + if (!desktop) await recordCopies(cdp); page.setDefaultTimeout(NAVIGATION_TIMEOUT_MS); page.setDefaultNavigationTimeout(NAVIGATION_TIMEOUT_MS); const control = new SessionControl(snapshot.automationOwner ?? null, () => @@ -780,6 +789,7 @@ const make = Effect.gen(function* () { tabId: snapshot.tabId, page, cdp, + frameSessions: new Map(), createdAt: Date.now(), usedAt: Date.now(), viewers: new Set(), @@ -828,7 +838,10 @@ const make = Effect.gen(function* () { page.on("framenavigated", (frame) => { // Same-document navigations (SPA routes) fire no load event. if (frame === page.mainFrame() && !tab.loading) void reportLoaded(tab); + if (!desktop) void recordFrameCopies(tab, frame); }); + // An adopted popup's frames may have loaded before the tab listened. + if (adopted) for (const frame of page.frames()) void recordFrameCopies(tab, frame); page.on("requestfailed", (request) => { const errorText = request.failure()?.errorText ?? ""; pushBounded(tab.networkEntries, { @@ -1129,16 +1142,81 @@ const make = Effect.gen(function* () { if (preparedContexts.has(context)) return; preparedContexts.add(context); await context.grantPermissions(["clipboard-write"]); - await context.exposeBinding(CLIPBOARD_BINDING, ({ page }, text: unknown) => { - if (typeof text !== "string") return; - const tab = [...tabs.values()].find((candidate) => candidate.page === page); - const controller = tab - ? [...tab.viewers].find((viewer) => viewer.id === tab.control.controller) - : undefined; - if (!controller || Date.now() - controller.inputAt > CLIPBOARD_GESTURE_MS) return; - controller.push({ _tag: "clipboard", text: text.slice(0, CLIPBOARD_TEXT_LIMIT) }); + }; + + /** Records the copies made in each frame a session reaches. */ + const recordCopies = async (cdp: CDPSession) => { + // Scripts for new documents run only while this session's Page domain is on. + await cdp.send("Page.enable"); + await cdp.send("Page.addScriptToEvaluateOnNewDocument", { + source: CLIPBOARD_SCRIPT, + worldName: CLIPBOARD_WORLD, + runImmediately: true, }); - await context.addInitScript(CLIPBOARD_SCRIPT); + }; + + /** + * A frame in its own process, such as one with an `Origin-Agent-Cluster` + * header, is out of the page session's reach, so it records copies through + * its own session until that target closes. + */ + const recordFrameCopies = async (tab: ServerTab, frame: Frame) => { + if (frame === tab.page.mainFrame() || tab.frameSessions.has(frame)) return; + // A frame in its parent's process has no session of its own. + const cdp = await tab.page + .context() + .newCDPSession(frame) + .catch(() => null); + if (!cdp) return; + if (tab.frameSessions.has(frame) || frame.isDetached()) { + await cdp.detach().catch(constVoid); + return; + } + tab.frameSessions.set(frame, cdp); + cdp.on("close", () => { + if (tab.frameSessions.get(frame) !== cdp) return; + tab.frameSessions.delete(frame); + // A frame that moved to another process records through its new target. + if (!frame.isDetached()) void recordFrameCopies(tab, frame); + }); + await recordCopies(cdp).catch(constVoid); + }; + + /** Clears and returns the copy of each frame a session reaches. */ + const takeSessionCopies = async (cdp: CDPSession) => { + // An out-of-process frame that closed meanwhile copied nothing. + const tree = await cdp.send("Page.getFrameTree").catch(() => null); + if (!tree) return []; + const frameIds = (node: typeof tree.frameTree): Array => [ + node.frame.id, + ...(node.childFrames ?? []).flatMap(frameIds), + ]; + return Promise.all( + frameIds(tree.frameTree).map((frameId) => + cdp + .send("Page.createIsolatedWorld", { frameId, worldName: CLIPBOARD_WORLD }) + .then(({ executionContextId }) => + cdp.send("Runtime.evaluate", { + contextId: executionContextId, + expression: "globalThis.takeCopy?.()", + returnByValue: true, + }), + ) + .then( + ({ result }) => (typeof result.value === "string" ? result.value : ""), + // A frame that detached meanwhile copied nothing. + () => "", + ), + ), + ); + }; + + /** Clears every frame's copy and returns what the key just handled copied or cut. */ + const takeCopiedText = async (tab: ServerTab) => { + const copies = await Promise.all( + [tab.cdp, ...tab.frameSessions.values()].map(takeSessionCopies), + ); + return copies.flat().find((text) => text.length > 0); }; const adoptPopup = async (opener: ServerTab, popup: Page) => { @@ -2058,7 +2136,6 @@ const make = Effect.gen(function* () { ) => { const message = asRecord(raw); if (!message) return; - viewer.inputAt = Date.now(); const modifiers = modifiersOf(message); switch (message.type) { case "mouse": { @@ -2104,19 +2181,29 @@ const make = Effect.gen(function* () { viewer.pressedKeys.delete(code || key); return; } + const command = editingCommand(key, modifiers); + // Only the controlling viewer's own copy and cut reach its clipboard. + const copying = + !tab.desktop && (command?.commands[0] === "copy" || command?.commands[0] === "cut"); + // Clear what any frame kept from an earlier key, so only this key's copy is read. + if (copying) await takeCopiedText(tab); await session.send("Input.dispatchKeyEvent", { type: text ? "keyDown" : "rawKeyDown", key, code, modifiers, ...(text ? { text, unmodifiedText: text } : {}), - ...editingCommand(key, modifiers), + ...command, windowsVirtualKeyCode: num( message.keyCode, key.length === 1 ? key.toUpperCase().charCodeAt(0) : 0, ), }); viewer.pressedKeys.set(code || key, { key, code }); + if (copying) { + const copied = await takeCopiedText(tab); + if (copied) viewer.push({ _tag: "clipboard", text: copied }); + } return; } case "text": @@ -2251,7 +2338,6 @@ const make = Effect.gen(function* () { }, resume: () => startScreencast(screencastScale), scrolledAt: 0, - inputAt: 0, requestedSize: null, }; yield* Effect.acquireRelease( -- 2.54.0