1"use client";
2
3import {
4 createPreviewFramePainter,
5 createPreviewStreamClient,
6 previewStreamControlLabel,
7 previewStreamModifiers,
8 type PreviewStreamClient,
9 type PreviewStreamControl,
10 type PreviewStreamDownload,
11 type PreviewStreamFileChooser,
12 uploadPreviewStreamFiles,
13 type PreviewStreamInput,
14 type PreviewStreamMouseButton,
15 type PreviewStreamPointer,
16 type PreviewStreamViewport,
17 previewStreamHostSetupMessage,
18} from "@t3tools/client-runtime/preview/server-browser-stream";
19import type {
20 EnvironmentId,
21 PreviewStreamHostSetup,
22 PreviewViewportSetting,
23} from "@t3tools/contracts";
24import {
25 type FormEvent,
26 type KeyboardEvent,
27 type PointerEvent,
28 type Ref,
29 useCallback,
30 useEffect,
31 useEffectEvent,
32 useImperativeHandle,
33 useRef,
34 useState,
35} from "react";
36
37import { AgentCursorMark } from "~/components/preview/AgentBrowserCursor";
38import { CommandBlock } from "~/components/CommandBlock";
39import { Button } from "~/components/ui/button";
40import { Input } from "~/components/ui/input";
41import { toastManager } from "~/components/ui/toast";
42import { cn } from "~/lib/utils";
43import { refreshPreviewStreamAccess, usePreviewStreamAccess } from "~/state/previewStream";
44import { observeResize } from "~/lib/observeResize";
45
46/** Chrome-row controls for a server tab; commands require current ownership. */
47export interface ServerBrowserHandle {
48 readonly navigate: (url: string) => void;
49 readonly history: (delta: -1 | 1) => void;
50 readonly reload: (options?: { readonly ignoreCache?: boolean }) => void;
51 readonly viewport: (setting: PreviewViewportSetting) => void;
52 readonly canvas: () => HTMLCanvasElement | null;
53}
54
55const RESIZE_THROTTLE_MS = 150;
56const ACCESS_RETRY_MS = 10_000;
57// A recent probe answer near a new tap stands in for that tap's own answer,
58// which on a slow link arrives after the tap ends.
59const PROBE_REUSE_PX = 24;
60const PROBE_REUSE_MS = 10_000;
61// Kept in the input so soft keyboards have something to delete: Android
62// reports Backspace only as a deletion of text, not as a key.
63const INPUT_SENTINEL = "\u200b";
64const BACKSPACE = { key: "Backspace", code: "Backspace", keyCode: 8 } as const;
65const DELETE = { key: "Delete", code: "Delete", keyCode: 46 } as const;
66const MAX_UNAUTHORIZED_REFUSALS = 3;
67const TAP_SLOP_PX = 8;
68const MULTI_CLICK_MS = 500;
69const MULTI_CLICK_SLOP_PX = 4;
70const WHEEL_LINE_PX = 16;
71
72type MouseInput = Extract<PreviewStreamInput, { type: "mouse" }>;
73type WheelInput = Extract<PreviewStreamInput, { type: "wheel" }>;
74
75interface PagePoint {
76 readonly x: number;
77 readonly y: number;
78 /** Page CSS px per client px. */
79 readonly scale: number;
80}
81
82interface TouchGesture {
83 readonly pointerId: number;
84 readonly startX: number;
85 readonly startY: number;
86 lastX: number;
87 lastY: number;
88 panning: boolean;
89}
90
91/** Whether the page point under a touch takes text; null until the server answers. */
92interface TouchProbe {
93 readonly x: number;
94 readonly y: number;
95 editable: boolean | null;
96 /** True only for this tap's own reply, never for a cached answer. */
97 answered: boolean;
98 /** The tap ended before the answer arrived. */
99 tapped: boolean;
100}
101
102const buttonOf = (button: number): PreviewStreamMouseButton =>
103 button === 0 ? "left" : button === 1 ? "middle" : button === 2 ? "right" : "none";
104
105const pressedButtonOf = (buttons: number): PreviewStreamMouseButton =>
106 buttons & 1 ? "left" : buttons & 2 ? "right" : buttons & 4 ? "middle" : "none";
107
108/** Browsers may refuse a write that arrives after the key press; a toast button is a fresh gesture. */
109async function copyPageText(text: string) {
110 try {
111 await navigator.clipboard.writeText(text);
112 } catch {
113 const id = toastManager.add({
114 type: "info",
115 title: "The page copied text",
116 actionProps: {
117 children: "Copy",
118 onClick: () => {
119 toastManager.close(id);
120 void navigator.clipboard.writeText(text).catch(() => undefined);
121 },
122 },
123 });
124 }
125}
126
127/** The download happens on the environment; this hands the finished file to this device. */
128function offerDownload(download: PreviewStreamDownload) {
129 const id = toastManager.add({
130 type: "info",
131 title: `Downloaded ${download.fileName}`,
132 actionProps: {
133 children: "Save",
134 onClick: () => {
135 toastManager.close(id);
136 const anchor = document.createElement("a");
137 anchor.href = download.url;
138 anchor.download = download.fileName;
139 anchor.click();
140 },
141 },
142 });
143}
144
145export function ServerBrowserSurface(props: {
146 readonly environmentId: EnvironmentId;
147 readonly threadId: string;
148 readonly tabId: string;
149 readonly visible: boolean;
150 /** Sends the surface size so fill-mode tabs follow it. The floating player scales the page instead. */
151 readonly followSize?: boolean;
152 /** Floating previews reserve the top edge for their existing hover controls. */
153 readonly controlPosition?: "top" | "bottom";
154 readonly onFirstFrame?: () => void;
155 readonly onViewport?: (viewport: PreviewStreamViewport) => void;
156 readonly onControl?: (control: PreviewStreamControl | null) => void;
157 /** The page opened a new tab from this viewer's click; the viewer should switch to it. */
158 readonly onPopup?: (tabId: string) => void;
159 readonly className?: string;
160 readonly ref?: Ref<ServerBrowserHandle>;
161}) {
162 const {
163 environmentId,
164 threadId,
165 tabId,
166 visible,
167 followSize = true,
168 controlPosition = "top",
169 onFirstFrame,
170 onViewport,
171 onControl,
172 onPopup,
173 className,
174 ref,
175 } = props;
176 const access = usePreviewStreamAccess(environmentId);
177 const canvasRef = useRef<HTMLCanvasElement | null>(null);
178 const inputRef = useRef<HTMLTextAreaElement | null>(null);
179 const keySentRef = useRef(false);
180 const clientRef = useRef<PreviewStreamClient | null>(null);
181 const viewportRef = useRef<PreviewStreamViewport | null>(null);
182 const sizeRef = useRef<{ width: number; height: number } | null>(null);
183 const hasFrameRef = useRef(false);
184 const controlRef = useRef<PreviewStreamControl | null>(null);
185 const [control, setControl] = useState<PreviewStreamControl | null>(null);
186 const [promptText, setPromptText] = useState("");
187 const [fileChooser, setFileChooser] = useState<PreviewStreamFileChooser | null>(null);
188 const [agentCursor, setAgentCursor] = useState<AgentCursorPlacement | null>(null);
189 const fileInputRef = useRef<HTMLInputElement | null>(null);
190 const answerFileChooser = (files: ReadonlyArray<File>) => {
191 const chooser = fileChooser;
192 if (!chooser) return;
193 setFileChooser(null);
194 void uploadPreviewStreamFiles(chooser, files).catch((cause: unknown) =>
195 toastManager.add({
196 type: "error",
197 title: "Could not send the files to the page",
198 description: cause instanceof Error ? cause.message : undefined,
199 }),
200 );
201 };
202 const unauthorizedRef = useRef(0);
203 const [accessDenied, setAccessDenied] = useState(false);
204 const [hostSetup, setHostSetup] = useState<PreviewStreamHostSetup | null>(null);
205 const pendingMoveRef = useRef<MouseInput | null>(null);
206 const pendingWheelRef = useRef<WheelInput | null>(null);
207 const inputFrameRef = useRef<number | null>(null);
208 const mouseButtonsRef = useRef(0);
209 const mouseClicksRef = useRef({ left: 1, middle: 1, right: 1, none: 1 });
210 const lastClickRef = useRef<{
211 button: PreviewStreamMouseButton;
212 time: number;
213 x: number;
214 y: number;
215 count: number;
216 } | null>(null);
217 const touchRef = useRef<TouchGesture | null>(null);
218 const probeRef = useRef<TouchProbe | null>(null);
219 const lastProbeRef = useRef<{ x: number; y: number; editable: boolean; at: number } | null>(null);
220 const firstFrame = useEffectEvent(() => onFirstFrame?.());
221 const viewportChanged = useEffectEvent((viewport: PreviewStreamViewport) =>
222 onViewport?.(viewport),
223 );
224 const controlChanged = useEffectEvent((next: PreviewStreamControl | null) => onControl?.(next));
225 const popupOpened = useEffectEvent((popupTabId: string) => onPopup?.(popupTabId));
226 // Frame cap in device px, fixed per socket. It grows with the surface and
227 // never shrinks, so only outgrowing it reconnects.
228 const [cap, setCap] = useState<{ width: number; height: number } | null>(null);
229
230 const send = useCallback((input: PreviewStreamInput) => {
231 clientRef.current?.send(input);
232 }, []);
233
234 const clearInput = useCallback(() => {
235 if (inputFrameRef.current !== null) cancelAnimationFrame(inputFrameRef.current);
236 inputFrameRef.current = null;
237 pendingMoveRef.current = pendingWheelRef.current = null;
238 mouseButtonsRef.current = 0;
239 mouseClicksRef.current = { left: 1, middle: 1, right: 1, none: 1 };
240 lastClickRef.current = null;
241 touchRef.current = probeRef.current = lastProbeRef.current = null;
242 keySentRef.current = false;
243 if (inputRef.current) inputRef.current.value = INPUT_SENTINEL;
244 inputRef.current?.blur();
245 }, []);
246
247 const flushInput = useCallback(() => {
248 if (inputFrameRef.current !== null) cancelAnimationFrame(inputFrameRef.current);
249 inputFrameRef.current = null;
250 const move = pendingMoveRef.current;
251 const wheel = pendingWheelRef.current;
252 pendingMoveRef.current = null;
253 pendingWheelRef.current = null;
254 if (move) send(move);
255 if (wheel) send(wheel);
256 }, [send]);
257
258 // Coalesce moves and wheel deltas to one message each per animation frame.
259 const scheduleFlush = useCallback(() => {
260 inputFrameRef.current ??= requestAnimationFrame(flushInput);
261 }, [flushInput]);
262
263 const queueWheel = useCallback(
264 (point: PagePoint, deltaX: number, deltaY: number, modifiers: number) => {
265 const pending = pendingWheelRef.current;
266 pendingWheelRef.current = {
267 type: "wheel",
268 x: point.x,
269 y: point.y,
270 deltaX: (pending?.deltaX ?? 0) + deltaX,
271 deltaY: (pending?.deltaY ?? 0) + deltaY,
272 modifiers,
273 };
274 scheduleFlush();
275 },
276 [scheduleFlush],
277 );
278
279 const pagePoint = useCallback(
280 (clientX: number, clientY: number, clamp: boolean): PagePoint | null => {
281 const canvas = canvasRef.current;
282 const viewport = viewportRef.current;
283 if (!canvas || !viewport || !hasFrameRef.current || controlRef.current?.controller !== "you")
284 return null;
285 const rect = canvas.getBoundingClientRect();
286 // `object-contain` letterboxes the frame inside the canvas box.
287 const fit = Math.min(rect.width / canvas.width, rect.height / canvas.height);
288 const width = canvas.width * fit;
289 const height = canvas.height * fit;
290 if (!(width > 0 && height > 0)) return null;
291 const scale = viewport.width / width;
292 const x = (clientX - rect.left - (rect.width - width) / 2) * scale;
293 const y = (clientY - rect.top - (rect.height - height) / 2) * (viewport.height / height);
294 const inside = x >= 0 && y >= 0 && x <= viewport.width && y <= viewport.height;
295 if (!inside && !clamp) return null;
296 return {
297 x: Math.min(Math.max(x, 0), viewport.width),
298 y: Math.min(Math.max(y, 0), viewport.height),
299 scale,
300 };
301 },
302 [],
303 );
304
305 const countClick = (button: PreviewStreamMouseButton, x: number, y: number, time: number) => {
306 const last = lastClickRef.current;
307 const count =
308 last &&
309 last.button === button &&
310 time - last.time < MULTI_CLICK_MS &&
311 Math.hypot(x - last.x, y - last.y) < MULTI_CLICK_SLOP_PX
312 ? last.count + 1
313 : 1;
314 lastClickRef.current = { button, time, x, y, count };
315 return count;
316 };
317
318 const focusInput = () => inputRef.current?.focus({ preventScroll: true });
319
320 useImperativeHandle(
321 ref,
322 () => ({
323 navigate: (url) => send({ type: "navigate", url }),
324 history: (delta) => send({ type: "history", delta }),
325 reload: (options) =>
326 send({ type: "reload", ...(options?.ignoreCache ? { ignoreCache: true } : {}) }),
327 viewport: (setting) => send({ type: "viewport", setting }),
328 canvas: () => (hasFrameRef.current ? canvasRef.current : null),
329 }),
330 [send],
331 );
332
333 useEffect(() => {
334 const element = canvasRef.current?.parentElement;
335 if (!element) return;
336 let timer: ReturnType<typeof setTimeout> | null = null;
337 let capTimer: ReturnType<typeof setTimeout> | null = null;
338 const growCap = () => {
339 capTimer = null;
340 const size = sizeRef.current;
341 if (!size) return;
342 const ratio = window.devicePixelRatio || 1;
343 const width = Math.round(size.width * ratio);
344 const height = Math.round(size.height * ratio);
345 setCap((current) =>
346 current !== null && current.width >= width && current.height >= height
347 ? current
348 : {
349 width: Math.max(width, current?.width ?? 0),
350 height: Math.max(height, current?.height ?? 0),
351 },
352 );
353 };
354 const measure = () => {
355 timer = null;
356 const rect = element.getBoundingClientRect();
357 if (rect.width < 1 || rect.height < 1) return;
358 const size = { width: Math.round(rect.width), height: Math.round(rect.height) };
359 const previous = sizeRef.current;
360 if (previous?.width === size.width && previous.height === size.height) return;
361 sizeRef.current = size;
362 if (followSize) clientRef.current?.send({ type: "resize", ...size });
363 // Outgrowing the cap reconnects the stream, so grow it once the size settles
364 // rather than on every throttled resize of a drag.
365 if (capTimer !== null) clearTimeout(capTimer);
366 capTimer = setTimeout(growCap, previous === null ? 0 : RESIZE_THROTTLE_MS * 2);
367 };
368 const stopObserving = observeResize(element, () => {
369 // A pending measure reads the latest size when it fires, so a drag resizes
370 // the page at most every RESIZE_THROTTLE_MS and still ends on the final size.
371 if (timer !== null) return;
372 // The first size connects right away.
373 timer = setTimeout(measure, sizeRef.current === null ? 0 : RESIZE_THROTTLE_MS);
374 });
375 return () => {
376 stopObserving();
377 if (timer !== null) clearTimeout(timer);
378 if (capTimer !== null) clearTimeout(capTimer);
379 };
380 }, [followSize]);
381
382 useEffect(() => {
383 const canvas = canvasRef.current;
384 if (!visible || accessDenied || hostSetup || !access || !cap || !canvas) return;
385 let refreshTimer: ReturnType<typeof setTimeout> | null = null;
386 const painter = createPreviewFramePainter(canvas, () => {
387 if (hasFrameRef.current) return;
388 hasFrameRef.current = true;
389 firstFrame();
390 });
391 const client = createPreviewStreamClient(
392 { access, threadId, tabId, maxWidth: cap.width, maxHeight: cap.height },
393 {
394 onFrame: (jpeg) => {
395 unauthorizedRef.current = 0;
396 painter.paint(jpeg);
397 },
398 onProbe: (result) => {
399 if (controlRef.current?.controller !== "you") return;
400 lastProbeRef.current = { ...result, at: performance.now() };
401 const probe = probeRef.current;
402 if (!probe || probe.x !== result.x || probe.y !== result.y) return;
403 probe.editable = result.editable;
404 probe.answered = true;
405 if (!probe.tapped) return;
406 // Late answer: Android still raises the keyboard; iOS waits for the next tap.
407 probeRef.current = null;
408 if (result.editable) inputRef.current?.focus({ preventScroll: true });
409 else inputRef.current?.blur();
410 },
411 onPointer: (pointer) => {
412 const placed = placeAgentCursor(pointer, canvasRef.current, viewportRef.current);
413 if (placed) setAgentCursor(placed);
414 },
415 onClipboard: (text) => void copyPageText(text),
416 onDownload: offerDownload,
417 onPopup: popupOpened,
418 onFileChooser: setFileChooser,
419 onViewport: (viewport) => {
420 viewportRef.current = viewport;
421 viewportChanged(viewport);
422 },
423 onControl: (next) => {
424 const previous = controlRef.current;
425 controlRef.current = next;
426 setControl(next);
427 controlChanged(next);
428 if (
429 next.dialog?.message !== previous?.dialog?.message ||
430 next.dialog?.defaultValue !== previous?.dialog?.defaultValue
431 )
432 setPromptText(next.dialog?.defaultValue ?? "");
433 if (next.controller !== "you") {
434 clearInput();
435 } else if (previous?.controller !== "you") {
436 const size = sizeRef.current;
437 if (followSize && size) client.send({ type: "resize", ...size });
438 }
439 },
440 onConnectedChange: (connected) => {
441 if (connected) return;
442 setFileChooser(null);
443 controlRef.current = null;
444 setControl(null);
445 controlChanged(null);
446 clearInput();
447 },
448 onHostSetup: setHostSetup,
449 onUnauthorized: () => {
450 // Fresh tickets re-run this effect. Repeated refusals need an explicit retry.
451 const refusals = ++unauthorizedRef.current;
452 if (refusals >= MAX_UNAUTHORIZED_REFUSALS) {
453 inputRef.current?.blur();
454 setAccessDenied(true);
455 return;
456 }
457 refreshTimer = setTimeout(
458 () => refreshPreviewStreamAccess(environmentId),
459 refusals === 1 ? 0 : 1_000 * 2 ** (refusals - 1),
460 );
461 },
462 },
463 );
464 clientRef.current = client;
465 return () => {
466 painter.stop();
467 if (refreshTimer !== null) clearTimeout(refreshTimer);
468 client.stop();
469 if (clientRef.current === client) clientRef.current = null;
470 controlRef.current = null;
471 setControl(null);
472 controlChanged(null);
473 clearInput();
474 };
475 }, [
476 access,
477 accessDenied,
478 cap,
479 clearInput,
480 environmentId,
481 followSize,
482 hostSetup,
483 tabId,
484 threadId,
485 visible,
486 ]);
487
488 useEffect(() => {
489 if (!visible || accessDenied || access !== null) return;
490 // A failed ticket mint, e.g. while the server restarts, never retries on its own.
491 const timer = setInterval(() => refreshPreviewStreamAccess(environmentId), ACCESS_RETRY_MS);
492 return () => clearInterval(timer);
493 }, [access, accessDenied, environmentId, visible]);
494
495 useEffect(() => {
496 const canvas = canvasRef.current;
497 if (!canvas) return;
498 // Registered natively: React's wheel listener is passive and cannot stop the panel scrolling.
499 const onWheel = (event: WheelEvent) => {
500 const point = pagePoint(event.clientX, event.clientY, false);
501 if (!point) return;
502 event.preventDefault();
503 const unit =
504 event.deltaMode === WheelEvent.DOM_DELTA_LINE
505 ? WHEEL_LINE_PX
506 : event.deltaMode === WheelEvent.DOM_DELTA_PAGE
507 ? (viewportRef.current?.height ?? 0)
508 : 1;
509 queueWheel(point, event.deltaX * unit, event.deltaY * unit, previewStreamModifiers(event));
510 };
511 canvas.addEventListener("wheel", onWheel, { passive: false });
512 return () => canvas.removeEventListener("wheel", onWheel);
513 }, [pagePoint, queueWheel]);
514
515 useEffect(
516 () => () => {
517 if (inputFrameRef.current !== null) cancelAnimationFrame(inputFrameRef.current);
518 },
519 [],
520 );
521
522 const handlePointer = (event: PointerEvent<HTMLCanvasElement>) => {
523 if (controlRef.current?.controller !== "you") return;
524 const { type, pointerId, clientX, clientY } = event;
525 const cancelled = type === "pointercancel";
526 const touch = touchRef.current;
527 if (cancelled && touch?.pointerId === pointerId) {
528 touchRef.current = probeRef.current = null;
529 return;
530 }
531 if (event.pointerType === "touch" && !cancelled) {
532 const point = pagePoint(clientX, clientY, type === "pointermove");
533 if (type === "pointerdown") {
534 if (!event.isPrimary) return;
535 touchRef.current = {
536 pointerId,
537 startX: clientX,
538 startY: clientY,
539 lastX: clientX,
540 lastY: clientY,
541 panning: false,
542 };
543 const last = lastProbeRef.current;
544 const reusable =
545 point &&
546 last &&
547 performance.now() - last.at < PROBE_REUSE_MS &&
548 Math.hypot(point.x - last.x, point.y - last.y) < PROBE_REUSE_PX;
549 probeRef.current = point
550 ? {
551 x: point.x,
552 y: point.y,
553 editable: reusable ? last.editable : null,
554 answered: false,
555 tapped: false,
556 }
557 : null;
558 if (point) send({ type: "probe", x: point.x, y: point.y });
559 return;
560 }
561 if (!touch || touch.pointerId !== pointerId) return;
562 if (type === "pointermove") {
563 if (
564 !touch.panning &&
565 Math.hypot(clientX - touch.startX, clientY - touch.startY) < TAP_SLOP_PX
566 )
567 return;
568 touch.panning = true;
569 if (point)
570 queueWheel(
571 point,
572 (touch.lastX - clientX) * point.scale,
573 (touch.lastY - clientY) * point.scale,
574 0,
575 );
576 touch.lastX = clientX;
577 touch.lastY = clientY;
578 return;
579 }
580 touchRef.current = null;
581 const probe = probeRef.current;
582 if (touch.panning) {
583 probeRef.current = null;
584 return;
585 }
586 if (!point) return;
587 // iOS requires focus during the tap; late probe replies only raise Android's keyboard.
588 if (probe?.editable === true) focusInput();
589 else if (probe?.editable === false) inputRef.current?.blur();
590 if (probe && !probe.answered) probe.tapped = true;
591 else probeRef.current = null;
592 flushInput();
593 const clickCount = countClick("left", clientX, clientY, event.timeStamp);
594 const at = { x: point.x, y: point.y, modifiers: 0 };
595 send({ type: "mouse", action: "move", ...at, button: "none", buttons: 0, clickCount: 0 });
596 send({ type: "mouse", action: "down", ...at, button: "left", buttons: 1, clickCount });
597 send({ type: "mouse", action: "up", ...at, button: "left", buttons: 0, clickCount });
598 return;
599 }
600 const previous = mouseButtonsRef.current;
601 if ((cancelled || type === "pointerup") && previous === 0) return;
602 if (cancelled || type === "pointerup")
603 mouseButtonsRef.current = cancelled ? 0 : event.buttons & 7;
604 const point = pagePoint(
605 clientX,
606 clientY,
607 type !== "pointerdown" &&
608 (cancelled || type === "pointerup" || previous !== 0 || event.buttons !== 0),
609 );
610 if (!point) return;
611 const at = { x: point.x, y: point.y, modifiers: previewStreamModifiers(event) };
612 const mouse = (action: "down" | "up", button: PreviewStreamMouseButton, buttons: number) => {
613 if (action === "down")
614 mouseClicksRef.current[button] = countClick(button, clientX, clientY, event.timeStamp);
615 send({
616 type: "mouse",
617 action,
618 ...at,
619 button,
620 buttons,
621 clickCount: mouseClicksRef.current[button],
622 });
623 };
624 if (type === "pointerdown") {
625 focusInput();
626 event.currentTarget.setPointerCapture(pointerId);
627 flushInput();
628 mouseButtonsRef.current = event.buttons & 7;
629 mouse("down", buttonOf(event.button), event.buttons);
630 } else if (type === "pointerup") {
631 flushInput();
632 mouse("up", buttonOf(event.button), event.buttons);
633 } else {
634 // Chords arrive as pointermove. Cancellation releases every held button.
635 const next = cancelled ? 0 : event.buttons & 7;
636 const changed = previous ^ next;
637 if (previous !== 0 && changed !== 0) {
638 flushInput();
639 let buttons = previous;
640 for (const bit of [1, 2, 4]) {
641 if (!(changed & bit)) continue;
642 buttons ^= bit;
643 mouse(next & bit ? "down" : "up", pressedButtonOf(bit), buttons);
644 }
645 mouseButtonsRef.current = next;
646 }
647 if (cancelled) return;
648 pendingMoveRef.current = {
649 type: "mouse",
650 action: "move",
651 ...at,
652 button: pressedButtonOf(event.buttons),
653 buttons: event.buttons,
654 clickCount: 0,
655 };
656 scheduleFlush();
657 }
658 };
659
660 const resetInput = (textarea: HTMLTextAreaElement) => {
661 keySentRef.current = false;
662 textarea.value = INPUT_SENTINEL;
663 textarea.setSelectionRange(INPUT_SENTINEL.length, INPUT_SENTINEL.length);
664 };
665
666 const sendKeyPress = (key: typeof BACKSPACE | typeof DELETE) => {
667 send({ type: "key", action: "down", ...key, modifiers: 0 });
668 send({ type: "key", action: "up", ...key, modifiers: 0 });
669 };
670
671 const handleKey = (action: "down" | "up", event: KeyboardEvent<HTMLTextAreaElement>) => {
672 if (controlRef.current?.controller !== "you") return;
673 keySentRef.current = false;
674 // IME and soft keyboards deliver text through composition and input events.
675 if (
676 event.nativeEvent.isComposing ||
677 event.keyCode === 229 ||
678 event.key === "Process" ||
679 event.key === "Unidentified"
680 ) {
681 return;
682 }
683 // Keep plain Escape and Tab in the page; Shift+Escape returns to the app.
684 if (event.key === "Escape" && event.shiftKey) {
685 event.preventDefault();
686 event.stopPropagation();
687 if (action === "down") event.currentTarget.blur();
688 return;
689 }
690 const shortcut = event.ctrlKey || event.metaKey;
691 // Paste arrives as a paste event carrying this device's clipboard. Copy and cut run
692 // in the page, which sends the copied text back. Shift+Insert is the same paste.
693 if (shortcut && event.key.toLowerCase() === "v") return;
694 if (event.shiftKey && event.key === "Insert") return;
695 // Enter carries "\r" like Puppeteer's key table, so forms submit and textareas break lines.
696 const text = shortcut
697 ? undefined
698 : [...event.key].length === 1
699 ? event.key
700 : event.key === "Enter"
701 ? "\r"
702 : undefined;
703 send({
704 type: "key",
705 action,
706 key: event.key,
707 code: event.code,
708 keyCode: event.keyCode,
709 ...(action === "down" && text !== undefined ? { text } : {}),
710 modifiers: previewStreamModifiers(event),
711 });
712 // Some Android keyboards edit the textarea even when keydown is prevented.
713 keySentRef.current =
714 action === "down" &&
715 !shortcut &&
716 (text !== undefined || event.key === "Backspace" || event.key === "Delete");
717 // Shortcuts stay with the app (copy, paste, keybindings); other keys belong to the page.
718 if (shortcut) return;
719 event.preventDefault();
720 event.stopPropagation();
721 };
722
723 const handleInput = (event: FormEvent<HTMLTextAreaElement>) => {
724 const native = event.nativeEvent;
725 if (native instanceof InputEvent && native.isComposing) return;
726 const textarea = event.currentTarget;
727 const inputType = native instanceof InputEvent ? native.inputType : "";
728 if (keySentRef.current) keySentRef.current = false;
729 else if (inputType === "deleteContentBackward") sendKeyPress(BACKSPACE);
730 else if (inputType === "deleteContentForward") sendKeyPress(DELETE);
731 else {
732 const text = textarea.value.replace(/^\u200b/, "");
733 if (text) send({ type: "text", text });
734 }
735 resetInput(textarea);
736 };
737
738 return (
739 <div
740 className={cn("relative flex flex-col overflow-hidden", className)}
741 data-server-browser-surface={tabId}
742 >
743 <div
744 className={cn(
745 "visible relative z-20 flex shrink-0 items-center justify-between gap-2 border-border bg-background px-2 py-1",
746 controlPosition === "bottom" ? "order-last border-t" : "border-b",
747 )}
748 >
749 <span role="status" className="text-xs text-muted-foreground">
750 {previewStreamControlLabel(control)}
751 </span>
752 {control?.canOperate ? (
753 <Button
754 variant="outline"
755 size="xs"
756 disabled={control.controller === "another-viewer"}
757 onClick={() =>
758 send({ type: control.controller === "you" ? "releaseControl" : "takeControl" })
759 }
760 >
761 {control.controller === "you" ? "Release control" : "Take control"}
762 </Button>
763 ) : null}
764 </div>
765 <div className="relative min-h-0 flex-1">
766 <canvas
767 ref={canvasRef}
768 className="block size-full touch-none object-contain"
769 onPointerDown={handlePointer}
770 onPointerMove={handlePointer}
771 onPointerUp={handlePointer}
772 onPointerCancel={handlePointer}
773 // Keeps focus in the page input below and stops native text selection.
774 onMouseDown={(event) => event.preventDefault()}
775 onContextMenu={(event) => event.preventDefault()}
776 />
777 {agentCursor && control?.controller !== "you" ? (
778 <AgentCursorMark
779 {...agentCursor}
780 controller={control?.controller === "agent" ? "agent" : "none"}
781 />
782 ) : null}
783 {/* Focus target for page keyboard input. Pinned top-left so focusing it never
784 scrolls the surface; 16px keeps iOS from zooming the app on focus. */}
785 <textarea
786 ref={inputRef}
787 aria-label="Browser page"
788 aria-description="Press Shift+Escape to leave the browser page."
789 autoCapitalize="off"
790 autoComplete="off"
791 autoCorrect="off"
792 spellCheck={false}
793 disabled={control?.controller !== "you"}
794 defaultValue={INPUT_SENTINEL}
795 // The caret must sit after the sentinel for a deletion to have something to delete.
796 onFocus={(event) => resetInput(event.currentTarget)}
797 className="sr-only top-0 left-0 text-base"
798 onKeyDown={(event) => handleKey("down", event)}
799 onKeyUp={(event) => handleKey("up", event)}
800 onInput={handleInput}
801 onCompositionEnd={(event) => {
802 const text = event.data.replace(/^\u200b/, "");
803 if (text) send({ type: "text", text });
804 resetInput(event.currentTarget);
805 }}
806 // Copying the input would put its sentinel on this device's clipboard.
807 onCopy={(event) => event.preventDefault()}
808 onCut={(event) => event.preventDefault()}
809 onPaste={(event) => {
810 event.preventDefault();
811 const text = event.clipboardData.getData("text/plain");
812 if (text) send({ type: "text", text });
813 }}
814 />
815 {fileChooser && control?.controller === "you" ? (
816 <div
817 className="absolute inset-x-2 top-2 z-10 flex flex-col gap-2 rounded-lg border border-border bg-background p-3 shadow-lg"
818 role="dialog"
819 aria-label="Choose files for the page"
820 >
821 <p className="text-sm">
822 The page asks for {fileChooser.multiple ? "files" : "a file"}.
823 </p>
824 <input
825 ref={fileInputRef}
826 type="file"
827 className="hidden"
828 multiple={fileChooser.multiple}
829 accept={fileChooser.accept}
830 onChange={(event) => {
831 const files = [...(event.currentTarget.files ?? [])];
832 event.currentTarget.value = "";
833 if (files.length > 0) answerFileChooser(files);
834 }}
835 />
836 <div className="flex justify-end gap-2">
837 <Button variant="outline" size="sm" onClick={() => answerFileChooser([])}>
838 Cancel
839 </Button>
840 <Button size="sm" onClick={() => fileInputRef.current?.click()}>
841 Choose {fileChooser.multiple ? "files" : "file"}
842 </Button>
843 </div>
844 </div>
845 ) : null}
846 {control?.dialog ? (
847 <div
848 className="absolute inset-x-2 top-2 z-10 flex flex-col gap-2 rounded-lg border border-border bg-background p-3 shadow-lg"
849 role="dialog"
850 aria-label="Browser dialog"
851 >
852 <p className="break-words text-sm">{control.dialog.message}</p>
853 {control.controller === "you" ? (
854 <>
855 {control.dialog.type === "prompt" ? (
856 <Input
857 aria-label="Dialog response"
858 value={promptText}
859 onChange={(event) => setPromptText(event.target.value)}
860 />
861 ) : null}
862 <div className="flex justify-end gap-2">
863 <Button
864 variant="outline"
865 size="sm"
866 onClick={() => send({ type: "dialog", accept: false })}
867 >
868 Dismiss
869 </Button>
870 <Button
871 size="sm"
872 onClick={() =>
873 send({
874 type: "dialog",
875 accept: true,
876 ...(control.dialog?.type === "prompt" ? { promptText } : {}),
877 })
878 }
879 >
880 Accept
881 </Button>
882 </div>
883 </>
884 ) : (
885 <p className="text-xs text-muted-foreground">Take control to respond.</p>
886 )}
887 </div>
888 ) : null}
889 </div>
890 {visible && hostSetup ? (
891 <div className="visible absolute inset-0 z-20 flex flex-col items-center justify-center gap-3 bg-background p-4 text-center">
892 <p role="alert" className="max-w-sm text-sm text-muted-foreground">
893 {previewStreamHostSetupMessage(hostSetup)}
894 </p>
895 <CommandBlock command={hostSetup.command} className="w-full max-w-md text-left" />
896 <Button variant="outline" size="sm" onClick={() => setHostSetup(null)}>
897 Try again
898 </Button>
899 </div>
900 ) : null}
901 {visible && accessDenied ? (
902 // The page can be invisible beneath an empty or unreachable state; reconnect must remain reachable.
903 <div className="visible absolute inset-0 z-20 flex flex-col items-center justify-center gap-2 bg-background p-3 text-center">
904 <p role="alert" className="text-xs text-muted-foreground">
905 Browser connection was refused.
906 </p>
907 <Button
908 variant="outline"
909 size="sm"
910 onClick={() => {
911 unauthorizedRef.current = 0;
912 setAccessDenied(false);
913 refreshPreviewStreamAccess(environmentId);
914 }}
915 >
916 Reconnect
917 </Button>
918 </div>
919 ) : null}
920 </div>
921 );
922}
923
924interface AgentCursorPlacement {
925 readonly phase: PreviewStreamPointer["phase"];
926 readonly sequence: number;
927 readonly left: number;
928 readonly top: number;
929}
930
931/** Places the agent cursor over the letterboxed frame, the inverse of the viewer's own pointer mapping. */
932function placeAgentCursor(
933 pointer: PreviewStreamPointer,
934 canvas: HTMLCanvasElement | null,
935 viewport: PreviewStreamViewport | null,
936): AgentCursorPlacement | null {
937 if (!canvas || !viewport || canvas.width < 1 || canvas.height < 1) return null;
938 const box = { width: canvas.clientWidth, height: canvas.clientHeight };
939 const fit = Math.min(box.width / canvas.width, box.height / canvas.height);
940 const width = canvas.width * fit;
941 const height = canvas.height * fit;
942 return {
943 phase: pointer.phase,
944 sequence: pointer.sequence,
945 left: (box.width - width) / 2 + (pointer.x * width) / viewport.width,
946 top: (box.height - height) / 2 + (pointer.y * height) / viewport.height,
947 };
948}