1import {
2 createPreviewFramePainter,
3 createPreviewStreamClient,
4 type PreviewStreamClient,
5} from "@t3tools/client-runtime/preview/server-browser-stream";
6import type { EnvironmentId } from "@t3tools/contracts";
7import { useSyncExternalStore } from "react";
8
9import { readPreviewStreamAccess } from "~/state/previewStream";
10
11interface WebKitVideo {
12 webkitSetPresentationMode?: (mode: string) => void;
13 webkitPresentationMode?: string;
14}
15
16interface ActivePictureInPicture {
17 readonly key: string;
18 readonly stop: () => void;
19}
20
21// Owns its stream outside React so the floating window survives panel unmounts.
22let active: ActivePictureInPicture | null = null;
23const listeners = new Set<() => void>();
24
25const emit = () => {
26 for (const listener of listeners) listener();
27};
28
29const subscribe = (listener: () => void) => {
30 listeners.add(listener);
31 return () => listeners.delete(listener);
32};
33
34export const serverPictureInPictureKey = (threadId: string, tabId: string) =>
35 JSON.stringify([threadId, tabId]);
36
37/** Chromium and Safari float a canvas-captured video; Firefox has no API for it. */
38export function supportsServerPictureInPicture(): boolean {
39 if (typeof document === "undefined") return false;
40 if (typeof HTMLCanvasElement.prototype.captureStream !== "function") return false;
41 const video = HTMLVideoElement.prototype as HTMLVideoElement & WebKitVideo;
42 return (
43 (document.pictureInPictureEnabled && typeof video.requestPictureInPicture === "function") ||
44 typeof video.webkitSetPresentationMode === "function"
45 );
46}
47
48export function useServerPictureInPictureKey(): string | null {
49 return useSyncExternalStore(
50 subscribe,
51 () => active?.key ?? null,
52 () => null,
53 );
54}
55
56export function closeServerPictureInPicture(): void {
57 const current = active;
58 active = null;
59 current?.stop();
60 emit();
61}
62
63/** Open during user activation, seeding the video with the currently visible frame. */
64export async function openServerPictureInPicture(input: {
65 readonly environmentId: EnvironmentId;
66 readonly threadId: string;
67 readonly tabId: string;
68 readonly seed: HTMLCanvasElement | null;
69}): Promise<void> {
70 closeServerPictureInPicture();
71 const canvas = document.createElement("canvas");
72 const context = canvas.getContext("2d");
73 const seed = input.seed && input.seed.width > 0 && input.seed.height > 0 ? input.seed : null;
74 canvas.width = seed?.width ?? 640;
75 canvas.height = seed?.height ?? 400;
76 // The capture emits a frame per draw, so the seed is drawn after it starts.
77 const stream = canvas.captureStream();
78 if (seed) context?.drawImage(seed, 0, 0);
79 else context?.fillRect(0, 0, canvas.width, canvas.height);
80 const video = document.createElement("video") as HTMLVideoElement & WebKitVideo;
81 video.muted = true;
82 video.playsInline = true;
83 video.srcObject = stream;
84 // Safari only floats a video that is in the document.
85 video.style.cssText =
86 "position:fixed;right:0;bottom:0;width:1px;height:1px;opacity:0;pointer-events:none";
87 document.body.append(video);
88
89 const painter = createPreviewFramePainter(canvas);
90 let client: PreviewStreamClient | null = null;
91 let stopped = false;
92 let refusals = 0;
93 let retryTimer: ReturnType<typeof setTimeout> | null = null;
94 const onLeave = () => {
95 if (active === current) closeServerPictureInPicture();
96 };
97 const connect = async (refresh: boolean) => {
98 const access = await readPreviewStreamAccess(input.environmentId, refresh);
99 if (stopped) return;
100 if (!access) {
101 closeServerPictureInPicture();
102 return;
103 }
104 client = createPreviewStreamClient(
105 {
106 access,
107 threadId: input.threadId,
108 tabId: input.tabId,
109 maxWidth: 1280,
110 maxHeight: 1280,
111 interactive: false,
112 },
113 {
114 onFrame: (jpeg) => {
115 refusals = 0;
116 painter.paint(jpeg);
117 },
118 onViewport: () => undefined,
119 onConnectedChange: () => undefined,
120 onUnauthorized: () => {
121 if (stopped) return;
122 if (refusals >= 5) {
123 onLeave();
124 return;
125 }
126 retryTimer = setTimeout(() => void connect(true), 1_000 * 2 ** refusals++);
127 },
128 onGone: onLeave,
129 },
130 );
131 };
132
133 const onPresentationMode = () => {
134 if (video.webkitPresentationMode !== "picture-in-picture") onLeave();
135 };
136 const current: ActivePictureInPicture = {
137 key: serverPictureInPictureKey(input.threadId, input.tabId),
138 stop: () => {
139 stopped = true;
140 if (retryTimer !== null) clearTimeout(retryTimer);
141 client?.stop();
142 painter.stop();
143 video.removeEventListener("leavepictureinpicture", onLeave);
144 video.removeEventListener("webkitpresentationmodechanged", onPresentationMode);
145 if (document.pictureInPictureElement === video) {
146 void document.exitPictureInPicture().catch(() => undefined);
147 } else if (video.webkitPresentationMode === "picture-in-picture") {
148 video.webkitSetPresentationMode?.("inline");
149 }
150 for (const track of stream.getTracks()) track.stop();
151 video.srcObject = null;
152 video.remove();
153 },
154 };
155 active = current;
156 emit();
157
158 try {
159 await video.play();
160 if (video.readyState < HTMLMediaElement.HAVE_METADATA) {
161 await new Promise((resolve) =>
162 video.addEventListener("loadedmetadata", resolve, { once: true }),
163 );
164 }
165 if (document.pictureInPictureEnabled && typeof video.requestPictureInPicture === "function") {
166 video.addEventListener("leavepictureinpicture", onLeave);
167 await video.requestPictureInPicture();
168 } else {
169 video.addEventListener("webkitpresentationmodechanged", onPresentationMode);
170 video.webkitSetPresentationMode?.("picture-in-picture");
171 }
172 } catch (error) {
173 onLeave();
174 throw error;
175 }
176 if (active === current) void connect(false);
177}