authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-06-13 21:53:00-07:00
committergravatar for me@theo.liTheo Browne <me@theo.li> 2026-06-13 22:18:37-07:00
logd4ba9e9618065d287bfe1d300e67759c55cb2c16
treeff8b188d8b21f15bef214dbc18ea20ea9767fd7f
parentc1792bc9a6f3c2fa62ee7cb66991b4d6508781a2

Document browser preview phase 0.5 findings and plans

- Add phase 0 and 0.5 ADRs, findings, and spike notes - Update browser preview docs and supporting UI/test files - Record the chosen renderer, automation, recording, tunnel, and input decisions

35 files changed, 5534 insertions(+), 90 deletions(-)

.plans/browser-phase-0-5/006-renderer-failure-boundary.md created+35
......@@ -0,0 +1,35 @@
1# ADR 006: Renderer Failure Boundary
2
3## Status
4
5Accepted, revising ADR 001 terminology and failure handling.
6
7## Decision
8
9Keep the renderer-hosted Electron `<webview>` for the next implementation iteration because it supports normal renderer composition, annotations, and the selected covered-surface recording strategy.
10
11Do not describe this as process-durable. It is:
12
13- durable across React unmounts, route changes, panel closure, and tab switching;
14- not durable across host-renderer reload, host-renderer crash, window destruction, or application restart.
15
16The logical `BrowserSession` and `BrowserTab` records survive host loss. The live page process state does not. When the host disappears, the environment server marks the host unavailable and tabs as lost rather than pretending the DOM, JavaScript heap, sockets, and history can be recovered.
17
18## Evidence
19
20The `renderer-reload` spike created both a renderer `<webview>` and a main-owned `WebContentsView`, modified state in each, and reloaded the host renderer:
21
22- `<webview>` guest id changed from `2` to `4`, the first guest was destroyed, and count reset from `1` to `0`;
23- `WebContentsView` retained id `3`, remained alive, and retained count `1`.
24
25## Product Requirements
26
27- Show an explicit "browser page was lost" state with last known URL, title, screenshot, and artifacts.
28- Offer restart-and-restore as a new browser process, clearly reporting which state cannot be restored.
29- Never silently bind a replacement guest to the old live-process identity.
30- Bind every desktop browser host to one window id and define window-close as host loss.
31- A future multi-window implementation must explicitly move or restart sessions; `<webview>` elements cannot be reparented across renderer documents.
32
33## Rejected for This Iteration
34
35Migrating the collaborative surface to `WebContentsView` solely for renderer-reload survival remains rejected because Phase 0 found hidden/detached recording and overlay composition unsuitable. The trade is now explicit: collaborative composition and recording are prioritized over host-renderer reload survival.
.plans/browser-phase-0-5/007-playwright-injected-runtime.md created+40
......@@ -0,0 +1,40 @@
1# ADR 007: Playwright Injected Runtime
2
3## Status
4
5Accepted as the initial implementation direction; supersedes the bespoke interpretation of ADR 002.
6
7## Decision
8
9Load a version-pinned Playwright injected runtime into each guest frame through the persistent CDP connection. Use it for selector parsing, role/name/text/test-id resolution, shadow DOM traversal, element-state checks, hit-target checks, and locator re-resolution.
10
11T3 continues to own:
12
13- browser sessions, hosts, tabs, command queues, authorization, and cancellation;
14- CDP target discovery and per-frame execution contexts;
15- native input dispatch, navigation, dialogs, downloads, artifacts, and remote reachability;
16- the stable T3 automation contract exposed to agents.
17
18Do not expose Playwright internal objects directly as the product API.
19
20## Evidence
21
22The `injected-runtime` spike extracted the runtime bundled with installed Playwright 1.60.0 and evaluated it inside the real Electron guest over CDP:
23
24- injected source size: `307,101` bytes;
25- install time: approximately `6.5–8.5 ms` across the recorded runs;
26- role/name locator matched the button;
27- role/name locator pierced an open shadow root;
28- the same locator re-resolved after the input element was replaced;
29- same-origin frames still require a separate injected runtime and routing context.
30
31## Constraints
32
33- Playwright's injected runtime is internal and version-coupled. Pin its version and run compatibility fixtures on upgrades.
34- OOPIFs remain separate CDP targets. The Browser Control Service must enable target auto-attach and maintain frame-to-session routing.
35- Locators are durable query descriptions resolved at action time. Snapshot node references are optional, short-lived accelerators only.
36- The T3 adapter must compare key behavior against real Playwright tests for actionability, shadow DOM, frames, rerenders, and strictness.
37
38## Fallback
39
40If a future Playwright release makes the injected runtime impractical to consume, retain the T3 contract and replace only the internal locator provider. Do not leak the vendored runtime across architecture boundaries.
.plans/browser-phase-0-5/008-recording-endurance.md created+32
......@@ -0,0 +1,32 @@
1# ADR 008: Covered Recording Is Provisional
2
3## Status
4
5Accepted with concurrency and platform gates.
6
7## Decision
8
9Keep CDP screencast plus Chromium MediaRecorder and the covered full-size parking surface for the first recording implementation.
10
11Default to one actively recorded tab per desktop window until multi-tab budgets are measured. Hidden idle tabs remain offscreen; hidden recording tabs remain composited and covered.
12
13## Evidence
14
15The `recording-endurance` spike recorded a `1600×1200` covered webview for ten seconds at a requested 12 fps:
16
17- 111–112 sampled frames, approximately 11.1–11.2 fps across the recorded runs;
18- 9.98-second seekable H.264 MP4;
19- main-process CPU sample approximately 3.5–3.8%;
20- observed renderer working sets approximately 146 MiB and 99 MiB, plus GPU working set approximately 149 MiB.
21
22This is stronger than the original 800×600 proof, but it is not a production concurrency budget.
23
24## Required Before Raising Concurrency
25
26- 30-minute soak with active timers, animation, HMR, and navigation;
27- two and four simultaneous recording tabs;
28- Retina device-scale-factor coverage;
29- window resize, minimization, macOS Spaces, sleep/wake, and display changes;
30- explicit `backgroundThrottling` policy and regression tests;
31- proof that the cover cannot receive or leak pointer input to the parked guest;
32- bounded frame queues with drop metrics and encoder-failure finalization.
.plans/browser-phase-0-5/009-loopback-threat-model.md created+33
......@@ -0,0 +1,33 @@
1# ADR 009: Loopback Is Not an Authorization Boundary
2
3## Status
4
5Accepted, revising ADR 004.
6
7## Threat
8
9A listener bound to `127.0.0.1` can be reached by any local process. Browser content can also cause cross-origin requests to it even when CORS prevents reading the response. Possession of a loopback port therefore cannot authorize access to a remote environment.
10
11## Evidence
12
13The `run-tunnel-security-spike.mjs` probe loaded an unrelated page in Chromium and issued a `no-cors` request to another loopback port. The request reached the preview listener. The request did not include an `Origin` header, so origin-header validation alone would not reject it.
14
15The same probe confirmed:
16
17- local storage survives when the exact loopback authority remains stable;
18- changing only the port creates a new origin and loses that storage;
19- an absolute HTTPS redirect passes through unchanged and escapes the loopback authority.
20
21## Decision
22
23- Allocate one stable local authority per environment/project/target tuple for the lifetime of its origin grant.
24- Treat the authority as a browser capability, not a random open port.
25- The desktop browser host must restrict navigation to authorities granted to that browser session and prevent unrelated tabs from using preview authorities.
26- Environment tunnel grants remain scoped to environment, session, target host, and target port, but server-side grants do not protect the desktop listener from local callers.
27- Add a desktop-side connection broker that verifies the connecting browser guest belongs to the authorized session. A bare TCP listener without browser attribution is insufficient for the final design.
28- Absolute redirects, OAuth callbacks, HTTPS upstreams, service workers, and configured public origins require explicit compatibility policy rather than a "no rewriting" guarantee.
29- Replace one-WebSocket-per-TCP-stream before relay rollout unless load testing proves the relay limits acceptable. The target design is bounded multiplexing over dedicated authenticated tunnel connections.
30
31## Open Implementation Choice
32
33The browser-attribution mechanism needs a dedicated spike. Candidates include an Electron protocol/proxy integration, per-session proxy configuration, or another browser-mediated connection path that preserves ordinary application semantics while identifying the requesting guest. Port secrecy is explicitly rejected.
.plans/browser-phase-0-5/010-human-input-interruption.md created+23
......@@ -0,0 +1,23 @@
1# ADR 010: Human Input Interruption
2
3## Status
4
5Accepted for the initial lease implementation, with regression coverage.
6
7## Decision
8
9Observe native user keyboard and pointer activity at the Electron guest/window boundary and revoke or pause the conflicting automation lease. Do not implement a timing-based suppression window unless a supported platform demonstrates that CDP-dispatched input is reported through the same native event path.
10
11## Evidence
12
13The `input-origin` spike attached Electron's `before-input-event` listener to the guest and dispatched keyboard input through CDP. No `before-input-event` was emitted.
14
15This contradicts the assumption that CDP keyboard input is necessarily indistinguishable from human input at this boundary. It does not prove every Electron version, platform, input method, or pointer path behaves identically.
16
17## Requirements
18
19- Cover keyboard, pointer, wheel, touch, IME, accessibility input, and DevTools interaction where available.
20- Record the interruption reason and the command that was cancelled.
21- Serialize lease transition and command cancellation so a late automation event cannot land after the user takes control.
22- Keep a regression fixture for CDP keyboard and mouse dispatch on every supported desktop platform.
23- Remove the undefined `shared` controller mode. Initial modes are `human`, `agent`, and `none`.
.plans/browser-phase-0-5/README.md created+23
......@@ -0,0 +1,23 @@
1# Browser Phase 0.5
2
3Phase 0.5 closes the production-risk questions raised after the original browser spikes. It does not add application behavior. It expands the executable harness and revises the architecture decisions before session implementation begins.
4
5## Commands
6
7```bash
8node .plans/browser-phase-0/spikes/run-electron-spikes.mjs renderer-reload
9node .plans/browser-phase-0/spikes/run-electron-spikes.mjs injected-runtime
10node .plans/browser-phase-0/spikes/run-electron-spikes.mjs recording-endurance
11node .plans/browser-phase-0/spikes/run-electron-spikes.mjs input-origin
12node .plans/browser-phase-0/spikes/run-tunnel-security-spike.mjs
13```
14
15## Decisions
16
17- `006-renderer-failure-boundary.md`: keep `<webview>` for this iteration, but describe it as route-durable rather than process-durable and specify session-loss behavior.
18- `007-playwright-injected-runtime.md`: use Playwright's injected runtime as the initial semantic locator/actionability substrate, behind a T3-owned adapter and compatibility tests.
19- `008-recording-endurance.md`: retain covered parking provisionally, with a one-recording default and further soak/platform tests required before widening concurrency.
20- `009-loopback-threat-model.md`: a plain loopback listener is not an authorization boundary; stable origins and a browser-enforced capability authority are required.
21- `010-human-input-interruption.md`: Electron `before-input-event` did not observe CDP keyboard dispatch in the tested guest, so human interruption can start with native input observation without a dispatch suppression window, while retaining regression coverage.
22
23See `findings.md` for measured results and the resulting roadmap changes.
.plans/browser-phase-0-5/findings.md created+27
......@@ -0,0 +1,27 @@
1# Phase 0.5 Findings
2
3## Executive Decisions
4
51. The selected `<webview>` is route-durable, not process-durable. Renderer reload destroys its guest and state; `WebContentsView` survives that specific failure boundary.
62. Use Playwright's injected runtime behind the T3 Browser Control Service instead of implementing locator and actionability semantics from scratch.
73. Use locator descriptions that re-resolve at action time. Treat snapshot element references as ephemeral.
84. Covered recording remains viable at 1600×1200 for one tab, but concurrency and long-duration operation remain gated.
95. A loopback port is not an authorization capability. Stable origin, browser attribution, and bounded tunnel multiplexing are required before remote preview ships.
106. CDP keyboard dispatch did not trigger Electron `before-input-event` in the tested guest. Start with native input interruption and no timing suppression hack.
11
12## Roadmap Changes
13
14- Add host loss, window identity, session-loss UX, annotation preservation, port discovery, and guest hardening to the session foundation phase.
15- Put persistent CDP, Playwright injected-runtime integration, console buffers, and network buffers in the automation phase.
16- Add OOPIF target routing and per-frame injected runtimes to the automation acceptance criteria.
17- Move remote preview behind a second browser-attribution spike; raw TCP transport feasibility alone is insufficient.
18- Keep one active recording per window as the initial policy.
19- Remove `shared` controller mode until concurrency semantics exist.
20
21## Remaining Open Questions
22
23- Browser-attributed local tunnel ingress that preserves application origin behavior.
24- Multi-tab recording budgets and 30-minute platform soak results.
25- Pointer/touch interruption behavior across macOS, Windows, and Linux.
26- Recovery fidelity after renderer/window loss.
27- Web-app viewing UX for separately hosted environment Chromium sessions.
.plans/browser-phase-0/001-browser-host.md created+24
......@@ -0,0 +1,24 @@
1# ADR 001: Durable Renderer-Hosted Webview
2
3## Status
4
5Accepted with the failure-boundary amendment in `.plans/browser-phase-0-5/006-renderer-failure-boundary.md`.
6
7## Decision
8
9Keep Electron `<webview>` as the collaborative browser surface, but move it out of thread panel lifecycle into a route-durable app-level browser host. This does not survive host-renderer reload, renderer crash, owning-window close, or application restart.
10
11The host owns one mounted webview per live browser tab. The visible browser panel supplies bounds and visibility intent. Hiding, switching routes, closing a sheet, or unmounting a panel must not remove the webview element or close its guest `WebContents`.
12
13The desktop main process remains authoritative for navigation, CDP, recording, security policy, and guest `WebContents` validation. The renderer host is responsible only for maintaining the required DOM attachment and reporting presentation bounds.
14
15## Hidden Presentation Policy
16
17- Visible tab: place the webview at the panel bounds.
18- Hidden idle tab: park it offscreen at its preserved viewport size; automation remains available but continuous screencast is not expected.
19- Hidden recording tab: place it on a full-size in-window parking surface covered by an opaque application layer so Chromium continues producing compositor frames.
20- Suspended tab: explicit lifecycle transition that may destroy and later reload the page; never perform silently while controlled or recording.
21
22## Rejected Alternative
23
24`WebContentsView` is not selected for this iteration. It is durable and Playwright-visible, but hidden/detached capture semantics failed the recording requirements and its native stacking model complicates overlays and clipping.
.plans/browser-phase-0/002-automation-adapter.md created+33
......@@ -0,0 +1,33 @@
1# ADR 002: Browser-Client-Style Semantic CDP Adapter
2
3## Status
4
5Superseded in implementation detail by `.plans/browser-phase-0-5/007-playwright-injected-runtime.md`.
6
7## Decision
8
9Build the primary automation adapter over a persistent Electron debugger connection using CDP accessibility, DOM, runtime, page, network, and input domains.
10
11The adapter should offer Playwright-like concepts—role/name locators, text locators, frames, auto-waiting, stale references, dialogs, downloads, and actionability checks—but it must not depend on Playwright owning the browser or recognizing the guest as a `Page`.
12
13Use two layers:
14
151. main-process CDP session and command/event transport
162. the version-pinned Playwright injected runtime behind a T3-owned compatibility adapter
17
18Locator descriptions re-resolve at action time. Snapshot-scoped backend-node references are optional and ephemeral.
19
20## Playwright Role
21
22Use Playwright as:
23
24- a behavior reference
25- a test oracle against standalone test pages
26- the initial injected selector/actionability runtime
27- an optional adapter for future environment-hosted Chromium sessions
28
29Do not expose a production Electron remote-debugging port merely to attach Playwright.
30
31## agent-browser Role
32
33agent-browser may be supported through a compatible CLI or command adapter backed by the same Browser Control Service. It must not launch an unrelated browser for collaborative preview sessions.
.plans/browser-phase-0/003-recording.md created+33
......@@ -0,0 +1,33 @@
1# ADR 003: CDP Capture with Chromium MediaRecorder
2
3## Status
4
5Accepted with the concurrency and soak gates in `.plans/browser-phase-0-5/008-recording-endurance.md`.
6
7## Decision
8
9Capture the collaborative guest page with `Page.startScreencast`, sample frames at the configured recording frame rate, draw them into an isolated recording canvas, and encode through Chromium MediaRecorder.
10
11Initial artifact format:
12
13- H.264 MP4
14- `video/mp4;codecs=avc1.42E01E`
15- default 12 fps
16- default viewport resolution, bounded by recording policy
17
18Encoding runs in a dedicated hidden renderer or utility surface, not in the primary React render path. Frame transport must be bounded and may drop intermediate frames under pressure rather than accumulating memory.
19
20## Hidden Recording
21
22An offscreen webview does not continuously produce screencast frames. While recording a hidden tab, the browser host temporarily places the full-size webview on a covered parking surface. The user continues seeing the normal application UI, not the frame stream.
23
24## Artifact Semantics
25
26Video is paired with structured action events and timestamps. Video is evidence, not browser state and not the interactive UX.
27
28## Rejected Alternatives
29
30- external ffmpeg as a required encoder
31- `desktopCapturer` capture of the whole application window
32- JPEG/video streaming as the browser UI
33- recording only when the preview panel is open
.plans/browser-phase-0/004-remote-preview-tunnel.md created+38
......@@ -0,0 +1,38 @@
1# ADR 004: Raw TCP Preview Tunnel
2
3## Status
4
5Transport accepted; desktop ingress and authorization amended by `.plans/browser-phase-0-5/009-loopback-threat-model.md`.
6
7## Decision
8
9Remote environment ports are exposed to the desktop browser through:
10
11```text
12browser webview
13 -> browser-attributed stable desktop authority
14 -> dedicated authenticated WebSocket tunnel
15 -> environment server tunnel endpoint
16 -> environment loopback TCP target
17```
18
19Multiplex TCP streams over a bounded number of dedicated authenticated tunnel connections before relay rollout. The protocol must preserve backpressure, half-close, cancellation, and connection-specific errors.
20
21The browser navigates to a stable desktop authority, preserving normal origin and root-relative URL behavior. A bare loopback listener is not an authorization boundary, and absolute redirects, OAuth callbacks, and HTTPS upstreams require explicit policy.
22
23## Isolation
24
25Tunnel data uses a dedicated connection and priority class. It must not share the normal RPC/event WebSocket because large responses or stalled browser streams must not delay thread control traffic.
26
27## Authorization
28
29Each listener is backed by a short-lived grant scoped to:
30
31- environment
32- browser session
33- target loopback host
34- target port
35- protocol policy
36- provider/thread authorization where agent-created
37
38The environment server validates the grant before opening any TCP target.
.plans/browser-phase-0/005-desktop-routing.md created+24
......@@ -0,0 +1,24 @@
1# ADR 005: Retain Renderer Transport Relay Initially
2
3## Status
4
5Accepted for Phase 1; revisit after lifecycle migration.
6
7## Decision
8
9Do not add a second environment-server connection from desktop main solely to remove renderer IPC.
10
11The current renderer-mediated transport can remain temporarily:
12
13```text
14environment server -> renderer connection -> typed IPC -> desktop Browser Control Service
15```
16
17The renderer must no longer decide browser existence, visible-only eligibility, or ownership. It relays authenticated commands and browser-host events between transports.
18
19## Rationale
20
21Measured renderer IPC overhead was approximately 0.04 ms at the median and approximately 0.20 ms at p95 above direct CDP for a no-op command. That is negligible compared with page work and remote network latency.
22
23The architectural problem is lifecycle coupling and failure ownership, not local IPC performance. A direct desktop-main network connection may still be introduced later for reliability or background operation, but it is not required to begin the durable session migration.
24
.plans/browser-phase-0/README.md created+33
......@@ -0,0 +1,33 @@
1# Browser Phase 0 Spikes
2
3These spikes validate the architectural decisions recorded beside this file. They exercise Electron 41 against real browser surfaces rather than mocking browser lifecycle or CDP behavior.
4
5## Environment Used for Recorded Results
6
7- macOS 26.5.1, arm64
8- Apple M4 Max, 64 GiB RAM
9- Electron 41.5.0
10- installed Playwright 1.60.0
11- Node.js 25.8.2 orchestration process
12
13Results are documented in `findings.md`. Generated frames, videos, screenshots, logs, and transient JSON files under `results/` are intentionally ignored.
14
15## Commands
16
17```bash
18node .plans/browser-phase-0/spikes/run-electron-spikes.mjs automation
19node .plans/browser-phase-0/spikes/run-electron-spikes.mjs view-automation
20node .plans/browser-phase-0/spikes/run-electron-spikes.mjs hidden
21node .plans/browser-phase-0/spikes/run-electron-spikes.mjs offscreen-recording
22node .plans/browser-phase-0/spikes/run-electron-spikes.mjs covered-recording
23node .plans/browser-phase-0/spikes/run-electron-spikes.mjs media-recorder
24node .plans/browser-phase-0/spikes/run-electron-spikes.mjs latency
25node .plans/browser-phase-0/spikes/run-gateway-spike.mjs
26```
27
28The Electron harness uses the repository's installed Electron and Playwright packages. The gateway harness uses the locked `ws` package already present in the pnpm installation.
29
30## Scope
31
32These are architectural probes, not production modules. They deliberately avoid changing application behavior. Production work must reimplement the selected patterns with typed contracts, Effect services, cancellation, security policy, tests, and resource cleanup.
33
.plans/browser-phase-0/findings.md created+156
......@@ -0,0 +1,156 @@
1# Browser Phase 0 Findings
2
3## Executive Result
4
5Phase 0 resolves the major architectural unknowns:
6
71. Keep a real Electron `<webview>` as the collaborative browser surface for the next iteration.
82. Move that webview into a durable app-level host independent of panel and route mounting.
93. Build a browser-client-style semantic automation adapter over persistent CDP, not direct Playwright attachment.
104. Use CDP screencast for capture and Chromium MediaRecorder for seekable H.264 MP4 encoding.
115. Use a desktop loopback listener plus a dedicated authenticated raw-TCP-over-WebSocket tunnel for remote environment ports.
126. Keep the renderer as the desktop/server transport relay initially; remove its browser lifecycle authority, not the relay itself.
13
14## Automation Attachment
15
16### Existing `<webview>`
17
18Electron remote debugging exposed two targets:
19
20- host renderer: target type `page`
21- guest browser: target type `webview`
22
23Playwright `chromium.connectOverCDP()` returned only the host renderer as a `Page`. Connecting directly to the guest target's debugger WebSocket succeeded at the transport level but produced zero Playwright pages.
24
25Conclusion: direct Playwright locators cannot drive the current guest webview through the supported `connectOverCDP` API.
26
27### Semantic CDP adapter
28
29The spike used:
30
31- `Accessibility.getFullAXTree`
32- role and accessible-name matching
33- `DOM.getBoxModel`
34- `Input.dispatchMouseEvent`
35- `DOM.resolveNode`
36- `Runtime.callFunctionOn`
37
38It located and clicked a button by role/name and filled a textbox by role/name against the exact guest page. The resulting page state was correct.
39
40Conclusion: a browser-client-style adapter can provide semantic locators without Playwright owning or directly attaching to the guest page.
41
42### `WebContentsView` comparison
43
44A main-process-owned `WebContentsView` appears as a normal Playwright `Page`; Playwright role and textbox locators succeeded.
45
46However:
47
48- hidden or detached view capture was inconsistent or unavailable
49- detached `capturePage()` reported no current display surface
50- detached CDP screenshot could hang without an explicit timeout
51- hidden/detached CDP screencast emitted zero frames
52- native child views add clipping, stacking, and renderer-overlay integration costs
53
54Conclusion: do not migrate the collaborative browser to `WebContentsView` in this iteration merely to obtain direct Playwright attachment.
55
56## Hidden Lifecycle
57
58For the existing `<webview>`:
59
60- the window was hidden for 1.5 seconds
61- the page timer advanced 15 ticks
62- semantic automation incremented page state
63- `capturePage()` returned a 1600×1200 Retina capture
64- the page remained alive and controllable
65
66Observed process snapshot for the minimal two-renderer harness:
67
68- main-process private memory: approximately 43 MiB
69- Electron Browser working set: approximately 172 MiB
70- GPU process working set: approximately 100 MiB
71- host renderer working set: approximately 88 MiB
72- guest renderer working set: approximately 93 MiB
73- measured main-process CPU during the 1.5-second hidden interval: approximately 0.04%
74
75These are development-machine snapshots, not production budgets. They prove lifecycle behavior and establish that each retained page has meaningful renderer memory cost.
76
77## Recording
78
79### CDP screencast behavior
80
81`Page.startScreencast` against the guest webview produced frames while:
82
83- the whole BrowserWindow was hidden
84- the webview remained full-size and was covered by an opaque renderer element
85
86Moving the webview offscreen reduced the stream to one frame. Therefore hidden idle parking and hidden recording require different presentation policies.
87
88### External ffmpeg proof
89
90The initial proof sampled CDP frames at 12 fps and encoded H.264 MP4 with ffmpeg:
91
92- 28 frames
93- 800×600
94- 12 fps
95- 2.33-second duration
96- valid seekable MP4
97
98This proved the capture pipeline but is not the selected production encoder because shipping an external ffmpeg executable is unnecessary.
99
100### Selected Chromium-native encoder
101
102The final proof sent sampled CDP frames into an isolated canvas and used Chromium MediaRecorder:
103
104- MIME: `video/mp4;codecs=avc1.42E01E`
105- codec: H.264
106- 800×600
107- approximately 11 fps from a requested 12 fps
108- 2.4617-second duration
109- 12,823-byte seekable MP4 artifact
110- no external ffmpeg process used for encoding
111
112Conclusion: select CDP screencast plus Chromium MediaRecorder H.264 MP4 for the initial recording backend.
113
114## Remote Preview Tunnel
115
116Two proxy shapes were tested:
117
1181. HTTP-aware reverse proxy with WebSocket upgrade handling.
1192. Desktop loopback TCP listener forwarding raw streams over a dedicated WebSocket to an environment-side TCP dialer.
120
121Both loaded the HTTP page and passed 100 WebSocket echo round trips. The raw TCP tunnel is selected because it preserves origin paths, root-relative assets, arbitrary HTTP semantics, HMR upgrades, and application WebSockets without rewriting response bodies.
122
123Five local runs of the raw TCP tunnel produced:
124
125- added HTTP median latency: 0.036–0.112 ms; median run 0.049 ms
126- WebSocket median round-trip: 0.075–0.138 ms; median run 0.102 ms
127- WebSocket p95 round-trip: 0.133–0.287 ms; median run 0.151 ms
128
129Network latency will dominate in real remote environments. The local result shows the framing/proxy architecture itself is not a meaningful latency source.
130
131## Desktop Routing Latency
132
133The spike compared 100 no-op CDP evaluations:
134
135- direct main-process CDP call
136- renderer `ipcRenderer.invoke` relay to the same main-process CDP call
137
138Across five runs:
139
140- direct median: 0.058–0.066 ms; median run 0.060 ms
141- renderer-relay median: 0.100 ms
142- direct p95: 0.101–0.115 ms; median run 0.105 ms
143- renderer-relay p95: approximately 0.300 ms
144
145Conclusion: renderer IPC overhead is negligible compared with browser work and remote networking. The renderer should lose lifecycle and ownership authority, but replacing the existing server-to-renderer transport relay is not a Phase 1 performance requirement.
146
147## Explicit Unsupported or Rejected Paths
148
149- Direct Playwright `connectOverCDP` to an Electron `<webview>` guest.
150- Enabling an unauthenticated process-wide Chromium remote-debugging port in production.
151- Switching to `WebContentsView` solely for Playwright compatibility.
152- Recording from an offscreen or detached surface without temporarily restoring a composited parking surface.
153- Using screenshots, JPEG frames, or video as the interactive browser presentation.
154- HTTP path-prefix rewriting as the general remote preview solution.
155- Sending preview tunnel data over the normal control/event WebSocket.
156
.plans/browser-phase-0/results/.gitignore created+3
......@@ -0,0 +1,3 @@
1*
2!.gitignore
3
.plans/browser-phase-0/spikes/electron-webview-bootstrap.cjs created+16
......@@ -0,0 +1,16 @@
1const { app } = require("electron");
2
3const portArgument = process.argv.find((value) => value.startsWith("--port="));
4if (portArgument) {
5 app.commandLine.appendSwitch("remote-debugging-address", "127.0.0.1");
6 app.commandLine.appendSwitch("remote-debugging-port", portArgument.slice("--port=".length));
7}
8app.commandLine.appendSwitch("disable-background-timer-throttling");
9
10app
11 .whenReady()
12 .then(() => import("./electron-webview-host.mjs"))
13 .catch((error) => {
14 console.error(error);
15 app.exit(1);
16 });
.plans/browser-phase-0/spikes/electron-webview-host.mjs created+736
......@@ -0,0 +1,736 @@
1import { spawnSync } from "node:child_process";
2import { appendFileSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
3import { dirname, join } from "node:path";
4import { fileURLToPath } from "node:url";
5
6import electron from "electron";
7
8const { app, BrowserWindow, ipcMain, WebContentsView } = electron;
9
10const args = Object.fromEntries(
11 process.argv
12 .slice(2)
13 .filter((value) => value.startsWith("--"))
14 .map((value) => {
15 const [key, ...rest] = value.slice(2).split("=");
16 return [key, rest.join("=") || "true"];
17 }),
18);
19
20const mode = args.mode ?? "automation";
21const debugPort = Number(args.port ?? 0);
22const outputDirectory = args.output;
23const playwrightCoreBundle = args["playwright-core-bundle"];
24
25function debug(message) {
26 if (!outputDirectory) return;
27 mkdirSync(outputDirectory, { recursive: true });
28 appendFileSync(join(outputDirectory, "debug.log"), `${new Date().toISOString()} ${message}\n`);
29}
30
31debug(`startup electron=${process.versions.electron ?? "missing"}`);
32
33const here = dirname(fileURLToPath(import.meta.url));
34const guestUrl = new URL("./guest.html", import.meta.url).href;
35
36function emitResult(result) {
37 if (outputDirectory) {
38 mkdirSync(outputDirectory, { recursive: true });
39 writeFileSync(join(outputDirectory, "result.json"), JSON.stringify(result, null, 2));
40 }
41 process.stdout.write(`PHASE0_RESULT ${JSON.stringify(result)}\n`);
42}
43
44function attachDebugger(contents) {
45 if (!contents.debugger.isAttached()) contents.debugger.attach("1.3");
46 return (method, params) => contents.debugger.sendCommand(method, params);
47}
48
49async function evaluate(send, expression) {
50 const result = await send("Runtime.evaluate", {
51 expression,
52 awaitPromise: true,
53 returnByValue: true,
54 });
55 if (result.exceptionDetails) {
56 const description = result.exceptionDetails.exception?.description;
57 throw new Error(description ?? result.exceptionDetails.text ?? "Evaluation failed");
58 }
59 return result.result?.value;
60}
61
62function axValue(node, key) {
63 return node[key]?.value ?? null;
64}
65
66async function runSemanticProbe(contents) {
67 const send = attachDebugger(contents);
68 await Promise.all([send("Runtime.enable"), send("DOM.enable"), send("Accessibility.enable")]);
69 const tree = await send("Accessibility.getFullAXTree");
70 const buttonNode = tree.nodes.find(
71 (node) => axValue(node, "role") === "button" && axValue(node, "name") === "Increment count",
72 );
73 const inputNode = tree.nodes.find(
74 (node) => axValue(node, "role") === "textbox" && axValue(node, "name") === "Message",
75 );
76 if (!buttonNode?.backendDOMNodeId || !inputNode?.backendDOMNodeId) {
77 throw new Error("Semantic nodes were not found in the accessibility tree");
78 }
79 const box = await send("DOM.getBoxModel", { backendNodeId: buttonNode.backendDOMNodeId });
80 const quad = box.model.content;
81 const x = (quad[0] + quad[2] + quad[4] + quad[6]) / 4;
82 const y = (quad[1] + quad[3] + quad[5] + quad[7]) / 4;
83 await send("Input.dispatchMouseEvent", { type: "mousePressed", x, y, button: "left", clickCount: 1 });
84 await send("Input.dispatchMouseEvent", { type: "mouseReleased", x, y, button: "left", clickCount: 1 });
85 const resolvedInput = await send("DOM.resolveNode", { backendNodeId: inputNode.backendDOMNodeId });
86 await send("Runtime.callFunctionOn", {
87 objectId: resolvedInput.object.objectId,
88 functionDeclaration: `function(value) {
89 this.focus();
90 const setter = Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value').set;
91 setter.call(this, value);
92 this.dispatchEvent(new Event('input', { bubbles: true }));
93 this.dispatchEvent(new Event('change', { bubbles: true }));
94 }`,
95 arguments: [{ value: "semantic CDP attached" }],
96 });
97 const state = await evaluate(
98 send,
99 `({ count: document.querySelector('#count').textContent, value: document.querySelector('#message').value })`,
100 );
101 return {
102 success: state.count === "1" && state.value === "semantic CDP attached",
103 button: { role: axValue(buttonNode, "role"), name: axValue(buttonNode, "name") },
104 input: { role: axValue(inputNode, "role"), name: axValue(inputNode, "name") },
105 state,
106 };
107}
108
109function readPlaywrightInjectedSource() {
110 if (!playwrightCoreBundle) throw new Error("Playwright core bundle path is required");
111 const bundle = readFileSync(playwrightCoreBundle, "utf8");
112 const marker = "source3 = ";
113 const start = bundle.indexOf(marker);
114 const suffix = ";\n }\n});\n\n// packages/playwright-core/src/server/dom.ts";
115 const end = bundle.indexOf(suffix, start);
116 if (start === -1 || end === -1) throw new Error("Could not locate Playwright injected source");
117 const literal = bundle.slice(start + marker.length, end);
118 return Function(`"use strict"; return (${literal});`)();
119}
120
121async function runInjectedRuntimeSpike(contents) {
122 const send = attachDebugger(contents);
123 await send("Runtime.enable");
124 const injectedSource = readPlaywrightInjectedSource();
125 const installStartedAt = performance.now();
126 const install = await evaluate(
127 send,
128 `(() => {
129 const module = {};
130 ${injectedSource}
131 const injected = new (module.exports.InjectedScript())(globalThis, {
132 isUnderTest: false,
133 sdkLanguage: "javascript",
134 testIdAttributeName: "data-testid",
135 stableRafCount: 2,
136 browserName: "chromium",
137 shouldPrependErrorPrefix: false,
138 isUtilityWorld: true,
139 customEngines: [],
140 });
141 globalThis.__phase05Injected = injected;
142 return {
143 methods: Object.getOwnPropertyNames(Object.getPrototypeOf(injected)).sort(),
144 bytes: ${Buffer.byteLength(injectedSource)},
145 };
146 })()`,
147 );
148 const installMs = performance.now() - installStartedAt;
149 const probe = await evaluate(
150 send,
151 `(async () => {
152 const injected = globalThis.__phase05Injected;
153 const resolve = (selector) => {
154 const parsed = injected.parseSelector(selector);
155 return injected.querySelectorAll(parsed, document.documentElement);
156 };
157 const roleSelector = 'internal:role=button[name="Increment count"i]';
158 const buttonBefore = resolve(roleSelector);
159 const shadow = resolve('internal:role=button[name="Shadow action"i]');
160 const inputLocator = 'internal:role=textbox[name="Message"i]';
161 const inputBefore = resolve(inputLocator)[0];
162 inputBefore.value = "before replacement";
163 document.querySelector("#replace").click();
164 const inputAfter = resolve(inputLocator)[0];
165 inputAfter.value = "after replacement";
166 const frame = document.querySelector("#same-origin-frame");
167 await new Promise((resolveReady) => {
168 if (frame.contentDocument?.readyState === "complete") resolveReady();
169 else frame.addEventListener("load", resolveReady, { once: true });
170 });
171 return {
172 roleMatches: buttonBefore.length,
173 shadowMatches: shadow.length,
174 locatorReresolvedAfterReplacement: inputBefore !== inputAfter && inputAfter.value === "after replacement",
175 sameOriginFrameRequiresSeparateRuntime: frame.contentDocument !== document,
176 };
177 })()`,
178 );
179 emitResult({ mode, install, installMs, probe });
180 app.quit();
181}
182
183let rendererReloadStarted = false;
184let rendererReloadNextGuestResolve;
185
186async function runRendererReloadSpike(window, firstGuest) {
187 rendererReloadStarted = true;
188 const firstSend = attachDebugger(firstGuest);
189 await firstSend("Runtime.enable");
190 await evaluate(firstSend, `document.querySelector("#increment").click()`);
191 const firstState = await evaluate(firstSend, `document.querySelector("#count").textContent`);
192 const firstGuestId = firstGuest.id;
193
194 const nativeView = new WebContentsView({
195 webPreferences: { contextIsolation: true, nodeIntegration: false, partition: "persist:t3-phase05-view" },
196 });
197 nativeView.setBounds({ x: 0, y: 0, width: 1, height: 1 });
198 window.contentView.addChildView(nativeView);
199 await nativeView.webContents.loadFile(join(here, "guest.html"));
200 await nativeView.webContents.executeJavaScript(`document.querySelector("#increment").click()`);
201 const nativeId = nativeView.webContents.id;
202
203 const nextGuest = new Promise((resolveGuest) => {
204 rendererReloadNextGuestResolve = resolveGuest;
205 });
206 await window.webContents.reload();
207 const secondGuest = await Promise.race([
208 nextGuest,
209 new Promise((_, reject) => setTimeout(() => reject(new Error("Replacement webview timed out")), 5_000)),
210 ]);
211 const secondSend = attachDebugger(secondGuest);
212 await secondSend("Runtime.enable");
213 const secondState = await evaluate(secondSend, `document.querySelector("#count").textContent`);
214 const nativeState = await nativeView.webContents.executeJavaScript(
215 `document.querySelector("#count").textContent`,
216 );
217 emitResult({
218 mode,
219 webview: {
220 firstGuestId,
221 secondGuestId: secondGuest.id,
222 firstState,
223 secondState,
224 firstDestroyed: firstGuest.isDestroyed(),
225 survivedRendererReload: firstGuestId === secondGuest.id,
226 },
227 webContentsView: {
228 idBefore: nativeId,
229 idAfter: nativeView.webContents.id,
230 stateAfter: nativeState,
231 destroyed: nativeView.webContents.isDestroyed(),
232 },
233 });
234 app.quit();
235}
236
237async function runInputOriginSpike(contents) {
238 const send = attachDebugger(contents);
239 await send("Input.enable").catch(() => undefined);
240 const beforeInputEvents = [];
241 contents.on("before-input-event", (_event, input) => beforeInputEvents.push(input));
242 await send("Input.dispatchKeyEvent", { type: "keyDown", key: "a", code: "KeyA", text: "a" });
243 await send("Input.dispatchKeyEvent", { type: "keyUp", key: "a", code: "KeyA" });
244 await new Promise((resolveDelay) => setTimeout(resolveDelay, 100));
245 emitResult({
246 mode,
247 beforeInputEvents: beforeInputEvents.map(({ type, key, code, isAutoRepeat }) => ({
248 type,
249 key,
250 code,
251 isAutoRepeat,
252 })),
253 cdpDispatchVisibleAsBeforeInput: beforeInputEvents.length > 0,
254 });
255 app.quit();
256}
257
258async function runHiddenSpike(window, contents) {
259 const send = attachDebugger(contents);
260 await send("Runtime.enable");
261 contents.setBackgroundThrottling(false);
262 const before = await evaluate(send, `({ ticks: Number(document.querySelector('#timer').textContent), count: Number(document.querySelector('#count').textContent) })`);
263 const cpuStart = process.getCPUUsage();
264 window.hide();
265 await new Promise((resolve) => setTimeout(resolve, 1_500));
266 await evaluate(send, `document.querySelector('#increment').click()`);
267 const after = await evaluate(send, `({ ticks: Number(document.querySelector('#timer').textContent), count: Number(document.querySelector('#count').textContent), hidden: document.hidden })`);
268 const image = await contents.capturePage();
269 const processMemory = await process.getProcessMemoryInfo();
270 const cpuUsage = process.getCPUUsage(cpuStart);
271 const metrics = app.getAppMetrics().map((metric) => ({
272 type: metric.type,
273 cpuPercent: metric.cpu.percentCPUUsage,
274 memoryKb: metric.memory.workingSetSize,
275 }));
276 if (outputDirectory) {
277 mkdirSync(outputDirectory, { recursive: true });
278 writeFileSync(join(outputDirectory, "hidden-capture.png"), image.toPNG());
279 }
280 emitResult({
281 mode,
282 before,
283 after,
284 captureSize: image.getSize(),
285 processPrivateKb: processMemory.private,
286 mainProcessCpuPercent: cpuUsage.percentCPUUsage,
287 metrics,
288 });
289 app.quit();
290}
291
292async function runRecordingSpike(window, contents, hideWindow = true) {
293 if (!outputDirectory) throw new Error("recording mode requires --output");
294 mkdirSync(outputDirectory, { recursive: true });
295 const send = attachDebugger(contents);
296 await Promise.all([send("Runtime.enable"), send("Page.enable")]);
297 let frameIndex = 0;
298 const timestamps = [];
299 let lastWrittenTimestamp = Number.NEGATIVE_INFINITY;
300 const onMessage = async (_event, method, params) => {
301 if (method !== "Page.screencastFrame") return;
302 const timestamp = params.metadata?.timestamp ?? 0;
303 if (timestamp - lastWrittenTimestamp < 1 / 12) {
304 await send("Page.screencastFrameAck", { sessionId: params.sessionId });
305 return;
306 }
307 lastWrittenTimestamp = timestamp;
308 frameIndex += 1;
309 timestamps.push(timestamp);
310 writeFileSync(
311 join(outputDirectory, `frame-${String(frameIndex).padStart(4, "0")}.jpg`),
312 Buffer.from(params.data, "base64"),
313 );
314 await send("Page.screencastFrameAck", { sessionId: params.sessionId });
315 };
316 contents.debugger.on("message", onMessage);
317 await send("Page.startScreencast", {
318 format: "jpeg",
319 quality: 80,
320 maxWidth: 800,
321 maxHeight: 600,
322 everyNthFrame: 1,
323 });
324 if (hideWindow) window.hide();
325 await new Promise((resolve) => setTimeout(resolve, 2_500));
326 await send("Page.stopScreencast");
327 contents.debugger.off("message", onMessage);
328
329 const videoPath = join(outputDirectory, "webview-recording.mp4");
330 const ffmpeg = spawnSync(
331 "ffmpeg",
332 [
333 "-y",
334 "-loglevel",
335 "error",
336 "-framerate",
337 "12",
338 "-i",
339 join(outputDirectory, "frame-%04d.jpg"),
340 "-c:v",
341 "libx264",
342 "-pix_fmt",
343 "yuv420p",
344 "-movflags",
345 "+faststart",
346 videoPath,
347 ],
348 { encoding: "utf8" },
349 );
350 const probe = spawnSync(
351 "ffprobe",
352 ["-v", "error", "-show_entries", "format=duration,size", "-of", "json", videoPath],
353 { encoding: "utf8" },
354 );
355 emitResult({
356 mode,
357 frames: frameIndex,
358 firstTimestamp: timestamps.at(0),
359 lastTimestamp: timestamps.at(-1),
360 ffmpegStatus: ffmpeg.status,
361 ffmpegError: ffmpeg.stderr.trim() || null,
362 probe: probe.status === 0 ? JSON.parse(probe.stdout) : { error: probe.stderr.trim() },
363 videoPath,
364 });
365 app.quit();
366}
367
368async function runMediaRecorderSpike(window, contents) {
369 if (!outputDirectory) throw new Error("media-recorder mode requires --output");
370 mkdirSync(outputDirectory, { recursive: true });
371 await window.webContents.executeJavaScript(`(() => {
372 const guest = document.querySelector('webview');
373 guest.style.position = 'fixed';
374 guest.style.left = '0';
375 guest.style.top = '0';
376 guest.style.width = '800px';
377 guest.style.height = '600px';
378 guest.style.zIndex = '1';
379 const cover = document.createElement('div');
380 cover.style.position = 'fixed';
381 cover.style.inset = '0';
382 cover.style.background = '#111827';
383 cover.style.zIndex = '2';
384 document.body.appendChild(cover);
385 })()`);
386 const recordingInfo = await window.webContents.executeJavaScript(
387 `window.phase0.startRecording({ width: 800, height: 600, fps: 12 })`,
388 );
389 const send = attachDebugger(contents);
390 await send("Page.enable");
391 let frameCount = 0;
392 let lastSentTimestamp = Number.NEGATIVE_INFINITY;
393 const onMessage = async (_event, method, params) => {
394 if (method !== "Page.screencastFrame") return;
395 const timestamp = params.metadata?.timestamp ?? 0;
396 if (timestamp - lastSentTimestamp >= 1 / 12) {
397 lastSentTimestamp = timestamp;
398 frameCount += 1;
399 window.webContents.send("phase0:recording-frame", `data:image/jpeg;base64,${params.data}`);
400 }
401 await send("Page.screencastFrameAck", { sessionId: params.sessionId });
402 };
403 contents.debugger.on("message", onMessage);
404 await send("Page.startScreencast", {
405 format: "jpeg",
406 quality: 80,
407 maxWidth: 800,
408 maxHeight: 600,
409 everyNthFrame: 1,
410 });
411 await new Promise((resolve) => setTimeout(resolve, 2_500));
412 await send("Page.stopScreencast");
413 contents.debugger.off("message", onMessage);
414 await new Promise((resolve) => setTimeout(resolve, 250));
415 const recordingResult = await window.webContents.executeJavaScript("window.phase0.stopRecording()");
416 const videoPath = join(
417 outputDirectory,
418 recordingResult.mimeType.startsWith("video/mp4")
419 ? "webview-recording.mp4"
420 : "webview-recording.webm",
421 );
422 writeFileSync(videoPath, Buffer.from(recordingResult.bytes));
423 const probe = spawnSync(
424 "ffprobe",
425 ["-v", "error", "-show_entries", "format=duration,size", "-show_entries", "stream=codec_name,width,height,avg_frame_rate", "-of", "json", videoPath],
426 { encoding: "utf8" },
427 );
428 emitResult({
429 mode,
430 frameCount,
431 recordingInfo,
432 videoPath,
433 bytes: recordingResult.bytes.length,
434 probe: probe.status === 0 ? JSON.parse(probe.stdout) : { error: probe.stderr.trim() },
435 });
436 app.quit();
437}
438
439async function runRecordingEnduranceSpike(window, contents) {
440 if (!outputDirectory) throw new Error("recording-endurance mode requires --output");
441 const width = 1600;
442 const height = 1200;
443 const fps = 12;
444 const durationMs = 10_000;
445 window.setSize(1700, 1300);
446 contents.setBackgroundThrottling(false);
447 await window.webContents.executeJavaScript(`(() => {
448 const guest = document.querySelector('webview');
449 guest.style.position = 'fixed';
450 guest.style.left = '0';
451 guest.style.top = '0';
452 guest.style.width = '${width}px';
453 guest.style.height = '${height}px';
454 guest.style.zIndex = '1';
455 const cover = document.createElement('div');
456 cover.style.position = 'fixed';
457 cover.style.inset = '0';
458 cover.style.background = '#111827';
459 cover.style.zIndex = '2';
460 document.body.appendChild(cover);
461 })()`);
462 const recordingInfo = await window.webContents.executeJavaScript(
463 `window.phase0.startRecording({ width: ${width}, height: ${height}, fps: ${fps} })`,
464 );
465 const send = attachDebugger(contents);
466 await send("Page.enable");
467 let frameCount = 0;
468 let lastSentTimestamp = Number.NEGATIVE_INFINITY;
469 const cpuStart = process.getCPUUsage();
470 const onMessage = async (_event, method, params) => {
471 if (method !== "Page.screencastFrame") return;
472 const timestamp = params.metadata?.timestamp ?? 0;
473 if (timestamp - lastSentTimestamp >= 1 / fps) {
474 lastSentTimestamp = timestamp;
475 frameCount += 1;
476 window.webContents.send("phase0:recording-frame", `data:image/jpeg;base64,${params.data}`);
477 }
478 await send("Page.screencastFrameAck", { sessionId: params.sessionId });
479 };
480 contents.debugger.on("message", onMessage);
481 await send("Page.startScreencast", {
482 format: "jpeg",
483 quality: 75,
484 maxWidth: width,
485 maxHeight: height,
486 everyNthFrame: 1,
487 });
488 await new Promise((resolveDelay) => setTimeout(resolveDelay, durationMs));
489 await send("Page.stopScreencast");
490 contents.debugger.off("message", onMessage);
491 await new Promise((resolveDelay) => setTimeout(resolveDelay, 250));
492 const recordingResult = await window.webContents.executeJavaScript("window.phase0.stopRecording()");
493 const videoPath = join(outputDirectory, "recording-endurance.mp4");
494 writeFileSync(videoPath, Buffer.from(recordingResult.bytes));
495 const probe = spawnSync(
496 "ffprobe",
497 ["-v", "error", "-show_entries", "format=duration,size", "-show_entries", "stream=codec_name,width,height,avg_frame_rate", "-of", "json", videoPath],
498 { encoding: "utf8" },
499 );
500 emitResult({
501 mode,
502 width,
503 height,
504 fps,
505 durationMs,
506 frameCount,
507 achievedFps: frameCount / (durationMs / 1_000),
508 recordingInfo,
509 bytes: recordingResult.bytes.length,
510 mainProcessCpu: process.getCPUUsage(cpuStart),
511 metrics: app.getAppMetrics().map((metric) => ({
512 type: metric.type,
513 cpuPercent: metric.cpu.percentCPUUsage,
514 memoryKb: metric.memory.workingSetSize,
515 })),
516 probe: probe.status === 0 ? JSON.parse(probe.stdout) : { error: probe.stderr.trim() },
517 });
518 app.quit();
519}
520
521function summarizeDurations(values) {
522 const sorted = [...values].sort((left, right) => left - right);
523 const percentile = (fraction) => sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * fraction))];
524 return {
525 count: sorted.length,
526 minMs: sorted[0],
527 medianMs: percentile(0.5),
528 p95Ms: percentile(0.95),
529 maxMs: sorted.at(-1),
530 meanMs: sorted.reduce((sum, value) => sum + value, 0) / sorted.length,
531 };
532}
533
534async function runLatencySpike(window, contents) {
535 const send = attachDebugger(contents);
536 await send("Runtime.enable");
537 const evaluateNoop = () => send("Runtime.evaluate", { expression: "1 + 1", returnByValue: true });
538 for (let index = 0; index < 10; index += 1) await evaluateNoop();
539 const direct = [];
540 for (let index = 0; index < 100; index += 1) {
541 const startedAt = performance.now();
542 await evaluateNoop();
543 direct.push(performance.now() - startedAt);
544 }
545 ipcMain.removeHandler("phase0:cdp-evaluate");
546 ipcMain.handle("phase0:cdp-evaluate", evaluateNoop);
547 const rendererRelay = await window.webContents.executeJavaScript(`(async () => {
548 const durations = [];
549 for (let index = 0; index < 100; index += 1) {
550 const startedAt = performance.now();
551 await window.phase0.invokeCdp();
552 durations.push(performance.now() - startedAt);
553 }
554 return durations;
555 })()`);
556 emitResult({
557 mode,
558 direct: summarizeDurations(direct),
559 rendererRelay: summarizeDurations(rendererRelay),
560 });
561 app.quit();
562}
563
564async function runDetachedViewSpike(window, view, detach) {
565 const contents = view.webContents;
566 const send = attachDebugger(contents);
567 await send("Runtime.enable");
568 contents.setBackgroundThrottling(false);
569 const before = await evaluate(
570 send,
571 `({ ticks: Number(document.querySelector('#timer').textContent), count: Number(document.querySelector('#count').textContent) })`,
572 );
573 if (detach) window.contentView.removeChildView(view);
574 else view.setVisible(false);
575 await new Promise((resolve) => setTimeout(resolve, 1_500));
576 const semanticProbe = await runSemanticProbe(contents);
577 const after = await evaluate(
578 send,
579 `({ ticks: Number(document.querySelector('#timer').textContent), count: Number(document.querySelector('#count').textContent), hidden: document.hidden })`,
580 );
581 let capture;
582 try {
583 const image = await contents.capturePage();
584 if (outputDirectory) writeFileSync(join(outputDirectory, "detached-view-capture.png"), image.toPNG());
585 capture = { success: true, size: image.getSize() };
586 } catch (error) {
587 capture = { success: false, error: error instanceof Error ? error.message : String(error) };
588 }
589 let cdpScreenshot;
590 try {
591 await send("Page.enable");
592 const screenshot = await Promise.race([
593 send("Page.captureScreenshot", {
594 format: "png",
595 captureBeyondViewport: false,
596 fromSurface: false,
597 }),
598 new Promise((_, reject) => setTimeout(() => reject(new Error("CDP screenshot timed out")), 3_000)),
599 ]);
600 const bytes = Buffer.from(screenshot.data, "base64");
601 if (outputDirectory) writeFileSync(join(outputDirectory, "hidden-cdp-screenshot.png"), bytes);
602 cdpScreenshot = { success: true, bytes: bytes.length };
603 } catch (error) {
604 cdpScreenshot = { success: false, error: error instanceof Error ? error.message : String(error) };
605 }
606 emitResult({
607 mode,
608 detach,
609 before,
610 after,
611 semanticProbe,
612 capture,
613 cdpScreenshot,
614 destroyed: contents.isDestroyed(),
615 });
616 contents.close();
617 app.quit();
618}
619
620await app.whenReady();
621debug("app-ready");
622
623const window = new BrowserWindow({
624 width: 900,
625 height: 700,
626 show: true,
627 webPreferences: {
628 contextIsolation: true,
629 nodeIntegration: false,
630 preload: join(here, "host-preload.cjs"),
631 webviewTag: true,
632 },
633});
634
635app.on("web-contents-created", (_event, contents) => {
636 debug(`web-contents-created type=${contents.getType()} id=${contents.id}`);
637 if (contents.getType() !== "webview") return;
638 contents.once("dom-ready", () => {
639 if (mode === "renderer-reload" && rendererReloadStarted) {
640 rendererReloadNextGuestResolve?.(contents);
641 rendererReloadNextGuestResolve = undefined;
642 return;
643 }
644 if (mode === "automation") {
645 runSemanticProbe(contents)
646 .then((semanticProbe) =>
647 emitResult({ mode, debugPort, webContentsId: contents.id, guestUrl, semanticProbe }),
648 )
649 .catch((error) => {
650 emitResult({ mode, error: error instanceof Error ? error.stack : String(error) });
651 app.exit(1);
652 });
653 return;
654 }
655 const operation =
656 mode === "hidden"
657 ? runHiddenSpike(window, contents)
658 : mode === "injected-runtime"
659 ? runInjectedRuntimeSpike(contents)
660 : mode === "renderer-reload"
661 ? runRendererReloadSpike(window, contents)
662 : mode === "input-origin"
663 ? runInputOriginSpike(contents)
664 : mode === "recording-endurance"
665 ? runRecordingEnduranceSpike(window, contents)
666 : mode === "recording"
667 ? runRecordingSpike(window, contents)
668 : mode === "offscreen-recording"
669 ? window.webContents
670 .executeJavaScript(`(() => {
671 const guest = document.querySelector('webview');
672 guest.style.position = 'fixed';
673 guest.style.left = '-10000px';
674 guest.style.top = '0';
675 guest.style.width = '800px';
676 guest.style.height = '600px';
677 })()`)
678 .then(() => runRecordingSpike(window, contents, false))
679 : mode === "covered-recording"
680 ? window.webContents
681 .executeJavaScript(`(() => {
682 const guest = document.querySelector('webview');
683 guest.style.position = 'fixed';
684 guest.style.left = '0';
685 guest.style.top = '0';
686 guest.style.width = '800px';
687 guest.style.height = '600px';
688 guest.style.zIndex = '1';
689 const cover = document.createElement('div');
690 cover.style.position = 'fixed';
691 cover.style.inset = '0';
692 cover.style.background = '#111827';
693 cover.style.zIndex = '2';
694 document.body.appendChild(cover);
695 })()`)
696 .then(() => runRecordingSpike(window, contents, false))
697 : mode === "media-recorder"
698 ? runMediaRecorderSpike(window, contents)
699 : runLatencySpike(window, contents);
700 operation.catch((error) => {
701 emitResult({ mode, error: error instanceof Error ? error.stack : String(error) });
702 app.exit(1);
703 });
704 });
705});
706
707if (mode.startsWith("view-")) {
708 await window.loadURL("data:text/html,<title>Phase0 View Host</title>");
709 const view = new WebContentsView({
710 webPreferences: {
711 contextIsolation: true,
712 nodeIntegration: false,
713 partition: "persist:t3-phase0-view",
714 },
715 });
716 view.setBounds({ x: 0, y: 0, width: 800, height: 600 });
717 window.contentView.addChildView(view);
718 await view.webContents.loadFile(join(here, "guest.html"));
719 if (mode === "view-hidden" || mode === "view-detached") {
720 await runDetachedViewSpike(window, view, mode === "view-detached");
721 } else if (mode === "view-recording") {
722 await runRecordingSpike(window, view.webContents);
723 } else {
724 const semanticProbe = await runSemanticProbe(view.webContents);
725 emitResult({
726 mode,
727 debugPort,
728 webContentsId: view.webContents.id,
729 guestUrl,
730 semanticProbe,
731 });
732 }
733} else {
734 await window.loadFile(join(here, "host.html"));
735 debug(`host-loaded url=${window.webContents.getURL()}`);
736}
.plans/browser-phase-0/spikes/guest.html created+41
......@@ -0,0 +1,41 @@
1<!doctype html>
2<html>
3 <head>
4 <meta charset="utf-8" />
5 <title>Phase0 Guest</title>
6 <style>
7 body { font-family: sans-serif; margin: 24px; }
8 #box { width: 80px; height: 80px; background: #7c3aed; position: relative; animation: move 1s infinite alternate linear; }
9 @keyframes move { from { left: 0; } to { left: 260px; } }
10 </style>
11 </head>
12 <body>
13 <button id="increment" aria-label="Increment count">Count: <span id="count">0</span></button>
14 <label>Message <input id="message" aria-label="Message" /></label>
15 <button id="replace" aria-label="Replace controls">Replace controls</button>
16 <div id="shadow-host"></div>
17 <iframe id="same-origin-frame" title="Same origin frame" srcdoc='<button aria-label="Frame action">Frame action</button>'></iframe>
18 <div id="timer">0</div>
19 <div id="box"></div>
20 <script>
21 let count = 0;
22 let ticks = 0;
23 document.querySelector("#increment").addEventListener("click", () => {
24 count += 1;
25 document.querySelector("#count").textContent = String(count);
26 });
27 document.querySelector("#replace").addEventListener("click", () => {
28 const input = document.querySelector("#message");
29 const replacement = input.cloneNode();
30 replacement.value = input.value;
31 input.replaceWith(replacement);
32 });
33 const shadowRoot = document.querySelector("#shadow-host").attachShadow({ mode: "open" });
34 shadowRoot.innerHTML = '<button aria-label="Shadow action">Shadow action</button>';
35 setInterval(() => {
36 ticks += 1;
37 document.querySelector("#timer").textContent = String(ticks);
38 }, 100);
39 </script>
40 </body>
41</html>
.plans/browser-phase-0/spikes/host-preload.cjs created+51
......@@ -0,0 +1,51 @@
1const { contextBridge, ipcRenderer } = require("electron");
2
3let recording = null;
4
5ipcRenderer.on("phase0:recording-frame", async (_event, dataUrl) => {
6 if (!recording) return;
7 const response = await fetch(dataUrl);
8 const bitmap = await createImageBitmap(await response.blob());
9 recording.context.drawImage(bitmap, 0, 0, recording.canvas.width, recording.canvas.height);
10 bitmap.close();
11});
12
13contextBridge.exposeInMainWorld("phase0", {
14 invokeCdp: () => ipcRenderer.invoke("phase0:cdp-evaluate"),
15 startRecording: ({ width, height, fps }) => {
16 const canvas = document.createElement("canvas");
17 canvas.width = width;
18 canvas.height = height;
19 const context = canvas.getContext("2d");
20 const stream = canvas.captureStream(fps);
21 const mimeTypes = [
22 "video/mp4;codecs=avc1.42E01E",
23 "video/mp4;codecs=h264",
24 "video/mp4",
25 "video/webm;codecs=vp9",
26 "video/webm;codecs=vp8",
27 "video/webm",
28 ];
29 const mimeType = mimeTypes.find((candidate) => MediaRecorder.isTypeSupported(candidate));
30 if (!mimeType) throw new Error("No supported WebM MediaRecorder codec");
31 const recorder = new MediaRecorder(stream, { mimeType, videoBitsPerSecond: 2_000_000 });
32 const chunks = [];
33 recorder.addEventListener("dataavailable", (event) => {
34 if (event.data.size > 0) chunks.push(event.data);
35 });
36 recorder.start(250);
37 recording = { canvas, context, recorder, chunks, mimeType };
38 return { mimeType };
39 },
40 stopRecording: async () => {
41 if (!recording) throw new Error("Recording was not started");
42 const activeRecording = recording;
43 recording = null;
44 await new Promise((resolve) => {
45 activeRecording.recorder.addEventListener("stop", resolve, { once: true });
46 activeRecording.recorder.stop();
47 });
48 const bytes = new Uint8Array(await new Blob(activeRecording.chunks).arrayBuffer());
49 return { mimeType: activeRecording.mimeType, bytes };
50 },
51});
.plans/browser-phase-0/spikes/host.html created+12
......@@ -0,0 +1,12 @@
1<!doctype html>
2<html>
3 <head><meta charset="utf-8" /><title>Phase0 Host</title></head>
4 <body style="margin: 0">
5 <webview
6 id="guest"
7 src="./guest.html"
8 style="width: 800px; height: 600px"
9 partition="persist:t3-phase0"
10 ></webview>
11 </body>
12</html>
.plans/browser-phase-0/spikes/run-electron-spikes.mjs created+191
......@@ -0,0 +1,191 @@
1import { spawn } from "node:child_process";
2import { createRequire } from "node:module";
3import { createServer } from "node:net";
4import { dirname, join, resolve } from "node:path";
5import { fileURLToPath } from "node:url";
6import { readFile, rm } from "node:fs/promises";
7
8const here = dirname(fileURLToPath(import.meta.url));
9const repositoryRoot = resolve(here, "../../..");
10const requireDesktop = createRequire(join(repositoryRoot, "apps/desktop/package.json"));
11const requireWeb = createRequire(join(repositoryRoot, "apps/web/package.json"));
12const electronBinary = requireDesktop("electron");
13const { chromium } = requireWeb("playwright");
14const playwrightPackage = requireWeb.resolve("playwright/package.json");
15const playwrightVersion = JSON.parse(await readFile(playwrightPackage, "utf8")).version;
16const playwrightCoreBundle = resolve(
17 dirname(playwrightPackage),
18 `../../../playwright-core@${playwrightVersion}/node_modules/playwright-core/lib/coreBundle.js`,
19);
20
21async function reservePort() {
22 return new Promise((resolvePort, reject) => {
23 const server = createServer();
24 server.once("error", reject);
25 server.listen(0, "127.0.0.1", () => {
26 const address = server.address();
27 const port = typeof address === "object" && address ? address.port : 0;
28 server.close(() => resolvePort(port));
29 });
30 });
31}
32
33function launchHost({ mode, port, output }) {
34 const childEnvironment = { ...process.env };
35 delete childEnvironment.ELECTRON_RUN_AS_NODE;
36 const command = {
37 electronPath: electronBinary,
38 args: [
39 join(here, "electron-webview-bootstrap.cjs"),
40 `--mode=${mode}`,
41 ...(port ? [`--port=${port}`] : []),
42 ...(output ? [`--output=${output}`] : []),
43 `--playwright-core-bundle=${playwrightCoreBundle}`,
44 ],
45 };
46 const child = spawn(
47 command.electronPath,
48 command.args,
49 { cwd: repositoryRoot, env: childEnvironment, stdio: ["ignore", "pipe", "pipe"] },
50 );
51 child.stderr.on("data", (chunk) => process.stderr.write(chunk));
52 return child;
53}
54
55async function waitForResult(child, output, timeoutMs = 20_000) {
56 const resultPath = join(output, "result.json");
57 const deadline = Date.now() + timeoutMs;
58 while (Date.now() < deadline) {
59 try {
60 return JSON.parse(await readFile(resultPath, "utf8"));
61 } catch {
62 await new Promise((resolveDelay) => setTimeout(resolveDelay, 100));
63 }
64 }
65 child.kill("SIGTERM");
66 throw new Error(`Timed out waiting for ${resultPath}`);
67}
68
69async function runAutomation(hostMode = "automation") {
70 const port = await reservePort();
71 const output = join(repositoryRoot, ".plans/browser-phase-0/results", hostMode);
72 await rm(output, { recursive: true, force: true });
73 const child = launchHost({ mode: hostMode, port, output });
74 const hostResult = await waitForResult(child, output);
75 let browser;
76 try {
77 const endpoint = `http://127.0.0.1:${port}`;
78 let lastError;
79 for (let attempt = 0; attempt < 30; attempt += 1) {
80 try {
81 browser = await chromium.connectOverCDP(endpoint);
82 break;
83 } catch (error) {
84 lastError = error;
85 await new Promise((resolveDelay) => setTimeout(resolveDelay, 100));
86 }
87 }
88 if (!browser) throw lastError;
89 const targets = await fetch(`${endpoint}/json/list`).then((response) => response.json());
90 const pages = browser.contexts().flatMap((context) => context.pages());
91 const pageMetadata = await Promise.all(
92 pages.map(async (page) => ({ url: page.url(), title: await page.title().catch(() => "") })),
93 );
94 const guestPage = pages.find((page, index) => pageMetadata[index]?.title === "Phase0 Guest");
95 if (!guestPage) {
96 const guestTarget = targets.find((target) => target.title === "Phase0 Guest");
97 let directTargetAttachment;
98 if (guestTarget?.webSocketDebuggerUrl) {
99 try {
100 const directBrowser = await chromium.connectOverCDP(guestTarget.webSocketDebuggerUrl);
101 const directPages = directBrowser.contexts().flatMap((context) => context.pages());
102 directTargetAttachment = {
103 success: true,
104 pages: await Promise.all(
105 directPages.map(async (page) => ({ url: page.url(), title: await page.title().catch(() => "") })),
106 ),
107 };
108 await directBrowser.close();
109 } catch (error) {
110 directTargetAttachment = {
111 success: false,
112 error: error instanceof Error ? error.message : String(error),
113 };
114 }
115 }
116 return {
117 hostResult,
118 targets: targets.map(({ id, title, type, url }) => ({ id, title, type, url })),
119 pageMetadata,
120 directTargetAttachment,
121 success: false,
122 reason: "Guest webview was not exposed as a Playwright Page",
123 };
124 }
125 const button = guestPage.getByRole("button", { name: "Increment count" });
126 await button.click();
127 const countAfterClick = await guestPage.locator("#count").textContent();
128 const input = guestPage.getByRole("textbox", { name: "Message" });
129 await input.fill("semantic locator attached");
130 const inputValue = await input.inputValue();
131 const expectedCount = String(Number(hostResult.semanticProbe?.state?.count ?? "0") + 1);
132 return {
133 hostResult,
134 pageMetadata,
135 success: countAfterClick === expectedCount && inputValue === "semantic locator attached",
136 countAfterClick,
137 inputValue,
138 };
139 } finally {
140 await browser?.close().catch(() => undefined);
141 child.kill("SIGTERM");
142 }
143}
144
145async function runManagedMode(mode) {
146 const output = join(repositoryRoot, ".plans/browser-phase-0/results", mode);
147 await rm(output, { recursive: true, force: true });
148 const child = launchHost({ mode, output });
149 return waitForResult(child, output, mode === "recording" ? 30_000 : 20_000);
150}
151
152const requestedMode = process.argv[2] ?? "all";
153const result = {};
154if (requestedMode === "all" || requestedMode === "automation") result.automation = await runAutomation();
155if (requestedMode === "all" || requestedMode === "view-automation") {
156 result.viewAutomation = await runAutomation("view-automation");
157}
158if (requestedMode === "all" || requestedMode === "hidden") result.hidden = await runManagedMode("hidden");
159if (requestedMode === "all" || requestedMode === "recording") result.recording = await runManagedMode("recording");
160if (requestedMode === "all" || requestedMode === "offscreen-recording") {
161 result.offscreenRecording = await runManagedMode("offscreen-recording");
162}
163if (requestedMode === "all" || requestedMode === "covered-recording") {
164 result.coveredRecording = await runManagedMode("covered-recording");
165}
166if (requestedMode === "all" || requestedMode === "media-recorder") {
167 result.mediaRecorder = await runManagedMode("media-recorder");
168}
169if (requestedMode === "all" || requestedMode === "latency") result.latency = await runManagedMode("latency");
170if (requestedMode === "all" || requestedMode === "injected-runtime") {
171 result.injectedRuntime = await runManagedMode("injected-runtime");
172}
173if (requestedMode === "all" || requestedMode === "renderer-reload") {
174 result.rendererReload = await runManagedMode("renderer-reload");
175}
176if (requestedMode === "all" || requestedMode === "input-origin") {
177 result.inputOrigin = await runManagedMode("input-origin");
178}
179if (requestedMode === "all" || requestedMode === "recording-endurance") {
180 result.recordingEndurance = await runManagedMode("recording-endurance");
181}
182if (requestedMode === "all" || requestedMode === "view-hidden") {
183 result.viewHidden = await runManagedMode("view-hidden");
184}
185if (requestedMode === "all" || requestedMode === "view-detached") {
186 result.viewDetached = await runManagedMode("view-detached");
187}
188if (requestedMode === "all" || requestedMode === "view-recording") {
189 result.viewRecording = await runManagedMode("view-recording");
190}
191process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
.plans/browser-phase-0/spikes/run-gateway-spike.mjs created+243
......@@ -0,0 +1,243 @@
1import { createServer, request } from "node:http";
2import { connect as connectTcp, createServer as createTcpServer } from "node:net";
3import { createRequire } from "node:module";
4import { readdirSync } from "node:fs";
5import { join, resolve } from "node:path";
6
7const repositoryRoot = resolve(import.meta.dirname, "../../..");
8const pnpmDirectory = join(repositoryRoot, "node_modules/.pnpm");
9const wsPackageDirectory = readdirSync(pnpmDirectory)
10 .filter((name) => name.startsWith("ws@8."))
11 .sort()
12 .at(-1);
13if (!wsPackageDirectory) throw new Error("The locked ws 8 package is not installed");
14const requireWs = createRequire(join(pnpmDirectory, wsPackageDirectory, "node_modules/ws/package.json"));
15const { WebSocket, WebSocketServer } = requireWs("ws");
16
17function listen(server) {
18 return new Promise((resolvePort, reject) => {
19 server.once("error", reject);
20 server.listen(0, "127.0.0.1", () => {
21 const address = server.address();
22 resolvePort(address.port);
23 });
24 });
25}
26
27function listenTcp(server) {
28 return new Promise((resolvePort, reject) => {
29 server.once("error", reject);
30 server.listen(0, "127.0.0.1", () => resolvePort(server.address().port));
31 });
32}
33
34function close(server) {
35 return new Promise((resolveClose, reject) => server.close((error) => (error ? reject(error) : resolveClose())));
36}
37
38function summarize(values) {
39 const sorted = [...values].sort((left, right) => left - right);
40 const percentile = (fraction) => sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * fraction))];
41 return {
42 count: sorted.length,
43 medianMs: percentile(0.5),
44 p95Ms: percentile(0.95),
45 maxMs: sorted.at(-1),
46 meanMs: sorted.reduce((sum, value) => sum + value, 0) / sorted.length,
47 };
48}
49
50const upstreamWebSockets = new WebSocketServer({ noServer: true });
51upstreamWebSockets.on("connection", (socket) => {
52 socket.on("message", (message) => socket.send(`upstream:${message}`));
53});
54const upstream = createServer((incoming, response) => {
55 response.writeHead(200, {
56 "content-type": incoming.url === "/app" ? "text/html" : "application/json",
57 "x-upstream-host": incoming.headers.host ?? "",
58 });
59 response.end(
60 incoming.url === "/app"
61 ? "<!doctype html><title>Remote Preview</title><h1>remote-loopback-ok</h1>"
62 : JSON.stringify({ ok: true, url: incoming.url }),
63 );
64});
65upstream.on("upgrade", (incoming, socket, head) => {
66 upstreamWebSockets.handleUpgrade(incoming, socket, head, (webSocket) => {
67 upstreamWebSockets.emit("connection", webSocket, incoming);
68 });
69});
70const upstreamPort = await listen(upstream);
71
72const gatewayWebSockets = new WebSocketServer({ noServer: true });
73gatewayWebSockets.on("connection", (clientSocket, incoming) => {
74 const upstreamSocket = new WebSocket(`ws://127.0.0.1:${upstreamPort}${incoming.url}`);
75 const pending = [];
76 clientSocket.on("message", (message) => {
77 if (upstreamSocket.readyState === WebSocket.OPEN) upstreamSocket.send(message);
78 else pending.push(message);
79 });
80 upstreamSocket.on("open", () => pending.splice(0).forEach((message) => upstreamSocket.send(message)));
81 upstreamSocket.on("message", (message) => clientSocket.send(message));
82 const closeBoth = () => {
83 if (clientSocket.readyState < WebSocket.CLOSING) clientSocket.close();
84 if (upstreamSocket.readyState < WebSocket.CLOSING) upstreamSocket.close();
85 };
86 clientSocket.on("close", closeBoth);
87 upstreamSocket.on("close", closeBoth);
88 upstreamSocket.on("error", closeBoth);
89});
90
91const gateway = createServer((incoming, response) => {
92 const upstreamRequest = request(
93 {
94 hostname: "127.0.0.1",
95 port: upstreamPort,
96 path: incoming.url,
97 method: incoming.method,
98 headers: { ...incoming.headers, host: `127.0.0.1:${upstreamPort}` },
99 },
100 (upstreamResponse) => {
101 response.writeHead(upstreamResponse.statusCode ?? 502, upstreamResponse.headers);
102 upstreamResponse.pipe(response);
103 },
104 );
105 upstreamRequest.on("error", (error) => response.destroy(error));
106 incoming.pipe(upstreamRequest);
107});
108gateway.on("upgrade", (incoming, socket, head) => {
109 gatewayWebSockets.handleUpgrade(incoming, socket, head, (webSocket) => {
110 gatewayWebSockets.emit("connection", webSocket, incoming);
111 });
112});
113const gatewayPort = await listen(gateway);
114
115const tunnelWebSockets = new WebSocketServer({ noServer: true });
116tunnelWebSockets.on("connection", (tunnelSocket) => {
117 const upstreamSocket = connectTcp({ host: "127.0.0.1", port: upstreamPort });
118 const pending = [];
119 tunnelSocket.on("message", (message) => {
120 const bytes = Buffer.from(message);
121 if (upstreamSocket.readyState === "open") upstreamSocket.write(bytes);
122 else pending.push(bytes);
123 });
124 upstreamSocket.on("connect", () => pending.splice(0).forEach((bytes) => upstreamSocket.write(bytes)));
125 upstreamSocket.on("data", (bytes) => {
126 if (tunnelSocket.readyState === WebSocket.OPEN) tunnelSocket.send(bytes, { binary: true });
127 });
128 const closeBoth = () => {
129 if (!upstreamSocket.destroyed) upstreamSocket.destroy();
130 if (tunnelSocket.readyState < WebSocket.CLOSING) tunnelSocket.close();
131 };
132 upstreamSocket.on("close", closeBoth);
133 upstreamSocket.on("error", closeBoth);
134 tunnelSocket.on("close", closeBoth);
135});
136const tunnelServer = createServer();
137tunnelServer.on("upgrade", (incoming, socket, head) => {
138 tunnelWebSockets.handleUpgrade(incoming, socket, head, (webSocket) => {
139 tunnelWebSockets.emit("connection", webSocket, incoming);
140 });
141});
142const tunnelPort = await listen(tunnelServer);
143
144const desktopLoopback = createTcpServer((browserSocket) => {
145 const tunnelSocket = new WebSocket(`ws://127.0.0.1:${tunnelPort}/tcp`);
146 const pending = [];
147 browserSocket.on("data", (bytes) => {
148 if (tunnelSocket.readyState === WebSocket.OPEN) tunnelSocket.send(bytes, { binary: true });
149 else pending.push(bytes);
150 });
151 tunnelSocket.on("open", () => pending.splice(0).forEach((bytes) => tunnelSocket.send(bytes, { binary: true })));
152 tunnelSocket.on("message", (message) => browserSocket.write(Buffer.from(message)));
153 const closeBoth = () => {
154 if (!browserSocket.destroyed) browserSocket.destroy();
155 if (tunnelSocket.readyState < WebSocket.CLOSING) tunnelSocket.close();
156 };
157 browserSocket.on("close", closeBoth);
158 browserSocket.on("error", closeBoth);
159 tunnelSocket.on("close", closeBoth);
160 tunnelSocket.on("error", closeBoth);
161});
162const desktopLoopbackPort = await listenTcp(desktopLoopback);
163
164async function measureFetch(url, count) {
165 const durations = [];
166 for (let index = 0; index < count; index += 1) {
167 const startedAt = performance.now();
168 const response = await fetch(url);
169 await response.arrayBuffer();
170 durations.push(performance.now() - startedAt);
171 }
172 return summarize(durations);
173}
174
175function websocketRoundTrips(url, count) {
176 return new Promise((resolveResult, reject) => {
177 const socket = new WebSocket(url);
178 const durations = [];
179 let startedAt = 0;
180 let completed = 0;
181 socket.on("open", () => {
182 startedAt = performance.now();
183 socket.send(String(completed));
184 });
185 socket.on("message", (message) => {
186 durations.push(performance.now() - startedAt);
187 if (String(message) !== `upstream:${completed}`) {
188 reject(new Error(`Unexpected WebSocket response: ${message}`));
189 return;
190 }
191 completed += 1;
192 if (completed === count) {
193 socket.close();
194 resolveResult(summarize(durations));
195 return;
196 }
197 startedAt = performance.now();
198 socket.send(String(completed));
199 });
200 socket.on("error", reject);
201 });
202}
203
204try {
205 const appResponse = await fetch(`http://127.0.0.1:${gatewayPort}/app`);
206 const appBody = await appResponse.text();
207 const directHttp = await measureFetch(`http://127.0.0.1:${upstreamPort}/bench`, 100);
208 const gatewayHttp = await measureFetch(`http://127.0.0.1:${gatewayPort}/bench`, 100);
209 const gatewayWebSocket = await websocketRoundTrips(`ws://127.0.0.1:${gatewayPort}/hmr`, 100);
210 const tunnelAppResponse = await fetch(`http://127.0.0.1:${desktopLoopbackPort}/app`);
211 const tunnelAppBody = await tunnelAppResponse.text();
212 const rawTunnelHttp = await measureFetch(`http://127.0.0.1:${desktopLoopbackPort}/bench`, 100);
213 const rawTunnelWebSocket = await websocketRoundTrips(
214 `ws://127.0.0.1:${desktopLoopbackPort}/hmr`,
215 100,
216 );
217 process.stdout.write(
218 `${JSON.stringify(
219 {
220 success: appBody.includes("remote-loopback-ok"),
221 upstreamPort,
222 gatewayPort,
223 responseHeaders: Object.fromEntries(appResponse.headers),
224 directHttp,
225 gatewayHttp,
226 addedHttpMedianMs: gatewayHttp.medianMs - directHttp.medianMs,
227 gatewayWebSocket,
228 rawTcpTunnel: {
229 success: tunnelAppBody.includes("remote-loopback-ok"),
230 environmentTunnelPort: tunnelPort,
231 desktopLoopbackPort,
232 http: rawTunnelHttp,
233 addedHttpMedianMs: rawTunnelHttp.medianMs - directHttp.medianMs,
234 webSocket: rawTunnelWebSocket,
235 },
236 },
237 null,
238 2,
239 )}\n`,
240 );
241} finally {
242 await Promise.all([close(desktopLoopback), close(tunnelServer), close(gateway), close(upstream)]);
243}
.plans/browser-phase-0/spikes/run-tunnel-security-spike.mjs created+98
......@@ -0,0 +1,98 @@
1import { spawn } from "node:child_process";
2import { createServer } from "node:http";
3import { createRequire } from "node:module";
4import { join, resolve } from "node:path";
5
6const repositoryRoot = resolve(import.meta.dirname, "../../..");
7const requireDesktop = createRequire(join(repositoryRoot, "apps/desktop/package.json"));
8const electronBinary = requireDesktop("electron");
9
10function listen(server) {
11 return new Promise((resolvePort, reject) => {
12 server.once("error", reject);
13 server.listen(0, "127.0.0.1", () => resolvePort(server.address().port));
14 });
15}
16
17function close(server) {
18 return new Promise((resolveClose, reject) =>
19 server.close((error) => (error ? reject(error) : resolveClose())),
20 );
21}
22
23const observed = [];
24const previewHandler = (request, response) => {
25 observed.push({ url: request.url, origin: request.headers.origin ?? null });
26 if (request.url === "/absolute-redirect") {
27 response.writeHead(302, { location: "https://remote.example.test/callback" });
28 response.end();
29 return;
30 }
31 response.writeHead(200, { "content-type": "text/html" });
32 response.end("<!doctype html><title>Preview target</title><h1>preview target</h1>");
33};
34const previewA = createServer(previewHandler);
35const previewB = createServer(previewHandler);
36const malicious = createServer((_request, response) => {
37 response.writeHead(200, { "content-type": "text/html" });
38 response.end("<!doctype html><title>Untrusted page</title><h1>untrusted page</h1>");
39});
40
41const [previewAPort, previewBPort, maliciousPort] = await Promise.all([
42 listen(previewA),
43 listen(previewB),
44 listen(malicious),
45]);
46const previewAUrl = `http://127.0.0.1:${previewAPort}`;
47const previewBUrl = `http://127.0.0.1:${previewBPort}`;
48const maliciousUrl = `http://127.0.0.1:${maliciousPort}`;
49
50const childEnvironment = { ...process.env };
51delete childEnvironment.ELECTRON_RUN_AS_NODE;
52const child = spawn(
53 electronBinary,
54 [join(import.meta.dirname, "tunnel-security-electron.cjs"), maliciousUrl, previewAUrl, previewBUrl],
55 { cwd: repositoryRoot, env: childEnvironment, stdio: ["ignore", "pipe", "pipe"] },
56);
57let stdout = "";
58let stderr = "";
59child.stdout.on("data", (bytes) => {
60 stdout += bytes;
61});
62child.stderr.on("data", (bytes) => {
63 stderr += bytes;
64});
65const exitCode = await new Promise((resolveExit) => child.once("exit", resolveExit));
66
67try {
68 const resultLine = stdout
69 .split("\n")
70 .find((line) => line.startsWith("PHASE05_TUNNEL "));
71 if (!resultLine) throw new Error(`Electron probe did not emit a result: ${stderr || stdout}`);
72 const browserResult = JSON.parse(resultLine.slice("PHASE05_TUNNEL ".length));
73 const redirectResponse = await fetch(`${previewAUrl}/absolute-redirect`, { redirect: "manual" });
74 process.stdout.write(
75 `${JSON.stringify(
76 {
77 success: exitCode === 0,
78 browserResult,
79 untrustedPageCausedLoopbackRequest: observed.some((request) => request.url === "/secret"),
80 loopbackRequestOriginHeader: observed.find((request) => request.url === "/secret")?.origin ?? null,
81 observed,
82 originStickiness: {
83 sameAuthorityPreservedStorage: browserResult.originalPortValue === "sticky",
84 differentPortChangedOrigin: browserResult.otherPortValue === null,
85 },
86 absoluteRedirect: {
87 status: redirectResponse.status,
88 location: redirectResponse.headers.get("location"),
89 escapedLoopbackAuthority: redirectResponse.headers.get("location")?.startsWith("https://") ?? false,
90 },
91 },
92 null,
93 2,
94 )}\n`,
95 );
96} finally {
97 await Promise.all([close(previewA), close(previewB), close(malicious)]);
98}
.plans/browser-phase-0/spikes/tunnel-security-electron.cjs created+16
......@@ -0,0 +1,16 @@
1const { app, BrowserWindow } = require('electron');
2const [maliciousUrl, previewA, previewB] = process.argv.slice(2);
3app.whenReady().then(async () => {
4 const window = new BrowserWindow({ show: false, webPreferences: { contextIsolation: true, nodeIntegration: false } });
5 await window.loadURL(maliciousUrl);
6 const maliciousFetch = await window.webContents.executeJavaScript(`fetch(${JSON.stringify(previewA + '/secret')}, { mode: 'no-cors' }).then(() => 'sent').catch(error => String(error))`);
7 await new Promise(resolve => setTimeout(resolve, 150));
8 await window.loadURL(previewA + '/storage');
9 await window.webContents.executeJavaScript(`localStorage.setItem('phase05', 'sticky')`);
10 await window.loadURL(previewB + '/storage');
11 const otherPortValue = await window.webContents.executeJavaScript(`localStorage.getItem('phase05')`);
12 await window.loadURL(previewA + '/storage');
13 const originalPortValue = await window.webContents.executeJavaScript(`localStorage.getItem('phase05')`);
14 process.stdout.write(`PHASE05_TUNNEL ${JSON.stringify({ maliciousFetch, otherPortValue, originalPortValue })}\n`);
15 app.quit();
16}).catch(error => { console.error(error); app.exit(1); });
.plans/collaborative-browser-architecture.html created+1981
......@@ -0,0 +1,1981 @@
1<!doctype html>
2<html lang="en">
3 <head>
4 <meta charset="utf-8" />
5 <meta name="viewport" content="width=device-width, initial-scale=1" />
6 <title>T3 Collaborative Browser Architecture</title>
7 <style>
8 :root {
9 color-scheme: dark;
10 --ink: #f2eee4;
11 --muted: #aaa99f;
12 --dim: #74766f;
13 --paper: #0b0d0e;
14 --panel: #111416;
15 --panel-2: #171b1d;
16 --line: #303638;
17 --line-bright: #596164;
18 --signal: #ff5e3a;
19 --signal-soft: #ff8d74;
20 --cyan: #69d6d2;
21 --yellow: #e7c75b;
22 --green: #7bcf92;
23 --blue: #78a8e8;
24 --purple: #b19ae8;
25 --danger: #ff6b72;
26 --sidebar: 286px;
27 --radius: 3px;
28 --shadow: 0 18px 70px rgb(0 0 0 / 0.32);
29 }
30
31 * {
32 box-sizing: border-box;
33 }
34
35 html {
36 scroll-behavior: smooth;
37 background: var(--paper);
38 }
39
40 body {
41 margin: 0;
42 color: var(--ink);
43 background:
44 linear-gradient(rgb(255 255 255 / 0.018) 1px, transparent 1px),
45 linear-gradient(90deg, rgb(255 255 255 / 0.018) 1px, transparent 1px),
46 radial-gradient(circle at 82% 4%, rgb(105 214 210 / 0.09), transparent 31rem),
47 radial-gradient(circle at 36% 0%, rgb(255 94 58 / 0.08), transparent 27rem),
48 var(--paper);
49 background-size: 32px 32px, 32px 32px, auto, auto, auto;
50 font-family: "IBM Plex Mono", "SFMono-Regular", Consolas, monospace;
51 line-height: 1.55;
52 }
53
54 button,
55 input {
56 font: inherit;
57 }
58
59 a {
60 color: inherit;
61 }
62
63 .shell {
64 min-height: 100vh;
65 }
66
67 .rail {
68 position: fixed;
69 inset: 0 auto 0 0;
70 z-index: 20;
71 width: var(--sidebar);
72 overflow: auto;
73 border-right: 1px solid var(--line);
74 background: rgb(11 13 14 / 0.94);
75 backdrop-filter: blur(18px);
76 }
77
78 .rail-brand {
79 padding: 28px 24px 22px;
80 border-bottom: 1px solid var(--line);
81 }
82
83 .eyebrow {
84 display: flex;
85 gap: 8px;
86 align-items: center;
87 margin: 0 0 10px;
88 color: var(--signal-soft);
89 font-size: 10px;
90 font-weight: 700;
91 letter-spacing: 0.16em;
92 text-transform: uppercase;
93 }
94
95 .eyebrow::before {
96 width: 18px;
97 height: 2px;
98 background: var(--signal);
99 content: "";
100 }
101
102 .rail h1 {
103 max-width: 220px;
104 margin: 0;
105 font-family: Georgia, "Times New Roman", serif;
106 font-size: 28px;
107 font-weight: 400;
108 line-height: 1.02;
109 letter-spacing: -0.035em;
110 }
111
112 .rail-meta {
113 margin-top: 16px;
114 color: var(--muted);
115 font-size: 11px;
116 }
117
118 .rail-meta strong {
119 color: var(--ink);
120 font-weight: 500;
121 }
122
123 .nav {
124 padding: 18px 12px 40px;
125 }
126
127 .nav-group {
128 margin-bottom: 22px;
129 }
130
131 .nav-label {
132 padding: 0 12px 7px;
133 color: var(--dim);
134 font-size: 9px;
135 font-weight: 700;
136 letter-spacing: 0.16em;
137 text-transform: uppercase;
138 }
139
140 .nav a {
141 display: grid;
142 grid-template-columns: 28px 1fr;
143 gap: 7px;
144 align-items: baseline;
145 padding: 7px 12px;
146 border-left: 2px solid transparent;
147 color: var(--muted);
148 font-size: 11px;
149 text-decoration: none;
150 transition: 120ms ease;
151 }
152
153 .nav a:hover,
154 .nav a.active {
155 border-left-color: var(--signal);
156 color: var(--ink);
157 background: rgb(255 255 255 / 0.035);
158 }
159
160 .nav-index {
161 color: var(--dim);
162 font-variant-numeric: tabular-nums;
163 }
164
165 main {
166 margin-left: var(--sidebar);
167 }
168
169 .hero {
170 position: relative;
171 min-height: 86vh;
172 display: grid;
173 grid-template-columns: minmax(0, 1.25fr) minmax(340px, 0.75fr);
174 gap: 60px;
175 align-items: end;
176 padding: 86px clamp(34px, 6vw, 96px) 68px;
177 overflow: hidden;
178 border-bottom: 1px solid var(--line);
179 }
180
181 .hero::after {
182 position: absolute;
183 top: 10%;
184 right: -11%;
185 width: 48vw;
186 aspect-ratio: 1;
187 border: 1px solid rgb(105 214 210 / 0.24);
188 border-radius: 50%;
189 box-shadow:
190 0 0 0 72px rgb(105 214 210 / 0.025),
191 0 0 0 144px rgb(105 214 210 / 0.018);
192 content: "";
193 pointer-events: none;
194 }
195
196 .hero-copy {
197 position: relative;
198 z-index: 1;
199 }
200
201 .hero-kicker {
202 margin-bottom: 24px;
203 color: var(--cyan);
204 font-size: 11px;
205 letter-spacing: 0.18em;
206 text-transform: uppercase;
207 }
208
209 .hero h2 {
210 max-width: 900px;
211 margin: 0;
212 font-family: Georgia, "Times New Roman", serif;
213 font-size: clamp(56px, 7.2vw, 116px);
214 font-weight: 400;
215 line-height: 0.88;
216 letter-spacing: -0.065em;
217 }
218
219 .hero h2 em {
220 color: var(--signal);
221 font-weight: inherit;
222 }
223
224 .hero-summary {
225 max-width: 760px;
226 margin: 34px 0 0;
227 color: #cbc8be;
228 font-family: Georgia, "Times New Roman", serif;
229 font-size: 19px;
230 line-height: 1.55;
231 }
232
233 .hero-side {
234 position: relative;
235 z-index: 1;
236 display: grid;
237 gap: 12px;
238 }
239
240 .status-card {
241 padding: 20px;
242 border: 1px solid var(--line-bright);
243 background: rgb(17 20 22 / 0.8);
244 box-shadow: var(--shadow);
245 }
246
247 .status-card.primary {
248 border-color: var(--signal);
249 }
250
251 .status-label {
252 color: var(--dim);
253 font-size: 9px;
254 letter-spacing: 0.16em;
255 text-transform: uppercase;
256 }
257
258 .status-value {
259 margin-top: 8px;
260 font-family: Georgia, "Times New Roman", serif;
261 font-size: 24px;
262 line-height: 1.15;
263 }
264
265 .status-detail {
266 margin-top: 10px;
267 color: var(--muted);
268 font-size: 11px;
269 }
270
271 section {
272 padding: 88px clamp(34px, 6vw, 96px);
273 border-bottom: 1px solid var(--line);
274 scroll-margin-top: 24px;
275 }
276
277 .section-head {
278 display: grid;
279 grid-template-columns: 110px minmax(0, 1fr);
280 gap: 22px;
281 margin-bottom: 42px;
282 }
283
284 .section-no {
285 padding-top: 8px;
286 color: var(--signal);
287 font-size: 12px;
288 letter-spacing: 0.14em;
289 }
290
291 .section-title h3 {
292 margin: 0;
293 font-family: Georgia, "Times New Roman", serif;
294 font-size: clamp(37px, 4vw, 64px);
295 font-weight: 400;
296 line-height: 1;
297 letter-spacing: -0.045em;
298 }
299
300 .section-title p {
301 max-width: 790px;
302 margin: 16px 0 0;
303 color: var(--muted);
304 font-size: 13px;
305 }
306
307 .callout {
308 display: grid;
309 grid-template-columns: auto 1fr;
310 gap: 16px;
311 padding: 18px 20px;
312 border: 1px solid var(--line);
313 border-left: 3px solid var(--cyan);
314 background: var(--panel);
315 }
316
317 .callout.warning {
318 border-left-color: var(--yellow);
319 }
320
321 .callout.signal {
322 border-left-color: var(--signal);
323 }
324
325 .callout-mark {
326 color: var(--cyan);
327 font-size: 18px;
328 }
329
330 .callout.warning .callout-mark {
331 color: var(--yellow);
332 }
333
334 .callout.signal .callout-mark {
335 color: var(--signal);
336 }
337
338 .callout strong {
339 display: block;
340 margin-bottom: 4px;
341 color: var(--ink);
342 font-size: 12px;
343 }
344
345 .callout span {
346 color: var(--muted);
347 font-size: 11px;
348 }
349
350 .system-map {
351 margin-top: 30px;
352 padding: 26px;
353 border: 1px solid var(--line-bright);
354 background: #0c0f10;
355 box-shadow: var(--shadow);
356 }
357
358 .map-row {
359 display: grid;
360 grid-template-columns: repeat(5, minmax(130px, 1fr));
361 gap: 22px;
362 align-items: stretch;
363 }
364
365 .map-node {
366 position: relative;
367 min-height: 136px;
368 padding: 17px;
369 border: 1px solid var(--line);
370 background: var(--panel);
371 }
372
373 .map-node:not(:last-child)::after {
374 position: absolute;
375 top: 50%;
376 right: -18px;
377 width: 13px;
378 border-top: 1px solid var(--signal);
379 content: "";
380 }
381
382 .map-node:not(:last-child)::before {
383 position: absolute;
384 top: calc(50% - 3px);
385 right: -19px;
386 width: 6px;
387 height: 6px;
388 border-top: 1px solid var(--signal);
389 border-right: 1px solid var(--signal);
390 transform: rotate(45deg);
391 content: "";
392 }
393
394 .node-tag {
395 color: var(--dim);
396 font-size: 9px;
397 letter-spacing: 0.14em;
398 text-transform: uppercase;
399 }
400
401 .map-node h4 {
402 margin: 10px 0 7px;
403 font-family: Georgia, "Times New Roman", serif;
404 font-size: 20px;
405 font-weight: 400;
406 line-height: 1.08;
407 }
408
409 .map-node p {
410 margin: 0;
411 color: var(--muted);
412 font-size: 10px;
413 }
414
415 .map-node[data-tone="signal"] {
416 border-color: var(--signal);
417 }
418
419 .map-node[data-tone="cyan"] {
420 border-color: var(--cyan);
421 }
422
423 .layer-grid {
424 display: grid;
425 grid-template-columns: repeat(3, minmax(0, 1fr));
426 gap: 15px;
427 }
428
429 .layer-card {
430 position: relative;
431 min-height: 290px;
432 padding: 22px;
433 overflow: hidden;
434 border: 1px solid var(--line);
435 background: var(--panel);
436 transition: 160ms ease;
437 }
438
439 .layer-card::before {
440 position: absolute;
441 inset: 0 auto 0 0;
442 width: 3px;
443 background: var(--accent, var(--signal));
444 content: "";
445 }
446
447 .layer-card:hover {
448 border-color: var(--line-bright);
449 transform: translateY(-3px);
450 box-shadow: 0 14px 42px rgb(0 0 0 / 0.24);
451 }
452
453 .layer-index {
454 color: var(--accent, var(--signal));
455 font-size: 10px;
456 }
457
458 .layer-card h4 {
459 margin: 30px 0 9px;
460 font-family: Georgia, "Times New Roman", serif;
461 font-size: 25px;
462 font-weight: 400;
463 line-height: 1.05;
464 }
465
466 .layer-card > p {
467 min-height: 54px;
468 margin: 0 0 18px;
469 color: var(--muted);
470 font-size: 11px;
471 }
472
473 .compact-list {
474 display: grid;
475 gap: 7px;
476 margin: 0;
477 padding: 0;
478 list-style: none;
479 }
480
481 .compact-list li {
482 position: relative;
483 padding-left: 14px;
484 color: #c8c7be;
485 font-size: 10px;
486 }
487
488 .compact-list li::before {
489 position: absolute;
490 top: 0.65em;
491 left: 0;
492 width: 6px;
493 border-top: 1px solid var(--accent, var(--signal));
494 content: "";
495 }
496
497 .responsibility-grid {
498 display: grid;
499 grid-template-columns: 1fr 1fr;
500 gap: 18px;
501 }
502
503 .responsibility {
504 padding: 24px;
505 border: 1px solid var(--line);
506 background: var(--panel);
507 }
508
509 .responsibility h4 {
510 margin: 0 0 16px;
511 color: var(--accent, var(--signal));
512 font-size: 11px;
513 letter-spacing: 0.12em;
514 text-transform: uppercase;
515 }
516
517 .responsibility p {
518 margin: 0 0 14px;
519 color: var(--muted);
520 font-size: 11px;
521 }
522
523 .flow {
524 display: grid;
525 gap: 1px;
526 border: 1px solid var(--line);
527 background: var(--line);
528 }
529
530 .flow-step {
531 display: grid;
532 grid-template-columns: 90px minmax(170px, 0.75fr) minmax(0, 1.25fr);
533 gap: 18px;
534 align-items: center;
535 padding: 18px 20px;
536 background: var(--panel);
537 }
538
539 .flow-step:hover {
540 background: var(--panel-2);
541 }
542
543 .flow-seq {
544 color: var(--signal);
545 font-size: 10px;
546 letter-spacing: 0.12em;
547 }
548
549 .flow-name {
550 font-family: Georgia, "Times New Roman", serif;
551 font-size: 18px;
552 }
553
554 .flow-detail {
555 color: var(--muted);
556 font-size: 10px;
557 }
558
559 .state-machine {
560 display: grid;
561 grid-template-columns: repeat(5, 1fr);
562 gap: 14px;
563 margin: 26px 0;
564 }
565
566 .state {
567 position: relative;
568 padding: 20px 16px;
569 border: 1px solid var(--line);
570 text-align: center;
571 background: var(--panel);
572 }
573
574 .state:not(:last-child)::after {
575 position: absolute;
576 top: 50%;
577 right: -11px;
578 width: 8px;
579 border-top: 1px solid var(--line-bright);
580 content: "";
581 }
582
583 .state strong {
584 display: block;
585 font-family: Georgia, "Times New Roman", serif;
586 font-size: 17px;
587 font-weight: 400;
588 }
589
590 .state span {
591 display: block;
592 margin-top: 6px;
593 color: var(--dim);
594 font-size: 9px;
595 }
596
597 .state[data-active="true"] {
598 border-color: var(--cyan);
599 box-shadow: inset 0 -2px var(--cyan);
600 }
601
602 .code-block {
603 overflow: auto;
604 padding: 20px;
605 border: 1px solid var(--line);
606 color: #d4d0c5;
607 background: #080a0b;
608 font-size: 10px;
609 line-height: 1.75;
610 white-space: pre;
611 }
612
613 .code-block .key {
614 color: var(--cyan);
615 }
616
617 .code-block .type {
618 color: var(--yellow);
619 }
620
621 .code-block .comment {
622 color: var(--dim);
623 }
624
625 .automation-stack {
626 display: grid;
627 grid-template-columns: minmax(0, 1fr) 340px;
628 gap: 20px;
629 }
630
631 .stack-layers {
632 display: grid;
633 gap: 8px;
634 }
635
636 .stack-layer {
637 display: grid;
638 grid-template-columns: 150px 1fr;
639 gap: 18px;
640 padding: 17px 19px;
641 border: 1px solid var(--line);
642 background: var(--panel);
643 }
644
645 .stack-layer strong {
646 color: var(--accent, var(--signal));
647 font-size: 10px;
648 letter-spacing: 0.08em;
649 text-transform: uppercase;
650 }
651
652 .stack-layer span {
653 color: var(--muted);
654 font-size: 10px;
655 }
656
657 .evidence-card {
658 padding: 22px;
659 border: 1px solid var(--yellow);
660 background:
661 linear-gradient(135deg, rgb(231 199 91 / 0.08), transparent 60%),
662 var(--panel);
663 }
664
665 .evidence-card h4 {
666 margin: 0 0 12px;
667 font-family: Georgia, "Times New Roman", serif;
668 font-size: 25px;
669 font-weight: 400;
670 }
671
672 .evidence-card p {
673 color: var(--muted);
674 font-size: 10px;
675 }
676
677 .recording-pipe {
678 display: grid;
679 gap: 8px;
680 margin-top: 18px;
681 }
682
683 .pipe-item {
684 padding: 10px 12px;
685 border-left: 2px solid var(--yellow);
686 background: rgb(255 255 255 / 0.028);
687 color: #d8d4c8;
688 font-size: 9px;
689 }
690
691 .matrix-wrap {
692 overflow-x: auto;
693 border: 1px solid var(--line);
694 }
695
696 table {
697 width: 100%;
698 min-width: 920px;
699 border-collapse: collapse;
700 background: var(--panel);
701 font-size: 10px;
702 }
703
704 th,
705 td {
706 padding: 14px 16px;
707 border-right: 1px solid var(--line);
708 border-bottom: 1px solid var(--line);
709 text-align: left;
710 vertical-align: top;
711 }
712
713 th {
714 position: sticky;
715 top: 0;
716 color: var(--ink);
717 background: #191d1f;
718 font-size: 9px;
719 letter-spacing: 0.09em;
720 text-transform: uppercase;
721 }
722
723 td {
724 color: var(--muted);
725 }
726
727 td:first-child {
728 color: var(--ink);
729 font-weight: 700;
730 }
731
732 .badge {
733 display: inline-block;
734 padding: 3px 7px;
735 border: 1px solid currentColor;
736 border-radius: 99px;
737 font-size: 8px;
738 font-weight: 700;
739 letter-spacing: 0.08em;
740 text-transform: uppercase;
741 }
742
743 .badge.green {
744 color: var(--green);
745 }
746
747 .badge.yellow {
748 color: var(--yellow);
749 }
750
751 .badge.blue {
752 color: var(--blue);
753 }
754
755 .badge.red {
756 color: var(--danger);
757 }
758
759 .transport-diagram {
760 display: grid;
761 gap: 18px;
762 }
763
764 .transport-lane {
765 display: grid;
766 grid-template-columns: 150px repeat(4, minmax(120px, 1fr));
767 gap: 12px;
768 align-items: center;
769 }
770
771 .lane-label {
772 color: var(--accent, var(--signal));
773 font-size: 10px;
774 font-weight: 700;
775 letter-spacing: 0.09em;
776 text-transform: uppercase;
777 }
778
779 .lane-hop {
780 position: relative;
781 min-height: 84px;
782 display: grid;
783 place-items: center;
784 padding: 12px;
785 border: 1px solid var(--line);
786 color: #d5d2c7;
787 text-align: center;
788 background: var(--panel);
789 font-size: 9px;
790 }
791
792 .lane-hop:not(:last-child)::after {
793 position: absolute;
794 top: 50%;
795 right: -10px;
796 width: 8px;
797 border-top: 1px dashed var(--line-bright);
798 content: "";
799 }
800
801 .ssh-note {
802 display: grid;
803 grid-template-columns: 1fr 1fr;
804 gap: 16px;
805 margin-top: 28px;
806 }
807
808 .decision {
809 padding: 20px;
810 border: 1px solid var(--line);
811 background: var(--panel);
812 }
813
814 .decision.selected {
815 border-color: var(--green);
816 }
817
818 .decision h4 {
819 margin: 0 0 8px;
820 font-family: Georgia, "Times New Roman", serif;
821 font-size: 20px;
822 font-weight: 400;
823 }
824
825 .decision p {
826 margin: 0;
827 color: var(--muted);
828 font-size: 10px;
829 }
830
831 .security-grid {
832 display: grid;
833 grid-template-columns: repeat(4, 1fr);
834 gap: 14px;
835 }
836
837 .security-card {
838 padding: 20px;
839 border-top: 2px solid var(--accent, var(--signal));
840 background: var(--panel);
841 }
842
843 .security-card h4 {
844 margin: 0 0 12px;
845 font-size: 10px;
846 letter-spacing: 0.1em;
847 text-transform: uppercase;
848 }
849
850 .security-card p {
851 margin: 0;
852 color: var(--muted);
853 font-size: 10px;
854 }
855
856 .roadmap {
857 position: relative;
858 display: grid;
859 gap: 14px;
860 }
861
862 .phase {
863 display: grid;
864 grid-template-columns: 110px minmax(200px, 0.7fr) minmax(0, 1.3fr);
865 gap: 20px;
866 padding: 22px;
867 border: 1px solid var(--line);
868 background: var(--panel);
869 }
870
871 .phase-status {
872 color: var(--dim);
873 font-size: 9px;
874 letter-spacing: 0.1em;
875 text-transform: uppercase;
876 }
877
878 .phase-status.done {
879 color: var(--green);
880 }
881
882 .phase h4 {
883 margin: 0;
884 font-family: Georgia, "Times New Roman", serif;
885 font-size: 21px;
886 font-weight: 400;
887 }
888
889 .phase p {
890 margin: 0;
891 color: var(--muted);
892 font-size: 10px;
893 }
894
895 .decision-log {
896 display: grid;
897 grid-template-columns: repeat(2, minmax(0, 1fr));
898 gap: 12px;
899 }
900
901 .decision-log article {
902 padding: 18px 20px;
903 border: 1px solid var(--line);
904 background: var(--panel);
905 }
906
907 .decision-log strong {
908 display: block;
909 color: var(--green);
910 font-size: 9px;
911 letter-spacing: 0.11em;
912 text-transform: uppercase;
913 }
914
915 .decision-log h4 {
916 margin: 11px 0 7px;
917 font-family: Georgia, "Times New Roman", serif;
918 font-size: 20px;
919 font-weight: 400;
920 }
921
922 .decision-log p {
923 margin: 0;
924 color: var(--muted);
925 font-size: 10px;
926 }
927
928 footer {
929 padding: 42px clamp(34px, 6vw, 96px) 70px;
930 color: var(--dim);
931 font-size: 10px;
932 }
933
934 .mobile-menu {
935 display: none;
936 }
937
938 .filter-bar {
939 display: flex;
940 flex-wrap: wrap;
941 gap: 8px;
942 margin-bottom: 22px;
943 }
944
945 .filter-button {
946 padding: 7px 10px;
947 border: 1px solid var(--line);
948 color: var(--muted);
949 background: transparent;
950 cursor: pointer;
951 }
952
953 .filter-button:hover,
954 .filter-button.active {
955 border-color: var(--signal);
956 color: var(--ink);
957 background: rgb(255 94 58 / 0.08);
958 }
959
960 [data-filtered="true"] {
961 display: none !important;
962 }
963
964 @media (max-width: 1180px) {
965 .map-row,
966 .layer-grid {
967 grid-template-columns: repeat(2, minmax(0, 1fr));
968 }
969
970 .map-node::before,
971 .map-node::after {
972 display: none;
973 }
974
975 .security-grid {
976 grid-template-columns: repeat(2, 1fr);
977 }
978
979 .automation-stack {
980 grid-template-columns: 1fr;
981 }
982
983 .transport-lane {
984 grid-template-columns: 120px repeat(4, minmax(105px, 1fr));
985 }
986 }
987
988 @media (max-width: 860px) {
989 :root {
990 --sidebar: 0px;
991 }
992
993 .rail {
994 display: none;
995 }
996
997 main {
998 margin-left: 0;
999 }
1000
1001 .mobile-menu {
1002 position: sticky;
1003 top: 0;
1004 z-index: 30;
1005 display: flex;
1006 justify-content: space-between;
1007 padding: 12px 18px;
1008 border-bottom: 1px solid var(--line);
1009 color: var(--muted);
1010 background: rgb(11 13 14 / 0.92);
1011 backdrop-filter: blur(14px);
1012 font-size: 10px;
1013 }
1014
1015 .hero {
1016 min-height: auto;
1017 grid-template-columns: 1fr;
1018 padding-top: 72px;
1019 }
1020
1021 .hero h2 {
1022 font-size: clamp(52px, 15vw, 84px);
1023 }
1024
1025 .section-head {
1026 grid-template-columns: 1fr;
1027 gap: 8px;
1028 }
1029
1030 .layer-grid,
1031 .responsibility-grid,
1032 .ssh-note,
1033 .decision-log {
1034 grid-template-columns: 1fr;
1035 }
1036
1037 .map-row {
1038 grid-template-columns: 1fr;
1039 }
1040
1041 .state-machine {
1042 grid-template-columns: 1fr 1fr;
1043 }
1044
1045 .state::after {
1046 display: none;
1047 }
1048
1049 .flow-step,
1050 .phase {
1051 grid-template-columns: 1fr;
1052 gap: 8px;
1053 }
1054
1055 .transport-lane {
1056 grid-template-columns: 1fr 1fr;
1057 }
1058
1059 .lane-label {
1060 grid-column: 1 / -1;
1061 }
1062
1063 .security-grid {
1064 grid-template-columns: 1fr;
1065 }
1066 }
1067
1068 @media print {
1069 :root {
1070 color-scheme: light;
1071 --ink: #121416;
1072 --muted: #4c5254;
1073 --dim: #747a7c;
1074 --paper: #fff;
1075 --panel: #f5f4ef;
1076 --panel-2: #eeece6;
1077 --line: #c8c9c4;
1078 --line-bright: #8b8f90;
1079 }
1080
1081 body {
1082 background: white;
1083 }
1084
1085 .rail,
1086 .mobile-menu,
1087 .filter-bar {
1088 display: none;
1089 }
1090
1091 main {
1092 margin-left: 0;
1093 }
1094
1095 .hero {
1096 min-height: auto;
1097 }
1098
1099 section {
1100 break-inside: avoid;
1101 }
1102 }
1103 </style>
1104 </head>
1105 <body>
1106 <div class="shell">
1107 <aside class="rail">
1108 <div class="rail-brand">
1109 <p class="eyebrow">Architecture 01</p>
1110 <h1>Collaborative Browser Runtime</h1>
1111 <div class="rail-meta">
1112 <strong>Status:</strong> Phase 0 decisions pinned<br />
1113 <strong>Branch:</strong> codex/browser-preview-port<br />
1114 <strong>Updated:</strong> June 12, 2026
1115 </div>
1116 </div>
1117 <nav class="nav" aria-label="Architecture sections">
1118 <div class="nav-group">
1119 <div class="nav-label">System</div>
1120 <a href="#north-star"><span class="nav-index">00</span><span>North star</span></a>
1121 <a href="#system-map"><span class="nav-index">01</span><span>System map</span></a>
1122 <a href="#layers"><span class="nav-index">02</span><span>Layer model</span></a>
1123 </div>
1124 <div class="nav-group">
1125 <div class="nav-label">Browser</div>
1126 <a href="#session-model"><span class="nav-index">03</span><span>Session + tab model</span></a>
1127 <a href="#webview-host"><span class="nav-index">04</span><span>Webview host</span></a>
1128 <a href="#lifecycle"><span class="nav-index">05</span><span>Lifecycle</span></a>
1129 </div>
1130 <div class="nav-group">
1131 <div class="nav-label">Control</div>
1132 <a href="#automation"><span class="nav-index">06</span><span>Automation</span></a>
1133 <a href="#evidence"><span class="nav-index">07</span><span>Recording + evidence</span></a>
1134 <a href="#collaboration"><span class="nav-index">08</span><span>Human + agent</span></a>
1135 </div>
1136 <div class="nav-group">
1137 <div class="nav-label">Remote</div>
1138 <a href="#reachability"><span class="nav-index">09</span><span>Reachability matrix</span></a>
1139 <a href="#tunnel"><span class="nav-index">10</span><span>Preview tunnel</span></a>
1140 <a href="#ssh-relay"><span class="nav-index">11</span><span>SSH + T3 Connect</span></a>
1141 </div>
1142 <div class="nav-group">
1143 <div class="nav-label">Delivery</div>
1144 <a href="#security"><span class="nav-index">12</span><span>Security</span></a>
1145 <a href="#roadmap"><span class="nav-index">13</span><span>Roadmap</span></a>
1146 <a href="#decisions"><span class="nav-index">14</span><span>Pinned decisions</span></a>
1147 </div>
1148 </nav>
1149 </aside>
1150
1151 <div class="mobile-menu"><span>T3 / Browser Architecture</span><span>2026.06.12</span></div>
1152
1153 <main>
1154 <header class="hero" id="north-star">
1155 <div class="hero-copy">
1156 <div class="hero-kicker">One real browser / multiple actors / any environment</div>
1157 <h2>A browser that is <em>shared state</em>, not a preview.</h2>
1158 <p class="hero-summary">
1159 T3 owns a route-durable Chromium page that users and agents manipulate together. The page
1160 survives panel changes—not renderer or window death—accepts semantic programmatic control, records evidence, and
1161 reaches development services whether they are local, SSH-hosted, private-networked,
1162 or connected through T3 Connect.
1163 </p>
1164 </div>
1165 <div class="hero-side">
1166 <div class="status-card primary">
1167 <div class="status-label">Core invariant</div>
1168 <div class="status-value">One session. One page state.</div>
1169 <div class="status-detail">
1170 Human interaction, agent automation, recording, DevTools, console, and network
1171 inspection all target the same guest WebContents.
1172 </div>
1173 </div>
1174 <div class="status-card">
1175 <div class="status-label">Selected browser host</div>
1176 <div class="status-value">Route-durable renderer-hosted &lt;webview&gt;</div>
1177 <div class="status-detail">
1178 App-level lifetime; desktop-main control; explicit lost state after renderer reload, crash, or window close.
1179 </div>
1180 </div>
1181 <div class="status-card">
1182 <div class="status-label">Remote model</div>
1183 <div class="status-value">Environment stream gateway</div>
1184 <div class="status-detail">
1185 A stable control endpoint carries authorized streams to arbitrary environment-local
1186 ports. Preview ports do not become public endpoints.
1187 </div>
1188 </div>
1189 </div>
1190 </header>
1191
1192 <section id="system-map">
1193 <div class="section-head">
1194 <div class="section-no">01 / SYSTEM</div>
1195 <div class="section-title">
1196 <h3>End-to-end ownership map</h3>
1197 <p>
1198 The internal browser API is session-oriented and transport-neutral. MCP, skills,
1199 CLI tools, and future adapters are clients of the control service—not alternate
1200 browser implementations.
1201 </p>
1202 </div>
1203 </div>
1204 <div class="callout signal">
1205 <div class="callout-mark">→</div>
1206 <div>
1207 <strong>Authority flows toward the real page; state flows back through revisions and events.</strong>
1208 <span>The renderer may relay transport, but it does not decide whether a browser exists or whether automation is permitted.</span>
1209 </div>
1210 </div>
1211 <div class="system-map" aria-label="End-to-end system map">
1212 <div class="map-row">
1213 <article class="map-node">
1214 <div class="node-tag">Clients</div>
1215 <h4>Agent + Human</h4>
1216 <p>MCP, skill client, CLI, direct UI input, DevTools, recordings.</p>
1217 </article>
1218 <article class="map-node" data-tone="signal">
1219 <div class="node-tag">Environment server</div>
1220 <h4>Browser Control Service</h4>
1221 <p>Authorization, sessions, routing, command queues, control leases, target resolution.</p>
1222 </article>
1223 <article class="map-node">
1224 <div class="node-tag">Desktop transport</div>
1225 <h4>Typed Relay + IPC</h4>
1226 <p>Environment connection enters renderer, then typed IPC reaches desktop main.</p>
1227 </article>
1228 <article class="map-node" data-tone="cyan">
1229 <div class="node-tag">Desktop host</div>
1230 <h4>Electron Browser Host</h4>
1231 <p>Persistent CDP, webview registry, presentation bounds, recording, guest security.</p>
1232 </article>
1233 <article class="map-node" data-tone="signal">
1234 <div class="node-tag">Shared state</div>
1235 <h4>Guest WebContents</h4>
1236 <p>The actual Chromium page both human and agent observe and manipulate.</p>
1237 </article>
1238 </div>
1239 </div>
1240 </section>
1241
1242 <section id="layers">
1243 <div class="section-head">
1244 <div class="section-no">02 / LAYERS</div>
1245 <div class="section-title">
1246 <h3>Six boundaries, one browser product</h3>
1247 <p>
1248 Each layer has a narrow authority. This prevents React visibility, provider quirks,
1249 network reachability, and artifact storage from leaking into the browser core.
1250 </p>
1251 </div>
1252 </div>
1253 <div class="layer-grid">
1254 <article class="layer-card" style="--accent: var(--purple)">
1255 <div class="layer-index">L1 / ENTRY</div>
1256 <h4>Agent Adapters</h4>
1257 <p>Provider-neutral ways to ask for browser work.</p>
1258 <ul class="compact-list">
1259 <li>MCP toolkit</li>
1260 <li>Browser skill + Node helper</li>
1261 <li>CLI / agent-browser compatibility</li>
1262 <li>Internal TypeScript test client</li>
1263 </ul>
1264 </article>
1265 <article class="layer-card" style="--accent: var(--signal)">
1266 <div class="layer-index">L2 / CONTROL</div>
1267 <h4>Browser Control Service</h4>
1268 <p>The internal API and policy boundary.</p>
1269 <ul class="compact-list">
1270 <li>Invocation authorization</li>
1271 <li>Per-tab command serialization</li>
1272 <li>Cancellation + deadlines</li>
1273 <li>Control leases + interruption</li>
1274 </ul>
1275 </article>
1276 <article class="layer-card" style="--accent: var(--yellow)">
1277 <div class="layer-index">L3 / STATE</div>
1278 <h4>Session Registry</h4>
1279 <p>Authoritative logical lifecycle and recovery state.</p>
1280 <ul class="compact-list">
1281 <li>Stable session + tab IDs</li>
1282 <li>Host assignment</li>
1283 <li>Lifecycle revisions</li>
1284 <li>Recording + artifact references</li>
1285 </ul>
1286 </article>
1287 <article class="layer-card" style="--accent: var(--cyan)">
1288 <div class="layer-index">L4 / HOST</div>
1289 <h4>Electron Browser Host</h4>
1290 <p>Owns the relationship between durable DOM webviews and desktop main.</p>
1291 <ul class="compact-list">
1292 <li>App-level mounted webviews</li>
1293 <li>Visible/offscreen/covered parking</li>
1294 <li>Guest WebContents validation</li>
1295 <li>Persistent debugger connections</li>
1296 </ul>
1297 </article>
1298 <article class="layer-card" style="--accent: var(--green)">
1299 <div class="layer-index">L5 / REACH</div>
1300 <h4>Target + Tunnel Service</h4>
1301 <p>Resolves environment-relative targets into browser-reachable loopback URLs.</p>
1302 <ul class="compact-list">
1303 <li>Direct URL classification</li>
1304 <li>Environment-port grants</li>
1305 <li>Raw TCP stream tunnels</li>
1306 <li>SSH, relay, LAN, Tailscale transport</li>
1307 </ul>
1308 </article>
1309 <article class="layer-card" style="--accent: var(--blue)">
1310 <div class="layer-index">L6 / EVIDENCE</div>
1311 <h4>Artifact Service</h4>
1312 <p>Stores durable evidence without blocking control/event traffic.</p>
1313 <ul class="compact-list">
1314 <li>H.264 MP4 recordings</li>
1315 <li>Action timeline + screenshots</li>
1316 <li>Console, exceptions, network</li>
1317 <li>Downloads, HAR, traces</li>
1318 </ul>
1319 </article>
1320 </div>
1321 </section>
1322
1323 <section id="session-model">
1324 <div class="section-head">
1325 <div class="section-no">03 / MODEL</div>
1326 <div class="section-title">
1327 <h3>Session, tab, view, and controller are separate</h3>
1328 <p>
1329 The old preview model conflates “panel mounted” with “browser exists.” The new model
1330 gives each concern its own state and lifecycle.
1331 </p>
1332 </div>
1333 </div>
1334 <div class="responsibility-grid">
1335 <article class="responsibility" style="--accent: var(--signal)">
1336 <h4>BrowserSession</h4>
1337 <p>Logical browser workspace bound to an environment, thread, host, storage partition, and artifact history.</p>
1338 <ul class="compact-list">
1339 <li>Outlives panel and route mounts</li>
1340 <li>Owns active tab selection</li>
1341 <li>Tracks recovery + host availability</li>
1342 <li>May be visible, hidden, detached, suspended, or closed</li>
1343 </ul>
1344 </article>
1345 <article class="responsibility" style="--accent: var(--cyan)">
1346 <h4>BrowserTab</h4>
1347 <p>One real Chromium page with origin, control, navigation, revision, and recording state.</p>
1348 <ul class="compact-list">
1349 <li>Created by human, agent, or system</li>
1350 <li>Can be adopted and handed off</li>
1351 <li>Has one host and one guest WebContents</li>
1352 <li>Element references expire with document revision</li>
1353 </ul>
1354 </article>
1355 </div>
1356 <div class="code-block" aria-label="Core contracts"><span class="comment">// contracts remain schema-only</span>
1357<span class="key">BrowserSession</span> {
1358 id, environmentId, threadId, hostId, activeTabId,
1359 lifecycle: <span class="type">creating | ready | suspended | recovering | closed</span>,
1360 visibility: <span class="type">visible | hidden | detached</span>,
1361 controller: <span class="type">human | agent | none</span>,
1362 partitionId, recordingId, revision
1363}
1364
1365<span class="key">BrowserTab</span> {
1366 id, sessionId, webContentsId, origin,
1367 state: <span class="type">opening | ready | closing | closed</span>,
1368 presentation: <span class="type">visible | parked-idle | parked-recording</span>,
1369 url, title, documentRevision, controller
1370}</div>
1371 </section>
1372
1373 <section id="webview-host">
1374 <div class="section-head">
1375 <div class="section-no">04 / HOST</div>
1376 <div class="section-title">
1377 <h3>The tab manager owns durable webviews</h3>
1378 <p>
1379 Keep the existing real Electron webview, but lift it out of `PreviewPanel` and thread
1380 route lifecycle. A window-level host maintains browser elements for every live tab.
1381 </p>
1382 </div>
1383 </div>
1384 <div class="flow">
1385 <div class="flow-step">
1386 <div class="flow-seq">HOST / 01</div>
1387 <div class="flow-name">Create tab</div>
1388 <div class="flow-detail">Session registry assigns tab ID and partition. App-level host mounts one webview and waits for `dom-ready`.</div>
1389 </div>
1390 <div class="flow-step">
1391 <div class="flow-seq">HOST / 02</div>
1392 <div class="flow-name">Register guest</div>
1393 <div class="flow-detail">Renderer reports `webContentsId`; desktop main verifies type=`webview` and correct host window before accepting it.</div>
1394 </div>
1395 <div class="flow-step">
1396 <div class="flow-seq">HOST / 03</div>
1397 <div class="flow-name">Attach control</div>
1398 <div class="flow-detail">Desktop main creates the persistent CDP connection, navigation listeners, console/network buffers, and recording controller.</div>
1399 </div>
1400 <div class="flow-step">
1401 <div class="flow-seq">HOST / 04</div>
1402 <div class="flow-name">Present surface</div>
1403 <div class="flow-detail">Panel reports bounds. Host places the same webview at visible bounds or an internal parking surface without recreating the page.</div>
1404 </div>
1405 <div class="flow-step">
1406 <div class="flow-seq">HOST / 05</div>
1407 <div class="flow-name">Explicit close</div>
1408 <div class="flow-detail">Only session/tab lifecycle commands close the guest. React unmounts, route changes, and panel closure never imply browser destruction.</div>
1409 </div>
1410 </div>
1411 <div class="ssh-note">
1412 <div class="decision selected">
1413 <span class="badge green">Selected</span>
1414 <h4>Renderer-hosted &lt;webview&gt;</h4>
1415 <p>Preserves the existing composited browser surface, overlays, clipping, capture behavior, and user interaction model.</p>
1416 </div>
1417 <div class="decision">
1418 <span class="badge red">Rejected for now</span>
1419 <h4>WebContentsView</h4>
1420 <p>Playwright-visible and main-owned, but hidden/detached capture failed and native view stacking complicates collaboration UI.</p>
1421 </div>
1422 </div>
1423 </section>
1424
1425 <section id="lifecycle">
1426 <div class="section-head">
1427 <div class="section-no">05 / LIFE</div>
1428 <div class="section-title">
1429 <h3>Presentation state is not process state</h3>
1430 <p>
1431 Tabs remain live while hidden. Resource policy can explicitly suspend inactive tabs,
1432 but never silently evicts a controlled or recording session.
1433 </p>
1434 </div>
1435 </div>
1436 <div class="state-machine">
1437 <div class="state"><strong>Creating</strong><span>webview mounting</span></div>
1438 <div class="state" data-active="true"><strong>Ready</strong><span>visible or parked</span></div>
1439 <div class="state"><strong>Suspended</strong><span>explicit reload required</span></div>
1440 <div class="state"><strong>Recovering</strong><span>host reconnect</span></div>
1441 <div class="state"><strong>Closed</strong><span>terminal</span></div>
1442 </div>
1443 <div class="responsibility-grid">
1444 <article class="responsibility" style="--accent: var(--cyan)">
1445 <h4>Idle hidden tab</h4>
1446 <p>Park offscreen at the preserved viewport size. Timers, network, CDP, and semantic automation continue.</p>
1447 <ul class="compact-list">
1448 <li>Lowest practical compositor cost</li>
1449 <li>No continuous recording frames</li>
1450 <li>Snapshot/capture available when surface remains valid</li>
1451 </ul>
1452 </article>
1453 <article class="responsibility" style="--accent: var(--yellow)">
1454 <h4>Hidden recording tab</h4>
1455 <p>Move to a covered full-size parking surface inside the window. The user sees app chrome, not the browser, while Chromium keeps painting frames.</p>
1456 <ul class="compact-list">
1457 <li>CDP screencast remains active</li>
1458 <li>Recording does not force panel open</li>
1459 <li>Same page and viewport are preserved</li>
1460 </ul>
1461 </article>
1462 </div>
1463 </section>
1464
1465 <section id="automation">
1466 <div class="section-head">
1467 <div class="section-no">06 / AUTO</div>
1468 <div class="section-title">
1469 <h3>Playwright semantics without Playwright ownership</h3>
1470 <p>
1471 Direct Playwright CDP attachment does not expose an Electron guest target as a
1472 `Page`. The selected adapter uses persistent CDP plus a versioned injected utility
1473 runtime to provide semantic, agent-friendly browser actions.
1474 </p>
1475 </div>
1476 </div>
1477 <div class="automation-stack">
1478 <div class="stack-layers">
1479 <div class="stack-layer" style="--accent: var(--purple)">
1480 <strong>Agent surface</strong>
1481 <span>`browser.open`, semantic locate, click, type, drag, wait, assert, record, console, network, tabs.</span>
1482 </div>
1483 <div class="stack-layer" style="--accent: var(--signal)">
1484 <strong>Control service</strong>
1485 <span>Capability checks, control lease, command ID, deadline, cancellation, page revision, result evidence.</span>
1486 </div>
1487 <div class="stack-layer" style="--accent: var(--yellow)">
1488 <strong>Semantic adapter</strong>
1489 <span>Version-pinned Playwright injected runtime, locator re-resolution, actionability, shadow DOM, and per-frame routing.</span>
1490 </div>
1491 <div class="stack-layer" style="--accent: var(--cyan)">
1492 <strong>Persistent CDP</strong>
1493 <span>Accessibility, DOM, Runtime, Input, Page, Network, Log; one serialized connection per tab.</span>
1494 </div>
1495 <div class="stack-layer" style="--accent: var(--green)">
1496 <strong>Guest page</strong>
1497 <span>The same live WebContents displayed to the user. No synchronized second browser.</span>
1498 </div>
1499 </div>
1500 <aside class="evidence-card">
1501 <span class="badge yellow">Phase 0.5 proof</span>
1502 <h4>Playwright's injected runtime works over CDP.</h4>
1503 <p>
1504 The 307 KB runtime installed in about 6.5–8.5 ms, resolved role/name through shadow DOM,
1505 and re-resolved a locator after its element was replaced. OOPIF routing remains T3-owned.
1506 </p>
1507 <div class="recording-pipe">
1508 <div class="pipe-item">locator description → fresh resolution</div>
1509 <div class="pipe-item">injected actionability → native dispatch</div>
1510 <div class="pipe-item">frame target → isolated runtime</div>
1511 <div class="pipe-item">ephemeral node reference → optional fast path</div>
1512 </div>
1513 </aside>
1514 </div>
1515 <div class="callout warning" style="margin-top: 22px">
1516 <div class="callout-mark">!</div>
1517 <div>
1518 <strong>Raw CDP is a privileged developer escape hatch.</strong>
1519 <span>Do not expose an unauthenticated process-wide Chromium debugging port in production. Normal agents use the Browser Control Service.</span>
1520 </div>
1521 </div>
1522 </section>
1523
1524 <section id="evidence">
1525 <div class="section-head">
1526 <div class="section-no">07 / PROOF</div>
1527 <div class="section-title">
1528 <h3>Record the work, not the browser UX</h3>
1529 <p>
1530 Video and structured traces are review artifacts. The interactive experience remains
1531 the real browser; frames are sampled only for evidence generation.
1532 </p>
1533 </div>
1534 </div>
1535 <div class="system-map">
1536 <div class="map-row" style="grid-template-columns: repeat(4, minmax(150px, 1fr))">
1537 <article class="map-node" data-tone="cyan">
1538 <div class="node-tag">Capture</div>
1539 <h4>CDP Screencast</h4>
1540 <p>Guest-only compositor frames. Sample to a bounded target frame rate.</p>
1541 </article>
1542 <article class="map-node">
1543 <div class="node-tag">Worker surface</div>
1544 <h4>Recording Canvas</h4>
1545 <p>Isolated renderer draws frames and optional agent/human cursor overlays.</p>
1546 </article>
1547 <article class="map-node" data-tone="signal">
1548 <div class="node-tag">Encode</div>
1549 <h4>MediaRecorder</h4>
1550 <p>H.264 MP4, initial 12 fps, no external ffmpeg requirement.</p>
1551 </article>
1552 <article class="map-node">
1553 <div class="node-tag">Persist</div>
1554 <h4>Artifact Service</h4>
1555 <p>Video, timeline, screenshots, logs, network, downloads, traces.</p>
1556 </article>
1557 </div>
1558 </div>
1559 <div class="responsibility-grid" style="margin-top: 18px">
1560 <article class="responsibility" style="--accent: var(--yellow)">
1561 <h4>Structured action timeline</h4>
1562 <ul class="compact-list">
1563 <li>Actor, action, input summary, status</li>
1564 <li>Before/after document revisions</li>
1565 <li>Recording timestamp and screenshot refs</li>
1566 <li>Failure, interruption, or timeout reason</li>
1567 </ul>
1568 </article>
1569 <article class="responsibility" style="--accent: var(--blue)">
1570 <h4>Diagnostic evidence</h4>
1571 <ul class="compact-list">
1572 <li>Console and uncaught exceptions</li>
1573 <li>Failed requests and response summaries</li>
1574 <li>Optional HAR / deeper trace mode</li>
1575 <li>Secret redaction before persistence</li>
1576 </ul>
1577 </article>
1578 </div>
1579 </section>
1580
1581 <section id="collaboration">
1582 <div class="section-head">
1583 <div class="section-no">08 / SHARE</div>
1584 <div class="section-title">
1585 <h3>Human interruption is a feature</h3>
1586 <p>
1587 The browser is collaborative, not locked. Explicit control leases make conflicting
1588 input predictable while allowing observation and immediate human takeover.
1589 </p>
1590 </div>
1591 </div>
1592 <div class="flow">
1593 <div class="flow-step">
1594 <div class="flow-seq">LEASE / READ</div>
1595 <div class="flow-name">Observe freely</div>
1596 <div class="flow-detail">Snapshots, URL, title, console, network, and action state can continue while another actor controls input.</div>
1597 </div>
1598 <div class="flow-step">
1599 <div class="flow-seq">LEASE / WRITE</div>
1600 <div class="flow-name">Serialize mutations</div>
1601 <div class="flow-detail">Clicks, typing, navigation, drag, and uploads run through the per-tab command queue with one active controller.</div>
1602 </div>
1603 <div class="flow-step">
1604 <div class="flow-seq">LEASE / BREAK</div>
1605 <div class="flow-name">Human interrupts</div>
1606 <div class="flow-detail">Pointer or keyboard input cancels the conflicting agent operation and returns a typed interruption result.</div>
1607 </div>
1608 <div class="flow-step">
1609 <div class="flow-seq">LEASE / HAND</div>
1610 <div class="flow-name">Adopt or hand off</div>
1611 <div class="flow-detail">Tabs carry human/agent origin and can be intentionally adopted, finalized, retained, or returned to the user.</div>
1612 </div>
1613 </div>
1614 </section>
1615
1616 <section id="reachability">
1617 <div class="section-head">
1618 <div class="section-no">09 / REACH</div>
1619 <div class="section-title">
1620 <h3>Connection and reachability matrix</h3>
1621 <p>
1622 The browser should accept environment-relative targets. Resolution decides whether
1623 the browser loads directly or through a private loopback tunnel.
1624 </p>
1625 </div>
1626 </div>
1627 <div class="filter-bar" aria-label="Reachability filters">
1628 <button class="filter-button active" data-filter="all">All scenarios</button>
1629 <button class="filter-button" data-filter="direct">Direct reachability</button>
1630 <button class="filter-button" data-filter="tunnel">Tunneled reachability</button>
1631 <button class="filter-button" data-filter="public">Public sharing</button>
1632 </div>
1633 <div class="matrix-wrap">
1634 <table>
1635 <thead>
1636 <tr>
1637 <th>Environment</th>
1638 <th>Target example</th>
1639 <th>Browser path</th>
1640 <th>Transport</th>
1641 <th>Product behavior</th>
1642 </tr>
1643 </thead>
1644 <tbody>
1645 <tr data-kind="direct">
1646 <td>Same machine</td>
1647 <td>`localhost:5173`</td>
1648 <td>Direct local URL</td>
1649 <td><span class="badge green">Direct</span></td>
1650 <td>No tunnel. Preserve normal localhost browser semantics.</td>
1651 </tr>
1652 <tr data-kind="direct">
1653 <td>LAN / Tailscale</td>
1654 <td>`mac-mini.local:5173`</td>
1655 <td>Direct private-network URL when policy allows</td>
1656 <td><span class="badge blue">Private network</span></td>
1657 <td>Prefer direct reachability; fall back to environment stream if unavailable.</td>
1658 </tr>
1659 <tr data-kind="tunnel">
1660 <td>SSH environment</td>
1661 <td>remote `127.0.0.1:5173`</td>
1662 <td>desktop loopback listener → preview stream → SSH-forwarded T3 server</td>
1663 <td><span class="badge yellow">T3 stream over SSH</span></td>
1664 <td>Default path. Native `ssh -L` may be a later optimization.</td>
1665 </tr>
1666 <tr data-kind="tunnel">
1667 <td>T3 Connect relay</td>
1668 <td>environment `127.0.0.1:3000`</td>
1669 <td>desktop loopback listener → preview stream → existing environment hostname</td>
1670 <td><span class="badge yellow">T3 stream over relay</span></td>
1671 <td>One stable public control endpoint; preview ports remain private logical streams.</td>
1672 </tr>
1673 <tr data-kind="tunnel">
1674 <td>Docker / WSL / VM</td>
1675 <td>environment-local service</td>
1676 <td>environment-port target → environment stream gateway</td>
1677 <td><span class="badge yellow">T3 stream</span></td>
1678 <td>Agent never guesses host IPs. Environment resolves its own loopback target.</td>
1679 </tr>
1680 <tr data-kind="public">
1681 <td>Public app sharing</td>
1682 <td>share preview with external person</td>
1683 <td>dedicated published endpoint</td>
1684 <td><span class="badge red">Separate product</span></td>
1685 <td>Not the browser-preview tunnel. Requires explicit publication and separate access policy.</td>
1686 </tr>
1687 </tbody>
1688 </table>
1689 </div>
1690 </section>
1691
1692 <section id="tunnel">
1693 <div class="section-head">
1694 <div class="section-no">10 / TUNNEL</div>
1695 <div class="section-title">
1696 <h3>One environment endpoint, many private streams</h3>
1697 <p>
1698 Do not provision a public Cloudflare route for every dev port. The existing T3 server
1699 endpoint carries authorized preview streams to environment-local TCP targets.
1700 </p>
1701 </div>
1702 </div>
1703 <div class="transport-diagram">
1704 <div class="transport-lane" style="--accent: var(--cyan)">
1705 <div class="lane-label">Browser side</div>
1706 <div class="lane-hop">Guest webview<br />`127.0.0.1:49101`</div>
1707 <div class="lane-hop">Desktop loopback<br />TCP listener</div>
1708 <div class="lane-hop">Authenticated<br />preview stream</div>
1709 <div class="lane-hop">Environment server<br />stream endpoint</div>
1710 </div>
1711 <div class="transport-lane" style="--accent: var(--green)">
1712 <div class="lane-label">Environment side</div>
1713 <div class="lane-hop">Grant validation</div>
1714 <div class="lane-hop">Target policy<br />loopback + port</div>
1715 <div class="lane-hop">TCP dial<br />`127.0.0.1:5173`</div>
1716 <div class="lane-hop">Vite / Next / app<br />HTTP + WebSocket</div>
1717 </div>
1718 </div>
1719 <div class="callout" style="margin-top: 24px">
1720 <div class="callout-mark">↔</div>
1721 <div>
1722 <strong>Raw TCP preserves browser behavior.</strong>
1723 <span>Root-relative assets, redirects, cookies, streaming responses, HMR upgrades, and application WebSockets pass without response rewriting.</span>
1724 </div>
1725 </div>
1726 <div class="responsibility-grid" style="margin-top: 18px">
1727 <article class="responsibility" style="--accent: var(--signal)">
1728 <h4>Initial stream model</h4>
1729 <ul class="compact-list">
1730 <li>One WebSocket per accepted browser TCP connection</li>
1731 <li>Dedicated from RPC/event traffic</li>
1732 <li>Backpressure, timeout, cancellation, half-close</li>
1733 <li>Easy to implement and diagnose</li>
1734 </ul>
1735 </article>
1736 <article class="responsibility" style="--accent: var(--purple)">
1737 <h4>Scale-up model</h4>
1738 <ul class="compact-list">
1739 <li>Multiplex streams per browser session</li>
1740 <li>Per-stream flow-control windows</li>
1741 <li>Bounded queues and concurrency</li>
1742 <li>Only after profiling real asset-heavy apps</li>
1743 </ul>
1744 </article>
1745 </div>
1746 </section>
1747
1748 <section id="ssh-relay">
1749 <div class="section-head">
1750 <div class="section-no">11 / ROUTES</div>
1751 <div class="section-title">
1752 <h3>SSH and T3 Connect carry the same preview protocol</h3>
1753 <p>
1754 Reachability transport changes; browser control does not. This keeps lifecycle,
1755 grants, diagnostics, recording, and agent behavior consistent across environments.
1756 </p>
1757 </div>
1758 </div>
1759 <div class="transport-diagram">
1760 <div class="transport-lane" style="--accent: var(--blue)">
1761 <div class="lane-label">SSH</div>
1762 <div class="lane-hop">Browser loopback</div>
1763 <div class="lane-hop">Preview stream</div>
1764 <div class="lane-hop">Existing `ssh -L`<br />to T3 server</div>
1765 <div class="lane-hop">Remote target port</div>
1766 </div>
1767 <div class="transport-lane" style="--accent: var(--yellow)">
1768 <div class="lane-label">T3 Connect</div>
1769 <div class="lane-hop">Browser loopback</div>
1770 <div class="lane-hop">Preview stream</div>
1771 <div class="lane-hop">One managed<br />Cloudflare endpoint</div>
1772 <div class="lane-hop">Environment target port</div>
1773 </div>
1774 <div class="transport-lane" style="--accent: var(--green)">
1775 <div class="lane-label">LAN / Tailnet</div>
1776 <div class="lane-hop">Browser target</div>
1777 <div class="lane-hop">Direct URL or stream</div>
1778 <div class="lane-hop">Private network</div>
1779 <div class="lane-hop">Environment target port</div>
1780 </div>
1781 </div>
1782 <div class="ssh-note">
1783 <div class="decision selected">
1784 <span class="badge green">Default</span>
1785 <h4>Stream through T3 server</h4>
1786 <p>One protocol across SSH, relay, and direct environments. Reuses authentication, grants, diagnostics, and lifecycle.</p>
1787 </div>
1788 <div class="decision">
1789 <span class="badge blue">Later optimization</span>
1790 <h4>Native SSH port forward</h4>
1791 <p>Generalize the existing `ssh -L local:127.0.0.1:remote` manager only if direct forwarding materially improves performance or compatibility.</p>
1792 </div>
1793 </div>
1794 <div class="callout warning" style="margin-top: 22px">
1795 <div class="callout-mark">!</div>
1796 <div>
1797 <strong>Public port publishing is not preview reachability.</strong>
1798 <span>“Share port 5173 with the internet” requires explicit publication, separate access control, and possibly additional managed ingress. It should not happen automatically when an agent opens a browser.</span>
1799 </div>
1800 </div>
1801 </section>
1802
1803 <section id="security">
1804 <div class="section-head">
1805 <div class="section-no">12 / TRUST</div>
1806 <div class="section-title">
1807 <h3>Capability, site, target, and artifact boundaries</h3>
1808 <p>Browser access is powerful. Authorization should be explicit but not turn every click into a dialog.</p>
1809 </div>
1810 </div>
1811 <div class="security-grid">
1812 <article class="security-card" style="--accent: var(--purple)">
1813 <h4>Invocation scope</h4>
1814 <p>`observe`, `interact`, `navigate`, `evaluate`, `record`, `network-inspect`, and privileged `raw-cdp` capabilities.</p>
1815 </article>
1816 <article class="security-card" style="--accent: var(--green)">
1817 <h4>Browser-attributed tunnel ingress</h4>
1818 <p>A loopback port is not authorization. Use a stable authority, identify the requesting guest, and scope the remote grant to one session and target.</p>
1819 </article>
1820 <article class="security-card" style="--accent: var(--signal)">
1821 <h4>Site policy</h4>
1822 <p>Origins, external navigation, auth pages, clipboard, uploads, downloads, permissions, evaluation, and raw CDP.</p>
1823 </article>
1824 <article class="security-card" style="--accent: var(--yellow)">
1825 <h4>Artifact policy</h4>
1826 <p>Retention, size bounds, redaction of cookies/auth headers, encrypted storage where required, explicit sharing.</p>
1827 </article>
1828 </div>
1829 <div class="callout signal" style="margin-top: 20px">
1830 <div class="callout-mark">×</div>
1831 <div>
1832 <strong>Normal tool input never supplies environment or thread identity.</strong>
1833 <span>Identity comes from the authenticated provider/session scope. Browser and tunnel commands cannot cross-control another thread by argument substitution.</span>
1834 </div>
1835 </div>
1836 <div class="callout warning" style="margin-top: 16px">
1837 <div class="callout-mark">!</div>
1838 <div>
1839 <strong>Phase 0.5 proved unrelated browser content can cause loopback requests.</strong>
1840 <span>Port secrecy and `Origin` checks are insufficient. Stable origins, browser attribution, bounded multiplexing, and explicit redirect/HTTPS policy gate the remote preview implementation.</span>
1841 </div>
1842 </div>
1843 </section>
1844
1845 <section id="roadmap">
1846 <div class="section-head">
1847 <div class="section-no">13 / BUILD</div>
1848 <div class="section-title">
1849 <h3>Implementation sequence</h3>
1850 <p>Build the durable host first; everything else becomes simpler once panel visibility no longer owns browser lifetime.</p>
1851 </div>
1852 </div>
1853 <div class="roadmap">
1854 <article class="phase">
1855 <div class="phase-status done">Phase 0 / complete</div>
1856 <h4>Prove decisions</h4>
1857 <p>Semantic CDP, hidden lifecycle, H.264 recording, raw TCP preview streams, routing latency, and host comparison.</p>
1858 </article>
1859 <article class="phase">
1860 <div class="phase-status done">Phase 0.5 / complete</div>
1861 <h4>Close production risks</h4>
1862 <p>Renderer-death blast radius, Playwright injected runtime, 1600×1200 recording endurance at about 11 fps, loopback threat model, origin stability, and input interruption.</p>
1863 </article>
1864 <article class="phase">
1865 <div class="phase-status">Phase 1</div>
1866 <h4>Durable session core</h4>
1867 <p>Session/tab/window contracts, app-level webview host, host-loss UX, annotations, source attribution, discovered servers, guest hardening, and parking states.</p>
1868 </article>
1869 <article class="phase">
1870 <div class="phase-status">Phase 2</div>
1871 <h4>Browser control service</h4>
1872 <p>Persistent CDP, Playwright injected runtime, OOPIF routing, locator re-resolution, console/network buffers, interruption, and tab APIs.</p>
1873 </article>
1874 <article class="phase">
1875 <div class="phase-status">Phase 3</div>
1876 <h4>Remote reachability</h4>
1877 <p>Environment-port targets, stable authorities, browser-attributed ingress, multiplexed TCP streams, redirect/HTTPS policy, SSH, and relay integration.</p>
1878 </article>
1879 <article class="phase">
1880 <div class="phase-status">Phase 4</div>
1881 <h4>Evidence pipeline</h4>
1882 <p>One-recording-per-window MediaRecorder worker, action timeline, artifact upload, thread viewer, retention, redaction, and soak gates.</p>
1883 </article>
1884 <article class="phase">
1885 <div class="phase-status">Phase 5</div>
1886 <h4>Collaboration + unattended hosts</h4>
1887 <p>Tab adoption/handoff, agent cursor, pause/revoke, optional environment-hosted Chromium for CI as a separate session type.</p>
1888 </article>
1889 </div>
1890 </section>
1891
1892 <section id="decisions">
1893 <div class="section-head">
1894 <div class="section-no">14 / LOCKED</div>
1895 <div class="section-title">
1896 <h3>Pinned architecture decisions</h3>
1897 <p>These are the working defaults until implementation evidence requires an explicit ADR replacement.</p>
1898 </div>
1899 </div>
1900 <div class="decision-log">
1901 <article>
1902 <strong>ADR 001 + 006</strong>
1903 <h4>Route-durable renderer-hosted webview</h4>
1904 <p>Keep the real guest webview in a window-level host. Renderer reload, crash, and window close are explicit live-page loss, not transparent recovery.</p>
1905 </article>
1906 <article>
1907 <strong>ADR 002 + 007</strong>
1908 <h4>Playwright-injected semantic adapter</h4>
1909 <p>Use persistent CDP plus the version-pinned Playwright injected runtime behind T3 contracts. Locators re-resolve at action time.</p>
1910 </article>
1911 <article>
1912 <strong>ADR 003 + 008</strong>
1913 <h4>Provisionally bounded recording</h4>
1914 <p>Capture guest frames and encode H.264 MP4 in Chromium. Start with one active recording per window and require soak/concurrency gates.</p>
1915 </article>
1916 <article>
1917 <strong>ADR 004 + 009</strong>
1918 <h4>Attributed, stable preview tunnel</h4>
1919 <p>Keep raw TCP remotely, but require browser-attributed local ingress, stable authorities, bounded multiplexing, and redirect/HTTPS policy.</p>
1920 </article>
1921 <article>
1922 <strong>ADR 005</strong>
1923 <h4>Keep renderer transport relay initially</h4>
1924 <p>IPC overhead is negligible. Remove renderer lifecycle authority before adding another desktop-main network connection.</p>
1925 </article>
1926 <article>
1927 <strong>ADR 010</strong>
1928 <h4>Native human interruption</h4>
1929 <p>CDP keyboard dispatch did not emit `before-input-event` in the spike. Start with native user-input interruption and retain platform regression fixtures.</p>
1930 </article>
1931 </div>
1932 <div class="callout warning" style="margin-top: 22px">
1933 <div class="callout-mark">≠</div>
1934 <div>
1935 <strong>Feasibility proof is not production proof.</strong>
1936 <span>Open gates remain: browser-attributed tunnel ingress, OOPIF routing, long-duration and multi-tab recording, platform input behavior, and recovery fidelity after host loss. Explicit non-goals still include browser-process survival, live migration, a synchronized second browser, and image-stream browser UX.</span>
1937 </div>
1938 </div>
1939 </section>
1940
1941 <footer>
1942 Source plans: `collaborative-browser-runtime-next-iteration.md`, `browser-phase-0/*`, and `browser-phase-0-5/*`.
1943 This HTML document is the consolidated visual architecture reference.
1944 </footer>
1945 </main>
1946 </div>
1947
1948 <script>
1949 const navLinks = [...document.querySelectorAll(".nav a")];
1950 const sections = navLinks
1951 .map((link) => document.querySelector(link.getAttribute("href")))
1952 .filter(Boolean);
1953
1954 const observer = new IntersectionObserver(
1955 (entries) => {
1956 const visible = entries
1957 .filter((entry) => entry.isIntersecting)
1958 .sort((left, right) => right.intersectionRatio - left.intersectionRatio)[0];
1959 if (!visible) return;
1960 navLinks.forEach((link) => {
1961 link.classList.toggle("active", link.getAttribute("href") === `#${visible.target.id}`);
1962 });
1963 },
1964 { rootMargin: "-20% 0px -65% 0px", threshold: [0.05, 0.25, 0.5] },
1965 );
1966 sections.forEach((section) => observer.observe(section));
1967
1968 const filterButtons = [...document.querySelectorAll(".filter-button")];
1969 const reachabilityRows = [...document.querySelectorAll("#reachability tbody tr")];
1970 filterButtons.forEach((button) => {
1971 button.addEventListener("click", () => {
1972 const filter = button.dataset.filter;
1973 filterButtons.forEach((candidate) => candidate.classList.toggle("active", candidate === button));
1974 reachabilityRows.forEach((row) => {
1975 row.dataset.filtered = String(filter !== "all" && row.dataset.kind !== filter);
1976 });
1977 });
1978 });
1979 </script>
1980 </body>
1981</html>
.plans/collaborative-browser-runtime-next-iteration.md created+1015
......@@ -0,0 +1,1015 @@
1# Collaborative Browser Runtime — Next Iteration
2
3## Purpose
4
5Turn the preview browser into a route-durable, programmable collaboration surface shared by users and agents.
6
7The browser is not an image preview and automation is not a secondary mode. The product should own a real browser session that:
8
9- remains alive independently of whether its panel is visible, while explicitly reporting host-renderer or window loss
10- can be driven programmatically with reliable, semantic browser primitives
11- can be inspected and manipulated by the user at any time
12- records actions, video, console, network, and page state as reviewable evidence
13- works when the development environment is remote
14- can support multiple agent providers without provider-specific browser implementations
15
16The Codex in-app browser is a strong implementation reference for shared browser ownership, hidden tab retention, browser-client ergonomics, and human/agent handoff. It is not the target product boundary. T3 should preserve its provider-neutral and remote-capable architecture and improve on Codex where those requirements demand a different shape.
17
18This plan supersedes assumptions in:
19
20- `.plans/visible-preview-browser-automation-via-cdp-mcp.md`
21- `.plans/shared-http-mcp-server-with-preview-automation-revised.md`
22
23Those plans remain useful implementation history, but their React-owned lifecycle, visibility-gated routing, fixed MCP command surface, and direct-only remote URL assumptions should not constrain this iteration.
24
25The Phase 0.5 review and follow-up spikes are authoritative amendments to Phase 0:
26
27- `.plans/browser-phase-0-5/findings.md`
28- `.plans/browser-phase-0-5/006-renderer-failure-boundary.md`
29- `.plans/browser-phase-0-5/007-playwright-injected-runtime.md`
30- `.plans/browser-phase-0-5/008-recording-endurance.md`
31- `.plans/browser-phase-0-5/009-loopback-threat-model.md`
32- `.plans/browser-phase-0-5/010-human-input-interruption.md`
33
34## Product Principles
35
36### 1. The browser session is the product primitive
37
38The primary object is a durable logical `BrowserSession`, not a React panel, webview component, MCP request, or screenshot. The logical record can survive host loss; the live Chromium page state cannot.
39
40The panel is one view onto the session. Agent automation, recording, DevTools, snapshots, and user interaction all target the same session.
41
42### 2. Human and agent use the same page
43
44When an agent clicks, types, scrolls, navigates, or changes page state, the user must see that state when the browser is visible. When the user interacts, the agent's next observation must include the resulting state.
45
46Do not run a hidden Playwright browser beside an unrelated visible preview and attempt to synchronize them.
47
48### 3. Visibility is not lifecycle
49
50Closing or switching away from the browser panel must not destroy the browser session. Visibility, focus, control ownership, process lifetime, and recording state are separate concepts.
51
52The selected Electron `<webview>` is route-durable, not process-durable. A host-renderer reload, renderer crash, owning-window close, or application restart destroys the guest and must transition the logical tab to an explicit lost state.
53
54### 4. Prefer existing browser automation ecosystems
55
56Do not grow a large bespoke click/type/wait/selector framework when Playwright, CDP, and accessibility-based tooling already solve those problems.
57
58T3 should provide session discovery, authorization, routing, collaboration state, remote access, and artifact storage. Automation adapters should translate established browser APIs onto the T3-owned session.
59
60### 5. Remote environments are normal
61
62A browser attached to a remote environment must be able to open services bound to that environment's loopback interface without requiring the user to manually expose a LAN address.
63
64Remote access cannot be an afterthought or a warning-only UX.
65
66### 6. Evidence is first-class
67
68An agent must be able to start and stop a recording, capture screenshots, inspect console and network activity, and produce a machine-readable action trace. Evidence should be attached to the thread and usable by both the agent and user.
69
70Video recording is an artifact and debugging tool. It is not the mechanism used to render the interactive browser UI.
71
72## Current Architecture and Constraints
73
74The current browser implementation already has the most important property: the human-visible Electron `<webview>` and agent automation target the same guest `WebContents`.
75
76Current path:
77
78```text
79agent
80 -> shared HTTP MCP tool
81 -> environment server PreviewAutomationBroker
82 -> WebSocket request to renderer owner
83 -> renderer PreviewAutomationOwner
84 -> Electron IPC
85 -> PreviewViewManager
86 -> guest webview WebContents through CDP
87```
88
89Current strengths:
90
91- real shared browser rather than a screenshot-driven duplicate
92- provider-neutral MCP entry point
93- environment and thread scoped authorization
94- server-side routing compatible with remote T3 clients
95- schema-validated contracts and tested broker behavior
96- normal browser UX including navigation, history, zoom, DevTools, and annotations
97
98Current constraints to remove:
99
100- browser lifetime is anchored to React component mounting
101- only three hidden preview threads are retained
102- sheet and route behavior can unmount the browser
103- most automation requires the owner to report `visible`
104- ownership is inferred from a focused renderer rather than a durable browser host
105- one global persistent preview partition is shared across projects and threads
106- CDP attaches and detaches around individual operations
107- the custom automation surface has limited selector and locator semantics
108- requests take an avoidable renderer-mediated hop before reaching desktop browser control
109- remote loopback URLs are not resolved to the remote environment
110- recording, trace, console, network, and download artifacts are not modeled as one system
111- a browser session cannot be intentionally handed between user and agent control modes
112
113The implementation may remove or replace these patterns rather than preserving compatibility internally. User-facing session state and existing thread behavior should be migrated deliberately.
114
115## Target Architecture
116
117Split the system into five boundaries:
118
119```text
120Agent adapter / skill / MCP
121 |
122 v
123Browser Control Service <----> Browser Artifact Service
124 |
125 v
126Browser Session Registry
127 |
128 v
129Browser Host Adapter <----> Environment Preview Gateway
130 |
131 v
132Real Chromium page shown in T3
133```
134
135### Browser Session Registry
136
137The environment server owns authoritative logical session metadata:
138
139```ts
140interface BrowserSession {
141 id: BrowserSessionId;
142 environmentId: EnvironmentId;
143 threadId: ThreadId;
144 hostId: BrowserHostId | null;
145 activeTabId: BrowserTabId | null;
146 lifecycle: "creating" | "ready" | "suspended" | "recovering" | "closed";
147 visibility: "visible" | "hidden" | "detached";
148 controller: "human" | "agent" | "none";
149 partitionId: BrowserPartitionId;
150 recordingId: BrowserRecordingId | null;
151 revision: number;
152 createdAt: string;
153 updatedAt: string;
154}
155```
156
157The registry owns:
158
159- stable session and tab identities
160- environment and thread association
161- lifecycle and recovery state
162- host assignment
163- visibility and control state
164- capabilities reported by the host
165- recording and artifact references
166- event revisions for reconnect and stale-command rejection
167
168The registry does not own Chromium directly. A browser host does.
169
170### Browser Host
171
172A `BrowserHost` is a process capable of owning and controlling real browser pages.
173
174Initial host adapter:
175
176- `ElectronBrowserHost`
177 - uses route-durable app-level renderer-hosted `<webview>` elements
178 - is mounted independently of thread routes, panel sheets, and preview components
179 - registers each guest `WebContents` with the desktop main process
180 - keeps desktop main authoritative for CDP, navigation, recording, and security
181 - moves tabs between visible panel bounds, offscreen idle parking, and covered recording parking
182 - binds each host to an owning window and reports renderer reload, renderer crash, or window close as host loss
183
184Future host adapter:
185
186- `EnvironmentChromiumHost`
187 - runs Chromium beside a remote/headless environment
188 - enables unattended testing and CI
189 - is not a second browser for the same active session
190 - creates a separate explicitly hosted session whose UI attachment strategy is a later phase
191
192Do not pretend a live browser process can migrate losslessly between hosts. A session is bound to one host. Migration requires an explicit restart/restore operation and must report what state cannot be preserved.
193
194### Browser Control Service
195
196The control service provides one internal session-oriented API regardless of the calling agent or host implementation.
197
198It owns:
199
200- authorization and invocation scope
201- host discovery and routing
202- command serialization per tab
203- cancellation, deadlines, and stale revision checks
204- automation adapter sessions
205- control lease and user-interruption behavior
206- structured event subscriptions
207- artifact creation triggers
208
209It should expose a transport-neutral internal protocol. MCP is one adapter, not the core API.
210
211### Browser Artifact Service
212
213The artifact service stores and indexes:
214
215- screenshots
216- video recordings
217- action timelines
218- console logs
219- uncaught exceptions
220- network request summaries
221- HAR exports where supported
222- DOM/accessibility snapshots
223- downloads
224- optional Playwright traces
225
226Artifacts are scoped to environment, thread, browser session, and provider session. Large payloads must not travel inline through the normal WebSocket event stream.
227
228### Environment Preview Gateway
229
230The gateway makes environment-local web services reachable by the browser host.
231
232Example:
233
234```text
235remote environment localhost:5173
236 -> environment TCP target
237 -> dedicated authenticated WebSocket tunnel
238 -> desktop loopback TCP listener
239 -> real webview navigation to desktop loopback
240```
241
242The browser may load a desktop-local authority so origin paths, root-relative assets, HMR, and application WebSockets retain normal semantics. Phase 0.5 proved that a bare loopback port is not an authorization boundary, changing ports loses origin storage, and absolute redirects can escape the authority. Browser-attributed ingress, stable authority assignment, and explicit redirect/HTTPS policy are required before the gateway is production-ready. The UI separately displays the environment-relative requested target.
243
244The gateway must support:
245
246- raw TCP forwarding with backpressure and half-close behavior
247- HTTP, streaming responses, and WebSocket upgrades without application-level rewriting
248- source maps, static assets, redirects, cookies, and arbitrary root-relative paths
249- deterministic target identity
250- connection loss and reconnect reporting
251- explicit port authorization
252- one dedicated tunnel connection per accepted TCP stream initially
253- a separate priority class from normal RPC and event traffic
254
255The browser automation layer should receive both requested and resolved URLs.
256
257## Browser Ownership and UI Model
258
259### Move lifetime out of React
260
261Replace `usePreviewBridge` mount/unmount ownership with explicit commands:
262
263- `browserSession.create`
264- `browserSession.attachView`
265- `browserSession.detachView`
266- `browserSession.setVisibility`
267- `browserSession.close`
268
269The app-level browser host remains mounted for the desktop window lifetime. React panels report display bounds and visibility intent; they do not mount, reparent, or destroy browser elements.
270
271### Hidden browser behavior
272
273When hidden:
274
275- keep the webview and `WebContents` alive
276- park idle tabs offscreen at their preserved viewport size to reduce compositor work
277- move recording tabs to a full-size covered parking surface because offscreen tabs stop producing screencast frames
278- preserve timers and network behavior by default
279- allow automation and recording according to session policy
280- expose an explicit low-resource suspension operation rather than silently evicting after an arbitrary count
281
282Resource pressure policy should be observable and deterministic:
283
284- warn before suspension
285- store restorable metadata
286- never silently destroy a session being controlled or recorded
287- allow configurable limits by memory pressure and session activity
288
289### Tabs
290
291Make tabs first-class:
292
293```ts
294interface BrowserTab {
295 id: BrowserTabId;
296 sessionId: BrowserSessionId;
297 origin: "human" | "agent" | "system";
298 state: "opening" | "ready" | "closing" | "closed";
299 url: string;
300 title: string;
301 active: boolean;
302 controller: "human" | "agent" | "none";
303}
304```
305
306Support:
307
308- agent-created tabs
309- user-created tabs
310- selecting and listing tabs
311- adopting an existing user tab
312- handing a tab back to the user
313- closing only tabs owned by the current workflow when requested
314- restoring tab metadata after client reconnect
315
316### Control and interruption
317
318Use an explicit control lease instead of assuming the most recently focused owner is always correct.
319
320Policy:
321
322- humans may always interrupt an agent action
323- active user pointer or keyboard input cancels or pauses conflicting agent input
324- automation receives `BrowserControlInterruptedError`
325- read-only observations may continue during human control
326- the UI shows when an agent controls a tab
327- agent cursor and current action are visible but do not block normal browser input
328- a user can pause, resume, or revoke automation for the session
329
330Avoid a heavyweight approval dialog for every action. Use session-level trust, site policy, and high-risk operation gates.
331
332## Programmatic Automation
333
334### Primary adapter decision
335
336Do not continue expanding a fixed set of hand-written CDP operations as the main agent interface.
337
338Implement an adapter boundary:
339
340```ts
341interface BrowserAutomationAdapter {
342 connect(session: BrowserAutomationSession): Effect.Effect<BrowserAutomationConnection, BrowserError>;
343}
344```
345
346Phase 0.5 selected a **Playwright-injected semantic adapter behind a T3-owned browser-client boundary**:
347
348- persistent Electron debugger connection owned by desktop main
349- the version-pinned Playwright injected runtime for selector parsing, semantic locators, shadow DOM, actionability, and hit-target behavior
350- one injected runtime and execution context per frame/target, including explicit OOPIF target routing
351- Playwright-like high-level semantics without Playwright owning the browser
352- locator descriptions re-resolved at action time
353- optional snapshot-scoped element references as ephemeral accelerators only
354
355Direct Playwright `connectOverCDP` does not expose Electron guest targets of type `webview` as Playwright pages. A `WebContentsView` does appear as a page, but it was rejected for this iteration because hidden/detached capture failed recording requirements and its native stacking model complicates collaboration UI.
356
357Playwright remains a behavior reference and test oracle. Its injected runtime is an internal, version-coupled dependency protected by compatibility fixtures and hidden behind T3 contracts.
358
359### Agent-facing API
360
361Offer multiple programmatic surfaces over the same control service:
362
363- MCP toolkit for providers that support MCP
364- skill instructions and helper client for Codex-like runtimes
365- CLI for debugging and providers that can execute shell commands
366- internal TypeScript client for T3 features and tests
367- optional raw CDP diagnostics only in explicitly trusted developer mode
368
369The default high-level surface should support:
370
371- create/open/list/close sessions and tabs
372- navigate, reload, history, viewport, and zoom
373- semantic locate, inspect, click, hover, drag, type, select, upload, and keyboard input
374- frames, dialogs, popups, downloads, and new tabs
375- wait for semantic conditions, requests, navigation, and page stability
376- JavaScript evaluation with bounded results
377- console and exception subscriptions
378- request/response observation
379- screenshots and recordings
380- action groups and assertions
381
382Keep raw CDP as a privileged developer escape hatch, not the normal workflow.
383
384### Observation model
385
386Agents should not need a full screenshot after every action.
387
388Provide composable observations:
389
390- accessibility snapshot with stable element references
391- semantic locator results
392- bounded visible text
393- viewport screenshot on demand
394- current URL/title/loading state
395- recent console and exception entries
396- recent network failures
397- DOM change summary since a known revision
398- current control and recording state
399
400Locator descriptions are the primary durable handle and must re-resolve immediately before action. Snapshot references expire on navigation, document replacement, frame replacement, HMR, or node detachment; when the originating locator is available the adapter may retry through that locator.
401
402### Command execution
403
404Commands to a tab are serialized unless explicitly marked read-only and concurrent-safe.
405
406Each command includes:
407
408- command id
409- browser session and tab id
410- expected session/document revision where relevant
411- provider and thread invocation scope
412- deadline and cancellation token
413- control requirement
414- artifact/evidence policy
415
416Each result includes:
417
418- before and after revisions
419- timing
420- resulting URL and title
421- structured error when applicable
422- references to generated evidence
423
424## Recording, Tracing, and Debug Evidence
425
426### Recording requirements
427
428Agents and users can:
429
430- start recording the active tab or browser session
431- stop recording and receive an artifact reference
432- mark recording chapters around action groups
433- capture a short rolling buffer after a failure
434- attach recordings to thread messages or task completion evidence
435- download or open recordings from the UI
436
437The browser remains a normal interactive webview while recording.
438
439### Selected capture pipeline
440
441Phase 0 selected:
442
443```text
444guest webview Page.startScreencast
445 -> bounded frame sampler
446 -> isolated recording canvas
447 -> Chromium MediaRecorder
448 -> H.264 MP4 artifact
449```
450
451Initial defaults:
452
453- `video/mp4;codecs=avc1.42E01E`
454- 12 frames per second
455- current browser viewport with policy bounds
456- dropped intermediate frames under encoder pressure instead of an unbounded queue
457
458When a recorded tab is hidden, the durable browser host places it on a covered full-size parking surface. Moving it offscreen stops continuous screencast frames. No external ffmpeg executable is required.
459
460CDP frame capture is internal to evidence generation. It does not replace the real browser as the interactive UX.
461
462### Action timeline
463
464Every recording can be paired with structured events:
465
466```ts
467interface BrowserActionEvent {
468 id: BrowserActionEventId;
469 sessionId: BrowserSessionId;
470 tabId: BrowserTabId;
471 actor: "human" | "agent" | "system";
472 action: string;
473 startedAt: string;
474 completedAt?: string;
475 status: "running" | "succeeded" | "failed" | "interrupted";
476 inputSummary: unknown;
477 pageRevisionBefore: number;
478 pageRevisionAfter?: number;
479 screenshotBeforeId?: BrowserArtifactId;
480 screenshotAfterId?: BrowserArtifactId;
481 recordingOffsetMs?: number;
482 error?: BrowserErrorSummary;
483}
484```
485
486This timeline is more useful to agents than video alone and allows the UI to jump from an action to the relevant recording timestamp.
487
488### Console and network capture
489
490Maintain bounded per-tab ring buffers for:
491
492- console entries
493- uncaught exceptions
494- failed requests
495- response status summaries
496- WebSocket connection failures
497- navigation timing
498
499Allow explicit HAR/trace recording for deeper debugging. Redact authorization headers, cookies, and configured sensitive fields before persistence.
500
501## Remote-First Behavior
502
503### Required remote scenarios
504
505The design must support:
506
5071. Desktop UI and browser on a MacBook, environment and agent on a Mac mini.
5082. Desktop UI and browser local, environment inside Docker, SSH, WSL, or a cloud VM.
5093. Multiple desktop clients connected to one environment without ambiguous browser ownership.
5104. Agent automation continuing when the browser panel is hidden.
5115. Reconnecting a desktop client to an existing logical browser session after network interruption.
5126. Unattended environment-side browser sessions for CI or automated verification in a later phase.
513
514### Preview target abstraction
515
516Agents should open an environment-relative target rather than constructing client-reachable URLs:
517
518```ts
519type BrowserNavigationTarget =
520 | { kind: "url"; url: string }
521 | { kind: "environment-port"; port: number; protocol?: "http" | "https"; path?: string }
522 | { kind: "run-action"; actionId: string; path?: string };
523```
524
525The environment resolves the target into:
526
527```ts
528interface ResolvedBrowserTarget {
529 requested: BrowserNavigationTarget;
530 displayUrl: string;
531 browserUrl: string;
532 resolutionKind: "direct" | "environment-gateway" | "tunnel";
533 environmentId: EnvironmentId;
534 expiresAt?: string;
535}
536```
537
538This removes remote hostname guessing from prompts and agent logic.
539
540### Gateway security
541
542- do not treat loopback binding or port secrecy as authorization
543- require browser-attributed ingress so unrelated pages and local processes cannot ride an authenticated environment tunnel
544- keep the desktop authority stable for the granted environment/project/target tuple to preserve cookies, storage, service workers, and origin behavior
545- scope gateway grants to environment, browser session, target host, and port
546- use short-lived credentials inaccessible to page JavaScript where possible
547- block cloud metadata and forbidden address ranges by policy
548- require explicit configuration for non-loopback arbitrary upstream hosts
549- validate redirects against the grant policy
550- audit port openings and navigation resolutions
551- do not place long-lived bearer tokens in visible URLs
552- use a dedicated tunnel connection rather than the normal control/event WebSocket
553- multiplex TCP streams over a bounded number of authenticated tunnel connections before T3 Connect rollout unless relay load tests prove per-stream connections safe
554- define explicit compatibility behavior for absolute redirects, configured public origins, OAuth callbacks, and HTTPS upstreams
555
556### Reconnect behavior
557
558If the desktop connection drops:
559
560- mark the host unavailable without deleting the logical session
561- fail or pause pending control commands deterministically
562- retain artifact metadata and last-known tab state
563- rebind when the same browser host reconnects and reports its session inventory
564- report when the underlying `WebContents` was lost and recovery requires reload
565
566Do not claim browser process survival when the owning desktop application exited.
567
568## Security Model
569
570### Invocation scope
571
572Keep the existing session-scoped MCP authentication model, generalized to browser capabilities:
573
574- `browser:observe`
575- `browser:interact`
576- `browser:navigate`
577- `browser:evaluate`
578- `browser:record`
579- `browser:network-inspect`
580- `browser:raw-cdp`
581
582The agent cannot override environment, thread, or provider session identity in normal tool arguments.
583
584### Site policy
585
586Add environment/user policy for:
587
588- allowed origins and port ranges
589- external internet navigation
590- authentication pages
591- file uploads and downloads
592- clipboard access
593- camera, microphone, geolocation, and notifications
594- JavaScript evaluation
595- raw CDP
596
597Local development origins may be trusted by default under a configurable policy. Sensitive external origins should require an explicit trust decision.
598
599### Browser partitions
600
601Replace the global `persist:t3code-preview` partition with deliberate isolation.
602
603Default proposal:
604
605- one persistent partition per environment and project
606- optional isolated partition per browser session
607- explicit user action to share authentication state between sessions
608- clear partition lifecycle and storage deletion UX
609
610The final partition key must not contain raw secrets or unsafe filesystem characters.
611
612## Contract and Module Direction
613
614Likely contracts:
615
616```text
617packages/contracts/src/browserSession.ts
618packages/contracts/src/browserControl.ts
619packages/contracts/src/browserArtifacts.ts
620packages/contracts/src/browserGateway.ts
621```
622
623Keep contracts schema-only.
624
625Likely runtime modules:
626
627```text
628apps/server/src/browser/
629 Services/
630 BrowserSessionRegistry.ts
631 BrowserControlService.ts
632 BrowserArtifactService.ts
633 BrowserTargetResolver.ts
634 Layers/
635 Rpc/
636 Mcp/
637
638apps/desktop/src/browser/
639 ElectronBrowserHost.ts
640 ElectronBrowserTab.ts
641 CdpConnection.ts
642 RecordingController.ts
643 ArtifactUploader.ts
644
645apps/web/src/browser/
646 BrowserPanel.tsx
647 BrowserSessionProvider.tsx
648 BrowserTabs.tsx
649 BrowserControlIndicator.tsx
650 BrowserArtifactViewer.tsx
651
652packages/browser-client/
653 session client
654 locator/action adapter
655 CLI or Node helper
656```
657
658Do not preserve `PreviewAutomationOwner` as the permanent routing boundary. Replace it with browser-host registration and session attachment protocols.
659
660## Migration Strategy
661
662### Phase 0: Technical Spikes and Decision Records
663
664**Status: completed.**
665
666Authoritative results and executable spikes:
667
668- `.plans/browser-phase-0/findings.md`
669- `.plans/browser-phase-0/001-browser-host.md`
670- `.plans/browser-phase-0/002-automation-adapter.md`
671- `.plans/browser-phase-0/003-recording.md`
672- `.plans/browser-phase-0/004-remote-preview-tunnel.md`
673- `.plans/browser-phase-0/005-desktop-routing.md`
674- `.plans/browser-phase-0/spikes/`
675
676Goals:
677
678- eliminate high-risk unknowns before reshaping contracts
679- produce small executable prototypes rather than design-only conclusions
680
681Spikes:
682
6831. Attach Playwright or a Playwright-compatible locator runtime to the existing Electron guest webview.
6842. Keep a guest `WebContents` alive and controllable while its panel is hidden and painting is reduced.
6853. Record the guest content to a seekable video without replacing the interactive browser UX.
6864. Proxy a remote environment's loopback HTTP and WebSocket dev server into the local desktop webview.
6875. Measure CDP command latency through direct desktop routing versus the current renderer-mediated route.
688
689Deliverables:
690
691- decision record for automation adapter
692- decision record for recording backend
693- decision record for remote preview gateway transport
694- measured CPU, memory, and latency results
695- explicit unsupported cases
696
697Exit criteria:
698
699- completed: semantic role/name click and input worked against the real webview through CDP
700- completed: Chromium MediaRecorder produced a seekable H.264 MP4 from covered webview capture
701- completed: raw TCP tunneling carried HTTP and WebSocket traffic through a desktop loopback listener
702
703### Phase 0.5: Production-Risk Closure
704
705**Status: completed for the targeted desktop probes; browser-attributed tunnel ingress remains an implementation prerequisite.**
706
707Authoritative results:
708
709- `.plans/browser-phase-0-5/findings.md`
710- `.plans/browser-phase-0-5/006-renderer-failure-boundary.md`
711- `.plans/browser-phase-0-5/007-playwright-injected-runtime.md`
712- `.plans/browser-phase-0-5/008-recording-endurance.md`
713- `.plans/browser-phase-0-5/009-loopback-threat-model.md`
714- `.plans/browser-phase-0-5/010-human-input-interruption.md`
715
716Confirmed:
717
718- host-renderer reload destroys the `<webview>` guest and live state, while a main-owned `WebContentsView` survives that specific reload
719- Playwright's installed injected runtime can be evaluated in the real guest over CDP, resolves role/name locators through shadow DOM, and re-resolves after element replacement
720- covered recording produced a seekable 1600×1200 H.264 MP4 for ten seconds at approximately 11.2 fps
721- an unrelated browser page can cause a request to a loopback preview listener; loopback binding and port secrecy are not authorization
722- changing the desktop port changes the browser origin and loses origin storage
723- CDP keyboard dispatch did not emit Electron `before-input-event` in the tested guest
724
725Required carry-forward:
726
727- route-durable terminology and explicit host-loss UX
728- version-pinned Playwright injected runtime with OOPIF routing and compatibility fixtures
729- one-recording-per-window default until soak and concurrency budgets pass
730- a browser-attributed desktop ingress design before remote preview implementation
731- annotation/source attribution, port discovery, and guest isolation hardening in Phase 1
732- console and network ring buffers in Phase 2
733
734### Phase 1: Durable Browser Session Core
735
736Goals:
737
738- separate browser lifetime from React
739- introduce stable sessions, tabs, and host registration
740
741Tasks:
742
7431. Add browser session, tab, host, lifecycle, and capability schemas.
7442. Add server `BrowserSessionRegistry` with revisioned events.
7453. Add durable app-level `ElectronBrowserHost` plus desktop-main registration and heartbeat.
7464. Move browser create/close ownership out of `usePreviewBridge` and panel components.
7475. Add panel-bound reporting and visible/offscreen/covered parking transitions.
7486. Remove the visibility requirement from control routing.
7497. Replace hidden-thread count eviction with explicit resource policy.
7508. Add per-environment/project partitioning.
7519. Migrate existing preview open/close state to browser sessions.
75210. Preserve the annotation/element-pick/source-attribution pipeline as a Browser Control Service capability.
75311. Preserve discovered-server and run-action preview UX through environment-relative target contracts.
75412. Harden guest isolation and document any narrowly required main-world injection.
75513. Add owning-window identity, host-loss transitions, and explicit session-loss UX.
756
757Acceptance criteria:
758
759- hiding the panel does not destroy the page
760- automation works while hidden
761- reopening shows the exact current page state
762- route changes do not accidentally close the session
763- desktop reconnect reports and reconciles existing browser inventory
764- browser closure is always an explicit lifecycle event
765
766### Phase 2: Control Service and Automation Adapter
767
768Goals:
769
770- replace fixed bespoke operations with a reusable automation connection
771- preserve provider-neutral invocation
772
773Tasks:
774
7751. Add `BrowserControlService` with per-tab command queues and cancellation.
7762. Add persistent CDP connection ownership in the desktop host.
7773. Implement the selected Playwright or browser-client-style adapter.
7784. Integrate the version-pinned Playwright injected runtime with locator re-resolution and compatibility fixtures.
7795. Add target auto-attach, OOPIF routing, frames, popups, dialogs, downloads, drag, hover, and file upload support.
7806. Add structured observation and incremental page revisions.
7817. Add bounded console, exception, and network ring buffers.
7828. Add a `packages/browser-client` helper and CLI.
7839. Refactor MCP tools into a thin adapter over the control service.
78410. Deprecate and remove redundant `PreviewAutomationOwner` operation dispatch.
785
786Acceptance criteria:
787
788- a provider can complete a multi-page workflow using semantic locators
789- the same workflow can be driven through MCP and the browser client
790- user interaction can interrupt an agent safely
791- command failures identify stale page state, lost control, timeout, or host loss distinctly
792- DevTools and automation coexist or fail with an explicit supported policy
793
794### Phase 3: Remote Preview Gateway
795
796Goals:
797
798- make remote loopback services first-class browser targets
799
800Tasks:
801
8021. Add environment-port and run-action target contracts.
8032. Add target resolution, stable desktop authorities, and short-lived gateway grants.
8043. Implement browser-attributed desktop ingress; reject a bare unauthenticated loopback listener.
8054. Implement bounded multiplexed raw TCP streams over dedicated authenticated tunnel connections.
8065. Add origin, cookie, service-worker, absolute-redirect, OAuth, HTTPS, and source-map compatibility tests.
8076. Integrate run actions so agents can request the declared preview target directly.
8087. Display requested and resolved target information in browser diagnostics.
8098. Add reconnect and expired-grant recovery.
810
811Acceptance criteria:
812
813- a desktop browser can open `localhost` services from a remote environment without LAN exposure
814- Vite HMR works through the gateway
815- application WebSockets work through the gateway
816- the agent never needs to guess the remote machine's IP address
817- gateway permissions cannot be reused for unrelated ports or hosts
818
819### Phase 4: Recording and Evidence
820
821Goals:
822
823- make browser work inspectable, replayable, and attachable to tasks
824
825Tasks:
826
8271. Add browser artifact contracts and storage abstraction.
8282. Add start/stop recording controls to agent and UI surfaces.
8293. Add structured action timeline generation.
8304. Add screenshot-before/after policies for action groups and failures.
8315. Add artifact upload outside the normal WS payload path.
8326. Add thread UI for video, screenshots, traces, and logs.
8337. Add retention, cleanup, and size limits.
8348. Add secret redaction for network and console artifacts.
8359. Enforce the initial one-active-recording-per-window policy and expose encoder/drop health.
836
837Acceptance criteria:
838
839- an agent can record a test flow and return an artifact reference
840- the user can watch the recording and jump to failed actions
841- recordings work while the panel is hidden
842- failed requests and console exceptions are included in the evidence bundle
843- large artifacts do not block normal thread event delivery
844
845### Phase 5: Multi-Tab Collaboration and Unattended Hosts
846
847Goals:
848
849- mature collaboration UX
850- add optional environment-side automation without compromising shared-session semantics
851
852Tasks:
853
8541. Add human/agent tab origin and ownership UX.
8552. Add tab adoption, handoff, finalization, and workflow cleanup policies.
8563. Add visible agent cursor and current-action overlays.
8574. Add session pause/revoke controls.
8585. Implement `EnvironmentChromiumHost` for CI and unattended verification.
8596. Define how a remote-hosted session is viewed or attached without claiming it is the local Electron webview.
8607. Add capability-based host selection.
8618. Add test-report integrations using browser artifacts.
862
863Acceptance criteria:
864
865- agent-created tabs are identifiable and cleanly handed to the user
866- user-created tabs can be intentionally adopted
867- unattended browser workflows use a separately identified environment-hosted session
868- no workflow silently operates a second browser while presenting another as the controlled browser
869
870## Testing Strategy
871
872### Contract tests
873
874- session, tab, host, control, artifact, and target schemas
875- version and revision validation
876- capability authorization
877- invalid lifecycle transitions
878- stale command rejection
879
880### Server tests
881
882- session registry lifecycle and event replay
883- host assignment and reconnect
884- per-tab command serialization
885- cancellation and timeout behavior
886- control lease interruption
887- provider/thread isolation
888- artifact metadata and cleanup
889- gateway grant scoping
890
891### Desktop tests
892
893- webview registration cannot target unrelated `WebContents`
894- hidden session remains navigable and controllable
895- persistent CDP connection reconnects after navigation or target loss
896- multiple tabs remain isolated
897- recording starts, survives navigation, and finalizes
898- resource policy never evicts active or recording sessions
899
900### Gateway integration tests
901
902- remote HTTP app
903- Vite HMR WebSocket
904- application WebSocket
905- redirects
906- cookies
907- source maps
908- upstream failure and reconnect
909- forbidden host/port rejection
910
911### End-to-end tests
912
9131. Start a remote test environment and dev server bound only to loopback.
9142. Open it through an environment-port target in the desktop browser.
9153. Hide the panel.
9164. Drive a semantic browser workflow through MCP or browser-client.
9175. Record the workflow.
9186. Reopen the panel and verify the same final page state.
9197. Inspect the recording, action timeline, console, and network artifacts.
920
921Additional scenarios:
922
923- user interrupts agent typing
924- desktop disconnects during navigation
925- HMR reload occurs during locator interaction
926- two desktop clients connect to the same environment
927- two provider sessions cannot cross-control threads
928- one provider loses authorization while another continues
929
930Do not mock core lifecycle, routing, or command-serialization logic. External Chromium, media encoding, filesystem artifact storage, and network transport boundaries may use test layers where necessary.
931
932## Performance and Reliability Budgets
933
934Measure and enforce budgets rather than treating performance as qualitative.
935
936Initial targets to validate during spikes:
937
938- control routing overhead excluding page work: p95 under 50 ms locally, under 150 ms over a typical remote T3 connection
939- hidden idle session CPU: near-zero absent page activity
940- hidden session memory: observable with configurable pressure policy
941- screenshot capture: under 500 ms for a normal 1280px viewport
942- semantic snapshot: under 300 ms for typical application pages
943- recording overhead: under 15% CPU on supported desktop hardware at the selected resolution/frame rate
944- browser host reconnect detection: under 5 seconds
945- no unbounded console, network, screenshot, or action buffers
946
947Budgets may be adjusted after measurement, but the final values must be documented and covered by benchmarks or diagnostics.
948
949## Observability
950
951Add structured diagnostics for:
952
953- browser session and host lifecycle transitions
954- tab creation, closure, and target changes
955- command queue length and duration
956- CDP connection state
957- control lease changes and interruptions
958- gateway resolution and proxy failures
959- recording start, encoder health, and finalization
960- artifact sizes and upload duration
961- dropped or redacted evidence
962
963Provide a user-accessible browser diagnostics view or export so remote failures do not require reading opaque server logs.
964
965## Explicit Non-Goals
966
967- synchronizing two independent browsers and presenting them as one session
968- replacing the interactive browser UI with JPEG or video streaming
969- building a full custom Playwright clone
970- preserving current internal APIs solely for compatibility
971- guaranteeing browser process survival after the owning host exits
972- transparent live migration of an active browser process between hosts
973- unrestricted proxying to arbitrary private-network targets
974- storing unlimited recordings or raw network bodies
975- making Codex's browser architecture the permanent product boundary
976
977## Pinned Phase 0 + 0.5 Decisions
978
9791. Keep a route-durable renderer-hosted Electron `<webview>`; do not migrate to `WebContentsView` in this iteration. Explicitly model host-renderer reload, crash, and window close as live-page loss.
9802. Use a T3-owned semantic CDP adapter backed initially by the version-pinned Playwright injected runtime. Do not depend on direct Playwright attachment and do not build locator/actionability semantics from scratch.
9813. Record with CDP screencast plus Chromium MediaRecorder H.264 MP4 encoding. Default to one active recording per window until soak and concurrency budgets pass.
9824. Keep raw TCP as the environment transport, but do not ship a bare loopback listener as the desktop authorization boundary. Require stable authorities, browser-attributed ingress, bounded multiplexing, and explicit redirect/HTTPS policy.
9835. Keep the renderer as the environment transport relay initially because measured IPC overhead is negligible; remove its lifecycle authority.
9846. Default browser partition scope is one persistent partition per environment and project, with optional isolated sessions.
9857. Capability and site policy gates remain as defined in the Security Model section; guest isolation and the annotation/source-attribution pipeline are explicit Phase 1 requirements.
9868. Artifact bytes use a dedicated storage/upload path; deployment-specific backends implement the shared artifact interface.
9879. Locator descriptions re-resolve at action time. Snapshot node references are optional and ephemeral.
98810. Initial controller modes are `human`, `agent`, and `none`; undefined shared control is removed.
989
990## Recommended Implementation Order
991
9921. Introduce session, tab, window, and host contracts from the completed Phase 0 and 0.5 ADRs.
9932. Add the durable app-level webview host and desktop-main registry.
9943. Move browser lifetime out of preview panels and `usePreviewBridge`.
9954. Remove visibility-gated automation.
9965. Add persistent CDP connections and command queues.
9976. Integrate the Playwright injected runtime, per-frame routing, and semantic automation adapter.
9987. Add console and network diagnostics.
9998. Complete browser-attributed ingress and then add the multiplexed raw TCP environment preview tunnel.
10009. Add the selected MediaRecorder evidence pipeline under the one-recording policy.
100110. Add collaboration, tab handoff, and cursor UX.
100211. Add optional environment-hosted Chromium sessions.
100312. Remove superseded preview automation code while preserving annotations, source attribution, and discovered-server UX.
1004
1005## Completion Gates
1006
1007Each implementation phase must include tests for backend changes and must pass:
1008
1009- `vp check`
1010- `vp run typecheck`
1011- `vp test`
1012
1013Run `vp run lint:mobile` only when native mobile code changes.
1014
1015The next iteration is not complete merely when an agent can click the page. It is complete when the same durable browser session can be controlled by an agent, inspected and interrupted by a user, reached from remote environments, and reviewed through reliable evidence artifacts.
apps/web/src/components/ChatView.tsx+18-4
......@@ -161,6 +161,7 @@ import {
161161 type ElementContextDraft,
162162 formatElementContextLabel,
163163} from "../lib/elementContext";
164import { appendPreviewAnnotationPrompt } from "../lib/previewAnnotation";
164165import { ChatComposer, type ChatComposerHandle } from "./chat/ChatComposer";
165166import { ExpandedImageDialog } from "./chat/ExpandedImageDialog";
166167import { PullRequestThreadDialog } from "./PullRequestThreadDialog";
......@@ -1111,6 +1112,9 @@ export default function ChatView(props: ChatViewProps) {
11111112 const setComposerDraftElementContexts = useComposerDraftStore(
11121113 (store) => store.setElementContexts,
11131114 );
1115 const setComposerDraftPreviewAnnotations = useComposerDraftStore(
1116 (store) => store.setPreviewAnnotations,
1117 );
11141118 const setComposerDraftModelSelection = useComposerDraftStore((store) => store.setModelSelection);
11151119 const setComposerDraftRuntimeMode = useComposerDraftStore((store) => store.setRuntimeMode);
11161120 const setComposerDraftInteractionMode = useComposerDraftStore(
......@@ -3555,6 +3559,7 @@ export default function ChatView(props: ChatViewProps) {
35553559 images: composerImages,
35563560 terminalContexts: composerTerminalContexts,
35573561 elementContexts: composerElementContexts,
3562 previewAnnotations: composerPreviewAnnotations,
35583563 selectedProvider: ctxSelectedProvider,
35593564 selectedModel: ctxSelectedModel,
35603565 selectedProviderModels: ctxSelectedProviderModels,
......@@ -3571,7 +3576,7 @@ export default function ChatView(props: ChatViewProps) {
35713576 prompt: promptForSend,
35723577 imageCount: composerImages.length,
35733578 terminalContexts: composerTerminalContexts,
3574 elementContextCount: composerElementContexts.length,
3579 elementContextCount: composerElementContexts.length + composerPreviewAnnotations.length,
35753580 });
35763581 if (showPlanFollowUpPrompt && activeProposedPlan) {
35773582 const followUp = resolvePlanFollowUpSubmission({
......@@ -3590,7 +3595,8 @@ export default function ChatView(props: ChatViewProps) {
35903595 const standaloneSlashCommand =
35913596 composerImages.length === 0 &&
35923597 sendableComposerTerminalContexts.length === 0 &&
3593 composerElementContexts.length === 0
3598 composerElementContexts.length === 0 &&
3599 composerPreviewAnnotations.length === 0
35943600 ? parseStandaloneComposerSlashCommand(trimmed)
35953601 : null;
35963602 if (standaloneSlashCommand) {
......@@ -3639,10 +3645,15 @@ export default function ChatView(props: ChatViewProps) {
36393645 const composerImagesSnapshot = [...composerImages];
36403646 const composerTerminalContextsSnapshot = [...sendableComposerTerminalContexts];
36413647 const composerElementContextsSnapshot = [...composerElementContexts];
3642 const messageTextForSend = appendElementContextsToPrompt(
3648 const composerPreviewAnnotationsSnapshot = [...composerPreviewAnnotations];
3649 const messageTextWithContexts = appendElementContextsToPrompt(
36433650 appendTerminalContextsToPrompt(promptForSend, composerTerminalContextsSnapshot),
36443651 composerElementContextsSnapshot,
36453652 );
3653 const messageTextForSend = composerPreviewAnnotationsSnapshot.reduce(
3654 (text, annotation) => appendPreviewAnnotationPrompt(text, annotation),
3655 messageTextWithContexts,
3656 );
36463657 const messageIdForSend = newMessageId();
36473658 const messageCreatedAt = new Date().toISOString();
36483659 const outgoingMessageText = formatOutgoingPrompt({
......@@ -3810,7 +3821,9 @@ export default function ChatView(props: ChatViewProps) {
38103821 promptRef.current.length === 0 &&
38113822 composerImagesRef.current.length === 0 &&
38123823 composerTerminalContextsRef.current.length === 0 &&
3813 composerElementContextsRef.current.length === 0
3824 composerElementContextsRef.current.length === 0 &&
3825 (useComposerDraftStore.getState().getComposerDraft(composerDraftTarget)?.previewAnnotations
3826 .length ?? 0) === 0
38143827 ) {
38153828 setOptimisticUserMessages((existing) => {
38163829 const removed = existing.filter((message) => message.id === messageIdForSend);
......@@ -3829,6 +3842,7 @@ export default function ChatView(props: ChatViewProps) {
38293842 addComposerDraftImages(composerDraftTarget, retryComposerImages);
38303843 setComposerDraftTerminalContexts(composerDraftTarget, composerTerminalContextsSnapshot);
38313844 setComposerDraftElementContexts(composerDraftTarget, composerElementContextsSnapshot);
3845 setComposerDraftPreviewAnnotations(composerDraftTarget, composerPreviewAnnotationsSnapshot);
38323846 composerRef.current?.resetCursorState({
38333847 cursor: collapseExpandedComposerCursor(promptForSend, promptForSend.length),
38343848 prompt: promptForSend,
apps/web/src/components/chat/ChatComposer.tsx+105-62
......@@ -2,6 +2,7 @@ import type {
22 ApprovalRequestId,
33 EnvironmentId,
44 ModelSelection,
5 PreviewAnnotationPayload,
56 ProviderApprovalDecision,
67 ProviderInteractionMode,
78 ResolvedKeybindingsConfig,
......@@ -55,6 +56,7 @@ import {
5556import { useComposerPathSearch } from "../../lib/composerPathSearchState";
5657import { type ElementContextDraft } from "../../lib/elementContext";
5758import { ComposerPendingElementContexts } from "./ComposerPendingElementContexts";
59import { ComposerPreviewAnnotationCards } from "./ComposerPreviewAnnotationCards";
5860import {
5961 shouldUseCompactComposerPrimaryActions,
6062 shouldUseCompactComposerFooter,
......@@ -403,6 +405,7 @@ export interface ChatComposerHandle {
403405 images: ComposerImageAttachment[];
404406 terminalContexts: TerminalContextDraft[];
405407 elementContexts: ElementContextDraft[];
408 previewAnnotations: PreviewAnnotationPayload[];
406409 selectedPromptEffort: string | null;
407410 selectedModelOptionsForDispatch: unknown;
408411 selectedModelSelection: ModelSelection;
......@@ -611,6 +614,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
611614 const composerImages = composerDraft.images;
612615 const composerTerminalContexts = composerDraft.terminalContexts;
613616 const composerElementContexts = composerDraft.elementContexts;
617 const composerPreviewAnnotations = composerDraft.previewAnnotations;
614618 const nonPersistedComposerImageIds = composerDraft.nonPersistedImageIds;
615619
616620 const setComposerDraftPrompt = useComposerDraftStore((store) => store.setPrompt);
......@@ -629,6 +633,9 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
629633 const removeComposerDraftElementContext = useComposerDraftStore(
630634 (store) => store.removeElementContext,
631635 );
636 const removeComposerDraftPreviewAnnotation = useComposerDraftStore(
637 (store) => store.removePreviewAnnotation,
638 );
632639 const clearComposerDraftPersistedAttachments = useComposerDraftStore(
633640 (store) => store.clearPersistedAttachments,
634641 );
......@@ -889,9 +896,15 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
889896 prompt,
890897 imageCount: composerImages.length,
891898 terminalContexts: composerTerminalContexts,
892 elementContextCount: composerElementContexts.length,
899 elementContextCount: composerElementContexts.length + composerPreviewAnnotations.length,
893900 }),
894 [composerElementContexts.length, composerImages.length, composerTerminalContexts, prompt],
901 [
902 composerElementContexts.length,
903 composerImages.length,
904 composerPreviewAnnotations.length,
905 composerTerminalContexts,
906 prompt,
907 ],
895908 );
896909
897910 // ------------------------------------------------------------------
......@@ -1984,6 +1997,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
19841997 images: composerImagesRef.current,
19851998 terminalContexts: composerTerminalContextsRef.current,
19861999 elementContexts: composerElementContextsRef.current,
2000 previewAnnotations: composerPreviewAnnotations,
19872001 selectedPromptEffort,
19882002 selectedModelOptionsForDispatch,
19892003 selectedModelSelection,
......@@ -2002,6 +2016,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
20022016 composerImagesRef,
20032017 composerTerminalContextsRef,
20042018 composerElementContextsRef,
2019 composerPreviewAnnotations,
20052020 isConnecting,
20062021 isComposerApprovalState,
20072022 pendingUserInputs.length,
......@@ -2239,6 +2254,24 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
22392254 </div>
22402255 )}
22412256
2257 {!isComposerCollapsedMobile &&
2258 !isComposerApprovalState &&
2259 pendingUserInputs.length === 0 &&
2260 composerPreviewAnnotations.length > 0 && (
2261 <ComposerPreviewAnnotationCards
2262 annotations={composerPreviewAnnotations}
2263 images={composerImages}
2264 onRemove={(annotationId) =>
2265 removeComposerDraftPreviewAnnotation(composerDraftTarget, annotationId)
2266 }
2267 onExpandImage={(imageId) => {
2268 const preview = buildExpandedImagePreview(composerImages, imageId);
2269 if (preview) onExpandImage(preview);
2270 }}
2271 className="mb-3"
2272 />
2273 )}
2274
22422275 {!isComposerCollapsedMobile &&
22432276 !isComposerApprovalState &&
22442277 pendingUserInputs.length === 0 &&
......@@ -2255,68 +2288,78 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
22552288 {!isComposerCollapsedMobile &&
22562289 !isComposerApprovalState &&
22572290 pendingUserInputs.length === 0 &&
2258 composerImages.length > 0 && (
2291 composerImages.some(
2292 (image) =>
2293 !composerPreviewAnnotations.some((annotation) => annotation.id === image.id),
2294 ) && (
22592295 <div className="mb-3 flex flex-wrap gap-2">
2260 {composerImages.map((image) => (
2261 <div
2262 key={image.id}
2263 className="relative h-16 w-16 overflow-hidden rounded-lg border border-border/80 bg-background"
2264 >
2265 {image.previewUrl ? (
2266 <button
2267 type="button"
2268 className="h-full w-full cursor-zoom-in"
2269 aria-label={`Preview ${image.name}`}
2270 onClick={() => {
2271 const preview = buildExpandedImagePreview(composerImages, image.id);
2272 if (!preview) return;
2273 onExpandImage(preview);
2274 }}
2275 >
2276 <img
2277 src={image.previewUrl}
2278 alt={image.name}
2279 className="h-full w-full object-cover"
2280 />
2281 </button>
2282 ) : (
2283 <div className="flex h-full w-full items-center justify-center px-1 text-center text-[10px] text-muted-foreground/70">
2284 {image.name}
2285 </div>
2286 )}
2287 {nonPersistedComposerImageIdSet.has(image.id) && (
2288 <Tooltip>
2289 <TooltipTrigger
2290 render={
2291 <span
2292 role="img"
2293 aria-label="Draft attachment may not persist"
2294 className="absolute left-1 top-1 inline-flex items-center justify-center rounded bg-background/85 p-0.5 text-amber-600"
2295 >
2296 <CircleAlertIcon className="size-3" />
2297 </span>
2298 }
2299 />
2300 <TooltipPopup
2301 side="top"
2302 className="max-w-64 whitespace-normal leading-tight"
2303 >
2304 Draft attachment could not be saved locally and may be lost on
2305 navigation.
2306 </TooltipPopup>
2307 </Tooltip>
2308 )}
2309 <Button
2310 variant="ghost"
2311 size="icon-xs"
2312 className="absolute right-1 top-1 bg-background/80 hover:bg-background/90"
2313 onClick={() => removeComposerImage(image.id)}
2314 aria-label={`Remove ${image.name}`}
2296 {composerImages
2297 .filter(
2298 (image) =>
2299 !composerPreviewAnnotations.some(
2300 (annotation) => annotation.id === image.id,
2301 ),
2302 )
2303 .map((image) => (
2304 <div
2305 key={image.id}
2306 className="relative h-16 w-16 overflow-hidden rounded-lg border border-border/80 bg-background"
23152307 >
2316 <XIcon />
2317 </Button>
2318 </div>
2319 ))}
2308 {image.previewUrl ? (
2309 <button
2310 type="button"
2311 className="h-full w-full cursor-zoom-in"
2312 aria-label={`Preview ${image.name}`}
2313 onClick={() => {
2314 const preview = buildExpandedImagePreview(composerImages, image.id);
2315 if (!preview) return;
2316 onExpandImage(preview);
2317 }}
2318 >
2319 <img
2320 src={image.previewUrl}
2321 alt={image.name}
2322 className="h-full w-full object-cover"
2323 />
2324 </button>
2325 ) : (
2326 <div className="flex h-full w-full items-center justify-center px-1 text-center text-[10px] text-muted-foreground/70">
2327 {image.name}
2328 </div>
2329 )}
2330 {nonPersistedComposerImageIdSet.has(image.id) && (
2331 <Tooltip>
2332 <TooltipTrigger
2333 render={
2334 <span
2335 role="img"
2336 aria-label="Draft attachment may not persist"
2337 className="absolute left-1 top-1 inline-flex items-center justify-center rounded bg-background/85 p-0.5 text-amber-600"
2338 >
2339 <CircleAlertIcon className="size-3" />
2340 </span>
2341 }
2342 />
2343 <TooltipPopup
2344 side="top"
2345 className="max-w-64 whitespace-normal leading-tight"
2346 >
2347 Draft attachment could not be saved locally and may be lost on
2348 navigation.
2349 </TooltipPopup>
2350 </Tooltip>
2351 )}
2352 <Button
2353 variant="ghost"
2354 size="icon-xs"
2355 className="absolute right-1 top-1 bg-background/80 hover:bg-background/90"
2356 onClick={() => removeComposerImage(image.id)}
2357 aria-label={`Remove ${image.name}`}
2358 >
2359 <XIcon />
2360 </Button>
2361 </div>
2362 ))}
23202363 </div>
23212364 )}
23222365
apps/web/src/components/chat/ComposerPreviewAnnotationCards.test.tsx created+46
......@@ -0,0 +1,46 @@
1import type { PreviewAnnotationPayload } from "@t3tools/contracts";
2import { renderToStaticMarkup } from "react-dom/server";
3import { describe, expect, it, vi } from "vite-plus/test";
4
5import { ComposerPreviewAnnotationCards } from "./ComposerPreviewAnnotationCards";
6
7const annotation: PreviewAnnotationPayload = {
8 id: "annotation_1",
9 pageUrl: "http://localhost:3000/welcome",
10 pageTitle: "Welcome",
11 comment: "Make this headline feel intentional.",
12 elements: [],
13 regions: [{ id: "region_1", rect: { x: 1, y: 2, width: 30, height: 20 } }],
14 strokes: [],
15 styleChanges: [
16 {
17 targetId: "element_1",
18 selector: "h1",
19 property: "font-size",
20 previousValue: "32px",
21 value: "40px",
22 },
23 ],
24 screenshot: null,
25 createdAt: "2026-06-13T00:00:00.000Z",
26};
27
28describe("ComposerPreviewAnnotationCards", () => {
29 it("presents the annotation as one contextual attachment", () => {
30 const markup = renderToStaticMarkup(
31 <ComposerPreviewAnnotationCards
32 annotations={[annotation]}
33 images={[]}
34 onRemove={vi.fn()}
35 onExpandImage={vi.fn()}
36 />,
37 );
38
39 expect(markup).toContain("Make this headline feel intentional.");
40 expect(markup).toContain('title="1 region"');
41 expect(markup).toContain('title="1 style change"');
42 expect(markup).not.toContain("Welcome");
43 expect(markup).not.toContain("localhost:3000");
44 expect(markup).not.toContain("Preview annotation");
45 });
46});
apps/web/src/components/chat/ComposerPreviewAnnotationCards.tsx created+144
......@@ -0,0 +1,144 @@
1import type { PreviewAnnotationPayload } from "@t3tools/contracts";
2import { Frame, MousePointerClick, Paintbrush, PenLine, X } from "lucide-react";
3import type { ReactNode } from "react";
4
5import type { ComposerImageAttachment } from "~/composerDraftStore";
6import { formatElementContextLabel, normalizeElementContextSelection } from "~/lib/elementContext";
7import { cn } from "~/lib/utils";
8
9interface ComposerPreviewAnnotationCardsProps {
10 annotations: ReadonlyArray<PreviewAnnotationPayload>;
11 images: ReadonlyArray<ComposerImageAttachment>;
12 onRemove: (annotationId: string) => void;
13 onExpandImage: (imageId: string) => void;
14 className?: string;
15}
16
17function TargetStat(props: { icon: ReactNode; count: number; label: string }) {
18 return (
19 <span
20 className="inline-flex items-center gap-1 text-[10px] font-medium text-muted-foreground"
21 title={`${props.count} ${props.label}${props.count === 1 ? "" : "s"}`}
22 >
23 {props.icon}
24 {props.count}
25 </span>
26 );
27}
28
29export function ComposerPreviewAnnotationCards({
30 annotations,
31 images,
32 onRemove,
33 onExpandImage,
34 className,
35}: ComposerPreviewAnnotationCardsProps) {
36 if (annotations.length === 0) return null;
37 const imagesById = new Map(images.map((image) => [image.id, image]));
38
39 return (
40 <div className={cn("flex flex-wrap gap-1.5", className)}>
41 {annotations.map((annotation) => {
42 const image = imagesById.get(annotation.id);
43 const elementLabels = annotation.elements.flatMap((target) => {
44 const context = normalizeElementContextSelection(target.element);
45 return context ? [{ id: target.id, label: formatElementContextLabel(context) }] : [];
46 });
47 return (
48 <section
49 key={annotation.id}
50 className="group/preview-annotation relative flex min-w-0 max-w-full items-center overflow-hidden rounded-lg border border-border/80 bg-background/72"
51 >
52 {image?.previewUrl ? (
53 <button
54 type="button"
55 aria-label={`Preview ${image.name}`}
56 className="size-14 shrink-0 cursor-zoom-in overflow-hidden border-r border-border/70 bg-muted"
57 onClick={() => onExpandImage(image.id)}
58 >
59 <img
60 src={image.previewUrl}
61 alt="Annotated preview crop"
62 className="size-full object-cover transition duration-200 group-hover/preview-annotation:scale-[1.03]"
63 />
64 </button>
65 ) : (
66 <span className="grid size-10 shrink-0 place-items-center border-r border-border/70 text-blue-500">
67 <MousePointerClick className="size-3.5" />
68 </span>
69 )}
70 <div className="min-w-0 px-2.5 py-2 pr-8">
71 {annotation.comment.trim() ? (
72 <p className="max-w-80 truncate text-xs font-medium text-foreground/90">
73 {annotation.comment.trim()}
74 </p>
75 ) : null}
76 <div
77 className={cn(
78 "flex min-w-0 items-center gap-2",
79 annotation.comment.trim() && "mt-1",
80 )}
81 >
82 {elementLabels.length > 0 ? (
83 <div className="flex min-w-0 items-center gap-1">
84 {elementLabels.slice(0, 2).map(({ id, label }) => (
85 <span
86 key={id}
87 className="max-w-40 truncate font-mono text-[10px] text-foreground/65"
88 >
89 {label}
90 </span>
91 ))}
92 {elementLabels.length > 2 ? (
93 <span className="text-[10px] text-muted-foreground">
94 +{elementLabels.length - 2}
95 </span>
96 ) : null}
97 </div>
98 ) : null}
99 <div className="flex shrink-0 items-center gap-2">
100 {annotation.elements.length > 0 ? (
101 <TargetStat
102 icon={<MousePointerClick className="size-3" />}
103 count={annotation.elements.length}
104 label="element"
105 />
106 ) : null}
107 {annotation.regions.length > 0 ? (
108 <TargetStat
109 icon={<Frame className="size-3" />}
110 count={annotation.regions.length}
111 label="region"
112 />
113 ) : null}
114 {annotation.strokes.length > 0 ? (
115 <TargetStat
116 icon={<PenLine className="size-3" />}
117 count={annotation.strokes.length}
118 label="drawing"
119 />
120 ) : null}
121 {annotation.styleChanges.length > 0 ? (
122 <TargetStat
123 icon={<Paintbrush className="size-3" />}
124 count={annotation.styleChanges.length}
125 label="style change"
126 />
127 ) : null}
128 </div>
129 </div>
130 </div>
131 <button
132 type="button"
133 aria-label="Remove preview annotation"
134 className="absolute right-1.5 top-1.5 grid size-5 place-items-center rounded text-muted-foreground/60 transition hover:bg-muted hover:text-foreground"
135 onClick={() => onRemove(annotation.id)}
136 >
137 <X className="size-3" />
138 </button>
139 </section>
140 );
141 })}
142 </div>
143 );
144}
apps/web/src/components/chat/MessagesTimeline.tsx+78-4
......@@ -49,6 +49,7 @@ import {
4949 HammerIcon,
5050 MessageCircleIcon,
5151 MousePointerClickIcon,
52 PaintbrushIcon,
5253 MinusIcon,
5354 SquarePenIcon,
5455 TerminalIcon,
......@@ -83,6 +84,10 @@ import {
8384 extractTrailingElementContexts,
8485 type ParsedElementContextEntry,
8586} from "~/lib/elementContext";
87import {
88 extractTrailingPreviewAnnotation,
89 type ParsedPreviewAnnotation,
90} from "~/lib/previewAnnotation";
8691import { cn } from "~/lib/utils";
8792import { useUiStateStore } from "~/uiStateStore";
8893import { type TimestampFormat } from "@t3tools/contracts/settings";
......@@ -433,15 +438,25 @@ function UserTimelineRow({ row }: { row: Extract<TimelineRow, { kind: "message"
433438 const userImages = row.message.attachments ?? [];
434439 const displayedUserMessage = deriveDisplayedUserMessageState(row.message.text);
435440 const terminalContexts = displayedUserMessage.contexts;
436 const elementContextState = extractTrailingElementContexts(displayedUserMessage.visibleText);
441 const previewAnnotations: ParsedPreviewAnnotation[] = [];
442 let visibleText = displayedUserMessage.visibleText;
443 while (true) {
444 const extracted = extractTrailingPreviewAnnotation(visibleText);
445 if (!extracted.annotation) break;
446 previewAnnotations.unshift(extracted.annotation);
447 visibleText = extracted.promptText;
448 }
449 const elementContextState = extractTrailingElementContexts(visibleText);
450 const previewImages = userImages.filter((image) => image.name.startsWith("preview-annotation-"));
451 const regularImages = userImages.filter((image) => !image.name.startsWith("preview-annotation-"));
437452 const canRevertAgentWork = typeof row.revertTurnCount === "number";
438453
439454 return (
440455 <div className="group flex flex-col items-end gap-1">
441456 <div className="relative max-w-[80%] rounded-2xl border border-border bg-secondary p-3">
442 {userImages.length > 0 && (
457 {regularImages.length > 0 && (
443458 <div className="mb-2 grid max-w-[420px] grid-cols-2 gap-2">
444 {userImages.map((image: NonNullable<TimelineMessage["attachments"]>[number]) => (
459 {regularImages.map((image: NonNullable<TimelineMessage["attachments"]>[number]) => (
445460 <div
446461 key={image.id}
447462 className="overflow-hidden rounded-lg border border-border/80 bg-background/70"
......@@ -452,7 +467,7 @@ function UserTimelineRow({ row }: { row: Extract<TimelineRow, { kind: "message"
452467 className="h-full w-full cursor-zoom-in"
453468 aria-label={`Preview ${image.name}`}
454469 onClick={() => {
455 const preview = buildExpandedImagePreview(userImages, image.id);
470 const preview = buildExpandedImagePreview(regularImages, image.id);
456471 if (!preview) return;
457472 ctx.onImageExpand(preview);
458473 }}
......@@ -472,6 +487,13 @@ function UserTimelineRow({ row }: { row: Extract<TimelineRow, { kind: "message"
472487 ))}
473488 </div>
474489 )}
490 {previewAnnotations.map((annotation, index) => (
491 <UserMessagePreviewAnnotationCard
492 key={annotation.id}
493 annotation={annotation}
494 image={previewImages[index] ?? null}
495 />
496 ))}
475497 {elementContextState.contexts.length > 0 ? (
476498 <div className="mb-2 flex flex-wrap gap-1.5">
477499 {elementContextState.contexts.map((context) => (
......@@ -944,6 +966,58 @@ const UserMessageElementContextChip = memo(function UserMessageElementContextChi
944966 );
945967});
946968
969function UserMessagePreviewAnnotationCard(props: {
970 annotation: ParsedPreviewAnnotation;
971 image: NonNullable<TimelineMessage["attachments"]>[number] | null;
972}) {
973 const ctx = use(TimelineRowCtx);
974 return (
975 <div className="mb-2 flex max-w-full items-center overflow-hidden rounded-lg border border-border/70 bg-background/70">
976 {props.image?.previewUrl ? (
977 <button
978 type="button"
979 className="size-14 shrink-0 cursor-zoom-in overflow-hidden border-r border-border/70 bg-muted"
980 aria-label={`Preview ${props.image.name}`}
981 onClick={() => {
982 if (!props.image) return;
983 const preview = buildExpandedImagePreview([props.image], props.image.id);
984 if (preview) ctx.onImageExpand(preview);
985 }}
986 >
987 <img
988 src={props.image.previewUrl}
989 alt="Annotated preview crop"
990 className="size-full object-cover"
991 />
992 </button>
993 ) : null}
994 <div className="min-w-0 px-2.5 py-2">
995 {props.annotation.comment ? (
996 <div className="max-w-80 truncate text-xs font-medium text-foreground/90">
997 {props.annotation.comment}
998 </div>
999 ) : null}
1000 <div
1001 className={cn(
1002 "flex items-center gap-2 text-[10px] text-muted-foreground",
1003 props.annotation.comment && "mt-1",
1004 )}
1005 >
1006 {props.annotation.targetSummary ? (
1007 <span className="truncate">{props.annotation.targetSummary}</span>
1008 ) : null}
1009 {props.annotation.styleChanges.length > 0 ? (
1010 <span className="inline-flex shrink-0 items-center gap-1">
1011 <PaintbrushIcon className="size-3" />
1012 {props.annotation.styleChanges.length}
1013 </span>
1014 ) : null}
1015 </div>
1016 </div>
1017 </div>
1018 );
1019}
1020
9471021const MAX_COLLAPSED_USER_MESSAGE_LINES = 8;
9481022const MAX_COLLAPSED_USER_MESSAGE_LENGTH = 600;
9491023const COLLAPSED_USER_MESSAGE_FADE_HEIGHT_REM = 1.75;
apps/web/src/components/preview/PreviewView.tsx+4-15
......@@ -6,11 +6,7 @@ import { useCallback, useEffect, useRef, useState } from "react";
66
77import { useComposerDraftStore } from "~/composerDraftStore";
88import { ensureEnvironmentApi } from "~/environmentApi";
9import { normalizeElementContextSelection } from "~/lib/elementContext";
10import {
11 appendPreviewAnnotationPrompt,
12 previewAnnotationScreenshotFile,
13} from "~/lib/previewAnnotation";
9import { previewAnnotationScreenshotFile } from "~/lib/previewAnnotation";
1410import { ensureLocalApi } from "~/localApi";
1511import { selectThreadPreviewState, usePreviewStateStore } from "~/previewStateStore";
1612import { resolveDiscoveredServerUrl } from "~/browser/browserTargetResolver";
......@@ -62,10 +58,8 @@ export function PreviewView({ threadRef, tabId: requestedTabId, configuredUrls,
6258 );
6359 const applyServerSnapshot = usePreviewStateStore((state) => state.applyServerSnapshot);
6460 const rememberUrl = usePreviewStateStore((state) => state.rememberUrl);
65 const addElementContext = useComposerDraftStore((store) => store.addElementContext);
61 const addPreviewAnnotation = useComposerDraftStore((store) => store.addPreviewAnnotation);
6662 const addImage = useComposerDraftStore((store) => store.addImage);
67 const getComposerDraft = useComposerDraftStore((store) => store.getComposerDraft);
68 const setPrompt = useComposerDraftStore((store) => store.setPrompt);
6963
7064 usePreviewSession(threadRef);
7165
......@@ -421,12 +415,7 @@ export function PreviewView({ threadRef, tabId: requestedTabId, configuredUrls,
421415 try {
422416 const annotation = await previewBridge.pickElement(tabId);
423417 if (!annotation) return;
424 for (const target of annotation.elements) {
425 const selection = normalizeElementContextSelection(target.element);
426 if (selection) addElementContext(threadRef, selection);
427 }
428 const currentPrompt = getComposerDraft(threadRef)?.prompt ?? "";
429 setPrompt(threadRef, appendPreviewAnnotationPrompt(currentPrompt, annotation));
418 addPreviewAnnotation(threadRef, annotation);
430419 const screenshotFile = await previewAnnotationScreenshotFile(annotation);
431420 if (screenshotFile && annotation.screenshot) {
432421 addImage(threadRef, {
......@@ -462,7 +451,7 @@ export function PreviewView({ threadRef, tabId: requestedTabId, configuredUrls,
462451 }
463452 }
464453 })();
465 }, [addElementContext, addImage, getComposerDraft, setPrompt, tabId, threadRef]);
454 }, [addImage, addPreviewAnnotation, tabId, threadRef]);
466455
467456 // If the active tab changes mid-pick (close, thread switch, hot restart),
468457 // tell main to tear down the in-flight session AND reset our local toggle
apps/web/src/composerDraftStore.ts+93-1
......@@ -9,6 +9,8 @@ import {
99 ProviderInteractionMode,
1010 ProviderDriverKind,
1111 ProviderOptionSelection,
12 PreviewAnnotationPayloadSchema,
13 type PreviewAnnotationPayload,
1214 RuntimeMode,
1315 type ServerProvider,
1416 type ScopedProjectRef,
......@@ -52,7 +54,7 @@ const isRuntimeMode = Schema.is(RuntimeMode);
5254const isProviderDriverKind = Schema.is(ProviderDriverKind);
5355
5456export const COMPOSER_DRAFT_STORAGE_KEY = "t3code:composer-drafts:v1";
55const COMPOSER_DRAFT_STORAGE_VERSION = 6;
57const COMPOSER_DRAFT_STORAGE_VERSION = 7;
5658const DraftThreadEnvModeSchema = Schema.Literals(["local", "worktree"]);
5759export type DraftThreadEnvMode = typeof DraftThreadEnvModeSchema.Type;
5860
......@@ -125,6 +127,7 @@ const PersistedComposerThreadDraftState = Schema.Struct({
125127 attachments: Schema.Array(PersistedComposerImageAttachment),
126128 terminalContexts: Schema.optionalKey(Schema.Array(PersistedTerminalContextDraft)),
127129 elementContexts: Schema.optionalKey(Schema.Array(PersistedElementContextDraft)),
130 previewAnnotations: Schema.optionalKey(Schema.Array(PreviewAnnotationPayloadSchema)),
128131 // Keyed by `ProviderInstanceId` (open branded slug) so custom provider
129132 // instances (e.g. `codex_personal`) round-trip alongside the built-in
130133 // `codex` / `claudeAgent` / ... entries. Every prior `ProviderDriverKind`
......@@ -252,6 +255,7 @@ export interface ComposerThreadDraftState {
252255 * re-derive the snapshot from on reload.
253256 */
254257 elementContexts: ElementContextDraft[];
258 previewAnnotations: PreviewAnnotationPayload[];
255259 /**
256260 * Per-instance model selection. Keyed by `ProviderInstanceId` (open
257261 * branded slug) so a default `codex` instance and a user-authored
......@@ -451,6 +455,15 @@ interface ComposerDraftStoreState {
451455 ) => void;
452456 removeElementContext: (threadRef: ComposerThreadTarget, contextId: string) => void;
453457 clearElementContexts: (threadRef: ComposerThreadTarget) => void;
458 addPreviewAnnotation: (
459 threadRef: ComposerThreadTarget,
460 annotation: PreviewAnnotationPayload,
461 ) => void;
462 setPreviewAnnotations: (
463 threadRef: ComposerThreadTarget,
464 annotations: ReadonlyArray<PreviewAnnotationPayload>,
465 ) => void;
466 removePreviewAnnotation: (threadRef: ComposerThreadTarget, annotationId: string) => void;
454467 clearPersistedAttachments: (threadRef: ComposerThreadTarget) => void;
455468 syncPersistedAttachments: (
456469 threadRef: ComposerThreadTarget,
......@@ -527,10 +540,12 @@ const EMPTY_IDS: string[] = [];
527540const EMPTY_PERSISTED_ATTACHMENTS: PersistedComposerImageAttachment[] = [];
528541const EMPTY_TERMINAL_CONTEXTS: TerminalContextDraft[] = [];
529542const EMPTY_ELEMENT_CONTEXTS: ElementContextDraft[] = [];
543const EMPTY_PREVIEW_ANNOTATIONS: PreviewAnnotationPayload[] = [];
530544Object.freeze(EMPTY_IMAGES);
531545Object.freeze(EMPTY_IDS);
532546Object.freeze(EMPTY_PERSISTED_ATTACHMENTS);
533547Object.freeze(EMPTY_ELEMENT_CONTEXTS);
548Object.freeze(EMPTY_PREVIEW_ANNOTATIONS);
534549const EMPTY_MODEL_SELECTION_BY_PROVIDER: Partial<Record<ProviderDriverKind, ModelSelection>> =
535550 Object.freeze({});
536551const EMPTY_COMPOSER_DRAFT_MODEL_STATE = Object.freeze<ComposerDraftModelState>({
......@@ -545,6 +560,7 @@ const EMPTY_THREAD_DRAFT = Object.freeze<ComposerThreadDraftState>({
545560 persistedAttachments: EMPTY_PERSISTED_ATTACHMENTS,
546561 terminalContexts: EMPTY_TERMINAL_CONTEXTS,
547562 elementContexts: EMPTY_ELEMENT_CONTEXTS,
563 previewAnnotations: EMPTY_PREVIEW_ANNOTATIONS,
548564 modelSelectionByProvider: EMPTY_MODEL_SELECTION_BY_PROVIDER,
549565 activeProvider: null,
550566 runtimeMode: null,
......@@ -565,6 +581,7 @@ export function createEmptyThreadDraft(): ComposerThreadDraftState {
565581 persistedAttachments: [],
566582 terminalContexts: [],
567583 elementContexts: [],
584 previewAnnotations: [],
568585 modelSelectionByProvider: {},
569586 activeProvider: null,
570587 runtimeMode: null,
......@@ -636,6 +653,7 @@ function shouldRemoveDraft(draft: ComposerThreadDraftState): boolean {
636653 draft.persistedAttachments.length === 0 &&
637654 draft.terminalContexts.length === 0 &&
638655 draft.elementContexts.length === 0 &&
656 draft.previewAnnotations.length === 0 &&
639657 Object.keys(draft.modelSelectionByProvider).length === 0 &&
640658 draft.activeProvider === null &&
641659 draft.runtimeMode === null &&
......@@ -1776,6 +1794,7 @@ function partializeComposerDraftStoreState(
17761794 draft.persistedAttachments.length === 0 &&
17771795 draft.terminalContexts.length === 0 &&
17781796 draft.elementContexts.length === 0 &&
1797 draft.previewAnnotations.length === 0 &&
17791798 !hasModelData &&
17801799 draft.runtimeMode === null &&
17811800 draft.interactionMode === null
......@@ -1815,6 +1834,13 @@ function partializeComposerDraftStoreState(
18151834 })),
18161835 }
18171836 : {}),
1837 ...(draft.previewAnnotations.length > 0
1838 ? {
1839 previewAnnotations: draft.previewAnnotations.map(
1840 (annotation) => ({ ...annotation }) as DeepMutable<PreviewAnnotationPayload>,
1841 ),
1842 }
1843 : {}),
18181844 ...(hasModelData
18191845 ? {
18201846 modelSelectionByProvider: compactModelSelectionByProvider(
......@@ -2055,6 +2081,8 @@ function toHydratedThreadDraft(
20552081 persistedDraft.elementContexts?.map((context) => ({
20562082 ...context,
20572083 })) ?? [],
2084 previewAnnotations:
2085 persistedDraft.previewAnnotations?.map((annotation) => ({ ...annotation })) ?? [],
20582086 modelSelectionByProvider,
20592087 activeProvider,
20602088 runtimeMode: persistedDraft.runtimeMode ?? null,
......@@ -3057,6 +3085,69 @@ const composerDraftStore = create<ComposerDraftStoreState>()(
30573085 return { draftsByThreadKey: nextDraftsByThreadKey };
30583086 });
30593087 },
3088 addPreviewAnnotation: (threadRef, annotation) => {
3089 const threadKey = resolveComposerDraftKey(get(), threadRef);
3090 if (!threadKey) return;
3091 set((state) => {
3092 const existing = state.draftsByThreadKey[threadKey] ?? createEmptyThreadDraft();
3093 const nextAnnotations = existing.previewAnnotations.filter(
3094 (entry) => entry.id !== annotation.id,
3095 );
3096 const compactAnnotation: PreviewAnnotationPayload = {
3097 ...annotation,
3098 screenshot: annotation.screenshot ? { ...annotation.screenshot, dataUrl: "" } : null,
3099 };
3100 return {
3101 draftsByThreadKey: {
3102 ...state.draftsByThreadKey,
3103 [threadKey]: {
3104 ...existing,
3105 previewAnnotations: [...nextAnnotations, compactAnnotation],
3106 },
3107 },
3108 };
3109 });
3110 },
3111 setPreviewAnnotations: (threadRef, annotations) => {
3112 const threadKey = resolveComposerDraftKey(get(), threadRef);
3113 if (!threadKey) return;
3114 set((state) => {
3115 const existing = state.draftsByThreadKey[threadKey] ?? createEmptyThreadDraft();
3116 return {
3117 draftsByThreadKey: {
3118 ...state.draftsByThreadKey,
3119 [threadKey]: { ...existing, previewAnnotations: [...annotations] },
3120 },
3121 };
3122 });
3123 },
3124 removePreviewAnnotation: (threadRef, annotationId) => {
3125 const threadKey = resolveComposerDraftKey(get(), threadRef);
3126 if (!threadKey || !annotationId) return;
3127 set((state) => {
3128 const current = state.draftsByThreadKey[threadKey];
3129 if (!current) return state;
3130 const previewAnnotations = current.previewAnnotations.filter(
3131 (entry) => entry.id !== annotationId,
3132 );
3133 if (previewAnnotations.length === current.previewAnnotations.length) return state;
3134 const nextDraft = {
3135 ...current,
3136 previewAnnotations,
3137 images: current.images.filter((image) => image.id !== annotationId),
3138 persistedAttachments: current.persistedAttachments.filter(
3139 (image) => image.id !== annotationId,
3140 ),
3141 nonPersistedImageIds: current.nonPersistedImageIds.filter(
3142 (imageId) => imageId !== annotationId,
3143 ),
3144 };
3145 const nextDraftsByThreadKey = { ...state.draftsByThreadKey };
3146 if (shouldRemoveDraft(nextDraft)) delete nextDraftsByThreadKey[threadKey];
3147 else nextDraftsByThreadKey[threadKey] = nextDraft;
3148 return { draftsByThreadKey: nextDraftsByThreadKey };
3149 });
3150 },
30603151 clearPersistedAttachments: (threadRef) => {
30613152 const threadKey = resolveComposerDraftKey(get(), threadRef) ?? "";
30623153 if (threadKey.length === 0) {
......@@ -3130,6 +3221,7 @@ const composerDraftStore = create<ComposerDraftStoreState>()(
31303221 persistedAttachments: [],
31313222 terminalContexts: [],
31323223 elementContexts: [],
3224 previewAnnotations: [],
31333225 };
31343226 const nextDraftsByThreadKey = { ...state.draftsByThreadKey };
31353227 if (shouldRemoveDraft(nextDraft)) {
apps/web/src/lib/previewAnnotation.test.ts+29-2
......@@ -1,7 +1,11 @@
11import type { PreviewAnnotationPayload } from "@t3tools/contracts";
22import { describe, expect, it } from "vite-plus/test";
33
4import { appendPreviewAnnotationPrompt, buildPreviewAnnotationPrompt } from "./previewAnnotation";
4import {
5 appendPreviewAnnotationPrompt,
6 buildPreviewAnnotationPrompt,
7 extractTrailingPreviewAnnotation,
8} from "./previewAnnotation";
59
610const annotation: PreviewAnnotationPayload = {
711 id: "annotation_1",
......@@ -53,8 +57,31 @@ describe("preview annotations", () => {
5357 it("appends to an existing composer prompt", () => {
5458 expect(
5559 appendPreviewAnnotationPrompt("Fix this", annotation).startsWith(
56 "Fix this\n\nPreview annotation:",
60 "Fix this\n\n<preview_annotation>",
5761 ),
5862 ).toBe(true);
5963 });
64
65 it("extracts annotation presentation from a sent prompt", () => {
66 const result = extractTrailingPreviewAnnotation(
67 appendPreviewAnnotationPrompt("Fix this", annotation),
68 );
69 expect(result.promptText).toBe("Fix this");
70 expect(result.annotation).toMatchObject({
71 title: "Example",
72 targetSummary: "1 marked region, 1 drawing.",
73 hasScreenshot: true,
74 });
75 });
76
77 it("extracts multiple trailing annotations one at a time", () => {
78 const first = appendPreviewAnnotationPrompt("Fix this", annotation);
79 const secondAnnotation = { ...annotation, id: "annotation_2", pageTitle: "Details" };
80 const second = appendPreviewAnnotationPrompt(first, secondAnnotation);
81 const extractedSecond = extractTrailingPreviewAnnotation(second);
82 const extractedFirst = extractTrailingPreviewAnnotation(extractedSecond.promptText);
83 expect(extractedSecond.annotation?.id).toBe("annotation_2");
84 expect(extractedFirst.annotation?.id).toBe("annotation_1");
85 expect(extractedFirst.promptText).toBe("Fix this");
86 });
6087});
apps/web/src/lib/previewAnnotation.ts+60-2
......@@ -1,8 +1,29 @@
11import type { PreviewAnnotationPayload } from "@t3tools/contracts";
2import { buildElementContextBlock, normalizeElementContextSelection } from "./elementContext";
3
4const TRAILING_PREVIEW_ANNOTATION_BLOCK_PATTERN =
5 /\n*<preview_annotation>\n((?:(?!<preview_annotation>)[\s\S])*)\n<\/preview_annotation>\s*$/;
6
7export interface ParsedPreviewAnnotation {
8 id: string;
9 title: string;
10 comment: string;
11 targetSummary: string;
12 styleChanges: string[];
13 hasScreenshot: boolean;
14}
15
16export interface ExtractedPreviewAnnotation {
17 promptText: string;
18 annotation: ParsedPreviewAnnotation | null;
19}
220
321export function buildPreviewAnnotationPrompt(annotation: PreviewAnnotationPayload): string {
422 const lines = ["Preview annotation:"];
5 if (annotation.comment.trim()) lines.push(annotation.comment.trim());
23 lines.push(`Id: ${annotation.id}`);
24 const title = annotation.pageTitle?.trim() || annotation.pageUrl.trim() || "Preview";
25 lines.push(`Page: ${title}`);
26 if (annotation.comment.trim()) lines.push(`Comment: ${annotation.comment.trim()}`);
627 const targets: string[] = [];
728 if (annotation.elements.length > 0) {
829 targets.push(
......@@ -29,7 +50,12 @@ export function buildPreviewAnnotationPrompt(annotation: PreviewAnnotationPayloa
2950 if (annotation.screenshot) {
3051 lines.push("The attached screenshot is the annotated preview crop.");
3152 }
32 return lines.join("\n");
53 const elementContexts = annotation.elements
54 .map((target) => normalizeElementContextSelection(target.element))
55 .filter((context) => context !== null);
56 const elementBlock = buildElementContextBlock(elementContexts);
57 if (elementBlock) lines.push(elementBlock);
58 return ["<preview_annotation>", ...lines, "</preview_annotation>"].join("\n");
3359}
3460
3561export function appendPreviewAnnotationPrompt(
......@@ -41,6 +67,38 @@ export function appendPreviewAnnotationPrompt(
4167 return trimmed ? `${trimmed}\n\n${annotationText}` : annotationText;
4268}
4369
70export function extractTrailingPreviewAnnotation(prompt: string): ExtractedPreviewAnnotation {
71 const match = TRAILING_PREVIEW_ANNOTATION_BLOCK_PATTERN.exec(prompt);
72 if (!match) return { promptText: prompt, annotation: null };
73 const body = match[1] ?? "";
74 const lines = body.split("\n");
75 const pageLine = lines.find((line) => line.startsWith("Page: "));
76 const idLine = lines.find((line) => line.startsWith("Id: "));
77 const commentLine = lines.find((line) => line.startsWith("Comment: "));
78 const targetsLine = lines.find((line) => line.startsWith("Targets: "));
79 const styleHeadingIndex = lines.indexOf("Requested visual changes:");
80 const linesAfterStyleHeading = lines.slice(styleHeadingIndex + 1);
81 const elementContextIndex = linesAfterStyleHeading.indexOf("<element_context>");
82 const styleChanges =
83 styleHeadingIndex < 0
84 ? []
85 : linesAfterStyleHeading
86 .slice(0, elementContextIndex < 0 ? undefined : elementContextIndex)
87 .filter((line) => line.startsWith("- "))
88 .map((line) => line.slice(2));
89 return {
90 promptText: prompt.slice(0, match.index).replace(/\n+$/, ""),
91 annotation: {
92 id: idLine?.slice("Id: ".length).trim() || `${match.index}`,
93 title: pageLine?.slice("Page: ".length).trim() || "Preview annotation",
94 comment: commentLine?.slice("Comment: ".length).trim() || "",
95 targetSummary: targetsLine?.slice("Targets: ".length).trim() || "",
96 styleChanges,
97 hasScreenshot: body.includes("The attached screenshot is the annotated preview crop."),
98 },
99 };
100}
101
44102export async function previewAnnotationScreenshotFile(
45103 annotation: PreviewAnnotationPayload,
46104): Promise<File | null> {