diff --git a/.plans/shared-http-mcp-server-with-preview-automation-revised.md b/.plans/shared-http-mcp-server-with-preview-automation-revised.md new file mode 100644 index 0000000000000000000000000000000000000000..12b2be5f78bbe935450b775ded60b910e911df2c --- /dev/null +++ b/.plans/shared-http-mcp-server-with-preview-automation-revised.md @@ -0,0 +1,520 @@ +# Shared HTTP MCP Server with Preview Automation + +## Summary + +Embed one reusable HTTP MCP server in each T3 environment server. Every agent session connects to that shared MCP endpoint using a session-scoped bearer token. + +Preview browser automation is the first MCP toolkit, not the purpose or boundary of the MCP server. Future T3 toolkits register with the same server. + +Architecture: + +`agent session` -> `shared T3 HTTP MCP server` -> `tool dispatcher` -> `preview broker` -> `focused desktop client` -> `Electron webview via CDP` + +No per-thread MCP process, stdio transport, headless browser, or automatic remote port forwarding. + +## Process Model + +Each `apps/server` process owns exactly one MCP server instance. + +- MCP transport: HTTP. +- MCP endpoint: `/mcp`. +- MCP lifetime: environment server lifetime. +- Agent sessions create MCP protocol sessions/connections, not OS processes. +- Toolkit registration happens once during server startup. +- Provider session termination revokes only its scoped credential. + +Implement the endpoint with: + +```ts +McpServer.layerHttp({ + name: "T3 Code", + version, + path: "/mcp", +}); +``` + +Use the API from: + +`effect/unstable/ai/McpServer` + +Reference source: + +`.repos/effect-smol/packages/effect/src/unstable/ai/McpServer.ts` + +Do not implement MCP framing, initialization, session management, or JSON-RPC manually. + +## Invocation Identity + +MCP `tools/call` does not include T3 thread identity. Bind identity to the MCP connection through authentication. + +When starting or resuming a provider session, issue an opaque bearer token associated internally with: + +```ts +interface McpInvocationScope { + environmentId: EnvironmentId; + threadId: ThreadId; + providerSessionId: string; + providerInstanceId: ProviderInstanceId; + allowedCapabilities: ReadonlySet; + issuedAt: string; + expiresAt: string; +} +``` + +Provider MCP configuration: + +```ts +{ + type: "http", + name: "t3", + url: `${environmentHttpBaseUrl}/mcp`, + headers: [ + { + name: "Authorization", + value: `Bearer ${token}`, + }, + ], +} +``` + +Requirements: + +- Agents never receive or pass `threadId` as a tool argument. +- Tool handlers obtain `McpInvocationScope` from request authentication middleware. +- Tokens are cryptographically random opaque values. +- Store only a hash of each token server-side. +- Revoke tokens when the provider session stops. +- Expire tokens after inactivity and at a fixed maximum lifetime. +- Server restart invalidates all tokens. +- Resuming a provider session issues a fresh token. + +## General MCP Architecture + +Add server modules such as: + +```text +apps/server/src/mcp/ + Services/ + McpSessionRegistry.ts + McpInvocationContext.ts + Layers/ + McpSessionRegistry.ts + McpHttpServer.ts + toolkits/ + preview/ + tools.ts + handlers.ts + layer.ts +``` + +### Toolkit Registration + +Define capabilities with Effect AI: + +```ts +import { McpServer, Tool, Toolkit } from "effect/unstable/ai"; +``` + +Each capability family owns: + +- `Tool.make` definitions +- a `Toolkit.make` collection +- handler services/layers +- an MCP registration layer + +Server startup merges all registration layers: + +```ts +const T3McpToolkits = Layer.mergeAll( + PreviewToolkitRegistration, + // Future toolkit registrations +); +``` + +Future filesystem, terminal, source-control, or environment tools must not require changes to MCP transport or authentication. + +### Naming + +Use stable capability-prefixed names: + +- `preview_status` +- `preview_open` +- `preview_navigate` +- `preview_snapshot` +- `preview_click` +- `preview_type` +- `preview_press` +- `preview_scroll` +- `preview_evaluate` +- `preview_wait_for` + +## MCP Authentication + +Integrate bearer authentication into the HTTP MCP route before MCP request handling. + +Authentication flow: + +1. Read `Authorization: Bearer `. +2. Hash token and resolve it through `McpSessionRegistry`. +3. Verify expiration and provider-session liveness. +4. Provide `McpInvocationContext` to MCP toolkit handlers. +5. Reject invalid credentials without invoking MCP tools. +6. Update token activity timestamp after authenticated requests. + +Capability authorization occurs inside handlers or common middleware: + +```ts +yield* McpInvocationContext.requireCapability("preview"); +``` + +## Preview Automation Contracts + +Add `packages/contracts/src/previewAutomation.ts`. + +Define schemas for: + +- preview status +- opening/showing preview +- navigation +- page snapshot +- selector or coordinate click +- text entry +- key press +- scrolling +- JavaScript evaluation +- waiting for selector, text, or URL + +Define tagged errors: + +- `PreviewAutomationUnavailableError` +- `PreviewAutomationNoFocusedOwnerError` +- `PreviewAutomationUnsupportedClientError` +- `PreviewAutomationTabNotFoundError` +- `PreviewAutomationTimeoutError` +- `PreviewAutomationExecutionError` +- `PreviewAutomationInvalidSelectorError` +- `PreviewAutomationResultTooLargeError` + +## Preview Toolkit + +Implement preview tools as an independent Effect AI toolkit. + +Apply annotations: + +- `preview_status` and `preview_snapshot`: read-only +- `preview_status`: idempotent +- navigation and page interaction tools: open-world +- browser operations: non-destructive +- all tools: human-readable title and precise description + +### `preview_open` + +Input: + +```ts +{ + url?: string; + show?: boolean; + reuseExistingTab?: boolean; +} +``` + +Defaults: + +- `show: true` +- `reuseExistingTab: true` + +Behavior: + +- Show the preview panel for the scoped thread. +- Reuse its active preview tab when available. +- Create and mount a tab otherwise. +- Navigate when `url` is supplied. +- Wait until the webview has registered before returning. + +### `preview_snapshot` + +Return: + +- current URL, title, loading state +- bounded visible text +- up to 200 interactive elements +- accessibility tree +- PNG screenshot scaled to a maximum width of 1280 pixels + +Expose the screenshot as MCP image content and metadata as structured content. + +### Browser Controls + +- `preview_click`: selector or viewport coordinates +- `preview_type`: optionally focus selector and clear existing value +- `preview_press`: common keys and modifiers +- `preview_scroll`: viewport or selector target +- `preview_evaluate`: execute bounded JavaScript +- `preview_wait_for`: selector, visible text, or URL substring +- `preview_navigate`: navigate and wait for selected readiness condition + +Default operation timeout: 15 seconds. + +Maximum serialized evaluation result: 64 KB. + +Maximum visible text: 20 KB. + +## Preview Broker + +Add `PreviewAutomationBroker` to `apps/server`. + +Responsibilities: + +- Track automation-capable desktop clients. +- Track preview ownership by environment and thread. +- Route operations to the correct desktop client. +- Correlate requests and responses. +- Enforce timeouts. +- Fail pending calls when clients disconnect. + +Owner state: + +```ts +interface PreviewAutomationOwner { + clientId: string; + environmentId: EnvironmentId; + threadId: ThreadId; + tabId: PreviewTabId | null; + visible: boolean; + supportsAutomation: boolean; + focusedAt: string; +} +``` + +Routing policy: + +- Use the most recently focused Electron window displaying the scoped thread. +- Never accept environment or thread overrides from tool arguments. +- Return `PreviewAutomationNoFocusedOwnerError` when no valid owner exists. +- Do not switch the UI to a different thread automatically. + +## Server-to-Desktop Protocol + +Add WS RPCs: + +- `previewAutomation.connect` +- `previewAutomation.respond` +- `previewAutomation.reportOwner` +- `previewAutomation.clearOwner` + +`connect` is a long-lived stream from the environment server to the desktop client. + +Request: + +```ts +{ + requestId: string; + threadId: ThreadId; + tabId?: PreviewTabId; + operation: PreviewAutomationOperation; + input: unknown; + timeoutMs: number; +} +``` + +Response: + +```ts +{ + requestId: string; + ok: boolean; + result?: unknown; + error?: { + _tag: string; + message: string; + detail?: unknown; + }; +} +``` + +## Desktop Automation + +Extend `PreviewViewManager` using Electron `webContents.debugger`. + +Use CDP domains: + +- `Runtime` +- `DOM` +- `Page` +- `Accessibility` +- `Input` + +Use `webContents.capturePage()` for screenshots. + +Create a scoped CDP helper that: + +1. Resolves the tab’s webContents. +2. Attaches lazily. +3. Enables required domains. +4. Executes one bounded operation. +5. Detaches in finalization. +6. Maps protocol failures to typed errors. + +If DevTools or another debugger owns the target, return a typed automation error rather than disrupting it. + +Place pure logic in separate modules: + +- DOM summary extraction +- selector generation +- result clamping +- key mapping +- CDP response parsing + +## Desktop IPC + +Extend `DesktopPreviewBridge` with: + +```ts +automation: { + status(...): Promise<...>; + snapshot(...): Promise<...>; + click(...): Promise<...>; + type(...): Promise<...>; + press(...): Promise<...>; + scroll(...): Promise<...>; + evaluate(...): Promise<...>; + waitFor(...): Promise<...>; +} +``` + +Add schema-validated IPC channels and handlers. + +## Web Client + +Add a preview ownership hook mounted with `PreviewView`. + +Report changes to: + +- active environment/thread +- tab id +- panel visibility +- window focus +- Electron automation availability + +Handle broker requests: + +- `preview_open` opens the right panel for the active scoped thread. +- Create a preview session and tab if needed. +- Wait for webview registration. +- Other operations invoke desktop automation IPC. +- Always send a correlated success or failure response. + +Clear ownership on unmount, thread change, panel close, or desktop disconnect. + +## Provider Integration + +Add `McpSessionRegistry` integration to provider lifecycle. + +For each provider session: + +1. Issue a scoped MCP bearer token. +2. Add the shared HTTP MCP configuration to session startup. +3. Start or resume the agent session. +4. Revoke the token during provider-session finalization. + +ACP providers use their existing HTTP MCP configuration fields. + +Codex uses its supported MCP/config override mechanism to register the same shared HTTP endpoint. + +Assume every supported provider can use HTTP MCP. Do not implement stdio fallback. + +## Remote Environment Behavior + +For a Mac mini environment viewed from a MacBook: + +1. Mac mini runs the T3 environment server and shared MCP endpoint. +2. Agent session connects to that endpoint locally/remotely using its scoped token. +3. Preview tool calls enter the Mac mini preview broker. +4. Broker routes them over the existing T3 connection to the focused MacBook desktop. +5. MacBook controls its visible Electron webview. + +URLs must already be reachable from the MacBook, such as: + +- `http://mac-mini.local:5173` +- `http://192.168.1.42:5173` + +Warn when a remote environment opens `localhost`, `127.0.0.1`, or `::1`, because loopback resolves on the preview client. + +Preserve URL-resolution metadata: + +```ts +{ + requestedUrl: string; + resolvedUrl: string; + resolutionKind: "direct"; + environmentId: EnvironmentId; +} +``` + +This leaves room for future SSH, relay, or Tailscale resolution. + +## Tests + +### MCP Server + +- one MCP server layer starts per environment server +- multiple authenticated MCP clients share the same server instance +- each client receives its own invocation scope +- toolkit registration is independent of transport +- a mock future toolkit can register without changing server runtime +- malformed parameters are rejected by Effect schemas +- tool annotations appear in `tools/list` + +### Authentication + +- valid token resolves correct thread and provider session +- concurrent tokens remain isolated +- revoked and expired tokens fail +- token cannot call unauthorized capability family +- thread identity cannot be overridden in arguments +- server restart invalidates tokens + +### Preview Broker + +- focused owner receives operation +- most recently focused client wins +- wrong-thread client is never selected +- no owner returns typed error +- disconnect fails pending calls +- stale responses are ignored + +### Desktop and Web + +- agent can show and open preview +- webview registration is awaited +- CDP click, typing, key press, scroll, evaluation, and wait work +- snapshot bounds screenshot, text, and interactive elements +- ownership updates on focus and visibility changes +- background threads are not automatically activated + +### End-to-End Integration + +Run two mocked agent sessions against one HTTP MCP server: + +1. Bind each bearer token to a different thread. +2. Call `preview_status` concurrently. +3. Verify each request routes to its own focused preview owner. +4. Verify no MCP child process is spawned. +5. Revoke one provider session and confirm only its MCP access fails. + +## Validation + +- `vp check` +- `vp run typecheck` +- `vp test` + +## Assumptions + +- HTTP MCP is supported by every target provider. +- One MCP server is embedded in each T3 environment server. +- MCP connection authentication supplies invocation identity. +- Agents never know or pass T3 thread IDs. +- Preview automation is the first of multiple future MCP toolkits. +- Only Electron desktop preview clients support browser automation in v1. +- No headless browser, stdio fallback, or automatic tunnel management is included. diff --git a/.plans/visible-preview-browser-automation-via-cdp-mcp.md b/.plans/visible-preview-browser-automation-via-cdp-mcp.md new file mode 100644 index 0000000000000000000000000000000000000000..0b040d7a79c2b231a2d0eee0f8059576432a140d --- /dev/null +++ b/.plans/visible-preview-browser-automation-via-cdp-mcp.md @@ -0,0 +1,670 @@ +# Visible Preview Browser Automation via CDP + MCP + +## Summary + +Implement agent control of the user-visible T3 preview browser only. Do not add headless browser support. Do not add SSH/relay/private forwarding in v1; preview URLs must already be reachable from the desktop client, such as a Mac mini private IP URL opened on a MacBook. + +Architecture: + +`agent` -> `stdio MCP server in environment` -> `private T3 server bridge` -> `focused desktop client/window` -> `Electron preview webview via CDP` + +The stdio MCP server is the agent-facing integration because all target agents can speak MCP. The MCP server is intentionally thin: it does not automate Chromium directly. It calls the T3 environment server, which routes commands to the focused desktop client that owns the visible preview. + +## Explicit Non-Goals + +- No headless browser runner. +- No Playwright-managed browser. +- No arbitrary SSH dev-port forwarding in v1. +- No Cloudflare/Tailscale/relay URL rewriting in v1. +- No automation of browser/web clients that do not have Electron preview support. +- No per-action user approval prompts in v1. + +## Key Decisions + +- **Transport to agents:** stdio MCP. +- **Browser being controlled:** the actual integrated Electron preview webview. +- **Remote URL handling:** manual reachable URLs only. Agents may open `http://mac-mini.local:5173` or `http://192.168.x.y:5173`; T3 will not tunnel `127.0.0.1:5173` from remote to local yet. +- **Client routing:** route tool requests to the most recently focused desktop window/client for the agent’s thread. Return a typed error if no focused desktop preview owner is available. +- **Scope:** full control of preview browser, including opening/showing the preview panel. +- **Primary automation engine:** Chrome DevTools Protocol through Electron `webContents.debugger`, with `webContents.capturePage()` where it is simpler and more reliable. + +## New Contracts + +Add `packages/contracts/src/previewAutomation.ts`. + +### Branded IDs + +- `PreviewAutomationRequestId` +- `PreviewAutomationClientId` +- `PreviewAutomationOwnerId` + +### Tool Input/Output Schemas + +Add Effect schemas for these operations: + +- `PreviewAutomationOpenInput` + - `url?: string` + - `show?: boolean` default `true` + - `reuseExistingTab?: boolean` default `true` +- `PreviewAutomationNavigateInput` + - `url: string` + - `waitUntil?: "load" | "domcontentloaded" | "network-idle" | "none"` default `"load"` + - `timeoutMs?: number` default `15000` +- `PreviewAutomationSnapshotInput` + - `includeScreenshot?: boolean` default `true` + - `includeDomSummary?: boolean` default `true` + - `includeAccessibilityTree?: boolean` default `true` + - `screenshotMaxWidth?: number` default `1280` +- `PreviewAutomationClickInput` + - one of: + - `{ selector: string }` + - `{ x: number; y: number }` + - `button?: "left" | "middle" | "right"` default `"left"` + - `clickCount?: number` default `1` +- `PreviewAutomationTypeInput` + - `text: string` + - `selector?: string` + - `clearFirst?: boolean` default `false` +- `PreviewAutomationPressInput` + - `key: string` + - `modifiers?: readonly ("alt" | "control" | "meta" | "shift")[]` +- `PreviewAutomationScrollInput` + - `deltaX?: number` + - `deltaY?: number` + - optional target `{ selector: string }` +- `PreviewAutomationEvaluateInput` + - `expression: string` + - `awaitPromise?: boolean` default `true` + - `returnByValue?: boolean` default `true` +- `PreviewAutomationWaitForInput` + - one of: + - `{ selector: string }` + - `{ text: string }` + - `{ urlIncludes: string }` + - `timeoutMs?: number` default `10000` +- `PreviewAutomationStatusResult` + - `available: boolean` + - `visible: boolean` + - `threadId` + - `tabId: string | null` + - `url: string | null` + - `title: string | null` + - `loading: boolean` + - `ownerClientId: string | null` + +### Result Shape + +Every mutating or stateful operation returns: + +```ts +{ + ok: boolean; + status: PreviewAutomationStatusResult; + message?: string; +} +``` + +`snapshot` additionally returns: + +```ts +{ + status: PreviewAutomationStatusResult; + screenshot?: { + mimeType: "image/png"; + dataBase64: string; + width: number; + height: number; + }; + domSummary?: { + url: string; + title: string; + activeElement: string | null; + text: string; + interactiveElements: readonly { + index: number; + tag: string; + role: string | null; + name: string; + text: string; + selector: string | null; + rect: { x: number; y: number; width: number; height: number } | null; + }[]; + }; + accessibilityTree?: unknown; +} +``` + +### Error Types + +Add tagged errors: + +- `PreviewAutomationUnavailableError` +- `PreviewAutomationNoFocusedOwnerError` +- `PreviewAutomationUnsupportedClientError` +- `PreviewAutomationTabNotFoundError` +- `PreviewAutomationTimeoutError` +- `PreviewAutomationExecutionError` +- `PreviewAutomationInvalidSelectorError` + +## Server-Side Broker + +Add `apps/server/src/previewAutomation/Services/PreviewAutomationBroker.ts`. + +Responsibilities: + +- Track connected desktop automation clients. +- Track focus ownership by `(environmentId, threadId)`. +- Accept tool calls from MCP/stdin proxy. +- Route each call to the focused owner for the thread. +- Enforce timeouts and cleanup pending requests on disconnect. +- Return typed failures when no client/window is available. + +### Owner State + +Store: + +```ts +{ + clientId: PreviewAutomationClientId; + environmentId: EnvironmentId; + threadId: ThreadId; + tabId: PreviewTabId | null; + focusedAt: string; + visible: boolean; + supportsAutomation: boolean; +} +``` + +Ownership updates come from the web/desktop client when: + +- route/thread changes, +- preview panel opens/closes, +- window focus changes, +- tab id changes, +- desktop bridge availability changes. + +## WS Bridge: Server to Desktop Client + +The current preview RPCs are client-to-server plus server event streams. Add a request/response bridge using stream-style RPCs to avoid introducing bidirectional RPC infrastructure. + +### New WS Methods + +Add to `packages/contracts/src/rpc.ts`: + +- `previewAutomation.connect` + - client calls this as a long-lived stream + - input: + - `clientId` + - `capabilities` + - stream output: + - `PreviewAutomationClientRequest` +- `previewAutomation.respond` + - client sends response for a request id +- `previewAutomation.reportOwner` + - client reports focus/visibility/thread ownership +- `previewAutomation.clearOwner` + - client clears stale ownership on unmount/disconnect + +### Request Shape + +```ts +{ + requestId: string; + threadId: string; + tabId?: string; + operation: + | "open" + | "navigate" + | "snapshot" + | "click" + | "type" + | "press" + | "scroll" + | "evaluate" + | "waitFor" + | "status"; + input: unknown; + timeoutMs: number; +} +``` + +### Response Shape + +```ts +{ + requestId: string; + ok: boolean; + result?: unknown; + error?: { + _tag: string; + message: string; + detail?: unknown; + }; +} +``` + +## Desktop Preview Automation + +Extend `apps/desktop/src/preview-view-manager.ts`. + +### Add Methods + +- `getAutomationStatus(tabId)` +- `captureSnapshot(tabId, options)` +- `click(tabId, input)` +- `type(tabId, input)` +- `press(tabId, input)` +- `scroll(tabId, input)` +- `evaluate(tabId, input)` +- `waitFor(tabId, input)` + +### CDP Session Handling + +Add a small helper inside desktop preview code: + +- Attach `webContents.debugger` lazily per operation. +- Do not keep debugger attached forever unless needed. +- If already attached by DevTools or another debugger, return `PreviewAutomationExecutionError`. +- Use CDP domains: + - `Runtime.evaluate` + - `DOM.getDocument` + - `DOM.querySelector` + - `DOM.getBoxModel` + - `Accessibility.getFullAXTree` + - `Input.dispatchMouseEvent` + - `Input.dispatchKeyEvent` + - optionally `Page.captureScreenshot` if `webContents.capturePage()` is insufficient + +For screenshots, prefer `webContents.capturePage()` first because it is already used safely for annotations. + +### DOM Summary Script + +Use `Runtime.evaluate` with a bounded page script that returns: + +- `document.URL` +- `document.title` +- active element summary +- visible text truncated to a fixed limit, e.g. 20k chars +- up to 200 interactive elements: + - buttons + - links + - inputs + - selects + - textareas + - elements with roles + - elements with click handlers where detectable +- stable-ish CSS selectors generated in page context +- bounding rects + +Do not return full HTML by default. + +### Input Behavior + +- `click(selector)`: + - resolve selector in page + - scroll into view + - compute center of bounding box + - dispatch mouse move/down/up through CDP +- `click(x, y)`: + - dispatch at viewport coordinates +- `type(selector, text)`: + - focus selector if provided + - optionally clear existing value with platform shortcut + - dispatch text via CDP keyboard events or `Input.insertText` +- `press(key)`: + - map common key names: Enter, Escape, Tab, Backspace, Arrow keys + - support modifiers +- `scroll`: + - use CDP mouse wheel or page `scrollBy` fallback + +## Web Client Changes + +Update `apps/web/src/components/preview`. + +### Ownership Reporting + +Add a hook, likely `usePreviewAutomationOwner`, mounted near `PreviewView`. + +It reports owner state when: + +- the current route thread ref changes, +- preview panel visibility changes, +- browser window focus/blur changes, +- tab id changes, +- desktop preview bridge exists. + +Policy: + +- Only Electron desktop clients with `window.desktopBridge.preview` can report `supportsAutomation: true`. +- A visible preview panel gets ownership. +- The most recently focused window wins. +- On unmount or preview close, clear ownership. + +### Handling Server Requests + +Add a client-side subscriber using the new `previewAutomation.connect` stream. + +When a request arrives: + +- if operation is `open`, ensure preview panel is visible for the thread, call `api.preview.open` if needed, mount/create desktop tab, then navigate if URL is provided. +- for browser operations, call new `desktopBridge.preview.automation.*` methods. +- send `previewAutomation.respond`. + +Opening behavior: + +- If the right panel is closed, open it to preview. +- If the thread is not currently active in the UI, do not navigate the whole app in v1. Return `PreviewAutomationNoFocusedOwnerError`. +- If preview is supported but no tab exists and the agent calls `open`, create one. +- If no tab exists and the agent calls `snapshot/click/type/...`, return a clear “preview not open” error suggesting `preview_open`. + +## Desktop IPC / Preload + +Extend `packages/contracts/src/ipc.ts` `DesktopPreviewBridge`. + +Add: + +```ts +automation: { + status(tabId: string): Promise; + snapshot(tabId: string, input: PreviewAutomationSnapshotInput): Promise; + click(tabId: string, input: PreviewAutomationClickInput): Promise; + type(tabId: string, input: PreviewAutomationTypeInput): Promise; + press(tabId: string, input: PreviewAutomationPressInput): Promise; + scroll(tabId: string, input: PreviewAutomationScrollInput): Promise; + evaluate(tabId: string, input: PreviewAutomationEvaluateInput): Promise; + waitFor(tabId: string, input: PreviewAutomationWaitForInput): Promise; +} +``` + +Add IPC channels in `apps/desktop/src/ipc/channels.ts` and handlers in `apps/desktop/src/ipc/methods/preview.ts`. + +## Stdio MCP Server + +Add package: `packages/preview-mcp`. + +Purpose: + +- Implements the MCP stdio protocol. +- Exposes T3 preview tools. +- Calls a private loopback endpoint or local JSON-RPC bridge on the environment server. +- Does not know Electron/CDP details. + +### Binary + +Expose bin: + +```json +{ + "bin": { + "t3-preview-mcp": "./dist/index.js" + } +} +``` + +During local dev, provider config can call the source runner via workspace package script; production package uses built JS. + +### MCP Environment Variables + +Provider sessions launch the MCP server with: + +- `T3_PREVIEW_MCP_SERVER_URL` + - environment server loopback URL +- `T3_PREVIEW_MCP_TOKEN` + - short-lived token scoped to the provider session/thread +- `T3_PREVIEW_ENVIRONMENT_ID` +- `T3_PREVIEW_THREAD_ID` + +The token must be generated by the environment server and expire when the provider session ends. + +### MCP Tools + +Expose these tools: + +- `preview_status` +- `preview_open` +- `preview_navigate` +- `preview_snapshot` +- `preview_click` +- `preview_type` +- `preview_press` +- `preview_scroll` +- `preview_evaluate` +- `preview_wait_for` + +Tool descriptions must explicitly say they operate the visible T3 preview browser for the current thread. + +### MCP Output + +- Text results include concise status and URL/title. +- `preview_snapshot` returns: + - text summary + - MCP image content for screenshot when available +- Errors are MCP tool errors with the tagged T3 error message included. + +## Private MCP Bridge Endpoint + +Add an internal server route under `apps/server`, not public app UI: + +- `POST /internal/preview-automation/tool` +- Auth: bearer `T3_PREVIEW_MCP_TOKEN` +- Body: + - `{ tool: string, input: unknown }` +- Response: + - `{ ok: true, result } | { ok: false, error }` + +This endpoint is only for the stdio MCP proxy. It calls `PreviewAutomationBroker`. + +Bind it to the same host/port as the environment server, but require the short-lived token. The endpoint must reject requests without a token or with an expired/stale thread/session. + +## Provider Integration + +### Codex + +When starting a Codex provider session, add the T3 preview MCP server to Codex configuration if the app-server config path supports per-thread MCP injection. + +Implementation path: + +1. Add `PreviewMcpSessionService` in `apps/server`. +2. On provider session start: + - create scoped MCP token for `(environmentId, threadId, providerSessionId)`. + - build MCP server config: + - name: `t3-preview` + - command: `t3-preview-mcp` + - env vars listed above +3. Thread/session start passes the MCP server config through the provider’s supported config field. +4. If Codex app-server cannot accept injected MCP servers through typed params, use its `config` override field with Codex-compatible MCP config. + +### ACP Providers + +ACP session creation already passes `mcpServers: []`. Replace that with the same `t3-preview` MCP server config for providers that support MCP. + +### Provider Fallback + +If a provider does not support MCP injection yet, do not add provider-specific native tools in v1. The shared MCP server remains available for later provider wiring. + +## Remote Environment Behavior + +### Mac mini dev server viewed from MacBook + +Expected v1 workflow: + +1. Agent runs dev server on Mac mini. +2. Agent or user opens a reachable URL, e.g.: + - `http://mac-mini.local:5173` + - `http://192.168.1.42:5173` +3. T3 desktop on MacBook opens that URL in the local Electron preview. +4. Agent uses MCP tools. +5. T3 routes tool calls from Mac mini environment server to the focused MacBook preview client. + +### `localhost` Caveat + +In v1, if an agent opens `http://localhost:5173` from a remote environment, the MacBook preview will interpret that as MacBook localhost. The tool result should include a warning when: + +- environment is not the primary/local environment, and +- URL hostname is `localhost`, `127.0.0.1`, or `::1`. + +Warning text: + +`This URL is loopback on the preview client, not necessarily the remote environment. Use a client-reachable host/IP for remote dev servers.` + +### Future-Proofing + +Do not bake manual URL assumptions into the automation layer. Represent opened URLs as: + +```ts +{ + displayUrl: string; + requestedUrl: string; + resolutionKind: "direct"; + environmentId: string; +} +``` + +Later `resolutionKind` can add: + +- `ssh-forward` +- `relay` +- `tailscale` +- `cloudflare-tunnel` + +## Security and Safety + +- MCP tokens are scoped to one provider session/thread. +- MCP tokens expire on provider session stop and server restart. +- Browser automation only routes to focused desktop owner for that same thread. +- Do not allow MCP input to specify arbitrary `environmentId` or `threadId`; infer both from token/session. +- `preview_evaluate` is powerful. Keep it enabled in v1 because the user requested full control, but: + - limit result serialization size, e.g. 64 KB + - timeout evaluation + - return by value by default + - document that it executes in the preview page context +- Screenshot output should be bounded: + - max dimensions + - max base64 size + - return error if too large after scaling attempts +- Clear pending broker requests when desktop disconnects. +- CDP operations must timeout and detach debugger in `finally`. + +## Failure Modes + +Return typed errors for: + +- no desktop client connected +- desktop client connected but preview unsupported +- no focused owner for thread +- preview panel not open and operation is not `preview_open` +- webview not initialized yet +- navigation timeout +- selector not found +- CDP debugger unavailable +- page execution error +- screenshot too large +- stale request id or response after timeout + +## Tests + +### Contracts + +Add tests for: + +- schema decoding for every preview automation input/result +- error schema decoding +- invalid selector/click union inputs rejected +- snapshot options defaulting + +### Desktop Unit Tests + +Add tests around `PreviewViewManager` helpers: + +- owner-independent status when tab exists/does not exist +- selector summary script clamps output +- selector generation handles ids/classes/nth-child fallback +- error mapping for missing webContents +- screenshot result shape + +Where Electron/CDP is hard to unit test, isolate pure helpers and cover IPC handler validation. + +### Server Broker Tests + +Add tests for: + +- registering clients +- ownership updates +- focused owner wins +- request routed to focused owner +- request timeout +- client disconnect fails pending requests +- response with unknown request id ignored/rejected +- token scoped to thread/session +- remote loopback URL warning generated + +### Web Client Tests + +Add tests for: + +- ownership report sent only in Electron preview-supported runtime +- opening preview panel on `preview_open` +- no route switch for background thread +- clear ownership on unmount +- response sent for successful and failed desktop bridge calls + +### MCP Tests + +Add tests for: + +- tool list includes all expected preview tools +- each tool maps to internal bridge request +- token missing/invalid returns MCP error +- snapshot maps screenshot to MCP image content +- tool errors preserve tagged T3 error message + +### Integration Tests + +Add a focused integration test using mocked desktop client: + +1. Start server broker. +2. Register fake desktop automation client. +3. Mark it focused for thread. +4. Invoke MCP `preview_open`. +5. Assert fake client received open request. +6. Respond success. +7. Assert MCP response is successful. + +Add a second integration test: + +1. No focused owner. +2. Invoke `preview_snapshot`. +3. Assert `PreviewAutomationNoFocusedOwnerError`. + +## Validation Commands + +Before completion: + +- `vp check` +- `vp run typecheck` +- `vp test` + +No `vp run lint:mobile` required unless mobile code is changed. + +## Implementation Order + +1. Add contracts for preview automation schemas and WS methods. +2. Add `PreviewAutomationBroker` server service. +3. Add WS stream bridge for desktop clients. +4. Add desktop IPC and `PreviewViewManager` CDP automation methods. +5. Add web ownership reporting and request handling. +6. Add private `/internal/preview-automation/tool` endpoint with scoped token auth. +7. Add `packages/preview-mcp` stdio MCP server. +8. Wire provider sessions to launch/register `t3-preview` MCP server where provider protocols support MCP config. +9. Add remote loopback URL warning. +10. Add tests. +11. Run validation commands. + +## Assumptions + +- v1 only supports Electron desktop preview clients. +- The active/focused thread preview is the right target for agent control. +- Manual client-reachable URLs are acceptable for remote dev servers. +- Full browser control includes `evaluate`. +- Stdio MCP is the agent-facing transport. +- A private server bridge behind the stdio MCP server is acceptable and necessary because the browser lives on the desktop client, not beside the remote agent. diff --git a/apps/desktop/scripts/dev-electron.mjs b/apps/desktop/scripts/dev-electron.mjs index 2a2e52449be9a5c80261bbce89d55f3e2edff917..6c8b94188a20139ca4b05e98baa1caa8a0a668da 100644 --- a/apps/desktop/scripts/dev-electron.mjs +++ b/apps/desktop/scripts/dev-electron.mjs @@ -2,7 +2,11 @@ import { spawn, spawnSync } from "node:child_process"; import { watch } from "node:fs"; import { join } from "node:path"; -import { desktopDir, resolveDevProtocolClient, resolveElectronPath } from "./electron-launcher.mjs"; +import { + desktopDir, + resolveDevProtocolClient, + resolveElectronLaunchCommand, +} from "./electron-launcher.mjs"; import { waitForResources } from "./wait-for-resources.mjs"; const devServerUrl = process.env.VITE_DEV_SERVER_URL?.trim(); @@ -79,7 +83,8 @@ function startApp() { const launchArgs = devProtocolClient ? electronArgs : [...electronArgs, `--t3code-dev-root=${desktopDir}`, "dist-electron/main.cjs"]; - const app = spawn(resolveElectronPath(), launchArgs, { + const electronCommand = resolveElectronLaunchCommand(launchArgs); + const app = spawn(electronCommand.electronPath, electronCommand.args, { cwd: desktopDir, env: childEnv, stdio: "inherit", diff --git a/apps/desktop/scripts/electron-launcher.mjs b/apps/desktop/scripts/electron-launcher.mjs index 8f20001bbb08977601b17d67160c65556672a90b..1fc956b39dc9ee74c207930243fc261376300db6 100644 --- a/apps/desktop/scripts/electron-launcher.mjs +++ b/apps/desktop/scripts/electron-launcher.mjs @@ -307,6 +307,31 @@ function buildMacLauncher(electronBinaryPath) { return targetBinaryPath; } +function isLinuxSetuidSandboxConfigured(electronBinaryPath) { + if (process.platform !== "linux") { + return true; + } + + const sandboxPath = join(dirname(electronBinaryPath), "chrome-sandbox"); + try { + const sandboxStat = statSync(sandboxPath); + return sandboxStat.uid === 0 && (sandboxStat.mode & 0o4777) === 0o4755; + } catch { + return false; + } +} + +function resolveLinuxSandboxArgs(electronBinaryPath) { + if (isLinuxSetuidSandboxConfigured(electronBinaryPath)) { + return []; + } + + console.warn( + "[desktop-launcher] Electron chrome-sandbox is not root-owned with mode 4755; launching local Electron with --no-sandbox.", + ); + return ["--no-sandbox"]; +} + export function resolveElectronPath() { ensureElectronRuntime(); @@ -320,6 +345,14 @@ export function resolveElectronPath() { return buildMacLauncher(electronBinaryPath); } +export function resolveElectronLaunchCommand(args = []) { + const electronPath = resolveElectronPath(); + return { + electronPath, + args: [...resolveLinuxSandboxArgs(electronPath), ...args], + }; +} + export function resolveDevProtocolClient() { if (process.platform !== "darwin" || !isDevelopment) { return null; diff --git a/apps/desktop/scripts/smoke-test.mjs b/apps/desktop/scripts/smoke-test.mjs index fdbe69b77800f252ec64691fc64538536cb5c184..48a2e168a2b661463b19ce74ff8ccd4dba705e87 100644 --- a/apps/desktop/scripts/smoke-test.mjs +++ b/apps/desktop/scripts/smoke-test.mjs @@ -1,15 +1,16 @@ import { spawn } from "node:child_process"; import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import { resolveElectronLaunchCommand } from "./electron-launcher.mjs"; const __dirname = dirname(fileURLToPath(import.meta.url)); const desktopDir = resolve(__dirname, ".."); -const electronBin = resolve(desktopDir, "node_modules/.bin/electron"); const mainJs = resolve(desktopDir, "dist-electron/main.cjs"); console.log("\nLaunching Electron smoke test..."); -const child = spawn(electronBin, [mainJs], { +const electronCommand = resolveElectronLaunchCommand([mainJs]); +const child = spawn(electronCommand.electronPath, electronCommand.args, { stdio: ["pipe", "pipe", "pipe"], env: { ...process.env, diff --git a/apps/desktop/scripts/start-electron.mjs b/apps/desktop/scripts/start-electron.mjs index 375dbfe575f9098700a89b27504bab8a5622ac60..d959b4ab1f087cc61eea6c56270f04f5fa749892 100644 --- a/apps/desktop/scripts/start-electron.mjs +++ b/apps/desktop/scripts/start-electron.mjs @@ -1,11 +1,12 @@ import { spawn } from "node:child_process"; -import { desktopDir, resolveElectronPath } from "./electron-launcher.mjs"; +import { desktopDir, resolveElectronLaunchCommand } from "./electron-launcher.mjs"; const childEnv = { ...process.env }; delete childEnv.ELECTRON_RUN_AS_NODE; -const child = spawn(resolveElectronPath(), ["dist-electron/main.cjs"], { +const electronCommand = resolveElectronLaunchCommand(["dist-electron/main.cjs"]); +const child = spawn(electronCommand.electronPath, electronCommand.args, { stdio: "inherit", cwd: desktopDir, env: childEnv, diff --git a/apps/desktop/src/ipc/channels.ts b/apps/desktop/src/ipc/channels.ts index 9c11859f236e03d20c50bde3250f235575f130e0..11832664d88c32080902b024042dd1349ea35710 100644 --- a/apps/desktop/src/ipc/channels.ts +++ b/apps/desktop/src/ipc/channels.ts @@ -56,4 +56,12 @@ export const PREVIEW_CLEAR_CACHE_CHANNEL = "desktop:preview-clear-cache"; export const PREVIEW_GET_CONFIG_CHANNEL = "desktop:preview-get-config"; export const PREVIEW_PICK_ELEMENT_CHANNEL = "desktop:preview-pick-element"; export const PREVIEW_CANCEL_PICK_ELEMENT_CHANNEL = "desktop:preview-cancel-pick-element"; +export const PREVIEW_AUTOMATION_STATUS_CHANNEL = "desktop:preview-automation-status"; +export const PREVIEW_AUTOMATION_SNAPSHOT_CHANNEL = "desktop:preview-automation-snapshot"; +export const PREVIEW_AUTOMATION_CLICK_CHANNEL = "desktop:preview-automation-click"; +export const PREVIEW_AUTOMATION_TYPE_CHANNEL = "desktop:preview-automation-type"; +export const PREVIEW_AUTOMATION_PRESS_CHANNEL = "desktop:preview-automation-press"; +export const PREVIEW_AUTOMATION_SCROLL_CHANNEL = "desktop:preview-automation-scroll"; +export const PREVIEW_AUTOMATION_EVALUATE_CHANNEL = "desktop:preview-automation-evaluate"; +export const PREVIEW_AUTOMATION_WAIT_FOR_CHANNEL = "desktop:preview-automation-wait-for"; export const PREVIEW_STATE_CHANGE_CHANNEL = "desktop:preview-state-change"; diff --git a/apps/desktop/src/ipc/methods/preview.ts b/apps/desktop/src/ipc/methods/preview.ts index dca8d64110884a9882fa8b2f13d86013a545e71a..904cef30b41216d7245da1073f4f81a761bd14f7 100644 --- a/apps/desktop/src/ipc/methods/preview.ts +++ b/apps/desktop/src/ipc/methods/preview.ts @@ -28,6 +28,13 @@ const tabIdFrom = (raw: unknown): string => { return tabId; }; +const inputFrom = (raw: unknown): unknown => { + if (typeof raw !== "object" || raw === null || !("input" in raw)) { + throw new Error("preview automation input is required"); + } + return raw.input; +}; + class PreviewIpcError extends Data.TaggedError("PreviewIpcError")<{ readonly cause: unknown; }> {} @@ -102,4 +109,46 @@ export const previewMethods = [ method(IpcChannels.PREVIEW_CANCEL_PICK_ELEMENT_CHANNEL, (raw) => previewViewManager.cancelPickElement(tabIdFrom(raw)), ), + method(IpcChannels.PREVIEW_AUTOMATION_STATUS_CHANNEL, (raw) => + previewViewManager.automationStatus(tabIdFrom(raw)), + ), + method(IpcChannels.PREVIEW_AUTOMATION_SNAPSHOT_CHANNEL, (raw) => + previewViewManager.automationSnapshot(tabIdFrom(raw)), + ), + method(IpcChannels.PREVIEW_AUTOMATION_CLICK_CHANNEL, (raw) => + previewViewManager.automationClick( + tabIdFrom(raw), + inputFrom(raw) as Parameters[1], + ), + ), + method(IpcChannels.PREVIEW_AUTOMATION_TYPE_CHANNEL, (raw) => + previewViewManager.automationType( + tabIdFrom(raw), + inputFrom(raw) as Parameters[1], + ), + ), + method(IpcChannels.PREVIEW_AUTOMATION_PRESS_CHANNEL, (raw) => + previewViewManager.automationPress( + tabIdFrom(raw), + inputFrom(raw) as Parameters[1], + ), + ), + method(IpcChannels.PREVIEW_AUTOMATION_SCROLL_CHANNEL, (raw) => + previewViewManager.automationScroll( + tabIdFrom(raw), + inputFrom(raw) as Parameters[1], + ), + ), + method(IpcChannels.PREVIEW_AUTOMATION_EVALUATE_CHANNEL, (raw) => + previewViewManager.automationEvaluate( + tabIdFrom(raw), + inputFrom(raw) as Parameters[1], + ), + ), + method(IpcChannels.PREVIEW_AUTOMATION_WAIT_FOR_CHANNEL, (raw) => + previewViewManager.automationWaitFor( + tabIdFrom(raw), + inputFrom(raw) as Parameters[1], + ), + ), ] as const; diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.ts index a66a168040190246576e3d80f1559ced34af67c4..8f890c27e781c4529686959b2e90edd07c8956a7 100644 --- a/apps/desktop/src/preload.ts +++ b/apps/desktop/src/preload.ts @@ -163,6 +163,24 @@ contextBridge.exposeInMainWorld("desktopBridge", { pickElement: (tabId) => ipcRenderer.invoke(IpcChannels.PREVIEW_PICK_ELEMENT_CHANNEL, { tabId }), cancelPickElement: (tabId) => ipcRenderer.invoke(IpcChannels.PREVIEW_CANCEL_PICK_ELEMENT_CHANNEL, { tabId }), + automation: { + status: (tabId) => + ipcRenderer.invoke(IpcChannels.PREVIEW_AUTOMATION_STATUS_CHANNEL, { tabId }), + snapshot: (tabId) => + ipcRenderer.invoke(IpcChannels.PREVIEW_AUTOMATION_SNAPSHOT_CHANNEL, { tabId }), + click: (tabId, input) => + ipcRenderer.invoke(IpcChannels.PREVIEW_AUTOMATION_CLICK_CHANNEL, { tabId, input }), + type: (tabId, input) => + ipcRenderer.invoke(IpcChannels.PREVIEW_AUTOMATION_TYPE_CHANNEL, { tabId, input }), + press: (tabId, input) => + ipcRenderer.invoke(IpcChannels.PREVIEW_AUTOMATION_PRESS_CHANNEL, { tabId, input }), + scroll: (tabId, input) => + ipcRenderer.invoke(IpcChannels.PREVIEW_AUTOMATION_SCROLL_CHANNEL, { tabId, input }), + evaluate: (tabId, input) => + ipcRenderer.invoke(IpcChannels.PREVIEW_AUTOMATION_EVALUATE_CHANNEL, { tabId, input }), + waitFor: (tabId, input) => + ipcRenderer.invoke(IpcChannels.PREVIEW_AUTOMATION_WAIT_FOR_CHANNEL, { tabId, input }), + }, onStateChange: (listener) => { const wrappedListener = ( _event: Electron.IpcRendererEvent, diff --git a/apps/desktop/src/preview-view-manager.test.ts b/apps/desktop/src/preview-view-manager.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..54d33a8569986adebeb7d0f73b8bda64687e830e --- /dev/null +++ b/apps/desktop/src/preview-view-manager.test.ts @@ -0,0 +1,44 @@ +import { beforeEach, describe, expect, it, vi } from "vite-plus/test"; + +const fromId = vi.fn(() => null); + +vi.mock("electron", () => ({ + session: { + fromPartition: vi.fn(), + }, + webContents: { + fromId, + }, +})); + +describe("PreviewViewManager automation status", () => { + beforeEach(() => { + fromId.mockClear(); + }); + + it("reports an unregistered webview as temporarily unavailable", async () => { + const { PreviewViewManager } = await import("./preview-view-manager.ts"); + const manager = new PreviewViewManager(); + + expect(manager.automationStatus("tab_1")).toEqual({ + available: false, + visible: true, + tabId: "tab_1", + url: null, + title: null, + loading: false, + }); + + manager.createTab("tab_1"); + + expect(manager.automationStatus("tab_1")).toEqual({ + available: false, + visible: true, + tabId: "tab_1", + url: null, + title: null, + loading: false, + }); + expect(fromId).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/desktop/src/preview-view-manager.ts b/apps/desktop/src/preview-view-manager.ts index 1f378e90cea5628bf82f7715ac3c0fb3937fd6ed..ce8f9964529470ec191fb6630a79f27cf5203fd0 100644 --- a/apps/desktop/src/preview-view-manager.ts +++ b/apps/desktop/src/preview-view-manager.ts @@ -6,9 +6,21 @@ * elements live in the renderer; we only attach listeners and forward state * here). Single layer-scoped browser session partition. */ -import type { PreviewAnnotationPayload, PreviewAnnotationRect } from "@t3tools/contracts"; +import type { + PreviewAnnotationPayload, + PreviewAnnotationRect, + PreviewAutomationClickInput, + PreviewAutomationEvaluateInput, + PreviewAutomationPressInput, + PreviewAutomationScrollInput, + PreviewAutomationSnapshot, + PreviewAutomationStatus, + PreviewAutomationTypeInput, + PreviewAutomationWaitForInput, +} from "@t3tools/contracts"; import { normalizePreviewUrl } from "@t3tools/shared/preview"; import { type BrowserWindow, type Session, session, webContents } from "electron"; +import { setTimeout as sleep } from "node:timers/promises"; import { isPreviewAnnotationPayload } from "./picked-element-payload.ts"; @@ -53,6 +65,36 @@ const ZOOM_LEVELS: ReadonlyArray = [ const DEFAULT_ZOOM_FACTOR = 1.0; const ZOOM_EPSILON = 0.001; +const MAX_EVALUATION_BYTES = 64_000; +const MAX_VISIBLE_TEXT_LENGTH = 20_000; +const MAX_INTERACTIVE_ELEMENTS = 200; +const MAX_SCREENSHOT_WIDTH = 1280; + +interface CdpEvaluationResult { + readonly result?: { + readonly value?: unknown; + readonly description?: string; + }; + readonly exceptionDetails?: { + readonly text?: string; + readonly exception?: { readonly description?: string }; + }; +} + +const automationError = ( + tag: + | "PreviewAutomationExecutionError" + | "PreviewAutomationInvalidSelectorError" + | "PreviewAutomationResultTooLargeError" + | "PreviewAutomationTimeoutError", + message: string, + detail?: unknown, +): Error & { detail?: unknown } => { + const error = new Error(message) as Error & { detail?: unknown }; + error.name = tag; + if (detail !== undefined) error.detail = detail; + return error; +}; const normalizeCaptureRect = (value: unknown): PreviewAnnotationRect | null => { if (typeof value !== "object" || value === null) return null; @@ -455,6 +497,391 @@ export class PreviewViewManager { this.applyZoom(tabId, () => DEFAULT_ZOOM_FACTOR); } + automationStatus(tabId: string): PreviewAutomationStatus { + const tab = this.tabs.get(tabId); + if (!tab || tab.webContentsId == null) { + const navStatus = tab?.navStatus; + return { + available: false, + visible: true, + tabId, + url: !navStatus || navStatus.kind === "Idle" ? null : navStatus.url, + title: !navStatus || navStatus.kind === "Idle" ? null : navStatus.title, + loading: navStatus?.kind === "Loading", + }; + } + const wc = webContents.fromId(tab.webContentsId); + if (!wc || wc.isDestroyed()) { + return { + available: false, + visible: true, + tabId, + url: null, + title: null, + loading: false, + }; + } + return { + available: true, + visible: true, + tabId, + url: wc.getURL() || null, + title: wc.getTitle() || null, + loading: wc.isLoading(), + }; + } + + async automationSnapshot(tabId: string): Promise { + const wc = this.requireWebContents(tabId); + return this.withDebugger(wc, async (send) => { + await Promise.all([send("Runtime.enable"), send("Accessibility.enable")]); + const page = await this.evaluateWithDebugger<{ + url: string; + title: string; + loading: boolean; + visibleText: string; + interactiveElements: PreviewAutomationSnapshot["interactiveElements"]; + }>( + send, + `(() => { + const selectorFor = (element) => { + if (element.id) return "#" + CSS.escape(element.id); + for (const attribute of ["data-testid", "name"]) { + const value = element.getAttribute(attribute); + if (value) return element.tagName.toLowerCase() + "[" + attribute + "=" + JSON.stringify(value) + "]"; + } + const parts = []; + let current = element; + while (current && current.nodeType === Node.ELEMENT_NODE && parts.length < 8) { + let part = current.tagName.toLowerCase(); + const parent = current.parentElement; + if (parent) { + const siblings = Array.from(parent.children).filter((child) => child.tagName === current.tagName); + if (siblings.length > 1) part += ":nth-of-type(" + (siblings.indexOf(current) + 1) + ")"; + } + parts.unshift(part); + current = parent; + } + return parts.join(" > "); + }; + const visible = (element) => { + const style = getComputedStyle(element); + const rect = element.getBoundingClientRect(); + return style.visibility !== "hidden" && style.display !== "none" && rect.width > 0 && rect.height > 0; + }; + const elements = Array.from(document.querySelectorAll( + "a[href],button,input,textarea,select,[role],[tabindex]" + )).filter(visible).slice(0, ${MAX_INTERACTIVE_ELEMENTS}).map((element) => { + const rect = element.getBoundingClientRect(); + return { + tag: element.tagName.toLowerCase(), + role: element.getAttribute("role"), + name: element.getAttribute("aria-label") || element.innerText || element.getAttribute("name") || "", + selector: selectorFor(element), + x: rect.x, + y: rect.y, + width: rect.width, + height: rect.height + }; + }); + return { + url: location.href, + title: document.title, + loading: document.readyState !== "complete", + visibleText: (document.body?.innerText || "").slice(0, ${MAX_VISIBLE_TEXT_LENGTH}), + interactiveElements: elements + }; + })()`, + true, + ); + const accessibility = await send("Accessibility.getFullAXTree"); + let image = await wc.capturePage(); + let size = image.getSize(); + if (size.width > MAX_SCREENSHOT_WIDTH) { + image = image.resize({ width: MAX_SCREENSHOT_WIDTH }); + size = image.getSize(); + } + return { + ...page, + accessibilityTree: accessibility, + screenshot: { + mimeType: "image/png", + data: image.toPNG().toString("base64"), + width: size.width, + height: size.height, + }, + }; + }); + } + + async automationClick(tabId: string, input: PreviewAutomationClickInput): Promise { + const wc = this.requireWebContents(tabId); + await this.withDebugger(wc, async (send) => { + await Promise.all([ + send("Runtime.enable"), + send("Input.setIgnoreInputEvents", { ignore: false }), + ]); + let x: number; + let y: number; + if ("selector" in input) { + const point = await this.evaluateWithDebugger< + { x: number; y: number } | { invalidSelector: true; message: string } | { notFound: true } + >( + send, + `(() => { + try { + const element = document.querySelector(${JSON.stringify(input.selector)}); + if (!element) return { notFound: true }; + element.scrollIntoView({ block: "center", inline: "center" }); + const rect = element.getBoundingClientRect(); + return { x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 }; + } catch (error) { + return { invalidSelector: true, message: String(error) }; + } + })()`, + true, + ); + if ("invalidSelector" in point) { + throw automationError("PreviewAutomationInvalidSelectorError", point.message, { + selector: input.selector, + }); + } + if ("notFound" in point) { + throw automationError( + "PreviewAutomationExecutionError", + `No element matches selector ${input.selector}.`, + ); + } + x = point.x; + y = point.y; + } else { + x = input.x; + y = input.y; + } + const viewport = await this.evaluateWithDebugger<{ width: number; height: number }>( + send, + "({ width: window.innerWidth, height: window.innerHeight })", + true, + ); + if (x < 0 || y < 0 || x > viewport.width || y > viewport.height) { + throw automationError( + "PreviewAutomationExecutionError", + `Click coordinates (${x}, ${y}) are outside the preview viewport.`, + ); + } + await send("Input.dispatchMouseEvent", { + type: "mousePressed", + x, + y, + button: "left", + clickCount: 1, + }); + await send("Input.dispatchMouseEvent", { + type: "mouseReleased", + x, + y, + button: "left", + clickCount: 1, + }); + }); + } + + async automationType(tabId: string, input: PreviewAutomationTypeInput): Promise { + const wc = this.requireWebContents(tabId); + await this.withDebugger(wc, async (send) => { + await send("Runtime.enable"); + const focusResult = await this.evaluateWithDebugger< + { ok: true } | { invalidSelector: true; message: string } | { notFound: true } + >( + send, + `(() => { + try { + const element = ${ + input.selector + ? `document.querySelector(${JSON.stringify(input.selector)})` + : "document.activeElement" + }; + if (!element) return { notFound: true }; + element.focus(); + if (${input.clear ?? false}) { + if ("value" in element) element.value = ""; + else if (element.isContentEditable) element.textContent = ""; + element.dispatchEvent(new InputEvent("input", { bubbles: true, inputType: "deleteContentBackward" })); + } + return { ok: true }; + } catch (error) { + return { invalidSelector: true, message: String(error) }; + } + })()`, + true, + ); + if ("invalidSelector" in focusResult) { + throw automationError("PreviewAutomationInvalidSelectorError", focusResult.message, { + selector: input.selector ?? "", + }); + } + if ("notFound" in focusResult) { + throw automationError( + "PreviewAutomationExecutionError", + input.selector + ? `No element matches selector ${input.selector}.` + : "No element is focused in the preview.", + ); + } + await send("Input.insertText", { text: input.text }); + await this.evaluateWithDebugger( + send, + `(() => { + const element = document.activeElement; + element?.dispatchEvent(new InputEvent("input", { bubbles: true, inputType: "insertText", data: ${JSON.stringify(input.text)} })); + element?.dispatchEvent(new Event("change", { bubbles: true })); + })()`, + false, + ); + }); + } + + async automationPress(tabId: string, input: PreviewAutomationPressInput): Promise { + const wc = this.requireWebContents(tabId); + await this.withDebugger(wc, async (send) => { + const modifiers = (input.modifiers ?? []).reduce((value, modifier) => { + switch (modifier) { + case "Alt": + return value | 1; + case "Control": + return value | 2; + case "Meta": + return value | 4; + case "Shift": + return value | 8; + } + }, 0); + const key = input.key; + const text = key.length === 1 ? key : undefined; + const params = { + key, + code: key.length === 1 ? `Key${key.toUpperCase()}` : key, + modifiers, + ...(text ? { text, unmodifiedText: text } : {}), + }; + await send("Input.dispatchKeyEvent", { type: "keyDown", ...params }); + await send("Input.dispatchKeyEvent", { type: "keyUp", ...params }); + }); + } + + async automationScroll(tabId: string, input: PreviewAutomationScrollInput): Promise { + const wc = this.requireWebContents(tabId); + await this.withDebugger(wc, async (send) => { + await send("Runtime.enable"); + const result = await this.evaluateWithDebugger< + { ok: true } | { invalidSelector: true; message: string } | { notFound: true } + >( + send, + `(() => { + try { + const target = ${ + input.selector + ? `document.querySelector(${JSON.stringify(input.selector)})` + : "window" + }; + if (!target) return { notFound: true }; + target.scrollBy({ left: ${input.deltaX ?? 0}, top: ${input.deltaY ?? 0}, behavior: "instant" }); + return { ok: true }; + } catch (error) { + return { invalidSelector: true, message: String(error) }; + } + })()`, + true, + ); + if ("invalidSelector" in result) { + throw automationError("PreviewAutomationInvalidSelectorError", result.message, { + selector: input.selector ?? "", + }); + } + if ("notFound" in result) { + throw automationError( + "PreviewAutomationExecutionError", + `No element matches selector ${input.selector}.`, + ); + } + }); + } + + async automationEvaluate(tabId: string, input: PreviewAutomationEvaluateInput): Promise { + const wc = this.requireWebContents(tabId); + return this.withDebugger(wc, async (send) => { + await send("Runtime.enable"); + const value = await this.evaluateWithDebugger( + send, + input.expression, + input.returnByValue ?? true, + input.awaitPromise ?? true, + ); + const serialized = JSON.stringify(value); + if ( + serialized !== undefined && + Buffer.byteLength(serialized, "utf8") > MAX_EVALUATION_BYTES + ) { + throw automationError( + "PreviewAutomationResultTooLargeError", + `Evaluation result exceeds ${MAX_EVALUATION_BYTES} bytes.`, + { maximumBytes: MAX_EVALUATION_BYTES }, + ); + } + return value; + }); + } + + async automationWaitFor(tabId: string, input: PreviewAutomationWaitForInput): Promise { + const wc = this.requireWebContents(tabId); + const timeoutMs = input.timeoutMs ?? 15_000; + await this.withDebugger(wc, async (send) => { + await send("Runtime.enable"); + const deadline = Date.now() + timeoutMs; + while (Date.now() <= deadline) { + const result = await this.evaluateWithDebugger< + { matched: boolean } | { invalidSelector: true; message: string } + >( + send, + `(() => { + try { + const selectorMatched = ${ + input.selector + ? `document.querySelector(${JSON.stringify(input.selector)}) !== null` + : "true" + }; + const textMatched = ${ + input.text + ? `(document.body?.innerText || "").includes(${JSON.stringify(input.text)})` + : "true" + }; + const urlMatched = ${ + input.urlIncludes + ? `location.href.includes(${JSON.stringify(input.urlIncludes)})` + : "true" + }; + return { matched: selectorMatched && textMatched && urlMatched }; + } catch (error) { + return { invalidSelector: true, message: String(error) }; + } + })()`, + true, + ); + if ("invalidSelector" in result) { + throw automationError("PreviewAutomationInvalidSelectorError", result.message, { + selector: input.selector ?? "", + }); + } + if (result.matched) return; + await sleep(100); + } + throw automationError( + "PreviewAutomationTimeoutError", + `Preview condition did not match within ${timeoutMs}ms.`, + ); + }); + } + private applyZoom(tabId: string, transform: (current: number) => number): void { const tab = this.tabs.get(tabId); if (!tab) return; @@ -469,6 +896,62 @@ export class PreviewViewManager { this.update(tabId, { zoomFactor: next }); } + private async withDebugger( + wc: Electron.WebContents, + use: ( + send: (method: string, commandParams?: Record) => Promise, + ) => Promise, + ): Promise { + if (wc.debugger.isAttached()) { + throw automationError( + "PreviewAutomationExecutionError", + "Preview automation is unavailable while another debugger is attached.", + ); + } + wc.debugger.attach("1.3"); + try { + return await use((method, commandParams) => wc.debugger.sendCommand(method, commandParams)); + } catch (cause) { + if (cause instanceof Error && cause.name.startsWith("PreviewAutomation")) throw cause; + throw automationError( + "PreviewAutomationExecutionError", + cause instanceof Error ? cause.message : String(cause), + cause, + ); + } finally { + if (wc.debugger.isAttached()) { + try { + wc.debugger.detach(); + } catch { + // The target can disappear while an operation is completing. + } + } + } + } + + private async evaluateWithDebugger( + send: (method: string, commandParams?: Record) => Promise, + expression: string, + returnByValue: boolean, + awaitPromise = true, + ): Promise { + const response = (await send("Runtime.evaluate", { + expression, + awaitPromise, + returnByValue, + userGesture: true, + })) as CdpEvaluationResult; + if (response.exceptionDetails) { + throw automationError( + "PreviewAutomationExecutionError", + response.exceptionDetails.exception?.description ?? + response.exceptionDetails.text ?? + "JavaScript evaluation failed.", + ); + } + return response.result?.value as A; + } + onStateChange(listener: Listener): () => void { this.listeners.add(listener); return () => { diff --git a/apps/server/src/mcp/Layers/McpHttpServer.test.ts b/apps/server/src/mcp/Layers/McpHttpServer.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..8d01277ce3e7852460f302d4c2ec53060ce3ab6b --- /dev/null +++ b/apps/server/src/mcp/Layers/McpHttpServer.test.ts @@ -0,0 +1,133 @@ +import { expect, it } from "@effect/vitest"; +import { EnvironmentId, PreviewTabId, ProviderInstanceId, ThreadId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Stream from "effect/Stream"; +import { McpSchema, McpServer } from "effect/unstable/ai"; +import { HttpServerResponse } from "effect/unstable/http"; + +import { McpInvocationContext } from "../Services/McpInvocationContext.ts"; +import { normalizeMcpHttpResponse, PreviewToolkitRegistrationLive } from "./McpHttpServer.ts"; +import { previewAutomationBroker } from "./PreviewAutomationBroker.ts"; + +const environmentId = EnvironmentId.make("environment-mcp-test"); +const threadId = ThreadId.make("thread-mcp-test"); +const tabId = PreviewTabId.make("tab-mcp-test"); +const invocation = { + environmentId, + threadId, + providerSessionId: "provider-session-mcp-test", + providerInstanceId: ProviderInstanceId.make("codex"), + capabilities: new Set(["preview"] as const), + issuedAt: 1, + expiresAt: Number.MAX_SAFE_INTEGER, +}; +const client = McpSchema.McpServerClient.of({ + clientId: 1, + initializePayload: { + protocolVersion: "2025-03-26", + capabilities: {}, + clientInfo: { name: "mcp-test", version: "1.0.0" }, + }, + getClient: Effect.die("unused"), +}); +const TestLayer = PreviewToolkitRegistrationLive.pipe( + Layer.provideMerge(McpServer.McpServer.layer), +); + +it("normalizes empty successful notification responses to accepted", () => { + const notificationResponse = normalizeMcpHttpResponse( + HttpServerResponse.text("", { status: 200, contentType: "application/json" }), + ); + expect(notificationResponse.status).toBe(202); + + const resultResponse = normalizeMcpHttpResponse( + HttpServerResponse.jsonUnsafe({ jsonrpc: "2.0", id: 1, result: {} }), + ); + expect(resultResponse.status).toBe(200); +}); + +it.effect("registers annotated tools and preserves authenticated request context", () => + Effect.scoped( + Effect.gen(function* () { + const server = yield* McpServer.McpServer; + const requests = yield* previewAutomationBroker.connect("mcp-test-client"); + yield* Stream.runForEach(requests, (request) => + previewAutomationBroker.respond({ + requestId: request.requestId, + ok: true, + result: + request.operation === "snapshot" + ? { + url: "http://example.test/", + title: "Example", + loading: false, + visibleText: "Example", + interactiveElements: [], + accessibilityTree: {}, + screenshot: { + mimeType: "image/png", + data: Buffer.from("png").toString("base64"), + width: 10, + height: 5, + }, + } + : { + available: true, + visible: true, + tabId, + url: "http://example.test/", + title: "Example", + loading: false, + }, + }), + ).pipe(Effect.forkScoped); + yield* Effect.yieldNow; + yield* previewAutomationBroker.reportOwner({ + clientId: "mcp-test-client", + environmentId, + threadId, + tabId, + visible: true, + supportsAutomation: true, + focusedAt: "2026-06-11T00:00:00.000Z", + }); + + const statusTool = server.tools.find(({ tool }) => tool.name === "preview_status"); + expect(statusTool?.tool.annotations?.readOnlyHint).toBe(true); + expect(statusTool?.tool.annotations?.idempotentHint).toBe(true); + + const status = yield* server + .callTool({ name: "preview_status", arguments: {} }) + .pipe( + Effect.provideService(McpInvocationContext, invocation), + Effect.provideService(McpSchema.McpServerClient, client), + ); + expect(status.isError).toBe(false); + expect(status.structuredContent).toMatchObject({ + available: true, + tabId, + }); + + const malformed = yield* server + .callTool({ name: "preview_click", arguments: { selector: "" } }) + .pipe( + Effect.provideService(McpInvocationContext, invocation), + Effect.provideService(McpSchema.McpServerClient, client), + ); + expect(malformed.isError).toBe(true); + + const snapshot = yield* server + .callTool({ name: "preview_snapshot", arguments: {} }) + .pipe( + Effect.provideService(McpInvocationContext, invocation), + Effect.provideService(McpSchema.McpServerClient, client), + ); + expect(snapshot.isError).toBe(false); + expect(snapshot.content.some((content) => content.type === "image")).toBe(true); + expect(snapshot.structuredContent).toMatchObject({ + screenshot: { mimeType: "image/png", width: 10, height: 5 }, + }); + }), + ).pipe(Effect.provide(TestLayer)), +); diff --git a/apps/server/src/mcp/Layers/McpHttpServer.ts b/apps/server/src/mcp/Layers/McpHttpServer.ts new file mode 100644 index 0000000000000000000000000000000000000000..645a8fdc3c8456fcf4f75145204b799be45c5e4b --- /dev/null +++ b/apps/server/src/mcp/Layers/McpHttpServer.ts @@ -0,0 +1,162 @@ +import * as Cause from "effect/Cause"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Sink from "effect/Sink"; +import * as Stream from "effect/Stream"; +import { McpSchema, McpServer, Tool } from "effect/unstable/ai"; +import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/unstable/http"; + +import packageJson from "../../../package.json" with { type: "json" }; +import { McpInvocationContext } from "../Services/McpInvocationContext.ts"; +import { McpSessionRegistry } from "../Services/McpSessionRegistry.ts"; +import { PreviewToolkitHandlersLive } from "../toolkits/preview/handlers.ts"; +import { PreviewToolkit } from "../toolkits/preview/tools.ts"; + +const unauthorized = HttpServerResponse.jsonUnsafe( + { + error: "invalid_mcp_credential", + message: "A valid provider-scoped MCP bearer credential is required.", + }, + { + status: 401, + headers: { + "cache-control": "no-store", + "www-authenticate": "Bearer", + }, + }, +); + +export const normalizeMcpHttpResponse = ( + response: HttpServerResponse.HttpServerResponse, +): HttpServerResponse.HttpServerResponse => { + const bodyIsEmpty = + response.body._tag === "Empty" || + (response.body._tag === "Uint8Array" && response.body.contentLength === 0) || + (response.body._tag === "Raw" && response.body.contentLength === 0); + return response.status === 200 && bodyIsEmpty + ? HttpServerResponse.setStatus(response, 202) + : response; +}; + +const McpAuthMiddlewareLive = HttpRouter.middleware<{ + provides: McpInvocationContext; +}>()( + Effect.gen(function* () { + const registry = yield* McpSessionRegistry; + return (httpEffect) => + Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + const authorization = request.headers.authorization; + const token = + authorization?.startsWith("Bearer ") === true + ? authorization.slice("Bearer ".length).trim() + : ""; + const invocation = yield* registry.resolve(token); + if (!invocation) return unauthorized; + return yield* httpEffect.pipe( + Effect.provideService(McpInvocationContext, invocation), + Effect.map(normalizeMcpHttpResponse), + ); + }); + }), +).layer; + +const McpTransportLive = McpServer.layerHttp({ + name: "T3 Code", + version: packageJson.version, + path: "/mcp", +}).pipe(Layer.provide(McpAuthMiddlewareLive)); + +export const PreviewToolkitRegistrationLive = Layer.effectDiscard( + Effect.gen(function* () { + const server = yield* McpServer.McpServer; + const built = yield* PreviewToolkit; + const handleTool = built.handle as unknown as ( + name: keyof typeof built.tools, + payload: unknown, + ) => Effect.Effect< + Stream.Stream<{ readonly encodedResult: unknown }, Error>, + Error, + McpInvocationContext + >; + for (const tool of Object.values(built.tools)) { + yield* server.addTool({ + tool: new McpSchema.Tool({ + name: tool.name, + description: Tool.getDescription(tool), + inputSchema: Tool.getJsonSchema(tool), + annotations: { + ...Context.getOption(tool.annotations, Tool.Title).pipe( + Option.map((title) => ({ title })), + Option.getOrUndefined, + ), + readOnlyHint: Context.get(tool.annotations, Tool.Readonly), + destructiveHint: Context.get(tool.annotations, Tool.Destructive), + idempotentHint: Context.get(tool.annotations, Tool.Idempotent), + openWorldHint: Context.get(tool.annotations, Tool.OpenWorld), + }, + }), + annotations: tool.annotations, + handle: (payload) => + handleTool(tool.name as keyof typeof built.tools, payload).pipe( + Stream.unwrap, + Stream.run(Sink.last()), + Effect.flatMap(Effect.fromOption), + Effect.matchCause({ + onFailure: (cause) => + new McpSchema.CallToolResult({ + isError: true, + content: [{ type: "text", text: Cause.pretty(cause) }], + }), + onSuccess: (result) => { + if (tool.name === "preview_snapshot") { + const snapshot = result.encodedResult as { + readonly screenshot: { + readonly mimeType: "image/png"; + readonly data: string; + readonly width: number; + readonly height: number; + }; + readonly [key: string]: unknown; + }; + const { screenshot, ...page } = snapshot; + const metadata = { + ...page, + screenshot: { + mimeType: screenshot.mimeType, + width: screenshot.width, + height: screenshot.height, + }, + }; + return new McpSchema.CallToolResult({ + isError: false, + structuredContent: metadata, + content: [ + { type: "text", text: JSON.stringify(metadata) }, + { + type: "image", + data: new Uint8Array(Buffer.from(screenshot.data, "base64")), + mimeType: screenshot.mimeType, + }, + ], + }); + } + return new McpSchema.CallToolResult({ + isError: false, + structuredContent: + typeof result.encodedResult === "object" ? result.encodedResult : undefined, + content: [{ type: "text", text: JSON.stringify(result.encodedResult) }], + }); + }, + }), + ) as unknown as Effect.Effect, + }); + } + }), +).pipe(Layer.provide(PreviewToolkitHandlersLive)); + +export const McpHttpServerLive = Layer.mergeAll(PreviewToolkitRegistrationLive).pipe( + Layer.provideMerge(McpTransportLive), +); diff --git a/apps/server/src/mcp/Layers/McpSessionRegistry.test.ts b/apps/server/src/mcp/Layers/McpSessionRegistry.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..bbd27c63cac25e7045cb17c73309a03325f60c48 --- /dev/null +++ b/apps/server/src/mcp/Layers/McpSessionRegistry.test.ts @@ -0,0 +1,66 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { expect, it } from "@effect/vitest"; +import { EnvironmentId, ProviderInstanceId, ThreadId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import { HttpServer } from "effect/unstable/http"; + +import { ServerEnvironment } from "../../environment/Services/ServerEnvironment.ts"; +import { makeMcpSessionRegistry } from "./McpSessionRegistry.ts"; + +const environmentId = EnvironmentId.make("environment-1"); +const fakeHttpServer = HttpServer.HttpServer.of({ + address: { _tag: "TcpAddress", hostname: "127.0.0.1", port: 43123 }, + serve: (() => Effect.void) as HttpServer.HttpServer["Service"]["serve"], +}); +const fakeEnvironment = ServerEnvironment.of({ + getEnvironmentId: Effect.succeed(environmentId), + getDescriptor: Effect.die("unused"), +}); + +const makeRegistry = (now: () => number) => + makeMcpSessionRegistry({ + now, + idleTimeoutMs: 100, + maximumLifetimeMs: 1_000, + }).pipe( + Effect.provideService(HttpServer.HttpServer, fakeHttpServer), + Effect.provideService(ServerEnvironment, fakeEnvironment), + Effect.provide(NodeServices.layer), + ); + +it.effect("stores only a token hash, resolves the bearer token, and revokes by thread", () => + Effect.gen(function* () { + let timestamp = 1_000; + const registry = yield* makeRegistry(() => timestamp); + const threadId = ThreadId.make("thread-1"); + const issued = yield* registry.issue({ + threadId, + providerInstanceId: ProviderInstanceId.make("codex"), + }); + expect(issued.config.endpoint).toBe("http://127.0.0.1:43123/mcp"); + const token = issued.config.authorizationHeader.replace(/^Bearer\s+/, ""); + expect(token.length).toBeGreaterThan(20); + + const resolved = yield* registry.resolve(token); + expect(resolved?.threadId).toBe(threadId); + + yield* registry.revokeThread(threadId); + expect(yield* registry.resolve(token)).toBeUndefined(); + + timestamp += 2_000; + }), +); + +it.effect("expires credentials after inactivity", () => + Effect.gen(function* () { + let timestamp = 1_000; + const registry = yield* makeRegistry(() => timestamp); + const issued = yield* registry.issue({ + threadId: ThreadId.make("thread-2"), + providerInstanceId: ProviderInstanceId.make("claude"), + }); + const token = issued.config.authorizationHeader.replace(/^Bearer\s+/, ""); + timestamp += 101; + expect(yield* registry.resolve(token)).toBeUndefined(); + }), +); diff --git a/apps/server/src/mcp/Layers/McpSessionRegistry.ts b/apps/server/src/mcp/Layers/McpSessionRegistry.ts new file mode 100644 index 0000000000000000000000000000000000000000..a6aa95072a6faa23b2a151601af1b2801e07ae42 --- /dev/null +++ b/apps/server/src/mcp/Layers/McpSessionRegistry.ts @@ -0,0 +1,173 @@ +import { ProviderInstanceId, ThreadId } from "@t3tools/contracts"; +import * as Crypto from "effect/Crypto"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Ref from "effect/Ref"; +import { HttpServer } from "effect/unstable/http"; + +import { ServerEnvironment } from "../../environment/Services/ServerEnvironment.ts"; +import type { McpInvocationScope } from "../Services/McpInvocationContext.ts"; +import { + McpSessionRegistry, + type McpCredentialRequest, + type McpIssuedCredential, + type McpSessionRegistryShape, +} from "../Services/McpSessionRegistry.ts"; + +interface CredentialRecord { + readonly tokenHash: string; + readonly scope: McpInvocationScope; + readonly lastUsedAt: number; +} + +interface RegistryState { + readonly records: ReadonlyMap; +} + +export interface McpSessionRegistryOptions { + readonly idleTimeoutMs?: number; + readonly maximumLifetimeMs?: number; + readonly now?: () => number; +} + +const DEFAULT_IDLE_TIMEOUT_MS = 30 * 60 * 1_000; +const DEFAULT_MAXIMUM_LIFETIME_MS = 8 * 60 * 60 * 1_000; + +const bytesToHex = (bytes: Uint8Array): string => + Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join(""); + +const tokenFromBytes = (bytes: Uint8Array): string => Buffer.from(bytes).toString("base64url"); + +export const makeMcpSessionRegistry = (options: McpSessionRegistryOptions = {}) => + Effect.gen(function* () { + const crypto = yield* Crypto.Crypto; + const environment = yield* ServerEnvironment; + const environmentId = yield* environment.getEnvironmentId; + const httpServer = yield* HttpServer.HttpServer; + const state = yield* Ref.make({ records: new Map() }); + const now = options.now ?? Date.now; + const idleTimeoutMs = options.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS; + const maximumLifetimeMs = options.maximumLifetimeMs ?? DEFAULT_MAXIMUM_LIFETIME_MS; + const endpoint = + httpServer.address._tag === "TcpAddress" + ? `http://127.0.0.1:${httpServer.address.port}/mcp` + : "http://127.0.0.1/mcp"; + + const hashToken = (token: string) => + crypto + .digest("SHA-256", new TextEncoder().encode(token)) + .pipe(Effect.map(bytesToHex), Effect.orDie); + + const pruneExpired = (records: ReadonlyMap, timestamp: number) => { + let changed = false; + const next = new Map(); + for (const [hash, record] of records) { + if (timestamp <= record.scope.expiresAt && timestamp - record.lastUsedAt <= idleTimeoutMs) { + next.set(hash, record); + } else { + changed = true; + } + } + return changed ? next : records; + }; + + const issue: McpSessionRegistryShape["issue"] = (request) => + Effect.gen(function* () { + const issuedAt = now(); + const providerSessionId = yield* crypto.randomUUIDv4.pipe(Effect.orDie); + const rawToken = yield* crypto + .randomBytes(32) + .pipe(Effect.map(tokenFromBytes), Effect.orDie); + const tokenHash = yield* hashToken(rawToken); + const expiresAt = issuedAt + maximumLifetimeMs; + const scope: McpInvocationScope = { + environmentId, + threadId: ThreadId.make(request.threadId), + providerSessionId, + providerInstanceId: ProviderInstanceId.make(request.providerInstanceId), + capabilities: new Set(["preview"]), + issuedAt, + expiresAt, + }; + yield* Ref.update(state, ({ records }) => { + const next = new Map(pruneExpired(records, issuedAt)); + next.set(tokenHash, { tokenHash, scope, lastUsedAt: issuedAt }); + return { records: next }; + }); + return { + config: { + environmentId, + threadId: scope.threadId, + providerSessionId, + providerInstanceId: scope.providerInstanceId, + endpoint, + authorizationHeader: `Bearer ${rawToken}`, + }, + expiresAt, + }; + }); + + const resolve: McpSessionRegistryShape["resolve"] = (rawToken) => + Effect.gen(function* () { + if (rawToken.length === 0) return undefined; + const tokenHash = yield* hashToken(rawToken); + const timestamp = now(); + let resolved: McpInvocationScope | undefined; + yield* Ref.update(state, ({ records }) => { + const current = pruneExpired(records, timestamp); + const record = current.get(tokenHash); + if (!record) return { records: current }; + resolved = record.scope; + const next = new Map(current); + next.set(tokenHash, { ...record, lastUsedAt: timestamp }); + return { records: next }; + }); + return resolved; + }); + + const revokeWhere = (predicate: (record: CredentialRecord) => boolean) => + Ref.update(state, ({ records }) => ({ + records: new Map(Array.from(records).filter(([, record]) => !predicate(record))), + })); + + return McpSessionRegistry.of({ + issue, + resolve, + revokeProviderSession: (providerSessionId) => + revokeWhere((record) => record.scope.providerSessionId === providerSessionId), + revokeThread: (threadId) => revokeWhere((record) => record.scope.threadId === threadId), + revokeAll: Ref.set(state, { records: new Map() }), + }); + }); + +let activeMcpSessionRegistry: McpSessionRegistryShape | undefined; + +export const McpSessionRegistryLive: Layer.Layer< + McpSessionRegistry, + never, + Crypto.Crypto | ServerEnvironment | HttpServer.HttpServer +> = Layer.effect( + McpSessionRegistry, + makeMcpSessionRegistry().pipe( + Effect.tap((registry) => + Effect.sync(() => { + activeMcpSessionRegistry = registry; + }), + ), + ), +); + +export const issueActiveMcpCredential = ( + request: McpCredentialRequest, +): Effect.Effect => + activeMcpSessionRegistry + ? activeMcpSessionRegistry + .revokeThread(request.threadId) + .pipe(Effect.andThen(activeMcpSessionRegistry.issue(request))) + : Effect.sync((): McpIssuedCredential | undefined => undefined); + +export const revokeActiveMcpThread = (threadId: ThreadId): Effect.Effect => + activeMcpSessionRegistry ? activeMcpSessionRegistry.revokeThread(threadId) : Effect.void; + +export const revokeAllActiveMcpCredentials = (): Effect.Effect => + activeMcpSessionRegistry ? activeMcpSessionRegistry.revokeAll : Effect.void; diff --git a/apps/server/src/mcp/Layers/PreviewAutomationBroker.test.ts b/apps/server/src/mcp/Layers/PreviewAutomationBroker.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..d3e43e4c53cade415c3aa51e865b49c82ac58538 --- /dev/null +++ b/apps/server/src/mcp/Layers/PreviewAutomationBroker.test.ts @@ -0,0 +1,65 @@ +import { expect, it } from "@effect/vitest"; +import { + EnvironmentId, + PreviewAutomationNoFocusedOwnerError, + ProviderInstanceId, + ThreadId, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Stream from "effect/Stream"; + +import { makePreviewAutomationBroker } from "./PreviewAutomationBroker.ts"; + +const scope = { + environmentId: EnvironmentId.make("environment-1"), + threadId: ThreadId.make("thread-1"), + providerSessionId: "provider-session-1", + providerInstanceId: ProviderInstanceId.make("codex"), + capabilities: new Set(["preview"] as const), + issuedAt: 1, + expiresAt: 2, +}; + +it.effect("routes a request to the focused owner and correlates its response", () => + Effect.scoped( + Effect.gen(function* () { + const broker = makePreviewAutomationBroker(); + const requests = yield* broker.connect("client-1"); + yield* Stream.runForEach(requests, (request) => + broker.respond({ + requestId: request.requestId, + ok: true, + result: { available: true }, + }), + ).pipe(Effect.forkScoped); + yield* Effect.yieldNow; + yield* broker.reportOwner({ + clientId: "client-1", + environmentId: scope.environmentId, + threadId: scope.threadId, + tabId: null, + visible: false, + supportsAutomation: true, + focusedAt: "2026-06-11T00:00:00.000Z", + }); + + const result = yield* broker.invoke<{ available: boolean }>({ + scope, + operation: "open", + input: {}, + }); + + expect(result).toEqual({ available: true }); + }), + ), +); + +it.effect("rejects calls when no focused owner exists", () => + Effect.gen(function* () { + const broker = makePreviewAutomationBroker(); + const error = yield* broker + .invoke({ scope, operation: "status", input: {} }) + .pipe(Effect.flip); + expect(error).toBeInstanceOf(PreviewAutomationNoFocusedOwnerError); + }), +); diff --git a/apps/server/src/mcp/Layers/PreviewAutomationBroker.ts b/apps/server/src/mcp/Layers/PreviewAutomationBroker.ts new file mode 100644 index 0000000000000000000000000000000000000000..67d3a6ed5758e869a3650ea6772a5bf617f5e11f --- /dev/null +++ b/apps/server/src/mcp/Layers/PreviewAutomationBroker.ts @@ -0,0 +1,268 @@ +import { + PreviewAutomationExecutionError, + PreviewAutomationInvalidSelectorError, + PreviewAutomationNoFocusedOwnerError, + PreviewAutomationResultTooLargeError, + PreviewAutomationTabNotFoundError, + PreviewAutomationTimeoutError, + PreviewAutomationUnavailableError, + PreviewAutomationUnsupportedClientError, + type PreviewAutomationError, + type PreviewAutomationOwner, + type PreviewAutomationResponse, +} from "@t3tools/contracts"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Queue from "effect/Queue"; +import * as Ref from "effect/Ref"; +import * as Stream from "effect/Stream"; + +import { + PreviewAutomationBroker, + type PreviewAutomationBrokerShape, +} from "../Services/PreviewAutomationBroker.ts"; + +interface ClientConnection { + readonly clientId: string; + readonly queue: Queue.Queue< + Parameters[0] extends never + ? never + : import("@t3tools/contracts").PreviewAutomationRequest + >; +} + +interface PendingRequest { + readonly clientId: string; + readonly deferred: Deferred.Deferred; +} + +interface BrokerState { + readonly clients: ReadonlyMap; + readonly owners: ReadonlyMap; + readonly pending: ReadonlyMap; +} + +const makeResponseError = ( + error: NonNullable, +): PreviewAutomationError => { + switch (error._tag) { + case "PreviewAutomationNoFocusedOwnerError": + return new PreviewAutomationNoFocusedOwnerError({ message: error.message }); + case "PreviewAutomationUnsupportedClientError": + return new PreviewAutomationUnsupportedClientError({ message: error.message }); + case "PreviewAutomationTabNotFoundError": + return new PreviewAutomationTabNotFoundError({ message: error.message }); + case "PreviewAutomationTimeoutError": + return new PreviewAutomationTimeoutError({ message: error.message }); + case "PreviewAutomationInvalidSelectorError": { + const detail = + typeof error.detail === "object" && error.detail !== null ? error.detail : undefined; + return new PreviewAutomationInvalidSelectorError({ + message: error.message, + selector: + detail && "selector" in detail && typeof detail.selector === "string" + ? detail.selector + : "", + }); + } + case "PreviewAutomationResultTooLargeError": { + const detail = + typeof error.detail === "object" && error.detail !== null ? error.detail : undefined; + return new PreviewAutomationResultTooLargeError({ + message: error.message, + maximumBytes: + detail && "maximumBytes" in detail && typeof detail.maximumBytes === "number" + ? detail.maximumBytes + : 64_000, + }); + } + case "PreviewAutomationUnavailableError": + return new PreviewAutomationUnavailableError({ message: error.message }); + default: + return new PreviewAutomationExecutionError({ + message: error.message, + detail: error.detail, + }); + } +}; + +export const makePreviewAutomationBroker = (): PreviewAutomationBrokerShape => { + const state = Effect.runSync( + Ref.make({ + clients: new Map(), + owners: new Map(), + pending: new Map(), + }), + ); + let requestSequence = 0; + + const disconnect = (clientId: string, queue: ClientConnection["queue"]) => + Effect.gen(function* () { + const toFail: PendingRequest[] = []; + yield* Ref.update(state, (current) => { + if (current.clients.get(clientId)?.queue !== queue) return current; + const clients = new Map(current.clients); + const owners = new Map(current.owners); + const pending = new Map(current.pending); + clients.delete(clientId); + owners.delete(clientId); + for (const [requestId, entry] of pending) { + if (entry.clientId === clientId) { + pending.delete(requestId); + toFail.push(entry); + } + } + return { clients, owners, pending }; + }); + yield* Effect.forEach( + toFail, + ({ deferred }) => + Deferred.fail( + deferred, + new PreviewAutomationUnavailableError({ + message: "The preview automation client disconnected.", + }), + ), + { discard: true }, + ); + yield* Queue.shutdown(queue); + }); + + const connect: PreviewAutomationBrokerShape["connect"] = (clientId) => + Effect.gen(function* () { + const queue = yield* Queue.unbounded(); + let previous: ClientConnection | undefined; + yield* Ref.update(state, (current) => { + previous = current.clients.get(clientId); + const clients = new Map(current.clients); + clients.set(clientId, { clientId, queue }); + return { ...current, clients }; + }); + if (previous) yield* disconnect(clientId, previous.queue); + return Stream.fromQueue(queue).pipe(Stream.ensuring(disconnect(clientId, queue))); + }); + + const reportOwner: PreviewAutomationBrokerShape["reportOwner"] = (owner) => + Ref.update(state, (current) => { + const owners = new Map(current.owners); + owners.set(owner.clientId, owner); + return { ...current, owners }; + }); + + const clearOwner: PreviewAutomationBrokerShape["clearOwner"] = (clientId) => + Ref.update(state, (current) => { + const owners = new Map(current.owners); + owners.delete(clientId); + return { ...current, owners }; + }); + + const respond: PreviewAutomationBrokerShape["respond"] = (response) => + Effect.gen(function* () { + let pending: PendingRequest | undefined; + yield* Ref.update(state, (current) => { + pending = current.pending.get(response.requestId); + if (!pending) return current; + const next = new Map(current.pending); + next.delete(response.requestId); + return { ...current, pending: next }; + }); + if (!pending) return; + if (response.ok) { + yield* Deferred.succeed(pending.deferred, response.result); + } else { + yield* Deferred.fail( + pending.deferred, + response.error + ? makeResponseError(response.error) + : new PreviewAutomationExecutionError({ + message: "Preview automation failed without an error payload.", + }), + ); + } + }); + + const invoke = ( + input: Parameters[0], + ): Effect.Effect => + Effect.gen(function* () { + const current = yield* Ref.get(state); + const candidates = Array.from(current.owners.values()) + .filter( + (owner) => + owner.environmentId === input.scope.environmentId && + owner.threadId === input.scope.threadId && + owner.supportsAutomation && + (input.operation === "open" || input.operation === "status" || owner.visible), + ) + .sort((left, right) => right.focusedAt.localeCompare(left.focusedAt)); + const owner = candidates[0]; + if (!owner) { + return yield* new PreviewAutomationNoFocusedOwnerError({ + message: "No focused desktop preview owner is available for this thread.", + }); + } + const connection = current.clients.get(owner.clientId); + if (!connection) { + return yield* new PreviewAutomationUnavailableError({ + message: "The focused preview owner is not connected.", + }); + } + if ( + input.operation !== "open" && + input.operation !== "status" && + !owner.tabId && + !input.tabId + ) { + return yield* new PreviewAutomationTabNotFoundError({ + message: "The focused preview owner does not have an active tab.", + }); + } + const requestId = `preview-${requestSequence++}`; + const timeoutMs = input.timeoutMs ?? 15_000; + const deferred = yield* Deferred.make(); + yield* Ref.update(state, (next) => { + const pending = new Map(next.pending); + pending.set(requestId, { clientId: owner.clientId, deferred }); + return { ...next, pending }; + }); + const offered = yield* Queue.offer(connection.queue, { + requestId, + threadId: input.scope.threadId, + tabId: input.tabId ?? owner.tabId ?? undefined, + operation: input.operation, + input: input.input, + timeoutMs, + }); + if (!offered) { + return yield* new PreviewAutomationUnavailableError({ + message: "The preview automation client is no longer accepting requests.", + }); + } + const result = yield* Deferred.await(deferred).pipe(Effect.timeoutOption(timeoutMs)); + yield* Ref.update(state, (next) => { + const pending = new Map(next.pending); + pending.delete(requestId); + return { ...next, pending }; + }); + return yield* Option.match(result, { + onNone: () => + Effect.fail( + new PreviewAutomationTimeoutError({ + message: `Preview automation timed out after ${timeoutMs}ms.`, + }), + ), + onSome: (value) => Effect.succeed(value as A), + }); + }); + + return PreviewAutomationBroker.of({ connect, reportOwner, clearOwner, respond, invoke }); +}; + +export const previewAutomationBroker = makePreviewAutomationBroker(); + +export const PreviewAutomationBrokerLive: Layer.Layer = Layer.succeed( + PreviewAutomationBroker, + previewAutomationBroker, +); diff --git a/apps/server/src/mcp/Services/McpInvocationContext.ts b/apps/server/src/mcp/Services/McpInvocationContext.ts new file mode 100644 index 0000000000000000000000000000000000000000..89a3820479b654583f775b3ad04503c10c4aee72 --- /dev/null +++ b/apps/server/src/mcp/Services/McpInvocationContext.ts @@ -0,0 +1,33 @@ +import type { EnvironmentId, ProviderInstanceId, ThreadId } from "@t3tools/contracts"; +import { PreviewAutomationUnavailableError } from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; + +export type McpCapability = "preview"; + +export interface McpInvocationScope { + readonly environmentId: EnvironmentId; + readonly threadId: ThreadId; + readonly providerSessionId: string; + readonly providerInstanceId: ProviderInstanceId; + readonly capabilities: ReadonlySet; + readonly issuedAt: number; + readonly expiresAt: number; +} + +export class McpInvocationContext extends Context.Service< + McpInvocationContext, + McpInvocationScope +>()("t3/mcp/Services/McpInvocationContext") {} + +export const requireMcpCapability = Effect.fn("mcp.requireCapability")(function* ( + capability: McpCapability, +) { + const invocation = yield* McpInvocationContext; + if (!invocation.capabilities.has(capability)) { + return yield* new PreviewAutomationUnavailableError({ + message: `MCP credential does not grant the ${capability} capability.`, + }); + } + return invocation; +}); diff --git a/apps/server/src/mcp/Services/McpProviderSession.ts b/apps/server/src/mcp/Services/McpProviderSession.ts new file mode 100644 index 0000000000000000000000000000000000000000..b97cbb1e1c043ff115e2a64ac1b3826c915c2ca6 --- /dev/null +++ b/apps/server/src/mcp/Services/McpProviderSession.ts @@ -0,0 +1,34 @@ +import type { EnvironmentId, ProviderInstanceId, ThreadId } from "@t3tools/contracts"; +import * as Context from "effect/Context"; + +export interface McpProviderSessionConfig { + readonly environmentId: EnvironmentId; + readonly threadId: ThreadId; + readonly providerSessionId: string; + readonly providerInstanceId: ProviderInstanceId; + readonly endpoint: string; + readonly authorizationHeader: string; +} + +export class McpProviderSession extends Context.Service< + McpProviderSession, + McpProviderSessionConfig +>()("t3/mcp/Services/McpProviderSession") {} + +const sessionsByThread = new Map(); + +export function setMcpProviderSession(config: McpProviderSessionConfig): void { + sessionsByThread.set(config.threadId, config); +} + +export function readMcpProviderSession(threadId: ThreadId): McpProviderSessionConfig | undefined { + return sessionsByThread.get(threadId); +} + +export function clearMcpProviderSession(threadId: ThreadId): void { + sessionsByThread.delete(threadId); +} + +export function clearAllMcpProviderSessions(): void { + sessionsByThread.clear(); +} diff --git a/apps/server/src/mcp/Services/McpSessionRegistry.ts b/apps/server/src/mcp/Services/McpSessionRegistry.ts new file mode 100644 index 0000000000000000000000000000000000000000..df2ca991271e6dbebed1c2bb80f2f503ce9ef4b8 --- /dev/null +++ b/apps/server/src/mcp/Services/McpSessionRegistry.ts @@ -0,0 +1,29 @@ +import type { ProviderInstanceId, ThreadId } from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import type * as Effect from "effect/Effect"; + +import type { McpInvocationScope } from "./McpInvocationContext.ts"; +import type { McpProviderSessionConfig } from "./McpProviderSession.ts"; + +export interface McpCredentialRequest { + readonly threadId: ThreadId; + readonly providerInstanceId: ProviderInstanceId; +} + +export interface McpIssuedCredential { + readonly config: McpProviderSessionConfig; + readonly expiresAt: number; +} + +export interface McpSessionRegistryShape { + readonly issue: (request: McpCredentialRequest) => Effect.Effect; + readonly resolve: (rawToken: string) => Effect.Effect; + readonly revokeProviderSession: (providerSessionId: string) => Effect.Effect; + readonly revokeThread: (threadId: ThreadId) => Effect.Effect; + readonly revokeAll: Effect.Effect; +} + +export class McpSessionRegistry extends Context.Service< + McpSessionRegistry, + McpSessionRegistryShape +>()("t3/mcp/Services/McpSessionRegistry") {} diff --git a/apps/server/src/mcp/Services/PreviewAutomationBroker.ts b/apps/server/src/mcp/Services/PreviewAutomationBroker.ts new file mode 100644 index 0000000000000000000000000000000000000000..1af1a3d24bfc72adc162f2da504142b611fefba1 --- /dev/null +++ b/apps/server/src/mcp/Services/PreviewAutomationBroker.ts @@ -0,0 +1,40 @@ +import type { + PreviewAutomationError, + PreviewAutomationOperation, + PreviewAutomationOwner, + PreviewAutomationRequest, + PreviewAutomationResponse, + PreviewTabId, +} from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import type * as Effect from "effect/Effect"; +import type * as Stream from "effect/Stream"; + +import type { McpInvocationScope } from "./McpInvocationContext.ts"; + +export interface PreviewAutomationInvokeInput { + readonly scope: McpInvocationScope; + readonly operation: PreviewAutomationOperation; + readonly input: unknown; + readonly tabId?: PreviewTabId; + readonly timeoutMs?: number; +} + +export interface PreviewAutomationBrokerShape { + readonly connect: (clientId: string) => Effect.Effect>; + readonly reportOwner: ( + owner: PreviewAutomationOwner, + ) => Effect.Effect; + readonly clearOwner: (clientId: string) => Effect.Effect; + readonly respond: ( + response: PreviewAutomationResponse, + ) => Effect.Effect; + readonly invoke: ( + request: PreviewAutomationInvokeInput, + ) => Effect.Effect; +} + +export class PreviewAutomationBroker extends Context.Service< + PreviewAutomationBroker, + PreviewAutomationBrokerShape +>()("t3/mcp/Services/PreviewAutomationBroker") {} diff --git a/apps/server/src/mcp/toolkits/preview/handlers.ts b/apps/server/src/mcp/toolkits/preview/handlers.ts new file mode 100644 index 0000000000000000000000000000000000000000..f5e19bfcf3e372dccd0532c332005944bca384eb --- /dev/null +++ b/apps/server/src/mcp/toolkits/preview/handlers.ts @@ -0,0 +1,47 @@ +import * as Effect from "effect/Effect"; +import type { + PreviewAutomationOperation, + PreviewAutomationSnapshot, + PreviewAutomationStatus, +} from "@t3tools/contracts"; + +import { requireMcpCapability } from "../../Services/McpInvocationContext.ts"; +import { previewAutomationBroker } from "../../Layers/PreviewAutomationBroker.ts"; +import { PreviewToolkit } from "./tools.ts"; + +const invoke = ( + operation: PreviewAutomationOperation, + input: unknown, + timeoutMs?: number, +): Effect.Effect< + A, + import("@t3tools/contracts").PreviewAutomationError, + import("../../Services/McpInvocationContext.ts").McpInvocationContext +> => + Effect.gen(function* () { + const scope = yield* requireMcpCapability("preview"); + return yield* previewAutomationBroker.invoke({ + scope, + operation, + input, + ...(timeoutMs === undefined ? {} : { timeoutMs }), + }); + }); + +export const PreviewToolkitHandlersLive = PreviewToolkit.toLayer({ + preview_status: () => invoke("status", {}), + preview_open: (input) => + invoke("open", { + ...input, + show: input.show ?? true, + reuseExistingTab: input.reuseExistingTab ?? true, + }), + preview_navigate: (input) => invoke("navigate", input, input.timeoutMs), + preview_snapshot: () => invoke("snapshot", {}), + preview_click: (input) => invoke("click", input, input.timeoutMs), + preview_type: (input) => invoke("type", input, input.timeoutMs), + preview_press: (input) => invoke("press", input), + preview_scroll: (input) => invoke("scroll", input), + preview_evaluate: (input) => invoke("evaluate", input), + preview_wait_for: (input) => invoke("waitFor", input, input.timeoutMs), +}); diff --git a/apps/server/src/mcp/toolkits/preview/tools.ts b/apps/server/src/mcp/toolkits/preview/tools.ts new file mode 100644 index 0000000000000000000000000000000000000000..32baf050f2f4c0a9309a2b67a3b58cea62e55843 --- /dev/null +++ b/apps/server/src/mcp/toolkits/preview/tools.ts @@ -0,0 +1,138 @@ +import { + PreviewAutomationClickInput, + PreviewAutomationError, + PreviewAutomationEvaluateInput, + PreviewAutomationNavigateInput, + PreviewAutomationOpenInput, + PreviewAutomationPressInput, + PreviewAutomationScrollInput, + PreviewAutomationSnapshot, + PreviewAutomationStatus, + PreviewAutomationTypeInput, + PreviewAutomationWaitForInput, +} from "@t3tools/contracts"; +import * as Schema from "effect/Schema"; +import { Tool, Toolkit } from "effect/unstable/ai"; + +import { McpInvocationContext } from "../../Services/McpInvocationContext.ts"; + +const browserTool = (tool: T): T => + tool.annotate(Tool.Destructive, false).annotate(Tool.OpenWorld, true) as T; + +export const PreviewStatusTool = Tool.make("preview_status", { + description: + "Report whether the scoped thread has an automation-capable desktop preview, including its active tab, URL, title, visibility, and loading state.", + success: PreviewAutomationStatus, + failure: PreviewAutomationError, + dependencies: [McpInvocationContext], +}) + .annotate(Tool.Title, "Get preview status") + .annotate(Tool.Readonly, true) + .annotate(Tool.Destructive, false) + .annotate(Tool.Idempotent, true); + +export const PreviewOpenTool = browserTool( + Tool.make("preview_open", { + description: + "Show and initialize the browser preview for the scoped thread, optionally reusing its current tab and navigating to a URL.", + parameters: PreviewAutomationOpenInput, + success: PreviewAutomationStatus, + failure: PreviewAutomationError, + dependencies: [McpInvocationContext], + }).annotate(Tool.Title, "Open browser preview"), +); + +export const PreviewNavigateTool = browserTool( + Tool.make("preview_navigate", { + description: + "Navigate the scoped thread's active preview tab to a URL and wait for the requested readiness condition.", + parameters: PreviewAutomationNavigateInput, + success: PreviewAutomationStatus, + failure: PreviewAutomationError, + dependencies: [McpInvocationContext], + }).annotate(Tool.Title, "Navigate browser preview"), +); + +export const PreviewSnapshotTool = Tool.make("preview_snapshot", { + description: + "Capture bounded page metadata, visible text, interactive elements, accessibility data, and a PNG screenshot from the scoped preview tab.", + success: PreviewAutomationSnapshot, + failure: PreviewAutomationError, + dependencies: [McpInvocationContext], +}) + .annotate(Tool.Title, "Capture preview snapshot") + .annotate(Tool.Readonly, true) + .annotate(Tool.Destructive, false); + +export const PreviewClickTool = browserTool( + Tool.make("preview_click", { + description: + "Click an element selected by CSS selector or click viewport coordinates in the scoped preview tab.", + parameters: PreviewAutomationClickInput, + failure: PreviewAutomationError, + dependencies: [McpInvocationContext], + }).annotate(Tool.Title, "Click preview page"), +); + +export const PreviewTypeTool = browserTool( + Tool.make("preview_type", { + description: + "Type text into the focused element or a CSS-selected element, optionally clearing its existing value first.", + parameters: PreviewAutomationTypeInput, + failure: PreviewAutomationError, + dependencies: [McpInvocationContext], + }).annotate(Tool.Title, "Type into preview page"), +); + +export const PreviewPressTool = browserTool( + Tool.make("preview_press", { + description: "Dispatch a keyboard key with optional modifiers to the scoped preview tab.", + parameters: PreviewAutomationPressInput, + failure: PreviewAutomationError, + dependencies: [McpInvocationContext], + }).annotate(Tool.Title, "Press key in preview page"), +); + +export const PreviewScrollTool = browserTool( + Tool.make("preview_scroll", { + description: + "Scroll the preview viewport or a CSS-selected scroll container by the requested deltas.", + parameters: PreviewAutomationScrollInput, + failure: PreviewAutomationError, + dependencies: [McpInvocationContext], + }).annotate(Tool.Title, "Scroll preview page"), +); + +export const PreviewEvaluateTool = browserTool( + Tool.make("preview_evaluate", { + description: + "Evaluate bounded JavaScript in the scoped preview tab and return a serializable result of at most 64 KB.", + parameters: PreviewAutomationEvaluateInput, + success: Schema.Unknown, + failure: PreviewAutomationError, + dependencies: [McpInvocationContext], + }).annotate(Tool.Title, "Evaluate JavaScript in preview"), +); + +export const PreviewWaitForTool = browserTool( + Tool.make("preview_wait_for", { + description: + "Wait until a CSS selector, visible-text substring, or URL substring appears in the scoped preview tab.", + parameters: PreviewAutomationWaitForInput, + failure: PreviewAutomationError, + dependencies: [McpInvocationContext], + }).annotate(Tool.Title, "Wait for preview page condition"), +); + +export const PreviewToolkit = Toolkit.make( + PreviewStatusTool, + PreviewOpenTool, + PreviewNavigateTool, + PreviewSnapshotTool, + PreviewClickTool, + PreviewTypeTool, + PreviewPressTool, + PreviewScrollTool, + PreviewEvaluateTool, + PreviewWaitForTool, +); diff --git a/apps/server/src/provider/Layers/ClaudeAdapter.ts b/apps/server/src/provider/Layers/ClaudeAdapter.ts index 38b77c692621ee0b5a87fe3b106688b5ab77108d..b7cdd931ca632843cab3b36c2e3418db6a543407 100644 --- a/apps/server/src/provider/Layers/ClaudeAdapter.ts +++ b/apps/server/src/provider/Layers/ClaudeAdapter.ts @@ -69,6 +69,7 @@ import * as Stream from "effect/Stream"; import { resolveAttachmentPath } from "../../attachmentStore.ts"; import { ServerConfig } from "../../config.ts"; +import { readMcpProviderSession } from "../../mcp/Services/McpProviderSession.ts"; import { makeClaudeEnvironment } from "../Drivers/ClaudeHome.ts"; import { getClaudeModelCapabilities, @@ -3445,6 +3446,7 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( ...(fastMode ? { fastMode: true } : {}), ...(ultracode ? { ultracode: true } : {}), }; + const mcpSession = readMcpProviderSession(input.threadId); const queryOptions: ClaudeQueryOptions = { ...(input.cwd ? { cwd: input.cwd } : {}), ...(apiModelId ? { model: apiModelId } : {}), @@ -3470,6 +3472,19 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( env: claudeEnvironment, ...(input.cwd ? { additionalDirectories: [input.cwd] } : {}), ...(Object.keys(extraArgs).length > 0 ? { extraArgs } : {}), + ...(mcpSession + ? { + mcpServers: { + "t3-code": { + type: "http", + url: mcpSession.endpoint, + headers: { + Authorization: mcpSession.authorizationHeader, + }, + }, + }, + } + : {}), }; yield* Effect.annotateCurrentSpan({ diff --git a/apps/server/src/provider/Layers/CodexAdapter.ts b/apps/server/src/provider/Layers/CodexAdapter.ts index 8c9969e2bc4451f4de9968b272937d858835a31b..ea2d898730afebcaf5913a8301ebf651b6989338 100644 --- a/apps/server/src/provider/Layers/CodexAdapter.ts +++ b/apps/server/src/provider/Layers/CodexAdapter.ts @@ -39,6 +39,7 @@ import * as EffectCodexSchema from "effect-codex-app-server/schema"; import { getModelSelectionStringOptionValue } from "@t3tools/shared/model"; import { getCodexServiceTierOptionValue } from "../../codexModelOptions.ts"; +import { readMcpProviderSession } from "../../mcp/Services/McpProviderSession.ts"; import { ProviderAdapterRequestError, @@ -1382,6 +1383,7 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* ( input.modelSelection?.instanceId === boundInstanceId ? getCodexServiceTierOptionValue(input.modelSelection) : undefined; + const mcpSession = readMcpProviderSession(input.threadId); const runtimeInput: CodexSessionRuntimeOptions = { threadId: input.threadId, providerInstanceId: boundInstanceId, @@ -1397,6 +1399,20 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* ( ? { model: input.modelSelection.model } : {}), ...(serviceTier ? { serviceTier } : {}), + ...(mcpSession + ? { + environment: { + ...(options?.environment ?? process.env), + T3_MCP_BEARER_TOKEN: mcpSession.authorizationHeader.replace(/^Bearer\s+/, ""), + }, + appServerArgs: [ + "-c", + `mcp_servers.t3-code.url=${mcpSession.endpoint}`, + "-c", + 'mcp_servers.t3-code.bearer_token_env_var="T3_MCP_BEARER_TOKEN"', + ], + } + : {}), }; const sessionScope = yield* Scope.make("sequential"); let sessionScopeTransferred = false; diff --git a/apps/server/src/provider/Layers/CodexSessionRuntime.ts b/apps/server/src/provider/Layers/CodexSessionRuntime.ts index f9b9c6ab4fba29b5a662d9e32b21cf1791579266..fc31ca3aef72559d1861adde39cd21790972aef8 100644 --- a/apps/server/src/provider/Layers/CodexSessionRuntime.ts +++ b/apps/server/src/provider/Layers/CodexSessionRuntime.ts @@ -103,6 +103,7 @@ export interface CodexSessionRuntimeOptions { readonly model?: string; readonly serviceTier?: CodexServiceTier | undefined; readonly resumeCursor?: CodexResumeCursor; + readonly appServerArgs?: ReadonlyArray; } export interface CodexSessionRuntimeSendTurnInput { @@ -720,7 +721,7 @@ export const makeCodexSessionRuntime = ( }; const child = yield* spawner .spawn( - ChildProcess.make(options.binaryPath, ["app-server"], { + ChildProcess.make(options.binaryPath, ["app-server", ...(options.appServerArgs ?? [])], { cwd: options.cwd, env, forceKillAfter: CODEX_APP_SERVER_FORCE_KILL_AFTER, diff --git a/apps/server/src/provider/Layers/CursorAdapter.ts b/apps/server/src/provider/Layers/CursorAdapter.ts index cdb3c224b97eae1228136eab3e67494fa0ff834c..c9af2de35578adecaf9c22fd07dd238802d41b80 100644 --- a/apps/server/src/provider/Layers/CursorAdapter.ts +++ b/apps/server/src/provider/Layers/CursorAdapter.ts @@ -42,6 +42,7 @@ import type * as EffectAcpSchema from "effect-acp/schema"; import { resolveAttachmentPath } from "../../attachmentStore.ts"; import { ServerConfig } from "../../config.ts"; +import { readMcpProviderSession } from "../../mcp/Services/McpProviderSession.ts"; import { ProviderAdapterProcessError, ProviderAdapterRequestError, @@ -530,6 +531,7 @@ export function makeCursorAdapter( ? yield* options.resolveSettings : cursorSettings; + const mcpSession = readMcpProviderSession(input.threadId); const acp = yield* makeCursorAcpRuntime({ cursorSettings: effectiveCursorSettings, ...(options?.environment ? { environment: options.environment } : {}), @@ -537,6 +539,23 @@ export function makeCursorAdapter( cwd, ...(resumeSessionId ? { resumeSessionId } : {}), clientInfo: { name: "t3-code", version: "0.0.0" }, + ...(mcpSession + ? { + mcpServers: [ + { + type: "http" as const, + name: "t3-code", + url: mcpSession.endpoint, + headers: [ + { + name: "Authorization", + value: mcpSession.authorizationHeader, + }, + ], + }, + ], + } + : {}), ...acpNativeLoggers, }).pipe( Effect.provideService(Scope.Scope, sessionScope), diff --git a/apps/server/src/provider/Layers/GrokAdapter.ts b/apps/server/src/provider/Layers/GrokAdapter.ts index 0f1007f261b9af75ef55fdb7d0f3db6c0eda337c..35cf7fb98e9b06f586acddcf8fc4859d8db0b390 100644 --- a/apps/server/src/provider/Layers/GrokAdapter.ts +++ b/apps/server/src/provider/Layers/GrokAdapter.ts @@ -33,6 +33,7 @@ import type * as EffectAcpSchema from "effect-acp/schema"; import { resolveAttachmentPath } from "../../attachmentStore.ts"; import { ServerConfig } from "../../config.ts"; +import { readMcpProviderSession } from "../../mcp/Services/McpProviderSession.ts"; import { ProviderAdapterProcessError, ProviderAdapterRequestError, @@ -374,6 +375,7 @@ export function makeGrokAdapter(grokSettings: GrokSettings, options?: GrokAdapte threadId: input.threadId, }); + const mcpSession = readMcpProviderSession(input.threadId); const acp = yield* makeGrokAcpRuntime({ grokSettings, ...(options?.environment ? { environment: options.environment } : {}), @@ -381,6 +383,23 @@ export function makeGrokAdapter(grokSettings: GrokSettings, options?: GrokAdapte cwd, ...(resumeSessionId ? { resumeSessionId } : {}), clientInfo: { name: "t3-code", version: "0.0.0" }, + ...(mcpSession + ? { + mcpServers: [ + { + type: "http" as const, + name: "t3-code", + url: mcpSession.endpoint, + headers: [ + { + name: "Authorization", + value: mcpSession.authorizationHeader, + }, + ], + }, + ], + } + : {}), ...acpNativeLoggers, }).pipe( Effect.provideService(Scope.Scope, sessionScope), diff --git a/apps/server/src/provider/Layers/OpenCodeAdapter.ts b/apps/server/src/provider/Layers/OpenCodeAdapter.ts index 54444ce586dbfe5e98219344a7377f1af7dca436..7361638924d8d4f2eb871d660cfc11e783d6fe48 100644 --- a/apps/server/src/provider/Layers/OpenCodeAdapter.ts +++ b/apps/server/src/provider/Layers/OpenCodeAdapter.ts @@ -26,6 +26,7 @@ import { getModelSelectionStringOptionValue } from "@t3tools/shared/model"; import { resolveAttachmentPath } from "../../attachmentStore.ts"; import { ServerConfig } from "../../config.ts"; +import { readMcpProviderSession } from "../../mcp/Services/McpProviderSession.ts"; import { type EventNdjsonLogger, makeEventNdjsonLogger } from "./EventNdjsonLogger.ts"; import { ProviderAdapterProcessError, @@ -1053,6 +1054,22 @@ export function makeOpenCodeAdapter( directory, ...(server.external && serverPassword ? { serverPassword } : {}), }); + const mcpSession = readMcpProviderSession(input.threadId); + if (mcpSession && !server.external) { + yield* runOpenCodeSdk("mcp.add", () => + client.mcp.add({ + name: "t3-code", + config: { + type: "remote", + url: mcpSession.endpoint, + headers: { + Authorization: mcpSession.authorizationHeader, + }, + oauth: false, + }, + }), + ); + } const openCodeSession = yield* runOpenCodeSdk("session.create", () => client.session.create({ title: `T3 Code ${input.threadId}`, diff --git a/apps/server/src/provider/Layers/ProviderService.ts b/apps/server/src/provider/Layers/ProviderService.ts index 2bce1f483b743f1151b66ff25c0fc5e617eaba97..7d653419b27366fdc297806c3ae822ff9dc9e176 100644 --- a/apps/server/src/provider/Layers/ProviderService.ts +++ b/apps/server/src/provider/Layers/ProviderService.ts @@ -56,6 +56,16 @@ import { import { type EventNdjsonLogger } from "./EventNdjsonLogger.ts"; import { ProviderEventLoggers } from "./ProviderEventLoggers.ts"; import { AnalyticsService } from "../../telemetry/Services/AnalyticsService.ts"; +import { + issueActiveMcpCredential, + revokeActiveMcpThread, + revokeAllActiveMcpCredentials, +} from "../../mcp/Layers/McpSessionRegistry.ts"; +import { + clearAllMcpProviderSessions, + clearMcpProviderSession, + setMcpProviderSession, +} from "../../mcp/Services/McpProviderSession.ts"; const isModelSelection = Schema.is(ModelSelection); /** @@ -212,6 +222,16 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( const directory = yield* ProviderSessionDirectory; const runtimeEventPubSub = yield* PubSub.unbounded(); const nowIso = Effect.map(DateTime.now, DateTime.formatIso); + const prepareMcpSession = (threadId: ThreadId, providerInstanceId: ProviderInstanceId) => + issueActiveMcpCredential({ threadId, providerInstanceId }).pipe( + Effect.tap((credential) => + credential ? Effect.sync(() => setMcpProviderSession(credential.config)) : Effect.void, + ), + ); + const clearMcpSession = (threadId: ThreadId) => + revokeActiveMcpThread(threadId).pipe( + Effect.tap(() => Effect.sync(() => clearMcpProviderSession(threadId))), + ); const publishRuntimeEvent = (event: ProviderRuntimeEvent): Effect.Effect => Effect.succeed(event).pipe( @@ -383,16 +403,20 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( const persistedCwd = readPersistedCwd(input.binding.runtimePayload); const persistedModelSelection = readPersistedModelSelection(input.binding.runtimePayload); - const resumed = yield* adapter.startSession({ - threadId: input.binding.threadId, - provider: input.binding.provider, - providerInstanceId: bindingInstanceId, - ...(persistedCwd ? { cwd: persistedCwd } : {}), - ...(persistedModelSelection ? { modelSelection: persistedModelSelection } : {}), - ...(hasResumeCursor ? { resumeCursor: input.binding.resumeCursor } : {}), - runtimeMode: input.binding.runtimeMode ?? "full-access", - }); + yield* prepareMcpSession(input.binding.threadId, bindingInstanceId); + const resumed = yield* adapter + .startSession({ + threadId: input.binding.threadId, + provider: input.binding.provider, + providerInstanceId: bindingInstanceId, + ...(persistedCwd ? { cwd: persistedCwd } : {}), + ...(persistedModelSelection ? { modelSelection: persistedModelSelection } : {}), + ...(hasResumeCursor ? { resumeCursor: input.binding.resumeCursor } : {}), + runtimeMode: input.binding.runtimeMode ?? "full-access", + }) + .pipe(Effect.onError(() => clearMcpSession(input.binding.threadId))); if (resumed.provider !== adapter.provider) { + yield* clearMcpSession(input.binding.threadId); return yield* toValidationError( input.operation, `Adapter/provider mismatch while recovering thread '${input.binding.threadId}'. Expected '${adapter.provider}', received '${resumed.provider}'.`, @@ -572,14 +596,18 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( "provider.cwd.effective": effectiveCwd ?? "", }); const adapter = yield* registry.getByInstance(resolvedInstanceId); - const session = yield* adapter.startSession({ - ...input, - providerInstanceId: resolvedInstanceId, - ...(effectiveCwd !== undefined ? { cwd: effectiveCwd } : {}), - ...(effectiveResumeCursor !== undefined ? { resumeCursor: effectiveResumeCursor } : {}), - }); + yield* prepareMcpSession(threadId, resolvedInstanceId); + const session = yield* adapter + .startSession({ + ...input, + providerInstanceId: resolvedInstanceId, + ...(effectiveCwd !== undefined ? { cwd: effectiveCwd } : {}), + ...(effectiveResumeCursor !== undefined ? { resumeCursor: effectiveResumeCursor } : {}), + }) + .pipe(Effect.onError(() => clearMcpSession(threadId))); if (session.provider !== adapter.provider) { + yield* clearMcpSession(threadId); return yield* toValidationError( "ProviderService.startSession", `Adapter/provider mismatch: requested '${adapter.provider}', received '${session.provider}'.`, @@ -827,6 +855,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( if (routed.isActive) { yield* routed.adapter.stopSession(routed.threadId); } + yield* clearMcpSession(input.threadId); yield* directory.upsert({ threadId: input.threadId, provider: routed.adapter.provider, @@ -998,6 +1027,8 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( ), ).pipe(Effect.asVoid); yield* Effect.forEach(currentAdapters, ([, adapter]) => adapter.stopAll()).pipe(Effect.asVoid); + yield* revokeAllActiveMcpCredentials(); + clearAllMcpProviderSessions(); const bindings = yield* directory.listBindings().pipe(Effect.orElseSucceed(() => [])); yield* Effect.forEach(bindings, (binding) => Effect.gen(function* () { diff --git a/apps/server/src/provider/acp/AcpSessionRuntime.ts b/apps/server/src/provider/acp/AcpSessionRuntime.ts index 4ed64890fc3e998c00f91ad74be4defc7257cd1f..47a8c845e56265acc8503e231cd27e6d387444cd 100644 --- a/apps/server/src/provider/acp/AcpSessionRuntime.ts +++ b/apps/server/src/provider/acp/AcpSessionRuntime.ts @@ -47,6 +47,7 @@ export interface AcpSessionRuntimeOptions { readonly version: string; }; readonly authMethodId: string; + readonly mcpServers?: ReadonlyArray; readonly requestLogger?: (event: AcpSessionRequestLogEvent) => Effect.Effect; readonly protocolLogging?: { readonly logIncoming?: boolean; @@ -400,7 +401,7 @@ const makeAcpSessionRuntime = ( const loadPayload = { sessionId: options.resumeSessionId, cwd: options.cwd, - mcpServers: [], + mcpServers: options.mcpServers ?? [], } satisfies EffectAcpSchema.LoadSessionRequest; const resumed = yield* runLoggedRequest( "session/load", @@ -413,7 +414,7 @@ const makeAcpSessionRuntime = ( } else { const createPayload = { cwd: options.cwd, - mcpServers: [], + mcpServers: options.mcpServers ?? [], } satisfies EffectAcpSchema.NewSessionRequest; const created = yield* runLoggedRequest( "session/new", @@ -426,7 +427,7 @@ const makeAcpSessionRuntime = ( } else { const createPayload = { cwd: options.cwd, - mcpServers: [], + mcpServers: options.mcpServers ?? [], } satisfies EffectAcpSchema.NewSessionRequest; const created = yield* runLoggedRequest( "session/new", diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 378e4c6060b062cde16f29638d519b303523296c..e14a08b7952042a0149bbb71a950879aa41b5b3f 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -37,6 +37,8 @@ import { ProviderInstanceRegistryHydrationLive } from "./provider/Layers/Provide import { TerminalManagerLive } from "./terminal/Layers/Manager.ts"; import { PreviewManagerLive } from "./preview/Layers/Manager.ts"; import { PreviewPortScannerLive } from "./preview/Layers/PortScanner.ts"; +import { McpHttpServerLive } from "./mcp/Layers/McpHttpServer.ts"; +import { McpSessionRegistryLive } from "./mcp/Layers/McpSessionRegistry.ts"; import * as ProcessRunner from "./processRunner.ts"; import * as GitManager from "./git/GitManager.ts"; import { KeybindingsLive } from "./keybindings.ts"; @@ -335,18 +337,21 @@ const RuntimeServicesLive = ServerRuntimeStartupLive.pipe( ); export const makeRoutesLayer = Layer.mergeAll( - HttpApiBuilder.layer(EnvironmentHttpApi).pipe( - Layer.provide(authHttpApiLayer), - Layer.provide(connectHttpApiLayer), - Layer.provide(orchestrationHttpApiLayer), - Layer.provide(serverEnvironmentHttpApiLayer), - Layer.provide(environmentAuthenticatedAuthLayer), + Layer.mergeAll( + HttpApiBuilder.layer(EnvironmentHttpApi).pipe( + Layer.provide(authHttpApiLayer), + Layer.provide(connectHttpApiLayer), + Layer.provide(orchestrationHttpApiLayer), + Layer.provide(serverEnvironmentHttpApiLayer), + Layer.provide(environmentAuthenticatedAuthLayer), + ), + attachmentsRouteLayer, + otlpTracesProxyRouteLayer, + projectFaviconRouteLayer, + staticAndDevRouteLayer, + websocketRpcRouteLayer, ), - attachmentsRouteLayer, - otlpTracesProxyRouteLayer, - projectFaviconRouteLayer, - staticAndDevRouteLayer, - websocketRpcRouteLayer, + McpHttpServerLive.pipe(Layer.provide(McpSessionRegistryLive)), ).pipe(Layer.provide(browserApiCorsLayer)); export const makeServerLayer = Layer.unwrap( diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index afd7f569d596cf622be933af56763f4208a9e26a..900e80dfa15a99c77719e5bd875c0f9dff2a8b67 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -73,6 +73,7 @@ import { redactServerSettingsForClient, ServerSettingsService } from "./serverSe import { TerminalManager } from "./terminal/Services/Manager.ts"; import { PreviewManager } from "./preview/Services/Manager.ts"; import { PreviewPortScanner } from "./preview/Services/PortScanner.ts"; +import { previewAutomationBroker } from "./mcp/Layers/PreviewAutomationBroker.ts"; import { WorkspaceEntries } from "./workspace/Services/WorkspaceEntries.ts"; import { WorkspaceFileSystem } from "./workspace/Services/WorkspaceFileSystem.ts"; import { WorkspacePathOutsideRootError } from "./workspace/Services/WorkspacePaths.ts"; @@ -189,6 +190,10 @@ const RPC_REQUIRED_SCOPE = new Map([ [WS_METHODS.previewClose, AuthOrchestrationOperateScope], [WS_METHODS.previewList, AuthOrchestrationReadScope], [WS_METHODS.previewReportStatus, AuthOrchestrationOperateScope], + [WS_METHODS.previewAutomationConnect, AuthOrchestrationOperateScope], + [WS_METHODS.previewAutomationRespond, AuthOrchestrationOperateScope], + [WS_METHODS.previewAutomationReportOwner, AuthOrchestrationOperateScope], + [WS_METHODS.previewAutomationClearOwner, AuthOrchestrationOperateScope], [WS_METHODS.subscribePreviewEvents, AuthOrchestrationReadScope], [WS_METHODS.subscribeDiscoveredLocalServers, AuthOrchestrationReadScope], [WS_METHODS.subscribeServerConfig, AuthOrchestrationReadScope], @@ -1387,6 +1392,30 @@ const makeWsRpcLayer = (currentSession: AuthenticatedSession) => observeRpcEffect(WS_METHODS.previewReportStatus, previewManager.reportStatus(input), { "rpc.aggregate": "preview", }), + [WS_METHODS.previewAutomationConnect]: (input) => + observeRpcStreamEffect( + WS_METHODS.previewAutomationConnect, + previewAutomationBroker.connect(input.clientId), + { "rpc.aggregate": "preview-automation" }, + ), + [WS_METHODS.previewAutomationRespond]: (input) => + observeRpcEffect( + WS_METHODS.previewAutomationRespond, + previewAutomationBroker.respond(input), + { "rpc.aggregate": "preview-automation" }, + ), + [WS_METHODS.previewAutomationReportOwner]: (input) => + observeRpcEffect( + WS_METHODS.previewAutomationReportOwner, + previewAutomationBroker.reportOwner(input), + { "rpc.aggregate": "preview-automation" }, + ), + [WS_METHODS.previewAutomationClearOwner]: (input) => + observeRpcEffect( + WS_METHODS.previewAutomationClearOwner, + previewAutomationBroker.clearOwner(input.clientId), + { "rpc.aggregate": "preview-automation" }, + ), [WS_METHODS.subscribePreviewEvents]: (_input) => observeRpcStream(WS_METHODS.subscribePreviewEvents, previewManager.events, { "rpc.aggregate": "preview", diff --git a/apps/web/src/components/ChatView.browser.tsx b/apps/web/src/components/ChatView.browser.tsx index f0e9ebb7e0ca43c1c00243627363231ff1bc4e42..e1f43bda46f10255c3eca735b086e9842f2b4d76 100644 --- a/apps/web/src/components/ChatView.browser.tsx +++ b/apps/web/src/components/ChatView.browser.tsx @@ -285,6 +285,12 @@ function createMockEnvironmentApi(input: { reportStatus: () => { throw new Error("Not implemented in browser test."); }, + automation: { + connect: () => () => undefined, + respond: () => Promise.resolve(), + reportOwner: () => Promise.resolve(), + clearOwner: () => Promise.resolve(), + }, onEvent: () => () => undefined, subscribePorts: () => () => undefined, } as EnvironmentApi["preview"], diff --git a/apps/web/src/components/ChatView.logic.test.ts b/apps/web/src/components/ChatView.logic.test.ts index e08563b234b6473791c8e59ab2721fc6f55daf7c..bbb59fd6bb817e144a8f3386661fd0fd3ce52d71 100644 --- a/apps/web/src/components/ChatView.logic.test.ts +++ b/apps/web/src/components/ChatView.logic.test.ts @@ -12,6 +12,7 @@ import { type EnvironmentState, useStore } from "../store"; import { type Thread } from "../types"; import { + MAX_HIDDEN_MOUNTED_PREVIEW_THREADS, MAX_HIDDEN_MOUNTED_TERMINAL_THREADS, buildExpiredTerminalContextToastCopy, createLocalDispatchSnapshot, @@ -19,6 +20,7 @@ import { getStartedThreadModelChangeBlockReason, hasServerAcknowledgedLocalDispatch, reconcileMountedTerminalThreadIds, + reconcileRetainedMountedThreadIds, resolveSendEnvMode, shouldWriteThreadErrorToCurrentServerThread, waitForStartedServerThread, @@ -274,6 +276,50 @@ describe("reconcileMountedTerminalThreadIds", () => { }); }); +describe("reconcileRetainedMountedThreadIds", () => { + it("retains hidden open threads and adds the active open thread", () => { + expect( + reconcileRetainedMountedThreadIds({ + currentThreadIds: [ThreadId.make("thread-hidden")], + openThreadIds: [ThreadId.make("thread-hidden")], + activeThreadId: ThreadId.make("thread-active"), + activeThreadOpen: true, + maxHiddenThreadCount: MAX_HIDDEN_MOUNTED_PREVIEW_THREADS, + }), + ).toEqual([ThreadId.make("thread-hidden"), ThreadId.make("thread-active")]); + }); + + it("can retain the active thread as hidden when it is inactive", () => { + expect( + reconcileRetainedMountedThreadIds({ + currentThreadIds: [ThreadId.make("thread-active")], + openThreadIds: [ThreadId.make("thread-active")], + activeThreadId: ThreadId.make("thread-active"), + activeThreadOpen: false, + maxHiddenThreadCount: MAX_HIDDEN_MOUNTED_PREVIEW_THREADS, + retainInactiveActiveThread: true, + }), + ).toEqual([ThreadId.make("thread-active")]); + }); + + it("evicts the oldest hidden threads beyond the configured cap", () => { + const currentThreadIds = Array.from( + { length: MAX_HIDDEN_MOUNTED_PREVIEW_THREADS + 2 }, + (_, index) => ThreadId.make(`thread-${index + 1}`), + ); + + expect( + reconcileRetainedMountedThreadIds({ + currentThreadIds, + openThreadIds: currentThreadIds, + activeThreadId: null, + activeThreadOpen: false, + maxHiddenThreadCount: MAX_HIDDEN_MOUNTED_PREVIEW_THREADS, + }), + ).toEqual(currentThreadIds.slice(-MAX_HIDDEN_MOUNTED_PREVIEW_THREADS)); + }); +}); + describe("shouldWriteThreadErrorToCurrentServerThread", () => { it("routes errors to the active server thread when route and target match", () => { const threadId = ThreadId.make("thread-1"); diff --git a/apps/web/src/components/ChatView.logic.ts b/apps/web/src/components/ChatView.logic.ts index e1417773c32e041d9ce9521330a11130372e5a87..0012bee256b47819ba4636b9132fcc95c47f5195 100644 --- a/apps/web/src/components/ChatView.logic.ts +++ b/apps/web/src/components/ChatView.logic.ts @@ -22,6 +22,7 @@ import type { DraftThreadEnvMode } from "../composerDraftStore"; export const LAST_INVOKED_SCRIPT_BY_PROJECT_KEY = "t3code:last-invoked-script-by-project"; export const MAX_HIDDEN_MOUNTED_TERMINAL_THREADS = 10; +export const MAX_HIDDEN_MOUNTED_PREVIEW_THREADS = 3; export const LastInvokedScriptByProjectSchema = Schema.Record(ProjectId, Schema.String); @@ -79,15 +80,31 @@ export function reconcileMountedTerminalThreadIds(input: { activeThreadId: string | null; activeThreadTerminalOpen: boolean; maxHiddenThreadCount?: number; +}): string[] { + return reconcileRetainedMountedThreadIds({ + currentThreadIds: input.currentThreadIds, + openThreadIds: input.openThreadIds, + activeThreadId: input.activeThreadId, + activeThreadOpen: input.activeThreadTerminalOpen, + maxHiddenThreadCount: input.maxHiddenThreadCount ?? MAX_HIDDEN_MOUNTED_TERMINAL_THREADS, + }); +} + +export function reconcileRetainedMountedThreadIds(input: { + currentThreadIds: ReadonlyArray; + openThreadIds: ReadonlyArray; + activeThreadId: string | null; + activeThreadOpen: boolean; + maxHiddenThreadCount: number; + retainInactiveActiveThread?: boolean; }): string[] { const openThreadIdSet = new Set(input.openThreadIds); const hiddenThreadIds = input.currentThreadIds.filter( - (threadId) => threadId !== input.activeThreadId && openThreadIdSet.has(threadId), - ); - const maxHiddenThreadCount = Math.max( - 0, - input.maxHiddenThreadCount ?? MAX_HIDDEN_MOUNTED_TERMINAL_THREADS, + (threadId) => + (threadId !== input.activeThreadId || input.retainInactiveActiveThread === true) && + openThreadIdSet.has(threadId), ); + const maxHiddenThreadCount = Math.max(0, input.maxHiddenThreadCount); const nextThreadIds = hiddenThreadIds.length > maxHiddenThreadCount ? hiddenThreadIds.slice(-maxHiddenThreadCount) @@ -95,7 +112,7 @@ export function reconcileMountedTerminalThreadIds(input: { if ( input.activeThreadId && - input.activeThreadTerminalOpen && + input.activeThreadOpen && !nextThreadIds.includes(input.activeThreadId) ) { nextThreadIds.push(input.activeThreadId); diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index adc40747f74dbbedac4d724831193a4208ab0f0c..4dd6d9003dd975bebb64cdc3145422fc01f78367 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -99,7 +99,8 @@ import { buildTemporaryWorktreeBranchName } from "@t3tools/shared/git"; import { useMediaQuery } from "../hooks/useMediaQuery"; import { RIGHT_PANEL_INLINE_LAYOUT_MEDIA_QUERY } from "../rightPanelLayout"; import { selectActiveRightPanelKindWithUrl, useRightPanelStore } from "../rightPanelStore"; -import { isPreviewSupportedInRuntime } from "../previewStateStore"; +import { PreviewAutomationOwner } from "./preview/PreviewAutomationOwner"; +import { isPreviewSupportedInRuntime, usePreviewStateStore } from "../previewStateStore"; import { subscribePreviewAction } from "./preview/previewActionBus"; // Lazy: keeps the entire preview component graph (webview host, favicon // helper, Chromium error icon) out of the web bundle until first open. @@ -166,6 +167,7 @@ import { ProviderStatusBanner } from "./chat/ProviderStatusBanner"; import { ThreadErrorBanner } from "./chat/ThreadErrorBanner"; import { ComposerBannerStack, type ComposerBannerStackItem } from "./chat/ComposerBannerStack"; import { + MAX_HIDDEN_MOUNTED_PREVIEW_THREADS, MAX_HIDDEN_MOUNTED_TERMINAL_THREADS, buildExpiredTerminalContextToastCopy, buildLocalDraftThread, @@ -182,6 +184,7 @@ import { deriveLockedProvider, readFileAsDataUrl, reconcileMountedTerminalThreadIds, + reconcileRetainedMountedThreadIds, resolveSendEnvMode, revokeBlobPreviewUrl, revokeUserMessagePreviewUrls, @@ -996,6 +999,22 @@ export default function ChatView(props: ChatViewProps) { }), [mountedTerminalThreadKeys], ); + const previewSessionThreadKeys = usePreviewStateStore( + useShallow((state) => + Object.entries(state.byThreadKey).flatMap(([nextThreadKey, nextPreviewState]) => + nextPreviewState.snapshot ? [nextThreadKey] : [], + ), + ), + ); + const [mountedPreviewThreadKeys, setMountedPreviewThreadKeys] = useState([]); + const mountedPreviewThreadRefs = useMemo( + () => + mountedPreviewThreadKeys.flatMap((mountedThreadKey) => { + const mountedThreadRef = parseScopedThreadKey(mountedThreadKey); + return mountedThreadRef ? [{ key: mountedThreadKey, threadRef: mountedThreadRef }] : []; + }), + [mountedPreviewThreadKeys], + ); const fallbackDraftProjectRef = draftThread ? scopeProjectRef(draftThread.environmentId, draftThread.projectId) @@ -1092,6 +1111,12 @@ export default function ChatView(props: ChatViewProps) { const existingThreadKeys = new Set([...serverThreadKeys, ...draftThreadKeys]); return openTerminalThreadKeys.filter((nextThreadKey) => existingThreadKeys.has(nextThreadKey)); }, [draftThreadKeys, openTerminalThreadKeys, serverThreadKeys]); + const existingPreviewSessionThreadKeys = useMemo(() => { + const existingThreadKeys = new Set([...serverThreadKeys, ...draftThreadKeys]); + return previewSessionThreadKeys.filter((nextThreadKey) => + existingThreadKeys.has(nextThreadKey), + ); + }, [draftThreadKeys, previewSessionThreadKeys, serverThreadKeys]); const activeLatestTurn = activeThread?.latestTurn ?? null; const threadPlanCatalog = useThreadPlanCatalog( useMemo(() => { @@ -1121,6 +1146,27 @@ export default function ChatView(props: ChatViewProps) { : nextThreadIds; }); }, [activeThreadKey, existingOpenTerminalThreadKeys, terminalUiState.terminalOpen]); + useEffect(() => { + setMountedPreviewThreadKeys((currentThreadIds) => { + const nextThreadIds = reconcileRetainedMountedThreadIds({ + currentThreadIds, + openThreadIds: existingPreviewSessionThreadKeys, + activeThreadId: activeThreadKey, + activeThreadOpen: Boolean(activeThreadKey && !shouldUsePlanSidebarSheet), + maxHiddenThreadCount: MAX_HIDDEN_MOUNTED_PREVIEW_THREADS, + retainInactiveActiveThread: true, + }); + return currentThreadIds.length === nextThreadIds.length && + currentThreadIds.every((nextThreadId, index) => nextThreadId === nextThreadIds[index]) + ? currentThreadIds + : nextThreadIds; + }); + }, [ + activeThreadKey, + existingPreviewSessionThreadKeys, + previewPanelOpen, + shouldUsePlanSidebarSheet, + ]); const latestTurnSettled = isLatestTurnSettled(activeLatestTurn, activeThread?.session ?? null); const activeProjectRef = activeThread ? scopeProjectRef(activeThread.environmentId, activeThread.projectId) @@ -4148,14 +4194,35 @@ export default function ChatView(props: ChatViewProps) { onClose={closePlanSidebar} /> ) : null} - {previewPanelOpen && !shouldUsePlanSidebarSheet && activeThreadRef ? ( - - - - ) : null} + {!shouldUsePlanSidebarSheet + ? mountedPreviewThreadRefs.map( + ({ key: mountedThreadKey, threadRef: mountedThreadRef }) => { + const visible = previewPanelOpen && mountedThreadKey === activeThreadKey; + return ( +
+ + + +
+ ); + }, + ) + : null} {/* end horizontal flex container */} + {activeThreadRef ? ( + + ) : null} + {mountedTerminalThreadRefs.map(({ key: mountedThreadKey, threadRef: mountedThreadRef }) => ( => { + const deadline = Date.now() + timeoutMs; + while (Date.now() <= deadline) { + const state = selectThreadPreviewState(usePreviewStateStore.getState().byThreadKey, threadRef); + const tabId = state.snapshot?.tabId; + if (tabId && state.desktopOverlay && previewBridge) { + const status = await previewBridge.automation.status(tabId); + if (status.available) return; + } + await new Promise((resolve) => window.setTimeout(resolve, 50)); + } + const error = new Error(`Preview webview did not register within ${timeoutMs}ms.`); + error.name = "PreviewAutomationTimeoutError"; + throw error; +}; + +const waitForNavigationReadiness = async ( + tabId: string, + readiness: PreviewAutomationNavigateInput["readiness"], + timeoutMs: number, +): Promise => { + if (!previewBridge || readiness === "none") return; + const deadline = Date.now() + timeoutMs; + while (Date.now() <= deadline) { + if (readiness === "domContentLoaded") { + const readyState = await previewBridge.automation.evaluate(tabId, { + expression: "document.readyState", + }); + if (readyState === "interactive" || readyState === "complete") return; + } else { + const status = await previewBridge.automation.status(tabId); + if (!status.loading) return; + } + await new Promise((resolve) => window.setTimeout(resolve, 50)); + } + const error = new Error(`Preview navigation did not become ready within ${timeoutMs}ms.`); + error.name = "PreviewAutomationTimeoutError"; + throw error; +}; + +const currentStatus = async ( + threadRef: ScopedThreadRef, + visible: boolean, +): Promise => { + const state = selectThreadPreviewState(usePreviewStateStore.getState().byThreadKey, threadRef); + const tabId = state.snapshot?.tabId ?? null; + if (tabId && previewBridge && state.desktopOverlay) { + const status = await previewBridge.automation.status(tabId); + return { ...status, visible }; + } + const navStatus = state.snapshot?.navStatus; + return { + available: Boolean(previewBridge?.automation), + visible, + tabId, + url: navStatus && navStatus._tag !== "Idle" ? navStatus.url : null, + title: navStatus && navStatus._tag !== "Idle" ? navStatus.title : null, + loading: navStatus?._tag === "Loading", + }; +}; + +const serializeError = (error: unknown): NonNullable => { + if (error instanceof Error) { + const detail = + "detail" in error && (error as { detail?: unknown }).detail !== undefined + ? (error as { detail?: unknown }).detail + : undefined; + return { + _tag: error.name.startsWith("PreviewAutomation") + ? error.name + : "PreviewAutomationExecutionError", + message: error.message, + ...(detail === undefined ? {} : { detail }), + }; + } + return { + _tag: "PreviewAutomationExecutionError", + message: String(error), + }; +}; + +export function PreviewAutomationOwner(props: { + readonly threadRef: ScopedThreadRef; + readonly visible: boolean; +}) { + const { threadRef, visible } = props; + const ownerStateRef = useRef({ threadRef, visible }); + const handlerRef = useRef<(request: PreviewAutomationRequest) => Promise>( + async () => undefined, + ); + ownerStateRef.current = { threadRef, visible }; + + const handleRequest = useCallback( + async (request: PreviewAutomationRequest): Promise => { + if (request.threadId !== threadRef.threadId) { + const error = new Error("Preview automation request targeted a stale thread owner."); + error.name = "PreviewAutomationUnavailableError"; + throw error; + } + const api = ensureEnvironmentApi(threadRef.environmentId); + const state = selectThreadPreviewState( + usePreviewStateStore.getState().byThreadKey, + threadRef, + ); + const tabId = request.tabId ?? state.snapshot?.tabId ?? null; + switch (request.operation) { + case "status": + return currentStatus(threadRef, visible); + case "open": { + const input = request.input as PreviewAutomationOpenInput; + if (input.show ?? true) { + useRightPanelStore.getState().open(threadRef, "preview"); + } + let activeTabId = + (input.reuseExistingTab ?? true) ? (state.snapshot?.tabId ?? null) : null; + if (!activeTabId) { + const snapshot = await api.preview.open({ + threadId: threadRef.threadId, + ...(input.url ? { url: input.url } : {}), + }); + usePreviewStateStore.getState().applyServerSnapshot(threadRef, snapshot); + activeTabId = snapshot.tabId; + } else if (input.url && previewBridge) { + await previewBridge.navigate(activeTabId, input.url); + } + await waitForDesktopOverlay(threadRef, request.timeoutMs); + return currentStatus(threadRef, input.show ?? true); + } + case "navigate": { + if (!previewBridge || !tabId) throw new Error("Preview tab is not initialized."); + const input = request.input as PreviewAutomationNavigateInput; + await previewBridge.navigate(tabId, input.url); + await waitForNavigationReadiness( + tabId, + input.readiness ?? "load", + input.timeoutMs ?? request.timeoutMs, + ); + return currentStatus(threadRef, visible); + } + case "snapshot": + if (!previewBridge || !tabId) throw new Error("Preview tab is not initialized."); + return previewBridge.automation.snapshot(tabId); + case "click": + if (!previewBridge || !tabId) throw new Error("Preview tab is not initialized."); + return previewBridge.automation.click( + tabId, + request.input as Parameters[1], + ); + case "type": + if (!previewBridge || !tabId) throw new Error("Preview tab is not initialized."); + return previewBridge.automation.type( + tabId, + request.input as Parameters[1], + ); + case "press": + if (!previewBridge || !tabId) throw new Error("Preview tab is not initialized."); + return previewBridge.automation.press( + tabId, + request.input as Parameters[1], + ); + case "scroll": + if (!previewBridge || !tabId) throw new Error("Preview tab is not initialized."); + return previewBridge.automation.scroll( + tabId, + request.input as Parameters[1], + ); + case "evaluate": + if (!previewBridge || !tabId) throw new Error("Preview tab is not initialized."); + return previewBridge.automation.evaluate( + tabId, + request.input as Parameters[1], + ); + case "waitFor": + if (!previewBridge || !tabId) throw new Error("Preview tab is not initialized."); + return previewBridge.automation.waitFor( + tabId, + request.input as Parameters[1], + ); + } + }, + [threadRef, visible], + ); + handlerRef.current = handleRequest; + + useEffect(() => { + const api = ensureEnvironmentApi(threadRef.environmentId); + return api.preview.automation.connect( + { clientId: automationClientId }, + (request) => { + void handlerRef.current(request).then( + (result) => + api.preview.automation.respond({ + requestId: request.requestId, + ok: true, + ...(result === undefined ? {} : { result }), + }), + (error) => + api.preview.automation.respond({ + requestId: request.requestId, + ok: false, + error: serializeError(error), + }), + ); + }, + { + onResubscribe: () => { + const ownerState = ownerStateRef.current; + const state = selectThreadPreviewState( + usePreviewStateStore.getState().byThreadKey, + ownerState.threadRef, + ); + void api.preview.automation.reportOwner({ + clientId: automationClientId, + environmentId: ownerState.threadRef.environmentId, + threadId: ownerState.threadRef.threadId, + tabId: state.snapshot?.tabId ?? null, + visible: ownerState.visible, + supportsAutomation: Boolean(previewBridge?.automation), + focusedAt: new Date().toISOString(), + }); + }, + }, + ); + }, [threadRef.environmentId]); + + useEffect(() => { + const api = ensureEnvironmentApi(threadRef.environmentId); + const report = () => { + const state = selectThreadPreviewState( + usePreviewStateStore.getState().byThreadKey, + threadRef, + ); + void api.preview.automation.reportOwner({ + clientId: automationClientId, + environmentId: threadRef.environmentId, + threadId: threadRef.threadId, + tabId: state.snapshot?.tabId ?? null, + visible, + supportsAutomation: Boolean(previewBridge?.automation), + focusedAt: new Date().toISOString(), + }); + }; + report(); + window.addEventListener("focus", report); + const unsubscribe = usePreviewStateStore.subscribe((state, previous) => { + const key = scopedThreadKey(threadRef); + if (state.byThreadKey[key]?.snapshot?.tabId !== previous.byThreadKey[key]?.snapshot?.tabId) { + report(); + } + }); + return () => { + window.removeEventListener("focus", report); + unsubscribe(); + void api.preview.automation.clearOwner({ clientId: automationClientId }); + }; + }, [threadRef, visible]); + + return null; +} diff --git a/apps/web/src/environmentApi.ts b/apps/web/src/environmentApi.ts index 0268d98dcd38e7736d3cc8619a11db43eba2d054..2c2166d50856f668c0f2b607f8af578a044cde72 100644 --- a/apps/web/src/environmentApi.ts +++ b/apps/web/src/environmentApi.ts @@ -65,6 +65,13 @@ export function createEnvironmentApi(rpcClient: WsRpcClient): EnvironmentApi { close: (input) => rpcClient.preview.close(input as never), list: (input) => rpcClient.preview.list(input as never), reportStatus: (input) => rpcClient.preview.reportStatus(input as never), + automation: { + connect: (input, callback, options) => + rpcClient.preview.automation.connect(input as never, callback, options), + respond: (response) => rpcClient.preview.automation.respond(response as never), + reportOwner: (owner) => rpcClient.preview.automation.reportOwner(owner as never), + clearOwner: (input) => rpcClient.preview.automation.clearOwner(input as never), + }, onEvent: (callback) => rpcClient.preview.onEvent(callback), subscribePorts: (callback, options) => rpcClient.preview.subscribePorts(callback, options), }, diff --git a/apps/web/src/environments/runtime/service.threadSubscriptions.test.ts b/apps/web/src/environments/runtime/service.threadSubscriptions.test.ts index b1fe715f491862ddf5d87c5a78bfcaf47bf8c843..94292da1d96904dda5bc79deefefde51f2708f55 100644 --- a/apps/web/src/environments/runtime/service.threadSubscriptions.test.ts +++ b/apps/web/src/environments/runtime/service.threadSubscriptions.test.ts @@ -111,6 +111,12 @@ vi.mock("@t3tools/client-runtime", async (importOriginal) => { close: vi.fn(), list: vi.fn(), reportStatus: vi.fn(), + automation: { + connect: vi.fn(() => () => undefined), + respond: vi.fn(), + reportOwner: vi.fn(), + clearOwner: vi.fn(), + }, onEvent: vi.fn(() => () => undefined), subscribePorts: vi.fn(() => () => undefined), }, diff --git a/packages/client-runtime/src/wsRpcClient.ts b/packages/client-runtime/src/wsRpcClient.ts index 733c65ee13eb6972a5a8f2540557976a464a7bf7..ca01302cbe3759b1bc0e35a122bb09c542a90e5e 100644 --- a/packages/client-runtime/src/wsRpcClient.ts +++ b/packages/client-runtime/src/wsRpcClient.ts @@ -86,6 +86,12 @@ export interface WsRpcClient { readonly close: RpcUnaryMethod; readonly list: RpcUnaryMethod; readonly reportStatus: RpcUnaryMethod; + readonly automation: { + readonly connect: RpcInputStreamMethod; + readonly respond: RpcUnaryMethod; + readonly reportOwner: RpcUnaryMethod; + readonly clearOwner: RpcUnaryMethod; + }; readonly onEvent: RpcStreamMethod; readonly subscribePorts: RpcStreamMethod; }; @@ -230,6 +236,20 @@ export function createWsRpcClient( list: (input) => transport.request((client) => client[WS_METHODS.previewList](input)), reportStatus: (input) => transport.request((client) => client[WS_METHODS.previewReportStatus](input)), + automation: { + connect: (input, listener, options) => + transport.subscribe( + (client) => client[WS_METHODS.previewAutomationConnect](input), + listener, + subscriptionOptions(options, WS_METHODS.previewAutomationConnect), + ), + respond: (input) => + transport.request((client) => client[WS_METHODS.previewAutomationRespond](input)), + reportOwner: (input) => + transport.request((client) => client[WS_METHODS.previewAutomationReportOwner](input)), + clearOwner: (input) => + transport.request((client) => client[WS_METHODS.previewAutomationClearOwner](input)), + }, onEvent: (listener, options) => transport.subscribe( (client) => client[WS_METHODS.subscribePreviewEvents]({}), diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index 1cb46f6c79d9e41868cc7de1e79479904b1cc3d2..4ffb7642d17401ce36a54d2865148952c8e6f95e 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -23,4 +23,5 @@ export * from "./project.ts"; export * from "./filesystem.ts"; export * from "./review.ts"; export * from "./preview.ts"; +export * from "./previewAutomation.ts"; export * from "./rpc.ts"; diff --git a/packages/contracts/src/ipc.ts b/packages/contracts/src/ipc.ts index 161aaf69a6be7fecad48dfca5de38c18f7b47020..bb0a1bd1eb5382807978a34ca6ccc37ae877c737 100644 --- a/packages/contracts/src/ipc.ts +++ b/packages/contracts/src/ipc.ts @@ -66,6 +66,19 @@ import type { PreviewReportStatusInput, PreviewSessionSnapshot, } from "./preview.ts"; +import type { + PreviewAutomationClickInput, + PreviewAutomationEvaluateInput, + PreviewAutomationOwner, + PreviewAutomationPressInput, + PreviewAutomationRequest, + PreviewAutomationResponse, + PreviewAutomationScrollInput, + PreviewAutomationSnapshot, + PreviewAutomationStatus, + PreviewAutomationTypeInput, + PreviewAutomationWaitForInput, +} from "./previewAutomation.ts"; import type { ClientOrchestrationCommand, OrchestrationGetFullThreadDiffInput, @@ -668,6 +681,16 @@ export interface DesktopPreviewBridge { pickElement: (tabId: string) => Promise; /** Cancel an in-flight preview annotation session. */ cancelPickElement: (tabId: string) => Promise; + automation: { + status: (tabId: string) => Promise; + snapshot: (tabId: string) => Promise; + click: (tabId: string, input: PreviewAutomationClickInput) => Promise; + type: (tabId: string, input: PreviewAutomationTypeInput) => Promise; + press: (tabId: string, input: PreviewAutomationPressInput) => Promise; + scroll: (tabId: string, input: PreviewAutomationScrollInput) => Promise; + evaluate: (tabId: string, input: PreviewAutomationEvaluateInput) => Promise; + waitFor: (tabId: string, input: PreviewAutomationWaitForInput) => Promise; + }; onStateChange: (listener: (tabId: string, state: DesktopPreviewTabState) => void) => () => void; } @@ -835,6 +858,16 @@ export interface EnvironmentApi { close: (input: typeof PreviewCloseInput.Encoded) => Promise; list: (input: typeof PreviewListInput.Encoded) => Promise; reportStatus: (input: typeof PreviewReportStatusInput.Encoded) => Promise; + automation: { + connect: ( + input: { clientId: string }, + callback: (request: PreviewAutomationRequest) => void, + options?: { onResubscribe?: () => void }, + ) => () => void; + respond: (response: PreviewAutomationResponse) => Promise; + reportOwner: (owner: PreviewAutomationOwner) => Promise; + clearOwner: (input: { clientId: string }) => Promise; + }; onEvent: (callback: (event: PreviewEvent) => void) => () => void; subscribePorts: ( callback: (servers: DiscoveredLocalServerList) => void, diff --git a/packages/contracts/src/previewAutomation.ts b/packages/contracts/src/previewAutomation.ts new file mode 100644 index 0000000000000000000000000000000000000000..eaadbcb48c9a4d6e5ce5da941de1b212a90b56f0 --- /dev/null +++ b/packages/contracts/src/previewAutomation.ts @@ -0,0 +1,219 @@ +import { Schema } from "effect"; + +import { EnvironmentId, ThreadId, TrimmedNonEmptyString } from "./baseSchemas.ts"; +import { PreviewTabId } from "./preview.ts"; + +const BoundedUrl = TrimmedNonEmptyString.check(Schema.isMaxLength(2048)); +const OptionalTimeoutMs = Schema.optional( + Schema.Int.check(Schema.isGreaterThan(0)).check(Schema.isLessThanOrEqualTo(60_000)), +); + +export const PreviewAutomationOperation = Schema.Literals([ + "status", + "open", + "navigate", + "snapshot", + "click", + "type", + "press", + "scroll", + "evaluate", + "waitFor", +]); +export type PreviewAutomationOperation = typeof PreviewAutomationOperation.Type; + +export const PreviewAutomationStatus = Schema.Struct({ + available: Schema.Boolean, + visible: Schema.Boolean, + tabId: Schema.NullOr(PreviewTabId), + url: Schema.NullOr(Schema.String), + title: Schema.NullOr(Schema.String), + loading: Schema.Boolean, +}); +export type PreviewAutomationStatus = typeof PreviewAutomationStatus.Type; + +export const PreviewAutomationOpenInput = Schema.Struct({ + url: Schema.optional(BoundedUrl), + show: Schema.optional(Schema.Boolean), + reuseExistingTab: Schema.optional(Schema.Boolean), +}); +export type PreviewAutomationOpenInput = typeof PreviewAutomationOpenInput.Type; + +export const PreviewAutomationNavigateInput = Schema.Struct({ + url: BoundedUrl, + readiness: Schema.optional(Schema.Literals(["load", "domContentLoaded", "none"])), + timeoutMs: OptionalTimeoutMs, +}); +export type PreviewAutomationNavigateInput = typeof PreviewAutomationNavigateInput.Type; + +export const PreviewAutomationClickInput = Schema.Union([ + Schema.Struct({ + selector: TrimmedNonEmptyString, + timeoutMs: OptionalTimeoutMs, + }), + Schema.Struct({ + x: Schema.Number, + y: Schema.Number, + timeoutMs: OptionalTimeoutMs, + }), +]); +export type PreviewAutomationClickInput = typeof PreviewAutomationClickInput.Type; + +export const PreviewAutomationTypeInput = Schema.Struct({ + text: Schema.String, + selector: Schema.optional(TrimmedNonEmptyString), + clear: Schema.optional(Schema.Boolean), + timeoutMs: OptionalTimeoutMs, +}); +export type PreviewAutomationTypeInput = typeof PreviewAutomationTypeInput.Type; + +export const PreviewAutomationPressInput = Schema.Struct({ + key: TrimmedNonEmptyString, + modifiers: Schema.optional(Schema.Array(Schema.Literals(["Alt", "Control", "Meta", "Shift"]))), +}); +export type PreviewAutomationPressInput = typeof PreviewAutomationPressInput.Type; + +export const PreviewAutomationScrollInput = Schema.Struct({ + deltaX: Schema.optional(Schema.Number), + deltaY: Schema.optional(Schema.Number), + selector: Schema.optional(TrimmedNonEmptyString), +}); +export type PreviewAutomationScrollInput = typeof PreviewAutomationScrollInput.Type; + +export const PreviewAutomationEvaluateInput = Schema.Struct({ + expression: TrimmedNonEmptyString.check(Schema.isMaxLength(64_000)), + awaitPromise: Schema.optional(Schema.Boolean), + returnByValue: Schema.optional(Schema.Boolean), +}); +export type PreviewAutomationEvaluateInput = typeof PreviewAutomationEvaluateInput.Type; + +export const PreviewAutomationWaitForInput = Schema.Struct({ + selector: Schema.optional(TrimmedNonEmptyString), + text: Schema.optional(TrimmedNonEmptyString), + urlIncludes: Schema.optional(TrimmedNonEmptyString), + timeoutMs: OptionalTimeoutMs, +}); +export type PreviewAutomationWaitForInput = typeof PreviewAutomationWaitForInput.Type; + +export const PreviewAutomationElement = Schema.Struct({ + tag: Schema.String, + role: Schema.NullOr(Schema.String), + name: Schema.String, + selector: Schema.String, + x: Schema.Number, + y: Schema.Number, + width: Schema.Number, + height: Schema.Number, +}); +export type PreviewAutomationElement = typeof PreviewAutomationElement.Type; + +export const PreviewAutomationSnapshot = Schema.Struct({ + url: Schema.String, + title: Schema.String, + loading: Schema.Boolean, + visibleText: Schema.String, + interactiveElements: Schema.Array(PreviewAutomationElement), + accessibilityTree: Schema.Unknown, + screenshot: Schema.Struct({ + mimeType: Schema.Literal("image/png"), + data: Schema.String, + width: Schema.Int, + height: Schema.Int, + }), +}); +export type PreviewAutomationSnapshot = typeof PreviewAutomationSnapshot.Type; + +export const PreviewAutomationOwner = Schema.Struct({ + clientId: TrimmedNonEmptyString, + environmentId: EnvironmentId, + threadId: ThreadId, + tabId: Schema.NullOr(PreviewTabId), + visible: Schema.Boolean, + supportsAutomation: Schema.Boolean, + focusedAt: Schema.String, +}); +export type PreviewAutomationOwner = typeof PreviewAutomationOwner.Type; + +export const PreviewAutomationRequest = Schema.Struct({ + requestId: TrimmedNonEmptyString, + threadId: ThreadId, + tabId: Schema.optional(PreviewTabId), + operation: PreviewAutomationOperation, + input: Schema.Unknown, + timeoutMs: Schema.Int.check(Schema.isGreaterThan(0)), +}); +export type PreviewAutomationRequest = typeof PreviewAutomationRequest.Type; + +export const PreviewAutomationResponse = Schema.Struct({ + requestId: TrimmedNonEmptyString, + ok: Schema.Boolean, + result: Schema.optional(Schema.Unknown), + error: Schema.optional( + Schema.Struct({ + _tag: TrimmedNonEmptyString, + message: Schema.String, + detail: Schema.optional(Schema.Unknown), + }), + ), +}); +export type PreviewAutomationResponse = typeof PreviewAutomationResponse.Type; + +export class PreviewAutomationUnavailableError extends Schema.TaggedErrorClass()( + "PreviewAutomationUnavailableError", + { message: Schema.String }, +) {} + +export class PreviewAutomationNoFocusedOwnerError extends Schema.TaggedErrorClass()( + "PreviewAutomationNoFocusedOwnerError", + { message: Schema.String }, +) {} + +export class PreviewAutomationUnsupportedClientError extends Schema.TaggedErrorClass()( + "PreviewAutomationUnsupportedClientError", + { message: Schema.String }, +) {} + +export class PreviewAutomationTabNotFoundError extends Schema.TaggedErrorClass()( + "PreviewAutomationTabNotFoundError", + { message: Schema.String }, +) {} + +export class PreviewAutomationTimeoutError extends Schema.TaggedErrorClass()( + "PreviewAutomationTimeoutError", + { message: Schema.String }, +) {} + +export class PreviewAutomationExecutionError extends Schema.TaggedErrorClass()( + "PreviewAutomationExecutionError", + { message: Schema.String, detail: Schema.optional(Schema.Unknown) }, +) {} + +export class PreviewAutomationInvalidSelectorError extends Schema.TaggedErrorClass()( + "PreviewAutomationInvalidSelectorError", + { message: Schema.String, selector: Schema.String }, +) {} + +export class PreviewAutomationResultTooLargeError extends Schema.TaggedErrorClass()( + "PreviewAutomationResultTooLargeError", + { message: Schema.String, maximumBytes: Schema.Int }, +) {} + +export const PreviewAutomationError = Schema.Union([ + PreviewAutomationUnavailableError, + PreviewAutomationNoFocusedOwnerError, + PreviewAutomationUnsupportedClientError, + PreviewAutomationTabNotFoundError, + PreviewAutomationTimeoutError, + PreviewAutomationExecutionError, + PreviewAutomationInvalidSelectorError, + PreviewAutomationResultTooLargeError, +]); +export type PreviewAutomationError = typeof PreviewAutomationError.Type; + +export const PreviewUrlResolution = Schema.Struct({ + requestedUrl: Schema.String, + resolvedUrl: Schema.String, + resolutionKind: Schema.Literal("direct"), + environmentId: EnvironmentId, +}); +export type PreviewUrlResolution = typeof PreviewUrlResolution.Type; diff --git a/packages/contracts/src/rpc.ts b/packages/contracts/src/rpc.ts index e3d462bb82a9e803a4e7bbbc54f8462839fffe16..189b1bdb6796e35fb2c37fd8f25cfc8d2f7e4f0a 100644 --- a/packages/contracts/src/rpc.ts +++ b/packages/contracts/src/rpc.ts @@ -98,6 +98,12 @@ import { PreviewReportStatusInput, PreviewSessionSnapshot, } from "./preview.ts"; +import { + PreviewAutomationError, + PreviewAutomationOwner, + PreviewAutomationRequest, + PreviewAutomationResponse, +} from "./previewAutomation.ts"; import { ServerConfigStreamEvent, ServerConfig, @@ -177,6 +183,10 @@ export const WS_METHODS = { previewClose: "preview.close", previewList: "preview.list", previewReportStatus: "preview.reportStatus", + previewAutomationConnect: "previewAutomation.connect", + previewAutomationRespond: "previewAutomation.respond", + previewAutomationReportOwner: "previewAutomation.reportOwner", + previewAutomationClearOwner: "previewAutomation.clearOwner", // Server meta serverGetConfig: "server.getConfig", @@ -510,6 +520,28 @@ export const WsPreviewReportStatusRpc = Rpc.make(WS_METHODS.previewReportStatus, error: Schema.Union([PreviewError, EnvironmentAuthorizationError]), }); +export const WsPreviewAutomationConnectRpc = Rpc.make(WS_METHODS.previewAutomationConnect, { + payload: Schema.Struct({ clientId: Schema.String }), + success: PreviewAutomationRequest, + error: Schema.Union([PreviewAutomationError, EnvironmentAuthorizationError]), + stream: true, +}); + +export const WsPreviewAutomationRespondRpc = Rpc.make(WS_METHODS.previewAutomationRespond, { + payload: PreviewAutomationResponse, + error: Schema.Union([PreviewAutomationError, EnvironmentAuthorizationError]), +}); + +export const WsPreviewAutomationReportOwnerRpc = Rpc.make(WS_METHODS.previewAutomationReportOwner, { + payload: PreviewAutomationOwner, + error: Schema.Union([PreviewAutomationError, EnvironmentAuthorizationError]), +}); + +export const WsPreviewAutomationClearOwnerRpc = Rpc.make(WS_METHODS.previewAutomationClearOwner, { + payload: Schema.Struct({ clientId: Schema.String }), + error: Schema.Union([PreviewAutomationError, EnvironmentAuthorizationError]), +}); + export const WsSubscribePreviewEventsRpc = Rpc.make(WS_METHODS.subscribePreviewEvents, { payload: Schema.Struct({}), success: PreviewEvent, @@ -668,6 +700,10 @@ export const WsRpcGroup = RpcGroup.make( WsPreviewCloseRpc, WsPreviewListRpc, WsPreviewReportStatusRpc, + WsPreviewAutomationConnectRpc, + WsPreviewAutomationRespondRpc, + WsPreviewAutomationReportOwnerRpc, + WsPreviewAutomationClearOwnerRpc, WsSubscribePreviewEventsRpc, WsSubscribeDiscoveredLocalServersRpc, WsSubscribeServerConfigRpc, diff --git a/vite.config.ts b/vite.config.ts index 1ce94c687542eaeb58ae987296e5efb3cd694530..d235efd60be3b7829575198dae1b7a422304a800 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -1,7 +1,13 @@ import "vite-plus/test/config"; import { defineConfig } from "vite-plus"; +import { fileURLToPath } from "node:url"; export default defineConfig({ + resolve: { + alias: { + "~": fileURLToPath(new URL("./apps/web/src", import.meta.url)), + }, + }, test: { environment: "node", exclude: [