| author |  Julius Marminge <julius0216@outlook.com> 2026-10-06 17:12:49-07:00 |
| committer |  GitHub <noreply@github.com> 2026-10-06 17:12:49-07:00 |
| log | 4b5c60485366421794787aee7513913f92eced29 |
| tree | 3e39f5eadad63939cd81d17dd07507a135dd889c |
| parent | 7202ba666080cfca1351c080d7aaa2d81bfa9a66 |
| signature | Signed by PGP key B5690EEEBB952194 |
fix(server): provider sessions clean up when their start is interrupted (#15571)
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>10 files changed, 1213 insertions(+), 210 deletions(-)
apps/server/src/device/LocalDeviceHost.ts+81-75
| ... | ... | @@ -383,62 +383,63 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { |
| 383 | 383 | ); |
| 384 | 384 | const origin = `http://127.0.0.1:${port}`; |
| 385 | 385 | const scope = yield* Scope.make("sequential"); |
| 386 | const child = yield* spawner | |
| 387 | .spawn( | |
| 388 | ChildProcess.make( | |
| 389 | nodePath, | |
| 390 | [ | |
| 391 | hubTool.entryPath, | |
| 392 | "--port", | |
| 393 | String(port), | |
| 394 | "--host", | |
| 395 | "127.0.0.1", | |
| 396 | "--hide-sidebar", | |
| 397 | "--hide-boot-device", | |
| 398 | ], | |
| 399 | { | |
| 400 | detached: false, | |
| 401 | shell: false, | |
| 402 | stdout: "pipe", | |
| 403 | stderr: "pipe", | |
| 404 | env: yield* hubEnvironment(hostEnvironment).pipe( | |
| 405 | Effect.provideService(FileSystem.FileSystem, fs), | |
| 406 | Effect.provideService(HostProcessPlatform, hostPlatform), | |
| 407 | ), | |
| 408 | }, | |
| 409 | ), | |
| 410 | ) | |
| 411 | .pipe( | |
| 412 | Effect.provideService(Scope.Scope, scope), | |
| 413 | Effect.mapError( | |
| 414 | (cause) => | |
| 415 | new DeviceHost.DeviceHostError({ | |
| 416 | hostId, | |
| 417 | step: "starting the device hub", | |
| 418 | cause, | |
| 419 | }), | |
| 420 | ), | |
| 421 | ); | |
| 422 | const startedAtMillis = yield* Clock.currentTimeMillis; | |
| 423 | const hub: HubProcess = { child, scope, origin, startedAtMillis, nodePath }; | |
| 424 | yield* Effect.forkIn(observeHubOutput(hub), scope); | |
| 425 | yield* waitForHttpReady({ | |
| 426 | baseUrl: origin, | |
| 427 | path: "/readyz", | |
| 428 | timeoutMs: HUB_READY_TIMEOUT_MS, | |
| 429 | makeError: (info) => | |
| 430 | new DeviceHost.DeviceHostError({ | |
| 431 | hostId, | |
| 432 | step: "waiting for the device hub to answer", | |
| 433 | cause: info.cause, | |
| 434 | }), | |
| 435 | }).pipe( | |
| 436 | Effect.provideService(HttpClient.HttpClient, httpClient), | |
| 437 | Effect.tapError(() => stopHub(hub)), | |
| 438 | ); | |
| 439 | yield* recordHub(hub, hubTool); | |
| 440 | yield* Effect.logInfo("Device hub started", { pid: Number(child.pid), port }); | |
| 441 | return hub; | |
| 386 | // On failure or interrupt, from the spawn on: the hub is not recorded yet, | |
| 387 | // so nothing else stops it. | |
| 388 | return yield* Effect.gen(function* () { | |
| 389 | const child = yield* spawner | |
| 390 | .spawn( | |
| 391 | ChildProcess.make( | |
| 392 | nodePath, | |
| 393 | [ | |
| 394 | hubTool.entryPath, | |
| 395 | "--port", | |
| 396 | String(port), | |
| 397 | "--host", | |
| 398 | "127.0.0.1", | |
| 399 | "--hide-sidebar", | |
| 400 | "--hide-boot-device", | |
| 401 | ], | |
| 402 | { | |
| 403 | detached: false, | |
| 404 | shell: false, | |
| 405 | stdout: "pipe", | |
| 406 | stderr: "pipe", | |
| 407 | env: yield* hubEnvironment(hostEnvironment).pipe( | |
| 408 | Effect.provideService(FileSystem.FileSystem, fs), | |
| 409 | Effect.provideService(HostProcessPlatform, hostPlatform), | |
| 410 | ), | |
| 411 | }, | |
| 412 | ), | |
| 413 | ) | |
| 414 | .pipe( | |
| 415 | Effect.provideService(Scope.Scope, scope), | |
| 416 | Effect.mapError( | |
| 417 | (cause) => | |
| 418 | new DeviceHost.DeviceHostError({ | |
| 419 | hostId, | |
| 420 | step: "starting the device hub", | |
| 421 | cause, | |
| 422 | }), | |
| 423 | ), | |
| 424 | ); | |
| 425 | const startedAtMillis = yield* Clock.currentTimeMillis; | |
| 426 | const hub: HubProcess = { child, scope, origin, startedAtMillis, nodePath }; | |
| 427 | yield* Effect.forkIn(observeHubOutput(hub), scope); | |
| 428 | yield* waitForHttpReady({ | |
| 429 | baseUrl: origin, | |
| 430 | path: "/readyz", | |
| 431 | timeoutMs: HUB_READY_TIMEOUT_MS, | |
| 432 | makeError: (info) => | |
| 433 | new DeviceHost.DeviceHostError({ | |
| 434 | hostId, | |
| 435 | step: "waiting for the device hub to answer", | |
| 436 | cause: info.cause, | |
| 437 | }), | |
| 438 | }).pipe(Effect.provideService(HttpClient.HttpClient, httpClient)); | |
| 439 | yield* recordHub(hub, hubTool); | |
| 440 | yield* Effect.logInfo("Device hub started", { pid: Number(child.pid), port }); | |
| 441 | return hub; | |
| 442 | }).pipe(Effect.onError(() => Scope.close(scope, Exit.void).pipe(Effect.ignore))); | |
| 442 | 443 | }); |
| 443 | 444 | |
| 444 | 445 | const observeHubOutput = (hub: HubProcess) => |
| ... | ... | @@ -622,25 +623,30 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { |
| 622 | 623 | ); |
| 623 | 624 | yield* onPhase("starting"); |
| 624 | 625 | const hub = yield* spawnHub(hubTool, nodePath); |
| 625 | yield* pruneLocalDeviceTools(config.baseDir, nodePath, "hub").pipe( | |
| 626 | Effect.provideService(Path.Path, path), | |
| 627 | Effect.provideService(ProcessRunner.ProcessRunner, runner), | |
| 628 | Effect.ignore, | |
| 629 | ); | |
| 630 | const candidate = helperPaths(hubTool); | |
| 631 | const [axExists, cliExists] = yield* Effect.all([ | |
| 632 | fs.exists(candidate.serveSimAxSettings).pipe(Effect.orElseSucceed(() => false)), | |
| 633 | fs.exists(candidate.serveSimCli).pipe(Effect.orElseSucceed(() => false)), | |
| 634 | ]); | |
| 635 | const next: RunningHost = { | |
| 636 | hub, | |
| 637 | agentDevice: null, | |
| 638 | helpers: { | |
| 639 | serveSimAxSettings: axExists ? candidate.serveSimAxSettings : null, | |
| 640 | serveSimCli: cliExists ? candidate.serveSimCli : null, | |
| 641 | }, | |
| 642 | }; | |
| 643 | yield* Ref.set(runningRef, next); | |
| 626 | // Until the hub is in runningRef, nothing else stops it, so publishing it | |
| 627 | // is part of the guarded step. | |
| 628 | const next = yield* Effect.gen(function* () { | |
| 629 | yield* pruneLocalDeviceTools(config.baseDir, nodePath, "hub").pipe( | |
| 630 | Effect.provideService(Path.Path, path), | |
| 631 | Effect.provideService(ProcessRunner.ProcessRunner, runner), | |
| 632 | Effect.ignore, | |
| 633 | ); | |
| 634 | const candidate = helperPaths(hubTool); | |
| 635 | const [axExists, cliExists] = yield* Effect.all([ | |
| 636 | fs.exists(candidate.serveSimAxSettings).pipe(Effect.orElseSucceed(() => false)), | |
| 637 | fs.exists(candidate.serveSimCli).pipe(Effect.orElseSucceed(() => false)), | |
| 638 | ]); | |
| 639 | const next: RunningHost = { | |
| 640 | hub, | |
| 641 | agentDevice: null, | |
| 642 | helpers: { | |
| 643 | serveSimAxSettings: axExists ? candidate.serveSimAxSettings : null, | |
| 644 | serveSimCli: cliExists ? candidate.serveSimCli : null, | |
| 645 | }, | |
| 646 | }; | |
| 647 | yield* Ref.set(runningRef, next); | |
| 648 | return next; | |
| 649 | }).pipe(Effect.onError(() => stopHub(hub))); | |
| 644 | 650 | yield* Ref.set(restartDelayRef, 0); |
| 645 | 651 | yield* Effect.forkDetach(superviseHub(hub, hubTool)); |
| 646 | 652 | return next; |
apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts+2-1
| ... | ... | @@ -7148,7 +7148,8 @@ export function makeClaudeAdapterV2( |
| 7148 | 7148 | options: queryOptions, |
| 7149 | 7149 | }) |
| 7150 | 7150 | .pipe( |
| 7151 | Effect.tapError(() => | |
| 7151 | // An interrupted open leaves the old process just as dead. | |
| 7152 | Effect.onError(() => | |
| 7152 | 7153 | // Same-native-thread replacement: the old process is already |
| 7153 | 7154 | // dead, so its process-scoped roster is not authoritative. |
| 7154 | 7155 | // First-ever failed open (no prior live query) must not invent |
apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts+3-1
| ... | ... | @@ -6174,8 +6174,10 @@ export function makeCodexAdapterV2(adapterOptions: CodexAdapterV2Options): Provi |
| 6174 | 6174 | yield* Ref.update(pendingRootTurns, (current) => |
| 6175 | 6175 | new Map(current).set(threadId, turnInput), |
| 6176 | 6176 | ); |
| 6177 | // Cleared on interrupt too, as startTurn does: an interrupted start | |
| 6178 | // must not adopt the native turn that a late turn/started reports. | |
| 6177 | 6179 | yield* client.request("thread/compact/start", { threadId }).pipe( |
| 6178 | Effect.tapError(() => | |
| 6180 | Effect.onError(() => | |
| 6179 | 6181 | Ref.update(pendingRootTurns, (current) => { |
| 6180 | 6182 | const next = new Map(current); |
| 6181 | 6183 | next.delete(threadId); |
apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts+19
| ... | ... | @@ -428,6 +428,25 @@ describe("OpenCode2 adapter", () => { |
| 428 | 428 | }).pipe(Effect.scoped), |
| 429 | 429 | ); |
| 430 | 430 | |
| 431 | it.effect("ends a turn on the provider thread it started on", () => | |
| 432 | Effect.gen(function* () { | |
| 433 | const { runtime, thread } = yield* resumed([ | |
| 434 | out("session.prompt", { sessionID: SESSION, text: "<any>" }), | |
| 435 | promptAccepted, | |
| 436 | event("session.execution.succeeded", { sessionID: SESSION }), | |
| 437 | ]); | |
| 438 | // A forked run starts on its own row for the same native session, while | |
| 439 | // the adapter tracks the session under the id it minted for the fork. | |
| 440 | const forkedRow = { | |
| 441 | ...thread, | |
| 442 | id: ProviderThreadId.make("provider-thread:opencode2-adapter:forked-run-row"), | |
| 443 | }; | |
| 444 | const terminal = yield* terminalOf(runtime).pipe(Effect.forkScoped); | |
| 445 | yield* runtime.startTurn(turnInput(forkedRow)); | |
| 446 | assert.equal((yield* Fiber.join(terminal))?.providerThreadId, forkedRow.id); | |
| 447 | }).pipe(Effect.scoped), | |
| 448 | ); | |
| 449 | ||
| 431 | 450 | it.effect("ends the turn when its terminal event is one this build cannot decode", () => |
| 432 | 451 | Effect.gen(function* () { |
| 433 | 452 | const { runtime, thread } = yield* resumed([ |
apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.ts+10-6
| ... | ... | @@ -1786,7 +1786,10 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr |
| 1786 | 1786 | const base = { |
| 1787 | 1787 | type: "turn.terminal" as const, |
| 1788 | 1788 | driver, |
| 1789 | providerThreadId: state.providerThread.id, | |
| 1789 | // The provider thread the turn started on. After a native fork the | |
| 1790 | // session's own thread has a fresh id, and the terminal must name the | |
| 1791 | // thread the start was recorded against. | |
| 1792 | providerThreadId: turn.providerTurn.providerThreadId, | |
| 1790 | 1793 | providerTurnId: turn.providerTurn.id, |
| 1791 | 1794 | runOrdinal: turn.input.runOrdinal, |
| 1792 | 1795 | threadDisposition, |
| ... | ... | @@ -2825,12 +2828,13 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr |
| 2825 | 2828 | * reconciles. Retried a few times; the caller fails everything after that. |
| 2826 | 2829 | */ |
| 2827 | 2830 | const reconnect = Effect.gen(function* () { |
| 2831 | // Returned on any failure, including the interrupt a closing session | |
| 2832 | // sends a reconnect still in flight, so a spawned server can idle-stop. | |
| 2828 | 2833 | const scope = yield* Scope.make(); |
| 2829 | const next = yield* borrow.pipe( | |
| 2830 | Effect.provideService(Scope.Scope, scope), | |
| 2831 | Effect.tapError(() => Scope.close(scope, Exit.void)), | |
| 2832 | ); | |
| 2833 | const stream = yield* next.events.pipe(Effect.tapError(() => Scope.close(scope, Exit.void))); | |
| 2834 | const { next, stream } = yield* Effect.gen(function* () { | |
| 2835 | const next = yield* borrow.pipe(Effect.provideService(Scope.Scope, scope)); | |
| 2836 | return { next, stream: yield* next.events }; | |
| 2837 | }).pipe(Effect.onError(() => Scope.close(scope, Exit.void))); | |
| 2834 | 2838 | const previous = currentScope; |
| 2835 | 2839 | connection = next; |
| 2836 | 2840 | client = next.client; |
apps/server/src/orchestration-v2/Adapters/OpenCodeAdapterV2.test.ts+44-2
| ... | ... | @@ -182,7 +182,10 @@ const makeOpenCodeRuntimeHarness = Effect.fn("makeOpenCodeRuntimeHarness")(funct |
| 182 | 182 | runtimePolicy: policy, |
| 183 | 183 | }); |
| 184 | 184 | const now = yield* DateTime.now; |
| 185 | const startTurn = (text = "hello") => | |
| 185 | const startTurn = ( | |
| 186 | text = "hello", | |
| 187 | startProviderThread: OrchestrationV2ProviderThread = providerThread, | |
| 188 | ) => | |
| 186 | 189 | runtime.startTurn({ |
| 187 | 190 | appThread: { |
| 188 | 191 | id: threadId, |
| ... | ... | @@ -213,7 +216,7 @@ const makeOpenCodeRuntimeHarness = Effect.fn("makeOpenCodeRuntimeHarness")(funct |
| 213 | 216 | providerTurnOrdinal: 1, |
| 214 | 217 | attemptId: RunAttemptId.make(`attempt-opencode-${suffix}`), |
| 215 | 218 | rootNodeId: NodeId.make(`node-opencode-${suffix}`), |
| 216 | providerThread, | |
| 219 | providerThread: startProviderThread, | |
| 217 | 220 | message: { |
| 218 | 221 | createdBy: "user", |
| 219 | 222 | creationSource: "web", |
| ... | ... | @@ -1291,6 +1294,45 @@ describe("OpenCodeAdapterV2", () => { |
| 1291 | 1294 | }).pipe(Effect.provide(IdAllocator.layer), Effect.scoped), |
| 1292 | 1295 | ); |
| 1293 | 1296 | |
| 1297 | it.effect("ends a turn on the provider thread it started on", () => | |
| 1298 | Effect.gen(function* () { | |
| 1299 | const nativeEvents = asyncEventStream(); | |
| 1300 | const harness = yield* makeOpenCodeRuntimeHarness( | |
| 1301 | "forked-row", | |
| 1302 | "native-opencode-forked-row", | |
| 1303 | { | |
| 1304 | event: { | |
| 1305 | subscribe: async (_input: unknown, options: { signal?: AbortSignal }) => { | |
| 1306 | options.signal?.addEventListener("abort", () => nativeEvents.close(), { once: true }); | |
| 1307 | return { stream: nativeEvents.stream }; | |
| 1308 | }, | |
| 1309 | }, | |
| 1310 | session: { | |
| 1311 | create: async () => ({ | |
| 1312 | data: { id: "native-opencode-forked-row", time: { created: 1, updated: 1 } }, | |
| 1313 | }), | |
| 1314 | summarize: async () => ({ data: true }), | |
| 1315 | }, | |
| 1316 | }, | |
| 1317 | ); | |
| 1318 | // A forked run starts on its own row for the same native session, while | |
| 1319 | // the adapter tracks the session under the id it minted. | |
| 1320 | const forkedRow = { | |
| 1321 | ...harness.providerThread, | |
| 1322 | id: ProviderThreadId.make("provider-thread:opencode-test:forked-run-row"), | |
| 1323 | }; | |
| 1324 | yield* harness.startTurn("/compact", forkedRow); | |
| 1325 | const terminal = yield* harness.runtime.events.pipe( | |
| 1326 | Stream.filter( | |
| 1327 | (event): event is Extract<typeof event, { type: "turn.terminal" }> => | |
| 1328 | event.type === "turn.terminal", | |
| 1329 | ), | |
| 1330 | Stream.runHead, | |
| 1331 | ); | |
| 1332 | assert.equal(Option.getOrUndefined(terminal)?.providerThreadId, forkedRow.id); | |
| 1333 | }).pipe(Effect.provide(IdAllocator.layer), Effect.scoped), | |
| 1334 | ); | |
| 1335 | ||
| 1294 | 1336 | it.effect("does not restore messages beyond OpenCode's persisted revert boundary", () => |
| 1295 | 1337 | Effect.gen(function* () { |
| 1296 | 1338 | const nativeEvents = asyncEventStream(); |
apps/server/src/orchestration-v2/Adapters/OpenCodeAdapterV2.ts+6-2
| ... | ... | @@ -279,6 +279,8 @@ interface ActiveOpenCodeTurn { |
| 279 | 279 | readonly modelSelection: ModelSelection; |
| 280 | 280 | readonly runtimePolicy: ProviderAdapter.ProviderAdapterV2RuntimePolicy; |
| 281 | 281 | readonly providerTurnId: OrchestrationV2ProviderTurn["id"]; |
| 282 | /** The provider thread the turn started on, which its terminal names. */ | |
| 283 | readonly providerThreadId: OrchestrationV2ProviderTurn["providerThreadId"]; | |
| 282 | 284 | readonly providerTurnOrdinal: number; |
| 283 | 285 | readonly runOrdinal: number; |
| 284 | 286 | readonly runAttemptId: OrchestrationV2ProviderTurn["runAttemptId"]; |
| ... | ... | @@ -2118,7 +2120,7 @@ export function makeOpenCodeAdapterV2( |
| 2118 | 2120 | ? { |
| 2119 | 2121 | type: "turn.terminal", |
| 2120 | 2122 | driver: OPENCODE_PROVIDER, |
| 2121 | providerThreadId: state.providerThread.id, | |
| 2123 | providerThreadId: turn.providerThreadId, | |
| 2122 | 2124 | providerTurnId: turn.providerTurnId, |
| 2123 | 2125 | runOrdinal: turn.runOrdinal, |
| 2124 | 2126 | failureItemOrdinal: itemOrdinal(turn, `terminal-failure:${turn.providerTurnId}`), |
| ... | ... | @@ -2134,7 +2136,7 @@ export function makeOpenCodeAdapterV2( |
| 2134 | 2136 | : { |
| 2135 | 2137 | type: "turn.terminal", |
| 2136 | 2138 | driver: OPENCODE_PROVIDER, |
| 2137 | providerThreadId: state.providerThread.id, | |
| 2139 | providerThreadId: turn.providerThreadId, | |
| 2138 | 2140 | providerTurnId: turn.providerTurnId, |
| 2139 | 2141 | runOrdinal: turn.runOrdinal, |
| 2140 | 2142 | status, |
| ... | ... | @@ -2256,6 +2258,7 @@ export function makeOpenCodeAdapterV2( |
| 2256 | 2258 | modelSelection: state.appThread.modelSelection, |
| 2257 | 2259 | runtimePolicy: state.parentSubagent.parentTurn.runtimePolicy, |
| 2258 | 2260 | providerTurnId, |
| 2261 | providerThreadId: providerTurn.providerThreadId, | |
| 2259 | 2262 | providerTurnOrdinal: providerTurn.ordinal, |
| 2260 | 2263 | runOrdinal: state.parentSubagent.parentTurn.runOrdinal, |
| 2261 | 2264 | runAttemptId: null, |
| ... | ... | @@ -3192,6 +3195,7 @@ export function makeOpenCodeAdapterV2( |
| 3192 | 3195 | modelSelection: turnInput.modelSelection, |
| 3193 | 3196 | runtimePolicy: turnInput.runtimePolicy, |
| 3194 | 3197 | providerTurnId, |
| 3198 | providerThreadId: turnInput.providerThread.id, | |
| 3195 | 3199 | providerTurnOrdinal: turnInput.providerTurnOrdinal, |
| 3196 | 3200 | runOrdinal: turnInput.runOrdinal, |
| 3197 | 3201 | runAttemptId: turnInput.attemptId, |
apps/server/src/orchestration-v2/Adapters/PiAdapterV2.ts+2-1
| ... | ... | @@ -2679,8 +2679,9 @@ export function makePiAdapterV2( |
| 2679 | 2679 | } |
| 2680 | 2680 | // Pi fork replaces the session file, including for rollback. Persist |
| 2681 | 2681 | // its new identity before any later request can fail or restart. |
| 2682 | // An interrupted read leaves the identity just as unknown as a failed one. | |
| 2682 | 2683 | const forkState = yield* request({ type: "get_state" }).pipe( |
| 2683 | Effect.tapError(() => | |
| 2684 | Effect.onError(() => | |
| 2684 | 2685 | Effect.sync(() => { |
| 2685 | 2686 | threadState = null; |
| 2686 | 2687 | }), |
apps/server/src/orchestration-v2/ProviderSessionManager.test.ts+827-20
| ... | ... | @@ -128,6 +128,30 @@ const layerFlakyReleaseEventSink = (flaky: FlakyReleaseWrites) => |
| 128 | 128 | }), |
| 129 | 129 | ).pipe(Layer.provide(layerTestEventSink)); |
| 130 | 130 | |
| 131 | // Once armed, holds the next attach write until the writer is interrupted. | |
| 132 | const layerPausingAttachEventSink = (pause: { | |
| 133 | readonly armed: Ref.Ref<boolean>; | |
| 134 | readonly paused: Deferred.Deferred<void>; | |
| 135 | }) => | |
| 136 | Layer.effect( | |
| 137 | EventSink.EventSinkV2, | |
| 138 | Effect.gen(function* () { | |
| 139 | const delegate = yield* EventSink.EventSinkV2; | |
| 140 | return EventSink.EventSinkV2.of({ | |
| 141 | ...delegate, | |
| 142 | write: (input) => | |
| 143 | Effect.gen(function* () { | |
| 144 | const attach = input.events.some((event) => event.type === "provider-session.attached"); | |
| 145 | if (attach && (yield* Ref.getAndSet(pause.armed, false))) { | |
| 146 | yield* Deferred.succeed(pause.paused, undefined); | |
| 147 | return yield* Effect.never; | |
| 148 | } | |
| 149 | return yield* delegate.write(input); | |
| 150 | }), | |
| 151 | }); | |
| 152 | }), | |
| 153 | ).pipe(Layer.provide(layerTestEventSink)); | |
| 154 | ||
| 131 | 155 | const CodexCapabilities: OrchestrationV2ProviderCapabilities = CodexProviderCapabilitiesV2; |
| 132 | 156 | const ExclusiveCapabilities: OrchestrationV2ProviderCapabilities = { |
| 133 | 157 | ...CodexCapabilities, |
| ... | ... | @@ -244,11 +268,13 @@ function makeProviderThread(input: { |
| 244 | 268 | readonly threadId: ThreadId; |
| 245 | 269 | readonly providerSessionId: ProviderSessionId; |
| 246 | 270 | readonly now: DateTime.Utc; |
| 271 | readonly nativeThreadId?: string; | |
| 247 | 272 | }): OrchestrationV2ProviderThread { |
| 273 | const nativeThreadId = input.nativeThreadId ?? "native-thread"; | |
| 248 | 274 | return { |
| 249 | 275 | id: input.idAllocator.derive.providerThread({ |
| 250 | 276 | driver: CODEX_DRIVER, |
| 251 | nativeThreadId: "native-thread", | |
| 277 | nativeThreadId, | |
| 252 | 278 | }), |
| 253 | 279 | driver: CODEX_DRIVER, |
| 254 | 280 | providerInstanceId: modelSelection.instanceId, |
| ... | ... | @@ -257,7 +283,7 @@ function makeProviderThread(input: { |
| 257 | 283 | ownerNodeId: null, |
| 258 | 284 | nativeThreadRef: { |
| 259 | 285 | driver: CODEX_DRIVER, |
| 260 | nativeId: "native-thread", | |
| 286 | nativeId: nativeThreadId, | |
| 261 | 287 | strength: "strong", |
| 262 | 288 | }, |
| 263 | 289 | nativeConversationHeadRef: null, |
| ... | ... | @@ -294,9 +320,29 @@ function makeProviderAdapter( |
| 294 | 320 | }) => Effect.Effect<void>; |
| 295 | 321 | readonly hasPendingBackgroundWork?: Effect.Effect<boolean>; |
| 296 | 322 | readonly hangSessionScopeClose?: boolean; |
| 323 | readonly startTurn?: Effect.Effect<void>; | |
| 297 | 324 | readonly beforeUnload?: Effect.Effect<void>; |
| 325 | /** Registers the process's closeCount finalizer before `beforeOpen` runs. */ | |
| 326 | readonly spawnBeforeOpen?: boolean; | |
| 327 | /** Completed when a hanging scope close reaches its wedged finalizer. */ | |
| 328 | readonly scopeCloseReached?: Deferred.Deferred<void>; | |
| 298 | 329 | } = {}, |
| 299 | 330 | ): ProviderAdapterV2Shape { |
| 331 | const countClose = Effect.addFinalizer(() => | |
| 332 | Ref.update(state, (current) => ({ | |
| 333 | ...current, | |
| 334 | closeCount: current.closeCount + 1, | |
| 335 | })), | |
| 336 | ); | |
| 337 | // Registered after countClose so it runs first on scope close, wedging the | |
| 338 | // close before the closeCount finalizer, like a provider process that never | |
| 339 | // yields its message stream. | |
| 340 | const hangClose = Effect.addFinalizer(() => | |
| 341 | (options.scopeCloseReached === undefined | |
| 342 | ? Effect.void | |
| 343 | : Deferred.succeed(options.scopeCloseReached, undefined) | |
| 344 | ).pipe(Effect.andThen(Effect.never)), | |
| 345 | ); | |
| 300 | 346 | return { |
| 301 | 347 | instanceId: ProviderInstanceId.make("codex"), |
| 302 | 348 | driver: CODEX_DRIVER, |
| ... | ... | @@ -304,10 +350,20 @@ function makeProviderAdapter( |
| 304 | 350 | planSelectionTransition: () => Effect.succeed({ type: "apply_on_next_turn" }), |
| 305 | 351 | openSession: (input) => |
| 306 | 352 | Effect.gen(function* () { |
| 353 | if (options.spawnBeforeOpen === true) { | |
| 354 | yield* countClose; | |
| 355 | if (options.hangSessionScopeClose === true) yield* hangClose; | |
| 356 | } | |
| 357 | if (options.mcpConfigs !== undefined && options.spawnBeforeOpen === true) { | |
| 358 | yield* Ref.update(options.mcpConfigs, (configs) => [ | |
| 359 | ...configs, | |
| 360 | McpProviderSession.readMcpProviderSession(input.threadId), | |
| 361 | ]); | |
| 362 | } | |
| 307 | 363 | if (options.beforeOpen !== undefined) { |
| 308 | 364 | yield* options.beforeOpen(input); |
| 309 | 365 | } |
| 310 | if (options.mcpConfigs !== undefined) { | |
| 366 | if (options.mcpConfigs !== undefined && options.spawnBeforeOpen !== true) { | |
| 311 | 367 | yield* Ref.update(options.mcpConfigs, (configs) => [ |
| 312 | 368 | ...configs, |
| 313 | 369 | McpProviderSession.readMcpProviderSession(input.threadId), |
| ... | ... | @@ -329,17 +385,9 @@ function makeProviderAdapter( |
| 329 | 385 | eventQueues, |
| 330 | 386 | }; |
| 331 | 387 | }); |
| 332 | yield* Effect.addFinalizer(() => | |
| 333 | Ref.update(state, (current) => ({ | |
| 334 | ...current, | |
| 335 | closeCount: current.closeCount + 1, | |
| 336 | })), | |
| 337 | ); | |
| 338 | if (options.hangSessionScopeClose === true) { | |
| 339 | // Registered last so it runs first on scope close, wedging the | |
| 340 | // close before the closeCount finalizer, like a provider process | |
| 341 | // that never yields its message stream. | |
| 342 | yield* Effect.addFinalizer(() => Effect.never); | |
| 388 | if (options.spawnBeforeOpen !== true) { | |
| 389 | yield* countClose; | |
| 390 | if (options.hangSessionScopeClose === true) yield* hangClose; | |
| 343 | 391 | } |
| 344 | 392 | |
| 345 | 393 | return { |
| ... | ... | @@ -365,7 +413,7 @@ function makeProviderAdapter( |
| 365 | 413 | ...current, |
| 366 | 414 | resumeCount: current.resumeCount + 1, |
| 367 | 415 | })).pipe(Effect.as(threadInput.providerThread)), |
| 368 | startTurn: () => Effect.void, | |
| 416 | startTurn: () => options.startTurn ?? Effect.void, | |
| 369 | 417 | steerTurn: () => Effect.void, |
| 370 | 418 | interruptTurn: () => |
| 371 | 419 | Ref.update(state, (current) => ({ |
| ... | ... | @@ -408,18 +456,29 @@ function layerTest(input: { |
| 408 | 456 | }) => Effect.Effect<void>; |
| 409 | 457 | readonly failReleaseEventWrites?: boolean; |
| 410 | 458 | readonly flakyReleaseWrites?: FlakyReleaseWrites; |
| 459 | readonly pauseAttachWrite?: Parameters<typeof layerPausingAttachEventSink>[0]; | |
| 460 | /** Once armed, holds the next credential lookup until it is interrupted. */ | |
| 461 | readonly pauseResolve?: { | |
| 462 | readonly armed: Ref.Ref<boolean>; | |
| 463 | readonly paused: Deferred.Deferred<void>; | |
| 464 | }; | |
| 411 | 465 | readonly hasPendingBackgroundWork?: Effect.Effect<boolean>; |
| 412 | 466 | readonly hangSessionScopeClose?: boolean; |
| 467 | readonly startTurn?: Effect.Effect<void>; | |
| 413 | 468 | readonly beforeUnload?: Effect.Effect<void>; |
| 469 | readonly spawnBeforeOpen?: boolean; | |
| 470 | readonly scopeCloseReached?: Deferred.Deferred<void>; | |
| 414 | 471 | readonly serverSettingsLayer?: ReturnType<typeof ServerSettings.layerTest>; |
| 415 | 472 | readonly projectServiceLayer?: Layer.Layer<ProjectService.ProjectService>; |
| 416 | 473 | }) { |
| 417 | 474 | const layerConfiguredEventSink = |
| 418 | 475 | input.flakyReleaseWrites !== undefined |
| 419 | 476 | ? layerFlakyReleaseEventSink(input.flakyReleaseWrites) |
| 420 | : input.failReleaseEventWrites | |
| 421 | ? layerFailingReleaseEventSink | |
| 422 | : layerTestEventSink; | |
| 477 | : input.pauseAttachWrite !== undefined | |
| 478 | ? layerPausingAttachEventSink(input.pauseAttachWrite) | |
| 479 | : input.failReleaseEventWrites | |
| 480 | ? layerFailingReleaseEventSink | |
| 481 | : layerTestEventSink; | |
| 423 | 482 | const layerRegistry = ProviderAdapterRegistry.layerSingle( |
| 424 | 483 | makeProviderAdapter(input.state, { |
| 425 | 484 | failEventStream: input.failEventStream ?? false, |
| ... | ... | @@ -432,9 +491,18 @@ function layerTest(input: { |
| 432 | 491 | ...(input.hangSessionScopeClose === undefined |
| 433 | 492 | ? {} |
| 434 | 493 | : { hangSessionScopeClose: input.hangSessionScopeClose }), |
| 494 | ...(input.startTurn === undefined ? {} : { startTurn: input.startTurn }), | |
| 435 | 495 | ...(input.beforeUnload === undefined ? {} : { beforeUnload: input.beforeUnload }), |
| 496 | ...(input.spawnBeforeOpen === undefined ? {} : { spawnBeforeOpen: input.spawnBeforeOpen }), | |
| 497 | ...(input.scopeCloseReached === undefined | |
| 498 | ? {} | |
| 499 | : { scopeCloseReached: input.scopeCloseReached }), | |
| 436 | 500 | }), |
| 437 | 501 | ); |
| 502 | const layerConfiguredMcpRegistry = | |
| 503 | input.pauseResolve === undefined | |
| 504 | ? layerTestMcpRegistry | |
| 505 | : layerPausingMcpRegistry(input.pauseResolve); | |
| 438 | 506 | const layerProviderEventIngestorTest = ProviderEventIngestor.layer.pipe( |
| 439 | 507 | Layer.provide( |
| 440 | 508 | Layer.mergeAll( |
| ... | ... | @@ -449,7 +517,7 @@ function layerTest(input: { |
| 449 | 517 | layerTestStores, |
| 450 | 518 | layerConfiguredEventSink, |
| 451 | 519 | IdAllocator.layer, |
| 452 | layerTestMcpRegistry, | |
| 520 | layerConfiguredMcpRegistry, | |
| 453 | 521 | ProviderSessionManager.layerWithOptions({ |
| 454 | 522 | idleTimeoutMs: input.idleTimeoutMs, |
| 455 | 523 | ...(input.maxIdlePinMs === undefined ? {} : { maxIdlePinMs: input.maxIdlePinMs }), |
| ... | ... | @@ -460,7 +528,7 @@ function layerTest(input: { |
| 460 | 528 | layerConfiguredEventSink, |
| 461 | 529 | IdAllocator.layer, |
| 462 | 530 | layerProviderEventIngestorTest, |
| 463 | layerTestMcpRegistry, | |
| 531 | layerConfiguredMcpRegistry, | |
| 464 | 532 | layerTestStores, |
| 465 | 533 | ...(input.serverSettingsLayer === undefined ? [] : [input.serverSettingsLayer]), |
| 466 | 534 | ...(input.projectServiceLayer === undefined ? [] : [input.projectServiceLayer]), |
| ... | ... | @@ -489,6 +557,28 @@ const layerTestMcpRegistry = Layer.effect( |
| 489 | 557 | Layer.provide(NodeServices.layer), |
| 490 | 558 | ); |
| 491 | 559 | |
| 560 | const layerPausingMcpRegistry = (pause: { | |
| 561 | readonly armed: Ref.Ref<boolean>; | |
| 562 | readonly paused: Deferred.Deferred<void>; | |
| 563 | }) => | |
| 564 | Layer.effect( | |
| 565 | McpSessionRegistry.McpSessionRegistry, | |
| 566 | Effect.gen(function* () { | |
| 567 | const delegate = yield* McpSessionRegistry.McpSessionRegistry; | |
| 568 | return McpSessionRegistry.McpSessionRegistry.of({ | |
| 569 | ...delegate, | |
| 570 | resolve: (rawToken) => | |
| 571 | Ref.getAndSet(pause.armed, false).pipe( | |
| 572 | Effect.flatMap((armed) => | |
| 573 | armed | |
| 574 | ? Deferred.succeed(pause.paused, undefined).pipe(Effect.andThen(Effect.never)) | |
| 575 | : delegate.resolve(rawToken), | |
| 576 | ), | |
| 577 | ), | |
| 578 | }); | |
| 579 | }), | |
| 580 | ).pipe(Layer.provide(layerTestMcpRegistry)); | |
| 581 | ||
| 492 | 582 | function makeBrowserAccessProject(projectId: ProjectId): Project { |
| 493 | 583 | return { |
| 494 | 584 | id: projectId, |
| ... | ... | @@ -997,6 +1087,498 @@ it.effect("ProviderSessionManagerV2 opens a duplicate session only once", () => |
| 997 | 1087 | }), |
| 998 | 1088 | ); |
| 999 | 1089 | |
| 1090 | it.effect("ProviderSessionManagerV2 cleans up an open interrupted mid-handshake", () => | |
| 1091 | Effect.gen(function* () { | |
| 1092 | const state = yield* Ref.make(emptyState); | |
| 1093 | const mcpConfigs = yield* Ref.make< | |
| 1094 | ReadonlyArray<McpProviderSession.McpProviderSessionConfig | undefined> | |
| 1095 | >([]); | |
| 1096 | const handshakeStarted = yield* Deferred.make<void>(); | |
| 1097 | const holdHandshake = yield* Ref.make(true); | |
| 1098 | const effect = Effect.gen(function* () { | |
| 1099 | const eventSink = yield* EventSink.EventSinkV2; | |
| 1100 | const idAllocator = yield* IdAllocator.IdAllocatorV2; | |
| 1101 | const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; | |
| 1102 | const registry = yield* McpSessionRegistry.McpSessionRegistry; | |
| 1103 | const projectionStore = yield* ProjectionStore.ProjectionStoreV2; | |
| 1104 | const now = yield* DateTime.now; | |
| 1105 | const threadId = ThreadId.make("thread-provider-session-manager-interrupted-open"); | |
| 1106 | const providerSessionId = yield* idAllocator.allocate.providerSession({ | |
| 1107 | providerInstanceId: modelSelection.instanceId, | |
| 1108 | threadId, | |
| 1109 | }); | |
| 1110 | yield* eventSink.write({ | |
| 1111 | events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], | |
| 1112 | }); | |
| 1113 | ||
| 1114 | const opening = yield* manager | |
| 1115 | .open({ threadId, providerSessionId, modelSelection, runtimePolicy }) | |
| 1116 | .pipe(Effect.forkScoped); | |
| 1117 | yield* Deferred.await(handshakeStarted); | |
| 1118 | const issued = (yield* Ref.get(mcpConfigs)).at(-1); | |
| 1119 | const token = issued?.authorizationHeader.replace(/^Bearer\s+/, ""); | |
| 1120 | assert.isDefined(token); | |
| 1121 | assert.isDefined(yield* registry.resolve(token!)); | |
| 1122 | ||
| 1123 | // A Stop while the provider is still starting. | |
| 1124 | yield* Fiber.interrupt(opening); | |
| 1125 | ||
| 1126 | // The process started for this open is stopped, and the credential minted | |
| 1127 | // for it revoked. | |
| 1128 | assert.equal((yield* Ref.get(state)).closeCount, 1); | |
| 1129 | assert.isUndefined(yield* registry.resolve(token!)); | |
| 1130 | assert.isUndefined(McpProviderSession.readMcpProviderSession(threadId)); | |
| 1131 | assert.isTrue(Option.isNone(yield* manager.get(providerSessionId))); | |
| 1132 | ||
| 1133 | // Nothing of the interrupted open is left behind: the next open starts a | |
| 1134 | // fresh process with a fresh credential that a later release revokes, | |
| 1135 | // which a leaked reservation would prevent. | |
| 1136 | yield* Ref.set(holdHandshake, false); | |
| 1137 | yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy }); | |
| 1138 | const replacement = (yield* Ref.get(mcpConfigs)).at(-1); | |
| 1139 | const replacementToken = replacement?.authorizationHeader.replace(/^Bearer\s+/, ""); | |
| 1140 | assert.isDefined(replacementToken); | |
| 1141 | assert.notEqual(replacementToken, token); | |
| 1142 | const projection = yield* projectionStore.getThreadProjection(threadId); | |
| 1143 | assert.equal(projection.providerSessions.at(-1)?.status, "ready"); | |
| 1144 | ||
| 1145 | yield* manager.close(providerSessionId); | |
| 1146 | assert.isUndefined(yield* registry.resolve(replacementToken!)); | |
| 1147 | }); | |
| 1148 | ||
| 1149 | yield* effect.pipe( | |
| 1150 | Effect.provide( | |
| 1151 | layerTest({ | |
| 1152 | state, | |
| 1153 | idleTimeoutMs: 60_000, | |
| 1154 | mcpConfigs, | |
| 1155 | beforeOpen: () => | |
| 1156 | Ref.get(holdHandshake).pipe( | |
| 1157 | Effect.flatMap((hold) => | |
| 1158 | hold | |
| 1159 | ? Deferred.succeed(handshakeStarted, undefined).pipe(Effect.andThen(Effect.never)) | |
| 1160 | : Effect.void, | |
| 1161 | ), | |
| 1162 | ), | |
| 1163 | // The process is spawned before the handshake that is interrupted. | |
| 1164 | spawnBeforeOpen: true, | |
| 1165 | }), | |
| 1166 | ), | |
| 1167 | ); | |
| 1168 | }), | |
| 1169 | ); | |
| 1170 | ||
| 1171 | it.effect("ProviderSessionManagerV2 cleans up an interrupted open whose scope close hangs", () => | |
| 1172 | Effect.gen(function* () { | |
| 1173 | const state = yield* Ref.make(emptyState); | |
| 1174 | const mcpConfigs = yield* Ref.make< | |
| 1175 | ReadonlyArray<McpProviderSession.McpProviderSessionConfig | undefined> | |
| 1176 | >([]); | |
| 1177 | const handshakeStarted = yield* Deferred.make<void>(); | |
| 1178 | const scopeCloseReached = yield* Deferred.make<void>(); | |
| 1179 | const releaseRetry = yield* Deferred.make<void>(); | |
| 1180 | yield* Effect.gen(function* () { | |
| 1181 | const eventSink = yield* EventSink.EventSinkV2; | |
| 1182 | const idAllocator = yield* IdAllocator.IdAllocatorV2; | |
| 1183 | const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; | |
| 1184 | const registry = yield* McpSessionRegistry.McpSessionRegistry; | |
| 1185 | const now = yield* DateTime.now; | |
| 1186 | const threadId = ThreadId.make("thread-provider-session-manager-interrupted-hung-open"); | |
| 1187 | const providerSessionId = yield* idAllocator.allocate.providerSession({ | |
| 1188 | providerInstanceId: modelSelection.instanceId, | |
| 1189 | threadId, | |
| 1190 | }); | |
| 1191 | yield* eventSink.write({ | |
| 1192 | events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], | |
| 1193 | }); | |
| 1194 | ||
| 1195 | const opening = yield* manager | |
| 1196 | .open({ threadId, providerSessionId, modelSelection, runtimePolicy }) | |
| 1197 | .pipe(Effect.forkChild); | |
| 1198 | yield* Deferred.await(handshakeStarted); | |
| 1199 | const issued = (yield* Ref.get(mcpConfigs)).at(-1); | |
| 1200 | const token = issued?.authorizationHeader.replace(/^Bearer\s+/, ""); | |
| 1201 | assert.isDefined(token); | |
| 1202 | ||
| 1203 | // The interrupt starts cleanup; the scope close then never finishes. | |
| 1204 | const interrupter = yield* Fiber.interrupt(opening).pipe(Effect.forkChild); | |
| 1205 | yield* Deferred.await(scopeCloseReached); | |
| 1206 | ||
| 1207 | // The session cleanup already ran, ahead of the stuck close. | |
| 1208 | assert.isUndefined(yield* registry.resolve(token!)); | |
| 1209 | assert.isUndefined(McpProviderSession.readMcpProviderSession(threadId)); | |
| 1210 | assert.equal((yield* Ref.get(state)).closeCount, 0); | |
| 1211 | ||
| 1212 | // The close is time-boxed, so the interrupter returns and the session's | |
| 1213 | // open lock is free for the next open. | |
| 1214 | yield* TestClock.adjust("30 seconds"); | |
| 1215 | yield* Fiber.join(interrupter); | |
| 1216 | yield* Deferred.succeed(releaseRetry, undefined); | |
| 1217 | yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy }); | |
| 1218 | assert.equal((yield* Ref.get(state)).openCount, 1); | |
| 1219 | ||
| 1220 | // Its close hangs as well; release it while the test clock can still move. | |
| 1221 | const stopping = yield* manager.shutdown.pipe(Effect.forkChild); | |
| 1222 | yield* TestClock.adjust("30 seconds"); | |
| 1223 | yield* Fiber.join(stopping); | |
| 1224 | }).pipe( | |
| 1225 | Effect.provide( | |
| 1226 | layerTest({ | |
| 1227 | state, | |
| 1228 | idleTimeoutMs: 60_000, | |
| 1229 | mcpConfigs, | |
| 1230 | // The first open hangs in its handshake; the retry completes. | |
| 1231 | beforeOpen: () => | |
| 1232 | Deferred.isDone(handshakeStarted).pipe( | |
| 1233 | Effect.flatMap((retry) => | |
| 1234 | retry | |
| 1235 | ? Deferred.await(releaseRetry) | |
| 1236 | : Deferred.succeed(handshakeStarted, undefined).pipe( | |
| 1237 | Effect.andThen(Effect.never), | |
| 1238 | ), | |
| 1239 | ), | |
| 1240 | ), | |
| 1241 | spawnBeforeOpen: true, | |
| 1242 | hangSessionScopeClose: true, | |
| 1243 | scopeCloseReached, | |
| 1244 | }), | |
| 1245 | ), | |
| 1246 | ); | |
| 1247 | }), | |
| 1248 | ); | |
| 1249 | ||
| 1250 | it.effect("ProviderSessionManagerV2 releases an idle session whose turn start was stopped", () => | |
| 1251 | Effect.gen(function* () { | |
| 1252 | const state = yield* Ref.make(emptyState); | |
| 1253 | const startTurnReached = yield* Deferred.make<void>(); | |
| 1254 | yield* Effect.gen(function* () { | |
| 1255 | const eventSink = yield* EventSink.EventSinkV2; | |
| 1256 | const idAllocator = yield* IdAllocator.IdAllocatorV2; | |
| 1257 | const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; | |
| 1258 | const projectionStore = yield* ProjectionStore.ProjectionStoreV2; | |
| 1259 | const now = yield* DateTime.now; | |
| 1260 | const threadId = ThreadId.make("thread-provider-session-manager-stopped-turn-start"); | |
| 1261 | const providerSessionId = yield* idAllocator.allocate.providerSession({ | |
| 1262 | providerInstanceId: modelSelection.instanceId, | |
| 1263 | threadId, | |
| 1264 | }); | |
| 1265 | yield* eventSink.write({ | |
| 1266 | events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], | |
| 1267 | }); | |
| 1268 | const runtime = yield* manager.open({ | |
| 1269 | threadId, | |
| 1270 | providerSessionId, | |
| 1271 | modelSelection, | |
| 1272 | runtimePolicy, | |
| 1273 | }); | |
| 1274 | const runId = idAllocator.derive.run({ threadId, ordinal: 1 }); | |
| 1275 | const starting = yield* runtime | |
| 1276 | .startTurn({ | |
| 1277 | appThread: (yield* projectionStore.getThreadProjection(threadId)).thread, | |
| 1278 | threadId, | |
| 1279 | runId, | |
| 1280 | runOrdinal: 1, | |
| 1281 | providerTurnOrdinal: 1, | |
| 1282 | attemptId: idAllocator.derive.runAttempt({ runId, attemptOrdinal: 1 }), | |
| 1283 | rootNodeId: idAllocator.derive.rootNode({ runId }), | |
| 1284 | providerThread: makeProviderThread({ idAllocator, threadId, providerSessionId, now }), | |
| 1285 | message: { | |
| 1286 | createdBy: "user", | |
| 1287 | creationSource: "web", | |
| 1288 | messageId: yield* idAllocator.allocate.message({ threadId, ordinal: 1 }), | |
| 1289 | text: "stopped before the provider accepted it", | |
| 1290 | attachments: [], | |
| 1291 | }, | |
| 1292 | modelSelection, | |
| 1293 | runtimePolicy, | |
| 1294 | }) | |
| 1295 | .pipe(Effect.forkChild); | |
| 1296 | yield* Deferred.await(startTurnReached); | |
| 1297 | ||
| 1298 | // Stop lands while the provider is still accepting the turn. No terminal | |
| 1299 | // event follows, so the session must count itself idle again. | |
| 1300 | yield* Fiber.interrupt(starting); | |
| 1301 | yield* TestClock.adjust("2 seconds"); | |
| 1302 | yield* Effect.yieldNow; | |
| 1303 | assert.isTrue(Option.isNone(yield* manager.get(providerSessionId))); | |
| 1304 | assert.equal((yield* Ref.get(state)).closeCount, 1); | |
| 1305 | }).pipe( | |
| 1306 | Effect.provide( | |
| 1307 | layerTest({ | |
| 1308 | state, | |
| 1309 | idleTimeoutMs: 1000, | |
| 1310 | startTurn: Deferred.succeed(startTurnReached, undefined).pipe( | |
| 1311 | Effect.andThen(Effect.never), | |
| 1312 | ), | |
| 1313 | }), | |
| 1314 | ), | |
| 1315 | ); | |
| 1316 | }), | |
| 1317 | ); | |
| 1318 | ||
| 1319 | it.effect( | |
| 1320 | "ProviderSessionManagerV2 keeps a shared session busy when another thread's start is stopped early", | |
| 1321 | () => | |
| 1322 | Effect.gen(function* () { | |
| 1323 | const state = yield* Ref.make(emptyState); | |
| 1324 | const projectId = ProjectId.make("project-provider-session-manager-shared-stopped-start"); | |
| 1325 | // Blocks the next project read, so a re-attach can be stopped before its | |
| 1326 | // start marks the session busy. | |
| 1327 | const holdProjectRead = yield* Ref.make(false); | |
| 1328 | const projectReadHeld = yield* Deferred.make<void>(); | |
| 1329 | const projectServiceLayer = Layer.mock(ProjectService.ProjectService)({ | |
| 1330 | getById: (requestedProjectId) => | |
| 1331 | Ref.get(holdProjectRead).pipe( | |
| 1332 | Effect.flatMap((hold) => | |
| 1333 | hold | |
| 1334 | ? Deferred.succeed(projectReadHeld, undefined).pipe(Effect.andThen(Effect.never)) | |
| 1335 | : Effect.succeed(Option.some(makeBrowserAccessProject(requestedProjectId))), | |
| 1336 | ), | |
| 1337 | ), | |
| 1338 | }); | |
| 1339 | yield* Effect.gen(function* () { | |
| 1340 | const eventSink = yield* EventSink.EventSinkV2; | |
| 1341 | const idAllocator = yield* IdAllocator.IdAllocatorV2; | |
| 1342 | const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; | |
| 1343 | const projectionStore = yield* ProjectionStore.ProjectionStoreV2; | |
| 1344 | const now = yield* DateTime.now; | |
| 1345 | const threadA = ThreadId.make("thread-provider-session-manager-shared-stopped-a"); | |
| 1346 | const threadB = ThreadId.make("thread-provider-session-manager-shared-stopped-b"); | |
| 1347 | const providerSessionId = idAllocator.derive.providerSession({ | |
| 1348 | providerInstanceId: modelSelection.instanceId, | |
| 1349 | }); | |
| 1350 | yield* eventSink.write({ | |
| 1351 | events: [ | |
| 1352 | yield* makeThreadCreatedEvent({ idAllocator, threadId: threadA, now, projectId }), | |
| 1353 | yield* makeThreadCreatedEvent({ idAllocator, threadId: threadB, now, projectId }), | |
| 1354 | ], | |
| 1355 | }); | |
| 1356 | const runtime = yield* manager.open({ | |
| 1357 | threadId: threadA, | |
| 1358 | providerSessionId, | |
| 1359 | modelSelection, | |
| 1360 | runtimePolicy, | |
| 1361 | }); | |
| 1362 | yield* manager.open({ | |
| 1363 | threadId: threadB, | |
| 1364 | providerSessionId, | |
| 1365 | modelSelection, | |
| 1366 | runtimePolicy, | |
| 1367 | }); | |
| 1368 | const startTurn = (threadId: ThreadId) => | |
| 1369 | Effect.gen(function* () { | |
| 1370 | const runId = idAllocator.derive.run({ threadId, ordinal: 1 }); | |
| 1371 | return yield* runtime.startTurn({ | |
| 1372 | appThread: (yield* projectionStore.getThreadProjection(threadId)).thread, | |
| 1373 | threadId, | |
| 1374 | runId, | |
| 1375 | runOrdinal: 1, | |
| 1376 | providerTurnOrdinal: 1, | |
| 1377 | attemptId: idAllocator.derive.runAttempt({ runId, attemptOrdinal: 1 }), | |
| 1378 | rootNodeId: idAllocator.derive.rootNode({ runId }), | |
| 1379 | providerThread: makeProviderThread({ idAllocator, threadId, providerSessionId, now }), | |
| 1380 | message: { | |
| 1381 | createdBy: "user", | |
| 1382 | creationSource: "web", | |
| 1383 | messageId: yield* idAllocator.allocate.message({ threadId, ordinal: 1 }), | |
| 1384 | text: "turn", | |
| 1385 | attachments: [], | |
| 1386 | }, | |
| 1387 | modelSelection, | |
| 1388 | runtimePolicy, | |
| 1389 | }); | |
| 1390 | }); | |
| 1391 | ||
| 1392 | // B's turn is accepted and still running. | |
| 1393 | yield* startTurn(threadB); | |
| 1394 | ||
| 1395 | // A detaches, and its next start is stopped while re-attaching, before | |
| 1396 | // it marks the session busy. | |
| 1397 | yield* manager.detach({ providerSessionId, threadId: threadA }); | |
| 1398 | yield* Ref.set(holdProjectRead, true); | |
| 1399 | const startingA = yield* startTurn(threadA).pipe(Effect.forkChild); | |
| 1400 | yield* Deferred.await(projectReadHeld); | |
| 1401 | yield* Fiber.interrupt(startingA); | |
| 1402 | ||
| 1403 | // B's running turn keeps the session busy past the idle timeout. | |
| 1404 | yield* TestClock.adjust("2 seconds"); | |
| 1405 | yield* Effect.yieldNow; | |
| 1406 | assert.isTrue(Option.isSome(yield* manager.get(providerSessionId))); | |
| 1407 | assert.equal((yield* Ref.get(state)).closeCount, 0); | |
| 1408 | }).pipe( | |
| 1409 | Effect.provide( | |
| 1410 | layerTest({ | |
| 1411 | state, | |
| 1412 | idleTimeoutMs: 1000, | |
| 1413 | projectServiceLayer, | |
| 1414 | serverSettingsLayer: ServerSettings.layerTest({ | |
| 1415 | projectSettingsOverrides: { [projectId]: { enableAgentBrowserAccess: true } }, | |
| 1416 | }), | |
| 1417 | }), | |
| 1418 | ), | |
| 1419 | ); | |
| 1420 | }), | |
| 1421 | ); | |
| 1422 | ||
| 1423 | it.effect( | |
| 1424 | "ProviderSessionManagerV2 keeps a shared session busy when a stopped start still ends its turn", | |
| 1425 | () => | |
| 1426 | Effect.gen(function* () { | |
| 1427 | const state = yield* Ref.make(emptyState); | |
| 1428 | // A's adapter start installs its turn, then is stopped mid-request; like | |
| 1429 | // OpenCode2, the adapter still ends that turn with turn.terminal later. | |
| 1430 | const holdStart = yield* Ref.make(false); | |
| 1431 | const startHeld = yield* Deferred.make<void>(); | |
| 1432 | yield* Effect.gen(function* () { | |
| 1433 | const eventSink = yield* EventSink.EventSinkV2; | |
| 1434 | const idAllocator = yield* IdAllocator.IdAllocatorV2; | |
| 1435 | const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; | |
| 1436 | const projectionStore = yield* ProjectionStore.ProjectionStoreV2; | |
| 1437 | const now = yield* DateTime.now; | |
| 1438 | const threadA = ThreadId.make("thread-provider-session-manager-stopped-ended-a"); | |
| 1439 | const threadB = ThreadId.make("thread-provider-session-manager-stopped-ended-b"); | |
| 1440 | const providerSessionId = idAllocator.derive.providerSession({ | |
| 1441 | providerInstanceId: modelSelection.instanceId, | |
| 1442 | }); | |
| 1443 | yield* eventSink.write({ | |
| 1444 | events: [ | |
| 1445 | yield* makeThreadCreatedEvent({ idAllocator, threadId: threadA, now }), | |
| 1446 | yield* makeThreadCreatedEvent({ idAllocator, threadId: threadB, now }), | |
| 1447 | ], | |
| 1448 | }); | |
| 1449 | const runtime = yield* manager.open({ | |
| 1450 | threadId: threadA, | |
| 1451 | providerSessionId, | |
| 1452 | modelSelection, | |
| 1453 | runtimePolicy, | |
| 1454 | }); | |
| 1455 | yield* manager.open({ | |
| 1456 | threadId: threadB, | |
| 1457 | providerSessionId, | |
| 1458 | modelSelection, | |
| 1459 | runtimePolicy, | |
| 1460 | }); | |
| 1461 | const providerThreadOf = (threadId: ThreadId) => | |
| 1462 | makeProviderThread({ | |
| 1463 | idAllocator, | |
| 1464 | threadId, | |
| 1465 | providerSessionId, | |
| 1466 | now, | |
| 1467 | nativeThreadId: `native-${threadId}`, | |
| 1468 | }); | |
| 1469 | const startTurn = (threadId: ThreadId) => | |
| 1470 | Effect.gen(function* () { | |
| 1471 | const runId = idAllocator.derive.run({ threadId, ordinal: 1 }); | |
| 1472 | return yield* runtime.startTurn({ | |
| 1473 | appThread: (yield* projectionStore.getThreadProjection(threadId)).thread, | |
| 1474 | threadId, | |
| 1475 | runId, | |
| 1476 | runOrdinal: 1, | |
| 1477 | providerTurnOrdinal: 1, | |
| 1478 | attemptId: idAllocator.derive.runAttempt({ runId, attemptOrdinal: 1 }), | |
| 1479 | rootNodeId: idAllocator.derive.rootNode({ runId }), | |
| 1480 | providerThread: providerThreadOf(threadId), | |
| 1481 | message: { | |
| 1482 | createdBy: "user", | |
| 1483 | creationSource: "web", | |
| 1484 | messageId: yield* idAllocator.allocate.message({ threadId, ordinal: 1 }), | |
| 1485 | text: "turn", | |
| 1486 | attachments: [], | |
| 1487 | }, | |
| 1488 | modelSelection, | |
| 1489 | runtimePolicy, | |
| 1490 | }); | |
| 1491 | }); | |
| 1492 | ||
| 1493 | // B's turn is accepted and still running. | |
| 1494 | yield* startTurn(threadB); | |
| 1495 | ||
| 1496 | // A's start is stopped after the adapter began the turn. | |
| 1497 | yield* Ref.set(holdStart, true); | |
| 1498 | const startingA = yield* startTurn(threadA).pipe(Effect.forkChild); | |
| 1499 | yield* Deferred.await(startHeld); | |
| 1500 | yield* Fiber.interrupt(startingA); | |
| 1501 | ||
| 1502 | // The adapter ends A's turn anyway. | |
| 1503 | const queue = (yield* Ref.get(state)).eventQueues.get(String(providerSessionId)); | |
| 1504 | assert.isDefined(queue); | |
| 1505 | yield* Queue.offer(queue!, { | |
| 1506 | type: "turn.terminal", | |
| 1507 | driver: CODEX_DRIVER, | |
| 1508 | providerThreadId: providerThreadOf(threadA).id, | |
| 1509 | providerTurnId: idAllocator.derive.providerTurn({ | |
| 1510 | driver: CODEX_DRIVER, | |
| 1511 | nativeTurnId: "native-turn-stopped-a", | |
| 1512 | }), | |
| 1513 | runOrdinal: 1, | |
| 1514 | status: "completed", | |
| 1515 | failure: null, | |
| 1516 | threadDisposition: "reusable", | |
| 1517 | }); | |
| 1518 | ||
| 1519 | // B's running turn keeps the session busy past the idle timeout. | |
| 1520 | yield* TestClock.adjust("2 seconds"); | |
| 1521 | yield* Effect.yieldNow; | |
| 1522 | assert.isTrue(Option.isSome(yield* manager.get(providerSessionId))); | |
| 1523 | assert.equal((yield* Ref.get(state)).closeCount, 0); | |
| 1524 | }).pipe( | |
| 1525 | Effect.provide( | |
| 1526 | layerTest({ | |
| 1527 | state, | |
| 1528 | idleTimeoutMs: 1000, | |
| 1529 | startTurn: Ref.get(holdStart).pipe( | |
| 1530 | Effect.flatMap((hold) => | |
| 1531 | hold | |
| 1532 | ? Deferred.succeed(startHeld, undefined).pipe(Effect.andThen(Effect.never)) | |
| 1533 | : Effect.void, | |
| 1534 | ), | |
| 1535 | ), | |
| 1536 | }), | |
| 1537 | ), | |
| 1538 | ); | |
| 1539 | }), | |
| 1540 | ); | |
| 1541 | ||
| 1542 | it.effect("ProviderSessionManagerV2 stops a session still opening when its layer shuts down", () => | |
| 1543 | Effect.gen(function* () { | |
| 1544 | const state = yield* Ref.make(emptyState); | |
| 1545 | const handshakeStarted = yield* Deferred.make<void>(); | |
| 1546 | yield* Effect.gen(function* () { | |
| 1547 | const eventSink = yield* EventSink.EventSinkV2; | |
| 1548 | const idAllocator = yield* IdAllocator.IdAllocatorV2; | |
| 1549 | const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; | |
| 1550 | const now = yield* DateTime.now; | |
| 1551 | const threadId = ThreadId.make("thread-provider-session-manager-shutdown-during-open"); | |
| 1552 | const providerSessionId = yield* idAllocator.allocate.providerSession({ | |
| 1553 | providerInstanceId: modelSelection.instanceId, | |
| 1554 | threadId, | |
| 1555 | }); | |
| 1556 | yield* eventSink.write({ | |
| 1557 | events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], | |
| 1558 | }); | |
| 1559 | // Detached, like an open the layer does not own: only the session | |
| 1560 | // scope's parent can stop its process when the layer closes. | |
| 1561 | yield* manager | |
| 1562 | .open({ threadId, providerSessionId, modelSelection, runtimePolicy }) | |
| 1563 | .pipe(Effect.forkDetach); | |
| 1564 | yield* Deferred.await(handshakeStarted); | |
| 1565 | assert.equal((yield* Ref.get(state)).closeCount, 0); | |
| 1566 | }).pipe( | |
| 1567 | Effect.provide( | |
| 1568 | layerTest({ | |
| 1569 | state, | |
| 1570 | idleTimeoutMs: 60_000, | |
| 1571 | beforeOpen: () => | |
| 1572 | Deferred.succeed(handshakeStarted, undefined).pipe(Effect.andThen(Effect.never)), | |
| 1573 | spawnBeforeOpen: true, | |
| 1574 | }), | |
| 1575 | ), | |
| 1576 | ); | |
| 1577 | ||
| 1578 | assert.equal((yield* Ref.get(state)).closeCount, 1); | |
| 1579 | }), | |
| 1580 | ); | |
| 1581 | ||
| 1000 | 1582 | it.effect("ProviderSessionManagerV2 releases live sessions when its layer shuts down", () => |
| 1001 | 1583 | Effect.gen(function* () { |
| 1002 | 1584 | const state = yield* Ref.make(emptyState); |
| ... | ... | @@ -1368,6 +1950,228 @@ it.effect("ProviderSessionManagerV2 revokes MCP credentials when release persist |
| 1368 | 1950 | }), |
| 1369 | 1951 | ); |
| 1370 | 1952 | |
| 1953 | it.effect( | |
| 1954 | "ProviderSessionManagerV2 keeps a thread attached when an earlier attach of it is interrupted", | |
| 1955 | () => | |
| 1956 | Effect.gen(function* () { | |
| 1957 | const state = yield* Ref.make(emptyState); | |
| 1958 | const armed = yield* Ref.make(false); | |
| 1959 | const paused = yield* Deferred.make<void>(); | |
| 1960 | yield* Effect.gen(function* () { | |
| 1961 | const eventSink = yield* EventSink.EventSinkV2; | |
| 1962 | const idAllocator = yield* IdAllocator.IdAllocatorV2; | |
| 1963 | const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; | |
| 1964 | const registry = yield* McpSessionRegistry.McpSessionRegistry; | |
| 1965 | const now = yield* DateTime.now; | |
| 1966 | const owner = ThreadId.make("thread-provider-session-manager-attach-race-owner"); | |
| 1967 | const threadId = ThreadId.make("thread-provider-session-manager-attach-race"); | |
| 1968 | const providerSessionId = idAllocator.derive.providerSession({ | |
| 1969 | providerInstanceId: modelSelection.instanceId, | |
| 1970 | }); | |
| 1971 | yield* eventSink.write({ | |
| 1972 | events: [ | |
| 1973 | yield* makeThreadCreatedEvent({ idAllocator, threadId: owner, now }), | |
| 1974 | yield* makeThreadCreatedEvent({ idAllocator, threadId, now }), | |
| 1975 | ], | |
| 1976 | }); | |
| 1977 | const runtime = yield* manager.open({ | |
| 1978 | threadId: owner, | |
| 1979 | providerSessionId, | |
| 1980 | modelSelection, | |
| 1981 | runtimePolicy, | |
| 1982 | }); | |
| 1983 | const resume = runtime.resumeThread({ | |
| 1984 | threadId, | |
| 1985 | providerThread: makeProviderThread({ | |
| 1986 | idAllocator, | |
| 1987 | threadId, | |
| 1988 | providerSessionId, | |
| 1989 | now, | |
| 1990 | nativeThreadId: "native-attach-race", | |
| 1991 | }), | |
| 1992 | }); | |
| 1993 | ||
| 1994 | // The first attach of the thread is stopped after attaching it, while | |
| 1995 | // a second attach of the same thread is already on its way. | |
| 1996 | yield* Ref.set(armed, true); | |
| 1997 | const first = yield* resume.pipe(Effect.forkChild({ startImmediately: true })); | |
| 1998 | yield* Deferred.await(paused); | |
| 1999 | const second = yield* resume.pipe(Effect.forkChild({ startImmediately: true })); | |
| 2000 | yield* Fiber.interrupt(first); | |
| 2001 | yield* Fiber.join(second); | |
| 2002 | ||
| 2003 | // The second attach owns the thread's attachment and credential. | |
| 2004 | const config = McpProviderSession.readMcpProviderSession(threadId); | |
| 2005 | assert.isDefined(config); | |
| 2006 | const token = config!.authorizationHeader.replace(/^Bearer\s+/, ""); | |
| 2007 | assert.equal((yield* registry.resolve(token))?.thread.threadId, threadId); | |
| 2008 | }).pipe( | |
| 2009 | Effect.provide( | |
| 2010 | layerTest({ state, idleTimeoutMs: 60_000, pauseAttachWrite: { armed, paused } }), | |
| 2011 | ), | |
| 2012 | ); | |
| 2013 | }), | |
| 2014 | ); | |
| 2015 | ||
| 2016 | it.effect( | |
| 2017 | "ProviderSessionManagerV2 keeps a replacement session's attachment when a stale attach is interrupted", | |
| 2018 | () => | |
| 2019 | Effect.gen(function* () { | |
| 2020 | const state = yield* Ref.make(emptyState); | |
| 2021 | const armed = yield* Ref.make(false); | |
| 2022 | const paused = yield* Deferred.make<void>(); | |
| 2023 | yield* Effect.gen(function* () { | |
| 2024 | const eventSink = yield* EventSink.EventSinkV2; | |
| 2025 | const idAllocator = yield* IdAllocator.IdAllocatorV2; | |
| 2026 | const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; | |
| 2027 | const registry = yield* McpSessionRegistry.McpSessionRegistry; | |
| 2028 | const eventStore = yield* EventStore.EventStoreV2; | |
| 2029 | const now = yield* DateTime.now; | |
| 2030 | const owner = ThreadId.make("thread-provider-session-manager-stale-attach-owner"); | |
| 2031 | const threadId = ThreadId.make("thread-provider-session-manager-stale-attach"); | |
| 2032 | const providerSessionId = idAllocator.derive.providerSession({ | |
| 2033 | providerInstanceId: modelSelection.instanceId, | |
| 2034 | }); | |
| 2035 | yield* eventSink.write({ | |
| 2036 | events: [ | |
| 2037 | yield* makeThreadCreatedEvent({ idAllocator, threadId: owner, now }), | |
| 2038 | yield* makeThreadCreatedEvent({ idAllocator, threadId, now }), | |
| 2039 | ], | |
| 2040 | }); | |
| 2041 | const runtime = yield* manager.open({ | |
| 2042 | threadId: owner, | |
| 2043 | providerSessionId, | |
| 2044 | modelSelection, | |
| 2045 | runtimePolicy, | |
| 2046 | }); | |
| 2047 | ||
| 2048 | // An attach of the thread to this session stalls mid-write... | |
| 2049 | yield* Ref.set(armed, true); | |
| 2050 | const stale = yield* runtime | |
| 2051 | .resumeThread({ | |
| 2052 | threadId, | |
| 2053 | providerThread: makeProviderThread({ | |
| 2054 | idAllocator, | |
| 2055 | threadId, | |
| 2056 | providerSessionId, | |
| 2057 | now, | |
| 2058 | nativeThreadId: "native-stale-attach", | |
| 2059 | }), | |
| 2060 | }) | |
| 2061 | .pipe(Effect.forkChild({ startImmediately: true })); | |
| 2062 | yield* Deferred.await(paused); | |
| 2063 | ||
| 2064 | // ...while the session is replaced and the thread opens the new one. | |
| 2065 | yield* manager.release({ providerSessionId, reason: "runtime_error" }); | |
| 2066 | const replacement = yield* manager.open({ | |
| 2067 | threadId, | |
| 2068 | providerSessionId, | |
| 2069 | modelSelection, | |
| 2070 | runtimePolicy, | |
| 2071 | }); | |
| 2072 | const config = McpProviderSession.readMcpProviderSession(threadId); | |
| 2073 | assert.isDefined(config); | |
| 2074 | ||
| 2075 | yield* Fiber.interrupt(stale); | |
| 2076 | ||
| 2077 | // The replacement keeps the thread and its credential. | |
| 2078 | assert.equal( | |
| 2079 | McpProviderSession.readMcpProviderSession(threadId)?.providerSessionId, | |
| 2080 | config!.providerSessionId, | |
| 2081 | ); | |
| 2082 | const token = config!.authorizationHeader.replace(/^Bearer\s+/, ""); | |
| 2083 | assert.equal((yield* registry.resolve(token))?.thread.threadId, threadId); | |
| 2084 | // Still attached: resuming on the replacement does not attach the thread again. | |
| 2085 | const attachedEvents = eventStore | |
| 2086 | .read({ threadId, eventType: "provider-session.attached" }) | |
| 2087 | .pipe( | |
| 2088 | Stream.runCollect, | |
| 2089 | Effect.map((events) => events.length), | |
| 2090 | ); | |
| 2091 | const attachedBefore = yield* attachedEvents; | |
| 2092 | yield* replacement.resumeThread({ | |
| 2093 | threadId, | |
| 2094 | providerThread: makeProviderThread({ | |
| 2095 | idAllocator, | |
| 2096 | threadId, | |
| 2097 | providerSessionId, | |
| 2098 | now, | |
| 2099 | nativeThreadId: "native-stale-attach", | |
| 2100 | }), | |
| 2101 | }); | |
| 2102 | assert.equal(yield* attachedEvents, attachedBefore); | |
| 2103 | }).pipe( | |
| 2104 | Effect.provide( | |
| 2105 | layerTest({ state, idleTimeoutMs: 60_000, pauseAttachWrite: { armed, paused } }), | |
| 2106 | ), | |
| 2107 | ); | |
| 2108 | }), | |
| 2109 | ); | |
| 2110 | ||
| 2111 | it.effect( | |
| 2112 | "ProviderSessionManagerV2 revokes a reused credential after a resume stopped while checking it", | |
| 2113 | () => | |
| 2114 | Effect.gen(function* () { | |
| 2115 | const state = yield* Ref.make(emptyState); | |
| 2116 | const armed = yield* Ref.make(false); | |
| 2117 | const paused = yield* Deferred.make<void>(); | |
| 2118 | yield* Effect.gen(function* () { | |
| 2119 | const eventSink = yield* EventSink.EventSinkV2; | |
| 2120 | const idAllocator = yield* IdAllocator.IdAllocatorV2; | |
| 2121 | const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; | |
| 2122 | const registry = yield* McpSessionRegistry.McpSessionRegistry; | |
| 2123 | const now = yield* DateTime.now; | |
| 2124 | const owner = ThreadId.make("thread-provider-session-manager-resolve-stop-owner"); | |
| 2125 | const threadId = ThreadId.make("thread-provider-session-manager-resolve-stop"); | |
| 2126 | const providerSessionId = idAllocator.derive.providerSession({ | |
| 2127 | providerInstanceId: modelSelection.instanceId, | |
| 2128 | }); | |
| 2129 | yield* eventSink.write({ | |
| 2130 | events: [ | |
| 2131 | yield* makeThreadCreatedEvent({ idAllocator, threadId: owner, now }), | |
| 2132 | yield* makeThreadCreatedEvent({ idAllocator, threadId, now }), | |
| 2133 | ], | |
| 2134 | }); | |
| 2135 | const runtime = yield* manager.open({ | |
| 2136 | threadId: owner, | |
| 2137 | providerSessionId, | |
| 2138 | modelSelection, | |
| 2139 | runtimePolicy, | |
| 2140 | }); | |
| 2141 | const resume = runtime.resumeThread({ | |
| 2142 | threadId, | |
| 2143 | providerThread: makeProviderThread({ | |
| 2144 | idAllocator, | |
| 2145 | threadId, | |
| 2146 | providerSessionId, | |
| 2147 | now, | |
| 2148 | nativeThreadId: "native-resolve-stop", | |
| 2149 | }), | |
| 2150 | }); | |
| 2151 | // The thread gets a credential, then detaches and keeps it for a re-attach. | |
| 2152 | yield* resume; | |
| 2153 | yield* manager.detach({ providerSessionId, threadId }); | |
| 2154 | const config = McpProviderSession.readMcpProviderSession(threadId); | |
| 2155 | assert.isDefined(config); | |
| 2156 | const token = config!.authorizationHeader.replace(/^Bearer\s+/, ""); | |
| 2157 | ||
| 2158 | // A re-attach is stopped while it checks whether that credential is reusable. | |
| 2159 | yield* Ref.set(armed, true); | |
| 2160 | const stopped = yield* resume.pipe(Effect.forkChild({ startImmediately: true })); | |
| 2161 | yield* Deferred.await(paused); | |
| 2162 | yield* Fiber.interrupt(stopped); | |
| 2163 | ||
| 2164 | // Nothing holds the credential now, so a terminal release revokes it. | |
| 2165 | yield* manager.release({ providerSessionId, reason: "manual_shutdown" }); | |
| 2166 | assert.isUndefined(yield* registry.resolve(token)); | |
| 2167 | }).pipe( | |
| 2168 | Effect.provide( | |
| 2169 | layerTest({ state, idleTimeoutMs: 60_000, pauseResolve: { armed, paused } }), | |
| 2170 | ), | |
| 2171 | ); | |
| 2172 | }), | |
| 2173 | ); | |
| 2174 | ||
| 1371 | 2175 | it.effect("ProviderSessionManagerV2 duplicate detach preserves replacement MCP credentials", () => |
| 1372 | 2176 | Effect.gen(function* () { |
| 1373 | 2177 | const state = yield* Ref.make(emptyState); |
| ... | ... | @@ -3011,17 +3815,20 @@ it.effect( |
| 3011 | 3815 | providerInstanceId: modelSelection.instanceId, |
| 3012 | 3816 | threadId: firstThreadId, |
| 3013 | 3817 | }); |
| 3818 | // Each thread has its own native thread on the shared session. | |
| 3014 | 3819 | const firstProviderThread = makeProviderThread({ |
| 3015 | 3820 | idAllocator, |
| 3016 | 3821 | threadId: firstThreadId, |
| 3017 | 3822 | providerSessionId, |
| 3018 | 3823 | now, |
| 3824 | nativeThreadId: "native-thread-a", | |
| 3019 | 3825 | }); |
| 3020 | 3826 | const secondProviderThread = makeProviderThread({ |
| 3021 | 3827 | idAllocator, |
| 3022 | 3828 | threadId: secondThreadId, |
| 3023 | 3829 | providerSessionId, |
| 3024 | 3830 | now, |
| 3831 | nativeThreadId: "native-thread-b", | |
| 3025 | 3832 | }); |
| 3026 | 3833 | const firstRunId = idAllocator.derive.run({ threadId: firstThreadId, ordinal: 1 }); |
| 3027 | 3834 | const secondRunId = idAllocator.derive.run({ threadId: secondThreadId, ordinal: 1 }); |
apps/server/src/orchestration-v2/ProviderSessionManager.ts+219-102
| ... | ... | @@ -9,6 +9,7 @@ import { |
| 9 | 9 | ProviderInstanceId, |
| 10 | 10 | ProviderSessionId, |
| 11 | 11 | ThreadId, |
| 12 | type ProviderThreadId, | |
| 12 | 13 | } from "@t3tools/contracts"; |
| 13 | 14 | import * as Cause from "effect/Cause"; |
| 14 | 15 | import * as Clock from "effect/Clock"; |
| ... | ... | @@ -59,6 +60,10 @@ import * as ProjectionStore from "./ProjectionStore.ts"; |
| 59 | 60 | const DEFAULT_IDLE_TIMEOUT_MS = 30 * 60 * 1000; |
| 60 | 61 | const DEFAULT_MAX_IDLE_PIN_MS = 4 * 60 * 60 * 1000; |
| 61 | 62 | const RELEASE_SCOPE_CLOSE_TIMEOUT_MS = 30 * 1000; |
| 63 | ||
| 64 | /** The identity a turn's start and its `turn.terminal` share. */ | |
| 65 | const busyTurnKey = (providerThreadId: ProviderThreadId, runOrdinal: number) => | |
| 66 | `${providerThreadId}#${runOrdinal}`; | |
| 62 | 67 | const UNLOAD_THREAD_TIMEOUT_MS = 10 * 1000; |
| 63 | 68 | |
| 64 | 69 | export const ProviderSessionReleaseReason = Schema.Literals([ |
| ... | ... | @@ -213,7 +218,12 @@ interface LiveSessionEntry { |
| 213 | 218 | readonly requestEventPermit: Semaphore.Semaphore; |
| 214 | 219 | readonly scope: Scope.Closeable; |
| 215 | 220 | readonly idleGeneration: number; |
| 216 | readonly busyCount: number; | |
| 221 | /** | |
| 222 | * Turns this session is running, keyed by `busyTurnKey`. A turn's start adds | |
| 223 | * it and its `turn.terminal` (or a failed start) removes it, so a turn can | |
| 224 | * only clear itself and the session is idle when the set is empty. | |
| 225 | */ | |
| 226 | readonly busyTurns: ReadonlySet<string>; | |
| 217 | 227 | readonly lastActivityAtMs: number; |
| 218 | 228 | readonly idleFiber: Fiber.Fiber<void, never> | null; |
| 219 | 229 | /** Set when idle release is deferred for pending background work; bounds total deferral. */ |
| ... | ... | @@ -472,7 +482,17 @@ export const layerWithOptions = ( |
| 472 | 482 | // revoke the credential between validation and reservation. |
| 473 | 483 | reserveMcpCredential(threadId, existing.providerSessionId); |
| 474 | 484 | const rawToken = existing.authorizationHeader.replace(/^Bearer\s+/, ""); |
| 475 | const resolved = yield* mcpSessionRegistry.resolve(rawToken); | |
| 485 | // The caller only learns of the reservation once this returns, | |
| 486 | // so a stop while resolving must drop it here. | |
| 487 | const resolved = yield* mcpSessionRegistry | |
| 488 | .resolve(rawToken) | |
| 489 | .pipe( | |
| 490 | Effect.onInterrupt(() => | |
| 491 | Effect.sync(() => | |
| 492 | dropMcpCredentialReservation(threadId, existing.providerSessionId), | |
| 493 | ), | |
| 494 | ), | |
| 495 | ); | |
| 476 | 496 | if ( |
| 477 | 497 | resolved !== undefined && |
| 478 | 498 | resolved.thread.threadId === threadId && |
| ... | ... | @@ -842,7 +862,8 @@ export const layerWithOptions = ( |
| 842 | 862 | } |
| 843 | 863 | if ( |
| 844 | 864 | input.onlyIfIdleGeneration !== undefined && |
| 845 | (existing.busyCount > 0 || existing.idleGeneration !== input.onlyIfIdleGeneration) | |
| 865 | (existing.busyTurns.size > 0 || | |
| 866 | existing.idleGeneration !== input.onlyIfIdleGeneration) | |
| 846 | 867 | ) { |
| 847 | 868 | return ["kept", current] as const; |
| 848 | 869 | } |
| ... | ... | @@ -861,6 +882,45 @@ export const layerWithOptions = ( |
| 861 | 882 | ] as const; |
| 862 | 883 | }); |
| 863 | 884 | |
| 885 | // Scope close can wedge on a misbehaving adapter finalizer (e.g. a | |
| 886 | // provider process that never yields its message stream). Time-box it so | |
| 887 | // the caller, and any lock or worker it holds, moves on and leaves a | |
| 888 | // diagnosable trail. A close that finishes late is still logged. | |
| 889 | const closeScopeWithin = ( | |
| 890 | scope: Scope.Closeable, | |
| 891 | annotations: { readonly providerSessionId?: ProviderSessionId; readonly reason: string }, | |
| 892 | ) => | |
| 893 | Effect.gen(function* () { | |
| 894 | const closeFiber = yield* Scope.close(scope, Exit.void).pipe( | |
| 895 | Effect.exit, | |
| 896 | Effect.forkDetach({ startImmediately: true }), | |
| 897 | ); | |
| 898 | const closeExit = yield* Fiber.join(closeFiber).pipe( | |
| 899 | Effect.timeoutOption(RELEASE_SCOPE_CLOSE_TIMEOUT_MS), | |
| 900 | ); | |
| 901 | if (Option.isNone(closeExit)) { | |
| 902 | yield* Effect.logWarning("orchestration-v2.provider-session-scope-close-timeout", { | |
| 903 | ...annotations, | |
| 904 | timeoutMs: RELEASE_SCOPE_CLOSE_TIMEOUT_MS, | |
| 905 | }); | |
| 906 | yield* Fiber.join(closeFiber).pipe( | |
| 907 | Effect.flatMap((exit) => | |
| 908 | Exit.isFailure(exit) | |
| 909 | ? Effect.logWarning("orchestration-v2.provider-session-scope-close-failed", { | |
| 910 | ...annotations, | |
| 911 | cause: exit.cause, | |
| 912 | }) | |
| 913 | : Effect.logInfo( | |
| 914 | "orchestration-v2.provider-session-scope-close-completed-late", | |
| 915 | annotations, | |
| 916 | ), | |
| 917 | ), | |
| 918 | Effect.forkDetach, | |
| 919 | ); | |
| 920 | } | |
| 921 | return closeExit; | |
| 922 | }); | |
| 923 | ||
| 864 | 924 | const releaseEntry = (input: { |
| 865 | 925 | readonly providerSessionId: ProviderSessionId; |
| 866 | 926 | readonly reason: ProviderSessionReleaseReason; |
| ... | ... | @@ -889,49 +949,10 @@ export const layerWithOptions = ( |
| 889 | 949 | input.detail ?? `Provider session released: ${input.reason}.`, |
| 890 | 950 | ); |
| 891 | 951 | } |
| 892 | // Scope close can wedge on a misbehaving adapter finalizer | |
| 893 | // (e.g. a provider process that never yields its message | |
| 894 | // stream). Time-box it so release still persists released | |
| 895 | // events and leaves a diagnosable trail instead of silently | |
| 896 | // parking the session as "ready" forever. | |
| 897 | const closeFiber = yield* Scope.close(entry.scope, Exit.void).pipe( | |
| 898 | Effect.exit, | |
| 899 | Effect.forkDetach({ startImmediately: true }), | |
| 900 | ); | |
| 901 | const closeExit = yield* Fiber.join(closeFiber).pipe( | |
| 902 | Effect.timeoutOption(RELEASE_SCOPE_CLOSE_TIMEOUT_MS), | |
| 903 | ); | |
| 904 | if (Option.isNone(closeExit)) { | |
| 905 | yield* Effect.logWarning( | |
| 906 | "orchestration-v2.provider-session-scope-close-timeout", | |
| 907 | { | |
| 908 | providerSessionId: input.providerSessionId, | |
| 909 | reason: input.reason, | |
| 910 | timeoutMs: RELEASE_SCOPE_CLOSE_TIMEOUT_MS, | |
| 911 | }, | |
| 912 | ); | |
| 913 | yield* Fiber.join(closeFiber).pipe( | |
| 914 | Effect.flatMap((exit) => | |
| 915 | Exit.isFailure(exit) | |
| 916 | ? Effect.logWarning( | |
| 917 | "orchestration-v2.provider-session-scope-close-failed", | |
| 918 | { | |
| 919 | providerSessionId: input.providerSessionId, | |
| 920 | reason: input.reason, | |
| 921 | cause: exit.cause, | |
| 922 | }, | |
| 923 | ) | |
| 924 | : Effect.logInfo( | |
| 925 | "orchestration-v2.provider-session-scope-close-completed-late", | |
| 926 | { | |
| 927 | providerSessionId: input.providerSessionId, | |
| 928 | reason: input.reason, | |
| 929 | }, | |
| 930 | ), | |
| 931 | ), | |
| 932 | Effect.forkDetach, | |
| 933 | ); | |
| 934 | } | |
| 952 | const closeExit = yield* closeScopeWithin(entry.scope, { | |
| 953 | providerSessionId: input.providerSessionId, | |
| 954 | reason: input.reason, | |
| 955 | }); | |
| 935 | 956 | const records = { |
| 936 | 957 | entry, |
| 937 | 958 | reason: input.reason, |
| ... | ... | @@ -1022,7 +1043,7 @@ export const layerWithOptions = ( |
| 1022 | 1043 | const entry = current.get(key); |
| 1023 | 1044 | if ( |
| 1024 | 1045 | entry === undefined || |
| 1025 | entry.busyCount > 0 || | |
| 1046 | entry.busyTurns.size > 0 || | |
| 1026 | 1047 | entry.idleGeneration !== input.generation |
| 1027 | 1048 | ) { |
| 1028 | 1049 | return; |
| ... | ... | @@ -1044,7 +1065,7 @@ export const layerWithOptions = ( |
| 1044 | 1065 | const latestEntry = latest.get(key); |
| 1045 | 1066 | if ( |
| 1046 | 1067 | latestEntry === undefined || |
| 1047 | latestEntry.busyCount > 0 || | |
| 1068 | latestEntry.busyTurns.size > 0 || | |
| 1048 | 1069 | latestEntry.idleGeneration !== input.generation || |
| 1049 | 1070 | latestEntry.runtime !== probedRuntime |
| 1050 | 1071 | ) { |
| ... | ... | @@ -1076,7 +1097,7 @@ export const layerWithOptions = ( |
| 1076 | 1097 | } |
| 1077 | 1098 | // hasPendingBackgroundWork yields to the adapter, so the idle |
| 1078 | 1099 | // decision above can go stale; the generation guard revalidates |
| 1079 | // busyCount and idleGeneration inside releaseEntry's atomic | |
| 1100 | // busyTurns and idleGeneration inside releaseEntry's atomic | |
| 1080 | 1101 | // entry removal. |
| 1081 | 1102 | yield* releaseEntry({ |
| 1082 | 1103 | providerSessionId: input.providerSessionId, |
| ... | ... | @@ -1113,7 +1134,7 @@ export const layerWithOptions = ( |
| 1113 | 1134 | const key = sessionKey(providerSessionId); |
| 1114 | 1135 | const current = yield* Ref.get(sessions); |
| 1115 | 1136 | const entry = current.get(key); |
| 1116 | if (entry === undefined || entry.busyCount > 0) { | |
| 1137 | if (entry === undefined || entry.busyTurns.size > 0) { | |
| 1117 | 1138 | return; |
| 1118 | 1139 | } |
| 1119 | 1140 | |
| ... | ... | @@ -1126,7 +1147,7 @@ export const layerWithOptions = ( |
| 1126 | 1147 | const lastActivityAtMs = yield* Clock.currentTimeMillis; |
| 1127 | 1148 | yield* Ref.update(sessions, (latest) => { |
| 1128 | 1149 | const latestEntry = latest.get(key); |
| 1129 | if (latestEntry === undefined || latestEntry.busyCount > 0) { | |
| 1150 | if (latestEntry === undefined || latestEntry.busyTurns.size > 0) { | |
| 1130 | 1151 | return latest; |
| 1131 | 1152 | } |
| 1132 | 1153 | const updated = new Map(latest); |
| ... | ... | @@ -1164,6 +1185,7 @@ export const layerWithOptions = ( |
| 1164 | 1185 | }), |
| 1165 | 1186 | ); |
| 1166 | 1187 | |
| 1188 | /** Returns the runtime the thread was attached to, or undefined if it already was. */ | |
| 1167 | 1189 | const attachThread = (input: { |
| 1168 | 1190 | readonly providerSessionId: ProviderSessionId; |
| 1169 | 1191 | readonly threadId: ThreadId; |
| ... | ... | @@ -1173,25 +1195,34 @@ export const layerWithOptions = ( |
| 1173 | 1195 | Ref.modify(sessions, (current) => { |
| 1174 | 1196 | const entry = current.get(sessionKey(input.providerSessionId)); |
| 1175 | 1197 | if (entry === undefined || entry.attachedThreadIds.has(input.threadId)) { |
| 1176 | return [false, current] as const; | |
| 1198 | return [undefined, current] as const; | |
| 1177 | 1199 | } |
| 1178 | 1200 | const updated = new Map(current); |
| 1179 | 1201 | updated.set(sessionKey(input.providerSessionId), { |
| 1180 | 1202 | ...entry, |
| 1181 | 1203 | attachedThreadIds: new Set([...entry.attachedThreadIds, input.threadId]), |
| 1182 | 1204 | }); |
| 1183 | return [true, updated] as const; | |
| 1205 | return [entry.runtime, updated] as const; | |
| 1184 | 1206 | }), |
| 1185 | 1207 | ); |
| 1186 | 1208 | |
| 1209 | /** | |
| 1210 | * Undoes an attach to `runtime`. A replacement session that reopened under | |
| 1211 | * the same id since is left alone. | |
| 1212 | */ | |
| 1187 | 1213 | const removeThreadAttachment = (input: { |
| 1188 | 1214 | readonly providerSessionId: ProviderSessionId; |
| 1189 | 1215 | readonly threadId: ThreadId; |
| 1216 | readonly runtime: ProviderAdapterV2SessionRuntime; | |
| 1190 | 1217 | }) => |
| 1191 | 1218 | Ref.update(sessions, (current) => { |
| 1192 | 1219 | const key = sessionKey(input.providerSessionId); |
| 1193 | 1220 | const entry = current.get(key); |
| 1194 | if (entry === undefined || !entry.attachedThreadIds.has(input.threadId)) { | |
| 1221 | if ( | |
| 1222 | entry === undefined || | |
| 1223 | entry.runtime !== input.runtime || | |
| 1224 | !entry.attachedThreadIds.has(input.threadId) | |
| 1225 | ) { | |
| 1195 | 1226 | return current; |
| 1196 | 1227 | } |
| 1197 | 1228 | const attachedThreadIds = new Set(entry.attachedThreadIds); |
| ... | ... | @@ -1245,6 +1276,7 @@ export const layerWithOptions = ( |
| 1245 | 1276 | readonly providerInstanceId: ProviderInstanceId; |
| 1246 | 1277 | }) => |
| 1247 | 1278 | Effect.suspend(() => { |
| 1279 | let attachedTo: ProviderAdapterV2SessionRuntime | undefined; | |
| 1248 | 1280 | let preparedForCleanup: PreparedMcpCredential | undefined; |
| 1249 | 1281 | let reservationDropped = false; |
| 1250 | 1282 | const dropReservation = () => { |
| ... | ... | @@ -1253,12 +1285,17 @@ export const layerWithOptions = ( |
| 1253 | 1285 | dropMcpCredentialReservation(input.threadId, preparedForCleanup.mcpCredentialId); |
| 1254 | 1286 | } |
| 1255 | 1287 | }; |
| 1256 | return Effect.gen(function* () { | |
| 1257 | const attached = yield* threadAttachment.withLock( | |
| 1258 | threadAttachmentKey(input), | |
| 1259 | attachThread(input), | |
| 1288 | // The whole attach, including undoing a failed one, holds the | |
| 1289 | // thread's lock: a concurrent attach of the same thread waits, so it | |
| 1290 | // never sees an attachment that this call is about to roll back. | |
| 1291 | const attach = Effect.gen(function* () { | |
| 1292 | const attached = yield* attachThread(input).pipe( | |
| 1293 | // Recorded with no gap for an interrupt: cleanup undoes only an | |
| 1294 | // attach this call made, never one an earlier open made. | |
| 1295 | Effect.tap((runtime) => Effect.sync(() => (attachedTo = runtime))), | |
| 1296 | Effect.uninterruptible, | |
| 1260 | 1297 | ); |
| 1261 | if (attached) { | |
| 1298 | if (attached !== undefined) { | |
| 1262 | 1299 | const prepared = yield* prepareMcpSession(input.threadId, input.providerInstanceId); |
| 1263 | 1300 | preparedForCleanup = prepared; |
| 1264 | 1301 | if (prepared.mcpCredentialId !== undefined) { |
| ... | ... | @@ -1288,22 +1325,46 @@ export const layerWithOptions = ( |
| 1288 | 1325 | } |
| 1289 | 1326 | } |
| 1290 | 1327 | }).pipe( |
| 1291 | Effect.tapError(() => | |
| 1292 | removeThreadAttachment(input).pipe( | |
| 1293 | // Revoke only a credential this attach freshly minted: a REUSED | |
| 1294 | // credential is by definition held by another live provider | |
| 1295 | // process, and revoking it thread-wide would break that | |
| 1296 | // process's MCP client mid-conversation. | |
| 1297 | Effect.andThen( | |
| 1298 | Effect.suspend(() => { | |
| 1299 | dropReservation(); | |
| 1300 | return preparedForCleanup?.issued === true | |
| 1301 | ? clearMcpSession(input.threadId, preparedForCleanup.mcpCredentialId) | |
| 1302 | : Effect.void; | |
| 1303 | }), | |
| 1304 | ), | |
| 1305 | ), | |
| 1328 | // An interrupted attach is undone too, so the next attach writes | |
| 1329 | // the attachment instead of finding the thread already attached. | |
| 1330 | Effect.onError(() => | |
| 1331 | attachedTo === undefined | |
| 1332 | ? Effect.void | |
| 1333 | : removeThreadAttachment({ ...input, runtime: attachedTo }).pipe( | |
| 1334 | Effect.andThen( | |
| 1335 | Effect.suspend(() => { | |
| 1336 | dropReservation(); | |
| 1337 | // Revoke only a credential this attach freshly minted: a | |
| 1338 | // REUSED credential is by definition held by another | |
| 1339 | // live provider process, and revoking it thread-wide | |
| 1340 | // would break that process's MCP client mid-conversation. | |
| 1341 | if (preparedForCleanup?.issued !== true) return Effect.void; | |
| 1342 | const mcpCredentialId = preparedForCleanup.mcpCredentialId; | |
| 1343 | const attachedRuntime = attachedTo; | |
| 1344 | // As in release: a replacement session (or an open | |
| 1345 | // configuring one) may have taken the credential up. | |
| 1346 | return Ref.get(sessions).pipe( | |
| 1347 | Effect.flatMap((current) => | |
| 1348 | (mcpCredentialId !== undefined && | |
| 1349 | isMcpCredentialReserved(input.threadId, mcpCredentialId)) || | |
| 1350 | Array.from(current.values()).some( | |
| 1351 | (other) => | |
| 1352 | other.runtime !== attachedRuntime && | |
| 1353 | (other.attachedThreadIds.has(input.threadId) || | |
| 1354 | (mcpCredentialId !== undefined && | |
| 1355 | other.mcpCredentialIdByThread.get(input.threadId) === | |
| 1356 | mcpCredentialId)), | |
| 1357 | ) | |
| 1358 | ? Effect.void | |
| 1359 | : clearMcpSession(input.threadId, mcpCredentialId), | |
| 1360 | ), | |
| 1361 | ); | |
| 1362 | }), | |
| 1363 | ), | |
| 1364 | ), | |
| 1306 | 1365 | ), |
| 1366 | ); | |
| 1367 | return threadAttachment.withLock(threadAttachmentKey(input), attach).pipe( | |
| 1307 | 1368 | // The entry's own record (written above while the thread is |
| 1308 | 1369 | // attached) guards the credential from here on; the reservation |
| 1309 | 1370 | // is only needed until then. Ensuring covers defects/interrupts. |
| ... | ... | @@ -1311,7 +1372,7 @@ export const layerWithOptions = ( |
| 1311 | 1372 | ); |
| 1312 | 1373 | }); |
| 1313 | 1374 | |
| 1314 | const markBusy = (providerSessionId: ProviderSessionId) => | |
| 1375 | const markBusy = (providerSessionId: ProviderSessionId, turnKey: string) => | |
| 1315 | 1376 | withActivityError( |
| 1316 | 1377 | providerSessionId, |
| 1317 | 1378 | Effect.gen(function* () { |
| ... | ... | @@ -1325,7 +1386,7 @@ export const layerWithOptions = ( |
| 1325 | 1386 | const updated = new Map(current); |
| 1326 | 1387 | updated.set(key, { |
| 1327 | 1388 | ...entry, |
| 1328 | busyCount: entry.busyCount + 1, | |
| 1389 | busyTurns: new Set(entry.busyTurns).add(turnKey), | |
| 1329 | 1390 | idleFiber: null, |
| 1330 | 1391 | lastActivityAtMs: now, |
| 1331 | 1392 | pinnedSinceMs: null, |
| ... | ... | @@ -1336,7 +1397,10 @@ export const layerWithOptions = ( |
| 1336 | 1397 | }), |
| 1337 | 1398 | ); |
| 1338 | 1399 | |
| 1339 | const markIdle = (providerSessionId: ProviderSessionId) => | |
| 1400 | // Clearing a turn that is not marked busy (one whose failed start already | |
| 1401 | // cleared it, or a subagent turn the manager never started) only | |
| 1402 | // records activity. | |
| 1403 | const markIdle = (providerSessionId: ProviderSessionId, turnKey: string) => | |
| 1340 | 1404 | withActivityError( |
| 1341 | 1405 | providerSessionId, |
| 1342 | 1406 | Effect.gen(function* () { |
| ... | ... | @@ -1347,10 +1411,12 @@ export const layerWithOptions = ( |
| 1347 | 1411 | if (entry === undefined) { |
| 1348 | 1412 | return current; |
| 1349 | 1413 | } |
| 1414 | const busyTurns = new Set(entry.busyTurns); | |
| 1415 | busyTurns.delete(turnKey); | |
| 1350 | 1416 | const updated = new Map(current); |
| 1351 | 1417 | updated.set(key, { |
| 1352 | 1418 | ...entry, |
| 1353 | busyCount: Math.max(0, entry.busyCount - 1), | |
| 1419 | busyTurns, | |
| 1354 | 1420 | lastActivityAtMs: now, |
| 1355 | 1421 | }); |
| 1356 | 1422 | return updated; |
| ... | ... | @@ -1536,13 +1602,32 @@ export const layerWithOptions = ( |
| 1536 | 1602 | providerInstanceId: runtime.instanceId, |
| 1537 | 1603 | }), |
| 1538 | 1604 | ).pipe( |
| 1539 | Effect.andThen(observeActivity(providerSessionId, markBusy(providerSessionId))), | |
| 1605 | // A start that fails or is stopped may never emit turn.terminal, | |
| 1606 | // so it clears its own turn or the session never goes idle. If | |
| 1607 | // the adapter emits the terminal anyway, clearing the same turn | |
| 1608 | // again changes nothing, so another thread's turn on a shared | |
| 1609 | // session stays busy either way. | |
| 1540 | 1610 | Effect.andThen( |
| 1541 | runtime.startTurn(input).pipe(turnMetrics("send", input.modelSelection.model)), | |
| 1542 | ), | |
| 1543 | Effect.catch((error) => | |
| 1544 | observeActivity(providerSessionId, markIdle(providerSessionId)).pipe( | |
| 1545 | Effect.andThen(Effect.fail(error)), | |
| 1611 | Effect.acquireUseRelease( | |
| 1612 | observeActivity( | |
| 1613 | providerSessionId, | |
| 1614 | markBusy( | |
| 1615 | providerSessionId, | |
| 1616 | busyTurnKey(input.providerThread.id, input.runOrdinal), | |
| 1617 | ), | |
| 1618 | ), | |
| 1619 | () => | |
| 1620 | runtime.startTurn(input).pipe(turnMetrics("send", input.modelSelection.model)), | |
| 1621 | (_, exit) => | |
| 1622 | Exit.isFailure(exit) | |
| 1623 | ? observeActivity( | |
| 1624 | providerSessionId, | |
| 1625 | markIdle( | |
| 1626 | providerSessionId, | |
| 1627 | busyTurnKey(input.providerThread.id, input.runOrdinal), | |
| 1628 | ), | |
| 1629 | ) | |
| 1630 | : Effect.void, | |
| 1546 | 1631 | ), |
| 1547 | 1632 | ), |
| 1548 | 1633 | ), |
| ... | ... | @@ -1605,7 +1690,10 @@ export const layerWithOptions = ( |
| 1605 | 1690 | return observeActivity( |
| 1606 | 1691 | entry.runtime.providerSessionId, |
| 1607 | 1692 | event.type === "turn.terminal" |
| 1608 | ? markIdle(entry.runtime.providerSessionId) | |
| 1693 | ? markIdle( | |
| 1694 | entry.runtime.providerSessionId, | |
| 1695 | busyTurnKey(event.providerThreadId, event.runOrdinal), | |
| 1696 | ) | |
| 1609 | 1697 | : touchActivity(entry.runtime.providerSessionId), |
| 1610 | 1698 | ).pipe( |
| 1611 | 1699 | Effect.andThen( |
| ... | ... | @@ -1695,6 +1783,12 @@ export const layerWithOptions = ( |
| 1695 | 1783 | ); |
| 1696 | 1784 | }; |
| 1697 | 1785 | |
| 1786 | // Parent of every session scope. On layer close, shutdown releases the | |
| 1787 | // live sessions first, then closes any session whose open is still in | |
| 1788 | // flight, time-boxed so a stuck adapter cannot hold up server shutdown. | |
| 1789 | // Parallel, so one session whose close hangs does not stop the rest from | |
| 1790 | // closing within the time box. | |
| 1791 | const sessionScopes = yield* Scope.make("parallel"); | |
| 1698 | 1792 | const shutdown = Effect.gen(function* () { |
| 1699 | 1793 | const activeSessions = [...(yield* Ref.get(sessions)).values()]; |
| 1700 | 1794 | yield* Effect.forEach( |
| ... | ... | @@ -1714,7 +1808,11 @@ export const layerWithOptions = ( |
| 1714 | 1808 | { discard: true }, |
| 1715 | 1809 | ); |
| 1716 | 1810 | }); |
| 1717 | yield* Effect.addFinalizer(() => shutdown); | |
| 1811 | yield* Effect.addFinalizer(() => | |
| 1812 | shutdown.pipe( | |
| 1813 | Effect.ensuring(closeScopeWithin(sessionScopes, { reason: "server_shutdown" })), | |
| 1814 | ), | |
| 1815 | ); | |
| 1718 | 1816 | |
| 1719 | 1817 | return ProviderSessionManagerV2.of({ |
| 1720 | 1818 | shutdown, |
| ... | ... | @@ -1783,7 +1881,7 @@ export const layerWithOptions = ( |
| 1783 | 1881 | dropMcpCredentialReservation(input.threadId, mcpCredentialId); |
| 1784 | 1882 | } |
| 1785 | 1883 | }); |
| 1786 | const sessionScope = yield* Scope.make(); | |
| 1884 | const sessionScope = yield* Scope.fork(sessionScopes); | |
| 1787 | 1885 | const runtime = yield* adapter |
| 1788 | 1886 | .openSession({ |
| 1789 | 1887 | threadId: input.threadId, |
| ... | ... | @@ -1805,21 +1903,30 @@ export const layerWithOptions = ( |
| 1805 | 1903 | }) |
| 1806 | 1904 | .pipe( |
| 1807 | 1905 | Effect.provideService(Scope.Scope, sessionScope), |
| 1808 | Effect.tapError(() => | |
| 1809 | Scope.close(sessionScope, Exit.void).pipe( | |
| 1810 | Effect.ignore, | |
| 1811 | Effect.andThen(dropReservation), | |
| 1812 | // Revoke only a credential this open freshly minted: a | |
| 1813 | // reused credential is held by another live provider | |
| 1814 | // process and must survive this open's failure. | |
| 1906 | // Any failure, including a Stop that interrupts a slow | |
| 1907 | // handshake, stops the provider process this open started. | |
| 1908 | // The session cleanup runs first, and the close is | |
| 1909 | // time-boxed: this runs under the session's open lock, so an | |
| 1910 | // adapter finalizer that never finishes must not hold the | |
| 1911 | // interrupter, the lock, or later opens. | |
| 1912 | Effect.onError(() => | |
| 1913 | dropReservation.pipe( | |
| 1914 | // Clear only a session this open freshly set up: a reused | |
| 1915 | // one is held by another live provider process and must | |
| 1916 | // survive this open's failure. | |
| 1815 | 1917 | Effect.andThen( |
| 1816 | 1918 | prepared.issued |
| 1817 | 1919 | ? clearMcpSession(input.threadId, mcpCredentialId) |
| 1818 | 1920 | : Effect.void, |
| 1819 | 1921 | ), |
| 1922 | Effect.ensuring( | |
| 1923 | closeScopeWithin(sessionScope, { | |
| 1924 | providerSessionId: input.providerSessionId, | |
| 1925 | reason: "open_failed", | |
| 1926 | }), | |
| 1927 | ), | |
| 1820 | 1928 | ), |
| 1821 | 1929 | ), |
| 1822 | Effect.onInterrupt(() => dropReservation), | |
| 1823 | 1930 | Effect.mapError( |
| 1824 | 1931 | (cause) => |
| 1825 | 1932 | new ProviderSessionOpenError({ |
| ... | ... | @@ -1854,7 +1961,7 @@ export const layerWithOptions = ( |
| 1854 | 1961 | requestEventPermit: yield* Semaphore.make(1), |
| 1855 | 1962 | scope: sessionScope, |
| 1856 | 1963 | idleGeneration: 0, |
| 1857 | busyCount: 0, | |
| 1964 | busyTurns: new Set(), | |
| 1858 | 1965 | lastActivityAtMs: now, |
| 1859 | 1966 | idleFiber: null, |
| 1860 | 1967 | pinnedSinceMs: null, |
| ... | ... | @@ -1876,12 +1983,22 @@ export const layerWithOptions = ( |
| 1876 | 1983 | payload: runtime.providerSession, |
| 1877 | 1984 | }), |
| 1878 | 1985 | ).pipe( |
| 1879 | Effect.tapError(() => | |
| 1880 | releaseEntry({ | |
| 1881 | providerSessionId: input.providerSessionId, | |
| 1882 | reason: "runtime_error", | |
| 1883 | detail: "Failed to persist the provider-session attachment.", | |
| 1884 | }).pipe(logReleaseFailure(input.providerSessionId)), | |
| 1986 | // Released on interrupt too: this entry has no event pump or | |
| 1987 | // idle timer yet, so nothing else would ever release it. | |
| 1988 | Effect.onError((cause) => | |
| 1989 | releaseEntry( | |
| 1990 | Cause.hasInterruptsOnly(cause) | |
| 1991 | ? { | |
| 1992 | providerSessionId: input.providerSessionId, | |
| 1993 | reason: "manual_shutdown", | |
| 1994 | detail: "The provider session start was interrupted.", | |
| 1995 | } | |
| 1996 | : { | |
| 1997 | providerSessionId: input.providerSessionId, | |
| 1998 | reason: "runtime_error", | |
| 1999 | detail: "Failed to persist the provider-session attachment.", | |
| 2000 | }, | |
| 2001 | ).pipe(logReleaseFailure(input.providerSessionId)), | |
| 1885 | 2002 | ), |
| 1886 | 2003 | ); |
| 1887 | 2004 | yield* startEventPump(entry); |