| author |  Julius Marminge <julius0216@outlook.com> 2026-10-06 17:12:49-07:00 |
| committer |  GitHub <noreply@github.com> 2026-10-06 17:12:49-07:00 |
| log | 4b5c60485366421794787aee7513913f92eced29 |
| tree | 3e39f5eadad63939cd81d17dd07507a135dd889c |
| parent | 7202ba666080cfca1351c080d7aaa2d81bfa9a66 |
| signature | Signed by PGP key B5690EEEBB952194 |
fix(server): provider sessions clean up when their start is interrupted (#15571)
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>10 files changed, 1213 insertions(+), 210 deletions(-)
apps/server/src/device/LocalDeviceHost.ts+81-75
| ... | @@ -383,62 +383,63 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { | ... | @@ -383,62 +383,63 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { |
| 383 | ); | 383 | ); |
| 384 | const origin = `http://127.0.0.1:${port}`; | 384 | const origin = `http://127.0.0.1:${port}`; |
| 385 | const scope = yield* Scope.make("sequential"); | 385 | const scope = yield* Scope.make("sequential"); |
| 386 | const child = yield* spawner | 386 | // On failure or interrupt, from the spawn on: the hub is not recorded yet, |
| 387 | .spawn( | 387 | // so nothing else stops it. |
| 388 | ChildProcess.make( | 388 | return yield* Effect.gen(function* () { |
| 389 | nodePath, | 389 | const child = yield* spawner |
| 390 | [ | 390 | .spawn( |
| 391 | hubTool.entryPath, | 391 | ChildProcess.make( |
| 392 | "--port", | 392 | nodePath, |
| 393 | String(port), | 393 | [ |
| 394 | "--host", | 394 | hubTool.entryPath, |
| 395 | "127.0.0.1", | 395 | "--port", |
| 396 | "--hide-sidebar", | 396 | String(port), |
| 397 | "--hide-boot-device", | 397 | "--host", |
| 398 | ], | 398 | "127.0.0.1", |
| 399 | { | 399 | "--hide-sidebar", |
| 400 | detached: false, | 400 | "--hide-boot-device", |
| 401 | shell: false, | 401 | ], |
| 402 | stdout: "pipe", | 402 | { |
| 403 | stderr: "pipe", | 403 | detached: false, |
| 404 | env: yield* hubEnvironment(hostEnvironment).pipe( | 404 | shell: false, |
| 405 | Effect.provideService(FileSystem.FileSystem, fs), | 405 | stdout: "pipe", |
| 406 | Effect.provideService(HostProcessPlatform, hostPlatform), | 406 | stderr: "pipe", |
| 407 | ), | 407 | env: yield* hubEnvironment(hostEnvironment).pipe( |
| 408 | }, | 408 | Effect.provideService(FileSystem.FileSystem, fs), |
| 409 | ), | 409 | Effect.provideService(HostProcessPlatform, hostPlatform), |
| 410 | ) | 410 | ), |
| 411 | .pipe( | 411 | }, |
| 412 | Effect.provideService(Scope.Scope, scope), | 412 | ), |
| 413 | Effect.mapError( | 413 | ) |
| 414 | (cause) => | 414 | .pipe( |
| 415 | new DeviceHost.DeviceHostError({ | 415 | Effect.provideService(Scope.Scope, scope), |
| 416 | hostId, | 416 | Effect.mapError( |
| 417 | step: "starting the device hub", | 417 | (cause) => |
| 418 | cause, | 418 | new DeviceHost.DeviceHostError({ |
| 419 | }), | 419 | hostId, |
| 420 | ), | 420 | step: "starting the device hub", |
| 421 | ); | 421 | cause, |
| 422 | const startedAtMillis = yield* Clock.currentTimeMillis; | 422 | }), |
| 423 | const hub: HubProcess = { child, scope, origin, startedAtMillis, nodePath }; | 423 | ), |
| 424 | yield* Effect.forkIn(observeHubOutput(hub), scope); | 424 | ); |
| 425 | yield* waitForHttpReady({ | 425 | const startedAtMillis = yield* Clock.currentTimeMillis; |
| 426 | baseUrl: origin, | 426 | const hub: HubProcess = { child, scope, origin, startedAtMillis, nodePath }; |
| 427 | path: "/readyz", | 427 | yield* Effect.forkIn(observeHubOutput(hub), scope); |
| 428 | timeoutMs: HUB_READY_TIMEOUT_MS, | 428 | yield* waitForHttpReady({ |
| 429 | makeError: (info) => | 429 | baseUrl: origin, |
| 430 | new DeviceHost.DeviceHostError({ | 430 | path: "/readyz", |
| 431 | hostId, | 431 | timeoutMs: HUB_READY_TIMEOUT_MS, |
| 432 | step: "waiting for the device hub to answer", | 432 | makeError: (info) => |
| 433 | cause: info.cause, | 433 | new DeviceHost.DeviceHostError({ |
| 434 | }), | 434 | hostId, |
| 435 | }).pipe( | 435 | step: "waiting for the device hub to answer", |
| 436 | Effect.provideService(HttpClient.HttpClient, httpClient), | 436 | cause: info.cause, |
| 437 | Effect.tapError(() => stopHub(hub)), | 437 | }), |
| 438 | ); | 438 | }).pipe(Effect.provideService(HttpClient.HttpClient, httpClient)); |
| 439 | yield* recordHub(hub, hubTool); | 439 | yield* recordHub(hub, hubTool); |
| 440 | yield* Effect.logInfo("Device hub started", { pid: Number(child.pid), port }); | 440 | yield* Effect.logInfo("Device hub started", { pid: Number(child.pid), port }); |
| 441 | return hub; | 441 | return hub; |
| 442 | }).pipe(Effect.onError(() => Scope.close(scope, Exit.void).pipe(Effect.ignore))); | ||
| 442 | }); | 443 | }); |
| 443 | 444 | ||
| 444 | const observeHubOutput = (hub: HubProcess) => | 445 | const observeHubOutput = (hub: HubProcess) => |
| ... | @@ -622,25 +623,30 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { | ... | @@ -622,25 +623,30 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { |
| 622 | ); | 623 | ); |
| 623 | yield* onPhase("starting"); | 624 | yield* onPhase("starting"); |
| 624 | const hub = yield* spawnHub(hubTool, nodePath); | 625 | const hub = yield* spawnHub(hubTool, nodePath); |
| 625 | yield* pruneLocalDeviceTools(config.baseDir, nodePath, "hub").pipe( | 626 | // Until the hub is in runningRef, nothing else stops it, so publishing it |
| 626 | Effect.provideService(Path.Path, path), | 627 | // is part of the guarded step. |
| 627 | Effect.provideService(ProcessRunner.ProcessRunner, runner), | 628 | const next = yield* Effect.gen(function* () { |
| 628 | Effect.ignore, | 629 | yield* pruneLocalDeviceTools(config.baseDir, nodePath, "hub").pipe( |
| 629 | ); | 630 | Effect.provideService(Path.Path, path), |
| 630 | const candidate = helperPaths(hubTool); | 631 | Effect.provideService(ProcessRunner.ProcessRunner, runner), |
| 631 | const [axExists, cliExists] = yield* Effect.all([ | 632 | Effect.ignore, |
| 632 | fs.exists(candidate.serveSimAxSettings).pipe(Effect.orElseSucceed(() => false)), | 633 | ); |
| 633 | fs.exists(candidate.serveSimCli).pipe(Effect.orElseSucceed(() => false)), | 634 | const candidate = helperPaths(hubTool); |
| 634 | ]); | 635 | const [axExists, cliExists] = yield* Effect.all([ |
| 635 | const next: RunningHost = { | 636 | fs.exists(candidate.serveSimAxSettings).pipe(Effect.orElseSucceed(() => false)), |
| 636 | hub, | 637 | fs.exists(candidate.serveSimCli).pipe(Effect.orElseSucceed(() => false)), |
| 637 | agentDevice: null, | 638 | ]); |
| 638 | helpers: { | 639 | const next: RunningHost = { |
| 639 | serveSimAxSettings: axExists ? candidate.serveSimAxSettings : null, | 640 | hub, |
| 640 | serveSimCli: cliExists ? candidate.serveSimCli : null, | 641 | agentDevice: null, |
| 641 | }, | 642 | helpers: { |
| 642 | }; | 643 | serveSimAxSettings: axExists ? candidate.serveSimAxSettings : null, |
| 643 | yield* Ref.set(runningRef, next); | 644 | serveSimCli: cliExists ? candidate.serveSimCli : null, |
| 645 | }, | ||
| 646 | }; | ||
| 647 | yield* Ref.set(runningRef, next); | ||
| 648 | return next; | ||
| 649 | }).pipe(Effect.onError(() => stopHub(hub))); | ||
| 644 | yield* Ref.set(restartDelayRef, 0); | 650 | yield* Ref.set(restartDelayRef, 0); |
| 645 | yield* Effect.forkDetach(superviseHub(hub, hubTool)); | 651 | yield* Effect.forkDetach(superviseHub(hub, hubTool)); |
| 646 | return next; | 652 | return next; |
apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts+2-1
| ... | @@ -7148,7 +7148,8 @@ export function makeClaudeAdapterV2( | ... | @@ -7148,7 +7148,8 @@ export function makeClaudeAdapterV2( |
| 7148 | options: queryOptions, | 7148 | options: queryOptions, |
| 7149 | }) | 7149 | }) |
| 7150 | .pipe( | 7150 | .pipe( |
| 7151 | Effect.tapError(() => | 7151 | // An interrupted open leaves the old process just as dead. |
| 7152 | Effect.onError(() => | ||
| 7152 | // Same-native-thread replacement: the old process is already | 7153 | // Same-native-thread replacement: the old process is already |
| 7153 | // dead, so its process-scoped roster is not authoritative. | 7154 | // dead, so its process-scoped roster is not authoritative. |
| 7154 | // First-ever failed open (no prior live query) must not invent | 7155 | // First-ever failed open (no prior live query) must not invent |
apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts+3-1
| ... | @@ -6174,8 +6174,10 @@ export function makeCodexAdapterV2(adapterOptions: CodexAdapterV2Options): Provi | ... | @@ -6174,8 +6174,10 @@ export function makeCodexAdapterV2(adapterOptions: CodexAdapterV2Options): Provi |
| 6174 | yield* Ref.update(pendingRootTurns, (current) => | 6174 | yield* Ref.update(pendingRootTurns, (current) => |
| 6175 | new Map(current).set(threadId, turnInput), | 6175 | new Map(current).set(threadId, turnInput), |
| 6176 | ); | 6176 | ); |
| 6177 | // Cleared on interrupt too, as startTurn does: an interrupted start | ||
| 6178 | // must not adopt the native turn that a late turn/started reports. | ||
| 6177 | yield* client.request("thread/compact/start", { threadId }).pipe( | 6179 | yield* client.request("thread/compact/start", { threadId }).pipe( |
| 6178 | Effect.tapError(() => | 6180 | Effect.onError(() => |
| 6179 | Ref.update(pendingRootTurns, (current) => { | 6181 | Ref.update(pendingRootTurns, (current) => { |
| 6180 | const next = new Map(current); | 6182 | const next = new Map(current); |
| 6181 | next.delete(threadId); | 6183 | next.delete(threadId); |
apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts+19
| ... | @@ -428,6 +428,25 @@ describe("OpenCode2 adapter", () => { | ... | @@ -428,6 +428,25 @@ describe("OpenCode2 adapter", () => { |
| 428 | }).pipe(Effect.scoped), | 428 | }).pipe(Effect.scoped), |
| 429 | ); | 429 | ); |
| 430 | 430 | ||
| 431 | it.effect("ends a turn on the provider thread it started on", () => | ||
| 432 | Effect.gen(function* () { | ||
| 433 | const { runtime, thread } = yield* resumed([ | ||
| 434 | out("session.prompt", { sessionID: SESSION, text: "<any>" }), | ||
| 435 | promptAccepted, | ||
| 436 | event("session.execution.succeeded", { sessionID: SESSION }), | ||
| 437 | ]); | ||
| 438 | // A forked run starts on its own row for the same native session, while | ||
| 439 | // the adapter tracks the session under the id it minted for the fork. | ||
| 440 | const forkedRow = { | ||
| 441 | ...thread, | ||
| 442 | id: ProviderThreadId.make("provider-thread:opencode2-adapter:forked-run-row"), | ||
| 443 | }; | ||
| 444 | const terminal = yield* terminalOf(runtime).pipe(Effect.forkScoped); | ||
| 445 | yield* runtime.startTurn(turnInput(forkedRow)); | ||
| 446 | assert.equal((yield* Fiber.join(terminal))?.providerThreadId, forkedRow.id); | ||
| 447 | }).pipe(Effect.scoped), | ||
| 448 | ); | ||
| 449 | |||
| 431 | it.effect("ends the turn when its terminal event is one this build cannot decode", () => | 450 | it.effect("ends the turn when its terminal event is one this build cannot decode", () => |
| 432 | Effect.gen(function* () { | 451 | Effect.gen(function* () { |
| 433 | const { runtime, thread } = yield* resumed([ | 452 | const { runtime, thread } = yield* resumed([ |
apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.ts+10-6
| ... | @@ -1786,7 +1786,10 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr | ... | @@ -1786,7 +1786,10 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr |
| 1786 | const base = { | 1786 | const base = { |
| 1787 | type: "turn.terminal" as const, | 1787 | type: "turn.terminal" as const, |
| 1788 | driver, | 1788 | driver, |
| 1789 | providerThreadId: state.providerThread.id, | 1789 | // The provider thread the turn started on. After a native fork the |
| 1790 | // session's own thread has a fresh id, and the terminal must name the | ||
| 1791 | // thread the start was recorded against. | ||
| 1792 | providerThreadId: turn.providerTurn.providerThreadId, | ||
| 1790 | providerTurnId: turn.providerTurn.id, | 1793 | providerTurnId: turn.providerTurn.id, |
| 1791 | runOrdinal: turn.input.runOrdinal, | 1794 | runOrdinal: turn.input.runOrdinal, |
| 1792 | threadDisposition, | 1795 | threadDisposition, |
| ... | @@ -2825,12 +2828,13 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr | ... | @@ -2825,12 +2828,13 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr |
| 2825 | * reconciles. Retried a few times; the caller fails everything after that. | 2828 | * reconciles. Retried a few times; the caller fails everything after that. |
| 2826 | */ | 2829 | */ |
| 2827 | const reconnect = Effect.gen(function* () { | 2830 | const reconnect = Effect.gen(function* () { |
| 2831 | // Returned on any failure, including the interrupt a closing session | ||
| 2832 | // sends a reconnect still in flight, so a spawned server can idle-stop. | ||
| 2828 | const scope = yield* Scope.make(); | 2833 | const scope = yield* Scope.make(); |
| 2829 | const next = yield* borrow.pipe( | 2834 | const { next, stream } = yield* Effect.gen(function* () { |
| 2830 | Effect.provideService(Scope.Scope, scope), | 2835 | const next = yield* borrow.pipe(Effect.provideService(Scope.Scope, scope)); |
| 2831 | Effect.tapError(() => Scope.close(scope, Exit.void)), | 2836 | return { next, stream: yield* next.events }; |
| 2832 | ); | 2837 | }).pipe(Effect.onError(() => Scope.close(scope, Exit.void))); |
| 2833 | const stream = yield* next.events.pipe(Effect.tapError(() => Scope.close(scope, Exit.void))); | ||
| 2834 | const previous = currentScope; | 2838 | const previous = currentScope; |
| 2835 | connection = next; | 2839 | connection = next; |
| 2836 | client = next.client; | 2840 | client = next.client; |
apps/server/src/orchestration-v2/Adapters/OpenCodeAdapterV2.test.ts+44-2
| ... | @@ -182,7 +182,10 @@ const makeOpenCodeRuntimeHarness = Effect.fn("makeOpenCodeRuntimeHarness")(funct | ... | @@ -182,7 +182,10 @@ const makeOpenCodeRuntimeHarness = Effect.fn("makeOpenCodeRuntimeHarness")(funct |
| 182 | runtimePolicy: policy, | 182 | runtimePolicy: policy, |
| 183 | }); | 183 | }); |
| 184 | const now = yield* DateTime.now; | 184 | const now = yield* DateTime.now; |
| 185 | const startTurn = (text = "hello") => | 185 | const startTurn = ( |
| 186 | text = "hello", | ||
| 187 | startProviderThread: OrchestrationV2ProviderThread = providerThread, | ||
| 188 | ) => | ||
| 186 | runtime.startTurn({ | 189 | runtime.startTurn({ |
| 187 | appThread: { | 190 | appThread: { |
| 188 | id: threadId, | 191 | id: threadId, |
| ... | @@ -213,7 +216,7 @@ const makeOpenCodeRuntimeHarness = Effect.fn("makeOpenCodeRuntimeHarness")(funct | ... | @@ -213,7 +216,7 @@ const makeOpenCodeRuntimeHarness = Effect.fn("makeOpenCodeRuntimeHarness")(funct |
| 213 | providerTurnOrdinal: 1, | 216 | providerTurnOrdinal: 1, |
| 214 | attemptId: RunAttemptId.make(`attempt-opencode-${suffix}`), | 217 | attemptId: RunAttemptId.make(`attempt-opencode-${suffix}`), |
| 215 | rootNodeId: NodeId.make(`node-opencode-${suffix}`), | 218 | rootNodeId: NodeId.make(`node-opencode-${suffix}`), |
| 216 | providerThread, | 219 | providerThread: startProviderThread, |
| 217 | message: { | 220 | message: { |
| 218 | createdBy: "user", | 221 | createdBy: "user", |
| 219 | creationSource: "web", | 222 | creationSource: "web", |
| ... | @@ -1291,6 +1294,45 @@ describe("OpenCodeAdapterV2", () => { | ... | @@ -1291,6 +1294,45 @@ describe("OpenCodeAdapterV2", () => { |
| 1291 | }).pipe(Effect.provide(IdAllocator.layer), Effect.scoped), | 1294 | }).pipe(Effect.provide(IdAllocator.layer), Effect.scoped), |
| 1292 | ); | 1295 | ); |
| 1293 | 1296 | ||
| 1297 | it.effect("ends a turn on the provider thread it started on", () => | ||
| 1298 | Effect.gen(function* () { | ||
| 1299 | const nativeEvents = asyncEventStream(); | ||
| 1300 | const harness = yield* makeOpenCodeRuntimeHarness( | ||
| 1301 | "forked-row", | ||
| 1302 | "native-opencode-forked-row", | ||
| 1303 | { | ||
| 1304 | event: { | ||
| 1305 | subscribe: async (_input: unknown, options: { signal?: AbortSignal }) => { | ||
| 1306 | options.signal?.addEventListener("abort", () => nativeEvents.close(), { once: true }); | ||
| 1307 | return { stream: nativeEvents.stream }; | ||
| 1308 | }, | ||
| 1309 | }, | ||
| 1310 | session: { | ||
| 1311 | create: async () => ({ | ||
| 1312 | data: { id: "native-opencode-forked-row", time: { created: 1, updated: 1 } }, | ||
| 1313 | }), | ||
| 1314 | summarize: async () => ({ data: true }), | ||
| 1315 | }, | ||
| 1316 | }, | ||
| 1317 | ); | ||
| 1318 | // A forked run starts on its own row for the same native session, while | ||
| 1319 | // the adapter tracks the session under the id it minted. | ||
| 1320 | const forkedRow = { | ||
| 1321 | ...harness.providerThread, | ||
| 1322 | id: ProviderThreadId.make("provider-thread:opencode-test:forked-run-row"), | ||
| 1323 | }; | ||
| 1324 | yield* harness.startTurn("/compact", forkedRow); | ||
| 1325 | const terminal = yield* harness.runtime.events.pipe( | ||
| 1326 | Stream.filter( | ||
| 1327 | (event): event is Extract<typeof event, { type: "turn.terminal" }> => | ||
| 1328 | event.type === "turn.terminal", | ||
| 1329 | ), | ||
| 1330 | Stream.runHead, | ||
| 1331 | ); | ||
| 1332 | assert.equal(Option.getOrUndefined(terminal)?.providerThreadId, forkedRow.id); | ||
| 1333 | }).pipe(Effect.provide(IdAllocator.layer), Effect.scoped), | ||
| 1334 | ); | ||
| 1335 | |||
| 1294 | it.effect("does not restore messages beyond OpenCode's persisted revert boundary", () => | 1336 | it.effect("does not restore messages beyond OpenCode's persisted revert boundary", () => |
| 1295 | Effect.gen(function* () { | 1337 | Effect.gen(function* () { |
| 1296 | const nativeEvents = asyncEventStream(); | 1338 | const nativeEvents = asyncEventStream(); |
apps/server/src/orchestration-v2/Adapters/OpenCodeAdapterV2.ts+6-2
| ... | @@ -279,6 +279,8 @@ interface ActiveOpenCodeTurn { | ... | @@ -279,6 +279,8 @@ interface ActiveOpenCodeTurn { |
| 279 | readonly modelSelection: ModelSelection; | 279 | readonly modelSelection: ModelSelection; |
| 280 | readonly runtimePolicy: ProviderAdapter.ProviderAdapterV2RuntimePolicy; | 280 | readonly runtimePolicy: ProviderAdapter.ProviderAdapterV2RuntimePolicy; |
| 281 | readonly providerTurnId: OrchestrationV2ProviderTurn["id"]; | 281 | readonly providerTurnId: OrchestrationV2ProviderTurn["id"]; |
| 282 | /** The provider thread the turn started on, which its terminal names. */ | ||
| 283 | readonly providerThreadId: OrchestrationV2ProviderTurn["providerThreadId"]; | ||
| 282 | readonly providerTurnOrdinal: number; | 284 | readonly providerTurnOrdinal: number; |
| 283 | readonly runOrdinal: number; | 285 | readonly runOrdinal: number; |
| 284 | readonly runAttemptId: OrchestrationV2ProviderTurn["runAttemptId"]; | 286 | readonly runAttemptId: OrchestrationV2ProviderTurn["runAttemptId"]; |
| ... | @@ -2118,7 +2120,7 @@ export function makeOpenCodeAdapterV2( | ... | @@ -2118,7 +2120,7 @@ export function makeOpenCodeAdapterV2( |
| 2118 | ? { | 2120 | ? { |
| 2119 | type: "turn.terminal", | 2121 | type: "turn.terminal", |
| 2120 | driver: OPENCODE_PROVIDER, | 2122 | driver: OPENCODE_PROVIDER, |
| 2121 | providerThreadId: state.providerThread.id, | 2123 | providerThreadId: turn.providerThreadId, |
| 2122 | providerTurnId: turn.providerTurnId, | 2124 | providerTurnId: turn.providerTurnId, |
| 2123 | runOrdinal: turn.runOrdinal, | 2125 | runOrdinal: turn.runOrdinal, |
| 2124 | failureItemOrdinal: itemOrdinal(turn, `terminal-failure:${turn.providerTurnId}`), | 2126 | failureItemOrdinal: itemOrdinal(turn, `terminal-failure:${turn.providerTurnId}`), |
| ... | @@ -2134,7 +2136,7 @@ export function makeOpenCodeAdapterV2( | ... | @@ -2134,7 +2136,7 @@ export function makeOpenCodeAdapterV2( |
| 2134 | : { | 2136 | : { |
| 2135 | type: "turn.terminal", | 2137 | type: "turn.terminal", |
| 2136 | driver: OPENCODE_PROVIDER, | 2138 | driver: OPENCODE_PROVIDER, |
| 2137 | providerThreadId: state.providerThread.id, | 2139 | providerThreadId: turn.providerThreadId, |
| 2138 | providerTurnId: turn.providerTurnId, | 2140 | providerTurnId: turn.providerTurnId, |
| 2139 | runOrdinal: turn.runOrdinal, | 2141 | runOrdinal: turn.runOrdinal, |
| 2140 | status, | 2142 | status, |
| ... | @@ -2256,6 +2258,7 @@ export function makeOpenCodeAdapterV2( | ... | @@ -2256,6 +2258,7 @@ export function makeOpenCodeAdapterV2( |
| 2256 | modelSelection: state.appThread.modelSelection, | 2258 | modelSelection: state.appThread.modelSelection, |
| 2257 | runtimePolicy: state.parentSubagent.parentTurn.runtimePolicy, | 2259 | runtimePolicy: state.parentSubagent.parentTurn.runtimePolicy, |
| 2258 | providerTurnId, | 2260 | providerTurnId, |
| 2261 | providerThreadId: providerTurn.providerThreadId, | ||
| 2259 | providerTurnOrdinal: providerTurn.ordinal, | 2262 | providerTurnOrdinal: providerTurn.ordinal, |
| 2260 | runOrdinal: state.parentSubagent.parentTurn.runOrdinal, | 2263 | runOrdinal: state.parentSubagent.parentTurn.runOrdinal, |
| 2261 | runAttemptId: null, | 2264 | runAttemptId: null, |
| ... | @@ -3192,6 +3195,7 @@ export function makeOpenCodeAdapterV2( | ... | @@ -3192,6 +3195,7 @@ export function makeOpenCodeAdapterV2( |
| 3192 | modelSelection: turnInput.modelSelection, | 3195 | modelSelection: turnInput.modelSelection, |
| 3193 | runtimePolicy: turnInput.runtimePolicy, | 3196 | runtimePolicy: turnInput.runtimePolicy, |
| 3194 | providerTurnId, | 3197 | providerTurnId, |
| 3198 | providerThreadId: turnInput.providerThread.id, | ||
| 3195 | providerTurnOrdinal: turnInput.providerTurnOrdinal, | 3199 | providerTurnOrdinal: turnInput.providerTurnOrdinal, |
| 3196 | runOrdinal: turnInput.runOrdinal, | 3200 | runOrdinal: turnInput.runOrdinal, |
| 3197 | runAttemptId: turnInput.attemptId, | 3201 | runAttemptId: turnInput.attemptId, |
apps/server/src/orchestration-v2/Adapters/PiAdapterV2.ts+2-1
| ... | @@ -2679,8 +2679,9 @@ export function makePiAdapterV2( | ... | @@ -2679,8 +2679,9 @@ export function makePiAdapterV2( |
| 2679 | } | 2679 | } |
| 2680 | // Pi fork replaces the session file, including for rollback. Persist | 2680 | // Pi fork replaces the session file, including for rollback. Persist |
| 2681 | // its new identity before any later request can fail or restart. | 2681 | // its new identity before any later request can fail or restart. |
| 2682 | // An interrupted read leaves the identity just as unknown as a failed one. | ||
| 2682 | const forkState = yield* request({ type: "get_state" }).pipe( | 2683 | const forkState = yield* request({ type: "get_state" }).pipe( |
| 2683 | Effect.tapError(() => | 2684 | Effect.onError(() => |
| 2684 | Effect.sync(() => { | 2685 | Effect.sync(() => { |
| 2685 | threadState = null; | 2686 | threadState = null; |
| 2686 | }), | 2687 | }), |
apps/server/src/orchestration-v2/ProviderSessionManager.test.ts+827-20
| ... | @@ -128,6 +128,30 @@ const layerFlakyReleaseEventSink = (flaky: FlakyReleaseWrites) => | ... | @@ -128,6 +128,30 @@ const layerFlakyReleaseEventSink = (flaky: FlakyReleaseWrites) => |
| 128 | }), | 128 | }), |
| 129 | ).pipe(Layer.provide(layerTestEventSink)); | 129 | ).pipe(Layer.provide(layerTestEventSink)); |
| 130 | 130 | ||
| 131 | // Once armed, holds the next attach write until the writer is interrupted. | ||
| 132 | const layerPausingAttachEventSink = (pause: { | ||
| 133 | readonly armed: Ref.Ref<boolean>; | ||
| 134 | readonly paused: Deferred.Deferred<void>; | ||
| 135 | }) => | ||
| 136 | Layer.effect( | ||
| 137 | EventSink.EventSinkV2, | ||
| 138 | Effect.gen(function* () { | ||
| 139 | const delegate = yield* EventSink.EventSinkV2; | ||
| 140 | return EventSink.EventSinkV2.of({ | ||
| 141 | ...delegate, | ||
| 142 | write: (input) => | ||
| 143 | Effect.gen(function* () { | ||
| 144 | const attach = input.events.some((event) => event.type === "provider-session.attached"); | ||
| 145 | if (attach && (yield* Ref.getAndSet(pause.armed, false))) { | ||
| 146 | yield* Deferred.succeed(pause.paused, undefined); | ||
| 147 | return yield* Effect.never; | ||
| 148 | } | ||
| 149 | return yield* delegate.write(input); | ||
| 150 | }), | ||
| 151 | }); | ||
| 152 | }), | ||
| 153 | ).pipe(Layer.provide(layerTestEventSink)); | ||
| 154 | |||
| 131 | const CodexCapabilities: OrchestrationV2ProviderCapabilities = CodexProviderCapabilitiesV2; | 155 | const CodexCapabilities: OrchestrationV2ProviderCapabilities = CodexProviderCapabilitiesV2; |
| 132 | const ExclusiveCapabilities: OrchestrationV2ProviderCapabilities = { | 156 | const ExclusiveCapabilities: OrchestrationV2ProviderCapabilities = { |
| 133 | ...CodexCapabilities, | 157 | ...CodexCapabilities, |
| ... | @@ -244,11 +268,13 @@ function makeProviderThread(input: { | ... | @@ -244,11 +268,13 @@ function makeProviderThread(input: { |
| 244 | readonly threadId: ThreadId; | 268 | readonly threadId: ThreadId; |
| 245 | readonly providerSessionId: ProviderSessionId; | 269 | readonly providerSessionId: ProviderSessionId; |
| 246 | readonly now: DateTime.Utc; | 270 | readonly now: DateTime.Utc; |
| 271 | readonly nativeThreadId?: string; | ||
| 247 | }): OrchestrationV2ProviderThread { | 272 | }): OrchestrationV2ProviderThread { |
| 273 | const nativeThreadId = input.nativeThreadId ?? "native-thread"; | ||
| 248 | return { | 274 | return { |
| 249 | id: input.idAllocator.derive.providerThread({ | 275 | id: input.idAllocator.derive.providerThread({ |
| 250 | driver: CODEX_DRIVER, | 276 | driver: CODEX_DRIVER, |
| 251 | nativeThreadId: "native-thread", | 277 | nativeThreadId, |
| 252 | }), | 278 | }), |
| 253 | driver: CODEX_DRIVER, | 279 | driver: CODEX_DRIVER, |
| 254 | providerInstanceId: modelSelection.instanceId, | 280 | providerInstanceId: modelSelection.instanceId, |
| ... | @@ -257,7 +283,7 @@ function makeProviderThread(input: { | ... | @@ -257,7 +283,7 @@ function makeProviderThread(input: { |
| 257 | ownerNodeId: null, | 283 | ownerNodeId: null, |
| 258 | nativeThreadRef: { | 284 | nativeThreadRef: { |
| 259 | driver: CODEX_DRIVER, | 285 | driver: CODEX_DRIVER, |
| 260 | nativeId: "native-thread", | 286 | nativeId: nativeThreadId, |
| 261 | strength: "strong", | 287 | strength: "strong", |
| 262 | }, | 288 | }, |
| 263 | nativeConversationHeadRef: null, | 289 | nativeConversationHeadRef: null, |
| ... | @@ -294,9 +320,29 @@ function makeProviderAdapter( | ... | @@ -294,9 +320,29 @@ function makeProviderAdapter( |
| 294 | }) => Effect.Effect<void>; | 320 | }) => Effect.Effect<void>; |
| 295 | readonly hasPendingBackgroundWork?: Effect.Effect<boolean>; | 321 | readonly hasPendingBackgroundWork?: Effect.Effect<boolean>; |
| 296 | readonly hangSessionScopeClose?: boolean; | 322 | readonly hangSessionScopeClose?: boolean; |
| 323 | readonly startTurn?: Effect.Effect<void>; | ||
| 297 | readonly beforeUnload?: Effect.Effect<void>; | 324 | readonly beforeUnload?: Effect.Effect<void>; |
| 325 | /** Registers the process's closeCount finalizer before `beforeOpen` runs. */ | ||
| 326 | readonly spawnBeforeOpen?: boolean; | ||
| 327 | /** Completed when a hanging scope close reaches its wedged finalizer. */ | ||
| 328 | readonly scopeCloseReached?: Deferred.Deferred<void>; | ||
| 298 | } = {}, | 329 | } = {}, |
| 299 | ): ProviderAdapterV2Shape { | 330 | ): ProviderAdapterV2Shape { |
| 331 | const countClose = Effect.addFinalizer(() => | ||
| 332 | Ref.update(state, (current) => ({ | ||
| 333 | ...current, | ||
| 334 | closeCount: current.closeCount + 1, | ||
| 335 | })), | ||
| 336 | ); | ||
| 337 | // Registered after countClose so it runs first on scope close, wedging the | ||
| 338 | // close before the closeCount finalizer, like a provider process that never | ||
| 339 | // yields its message stream. | ||
| 340 | const hangClose = Effect.addFinalizer(() => | ||
| 341 | (options.scopeCloseReached === undefined | ||
| 342 | ? Effect.void | ||
| 343 | : Deferred.succeed(options.scopeCloseReached, undefined) | ||
| 344 | ).pipe(Effect.andThen(Effect.never)), | ||
| 345 | ); | ||
| 300 | return { | 346 | return { |
| 301 | instanceId: ProviderInstanceId.make("codex"), | 347 | instanceId: ProviderInstanceId.make("codex"), |
| 302 | driver: CODEX_DRIVER, | 348 | driver: CODEX_DRIVER, |
| ... | @@ -304,10 +350,20 @@ function makeProviderAdapter( | ... | @@ -304,10 +350,20 @@ function makeProviderAdapter( |
| 304 | planSelectionTransition: () => Effect.succeed({ type: "apply_on_next_turn" }), | 350 | planSelectionTransition: () => Effect.succeed({ type: "apply_on_next_turn" }), |
| 305 | openSession: (input) => | 351 | openSession: (input) => |
| 306 | Effect.gen(function* () { | 352 | Effect.gen(function* () { |
| 353 | if (options.spawnBeforeOpen === true) { | ||
| 354 | yield* countClose; | ||
| 355 | if (options.hangSessionScopeClose === true) yield* hangClose; | ||
| 356 | } | ||
| 357 | if (options.mcpConfigs !== undefined && options.spawnBeforeOpen === true) { | ||
| 358 | yield* Ref.update(options.mcpConfigs, (configs) => [ | ||
| 359 | ...configs, | ||
| 360 | McpProviderSession.readMcpProviderSession(input.threadId), | ||
| 361 | ]); | ||
| 362 | } | ||
| 307 | if (options.beforeOpen !== undefined) { | 363 | if (options.beforeOpen !== undefined) { |
| 308 | yield* options.beforeOpen(input); | 364 | yield* options.beforeOpen(input); |
| 309 | } | 365 | } |
| 310 | if (options.mcpConfigs !== undefined) { | 366 | if (options.mcpConfigs !== undefined && options.spawnBeforeOpen !== true) { |
| 311 | yield* Ref.update(options.mcpConfigs, (configs) => [ | 367 | yield* Ref.update(options.mcpConfigs, (configs) => [ |
| 312 | ...configs, | 368 | ...configs, |
| 313 | McpProviderSession.readMcpProviderSession(input.threadId), | 369 | McpProviderSession.readMcpProviderSession(input.threadId), |
| ... | @@ -329,17 +385,9 @@ function makeProviderAdapter( | ... | @@ -329,17 +385,9 @@ function makeProviderAdapter( |
| 329 | eventQueues, | 385 | eventQueues, |
| 330 | }; | 386 | }; |
| 331 | }); | 387 | }); |
| 332 | yield* Effect.addFinalizer(() => | 388 | if (options.spawnBeforeOpen !== true) { |
| 333 | Ref.update(state, (current) => ({ | 389 | yield* countClose; |
| 334 | ...current, | 390 | if (options.hangSessionScopeClose === true) yield* hangClose; |
| 335 | closeCount: current.closeCount + 1, | ||
| 336 | })), | ||
| 337 | ); | ||
| 338 | if (options.hangSessionScopeClose === true) { | ||
| 339 | // Registered last so it runs first on scope close, wedging the | ||
| 340 | // close before the closeCount finalizer, like a provider process | ||
| 341 | // that never yields its message stream. | ||
| 342 | yield* Effect.addFinalizer(() => Effect.never); | ||
| 343 | } | 391 | } |
| 344 | 392 | ||
| 345 | return { | 393 | return { |
| ... | @@ -365,7 +413,7 @@ function makeProviderAdapter( | ... | @@ -365,7 +413,7 @@ function makeProviderAdapter( |
| 365 | ...current, | 413 | ...current, |
| 366 | resumeCount: current.resumeCount + 1, | 414 | resumeCount: current.resumeCount + 1, |
| 367 | })).pipe(Effect.as(threadInput.providerThread)), | 415 | })).pipe(Effect.as(threadInput.providerThread)), |
| 368 | startTurn: () => Effect.void, | 416 | startTurn: () => options.startTurn ?? Effect.void, |
| 369 | steerTurn: () => Effect.void, | 417 | steerTurn: () => Effect.void, |
| 370 | interruptTurn: () => | 418 | interruptTurn: () => |
| 371 | Ref.update(state, (current) => ({ | 419 | Ref.update(state, (current) => ({ |
| ... | @@ -408,18 +456,29 @@ function layerTest(input: { | ... | @@ -408,18 +456,29 @@ function layerTest(input: { |
| 408 | }) => Effect.Effect<void>; | 456 | }) => Effect.Effect<void>; |
| 409 | readonly failReleaseEventWrites?: boolean; | 457 | readonly failReleaseEventWrites?: boolean; |
| 410 | readonly flakyReleaseWrites?: FlakyReleaseWrites; | 458 | readonly flakyReleaseWrites?: FlakyReleaseWrites; |
| 459 | readonly pauseAttachWrite?: Parameters<typeof layerPausingAttachEventSink>[0]; | ||
| 460 | /** Once armed, holds the next credential lookup until it is interrupted. */ | ||
| 461 | readonly pauseResolve?: { | ||
| 462 | readonly armed: Ref.Ref<boolean>; | ||
| 463 | readonly paused: Deferred.Deferred<void>; | ||
| 464 | }; | ||
| 411 | readonly hasPendingBackgroundWork?: Effect.Effect<boolean>; | 465 | readonly hasPendingBackgroundWork?: Effect.Effect<boolean>; |
| 412 | readonly hangSessionScopeClose?: boolean; | 466 | readonly hangSessionScopeClose?: boolean; |
| 467 | readonly startTurn?: Effect.Effect<void>; | ||
| 413 | readonly beforeUnload?: Effect.Effect<void>; | 468 | readonly beforeUnload?: Effect.Effect<void>; |
| 469 | readonly spawnBeforeOpen?: boolean; | ||
| 470 | readonly scopeCloseReached?: Deferred.Deferred<void>; | ||
| 414 | readonly serverSettingsLayer?: ReturnType<typeof ServerSettings.layerTest>; | 471 | readonly serverSettingsLayer?: ReturnType<typeof ServerSettings.layerTest>; |
| 415 | readonly projectServiceLayer?: Layer.Layer<ProjectService.ProjectService>; | 472 | readonly projectServiceLayer?: Layer.Layer<ProjectService.ProjectService>; |
| 416 | }) { | 473 | }) { |
| 417 | const layerConfiguredEventSink = | 474 | const layerConfiguredEventSink = |
| 418 | input.flakyReleaseWrites !== undefined | 475 | input.flakyReleaseWrites !== undefined |
| 419 | ? layerFlakyReleaseEventSink(input.flakyReleaseWrites) | 476 | ? layerFlakyReleaseEventSink(input.flakyReleaseWrites) |
| 420 | : input.failReleaseEventWrites | 477 | : input.pauseAttachWrite !== undefined |
| 421 | ? layerFailingReleaseEventSink | 478 | ? layerPausingAttachEventSink(input.pauseAttachWrite) |
| 422 | : layerTestEventSink; | 479 | : input.failReleaseEventWrites |
| 480 | ? layerFailingReleaseEventSink | ||
| 481 | : layerTestEventSink; | ||
| 423 | const layerRegistry = ProviderAdapterRegistry.layerSingle( | 482 | const layerRegistry = ProviderAdapterRegistry.layerSingle( |
| 424 | makeProviderAdapter(input.state, { | 483 | makeProviderAdapter(input.state, { |
| 425 | failEventStream: input.failEventStream ?? false, | 484 | failEventStream: input.failEventStream ?? false, |
| ... | @@ -432,9 +491,18 @@ function layerTest(input: { | ... | @@ -432,9 +491,18 @@ function layerTest(input: { |
| 432 | ...(input.hangSessionScopeClose === undefined | 491 | ...(input.hangSessionScopeClose === undefined |
| 433 | ? {} | 492 | ? {} |
| 434 | : { hangSessionScopeClose: input.hangSessionScopeClose }), | 493 | : { hangSessionScopeClose: input.hangSessionScopeClose }), |
| 494 | ...(input.startTurn === undefined ? {} : { startTurn: input.startTurn }), | ||
| 435 | ...(input.beforeUnload === undefined ? {} : { beforeUnload: input.beforeUnload }), | 495 | ...(input.beforeUnload === undefined ? {} : { beforeUnload: input.beforeUnload }), |
| 496 | ...(input.spawnBeforeOpen === undefined ? {} : { spawnBeforeOpen: input.spawnBeforeOpen }), | ||
| 497 | ...(input.scopeCloseReached === undefined | ||
| 498 | ? {} | ||
| 499 | : { scopeCloseReached: input.scopeCloseReached }), | ||
| 436 | }), | 500 | }), |
| 437 | ); | 501 | ); |
| 502 | const layerConfiguredMcpRegistry = | ||
| 503 | input.pauseResolve === undefined | ||
| 504 | ? layerTestMcpRegistry | ||
| 505 | : layerPausingMcpRegistry(input.pauseResolve); | ||
| 438 | const layerProviderEventIngestorTest = ProviderEventIngestor.layer.pipe( | 506 | const layerProviderEventIngestorTest = ProviderEventIngestor.layer.pipe( |
| 439 | Layer.provide( | 507 | Layer.provide( |
| 440 | Layer.mergeAll( | 508 | Layer.mergeAll( |
| ... | @@ -449,7 +517,7 @@ function layerTest(input: { | ... | @@ -449,7 +517,7 @@ function layerTest(input: { |
| 449 | layerTestStores, | 517 | layerTestStores, |
| 450 | layerConfiguredEventSink, | 518 | layerConfiguredEventSink, |
| 451 | IdAllocator.layer, | 519 | IdAllocator.layer, |
| 452 | layerTestMcpRegistry, | 520 | layerConfiguredMcpRegistry, |
| 453 | ProviderSessionManager.layerWithOptions({ | 521 | ProviderSessionManager.layerWithOptions({ |
| 454 | idleTimeoutMs: input.idleTimeoutMs, | 522 | idleTimeoutMs: input.idleTimeoutMs, |
| 455 | ...(input.maxIdlePinMs === undefined ? {} : { maxIdlePinMs: input.maxIdlePinMs }), | 523 | ...(input.maxIdlePinMs === undefined ? {} : { maxIdlePinMs: input.maxIdlePinMs }), |
| ... | @@ -460,7 +528,7 @@ function layerTest(input: { | ... | @@ -460,7 +528,7 @@ function layerTest(input: { |
| 460 | layerConfiguredEventSink, | 528 | layerConfiguredEventSink, |
| 461 | IdAllocator.layer, | 529 | IdAllocator.layer, |
| 462 | layerProviderEventIngestorTest, | 530 | layerProviderEventIngestorTest, |
| 463 | layerTestMcpRegistry, | 531 | layerConfiguredMcpRegistry, |
| 464 | layerTestStores, | 532 | layerTestStores, |
| 465 | ...(input.serverSettingsLayer === undefined ? [] : [input.serverSettingsLayer]), | 533 | ...(input.serverSettingsLayer === undefined ? [] : [input.serverSettingsLayer]), |
| 466 | ...(input.projectServiceLayer === undefined ? [] : [input.projectServiceLayer]), | 534 | ...(input.projectServiceLayer === undefined ? [] : [input.projectServiceLayer]), |
| ... | @@ -489,6 +557,28 @@ const layerTestMcpRegistry = Layer.effect( | ... | @@ -489,6 +557,28 @@ const layerTestMcpRegistry = Layer.effect( |
| 489 | Layer.provide(NodeServices.layer), | 557 | Layer.provide(NodeServices.layer), |
| 490 | ); | 558 | ); |
| 491 | 559 | ||
| 560 | const layerPausingMcpRegistry = (pause: { | ||
| 561 | readonly armed: Ref.Ref<boolean>; | ||
| 562 | readonly paused: Deferred.Deferred<void>; | ||
| 563 | }) => | ||
| 564 | Layer.effect( | ||
| 565 | McpSessionRegistry.McpSessionRegistry, | ||
| 566 | Effect.gen(function* () { | ||
| 567 | const delegate = yield* McpSessionRegistry.McpSessionRegistry; | ||
| 568 | return McpSessionRegistry.McpSessionRegistry.of({ | ||
| 569 | ...delegate, | ||
| 570 | resolve: (rawToken) => | ||
| 571 | Ref.getAndSet(pause.armed, false).pipe( | ||
| 572 | Effect.flatMap((armed) => | ||
| 573 | armed | ||
| 574 | ? Deferred.succeed(pause.paused, undefined).pipe(Effect.andThen(Effect.never)) | ||
| 575 | : delegate.resolve(rawToken), | ||
| 576 | ), | ||
| 577 | ), | ||
| 578 | }); | ||
| 579 | }), | ||
| 580 | ).pipe(Layer.provide(layerTestMcpRegistry)); | ||
| 581 | |||
| 492 | function makeBrowserAccessProject(projectId: ProjectId): Project { | 582 | function makeBrowserAccessProject(projectId: ProjectId): Project { |
| 493 | return { | 583 | return { |
| 494 | id: projectId, | 584 | id: projectId, |
| ... | @@ -997,6 +1087,498 @@ it.effect("ProviderSessionManagerV2 opens a duplicate session only once", () => | ... | @@ -997,6 +1087,498 @@ it.effect("ProviderSessionManagerV2 opens a duplicate session only once", () => |
| 997 | }), | 1087 | }), |
| 998 | ); | 1088 | ); |
| 999 | 1089 | ||
| 1090 | it.effect("ProviderSessionManagerV2 cleans up an open interrupted mid-handshake", () => | ||
| 1091 | Effect.gen(function* () { | ||
| 1092 | const state = yield* Ref.make(emptyState); | ||
| 1093 | const mcpConfigs = yield* Ref.make< | ||
| 1094 | ReadonlyArray<McpProviderSession.McpProviderSessionConfig | undefined> | ||
| 1095 | >([]); | ||
| 1096 | const handshakeStarted = yield* Deferred.make<void>(); | ||
| 1097 | const holdHandshake = yield* Ref.make(true); | ||
| 1098 | const effect = Effect.gen(function* () { | ||
| 1099 | const eventSink = yield* EventSink.EventSinkV2; | ||
| 1100 | const idAllocator = yield* IdAllocator.IdAllocatorV2; | ||
| 1101 | const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; | ||
| 1102 | const registry = yield* McpSessionRegistry.McpSessionRegistry; | ||
| 1103 | const projectionStore = yield* ProjectionStore.ProjectionStoreV2; | ||
| 1104 | const now = yield* DateTime.now; | ||
| 1105 | const threadId = ThreadId.make("thread-provider-session-manager-interrupted-open"); | ||
| 1106 | const providerSessionId = yield* idAllocator.allocate.providerSession({ | ||
| 1107 | providerInstanceId: modelSelection.instanceId, | ||
| 1108 | threadId, | ||
| 1109 | }); | ||
| 1110 | yield* eventSink.write({ | ||
| 1111 | events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], | ||
| 1112 | }); | ||
| 1113 | |||
| 1114 | const opening = yield* manager | ||
| 1115 | .open({ threadId, providerSessionId, modelSelection, runtimePolicy }) | ||
| 1116 | .pipe(Effect.forkScoped); | ||
| 1117 | yield* Deferred.await(handshakeStarted); | ||
| 1118 | const issued = (yield* Ref.get(mcpConfigs)).at(-1); | ||
| 1119 | const token = issued?.authorizationHeader.replace(/^Bearer\s+/, ""); | ||
| 1120 | assert.isDefined(token); | ||
| 1121 | assert.isDefined(yield* registry.resolve(token!)); | ||
| 1122 | |||
| 1123 | // A Stop while the provider is still starting. | ||
| 1124 | yield* Fiber.interrupt(opening); | ||
| 1125 | |||
| 1126 | // The process started for this open is stopped, and the credential minted | ||
| 1127 | // for it revoked. | ||
| 1128 | assert.equal((yield* Ref.get(state)).closeCount, 1); | ||
| 1129 | assert.isUndefined(yield* registry.resolve(token!)); | ||
| 1130 | assert.isUndefined(McpProviderSession.readMcpProviderSession(threadId)); | ||
| 1131 | assert.isTrue(Option.isNone(yield* manager.get(providerSessionId))); | ||
| 1132 | |||
| 1133 | // Nothing of the interrupted open is left behind: the next open starts a | ||
| 1134 | // fresh process with a fresh credential that a later release revokes, | ||
| 1135 | // which a leaked reservation would prevent. | ||
| 1136 | yield* Ref.set(holdHandshake, false); | ||
| 1137 | yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy }); | ||
| 1138 | const replacement = (yield* Ref.get(mcpConfigs)).at(-1); | ||
| 1139 | const replacementToken = replacement?.authorizationHeader.replace(/^Bearer\s+/, ""); | ||
| 1140 | assert.isDefined(replacementToken); | ||
| 1141 | assert.notEqual(replacementToken, token); | ||
| 1142 | const projection = yield* projectionStore.getThreadProjection(threadId); | ||
| 1143 | assert.equal(projection.providerSessions.at(-1)?.status, "ready"); | ||
| 1144 | |||
| 1145 | yield* manager.close(providerSessionId); | ||
| 1146 | assert.isUndefined(yield* registry.resolve(replacementToken!)); | ||
| 1147 | }); | ||
| 1148 | |||
| 1149 | yield* effect.pipe( | ||
| 1150 | Effect.provide( | ||
| 1151 | layerTest({ | ||
| 1152 | state, | ||
| 1153 | idleTimeoutMs: 60_000, | ||
| 1154 | mcpConfigs, | ||
| 1155 | beforeOpen: () => | ||
| 1156 | Ref.get(holdHandshake).pipe( | ||
| 1157 | Effect.flatMap((hold) => | ||
| 1158 | hold | ||
| 1159 | ? Deferred.succeed(handshakeStarted, undefined).pipe(Effect.andThen(Effect.never)) | ||
| 1160 | : Effect.void, | ||
| 1161 | ), | ||
| 1162 | ), | ||
| 1163 | // The process is spawned before the handshake that is interrupted. | ||
| 1164 | spawnBeforeOpen: true, | ||
| 1165 | }), | ||
| 1166 | ), | ||
| 1167 | ); | ||
| 1168 | }), | ||
| 1169 | ); | ||
| 1170 | |||
| 1171 | it.effect("ProviderSessionManagerV2 cleans up an interrupted open whose scope close hangs", () => | ||
| 1172 | Effect.gen(function* () { | ||
| 1173 | const state = yield* Ref.make(emptyState); | ||
| 1174 | const mcpConfigs = yield* Ref.make< | ||
| 1175 | ReadonlyArray<McpProviderSession.McpProviderSessionConfig | undefined> | ||
| 1176 | >([]); | ||
| 1177 | const handshakeStarted = yield* Deferred.make<void>(); | ||
| 1178 | const scopeCloseReached = yield* Deferred.make<void>(); | ||
| 1179 | const releaseRetry = yield* Deferred.make<void>(); | ||
| 1180 | yield* Effect.gen(function* () { | ||
| 1181 | const eventSink = yield* EventSink.EventSinkV2; | ||
| 1182 | const idAllocator = yield* IdAllocator.IdAllocatorV2; | ||
| 1183 | const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; | ||
| 1184 | const registry = yield* McpSessionRegistry.McpSessionRegistry; | ||
| 1185 | const now = yield* DateTime.now; | ||
| 1186 | const threadId = ThreadId.make("thread-provider-session-manager-interrupted-hung-open"); | ||
| 1187 | const providerSessionId = yield* idAllocator.allocate.providerSession({ | ||
| 1188 | providerInstanceId: modelSelection.instanceId, | ||
| 1189 | threadId, | ||
| 1190 | }); | ||
| 1191 | yield* eventSink.write({ | ||
| 1192 | events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], | ||
| 1193 | }); | ||
| 1194 | |||
| 1195 | const opening = yield* manager | ||
| 1196 | .open({ threadId, providerSessionId, modelSelection, runtimePolicy }) | ||
| 1197 | .pipe(Effect.forkChild); | ||
| 1198 | yield* Deferred.await(handshakeStarted); | ||
| 1199 | const issued = (yield* Ref.get(mcpConfigs)).at(-1); | ||
| 1200 | const token = issued?.authorizationHeader.replace(/^Bearer\s+/, ""); | ||
| 1201 | assert.isDefined(token); | ||
| 1202 | |||
| 1203 | // The interrupt starts cleanup; the scope close then never finishes. | ||
| 1204 | const interrupter = yield* Fiber.interrupt(opening).pipe(Effect.forkChild); | ||
| 1205 | yield* Deferred.await(scopeCloseReached); | ||
| 1206 | |||
| 1207 | // The session cleanup already ran, ahead of the stuck close. | ||
| 1208 | assert.isUndefined(yield* registry.resolve(token!)); | ||
| 1209 | assert.isUndefined(McpProviderSession.readMcpProviderSession(threadId)); | ||
| 1210 | assert.equal((yield* Ref.get(state)).closeCount, 0); | ||
| 1211 | |||
| 1212 | // The close is time-boxed, so the interrupter returns and the session's | ||
| 1213 | // open lock is free for the next open. | ||
| 1214 | yield* TestClock.adjust("30 seconds"); | ||
| 1215 | yield* Fiber.join(interrupter); | ||
| 1216 | yield* Deferred.succeed(releaseRetry, undefined); | ||
| 1217 | yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy }); | ||
| 1218 | assert.equal((yield* Ref.get(state)).openCount, 1); | ||
| 1219 | |||
| 1220 | // Its close hangs as well; release it while the test clock can still move. | ||
| 1221 | const stopping = yield* manager.shutdown.pipe(Effect.forkChild); | ||
| 1222 | yield* TestClock.adjust("30 seconds"); | ||
| 1223 | yield* Fiber.join(stopping); | ||
| 1224 | }).pipe( | ||
| 1225 | Effect.provide( | ||
| 1226 | layerTest({ | ||
| 1227 | state, | ||
| 1228 | idleTimeoutMs: 60_000, | ||
| 1229 | mcpConfigs, | ||
| 1230 | // The first open hangs in its handshake; the retry completes. | ||
| 1231 | beforeOpen: () => | ||
| 1232 | Deferred.isDone(handshakeStarted).pipe( | ||
| 1233 | Effect.flatMap((retry) => | ||
| 1234 | retry | ||
| 1235 | ? Deferred.await(releaseRetry) | ||
| 1236 | : Deferred.succeed(handshakeStarted, undefined).pipe( | ||
| 1237 | Effect.andThen(Effect.never), | ||
| 1238 | ), | ||
| 1239 | ), | ||
| 1240 | ), | ||
| 1241 | spawnBeforeOpen: true, | ||
| 1242 | hangSessionScopeClose: true, | ||
| 1243 | scopeCloseReached, | ||
| 1244 | }), | ||
| 1245 | ), | ||
| 1246 | ); | ||
| 1247 | }), | ||
| 1248 | ); | ||
| 1249 | |||
| 1250 | it.effect("ProviderSessionManagerV2 releases an idle session whose turn start was stopped", () => | ||
| 1251 | Effect.gen(function* () { | ||
| 1252 | const state = yield* Ref.make(emptyState); | ||
| 1253 | const startTurnReached = yield* Deferred.make<void>(); | ||
| 1254 | yield* Effect.gen(function* () { | ||
| 1255 | const eventSink = yield* EventSink.EventSinkV2; | ||
| 1256 | const idAllocator = yield* IdAllocator.IdAllocatorV2; | ||
| 1257 | const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; | ||
| 1258 | const projectionStore = yield* ProjectionStore.ProjectionStoreV2; | ||
| 1259 | const now = yield* DateTime.now; | ||
| 1260 | const threadId = ThreadId.make("thread-provider-session-manager-stopped-turn-start"); | ||
| 1261 | const providerSessionId = yield* idAllocator.allocate.providerSession({ | ||
| 1262 | providerInstanceId: modelSelection.instanceId, | ||
| 1263 | threadId, | ||
| 1264 | }); | ||
| 1265 | yield* eventSink.write({ | ||
| 1266 | events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], | ||
| 1267 | }); | ||
| 1268 | const runtime = yield* manager.open({ | ||
| 1269 | threadId, | ||
| 1270 | providerSessionId, | ||
| 1271 | modelSelection, | ||
| 1272 | runtimePolicy, | ||
| 1273 | }); | ||
| 1274 | const runId = idAllocator.derive.run({ threadId, ordinal: 1 }); | ||
| 1275 | const starting = yield* runtime | ||
| 1276 | .startTurn({ | ||
| 1277 | appThread: (yield* projectionStore.getThreadProjection(threadId)).thread, | ||
| 1278 | threadId, | ||
| 1279 | runId, | ||
| 1280 | runOrdinal: 1, | ||
| 1281 | providerTurnOrdinal: 1, | ||
| 1282 | attemptId: idAllocator.derive.runAttempt({ runId, attemptOrdinal: 1 }), | ||
| 1283 | rootNodeId: idAllocator.derive.rootNode({ runId }), | ||
| 1284 | providerThread: makeProviderThread({ idAllocator, threadId, providerSessionId, now }), | ||
| 1285 | message: { | ||
| 1286 | createdBy: "user", | ||
| 1287 | creationSource: "web", | ||
| 1288 | messageId: yield* idAllocator.allocate.message({ threadId, ordinal: 1 }), | ||
| 1289 | text: "stopped before the provider accepted it", | ||
| 1290 | attachments: [], | ||
| 1291 | }, | ||
| 1292 | modelSelection, | ||
| 1293 | runtimePolicy, | ||
| 1294 | }) | ||
| 1295 | .pipe(Effect.forkChild); | ||
| 1296 | yield* Deferred.await(startTurnReached); | ||
| 1297 | |||
| 1298 | // Stop lands while the provider is still accepting the turn. No terminal | ||
| 1299 | // event follows, so the session must count itself idle again. | ||
| 1300 | yield* Fiber.interrupt(starting); | ||
| 1301 | yield* TestClock.adjust("2 seconds"); | ||
| 1302 | yield* Effect.yieldNow; | ||
| 1303 | assert.isTrue(Option.isNone(yield* manager.get(providerSessionId))); | ||
| 1304 | assert.equal((yield* Ref.get(state)).closeCount, 1); | ||
| 1305 | }).pipe( | ||
| 1306 | Effect.provide( | ||
| 1307 | layerTest({ | ||
| 1308 | state, | ||
| 1309 | idleTimeoutMs: 1000, | ||
| 1310 | startTurn: Deferred.succeed(startTurnReached, undefined).pipe( | ||
| 1311 | Effect.andThen(Effect.never), | ||
| 1312 | ), | ||
| 1313 | }), | ||
| 1314 | ), | ||
| 1315 | ); | ||
| 1316 | }), | ||
| 1317 | ); | ||
| 1318 | |||
| 1319 | it.effect( | ||
| 1320 | "ProviderSessionManagerV2 keeps a shared session busy when another thread's start is stopped early", | ||
| 1321 | () => | ||
| 1322 | Effect.gen(function* () { | ||
| 1323 | const state = yield* Ref.make(emptyState); | ||
| 1324 | const projectId = ProjectId.make("project-provider-session-manager-shared-stopped-start"); | ||
| 1325 | // Blocks the next project read, so a re-attach can be stopped before its | ||
| 1326 | // start marks the session busy. | ||
| 1327 | const holdProjectRead = yield* Ref.make(false); | ||
| 1328 | const projectReadHeld = yield* Deferred.make<void>(); | ||
| 1329 | const projectServiceLayer = Layer.mock(ProjectService.ProjectService)({ | ||
| 1330 | getById: (requestedProjectId) => | ||
| 1331 | Ref.get(holdProjectRead).pipe( | ||
| 1332 | Effect.flatMap((hold) => | ||
| 1333 | hold | ||
| 1334 | ? Deferred.succeed(projectReadHeld, undefined).pipe(Effect.andThen(Effect.never)) | ||
| 1335 | : Effect.succeed(Option.some(makeBrowserAccessProject(requestedProjectId))), | ||
| 1336 | ), | ||
| 1337 | ), | ||
| 1338 | }); | ||
| 1339 | yield* Effect.gen(function* () { | ||
| 1340 | const eventSink = yield* EventSink.EventSinkV2; | ||
| 1341 | const idAllocator = yield* IdAllocator.IdAllocatorV2; | ||
| 1342 | const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; | ||
| 1343 | const projectionStore = yield* ProjectionStore.ProjectionStoreV2; | ||
| 1344 | const now = yield* DateTime.now; | ||
| 1345 | const threadA = ThreadId.make("thread-provider-session-manager-shared-stopped-a"); | ||
| 1346 | const threadB = ThreadId.make("thread-provider-session-manager-shared-stopped-b"); | ||
| 1347 | const providerSessionId = idAllocator.derive.providerSession({ | ||
| 1348 | providerInstanceId: modelSelection.instanceId, | ||
| 1349 | }); | ||
| 1350 | yield* eventSink.write({ | ||
| 1351 | events: [ | ||
| 1352 | yield* makeThreadCreatedEvent({ idAllocator, threadId: threadA, now, projectId }), | ||
| 1353 | yield* makeThreadCreatedEvent({ idAllocator, threadId: threadB, now, projectId }), | ||
| 1354 | ], | ||
| 1355 | }); | ||
| 1356 | const runtime = yield* manager.open({ | ||
| 1357 | threadId: threadA, | ||
| 1358 | providerSessionId, | ||
| 1359 | modelSelection, | ||
| 1360 | runtimePolicy, | ||
| 1361 | }); | ||
| 1362 | yield* manager.open({ | ||
| 1363 | threadId: threadB, | ||
| 1364 | providerSessionId, | ||
| 1365 | modelSelection, | ||
| 1366 | runtimePolicy, | ||
| 1367 | }); | ||
| 1368 | const startTurn = (threadId: ThreadId) => | ||
| 1369 | Effect.gen(function* () { | ||
| 1370 | const runId = idAllocator.derive.run({ threadId, ordinal: 1 }); | ||
| 1371 | return yield* runtime.startTurn({ | ||
| 1372 | appThread: (yield* projectionStore.getThreadProjection(threadId)).thread, | ||
| 1373 | threadId, | ||
| 1374 | runId, | ||
| 1375 | runOrdinal: 1, | ||
| 1376 | providerTurnOrdinal: 1, | ||
| 1377 | attemptId: idAllocator.derive.runAttempt({ runId, attemptOrdinal: 1 }), | ||
| 1378 | rootNodeId: idAllocator.derive.rootNode({ runId }), | ||
| 1379 | providerThread: makeProviderThread({ idAllocator, threadId, providerSessionId, now }), | ||
| 1380 | message: { | ||
| 1381 | createdBy: "user", | ||
| 1382 | creationSource: "web", | ||
| 1383 | messageId: yield* idAllocator.allocate.message({ threadId, ordinal: 1 }), | ||
| 1384 | text: "turn", | ||
| 1385 | attachments: [], | ||
| 1386 | }, | ||
| 1387 | modelSelection, | ||
| 1388 | runtimePolicy, | ||
| 1389 | }); | ||
| 1390 | }); | ||
| 1391 | |||
| 1392 | // B's turn is accepted and still running. | ||
| 1393 | yield* startTurn(threadB); | ||
| 1394 | |||
| 1395 | // A detaches, and its next start is stopped while re-attaching, before | ||
| 1396 | // it marks the session busy. | ||
| 1397 | yield* manager.detach({ providerSessionId, threadId: threadA }); | ||
| 1398 | yield* Ref.set(holdProjectRead, true); | ||
| 1399 | const startingA = yield* startTurn(threadA).pipe(Effect.forkChild); | ||
| 1400 | yield* Deferred.await(projectReadHeld); | ||
| 1401 | yield* Fiber.interrupt(startingA); | ||
| 1402 | |||
| 1403 | // B's running turn keeps the session busy past the idle timeout. | ||
| 1404 | yield* TestClock.adjust("2 seconds"); | ||
| 1405 | yield* Effect.yieldNow; | ||
| 1406 | assert.isTrue(Option.isSome(yield* manager.get(providerSessionId))); | ||
| 1407 | assert.equal((yield* Ref.get(state)).closeCount, 0); | ||
| 1408 | }).pipe( | ||
| 1409 | Effect.provide( | ||
| 1410 | layerTest({ | ||
| 1411 | state, | ||
| 1412 | idleTimeoutMs: 1000, | ||
| 1413 | projectServiceLayer, | ||
| 1414 | serverSettingsLayer: ServerSettings.layerTest({ | ||
| 1415 | projectSettingsOverrides: { [projectId]: { enableAgentBrowserAccess: true } }, | ||
| 1416 | }), | ||
| 1417 | }), | ||
| 1418 | ), | ||
| 1419 | ); | ||
| 1420 | }), | ||
| 1421 | ); | ||
| 1422 | |||
| 1423 | it.effect( | ||
| 1424 | "ProviderSessionManagerV2 keeps a shared session busy when a stopped start still ends its turn", | ||
| 1425 | () => | ||
| 1426 | Effect.gen(function* () { | ||
| 1427 | const state = yield* Ref.make(emptyState); | ||
| 1428 | // A's adapter start installs its turn, then is stopped mid-request; like | ||
| 1429 | // OpenCode2, the adapter still ends that turn with turn.terminal later. | ||
| 1430 | const holdStart = yield* Ref.make(false); | ||
| 1431 | const startHeld = yield* Deferred.make<void>(); | ||
| 1432 | yield* Effect.gen(function* () { | ||
| 1433 | const eventSink = yield* EventSink.EventSinkV2; | ||
| 1434 | const idAllocator = yield* IdAllocator.IdAllocatorV2; | ||
| 1435 | const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; | ||
| 1436 | const projectionStore = yield* ProjectionStore.ProjectionStoreV2; | ||
| 1437 | const now = yield* DateTime.now; | ||
| 1438 | const threadA = ThreadId.make("thread-provider-session-manager-stopped-ended-a"); | ||
| 1439 | const threadB = ThreadId.make("thread-provider-session-manager-stopped-ended-b"); | ||
| 1440 | const providerSessionId = idAllocator.derive.providerSession({ | ||
| 1441 | providerInstanceId: modelSelection.instanceId, | ||
| 1442 | }); | ||
| 1443 | yield* eventSink.write({ | ||
| 1444 | events: [ | ||
| 1445 | yield* makeThreadCreatedEvent({ idAllocator, threadId: threadA, now }), | ||
| 1446 | yield* makeThreadCreatedEvent({ idAllocator, threadId: threadB, now }), | ||
| 1447 | ], | ||
| 1448 | }); | ||
| 1449 | const runtime = yield* manager.open({ | ||
| 1450 | threadId: threadA, | ||
| 1451 | providerSessionId, | ||
| 1452 | modelSelection, | ||
| 1453 | runtimePolicy, | ||
| 1454 | }); | ||
| 1455 | yield* manager.open({ | ||
| 1456 | threadId: threadB, | ||
| 1457 | providerSessionId, | ||
| 1458 | modelSelection, | ||
| 1459 | runtimePolicy, | ||
| 1460 | }); | ||
| 1461 | const providerThreadOf = (threadId: ThreadId) => | ||
| 1462 | makeProviderThread({ | ||
| 1463 | idAllocator, | ||
| 1464 | threadId, | ||
| 1465 | providerSessionId, | ||
| 1466 | now, | ||
| 1467 | nativeThreadId: `native-${threadId}`, | ||
| 1468 | }); | ||
| 1469 | const startTurn = (threadId: ThreadId) => | ||
| 1470 | Effect.gen(function* () { | ||
| 1471 | const runId = idAllocator.derive.run({ threadId, ordinal: 1 }); | ||
| 1472 | return yield* runtime.startTurn({ | ||
| 1473 | appThread: (yield* projectionStore.getThreadProjection(threadId)).thread, | ||
| 1474 | threadId, | ||
| 1475 | runId, | ||
| 1476 | runOrdinal: 1, | ||
| 1477 | providerTurnOrdinal: 1, | ||
| 1478 | attemptId: idAllocator.derive.runAttempt({ runId, attemptOrdinal: 1 }), | ||
| 1479 | rootNodeId: idAllocator.derive.rootNode({ runId }), | ||
| 1480 | providerThread: providerThreadOf(threadId), | ||
| 1481 | message: { | ||
| 1482 | createdBy: "user", | ||
| 1483 | creationSource: "web", | ||
| 1484 | messageId: yield* idAllocator.allocate.message({ threadId, ordinal: 1 }), | ||
| 1485 | text: "turn", | ||
| 1486 | attachments: [], | ||
| 1487 | }, | ||
| 1488 | modelSelection, | ||
| 1489 | runtimePolicy, | ||
| 1490 | }); | ||
| 1491 | }); | ||
| 1492 | |||
| 1493 | // B's turn is accepted and still running. | ||
| 1494 | yield* startTurn(threadB); | ||
| 1495 | |||
| 1496 | // A's start is stopped after the adapter began the turn. | ||
| 1497 | yield* Ref.set(holdStart, true); | ||
| 1498 | const startingA = yield* startTurn(threadA).pipe(Effect.forkChild); | ||
| 1499 | yield* Deferred.await(startHeld); | ||
| 1500 | yield* Fiber.interrupt(startingA); | ||
| 1501 | |||
| 1502 | // The adapter ends A's turn anyway. | ||
| 1503 | const queue = (yield* Ref.get(state)).eventQueues.get(String(providerSessionId)); | ||
| 1504 | assert.isDefined(queue); | ||
| 1505 | yield* Queue.offer(queue!, { | ||
| 1506 | type: "turn.terminal", | ||
| 1507 | driver: CODEX_DRIVER, | ||
| 1508 | providerThreadId: providerThreadOf(threadA).id, | ||
| 1509 | providerTurnId: idAllocator.derive.providerTurn({ | ||
| 1510 | driver: CODEX_DRIVER, | ||
| 1511 | nativeTurnId: "native-turn-stopped-a", | ||
| 1512 | }), | ||
| 1513 | runOrdinal: 1, | ||
| 1514 | status: "completed", | ||
| 1515 | failure: null, | ||
| 1516 | threadDisposition: "reusable", | ||
| 1517 | }); | ||
| 1518 | |||
| 1519 | // B's running turn keeps the session busy past the idle timeout. | ||
| 1520 | yield* TestClock.adjust("2 seconds"); | ||
| 1521 | yield* Effect.yieldNow; | ||
| 1522 | assert.isTrue(Option.isSome(yield* manager.get(providerSessionId))); | ||
| 1523 | assert.equal((yield* Ref.get(state)).closeCount, 0); | ||
| 1524 | }).pipe( | ||
| 1525 | Effect.provide( | ||
| 1526 | layerTest({ | ||
| 1527 | state, | ||
| 1528 | idleTimeoutMs: 1000, | ||
| 1529 | startTurn: Ref.get(holdStart).pipe( | ||
| 1530 | Effect.flatMap((hold) => | ||
| 1531 | hold | ||
| 1532 | ? Deferred.succeed(startHeld, undefined).pipe(Effect.andThen(Effect.never)) | ||
| 1533 | : Effect.void, | ||
| 1534 | ), | ||
| 1535 | ), | ||
| 1536 | }), | ||
| 1537 | ), | ||
| 1538 | ); | ||
| 1539 | }), | ||
| 1540 | ); | ||
| 1541 | |||
| 1542 | it.effect("ProviderSessionManagerV2 stops a session still opening when its layer shuts down", () => | ||
| 1543 | Effect.gen(function* () { | ||
| 1544 | const state = yield* Ref.make(emptyState); | ||
| 1545 | const handshakeStarted = yield* Deferred.make<void>(); | ||
| 1546 | yield* Effect.gen(function* () { | ||
| 1547 | const eventSink = yield* EventSink.EventSinkV2; | ||
| 1548 | const idAllocator = yield* IdAllocator.IdAllocatorV2; | ||
| 1549 | const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; | ||
| 1550 | const now = yield* DateTime.now; | ||
| 1551 | const threadId = ThreadId.make("thread-provider-session-manager-shutdown-during-open"); | ||
| 1552 | const providerSessionId = yield* idAllocator.allocate.providerSession({ | ||
| 1553 | providerInstanceId: modelSelection.instanceId, | ||
| 1554 | threadId, | ||
| 1555 | }); | ||
| 1556 | yield* eventSink.write({ | ||
| 1557 | events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], | ||
| 1558 | }); | ||
| 1559 | // Detached, like an open the layer does not own: only the session | ||
| 1560 | // scope's parent can stop its process when the layer closes. | ||
| 1561 | yield* manager | ||
| 1562 | .open({ threadId, providerSessionId, modelSelection, runtimePolicy }) | ||
| 1563 | .pipe(Effect.forkDetach); | ||
| 1564 | yield* Deferred.await(handshakeStarted); | ||
| 1565 | assert.equal((yield* Ref.get(state)).closeCount, 0); | ||
| 1566 | }).pipe( | ||
| 1567 | Effect.provide( | ||
| 1568 | layerTest({ | ||
| 1569 | state, | ||
| 1570 | idleTimeoutMs: 60_000, | ||
| 1571 | beforeOpen: () => | ||
| 1572 | Deferred.succeed(handshakeStarted, undefined).pipe(Effect.andThen(Effect.never)), | ||
| 1573 | spawnBeforeOpen: true, | ||
| 1574 | }), | ||
| 1575 | ), | ||
| 1576 | ); | ||
| 1577 | |||
| 1578 | assert.equal((yield* Ref.get(state)).closeCount, 1); | ||
| 1579 | }), | ||
| 1580 | ); | ||
| 1581 | |||
| 1000 | it.effect("ProviderSessionManagerV2 releases live sessions when its layer shuts down", () => | 1582 | it.effect("ProviderSessionManagerV2 releases live sessions when its layer shuts down", () => |
| 1001 | Effect.gen(function* () { | 1583 | Effect.gen(function* () { |
| 1002 | const state = yield* Ref.make(emptyState); | 1584 | const state = yield* Ref.make(emptyState); |
| ... | @@ -1368,6 +1950,228 @@ it.effect("ProviderSessionManagerV2 revokes MCP credentials when release persist | ... | @@ -1368,6 +1950,228 @@ it.effect("ProviderSessionManagerV2 revokes MCP credentials when release persist |
| 1368 | }), | 1950 | }), |
| 1369 | ); | 1951 | ); |
| 1370 | 1952 | ||
| 1953 | it.effect( | ||
| 1954 | "ProviderSessionManagerV2 keeps a thread attached when an earlier attach of it is interrupted", | ||
| 1955 | () => | ||
| 1956 | Effect.gen(function* () { | ||
| 1957 | const state = yield* Ref.make(emptyState); | ||
| 1958 | const armed = yield* Ref.make(false); | ||
| 1959 | const paused = yield* Deferred.make<void>(); | ||
| 1960 | yield* Effect.gen(function* () { | ||
| 1961 | const eventSink = yield* EventSink.EventSinkV2; | ||
| 1962 | const idAllocator = yield* IdAllocator.IdAllocatorV2; | ||
| 1963 | const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; | ||
| 1964 | const registry = yield* McpSessionRegistry.McpSessionRegistry; | ||
| 1965 | const now = yield* DateTime.now; | ||
| 1966 | const owner = ThreadId.make("thread-provider-session-manager-attach-race-owner"); | ||
| 1967 | const threadId = ThreadId.make("thread-provider-session-manager-attach-race"); | ||
| 1968 | const providerSessionId = idAllocator.derive.providerSession({ | ||
| 1969 | providerInstanceId: modelSelection.instanceId, | ||
| 1970 | }); | ||
| 1971 | yield* eventSink.write({ | ||
| 1972 | events: [ | ||
| 1973 | yield* makeThreadCreatedEvent({ idAllocator, threadId: owner, now }), | ||
| 1974 | yield* makeThreadCreatedEvent({ idAllocator, threadId, now }), | ||
| 1975 | ], | ||
| 1976 | }); | ||
| 1977 | const runtime = yield* manager.open({ | ||
| 1978 | threadId: owner, | ||
| 1979 | providerSessionId, | ||
| 1980 | modelSelection, | ||
| 1981 | runtimePolicy, | ||
| 1982 | }); | ||
| 1983 | const resume = runtime.resumeThread({ | ||
| 1984 | threadId, | ||
| 1985 | providerThread: makeProviderThread({ | ||
| 1986 | idAllocator, | ||
| 1987 | threadId, | ||
| 1988 | providerSessionId, | ||
| 1989 | now, | ||
| 1990 | nativeThreadId: "native-attach-race", | ||
| 1991 | }), | ||
| 1992 | }); | ||
| 1993 | |||
| 1994 | // The first attach of the thread is stopped after attaching it, while | ||
| 1995 | // a second attach of the same thread is already on its way. | ||
| 1996 | yield* Ref.set(armed, true); | ||
| 1997 | const first = yield* resume.pipe(Effect.forkChild({ startImmediately: true })); | ||
| 1998 | yield* Deferred.await(paused); | ||
| 1999 | const second = yield* resume.pipe(Effect.forkChild({ startImmediately: true })); | ||
| 2000 | yield* Fiber.interrupt(first); | ||
| 2001 | yield* Fiber.join(second); | ||
| 2002 | |||
| 2003 | // The second attach owns the thread's attachment and credential. | ||
| 2004 | const config = McpProviderSession.readMcpProviderSession(threadId); | ||
| 2005 | assert.isDefined(config); | ||
| 2006 | const token = config!.authorizationHeader.replace(/^Bearer\s+/, ""); | ||
| 2007 | assert.equal((yield* registry.resolve(token))?.thread.threadId, threadId); | ||
| 2008 | }).pipe( | ||
| 2009 | Effect.provide( | ||
| 2010 | layerTest({ state, idleTimeoutMs: 60_000, pauseAttachWrite: { armed, paused } }), | ||
| 2011 | ), | ||
| 2012 | ); | ||
| 2013 | }), | ||
| 2014 | ); | ||
| 2015 | |||
| 2016 | it.effect( | ||
| 2017 | "ProviderSessionManagerV2 keeps a replacement session's attachment when a stale attach is interrupted", | ||
| 2018 | () => | ||
| 2019 | Effect.gen(function* () { | ||
| 2020 | const state = yield* Ref.make(emptyState); | ||
| 2021 | const armed = yield* Ref.make(false); | ||
| 2022 | const paused = yield* Deferred.make<void>(); | ||
| 2023 | yield* Effect.gen(function* () { | ||
| 2024 | const eventSink = yield* EventSink.EventSinkV2; | ||
| 2025 | const idAllocator = yield* IdAllocator.IdAllocatorV2; | ||
| 2026 | const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; | ||
| 2027 | const registry = yield* McpSessionRegistry.McpSessionRegistry; | ||
| 2028 | const eventStore = yield* EventStore.EventStoreV2; | ||
| 2029 | const now = yield* DateTime.now; | ||
| 2030 | const owner = ThreadId.make("thread-provider-session-manager-stale-attach-owner"); | ||
| 2031 | const threadId = ThreadId.make("thread-provider-session-manager-stale-attach"); | ||
| 2032 | const providerSessionId = idAllocator.derive.providerSession({ | ||
| 2033 | providerInstanceId: modelSelection.instanceId, | ||
| 2034 | }); | ||
| 2035 | yield* eventSink.write({ | ||
| 2036 | events: [ | ||
| 2037 | yield* makeThreadCreatedEvent({ idAllocator, threadId: owner, now }), | ||
| 2038 | yield* makeThreadCreatedEvent({ idAllocator, threadId, now }), | ||
| 2039 | ], | ||
| 2040 | }); | ||
| 2041 | const runtime = yield* manager.open({ | ||
| 2042 | threadId: owner, | ||
| 2043 | providerSessionId, | ||
| 2044 | modelSelection, | ||
| 2045 | runtimePolicy, | ||
| 2046 | }); | ||
| 2047 | |||
| 2048 | // An attach of the thread to this session stalls mid-write... | ||
| 2049 | yield* Ref.set(armed, true); | ||
| 2050 | const stale = yield* runtime | ||
| 2051 | .resumeThread({ | ||
| 2052 | threadId, | ||
| 2053 | providerThread: makeProviderThread({ | ||
| 2054 | idAllocator, | ||
| 2055 | threadId, | ||
| 2056 | providerSessionId, | ||
| 2057 | now, | ||
| 2058 | nativeThreadId: "native-stale-attach", | ||
| 2059 | }), | ||
| 2060 | }) | ||
| 2061 | .pipe(Effect.forkChild({ startImmediately: true })); | ||
| 2062 | yield* Deferred.await(paused); | ||
| 2063 | |||
| 2064 | // ...while the session is replaced and the thread opens the new one. | ||
| 2065 | yield* manager.release({ providerSessionId, reason: "runtime_error" }); | ||
| 2066 | const replacement = yield* manager.open({ | ||
| 2067 | threadId, | ||
| 2068 | providerSessionId, | ||
| 2069 | modelSelection, | ||
| 2070 | runtimePolicy, | ||
| 2071 | }); | ||
| 2072 | const config = McpProviderSession.readMcpProviderSession(threadId); | ||
| 2073 | assert.isDefined(config); | ||
| 2074 | |||
| 2075 | yield* Fiber.interrupt(stale); | ||
| 2076 | |||
| 2077 | // The replacement keeps the thread and its credential. | ||
| 2078 | assert.equal( | ||
| 2079 | McpProviderSession.readMcpProviderSession(threadId)?.providerSessionId, | ||
| 2080 | config!.providerSessionId, | ||
| 2081 | ); | ||
| 2082 | const token = config!.authorizationHeader.replace(/^Bearer\s+/, ""); | ||
| 2083 | assert.equal((yield* registry.resolve(token))?.thread.threadId, threadId); | ||
| 2084 | // Still attached: resuming on the replacement does not attach the thread again. | ||
| 2085 | const attachedEvents = eventStore | ||
| 2086 | .read({ threadId, eventType: "provider-session.attached" }) | ||
| 2087 | .pipe( | ||
| 2088 | Stream.runCollect, | ||
| 2089 | Effect.map((events) => events.length), | ||
| 2090 | ); | ||
| 2091 | const attachedBefore = yield* attachedEvents; | ||
| 2092 | yield* replacement.resumeThread({ | ||
| 2093 | threadId, | ||
| 2094 | providerThread: makeProviderThread({ | ||
| 2095 | idAllocator, | ||
| 2096 | threadId, | ||
| 2097 | providerSessionId, | ||
| 2098 | now, | ||
| 2099 | nativeThreadId: "native-stale-attach", | ||
| 2100 | }), | ||
| 2101 | }); | ||
| 2102 | assert.equal(yield* attachedEvents, attachedBefore); | ||
| 2103 | }).pipe( | ||
| 2104 | Effect.provide( | ||
| 2105 | layerTest({ state, idleTimeoutMs: 60_000, pauseAttachWrite: { armed, paused } }), | ||
| 2106 | ), | ||
| 2107 | ); | ||
| 2108 | }), | ||
| 2109 | ); | ||
| 2110 | |||
| 2111 | it.effect( | ||
| 2112 | "ProviderSessionManagerV2 revokes a reused credential after a resume stopped while checking it", | ||
| 2113 | () => | ||
| 2114 | Effect.gen(function* () { | ||
| 2115 | const state = yield* Ref.make(emptyState); | ||
| 2116 | const armed = yield* Ref.make(false); | ||
| 2117 | const paused = yield* Deferred.make<void>(); | ||
| 2118 | yield* Effect.gen(function* () { | ||
| 2119 | const eventSink = yield* EventSink.EventSinkV2; | ||
| 2120 | const idAllocator = yield* IdAllocator.IdAllocatorV2; | ||
| 2121 | const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; | ||
| 2122 | const registry = yield* McpSessionRegistry.McpSessionRegistry; | ||
| 2123 | const now = yield* DateTime.now; | ||
| 2124 | const owner = ThreadId.make("thread-provider-session-manager-resolve-stop-owner"); | ||
| 2125 | const threadId = ThreadId.make("thread-provider-session-manager-resolve-stop"); | ||
| 2126 | const providerSessionId = idAllocator.derive.providerSession({ | ||
| 2127 | providerInstanceId: modelSelection.instanceId, | ||
| 2128 | }); | ||
| 2129 | yield* eventSink.write({ | ||
| 2130 | events: [ | ||
| 2131 | yield* makeThreadCreatedEvent({ idAllocator, threadId: owner, now }), | ||
| 2132 | yield* makeThreadCreatedEvent({ idAllocator, threadId, now }), | ||
| 2133 | ], | ||
| 2134 | }); | ||
| 2135 | const runtime = yield* manager.open({ | ||
| 2136 | threadId: owner, | ||
| 2137 | providerSessionId, | ||
| 2138 | modelSelection, | ||
| 2139 | runtimePolicy, | ||
| 2140 | }); | ||
| 2141 | const resume = runtime.resumeThread({ | ||
| 2142 | threadId, | ||
| 2143 | providerThread: makeProviderThread({ | ||
| 2144 | idAllocator, | ||
| 2145 | threadId, | ||
| 2146 | providerSessionId, | ||
| 2147 | now, | ||
| 2148 | nativeThreadId: "native-resolve-stop", | ||
| 2149 | }), | ||
| 2150 | }); | ||
| 2151 | // The thread gets a credential, then detaches and keeps it for a re-attach. | ||
| 2152 | yield* resume; | ||
| 2153 | yield* manager.detach({ providerSessionId, threadId }); | ||
| 2154 | const config = McpProviderSession.readMcpProviderSession(threadId); | ||
| 2155 | assert.isDefined(config); | ||
| 2156 | const token = config!.authorizationHeader.replace(/^Bearer\s+/, ""); | ||
| 2157 | |||
| 2158 | // A re-attach is stopped while it checks whether that credential is reusable. | ||
| 2159 | yield* Ref.set(armed, true); | ||
| 2160 | const stopped = yield* resume.pipe(Effect.forkChild({ startImmediately: true })); | ||
| 2161 | yield* Deferred.await(paused); | ||
| 2162 | yield* Fiber.interrupt(stopped); | ||
| 2163 | |||
| 2164 | // Nothing holds the credential now, so a terminal release revokes it. | ||
| 2165 | yield* manager.release({ providerSessionId, reason: "manual_shutdown" }); | ||
| 2166 | assert.isUndefined(yield* registry.resolve(token)); | ||
| 2167 | }).pipe( | ||
| 2168 | Effect.provide( | ||
| 2169 | layerTest({ state, idleTimeoutMs: 60_000, pauseResolve: { armed, paused } }), | ||
| 2170 | ), | ||
| 2171 | ); | ||
| 2172 | }), | ||
| 2173 | ); | ||
| 2174 | |||
| 1371 | it.effect("ProviderSessionManagerV2 duplicate detach preserves replacement MCP credentials", () => | 2175 | it.effect("ProviderSessionManagerV2 duplicate detach preserves replacement MCP credentials", () => |
| 1372 | Effect.gen(function* () { | 2176 | Effect.gen(function* () { |
| 1373 | const state = yield* Ref.make(emptyState); | 2177 | const state = yield* Ref.make(emptyState); |
| ... | @@ -3011,17 +3815,20 @@ it.effect( | ... | @@ -3011,17 +3815,20 @@ it.effect( |
| 3011 | providerInstanceId: modelSelection.instanceId, | 3815 | providerInstanceId: modelSelection.instanceId, |
| 3012 | threadId: firstThreadId, | 3816 | threadId: firstThreadId, |
| 3013 | }); | 3817 | }); |
| 3818 | // Each thread has its own native thread on the shared session. | ||
| 3014 | const firstProviderThread = makeProviderThread({ | 3819 | const firstProviderThread = makeProviderThread({ |
| 3015 | idAllocator, | 3820 | idAllocator, |
| 3016 | threadId: firstThreadId, | 3821 | threadId: firstThreadId, |
| 3017 | providerSessionId, | 3822 | providerSessionId, |
| 3018 | now, | 3823 | now, |
| 3824 | nativeThreadId: "native-thread-a", | ||
| 3019 | }); | 3825 | }); |
| 3020 | const secondProviderThread = makeProviderThread({ | 3826 | const secondProviderThread = makeProviderThread({ |
| 3021 | idAllocator, | 3827 | idAllocator, |
| 3022 | threadId: secondThreadId, | 3828 | threadId: secondThreadId, |
| 3023 | providerSessionId, | 3829 | providerSessionId, |
| 3024 | now, | 3830 | now, |
| 3831 | nativeThreadId: "native-thread-b", | ||
| 3025 | }); | 3832 | }); |
| 3026 | const firstRunId = idAllocator.derive.run({ threadId: firstThreadId, ordinal: 1 }); | 3833 | const firstRunId = idAllocator.derive.run({ threadId: firstThreadId, ordinal: 1 }); |
| 3027 | const secondRunId = idAllocator.derive.run({ threadId: secondThreadId, ordinal: 1 }); | 3834 | const secondRunId = idAllocator.derive.run({ threadId: secondThreadId, ordinal: 1 }); |
apps/server/src/orchestration-v2/ProviderSessionManager.ts+219-102
| ... | @@ -9,6 +9,7 @@ import { | ... | @@ -9,6 +9,7 @@ import { |
| 9 | ProviderInstanceId, | 9 | ProviderInstanceId, |
| 10 | ProviderSessionId, | 10 | ProviderSessionId, |
| 11 | ThreadId, | 11 | ThreadId, |
| 12 | type ProviderThreadId, | ||
| 12 | } from "@t3tools/contracts"; | 13 | } from "@t3tools/contracts"; |
| 13 | import * as Cause from "effect/Cause"; | 14 | import * as Cause from "effect/Cause"; |
| 14 | import * as Clock from "effect/Clock"; | 15 | import * as Clock from "effect/Clock"; |
| ... | @@ -59,6 +60,10 @@ import * as ProjectionStore from "./ProjectionStore.ts"; | ... | @@ -59,6 +60,10 @@ import * as ProjectionStore from "./ProjectionStore.ts"; |
| 59 | const DEFAULT_IDLE_TIMEOUT_MS = 30 * 60 * 1000; | 60 | const DEFAULT_IDLE_TIMEOUT_MS = 30 * 60 * 1000; |
| 60 | const DEFAULT_MAX_IDLE_PIN_MS = 4 * 60 * 60 * 1000; | 61 | const DEFAULT_MAX_IDLE_PIN_MS = 4 * 60 * 60 * 1000; |
| 61 | const RELEASE_SCOPE_CLOSE_TIMEOUT_MS = 30 * 1000; | 62 | const RELEASE_SCOPE_CLOSE_TIMEOUT_MS = 30 * 1000; |
| 63 | |||
| 64 | /** The identity a turn's start and its `turn.terminal` share. */ | ||
| 65 | const busyTurnKey = (providerThreadId: ProviderThreadId, runOrdinal: number) => | ||
| 66 | `${providerThreadId}#${runOrdinal}`; | ||
| 62 | const UNLOAD_THREAD_TIMEOUT_MS = 10 * 1000; | 67 | const UNLOAD_THREAD_TIMEOUT_MS = 10 * 1000; |
| 63 | 68 | ||
| 64 | export const ProviderSessionReleaseReason = Schema.Literals([ | 69 | export const ProviderSessionReleaseReason = Schema.Literals([ |
| ... | @@ -213,7 +218,12 @@ interface LiveSessionEntry { | ... | @@ -213,7 +218,12 @@ interface LiveSessionEntry { |
| 213 | readonly requestEventPermit: Semaphore.Semaphore; | 218 | readonly requestEventPermit: Semaphore.Semaphore; |
| 214 | readonly scope: Scope.Closeable; | 219 | readonly scope: Scope.Closeable; |
| 215 | readonly idleGeneration: number; | 220 | readonly idleGeneration: number; |
| 216 | readonly busyCount: number; | 221 | /** |
| 222 | * Turns this session is running, keyed by `busyTurnKey`. A turn's start adds | ||
| 223 | * it and its `turn.terminal` (or a failed start) removes it, so a turn can | ||
| 224 | * only clear itself and the session is idle when the set is empty. | ||
| 225 | */ | ||
| 226 | readonly busyTurns: ReadonlySet<string>; | ||
| 217 | readonly lastActivityAtMs: number; | 227 | readonly lastActivityAtMs: number; |
| 218 | readonly idleFiber: Fiber.Fiber<void, never> | null; | 228 | readonly idleFiber: Fiber.Fiber<void, never> | null; |
| 219 | /** Set when idle release is deferred for pending background work; bounds total deferral. */ | 229 | /** Set when idle release is deferred for pending background work; bounds total deferral. */ |
| ... | @@ -472,7 +482,17 @@ export const layerWithOptions = ( | ... | @@ -472,7 +482,17 @@ export const layerWithOptions = ( |
| 472 | // revoke the credential between validation and reservation. | 482 | // revoke the credential between validation and reservation. |
| 473 | reserveMcpCredential(threadId, existing.providerSessionId); | 483 | reserveMcpCredential(threadId, existing.providerSessionId); |
| 474 | const rawToken = existing.authorizationHeader.replace(/^Bearer\s+/, ""); | 484 | const rawToken = existing.authorizationHeader.replace(/^Bearer\s+/, ""); |
| 475 | const resolved = yield* mcpSessionRegistry.resolve(rawToken); | 485 | // The caller only learns of the reservation once this returns, |
| 486 | // so a stop while resolving must drop it here. | ||
| 487 | const resolved = yield* mcpSessionRegistry | ||
| 488 | .resolve(rawToken) | ||
| 489 | .pipe( | ||
| 490 | Effect.onInterrupt(() => | ||
| 491 | Effect.sync(() => | ||
| 492 | dropMcpCredentialReservation(threadId, existing.providerSessionId), | ||
| 493 | ), | ||
| 494 | ), | ||
| 495 | ); | ||
| 476 | if ( | 496 | if ( |
| 477 | resolved !== undefined && | 497 | resolved !== undefined && |
| 478 | resolved.thread.threadId === threadId && | 498 | resolved.thread.threadId === threadId && |
| ... | @@ -842,7 +862,8 @@ export const layerWithOptions = ( | ... | @@ -842,7 +862,8 @@ export const layerWithOptions = ( |
| 842 | } | 862 | } |
| 843 | if ( | 863 | if ( |
| 844 | input.onlyIfIdleGeneration !== undefined && | 864 | input.onlyIfIdleGeneration !== undefined && |
| 845 | (existing.busyCount > 0 || existing.idleGeneration !== input.onlyIfIdleGeneration) | 865 | (existing.busyTurns.size > 0 || |
| 866 | existing.idleGeneration !== input.onlyIfIdleGeneration) | ||
| 846 | ) { | 867 | ) { |
| 847 | return ["kept", current] as const; | 868 | return ["kept", current] as const; |
| 848 | } | 869 | } |
| ... | @@ -861,6 +882,45 @@ export const layerWithOptions = ( | ... | @@ -861,6 +882,45 @@ export const layerWithOptions = ( |
| 861 | ] as const; | 882 | ] as const; |
| 862 | }); | 883 | }); |
| 863 | 884 | ||
| 885 | // Scope close can wedge on a misbehaving adapter finalizer (e.g. a | ||
| 886 | // provider process that never yields its message stream). Time-box it so | ||
| 887 | // the caller, and any lock or worker it holds, moves on and leaves a | ||
| 888 | // diagnosable trail. A close that finishes late is still logged. | ||
| 889 | const closeScopeWithin = ( | ||
| 890 | scope: Scope.Closeable, | ||
| 891 | annotations: { readonly providerSessionId?: ProviderSessionId; readonly reason: string }, | ||
| 892 | ) => | ||
| 893 | Effect.gen(function* () { | ||
| 894 | const closeFiber = yield* Scope.close(scope, Exit.void).pipe( | ||
| 895 | Effect.exit, | ||
| 896 | Effect.forkDetach({ startImmediately: true }), | ||
| 897 | ); | ||
| 898 | const closeExit = yield* Fiber.join(closeFiber).pipe( | ||
| 899 | Effect.timeoutOption(RELEASE_SCOPE_CLOSE_TIMEOUT_MS), | ||
| 900 | ); | ||
| 901 | if (Option.isNone(closeExit)) { | ||
| 902 | yield* Effect.logWarning("orchestration-v2.provider-session-scope-close-timeout", { | ||
| 903 | ...annotations, | ||
| 904 | timeoutMs: RELEASE_SCOPE_CLOSE_TIMEOUT_MS, | ||
| 905 | }); | ||
| 906 | yield* Fiber.join(closeFiber).pipe( | ||
| 907 | Effect.flatMap((exit) => | ||
| 908 | Exit.isFailure(exit) | ||
| 909 | ? Effect.logWarning("orchestration-v2.provider-session-scope-close-failed", { | ||
| 910 | ...annotations, | ||
| 911 | cause: exit.cause, | ||
| 912 | }) | ||
| 913 | : Effect.logInfo( | ||
| 914 | "orchestration-v2.provider-session-scope-close-completed-late", | ||
| 915 | annotations, | ||
| 916 | ), | ||
| 917 | ), | ||
| 918 | Effect.forkDetach, | ||
| 919 | ); | ||
| 920 | } | ||
| 921 | return closeExit; | ||
| 922 | }); | ||
| 923 | |||
| 864 | const releaseEntry = (input: { | 924 | const releaseEntry = (input: { |
| 865 | readonly providerSessionId: ProviderSessionId; | 925 | readonly providerSessionId: ProviderSessionId; |
| 866 | readonly reason: ProviderSessionReleaseReason; | 926 | readonly reason: ProviderSessionReleaseReason; |
| ... | @@ -889,49 +949,10 @@ export const layerWithOptions = ( | ... | @@ -889,49 +949,10 @@ export const layerWithOptions = ( |
| 889 | input.detail ?? `Provider session released: ${input.reason}.`, | 949 | input.detail ?? `Provider session released: ${input.reason}.`, |
| 890 | ); | 950 | ); |
| 891 | } | 951 | } |
| 892 | // Scope close can wedge on a misbehaving adapter finalizer | 952 | const closeExit = yield* closeScopeWithin(entry.scope, { |
| 893 | // (e.g. a provider process that never yields its message | 953 | providerSessionId: input.providerSessionId, |
| 894 | // stream). Time-box it so release still persists released | 954 | reason: input.reason, |
| 895 | // events and leaves a diagnosable trail instead of silently | 955 | }); |
| 896 | // parking the session as "ready" forever. | ||
| 897 | const closeFiber = yield* Scope.close(entry.scope, Exit.void).pipe( | ||
| 898 | Effect.exit, | ||
| 899 | Effect.forkDetach({ startImmediately: true }), | ||
| 900 | ); | ||
| 901 | const closeExit = yield* Fiber.join(closeFiber).pipe( | ||
| 902 | Effect.timeoutOption(RELEASE_SCOPE_CLOSE_TIMEOUT_MS), | ||
| 903 | ); | ||
| 904 | if (Option.isNone(closeExit)) { | ||
| 905 | yield* Effect.logWarning( | ||
| 906 | "orchestration-v2.provider-session-scope-close-timeout", | ||
| 907 | { | ||
| 908 | providerSessionId: input.providerSessionId, | ||
| 909 | reason: input.reason, | ||
| 910 | timeoutMs: RELEASE_SCOPE_CLOSE_TIMEOUT_MS, | ||
| 911 | }, | ||
| 912 | ); | ||
| 913 | yield* Fiber.join(closeFiber).pipe( | ||
| 914 | Effect.flatMap((exit) => | ||
| 915 | Exit.isFailure(exit) | ||
| 916 | ? Effect.logWarning( | ||
| 917 | "orchestration-v2.provider-session-scope-close-failed", | ||
| 918 | { | ||
| 919 | providerSessionId: input.providerSessionId, | ||
| 920 | reason: input.reason, | ||
| 921 | cause: exit.cause, | ||
| 922 | }, | ||
| 923 | ) | ||
| 924 | : Effect.logInfo( | ||
| 925 | "orchestration-v2.provider-session-scope-close-completed-late", | ||
| 926 | { | ||
| 927 | providerSessionId: input.providerSessionId, | ||
| 928 | reason: input.reason, | ||
| 929 | }, | ||
| 930 | ), | ||
| 931 | ), | ||
| 932 | Effect.forkDetach, | ||
| 933 | ); | ||
| 934 | } | ||
| 935 | const records = { | 956 | const records = { |
| 936 | entry, | 957 | entry, |
| 937 | reason: input.reason, | 958 | reason: input.reason, |
| ... | @@ -1022,7 +1043,7 @@ export const layerWithOptions = ( | ... | @@ -1022,7 +1043,7 @@ export const layerWithOptions = ( |
| 1022 | const entry = current.get(key); | 1043 | const entry = current.get(key); |
| 1023 | if ( | 1044 | if ( |
| 1024 | entry === undefined || | 1045 | entry === undefined || |
| 1025 | entry.busyCount > 0 || | 1046 | entry.busyTurns.size > 0 || |
| 1026 | entry.idleGeneration !== input.generation | 1047 | entry.idleGeneration !== input.generation |
| 1027 | ) { | 1048 | ) { |
| 1028 | return; | 1049 | return; |
| ... | @@ -1044,7 +1065,7 @@ export const layerWithOptions = ( | ... | @@ -1044,7 +1065,7 @@ export const layerWithOptions = ( |
| 1044 | const latestEntry = latest.get(key); | 1065 | const latestEntry = latest.get(key); |
| 1045 | if ( | 1066 | if ( |
| 1046 | latestEntry === undefined || | 1067 | latestEntry === undefined || |
| 1047 | latestEntry.busyCount > 0 || | 1068 | latestEntry.busyTurns.size > 0 || |
| 1048 | latestEntry.idleGeneration !== input.generation || | 1069 | latestEntry.idleGeneration !== input.generation || |
| 1049 | latestEntry.runtime !== probedRuntime | 1070 | latestEntry.runtime !== probedRuntime |
| 1050 | ) { | 1071 | ) { |
| ... | @@ -1076,7 +1097,7 @@ export const layerWithOptions = ( | ... | @@ -1076,7 +1097,7 @@ export const layerWithOptions = ( |
| 1076 | } | 1097 | } |
| 1077 | // hasPendingBackgroundWork yields to the adapter, so the idle | 1098 | // hasPendingBackgroundWork yields to the adapter, so the idle |
| 1078 | // decision above can go stale; the generation guard revalidates | 1099 | // decision above can go stale; the generation guard revalidates |
| 1079 | // busyCount and idleGeneration inside releaseEntry's atomic | 1100 | // busyTurns and idleGeneration inside releaseEntry's atomic |
| 1080 | // entry removal. | 1101 | // entry removal. |
| 1081 | yield* releaseEntry({ | 1102 | yield* releaseEntry({ |
| 1082 | providerSessionId: input.providerSessionId, | 1103 | providerSessionId: input.providerSessionId, |
| ... | @@ -1113,7 +1134,7 @@ export const layerWithOptions = ( | ... | @@ -1113,7 +1134,7 @@ export const layerWithOptions = ( |
| 1113 | const key = sessionKey(providerSessionId); | 1134 | const key = sessionKey(providerSessionId); |
| 1114 | const current = yield* Ref.get(sessions); | 1135 | const current = yield* Ref.get(sessions); |
| 1115 | const entry = current.get(key); | 1136 | const entry = current.get(key); |
| 1116 | if (entry === undefined || entry.busyCount > 0) { | 1137 | if (entry === undefined || entry.busyTurns.size > 0) { |
| 1117 | return; | 1138 | return; |
| 1118 | } | 1139 | } |
| 1119 | 1140 | ||
| ... | @@ -1126,7 +1147,7 @@ export const layerWithOptions = ( | ... | @@ -1126,7 +1147,7 @@ export const layerWithOptions = ( |
| 1126 | const lastActivityAtMs = yield* Clock.currentTimeMillis; | 1147 | const lastActivityAtMs = yield* Clock.currentTimeMillis; |
| 1127 | yield* Ref.update(sessions, (latest) => { | 1148 | yield* Ref.update(sessions, (latest) => { |
| 1128 | const latestEntry = latest.get(key); | 1149 | const latestEntry = latest.get(key); |
| 1129 | if (latestEntry === undefined || latestEntry.busyCount > 0) { | 1150 | if (latestEntry === undefined || latestEntry.busyTurns.size > 0) { |
| 1130 | return latest; | 1151 | return latest; |
| 1131 | } | 1152 | } |
| 1132 | const updated = new Map(latest); | 1153 | const updated = new Map(latest); |
| ... | @@ -1164,6 +1185,7 @@ export const layerWithOptions = ( | ... | @@ -1164,6 +1185,7 @@ export const layerWithOptions = ( |
| 1164 | }), | 1185 | }), |
| 1165 | ); | 1186 | ); |
| 1166 | 1187 | ||
| 1188 | /** Returns the runtime the thread was attached to, or undefined if it already was. */ | ||
| 1167 | const attachThread = (input: { | 1189 | const attachThread = (input: { |
| 1168 | readonly providerSessionId: ProviderSessionId; | 1190 | readonly providerSessionId: ProviderSessionId; |
| 1169 | readonly threadId: ThreadId; | 1191 | readonly threadId: ThreadId; |
| ... | @@ -1173,25 +1195,34 @@ export const layerWithOptions = ( | ... | @@ -1173,25 +1195,34 @@ export const layerWithOptions = ( |
| 1173 | Ref.modify(sessions, (current) => { | 1195 | Ref.modify(sessions, (current) => { |
| 1174 | const entry = current.get(sessionKey(input.providerSessionId)); | 1196 | const entry = current.get(sessionKey(input.providerSessionId)); |
| 1175 | if (entry === undefined || entry.attachedThreadIds.has(input.threadId)) { | 1197 | if (entry === undefined || entry.attachedThreadIds.has(input.threadId)) { |
| 1176 | return [false, current] as const; | 1198 | return [undefined, current] as const; |
| 1177 | } | 1199 | } |
| 1178 | const updated = new Map(current); | 1200 | const updated = new Map(current); |
| 1179 | updated.set(sessionKey(input.providerSessionId), { | 1201 | updated.set(sessionKey(input.providerSessionId), { |
| 1180 | ...entry, | 1202 | ...entry, |
| 1181 | attachedThreadIds: new Set([...entry.attachedThreadIds, input.threadId]), | 1203 | attachedThreadIds: new Set([...entry.attachedThreadIds, input.threadId]), |
| 1182 | }); | 1204 | }); |
| 1183 | return [true, updated] as const; | 1205 | return [entry.runtime, updated] as const; |
| 1184 | }), | 1206 | }), |
| 1185 | ); | 1207 | ); |
| 1186 | 1208 | ||
| 1209 | /** | ||
| 1210 | * Undoes an attach to `runtime`. A replacement session that reopened under | ||
| 1211 | * the same id since is left alone. | ||
| 1212 | */ | ||
| 1187 | const removeThreadAttachment = (input: { | 1213 | const removeThreadAttachment = (input: { |
| 1188 | readonly providerSessionId: ProviderSessionId; | 1214 | readonly providerSessionId: ProviderSessionId; |
| 1189 | readonly threadId: ThreadId; | 1215 | readonly threadId: ThreadId; |
| 1216 | readonly runtime: ProviderAdapterV2SessionRuntime; | ||
| 1190 | }) => | 1217 | }) => |
| 1191 | Ref.update(sessions, (current) => { | 1218 | Ref.update(sessions, (current) => { |
| 1192 | const key = sessionKey(input.providerSessionId); | 1219 | const key = sessionKey(input.providerSessionId); |
| 1193 | const entry = current.get(key); | 1220 | const entry = current.get(key); |
| 1194 | if (entry === undefined || !entry.attachedThreadIds.has(input.threadId)) { | 1221 | if ( |
| 1222 | entry === undefined || | ||
| 1223 | entry.runtime !== input.runtime || | ||
| 1224 | !entry.attachedThreadIds.has(input.threadId) | ||
| 1225 | ) { | ||
| 1195 | return current; | 1226 | return current; |
| 1196 | } | 1227 | } |
| 1197 | const attachedThreadIds = new Set(entry.attachedThreadIds); | 1228 | const attachedThreadIds = new Set(entry.attachedThreadIds); |
| ... | @@ -1245,6 +1276,7 @@ export const layerWithOptions = ( | ... | @@ -1245,6 +1276,7 @@ export const layerWithOptions = ( |
| 1245 | readonly providerInstanceId: ProviderInstanceId; | 1276 | readonly providerInstanceId: ProviderInstanceId; |
| 1246 | }) => | 1277 | }) => |
| 1247 | Effect.suspend(() => { | 1278 | Effect.suspend(() => { |
| 1279 | let attachedTo: ProviderAdapterV2SessionRuntime | undefined; | ||
| 1248 | let preparedForCleanup: PreparedMcpCredential | undefined; | 1280 | let preparedForCleanup: PreparedMcpCredential | undefined; |
| 1249 | let reservationDropped = false; | 1281 | let reservationDropped = false; |
| 1250 | const dropReservation = () => { | 1282 | const dropReservation = () => { |
| ... | @@ -1253,12 +1285,17 @@ export const layerWithOptions = ( | ... | @@ -1253,12 +1285,17 @@ export const layerWithOptions = ( |
| 1253 | dropMcpCredentialReservation(input.threadId, preparedForCleanup.mcpCredentialId); | 1285 | dropMcpCredentialReservation(input.threadId, preparedForCleanup.mcpCredentialId); |
| 1254 | } | 1286 | } |
| 1255 | }; | 1287 | }; |
| 1256 | return Effect.gen(function* () { | 1288 | // The whole attach, including undoing a failed one, holds the |
| 1257 | const attached = yield* threadAttachment.withLock( | 1289 | // thread's lock: a concurrent attach of the same thread waits, so it |
| 1258 | threadAttachmentKey(input), | 1290 | // never sees an attachment that this call is about to roll back. |
| 1259 | attachThread(input), | 1291 | const attach = Effect.gen(function* () { |
| 1292 | const attached = yield* attachThread(input).pipe( | ||
| 1293 | // Recorded with no gap for an interrupt: cleanup undoes only an | ||
| 1294 | // attach this call made, never one an earlier open made. | ||
| 1295 | Effect.tap((runtime) => Effect.sync(() => (attachedTo = runtime))), | ||
| 1296 | Effect.uninterruptible, | ||
| 1260 | ); | 1297 | ); |
| 1261 | if (attached) { | 1298 | if (attached !== undefined) { |
| 1262 | const prepared = yield* prepareMcpSession(input.threadId, input.providerInstanceId); | 1299 | const prepared = yield* prepareMcpSession(input.threadId, input.providerInstanceId); |
| 1263 | preparedForCleanup = prepared; | 1300 | preparedForCleanup = prepared; |
| 1264 | if (prepared.mcpCredentialId !== undefined) { | 1301 | if (prepared.mcpCredentialId !== undefined) { |
| ... | @@ -1288,22 +1325,46 @@ export const layerWithOptions = ( | ... | @@ -1288,22 +1325,46 @@ export const layerWithOptions = ( |
| 1288 | } | 1325 | } |
| 1289 | } | 1326 | } |
| 1290 | }).pipe( | 1327 | }).pipe( |
| 1291 | Effect.tapError(() => | 1328 | // An interrupted attach is undone too, so the next attach writes |
| 1292 | removeThreadAttachment(input).pipe( | 1329 | // the attachment instead of finding the thread already attached. |
| 1293 | // Revoke only a credential this attach freshly minted: a REUSED | 1330 | Effect.onError(() => |
| 1294 | // credential is by definition held by another live provider | 1331 | attachedTo === undefined |
| 1295 | // process, and revoking it thread-wide would break that | 1332 | ? Effect.void |
| 1296 | // process's MCP client mid-conversation. | 1333 | : removeThreadAttachment({ ...input, runtime: attachedTo }).pipe( |
| 1297 | Effect.andThen( | 1334 | Effect.andThen( |
| 1298 | Effect.suspend(() => { | 1335 | Effect.suspend(() => { |
| 1299 | dropReservation(); | 1336 | dropReservation(); |
| 1300 | return preparedForCleanup?.issued === true | 1337 | // Revoke only a credential this attach freshly minted: a |
| 1301 | ? clearMcpSession(input.threadId, preparedForCleanup.mcpCredentialId) | 1338 | // REUSED credential is by definition held by another |
| 1302 | : Effect.void; | 1339 | // live provider process, and revoking it thread-wide |
| 1303 | }), | 1340 | // would break that process's MCP client mid-conversation. |
| 1304 | ), | 1341 | if (preparedForCleanup?.issued !== true) return Effect.void; |
| 1305 | ), | 1342 | const mcpCredentialId = preparedForCleanup.mcpCredentialId; |
| 1343 | const attachedRuntime = attachedTo; | ||
| 1344 | // As in release: a replacement session (or an open | ||
| 1345 | // configuring one) may have taken the credential up. | ||
| 1346 | return Ref.get(sessions).pipe( | ||
| 1347 | Effect.flatMap((current) => | ||
| 1348 | (mcpCredentialId !== undefined && | ||
| 1349 | isMcpCredentialReserved(input.threadId, mcpCredentialId)) || | ||
| 1350 | Array.from(current.values()).some( | ||
| 1351 | (other) => | ||
| 1352 | other.runtime !== attachedRuntime && | ||
| 1353 | (other.attachedThreadIds.has(input.threadId) || | ||
| 1354 | (mcpCredentialId !== undefined && | ||
| 1355 | other.mcpCredentialIdByThread.get(input.threadId) === | ||
| 1356 | mcpCredentialId)), | ||
| 1357 | ) | ||
| 1358 | ? Effect.void | ||
| 1359 | : clearMcpSession(input.threadId, mcpCredentialId), | ||
| 1360 | ), | ||
| 1361 | ); | ||
| 1362 | }), | ||
| 1363 | ), | ||
| 1364 | ), | ||
| 1306 | ), | 1365 | ), |
| 1366 | ); | ||
| 1367 | return threadAttachment.withLock(threadAttachmentKey(input), attach).pipe( | ||
| 1307 | // The entry's own record (written above while the thread is | 1368 | // The entry's own record (written above while the thread is |
| 1308 | // attached) guards the credential from here on; the reservation | 1369 | // attached) guards the credential from here on; the reservation |
| 1309 | // is only needed until then. Ensuring covers defects/interrupts. | 1370 | // is only needed until then. Ensuring covers defects/interrupts. |
| ... | @@ -1311,7 +1372,7 @@ export const layerWithOptions = ( | ... | @@ -1311,7 +1372,7 @@ export const layerWithOptions = ( |
| 1311 | ); | 1372 | ); |
| 1312 | }); | 1373 | }); |
| 1313 | 1374 | ||
| 1314 | const markBusy = (providerSessionId: ProviderSessionId) => | 1375 | const markBusy = (providerSessionId: ProviderSessionId, turnKey: string) => |
| 1315 | withActivityError( | 1376 | withActivityError( |
| 1316 | providerSessionId, | 1377 | providerSessionId, |
| 1317 | Effect.gen(function* () { | 1378 | Effect.gen(function* () { |
| ... | @@ -1325,7 +1386,7 @@ export const layerWithOptions = ( | ... | @@ -1325,7 +1386,7 @@ export const layerWithOptions = ( |
| 1325 | const updated = new Map(current); | 1386 | const updated = new Map(current); |
| 1326 | updated.set(key, { | 1387 | updated.set(key, { |
| 1327 | ...entry, | 1388 | ...entry, |
| 1328 | busyCount: entry.busyCount + 1, | 1389 | busyTurns: new Set(entry.busyTurns).add(turnKey), |
| 1329 | idleFiber: null, | 1390 | idleFiber: null, |
| 1330 | lastActivityAtMs: now, | 1391 | lastActivityAtMs: now, |
| 1331 | pinnedSinceMs: null, | 1392 | pinnedSinceMs: null, |
| ... | @@ -1336,7 +1397,10 @@ export const layerWithOptions = ( | ... | @@ -1336,7 +1397,10 @@ export const layerWithOptions = ( |
| 1336 | }), | 1397 | }), |
| 1337 | ); | 1398 | ); |
| 1338 | 1399 | ||
| 1339 | const markIdle = (providerSessionId: ProviderSessionId) => | 1400 | // Clearing a turn that is not marked busy (one whose failed start already |
| 1401 | // cleared it, or a subagent turn the manager never started) only | ||
| 1402 | // records activity. | ||
| 1403 | const markIdle = (providerSessionId: ProviderSessionId, turnKey: string) => | ||
| 1340 | withActivityError( | 1404 | withActivityError( |
| 1341 | providerSessionId, | 1405 | providerSessionId, |
| 1342 | Effect.gen(function* () { | 1406 | Effect.gen(function* () { |
| ... | @@ -1347,10 +1411,12 @@ export const layerWithOptions = ( | ... | @@ -1347,10 +1411,12 @@ export const layerWithOptions = ( |
| 1347 | if (entry === undefined) { | 1411 | if (entry === undefined) { |
| 1348 | return current; | 1412 | return current; |
| 1349 | } | 1413 | } |
| 1414 | const busyTurns = new Set(entry.busyTurns); | ||
| 1415 | busyTurns.delete(turnKey); | ||
| 1350 | const updated = new Map(current); | 1416 | const updated = new Map(current); |
| 1351 | updated.set(key, { | 1417 | updated.set(key, { |
| 1352 | ...entry, | 1418 | ...entry, |
| 1353 | busyCount: Math.max(0, entry.busyCount - 1), | 1419 | busyTurns, |
| 1354 | lastActivityAtMs: now, | 1420 | lastActivityAtMs: now, |
| 1355 | }); | 1421 | }); |
| 1356 | return updated; | 1422 | return updated; |
| ... | @@ -1536,13 +1602,32 @@ export const layerWithOptions = ( | ... | @@ -1536,13 +1602,32 @@ export const layerWithOptions = ( |
| 1536 | providerInstanceId: runtime.instanceId, | 1602 | providerInstanceId: runtime.instanceId, |
| 1537 | }), | 1603 | }), |
| 1538 | ).pipe( | 1604 | ).pipe( |
| 1539 | Effect.andThen(observeActivity(providerSessionId, markBusy(providerSessionId))), | 1605 | // A start that fails or is stopped may never emit turn.terminal, |
| 1606 | // so it clears its own turn or the session never goes idle. If | ||
| 1607 | // the adapter emits the terminal anyway, clearing the same turn | ||
| 1608 | // again changes nothing, so another thread's turn on a shared | ||
| 1609 | // session stays busy either way. | ||
| 1540 | Effect.andThen( | 1610 | Effect.andThen( |
| 1541 | runtime.startTurn(input).pipe(turnMetrics("send", input.modelSelection.model)), | 1611 | Effect.acquireUseRelease( |
| 1542 | ), | 1612 | observeActivity( |
| 1543 | Effect.catch((error) => | 1613 | providerSessionId, |
| 1544 | observeActivity(providerSessionId, markIdle(providerSessionId)).pipe( | 1614 | markBusy( |
| 1545 | Effect.andThen(Effect.fail(error)), | 1615 | providerSessionId, |
| 1616 | busyTurnKey(input.providerThread.id, input.runOrdinal), | ||
| 1617 | ), | ||
| 1618 | ), | ||
| 1619 | () => | ||
| 1620 | runtime.startTurn(input).pipe(turnMetrics("send", input.modelSelection.model)), | ||
| 1621 | (_, exit) => | ||
| 1622 | Exit.isFailure(exit) | ||
| 1623 | ? observeActivity( | ||
| 1624 | providerSessionId, | ||
| 1625 | markIdle( | ||
| 1626 | providerSessionId, | ||
| 1627 | busyTurnKey(input.providerThread.id, input.runOrdinal), | ||
| 1628 | ), | ||
| 1629 | ) | ||
| 1630 | : Effect.void, | ||
| 1546 | ), | 1631 | ), |
| 1547 | ), | 1632 | ), |
| 1548 | ), | 1633 | ), |
| ... | @@ -1605,7 +1690,10 @@ export const layerWithOptions = ( | ... | @@ -1605,7 +1690,10 @@ export const layerWithOptions = ( |
| 1605 | return observeActivity( | 1690 | return observeActivity( |
| 1606 | entry.runtime.providerSessionId, | 1691 | entry.runtime.providerSessionId, |
| 1607 | event.type === "turn.terminal" | 1692 | event.type === "turn.terminal" |
| 1608 | ? markIdle(entry.runtime.providerSessionId) | 1693 | ? markIdle( |
| 1694 | entry.runtime.providerSessionId, | ||
| 1695 | busyTurnKey(event.providerThreadId, event.runOrdinal), | ||
| 1696 | ) | ||
| 1609 | : touchActivity(entry.runtime.providerSessionId), | 1697 | : touchActivity(entry.runtime.providerSessionId), |
| 1610 | ).pipe( | 1698 | ).pipe( |
| 1611 | Effect.andThen( | 1699 | Effect.andThen( |
| ... | @@ -1695,6 +1783,12 @@ export const layerWithOptions = ( | ... | @@ -1695,6 +1783,12 @@ export const layerWithOptions = ( |
| 1695 | ); | 1783 | ); |
| 1696 | }; | 1784 | }; |
| 1697 | 1785 | ||
| 1786 | // Parent of every session scope. On layer close, shutdown releases the | ||
| 1787 | // live sessions first, then closes any session whose open is still in | ||
| 1788 | // flight, time-boxed so a stuck adapter cannot hold up server shutdown. | ||
| 1789 | // Parallel, so one session whose close hangs does not stop the rest from | ||
| 1790 | // closing within the time box. | ||
| 1791 | const sessionScopes = yield* Scope.make("parallel"); | ||
| 1698 | const shutdown = Effect.gen(function* () { | 1792 | const shutdown = Effect.gen(function* () { |
| 1699 | const activeSessions = [...(yield* Ref.get(sessions)).values()]; | 1793 | const activeSessions = [...(yield* Ref.get(sessions)).values()]; |
| 1700 | yield* Effect.forEach( | 1794 | yield* Effect.forEach( |
| ... | @@ -1714,7 +1808,11 @@ export const layerWithOptions = ( | ... | @@ -1714,7 +1808,11 @@ export const layerWithOptions = ( |
| 1714 | { discard: true }, | 1808 | { discard: true }, |
| 1715 | ); | 1809 | ); |
| 1716 | }); | 1810 | }); |
| 1717 | yield* Effect.addFinalizer(() => shutdown); | 1811 | yield* Effect.addFinalizer(() => |
| 1812 | shutdown.pipe( | ||
| 1813 | Effect.ensuring(closeScopeWithin(sessionScopes, { reason: "server_shutdown" })), | ||
| 1814 | ), | ||
| 1815 | ); | ||
| 1718 | 1816 | ||
| 1719 | return ProviderSessionManagerV2.of({ | 1817 | return ProviderSessionManagerV2.of({ |
| 1720 | shutdown, | 1818 | shutdown, |
| ... | @@ -1783,7 +1881,7 @@ export const layerWithOptions = ( | ... | @@ -1783,7 +1881,7 @@ export const layerWithOptions = ( |
| 1783 | dropMcpCredentialReservation(input.threadId, mcpCredentialId); | 1881 | dropMcpCredentialReservation(input.threadId, mcpCredentialId); |
| 1784 | } | 1882 | } |
| 1785 | }); | 1883 | }); |
| 1786 | const sessionScope = yield* Scope.make(); | 1884 | const sessionScope = yield* Scope.fork(sessionScopes); |
| 1787 | const runtime = yield* adapter | 1885 | const runtime = yield* adapter |
| 1788 | .openSession({ | 1886 | .openSession({ |
| 1789 | threadId: input.threadId, | 1887 | threadId: input.threadId, |
| ... | @@ -1805,21 +1903,30 @@ export const layerWithOptions = ( | ... | @@ -1805,21 +1903,30 @@ export const layerWithOptions = ( |
| 1805 | }) | 1903 | }) |
| 1806 | .pipe( | 1904 | .pipe( |
| 1807 | Effect.provideService(Scope.Scope, sessionScope), | 1905 | Effect.provideService(Scope.Scope, sessionScope), |
| 1808 | Effect.tapError(() => | 1906 | // Any failure, including a Stop that interrupts a slow |
| 1809 | Scope.close(sessionScope, Exit.void).pipe( | 1907 | // handshake, stops the provider process this open started. |
| 1810 | Effect.ignore, | 1908 | // The session cleanup runs first, and the close is |
| 1811 | Effect.andThen(dropReservation), | 1909 | // time-boxed: this runs under the session's open lock, so an |
| 1812 | // Revoke only a credential this open freshly minted: a | 1910 | // adapter finalizer that never finishes must not hold the |
| 1813 | // reused credential is held by another live provider | 1911 | // interrupter, the lock, or later opens. |
| 1814 | // process and must survive this open's failure. | 1912 | Effect.onError(() => |
| 1913 | dropReservation.pipe( | ||
| 1914 | // Clear only a session this open freshly set up: a reused | ||
| 1915 | // one is held by another live provider process and must | ||
| 1916 | // survive this open's failure. | ||
| 1815 | Effect.andThen( | 1917 | Effect.andThen( |
| 1816 | prepared.issued | 1918 | prepared.issued |
| 1817 | ? clearMcpSession(input.threadId, mcpCredentialId) | 1919 | ? clearMcpSession(input.threadId, mcpCredentialId) |
| 1818 | : Effect.void, | 1920 | : Effect.void, |
| 1819 | ), | 1921 | ), |
| 1922 | Effect.ensuring( | ||
| 1923 | closeScopeWithin(sessionScope, { | ||
| 1924 | providerSessionId: input.providerSessionId, | ||
| 1925 | reason: "open_failed", | ||
| 1926 | }), | ||
| 1927 | ), | ||
| 1820 | ), | 1928 | ), |
| 1821 | ), | 1929 | ), |
| 1822 | Effect.onInterrupt(() => dropReservation), | ||
| 1823 | Effect.mapError( | 1930 | Effect.mapError( |
| 1824 | (cause) => | 1931 | (cause) => |
| 1825 | new ProviderSessionOpenError({ | 1932 | new ProviderSessionOpenError({ |
| ... | @@ -1854,7 +1961,7 @@ export const layerWithOptions = ( | ... | @@ -1854,7 +1961,7 @@ export const layerWithOptions = ( |
| 1854 | requestEventPermit: yield* Semaphore.make(1), | 1961 | requestEventPermit: yield* Semaphore.make(1), |
| 1855 | scope: sessionScope, | 1962 | scope: sessionScope, |
| 1856 | idleGeneration: 0, | 1963 | idleGeneration: 0, |
| 1857 | busyCount: 0, | 1964 | busyTurns: new Set(), |
| 1858 | lastActivityAtMs: now, | 1965 | lastActivityAtMs: now, |
| 1859 | idleFiber: null, | 1966 | idleFiber: null, |
| 1860 | pinnedSinceMs: null, | 1967 | pinnedSinceMs: null, |
| ... | @@ -1876,12 +1983,22 @@ export const layerWithOptions = ( | ... | @@ -1876,12 +1983,22 @@ export const layerWithOptions = ( |
| 1876 | payload: runtime.providerSession, | 1983 | payload: runtime.providerSession, |
| 1877 | }), | 1984 | }), |
| 1878 | ).pipe( | 1985 | ).pipe( |
| 1879 | Effect.tapError(() => | 1986 | // Released on interrupt too: this entry has no event pump or |
| 1880 | releaseEntry({ | 1987 | // idle timer yet, so nothing else would ever release it. |
| 1881 | providerSessionId: input.providerSessionId, | 1988 | Effect.onError((cause) => |
| 1882 | reason: "runtime_error", | 1989 | releaseEntry( |
| 1883 | detail: "Failed to persist the provider-session attachment.", | 1990 | Cause.hasInterruptsOnly(cause) |
| 1884 | }).pipe(logReleaseFailure(input.providerSessionId)), | 1991 | ? { |
| 1992 | providerSessionId: input.providerSessionId, | ||
| 1993 | reason: "manual_shutdown", | ||
| 1994 | detail: "The provider session start was interrupted.", | ||
| 1995 | } | ||
| 1996 | : { | ||
| 1997 | providerSessionId: input.providerSessionId, | ||
| 1998 | reason: "runtime_error", | ||
| 1999 | detail: "Failed to persist the provider-session attachment.", | ||
| 2000 | }, | ||
| 2001 | ).pipe(logReleaseFailure(input.providerSessionId)), | ||
| 1885 | ), | 2002 | ), |
| 1886 | ); | 2003 | ); |
| 1887 | yield* startEventPump(entry); | 2004 | yield* startEventPump(entry); |