| author |  Julius Marminge <julius0216@outlook.com> 2026-03-09 17:16:44-07:00 |
| committer |  Julius Marminge <jmarminge@gmail.com> 2026-03-09 17:21:03-07:00 |
| log | 4ed89e4ba72e47f7919b3b8e08d375bc912d89e4 |
| tree | 83116bd139b3dc72958228751ef66d6f59138a0e |
| parent | 99dbe4ff02d9ee8e5450e96d84e8f96170c64a71 |
Treat retryable Codex errors as runtime warnings
- Map Codex `error` notifications with `willRetry: true` to `runtime.warning`
- Keep active turns/sessions running during reconnect warnings
- Add adapter and ingestion tests for retry-warning behavior3 files changed, 83 insertions(+), 2 deletions(-)
apps
server
src
orchestration
provider
apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts+44
| ... | ... | @@ -1091,6 +1091,50 @@ describe("ProviderRuntimeIngestion", () => { |
| 1091 | 1091 | expect(thread.session?.lastError).toBe("runtime exploded"); |
| 1092 | 1092 | }); |
| 1093 | 1093 | |
| 1094 | it("keeps the session running when a runtime.warning arrives during an active turn", async () => { | |
| 1095 | const harness = await createHarness(); | |
| 1096 | const now = new Date().toISOString(); | |
| 1097 | ||
| 1098 | harness.emit({ | |
| 1099 | type: "turn.started", | |
| 1100 | eventId: asEventId("evt-warning-turn-started"), | |
| 1101 | provider: "codex", | |
| 1102 | createdAt: now, | |
| 1103 | threadId: asThreadId("thread-1"), | |
| 1104 | turnId: asTurnId("turn-warning"), | |
| 1105 | payload: {}, | |
| 1106 | }); | |
| 1107 | ||
| 1108 | harness.emit({ | |
| 1109 | type: "runtime.warning", | |
| 1110 | eventId: asEventId("evt-warning-runtime"), | |
| 1111 | provider: "codex", | |
| 1112 | createdAt: now, | |
| 1113 | threadId: asThreadId("thread-1"), | |
| 1114 | turnId: asTurnId("turn-warning"), | |
| 1115 | payload: { | |
| 1116 | message: "Reconnecting... 2/5", | |
| 1117 | detail: { | |
| 1118 | willRetry: true, | |
| 1119 | }, | |
| 1120 | }, | |
| 1121 | }); | |
| 1122 | ||
| 1123 | const thread = await waitForThread( | |
| 1124 | harness.engine, | |
| 1125 | (entry) => | |
| 1126 | entry.session?.status === "running" && | |
| 1127 | entry.session?.activeTurnId === "turn-warning" && | |
| 1128 | entry.activities.some( | |
| 1129 | (activity: ProviderRuntimeTestActivity) => | |
| 1130 | activity.id === "evt-warning-runtime" && activity.kind === "runtime.warning", | |
| 1131 | ), | |
| 1132 | ); | |
| 1133 | expect(thread.session?.status).toBe("running"); | |
| 1134 | expect(thread.session?.activeTurnId).toBe("turn-warning"); | |
| 1135 | expect(thread.session?.lastError).toBeNull(); | |
| 1136 | }); | |
| 1137 | ||
| 1094 | 1138 | it("maps session/thread lifecycle and item.started into session/activity projections", async () => { |
| 1095 | 1139 | const harness = await createHarness(); |
| 1096 | 1140 | const now = new Date().toISOString(); |
apps/server/src/provider/Layers/CodexAdapter.test.ts+36
| ... | ... | @@ -407,6 +407,42 @@ lifecycleLayer("CodexAdapterLive lifecycle", (it) => { |
| 407 | 407 | }), |
| 408 | 408 | ); |
| 409 | 409 | |
| 410 | it.effect("maps retryable Codex error notifications to runtime.warning", () => | |
| 411 | Effect.gen(function* () { | |
| 412 | const adapter = yield* CodexAdapter; | |
| 413 | const firstEventFiber = yield* Stream.runHead(adapter.streamEvents).pipe(Effect.forkChild); | |
| 414 | ||
| 415 | lifecycleManager.emit("event", { | |
| 416 | id: asEventId("evt-retryable-error"), | |
| 417 | kind: "notification", | |
| 418 | provider: "codex", | |
| 419 | threadId: asThreadId("thread-1"), | |
| 420 | createdAt: new Date().toISOString(), | |
| 421 | method: "error", | |
| 422 | turnId: asTurnId("turn-1"), | |
| 423 | payload: { | |
| 424 | error: { | |
| 425 | message: "Reconnecting... 2/5", | |
| 426 | }, | |
| 427 | willRetry: true, | |
| 428 | }, | |
| 429 | } satisfies ProviderEvent); | |
| 430 | ||
| 431 | const firstEvent = yield* Fiber.join(firstEventFiber); | |
| 432 | ||
| 433 | assert.equal(firstEvent._tag, "Some"); | |
| 434 | if (firstEvent._tag !== "Some") { | |
| 435 | return; | |
| 436 | } | |
| 437 | assert.equal(firstEvent.value.type, "runtime.warning"); | |
| 438 | if (firstEvent.value.type !== "runtime.warning") { | |
| 439 | return; | |
| 440 | } | |
| 441 | assert.equal(firstEvent.value.turnId, "turn-1"); | |
| 442 | assert.equal(firstEvent.value.payload.message, "Reconnecting... 2/5"); | |
| 443 | }), | |
| 444 | ); | |
| 445 | ||
| 410 | 446 | it.effect("preserves request type when mapping serverRequest/resolved", () => |
| 411 | 447 | Effect.gen(function* () { |
| 412 | 448 | const adapter = yield* CodexAdapter; |
apps/server/src/provider/Layers/CodexAdapter.ts+3-2
| ... | ... | @@ -1192,13 +1192,14 @@ function mapToRuntimeEvents( |
| 1192 | 1192 | if (event.method === "error") { |
| 1193 | 1193 | const message = |
| 1194 | 1194 | asString(asObject(payload?.error)?.message) ?? event.message ?? "Provider runtime error"; |
| 1195 | const willRetry = payload?.willRetry === true; | |
| 1195 | 1196 | return [ |
| 1196 | 1197 | { |
| 1197 | type: "runtime.error", | |
| 1198 | type: willRetry ? "runtime.warning" : "runtime.error", | |
| 1198 | 1199 | ...runtimeEventBase(event, canonicalThreadId), |
| 1199 | 1200 | payload: { |
| 1200 | 1201 | message, |
| 1201 | class: "provider_error", | |
| 1202 | ...(!willRetry ? { class: "provider_error" as const } : {}), | |
| 1202 | 1203 | ...(event.payload !== undefined ? { detail: event.payload } : {}), |
| 1203 | 1204 | }, |
| 1204 | 1205 | }, |