| author |  Julius Marminge <julius0216@outlook.com> 2026-03-09 17:16:44-07:00 |
| committer |  Julius Marminge <jmarminge@gmail.com> 2026-03-09 17:21:03-07:00 |
| log | 4ed89e4ba72e47f7919b3b8e08d375bc912d89e4 |
| tree | 83116bd139b3dc72958228751ef66d6f59138a0e |
| parent | 99dbe4ff02d9ee8e5450e96d84e8f96170c64a71 |
Treat retryable Codex errors as runtime warnings
- Map Codex `error` notifications with `willRetry: true` to `runtime.warning`
- Keep active turns/sessions running during reconnect warnings
- Add adapter and ingestion tests for retry-warning behavior3 files changed, 83 insertions(+), 2 deletions(-)
apps
server
src
orchestration
provider
apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts+44
| ... | @@ -1091,6 +1091,50 @@ describe("ProviderRuntimeIngestion", () => { | ... | @@ -1091,6 +1091,50 @@ describe("ProviderRuntimeIngestion", () => { |
| 1091 | expect(thread.session?.lastError).toBe("runtime exploded"); | 1091 | expect(thread.session?.lastError).toBe("runtime exploded"); |
| 1092 | }); | 1092 | }); |
| 1093 | 1093 | ||
| 1094 | it("keeps the session running when a runtime.warning arrives during an active turn", async () => { | ||
| 1095 | const harness = await createHarness(); | ||
| 1096 | const now = new Date().toISOString(); | ||
| 1097 | |||
| 1098 | harness.emit({ | ||
| 1099 | type: "turn.started", | ||
| 1100 | eventId: asEventId("evt-warning-turn-started"), | ||
| 1101 | provider: "codex", | ||
| 1102 | createdAt: now, | ||
| 1103 | threadId: asThreadId("thread-1"), | ||
| 1104 | turnId: asTurnId("turn-warning"), | ||
| 1105 | payload: {}, | ||
| 1106 | }); | ||
| 1107 | |||
| 1108 | harness.emit({ | ||
| 1109 | type: "runtime.warning", | ||
| 1110 | eventId: asEventId("evt-warning-runtime"), | ||
| 1111 | provider: "codex", | ||
| 1112 | createdAt: now, | ||
| 1113 | threadId: asThreadId("thread-1"), | ||
| 1114 | turnId: asTurnId("turn-warning"), | ||
| 1115 | payload: { | ||
| 1116 | message: "Reconnecting... 2/5", | ||
| 1117 | detail: { | ||
| 1118 | willRetry: true, | ||
| 1119 | }, | ||
| 1120 | }, | ||
| 1121 | }); | ||
| 1122 | |||
| 1123 | const thread = await waitForThread( | ||
| 1124 | harness.engine, | ||
| 1125 | (entry) => | ||
| 1126 | entry.session?.status === "running" && | ||
| 1127 | entry.session?.activeTurnId === "turn-warning" && | ||
| 1128 | entry.activities.some( | ||
| 1129 | (activity: ProviderRuntimeTestActivity) => | ||
| 1130 | activity.id === "evt-warning-runtime" && activity.kind === "runtime.warning", | ||
| 1131 | ), | ||
| 1132 | ); | ||
| 1133 | expect(thread.session?.status).toBe("running"); | ||
| 1134 | expect(thread.session?.activeTurnId).toBe("turn-warning"); | ||
| 1135 | expect(thread.session?.lastError).toBeNull(); | ||
| 1136 | }); | ||
| 1137 | |||
| 1094 | it("maps session/thread lifecycle and item.started into session/activity projections", async () => { | 1138 | it("maps session/thread lifecycle and item.started into session/activity projections", async () => { |
| 1095 | const harness = await createHarness(); | 1139 | const harness = await createHarness(); |
| 1096 | const now = new Date().toISOString(); | 1140 | const now = new Date().toISOString(); |
apps/server/src/provider/Layers/CodexAdapter.test.ts+36
| ... | @@ -407,6 +407,42 @@ lifecycleLayer("CodexAdapterLive lifecycle", (it) => { | ... | @@ -407,6 +407,42 @@ lifecycleLayer("CodexAdapterLive lifecycle", (it) => { |
| 407 | }), | 407 | }), |
| 408 | ); | 408 | ); |
| 409 | 409 | ||
| 410 | it.effect("maps retryable Codex error notifications to runtime.warning", () => | ||
| 411 | Effect.gen(function* () { | ||
| 412 | const adapter = yield* CodexAdapter; | ||
| 413 | const firstEventFiber = yield* Stream.runHead(adapter.streamEvents).pipe(Effect.forkChild); | ||
| 414 | |||
| 415 | lifecycleManager.emit("event", { | ||
| 416 | id: asEventId("evt-retryable-error"), | ||
| 417 | kind: "notification", | ||
| 418 | provider: "codex", | ||
| 419 | threadId: asThreadId("thread-1"), | ||
| 420 | createdAt: new Date().toISOString(), | ||
| 421 | method: "error", | ||
| 422 | turnId: asTurnId("turn-1"), | ||
| 423 | payload: { | ||
| 424 | error: { | ||
| 425 | message: "Reconnecting... 2/5", | ||
| 426 | }, | ||
| 427 | willRetry: true, | ||
| 428 | }, | ||
| 429 | } satisfies ProviderEvent); | ||
| 430 | |||
| 431 | const firstEvent = yield* Fiber.join(firstEventFiber); | ||
| 432 | |||
| 433 | assert.equal(firstEvent._tag, "Some"); | ||
| 434 | if (firstEvent._tag !== "Some") { | ||
| 435 | return; | ||
| 436 | } | ||
| 437 | assert.equal(firstEvent.value.type, "runtime.warning"); | ||
| 438 | if (firstEvent.value.type !== "runtime.warning") { | ||
| 439 | return; | ||
| 440 | } | ||
| 441 | assert.equal(firstEvent.value.turnId, "turn-1"); | ||
| 442 | assert.equal(firstEvent.value.payload.message, "Reconnecting... 2/5"); | ||
| 443 | }), | ||
| 444 | ); | ||
| 445 | |||
| 410 | it.effect("preserves request type when mapping serverRequest/resolved", () => | 446 | it.effect("preserves request type when mapping serverRequest/resolved", () => |
| 411 | Effect.gen(function* () { | 447 | Effect.gen(function* () { |
| 412 | const adapter = yield* CodexAdapter; | 448 | const adapter = yield* CodexAdapter; |
apps/server/src/provider/Layers/CodexAdapter.ts+3-2
| ... | @@ -1192,13 +1192,14 @@ function mapToRuntimeEvents( | ... | @@ -1192,13 +1192,14 @@ function mapToRuntimeEvents( |
| 1192 | if (event.method === "error") { | 1192 | if (event.method === "error") { |
| 1193 | const message = | 1193 | const message = |
| 1194 | asString(asObject(payload?.error)?.message) ?? event.message ?? "Provider runtime error"; | 1194 | asString(asObject(payload?.error)?.message) ?? event.message ?? "Provider runtime error"; |
| 1195 | const willRetry = payload?.willRetry === true; | ||
| 1195 | return [ | 1196 | return [ |
| 1196 | { | 1197 | { |
| 1197 | type: "runtime.error", | 1198 | type: willRetry ? "runtime.warning" : "runtime.error", |
| 1198 | ...runtimeEventBase(event, canonicalThreadId), | 1199 | ...runtimeEventBase(event, canonicalThreadId), |
| 1199 | payload: { | 1200 | payload: { |
| 1200 | message, | 1201 | message, |
| 1201 | class: "provider_error", | 1202 | ...(!willRetry ? { class: "provider_error" as const } : {}), |
| 1202 | ...(event.payload !== undefined ? { detail: event.payload } : {}), | 1203 | ...(event.payload !== undefined ? { detail: event.payload } : {}), |
| 1203 | }, | 1204 | }, |
| 1204 | }, | 1205 | }, |