| author |  Zach <tzachbonfil@gmail.com> 2026-10-10 04:11:36-07:00 |
| committer |  GitHub <noreply@github.com> 2026-10-10 06:11:36-05:00 |
| log | 4606f9d6f213952124d93792854b0c4e48658e60 |
| tree | 8cfe333835c5884110f730738191b80e80b5ec1c |
| parent | d6f3af77192adfb2304769096eeec59d3464aa21 |
| signature | Signed by PGP key B5690EEEBB952194 |
fix(server): restore auto resume for wrapped Claude gateway rate limits (#17778)
2 files changed, 89 insertions(+), 3 deletions(-)
apps
server
src
orchestration-v2
apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts+59-1
| ... | ... | @@ -3017,7 +3017,65 @@ describe("ClaudeAdapterV2 background wake turns", () => { |
| 3017 | 3017 | ), |
| 3018 | 3018 | ); |
| 3019 | 3019 | |
| 3020 | it.effect.each([429, 401, 529])( | |
| 3020 | it.effect.each([ | |
| 3021 | { status: 503, cooldown: "4m 6s", seconds: 246 }, | |
| 3022 | { status: 503, cooldown: "1h 2m 3s", seconds: 3723 }, | |
| 3023 | { status: 503, cooldown: "7s", seconds: 7 }, | |
| 3024 | { status: 503, cooldown: "0s", seconds: 0 }, | |
| 3025 | { status: 503, cooldown: "unknown", seconds: 0 }, | |
| 3026 | { status: 503, cooldown: "999999999999999999999999h", seconds: 0 }, | |
| 3027 | { status: 503, cooldown: "4m 6s", seconds: 0, errorType: "overloaded_error" }, | |
| 3028 | { status: 401, cooldown: "4m 6s", seconds: 0 }, | |
| 3029 | ])("recovers wrapped gateway limits with status $status and cooldown $cooldown", (input) => | |
| 3030 | Effect.gen(function* () { | |
| 3031 | const harness = yield* makeWakeHarness; | |
| 3032 | const now = yield* DateTime.now; | |
| 3033 | yield* harness.runtime.startTurn( | |
| 3034 | makeClaudeTestTurnInput({ | |
| 3035 | threadId: harness.threadId, | |
| 3036 | providerThread: harness.providerThread, | |
| 3037 | now, | |
| 3038 | attemptId: RunAttemptId.make("attempt-gateway-limit"), | |
| 3039 | text: "Continue.", | |
| 3040 | attachments: [], | |
| 3041 | }), | |
| 3042 | ); | |
| 3043 | yield* Queue.offerAll(harness.sdkMessages, [ | |
| 3044 | makeAssistantErrorFrame({ | |
| 3045 | uuid: "00000000-0000-4000-8000-000000000652", | |
| 3046 | error: "server_error", | |
| 3047 | }), | |
| 3048 | makeResultFrame({ | |
| 3049 | uuid: "00000000-0000-4000-8000-000000000653", | |
| 3050 | result: `API Error: ${input.status} [claude/claude-opus-5-5] [429]: {"type":"error","error":{"type":"${input.errorType ?? "rate_limit_error"}","message":"Please try again later."}} (reset after ${input.cooldown}).`, | |
| 3051 | terminalReason: "api_error", | |
| 3052 | isError: true, | |
| 3053 | apiErrorStatus: input.status, | |
| 3054 | }), | |
| 3055 | ]); | |
| 3056 | const terminal = yield* Queue.take(harness.terminalReceipts); | |
| 3057 | assert.equal(terminal.status, "failed"); | |
| 3058 | if (terminal.status !== "failed") return; | |
| 3059 | assert.equal(terminal.failure.class, input.seconds > 0 ? "usage_limit" : "provider_error"); | |
| 3060 | if (input.seconds > 0) { | |
| 3061 | assert.equal(terminal.failure.retryable, true); | |
| 3062 | assert.equal( | |
| 3063 | terminal.failure.resetAt, | |
| 3064 | DateTime.formatIso( | |
| 3065 | DateTime.makeUnsafe(DateTime.toEpochMillis(now) + input.seconds * 1_000), | |
| 3066 | ), | |
| 3067 | ); | |
| 3068 | } else { | |
| 3069 | assert.isUndefined(terminal.failure.resetAt); | |
| 3070 | } | |
| 3071 | }).pipe( | |
| 3072 | Effect.provide( | |
| 3073 | Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), | |
| 3074 | ), | |
| 3075 | ), | |
| 3076 | ); | |
| 3077 | ||
| 3078 | it.effect.each([429, 401, 503, 529])( | |
| 3021 | 3079 | "classifies the current Claude API status %s after rate-limit evidence", |
| 3022 | 3080 | (apiErrorStatus) => |
| 3023 | 3081 | Effect.gen(function* () { |
apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts+30-2
| ... | ... | @@ -2553,9 +2553,37 @@ function isClaudeTaskNotificationOriginResult(message: SDKMessage): message is S |
| 2553 | 2553 | |
| 2554 | 2554 | function providerFailureFromResult( |
| 2555 | 2555 | message: SDKResultMessage, |
| 2556 | completedAt: DateTime.Utc, | |
| 2556 | 2557 | failureHint?: string, |
| 2557 | 2558 | usageLimited = false, |
| 2558 | 2559 | ): OrchestrationV2ProviderFailure | null { |
| 2560 | // shortcut: accept 9router's wrapped 429 cooldown; prefer native reset metadata when available. | |
| 2561 | const gatewayLimit = | |
| 2562 | failureHint === undefined && | |
| 2563 | message.subtype === "success" && | |
| 2564 | message.is_error && | |
| 2565 | message.terminal_reason === "api_error" && | |
| 2566 | message.api_error_status === 503 | |
| 2567 | ? /^API Error: 503 \[[^\]\r\n]+\] \[429\]: \{"type":"error","error":\{"type":"rate_limit_error",[^\r\n]*\} \(reset after (?:(\d+)h ?)?(?:(\d+)m ?)?(?:(\d+)s)?\)/.exec( | |
| 2568 | message.result, | |
| 2569 | ) | |
| 2570 | : null; | |
| 2571 | if (gatewayLimit) { | |
| 2572 | const seconds = | |
| 2573 | Number(gatewayLimit[1] ?? 0) * 3_600 + | |
| 2574 | Number(gatewayLimit[2] ?? 0) * 60 + | |
| 2575 | Number(gatewayLimit[3] ?? 0); | |
| 2576 | const resetMs = DateTime.toEpochMillis(completedAt) + seconds * 1_000; | |
| 2577 | if (seconds > 0 && resetMs < 8.64e15) { | |
| 2578 | return makeProviderFailure({ | |
| 2579 | class: "usage_limit", | |
| 2580 | message: "Claude API rate limit reached. Try again later.", | |
| 2581 | code: "api_error_429", | |
| 2582 | retryable: true, | |
| 2583 | resetAt: DateTime.formatIso(DateTime.makeUnsafe(resetMs)), | |
| 2584 | }); | |
| 2585 | } | |
| 2586 | } | |
| 2559 | 2587 | const failureClass = |
| 2560 | 2588 | message.terminal_reason === "blocking_limit" || |
| 2561 | 2589 | (message.subtype === "success" && message.api_error_status === 429) || |
| ... | ... | @@ -6707,10 +6735,10 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( |
| 6707 | 6735 | : null; |
| 6708 | 6736 | const resultFailure = interrupted |
| 6709 | 6737 | ? null |
| 6710 | : providerFailureFromResult(message, failureHint, usageLimited); | |
| 6738 | : providerFailureFromResult(message, completedAt, failureHint, usageLimited); | |
| 6711 | 6739 | const terminalFailure = |
| 6712 | 6740 | resultFailure?.class === "usage_limit" |
| 6713 | ? { ...resultFailure, resetAt } | |
| 6741 | ? { ...resultFailure, resetAt: resetAt ?? resultFailure.resetAt ?? null } | |
| 6714 | 6742 | : resultFailure; |
| 6715 | 6743 | yield* finalizeActiveTurn({ |
| 6716 | 6744 | context, |