diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 6777f0a2902a9f7f43d1ef3302180fbc2cdcc3b4..4f89f58b331529bb9d61f1281002af65568b7453 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -3017,7 +3017,65 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); - it.effect.each([429, 401, 529])( + it.effect.each([ + { status: 503, cooldown: "4m 6s", seconds: 246 }, + { status: 503, cooldown: "1h 2m 3s", seconds: 3723 }, + { status: 503, cooldown: "7s", seconds: 7 }, + { status: 503, cooldown: "0s", seconds: 0 }, + { status: 503, cooldown: "unknown", seconds: 0 }, + { status: 503, cooldown: "999999999999999999999999h", seconds: 0 }, + { status: 503, cooldown: "4m 6s", seconds: 0, errorType: "overloaded_error" }, + { status: 401, cooldown: "4m 6s", seconds: 0 }, + ])("recovers wrapped gateway limits with status $status and cooldown $cooldown", (input) => + Effect.gen(function* () { + const harness = yield* makeWakeHarness; + const now = yield* DateTime.now; + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-gateway-limit"), + text: "Continue.", + attachments: [], + }), + ); + yield* Queue.offerAll(harness.sdkMessages, [ + makeAssistantErrorFrame({ + uuid: "00000000-0000-4000-8000-000000000652", + error: "server_error", + }), + makeResultFrame({ + uuid: "00000000-0000-4000-8000-000000000653", + result: `API Error: ${input.status} [claude/claude-opus-5-5] [429]: {"type":"error","error":{"type":"${input.errorType ?? "rate_limit_error"}","message":"Please try again later."}} (reset after ${input.cooldown}).`, + terminalReason: "api_error", + isError: true, + apiErrorStatus: input.status, + }), + ]); + const terminal = yield* Queue.take(harness.terminalReceipts); + assert.equal(terminal.status, "failed"); + if (terminal.status !== "failed") return; + assert.equal(terminal.failure.class, input.seconds > 0 ? "usage_limit" : "provider_error"); + if (input.seconds > 0) { + assert.equal(terminal.failure.retryable, true); + assert.equal( + terminal.failure.resetAt, + DateTime.formatIso( + DateTime.makeUnsafe(DateTime.toEpochMillis(now) + input.seconds * 1_000), + ), + ); + } else { + assert.isUndefined(terminal.failure.resetAt); + } + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ); + + it.effect.each([429, 401, 503, 529])( "classifies the current Claude API status %s after rate-limit evidence", (apiErrorStatus) => Effect.gen(function* () { diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 1451ec9ff62e65b52b9838c08e66fb84ca8510ea..79e46c25dc8c9fa3edc0f622c188f0fd6ca39044 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -2553,9 +2553,37 @@ function isClaudeTaskNotificationOriginResult(message: SDKMessage): message is S function providerFailureFromResult( message: SDKResultMessage, + completedAt: DateTime.Utc, failureHint?: string, usageLimited = false, ): OrchestrationV2ProviderFailure | null { + // shortcut: accept 9router's wrapped 429 cooldown; prefer native reset metadata when available. + const gatewayLimit = + failureHint === undefined && + message.subtype === "success" && + message.is_error && + message.terminal_reason === "api_error" && + message.api_error_status === 503 + ? /^API Error: 503 \[[^\]\r\n]+\] \[429\]: \{"type":"error","error":\{"type":"rate_limit_error",[^\r\n]*\} \(reset after (?:(\d+)h ?)?(?:(\d+)m ?)?(?:(\d+)s)?\)/.exec( + message.result, + ) + : null; + if (gatewayLimit) { + const seconds = + Number(gatewayLimit[1] ?? 0) * 3_600 + + Number(gatewayLimit[2] ?? 0) * 60 + + Number(gatewayLimit[3] ?? 0); + const resetMs = DateTime.toEpochMillis(completedAt) + seconds * 1_000; + if (seconds > 0 && resetMs < 8.64e15) { + return makeProviderFailure({ + class: "usage_limit", + message: "Claude API rate limit reached. Try again later.", + code: "api_error_429", + retryable: true, + resetAt: DateTime.formatIso(DateTime.makeUnsafe(resetMs)), + }); + } + } const failureClass = message.terminal_reason === "blocking_limit" || (message.subtype === "success" && message.api_error_status === 429) || @@ -6707,10 +6735,10 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( : null; const resultFailure = interrupted ? null - : providerFailureFromResult(message, failureHint, usageLimited); + : providerFailureFromResult(message, completedAt, failureHint, usageLimited); const terminalFailure = resultFailure?.class === "usage_limit" - ? { ...resultFailure, resetAt } + ? { ...resultFailure, resetAt: resetAt ?? resultFailure.resetAt ?? null } : resultFailure; yield* finalizeActiveTurn({ context,