| author | |
| committer | |
| log | d64335bb5913c21869633a4236d9409911e3da52 |
| tree | 6a7d32ddcbfeea2f94f4783ceeb0e92d22bdc243 |
| parent | 2c8e95a4b641898322c7312cb67535612606745e |
| signature | Signed by PGP key B5690EEEBB952194 |
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>5 files changed, 537 insertions(+), 2 deletions(-)
apps/server/src/provider/Layers/CodexAdapter.test.ts+299| ... | @@ -2694,3 +2694,302 @@ it.effect("flushes managed native logs when the adapter layer shuts down", () => | ... | @@ -2694,3 +2694,302 @@ it.effect("flushes managed native logs when the adapter layer shuts down", () => |
| 2694 | } | 2694 | } |
| 2695 | }), | 2695 | }), |
| 2696 | ); | 2696 | ); |
| 2697 | |||
| 2698 | const usageLimitRuntimeFactory = makeRuntimeFactory(); | ||
| 2699 | const usageLimitLayer = it.layer( | ||
| 2700 | Layer.effect( | ||
| 2701 | CodexAdapter, | ||
| 2702 | Effect.gen(function* () { | ||
| 2703 | const codexConfig = decodeCodexSettings({}); | ||
| 2704 | return yield* makeCodexAdapter(codexConfig, { | ||
| 2705 | makeRuntime: usageLimitRuntimeFactory.factory, | ||
| 2706 | }); | ||
| 2707 | }), | ||
| 2708 | ).pipe( | ||
| 2709 | Layer.provideMerge(ServerConfig.layerTest(process.cwd(), process.cwd())), | ||
| 2710 | Layer.provideMerge(ServerSettingsService.layerTest()), | ||
| 2711 | Layer.provideMerge(providerSessionDirectoryTestLayer), | ||
| 2712 | Layer.provideMerge(NodeServices.layer), | ||
| 2713 | ), | ||
| 2714 | ); | ||
| 2715 | |||
| 2716 | const USAGE_LIMIT_NOW = "2026-01-01T00:00:00.000Z"; | ||
| 2717 | const USAGE_LIMIT_NOW_SECONDS = Date.parse(USAGE_LIMIT_NOW) / 1000; | ||
| 2718 | const CODEX_OUT_OF_CREDITS = | ||
| 2719 | "Your workspace is out of credits. Ask your workspace owner to refill in order to continue."; | ||
| 2720 | |||
| 2721 | function startUsageLimitRuntime() { | ||
| 2722 | return Effect.gen(function* () { | ||
| 2723 | const adapter = yield* CodexAdapter; | ||
| 2724 | yield* adapter.startSession({ | ||
| 2725 | provider: ProviderDriverKind.make("codex"), | ||
| 2726 | threadId: asThreadId("thread-1"), | ||
| 2727 | runtimeMode: "full-access", | ||
| 2728 | }); | ||
| 2729 | const runtime = usageLimitRuntimeFactory.lastRuntime; | ||
| 2730 | NodeAssert.ok(runtime); | ||
| 2731 | return { adapter, runtime }; | ||
| 2732 | }); | ||
| 2733 | } | ||
| 2734 | |||
| 2735 | function codexErrorNotification(input: { | ||
| 2736 | readonly id: string; | ||
| 2737 | readonly message: string; | ||
| 2738 | readonly codexErrorInfo?: string; | ||
| 2739 | }): ProviderEvent { | ||
| 2740 | return { | ||
| 2741 | id: asEventId(input.id), | ||
| 2742 | kind: "notification", | ||
| 2743 | provider: ProviderDriverKind.make("codex"), | ||
| 2744 | threadId: asThreadId("thread-1"), | ||
| 2745 | turnId: asTurnId("turn-limit"), | ||
| 2746 | createdAt: USAGE_LIMIT_NOW, | ||
| 2747 | method: "error", | ||
| 2748 | payload: { | ||
| 2749 | threadId: "thread-1", | ||
| 2750 | turnId: "turn-limit", | ||
| 2751 | willRetry: false, | ||
| 2752 | error: { | ||
| 2753 | message: input.message, | ||
| 2754 | ...(input.codexErrorInfo ? { codexErrorInfo: input.codexErrorInfo } : {}), | ||
| 2755 | }, | ||
| 2756 | }, | ||
| 2757 | }; | ||
| 2758 | } | ||
| 2759 | |||
| 2760 | function codexRateLimitsNotification(input: { | ||
| 2761 | readonly id: string; | ||
| 2762 | readonly rateLimitReachedType?: string; | ||
| 2763 | readonly primary?: { readonly usedPercent: number; readonly resetsInSeconds: number }; | ||
| 2764 | readonly secondary?: { readonly usedPercent: number; readonly resetsInSeconds: number }; | ||
| 2765 | }): ProviderEvent { | ||
| 2766 | return { | ||
| 2767 | id: asEventId(input.id), | ||
| 2768 | kind: "notification", | ||
| 2769 | provider: ProviderDriverKind.make("codex"), | ||
| 2770 | threadId: asThreadId("thread-1"), | ||
| 2771 | turnId: asTurnId("turn-limit"), | ||
| 2772 | createdAt: USAGE_LIMIT_NOW, | ||
| 2773 | method: "account/rateLimits/updated", | ||
| 2774 | payload: { | ||
| 2775 | rateLimits: { | ||
| 2776 | limitId: "codex", | ||
| 2777 | ...(input.rateLimitReachedType ? { rateLimitReachedType: input.rateLimitReachedType } : {}), | ||
| 2778 | ...(input.primary | ||
| 2779 | ? { | ||
| 2780 | primary: { | ||
| 2781 | usedPercent: input.primary.usedPercent, | ||
| 2782 | resetsAt: USAGE_LIMIT_NOW_SECONDS + input.primary.resetsInSeconds, | ||
| 2783 | windowDurationMins: 300, | ||
| 2784 | }, | ||
| 2785 | } | ||
| 2786 | : {}), | ||
| 2787 | ...(input.secondary | ||
| 2788 | ? { | ||
| 2789 | secondary: { | ||
| 2790 | usedPercent: input.secondary.usedPercent, | ||
| 2791 | resetsAt: USAGE_LIMIT_NOW_SECONDS + input.secondary.resetsInSeconds, | ||
| 2792 | windowDurationMins: 10_080, | ||
| 2793 | }, | ||
| 2794 | } | ||
| 2795 | : {}), | ||
| 2796 | }, | ||
| 2797 | }, | ||
| 2798 | }; | ||
| 2799 | } | ||
| 2800 | |||
| 2801 | function codexUsageLimitTurnFailed(id: string, turnId = "turn-limit"): ProviderEvent { | ||
| 2802 | return { | ||
| 2803 | id: asEventId(id), | ||
| 2804 | kind: "notification", | ||
| 2805 | provider: ProviderDriverKind.make("codex"), | ||
| 2806 | threadId: asThreadId("thread-1"), | ||
| 2807 | turnId: asTurnId(turnId), | ||
| 2808 | createdAt: USAGE_LIMIT_NOW, | ||
| 2809 | method: "turn/completed", | ||
| 2810 | payload: { | ||
| 2811 | threadId: "thread-1", | ||
| 2812 | turn: { | ||
| 2813 | id: turnId, | ||
| 2814 | items: [], | ||
| 2815 | status: "failed", | ||
| 2816 | error: { message: CODEX_OUT_OF_CREDITS, codexErrorInfo: "usageLimitExceeded" }, | ||
| 2817 | }, | ||
| 2818 | }, | ||
| 2819 | }; | ||
| 2820 | } | ||
| 2821 | |||
| 2822 | usageLimitLayer("CodexAdapterLive usage limits", (it) => { | ||
| 2823 | it.effect("names the exhausted window and the workspace's missing credits", () => | ||
| 2824 | Effect.gen(function* () { | ||
| 2825 | const { adapter, runtime } = yield* startUsageLimitRuntime(); | ||
| 2826 | const eventsFiber = yield* adapter.streamEvents.pipe( | ||
| 2827 | Stream.take(5), | ||
| 2828 | Stream.runCollect, | ||
| 2829 | Effect.forkChild, | ||
| 2830 | ); | ||
| 2831 | |||
| 2832 | yield* runtime.emit( | ||
| 2833 | codexErrorNotification({ | ||
| 2834 | id: "evt-limit-error", | ||
| 2835 | message: CODEX_OUT_OF_CREDITS, | ||
| 2836 | codexErrorInfo: "usageLimitExceeded", | ||
| 2837 | }), | ||
| 2838 | ); | ||
| 2839 | yield* runtime.emit( | ||
| 2840 | codexRateLimitsNotification({ | ||
| 2841 | id: "evt-limit-rate-limits", | ||
| 2842 | rateLimitReachedType: "workspace_owner_credits_depleted", | ||
| 2843 | primary: { usedPercent: 40, resetsInSeconds: 3_600 }, | ||
| 2844 | secondary: { usedPercent: 100, resetsInSeconds: 5 * 86_400 + 5 * 3_600 }, | ||
| 2845 | }), | ||
| 2846 | ); | ||
| 2847 | yield* runtime.emit(codexUsageLimitTurnFailed("evt-limit-turn")); | ||
| 2848 | // A second turn stopping on the same limit says as much as the first. | ||
| 2849 | yield* runtime.emit(codexUsageLimitTurnFailed("evt-limit-turn-2", "turn-limit-2")); | ||
| 2850 | |||
| 2851 | const events = Array.from(yield* Fiber.join(eventsFiber)); | ||
| 2852 | const expected = | ||
| 2853 | "Codex usage limit reached. The weekly limit resets in 5d 5h. The workspace has no credits to continue sooner: ask your workspace owner to add credits, or send the message again once the limit resets."; | ||
| 2854 | NodeAssert.deepStrictEqual( | ||
| 2855 | events.map((event) => event.type), | ||
| 2856 | [ | ||
| 2857 | "account.rate-limits.updated", | ||
| 2858 | "runtime.error", | ||
| 2859 | "turn.completed", | ||
| 2860 | "runtime.error", | ||
| 2861 | "turn.completed", | ||
| 2862 | ], | ||
| 2863 | ); | ||
| 2864 | for (const event of events) { | ||
| 2865 | if (event.type === "runtime.error") { | ||
| 2866 | NodeAssert.equal(event.payload.message, expected); | ||
| 2867 | NodeAssert.equal(event.payload.detail, CODEX_OUT_OF_CREDITS); | ||
| 2868 | } | ||
| 2869 | if (event.type === "turn.completed") { | ||
| 2870 | NodeAssert.equal(event.payload.errorMessage, expected); | ||
| 2871 | } | ||
| 2872 | } | ||
| 2873 | }), | ||
| 2874 | ); | ||
| 2875 | |||
| 2876 | it.effect("names the session window for a plan limit", () => | ||
| 2877 | Effect.gen(function* () { | ||
| 2878 | const { adapter, runtime } = yield* startUsageLimitRuntime(); | ||
| 2879 | const eventsFiber = yield* adapter.streamEvents.pipe( | ||
| 2880 | Stream.take(3), | ||
| 2881 | Stream.runCollect, | ||
| 2882 | Effect.forkChild, | ||
| 2883 | ); | ||
| 2884 | |||
| 2885 | yield* runtime.emit( | ||
| 2886 | codexErrorNotification({ | ||
| 2887 | id: "evt-plan-error", | ||
| 2888 | message: "You've hit your usage limit.", | ||
| 2889 | codexErrorInfo: "usageLimitExceeded", | ||
| 2890 | }), | ||
| 2891 | ); | ||
| 2892 | yield* runtime.emit( | ||
| 2893 | codexRateLimitsNotification({ | ||
| 2894 | id: "evt-plan-rate-limits", | ||
| 2895 | rateLimitReachedType: "rate_limit_reached", | ||
| 2896 | primary: { usedPercent: 100, resetsInSeconds: 3 * 3_600 + 20 * 60 }, | ||
| 2897 | }), | ||
| 2898 | ); | ||
| 2899 | yield* runtime.emit(codexUsageLimitTurnFailed("evt-plan-turn")); | ||
| 2900 | |||
| 2901 | const events = Array.from(yield* Fiber.join(eventsFiber)); | ||
| 2902 | const completed = events.find((event) => event.type === "turn.completed"); | ||
| 2903 | NodeAssert.equal( | ||
| 2904 | completed?.payload.errorMessage, | ||
| 2905 | "Codex usage limit reached. The session limit resets in 3h 20m. Send the message again once the limit resets.", | ||
| 2906 | ); | ||
| 2907 | }), | ||
| 2908 | ); | ||
| 2909 | |||
| 2910 | it.effect("reads a rate-limit snapshot seen earlier in the session", () => | ||
| 2911 | Effect.gen(function* () { | ||
| 2912 | const { adapter, runtime } = yield* startUsageLimitRuntime(); | ||
| 2913 | const eventsFiber = yield* adapter.streamEvents.pipe( | ||
| 2914 | Stream.take(3), | ||
| 2915 | Stream.runCollect, | ||
| 2916 | Effect.forkChild, | ||
| 2917 | ); | ||
| 2918 | |||
| 2919 | // The window arrives long before the stop, and the update that reports the | ||
| 2920 | // limit as reached carries no windows of its own. | ||
| 2921 | yield* runtime.emit( | ||
| 2922 | codexRateLimitsNotification({ | ||
| 2923 | id: "evt-early-rate-limits", | ||
| 2924 | primary: { usedPercent: 100, resetsInSeconds: 3 * 3_600 + 20 * 60 }, | ||
| 2925 | }), | ||
| 2926 | ); | ||
| 2927 | yield* runtime.emit( | ||
| 2928 | codexRateLimitsNotification({ | ||
| 2929 | id: "evt-sparse-rate-limits", | ||
| 2930 | rateLimitReachedType: "rate_limit_reached", | ||
| 2931 | }), | ||
| 2932 | ); | ||
| 2933 | yield* runtime.emit(codexUsageLimitTurnFailed("evt-early-turn")); | ||
| 2934 | |||
| 2935 | const events = Array.from(yield* Fiber.join(eventsFiber)); | ||
| 2936 | const completed = events.find((event) => event.type === "turn.completed"); | ||
| 2937 | NodeAssert.equal( | ||
| 2938 | completed?.payload.errorMessage, | ||
| 2939 | "Codex usage limit reached. The session limit resets in 3h 20m. Send the message again once the limit resets.", | ||
| 2940 | ); | ||
| 2941 | }), | ||
| 2942 | ); | ||
| 2943 | |||
| 2944 | it.effect("falls back to the short message without a rate-limit snapshot", () => | ||
| 2945 | Effect.gen(function* () { | ||
| 2946 | const { adapter, runtime } = yield* startUsageLimitRuntime(); | ||
| 2947 | const eventsFiber = yield* adapter.streamEvents.pipe( | ||
| 2948 | Stream.take(2), | ||
| 2949 | Stream.runCollect, | ||
| 2950 | Effect.forkChild, | ||
| 2951 | ); | ||
| 2952 | |||
| 2953 | yield* runtime.emit( | ||
| 2954 | codexErrorNotification({ | ||
| 2955 | id: "evt-bare-error", | ||
| 2956 | message: CODEX_OUT_OF_CREDITS, | ||
| 2957 | codexErrorInfo: "usageLimitExceeded", | ||
| 2958 | }), | ||
| 2959 | ); | ||
| 2960 | yield* runtime.emit(codexUsageLimitTurnFailed("evt-bare-turn")); | ||
| 2961 | |||
| 2962 | const events = Array.from(yield* Fiber.join(eventsFiber)); | ||
| 2963 | const expected = "Codex usage limit reached. Send the message again once the limit resets."; | ||
| 2964 | NodeAssert.deepStrictEqual( | ||
| 2965 | events.map((event) => event.type), | ||
| 2966 | ["runtime.error", "turn.completed"], | ||
| 2967 | ); | ||
| 2968 | const runtimeError = events.find((event) => event.type === "runtime.error"); | ||
| 2969 | NodeAssert.equal(runtimeError?.payload.message, expected); | ||
| 2970 | const completed = events.find((event) => event.type === "turn.completed"); | ||
| 2971 | NodeAssert.equal(completed?.payload.errorMessage, expected); | ||
| 2972 | }), | ||
| 2973 | ); | ||
| 2974 | |||
| 2975 | it.effect("still relays other provider errors as they arrive", () => | ||
| 2976 | Effect.gen(function* () { | ||
| 2977 | const { adapter, runtime } = yield* startUsageLimitRuntime(); | ||
| 2978 | const firstEventFiber = yield* Stream.runHead(adapter.streamEvents).pipe(Effect.forkChild); | ||
| 2979 | |||
| 2980 | yield* runtime.emit( | ||
| 2981 | codexErrorNotification({ | ||
| 2982 | id: "evt-other-error", | ||
| 2983 | message: "Codex is temporarily unavailable.", | ||
| 2984 | codexErrorInfo: "internalServerError", | ||
| 2985 | }), | ||
| 2986 | ); | ||
| 2987 | |||
| 2988 | const first = yield* Fiber.join(firstEventFiber); | ||
| 2989 | NodeAssert.equal(first._tag, "Some"); | ||
| 2990 | if (first._tag !== "Some" || first.value.type !== "runtime.error") return; | ||
| 2991 | NodeAssert.equal(first.value.payload.message, "Codex is temporarily unavailable."); | ||
| 2992 | NodeAssert.equal(first.value.payload.class, "provider_error"); | ||
| 2993 | }), | ||
| 2994 | ); | ||
| 2995 | }); |
apps/server/src/provider/Layers/CodexAdapter.ts+60-2| ... | @@ -71,7 +71,12 @@ import { | ... | @@ -71,7 +71,12 @@ import { |
| 71 | } from "./CodexSessionRuntime.ts"; | 71 | } from "./CodexSessionRuntime.ts"; |
| 72 | import { type EventNdjsonLogger, makeEventNdjsonLogger } from "./EventNdjsonLogger.ts"; | 72 | import { type EventNdjsonLogger, makeEventNdjsonLogger } from "./EventNdjsonLogger.ts"; |
| 73 | import { resolveCodexLaunchArgs } from "./codexLaunchArgs.ts"; | 73 | import { resolveCodexLaunchArgs } from "./codexLaunchArgs.ts"; |
| 74 | import { codexRateLimitsToUpdate } from "./codexUsageLimits.ts"; | 74 | import { |
| 75 | type CodexRateLimitSnapshot, | ||
| 76 | codexRateLimitsToUpdate, | ||
| 77 | codexUsageLimitMessage, | ||
| 78 | mergeCodexRateLimits, | ||
| 79 | } from "./codexUsageLimits.ts"; | ||
| 75 | const isCodexAppServerProcessExitedError = Schema.is(CodexErrors.CodexAppServerProcessExitedError); | 80 | const isCodexAppServerProcessExitedError = Schema.is(CodexErrors.CodexAppServerProcessExitedError); |
| 76 | const isCodexAppServerTransportError = Schema.is(CodexErrors.CodexAppServerTransportError); | 81 | const isCodexAppServerTransportError = Schema.is(CodexErrors.CodexAppServerTransportError); |
| 77 | const isCodexSessionRuntimeThreadIdMissingError = Schema.is( | 82 | const isCodexSessionRuntimeThreadIdMissingError = Schema.is( |
| ... | @@ -2282,6 +2287,12 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* ( | ... | @@ -2282,6 +2287,12 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* ( |
| 2282 | : {}), | 2287 | : {}), |
| 2283 | }; | 2288 | }; |
| 2284 | const turnTokenUsage = makeCodexTurnTokenUsageState(); | 2289 | const turnTokenUsage = makeCodexTurnTokenUsageState(); |
| 2290 | // Codex reports a usage-limit stop as OpenAI's own sentence, which on a | ||
| 2291 | // Business workspace blames credits for a window that ran out. The | ||
| 2292 | // snapshot naming that window arrives in its own notification, before or | ||
| 2293 | // after the stop and often sparse, so keep the session's merged view of | ||
| 2294 | // it and read it when a turn fails on the limit. | ||
| 2295 | let rateLimits: CodexRateLimitSnapshot | undefined; | ||
| 2285 | const sessionScope = yield* Scope.make("sequential"); | 2296 | const sessionScope = yield* Scope.make("sequential"); |
| 2286 | let sessionScopeTransferred = false; | 2297 | let sessionScopeTransferred = false; |
| 2287 | yield* Effect.addFinalizer(() => | 2298 | yield* Effect.addFinalizer(() => |
| ... | @@ -2339,12 +2350,56 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* ( | ... | @@ -2339,12 +2350,56 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* ( |
| 2339 | } | 2350 | } |
| 2340 | } | 2351 | } |
| 2341 | 2352 | ||
| 2342 | const runtimeEvents = mapToRuntimeEvents(event, event.threadId).map((runtimeEvent) => { | 2353 | if (event.method === "account/rateLimits/updated") { |
| 2354 | const limitsPayload = readPayload( | ||
| 2355 | EffectCodexSchema.V2AccountRateLimitsUpdatedNotification, | ||
| 2356 | event.payload, | ||
| 2357 | ); | ||
| 2358 | if (limitsPayload) { | ||
| 2359 | rateLimits = mergeCodexRateLimits(rateLimits, limitsPayload.rateLimits); | ||
| 2360 | } | ||
| 2361 | } else if (event.method === "error") { | ||
| 2362 | const errorPayload = readPayload( | ||
| 2363 | EffectCodexSchema.V2ErrorNotification, | ||
| 2364 | event.payload, | ||
| 2365 | ); | ||
| 2366 | // The failed `turn/completed` repeats this sentence and is answered | ||
| 2367 | // below; relaying both would show the limit twice. | ||
| 2368 | if (errorPayload?.error.codexErrorInfo === "usageLimitExceeded") return; | ||
| 2369 | } | ||
| 2370 | |||
| 2371 | let usageLimitError: ProviderRuntimeEvent | undefined; | ||
| 2372 | let usageLimitMessage: string | undefined; | ||
| 2373 | if (event.method === "turn/completed") { | ||
| 2374 | const completedPayload = readPayload( | ||
| 2375 | EffectCodexSchema.V2TurnCompletedNotification, | ||
| 2376 | event.payload, | ||
| 2377 | ); | ||
| 2378 | const turnError = | ||
| 2379 | completedPayload?.turn.status === "failed" | ||
| 2380 | ? completedPayload.turn.error | ||
| 2381 | : undefined; | ||
| 2382 | if (turnError?.codexErrorInfo === "usageLimitExceeded") { | ||
| 2383 | usageLimitMessage = codexUsageLimitMessage(rateLimits, event.createdAt); | ||
| 2384 | usageLimitError = { | ||
| 2385 | ...runtimeEventBase(event, event.threadId), | ||
| 2386 | type: "runtime.error", | ||
| 2387 | payload: { | ||
| 2388 | message: usageLimitMessage, | ||
| 2389 | class: "provider_error", | ||
| 2390 | ...(turnError.message ? { detail: turnError.message } : {}), | ||
| 2391 | }, | ||
| 2392 | }; | ||
| 2393 | } | ||
| 2394 | } | ||
| 2395 | |||
| 2396 | const mappedEvents = mapToRuntimeEvents(event, event.threadId).map((runtimeEvent) => { | ||
| 2343 | if (runtimeEvent.type === "turn.completed" && runtimeEvent.turnId) { | 2397 | if (runtimeEvent.type === "turn.completed" && runtimeEvent.turnId) { |
| 2344 | return { | 2398 | return { |
| 2345 | ...runtimeEvent, | 2399 | ...runtimeEvent, |
| 2346 | payload: { | 2400 | payload: { |
| 2347 | ...runtimeEvent.payload, | 2401 | ...runtimeEvent.payload, |
| 2402 | ...(usageLimitMessage ? { errorMessage: usageLimitMessage } : {}), | ||
| 2348 | tokenUsage: completeCodexTurnTokenUsage( | 2403 | tokenUsage: completeCodexTurnTokenUsage( |
| 2349 | turnTokenUsage, | 2404 | turnTokenUsage, |
| 2350 | String(runtimeEvent.turnId), | 2405 | String(runtimeEvent.turnId), |
| ... | @@ -2368,6 +2423,9 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* ( | ... | @@ -2368,6 +2423,9 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* ( |
| 2368 | } | 2423 | } |
| 2369 | return runtimeEvent; | 2424 | return runtimeEvent; |
| 2370 | }); | 2425 | }); |
| 2426 | const runtimeEvents = usageLimitError | ||
| 2427 | ? [usageLimitError, ...mappedEvents] | ||
| 2428 | : mappedEvents; | ||
| 2371 | if (runtimeEvents.length === 0) { | 2429 | if (runtimeEvents.length === 0) { |
| 2372 | yield* Effect.logDebug("ignoring unhandled Codex provider event", { | 2430 | yield* Effect.logDebug("ignoring unhandled Codex provider event", { |
| 2373 | method: event.method, | 2431 | method: event.method, |
apps/server/src/provider/Layers/codexUsageLimits.test.ts+96| ... | @@ -6,6 +6,8 @@ import { | ... | @@ -6,6 +6,8 @@ import { |
| 6 | codexRateLimitsToLimits, | 6 | codexRateLimitsToLimits, |
| 7 | codexRateLimitsToUpdate, | 7 | codexRateLimitsToUpdate, |
| 8 | codexResetCreditsToContract, | 8 | codexResetCreditsToContract, |
| 9 | codexUsageLimitMessage, | ||
| 10 | mergeCodexRateLimits, | ||
| 9 | } from "./codexUsageLimits.ts"; | 11 | } from "./codexUsageLimits.ts"; |
| 10 | 12 | ||
| 11 | const checkedAt = "2026-07-18T10:00:00.000Z"; | 13 | const checkedAt = "2026-07-18T10:00:00.000Z"; |
| ... | @@ -201,3 +203,97 @@ describe("codexResetCreditsToContract", () => { | ... | @@ -201,3 +203,97 @@ describe("codexResetCreditsToContract", () => { |
| 201 | ).toEqual({ availableCount: 1 }); | 203 | ).toEqual({ availableCount: 1 }); |
| 202 | }); | 204 | }); |
| 203 | }); | 205 | }); |
| 206 | |||
| 207 | describe("codexUsageLimitMessage", () => { | ||
| 208 | const at = "2026-01-01T00:00:00.000Z"; | ||
| 209 | const atSeconds = Date.parse(at) / 1000; | ||
| 210 | |||
| 211 | it("names the exhausted window and the workspace's missing credits", () => { | ||
| 212 | expect( | ||
| 213 | codexUsageLimitMessage( | ||
| 214 | { | ||
| 215 | limitId: "codex", | ||
| 216 | rateLimitReachedType: "workspace_owner_credits_depleted", | ||
| 217 | primary: { usedPercent: 40, resetsAt: atSeconds + 3_600, windowDurationMins: 300 }, | ||
| 218 | secondary: { | ||
| 219 | usedPercent: 100, | ||
| 220 | resetsAt: atSeconds + 5 * 86_400 + 5 * 3_600, | ||
| 221 | windowDurationMins: 10_080, | ||
| 222 | }, | ||
| 223 | }, | ||
| 224 | at, | ||
| 225 | ), | ||
| 226 | ).toBe( | ||
| 227 | "Codex usage limit reached. The weekly limit resets in 5d 5h. The workspace has no credits to continue sooner: ask your workspace owner to add credits, or send the message again once the limit resets.", | ||
| 228 | ); | ||
| 229 | }); | ||
| 230 | |||
| 231 | it("points a reached spend cap at the workspace owner", () => { | ||
| 232 | expect( | ||
| 233 | codexUsageLimitMessage( | ||
| 234 | { | ||
| 235 | limitId: "codex", | ||
| 236 | rateLimitReachedType: "workspace_member_usage_limit_reached", | ||
| 237 | primary: { | ||
| 238 | usedPercent: 100, | ||
| 239 | resetsAt: atSeconds + 3 * 3_600 + 20 * 60, | ||
| 240 | windowDurationMins: 300, | ||
| 241 | }, | ||
| 242 | }, | ||
| 243 | at, | ||
| 244 | ), | ||
| 245 | ).toBe( | ||
| 246 | "Codex usage limit reached. The session limit resets in 3h 20m. The workspace spend limit is reached: ask your workspace owner to raise it, or send the message again once the limit resets.", | ||
| 247 | ); | ||
| 248 | }); | ||
| 249 | |||
| 250 | it("names no window when credits run out without one", () => { | ||
| 251 | expect( | ||
| 252 | codexUsageLimitMessage( | ||
| 253 | { limitId: "codex", rateLimitReachedType: "workspace_member_credits_depleted" }, | ||
| 254 | at, | ||
| 255 | ), | ||
| 256 | ).toBe( | ||
| 257 | "Codex usage limit reached. The workspace has no credits to continue sooner: ask your workspace owner to add credits, or send the message again once the limit resets.", | ||
| 258 | ); | ||
| 259 | }); | ||
| 260 | |||
| 261 | it("says only what it knows without a snapshot", () => { | ||
| 262 | expect(codexUsageLimitMessage(undefined, at)).toBe( | ||
| 263 | "Codex usage limit reached. Send the message again once the limit resets.", | ||
| 264 | ); | ||
| 265 | }); | ||
| 266 | }); | ||
| 267 | |||
| 268 | describe("mergeCodexRateLimits", () => { | ||
| 269 | it("keeps windows an update does not carry", () => { | ||
| 270 | const merged = mergeCodexRateLimits( | ||
| 271 | { | ||
| 272 | limitId: "codex", | ||
| 273 | planType: "business", | ||
| 274 | primary: { usedPercent: 100, resetsAt: 1_800_000_000, windowDurationMins: 300 }, | ||
| 275 | }, | ||
| 276 | { rateLimitReachedType: "rate_limit_reached" }, | ||
| 277 | ); | ||
| 278 | |||
| 279 | expect(merged).toEqual({ | ||
| 280 | limitId: "codex", | ||
| 281 | planType: "business", | ||
| 282 | rateLimitReachedType: "rate_limit_reached", | ||
| 283 | primary: { usedPercent: 100, resetsAt: 1_800_000_000, windowDurationMins: 300 }, | ||
| 284 | }); | ||
| 285 | }); | ||
| 286 | |||
| 287 | it("ignores a model-specific snapshot so it cannot replace the main allowance", () => { | ||
| 288 | const main = { | ||
| 289 | limitId: "codex", | ||
| 290 | primary: { usedPercent: 100, resetsAt: 1_800_000_000, windowDurationMins: 300 }, | ||
| 291 | }; | ||
| 292 | expect( | ||
| 293 | mergeCodexRateLimits(main, { | ||
| 294 | limitId: "spark", | ||
| 295 | primary: { usedPercent: 3, resetsAt: 1_800_000_000, windowDurationMins: 300 }, | ||
| 296 | }), | ||
| 297 | ).toBe(main); | ||
| 298 | }); | ||
| 299 | }); |
apps/server/src/provider/Layers/codexUsageLimits.ts+75| ... | @@ -28,6 +28,7 @@ interface CodexRateLimitWindow { | ... | @@ -28,6 +28,7 @@ interface CodexRateLimitWindow { |
| 28 | export interface CodexRateLimitSnapshot { | 28 | export interface CodexRateLimitSnapshot { |
| 29 | readonly limitId?: string | null; | 29 | readonly limitId?: string | null; |
| 30 | readonly planType?: string | null; | 30 | readonly planType?: string | null; |
| 31 | readonly rateLimitReachedType?: string | null; | ||
| 31 | readonly primary?: CodexRateLimitWindow | null; | 32 | readonly primary?: CodexRateLimitWindow | null; |
| 32 | readonly secondary?: CodexRateLimitWindow | null; | 33 | readonly secondary?: CodexRateLimitWindow | null; |
| 33 | } | 34 | } |
| ... | @@ -157,3 +158,77 @@ export function codexRateLimitsFailureMessage(error: CodexErrors.CodexAppServerE | ... | @@ -157,3 +158,77 @@ export function codexRateLimitsFailureMessage(error: CodexErrors.CodexAppServerE |
| 157 | return "Codex did not answer the usage request."; | 158 | return "Codex did not answer the usage request."; |
| 158 | } | 159 | } |
| 159 | } | 160 | } |
| 161 | |||
| 162 | /** | ||
| 163 | * Codex sends `account/rateLimits/updated` as a partial view of the snapshot: a | ||
| 164 | * field the update omits keeps the value observed earlier in the session, so a | ||
| 165 | * later notification that only names the limit it reached must not drop the | ||
| 166 | * windows an earlier one carried. | ||
| 167 | */ | ||
| 168 | export function mergeCodexRateLimits( | ||
| 169 | previous: CodexRateLimitSnapshot | undefined, | ||
| 170 | update: CodexRateLimitSnapshot, | ||
| 171 | ): CodexRateLimitSnapshot | undefined { | ||
| 172 | // Model-specific snapshots (such as Spark) describe a different allowance | ||
| 173 | // and must not overwrite the main one, the same rule the usage rows apply. | ||
| 174 | if (update.limitId && update.limitId !== "codex") return previous; | ||
| 175 | if (!previous) return update; | ||
| 176 | return { | ||
| 177 | ...previous, | ||
| 178 | ...(update.limitId !== undefined ? { limitId: update.limitId } : {}), | ||
| 179 | ...(update.planType !== undefined ? { planType: update.planType } : {}), | ||
| 180 | ...(update.rateLimitReachedType !== undefined | ||
| 181 | ? { rateLimitReachedType: update.rateLimitReachedType } | ||
| 182 | : {}), | ||
| 183 | ...(update.primary !== undefined ? { primary: update.primary } : {}), | ||
| 184 | ...(update.secondary !== undefined ? { secondary: update.secondary } : {}), | ||
| 185 | }; | ||
| 186 | } | ||
| 187 | |||
| 188 | /** Coarse remaining wait, matching how the usage rows read: `5d 5h`, `3h 20m`, `12m`. */ | ||
| 189 | function formatCodexUsageLimitWait(waitMs: number): string { | ||
| 190 | const totalMinutes = Math.ceil(waitMs / 60_000); | ||
| 191 | const days = Math.floor(totalMinutes / (24 * 60)); | ||
| 192 | const hours = Math.floor((totalMinutes % (24 * 60)) / 60); | ||
| 193 | const minutes = totalMinutes % 60; | ||
| 194 | if (days > 0) return hours === 0 ? `${days}d` : `${days}d ${hours}h`; | ||
| 195 | if (hours === 0) return `${totalMinutes}m`; | ||
| 196 | return minutes === 0 ? `${hours}h` : `${hours}h ${minutes}m`; | ||
| 197 | } | ||
| 198 | |||
| 199 | function codexUsageLimitNextStep(rateLimitReachedType: string | null | undefined): string { | ||
| 200 | switch (rateLimitReachedType) { | ||
| 201 | case "workspace_owner_credits_depleted": | ||
| 202 | case "workspace_member_credits_depleted": | ||
| 203 | return " The workspace has no credits to continue sooner: ask your workspace owner to add credits, or send the message again once the limit resets."; | ||
| 204 | case "workspace_owner_usage_limit_reached": | ||
| 205 | case "workspace_member_usage_limit_reached": | ||
| 206 | return " The workspace spend limit is reached: ask your workspace owner to raise it, or send the message again once the limit resets."; | ||
| 207 | default: | ||
| 208 | return " Send the message again once the limit resets."; | ||
| 209 | } | ||
| 210 | } | ||
| 211 | |||
| 212 | /** | ||
| 213 | * The message a usage-limit stop shows instead of the provider sentence, which | ||
| 214 | * on a Business workspace blames credits for a window that simply ran out. The | ||
| 215 | * window named is the exhausted one that has yet to reset, latest first; `atIso` | ||
| 216 | * is the stopping event's timestamp, not the wall clock. | ||
| 217 | */ | ||
| 218 | export function codexUsageLimitMessage( | ||
| 219 | snapshot: CodexRateLimitSnapshot | undefined, | ||
| 220 | atIso: string, | ||
| 221 | ): string { | ||
| 222 | const atMs = Date.parse(atIso); | ||
| 223 | const windows = snapshot && Number.isFinite(atMs) ? codexRateLimitsToWindows(snapshot) : []; | ||
| 224 | let reset = ""; | ||
| 225 | let latestResetMs = Number.NEGATIVE_INFINITY; | ||
| 226 | for (const window of windows) { | ||
| 227 | if (window.usedPercent < 100 || !window.resetsAt) continue; | ||
| 228 | const resetMs = Date.parse(window.resetsAt); | ||
| 229 | if (!Number.isFinite(resetMs) || resetMs <= atMs || resetMs <= latestResetMs) continue; | ||
| 230 | latestResetMs = resetMs; | ||
| 231 | reset = ` The ${window.kind} limit resets in ${formatCodexUsageLimitWait(resetMs - atMs)}.`; | ||
| 232 | } | ||
| 233 | return `Codex usage limit reached.${reset}${codexUsageLimitNextStep(snapshot?.rateLimitReachedType)}`; | ||
| 234 | } |
docs/user/providers-codex.md+7| ... | @@ -65,6 +65,13 @@ in the thread on web, desktop, or mobile. Some tools offer access for one reques | ... | @@ -65,6 +65,13 @@ in the thread on web, desktop, or mobile. Some tools offer access for one reques |
| 65 | the current session, or permanently. See [Permission modes](./permission-modes.md) | 65 | the current session, or permanently. See [Permission modes](./permission-modes.md) |
| 66 | for command and file approvals. | 66 | for command and file approvals. |
| 67 | 67 | ||
| 68 | ## Codex says I hit a usage limit | ||
| 69 | |||
| 70 | When Codex stops on a usage limit, the thread names the window that ran out and | ||
| 71 | when it resets, when Codex reports them. Send the message again after the reset. On a workspace plan the | ||
| 72 | message also says whether your workspace owner needs to add credits or raise the | ||
| 73 | spend limit to continue sooner. | ||
| 74 | |||
| 68 | ## Send feedback to OpenAI | 75 | ## Send feedback to OpenAI |
| 69 | 76 | ||
| 70 | In an existing Codex thread, send `/feedback` with an optional description, for | 77 | In an existing Codex thread, send `/feedback` with an optional description, for |