authorgravatar for 76637177+SunkenInTime@users.noreply.github.comDara Adedeji <76637177+SunkenInTime@users.noreply.github.com> 2026-08-30 18:02:19-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-08-30 18:02:19-04:00
log9072aa1fd711170af425bf6a1a2828d0d54bbfa6
tree63d9a7c6eaeab1d8750638aded6c613f846f7b99
parent60f2ce0279d524bd70a573f6e0b6e9fab56e4b3e
signature Signed by PGP key B5690EEEBB952194

fix(server): stop overpricing cached Claude tokens (#8806)

4 files changed, 105 insertions(+), 12 deletions(-)

apps/server/src/usage/usagePricing.test.ts created+55
......@@ -0,0 +1,55 @@
1import { describe, expect, it } from "@effect/vitest";
2
3import { lookupRate, normalizeModelName, parseRateTable } from "./usagePricing.ts";
4
5const rate = (input: number, cacheRead?: number) => ({
6 input_cost_per_token: input,
7 output_cost_per_token: input * 5,
8 ...(cacheRead === undefined ? {} : { cache_read_input_token_cost: cacheRead }),
9});
10
11describe("usage pricing", () => {
12 it("keeps the existing model-name normalization contract", () => {
13 expect(normalizeModelName(" Anthropic/Claude-Opus-5 ")).toBe("claude-opus-5");
14 });
15
16 it("keeps the canonical Fable rate separate from DeepInfra in either order", () => {
17 const canonical = ["claude-fable-5", rate(1e-5, 1e-6)] as const;
18 const deepInfra = ["deepinfra/anthropic/claude-fable-5", rate(1e-5)] as const;
19
20 for (const entries of [
21 [canonical, deepInfra],
22 [deepInfra, canonical],
23 ]) {
24 const table = parseRateTable(Object.fromEntries(entries));
25
26 expect(lookupRate(table, "claude-fable-5")?.cacheReadCostPerToken).toBe(1e-6);
27 expect(lookupRate(table, "deepinfra/anthropic/claude-fable-5")?.cacheReadCostPerToken).toBe(
28 1e-5,
29 );
30 expect(lookupRate(table, "other/claude-fable-5")).toBeNull();
31 }
32 });
33
34 it("adds a bare alias when every qualified entry has the same rate", () => {
35 const table = parseRateTable({
36 "provider-a/example-model": rate(1),
37 "provider-b/example-model": rate(1),
38 });
39
40 expect(lookupRate(table, "example-model")).toEqual(
41 lookupRate(table, "provider-a/example-model"),
42 );
43 });
44
45 it("leaves an ambiguous bare name unpriced", () => {
46 const table = parseRateTable({
47 "provider-a/example-model": rate(1),
48 "provider-b/example-model": rate(3),
49 });
50
51 expect(lookupRate(table, "provider-a/example-model")?.inputCostPerToken).toBe(1);
52 expect(lookupRate(table, "provider-b/example-model")?.inputCostPerToken).toBe(3);
53 expect(lookupRate(table, "example-model")).toBeNull();
54 });
55});
apps/server/src/usage/usagePricing.ts+48-10
......@@ -44,6 +44,9 @@ function finiteNumber(value: unknown): number | null {
4444 * Entries without both an input and an output rate are dropped: a half-priced
4545 * model would silently under-report cost, which is worse than reporting the
4646 * model as unpriced.
47 *
48 * Entries keep their full normalized key; a bare name is aliased only when no
49 * canonical entry exists and every qualified entry has the same rate.
4750 */
4851export function parseRateTable(document: unknown): RateTable {
4952 const table = new Map<string, ModelRate>();
......@@ -56,7 +59,9 @@ export function parseRateTable(document: unknown): RateTable {
5659 const output = finiteNumber(entry.output_cost_per_token);
5760 if (input === null || output === null) continue;
5861
59 table.set(normalizeModelName(name), {
62 const key = normalizeRateKey(name);
63 if (key.length === 0) continue;
64 table.set(key, {
6065 inputCostPerToken: input,
6166 outputCostPerToken: output,
6267 // Anthropic bills cache reads at a discount and cache writes at a
......@@ -66,20 +71,52 @@ export function parseRateTable(document: unknown): RateTable {
6671 cacheCreationCostPerToken: finiteNumber(entry.cache_creation_input_token_cost) ?? input,
6772 });
6873 }
74
75 // `null` marks a bare name claimed at conflicting rates: no alias for it.
76 const aliasCandidates = new Map<string, ModelRate | null>();
77 for (const [key, rate] of table) {
78 const alias = bareModelName(key);
79 if (alias.length === 0 || alias === key || table.has(alias)) continue;
80 const held = aliasCandidates.get(alias);
81 if (held === undefined) {
82 aliasCandidates.set(alias, rate);
83 } else if (held !== null && !sameRate(held, rate)) {
84 aliasCandidates.set(alias, null);
85 }
86 }
87 for (const [alias, rate] of aliasCandidates) {
88 if (rate !== null) table.set(alias, rate);
89 }
90
6991 return table;
7092}
7193
94function sameRate(a: ModelRate, b: ModelRate): boolean {
95 return (
96 a.inputCostPerToken === b.inputCostPerToken &&
97 a.outputCostPerToken === b.outputCostPerToken &&
98 a.cacheReadCostPerToken === b.cacheReadCostPerToken &&
99 a.cacheCreationCostPerToken === b.cacheCreationCostPerToken
100 );
101}
102
103function normalizeRateKey(model: string): string {
104 return model.trim().toLowerCase();
105}
106
72107/**
73108 * Canonicalises a model name for lookup.
74109 *
75 * Strips a `provider/` prefix (LiteLLM publishes both `claude-opus-5` and
76 * `anthropic/claude-opus-5`) and lowercases, since transcripts are inconsistent
77 * about casing.
110 * Strips a `provider/` prefix and lowercases, since transcripts are
111 * inconsistent about casing.
78112 */
79113export function normalizeModelName(model: string): string {
80 const trimmed = model.trim().toLowerCase();
81 const slash = trimmed.lastIndexOf("/");
82 return slash === -1 ? trimmed : trimmed.slice(slash + 1);
114 return bareModelName(normalizeRateKey(model));
115}
116
117function bareModelName(key: string): string {
118 const slash = key.lastIndexOf("/");
119 return slash === -1 ? key : key.slice(slash + 1);
83120}
84121
85122/**
......@@ -99,9 +136,10 @@ const UNPRICEABLE_MODELS = new Set([
99136]);
100137
101138export function lookupRate(table: RateTable, model: string): ModelRate | null {
102 const normalized = normalizeModelName(model);
103 if (normalized.length === 0 || UNPRICEABLE_MODELS.has(normalized)) return null;
104 return table.get(normalized) ?? null;
139 const key = normalizeRateKey(model);
140 const bareName = bareModelName(key);
141 if (bareName.length === 0 || UNPRICEABLE_MODELS.has(bareName)) return null;
142 return table.get(key) ?? null;
105143}
106144
107145export interface PricedUsage {
apps/server/src/usage/usageScanCache.test.ts+1-1
......@@ -88,7 +88,7 @@ describe("scan cache round trip", () => {
8888
8989 it("rejects the whole cache when an intern table holds a non-string", () => {
9090 // models: [1] would pass the undefined guard, put a number in a record's
91 // model, and crash normalizeModelName at aggregate time.
91 // model, and crash lookupRate at aggregate time.
9292 const encoded = encodeScanCache(cacheWith([["/a.jsonl", 100, [record()]]]));
9393 const poisoned = { ...encoded, models: [1] };
9494
apps/server/src/usage/usageScanCache.ts+1-1
......@@ -124,7 +124,7 @@ export function decodeScanCache(document: unknown): ScanCache {
124124
125125 // The intern tables must be all strings: a numeric entry would pass the
126126 // undefined guard below, land in a record's model, and crash the aggregate
127 // at normalizeModelName. A corrupt table rejects the whole cache.
127 // at lookupRate. A corrupt table rejects the whole cache.
128128 if (!root.models.every((value) => typeof value === "string")) return cache;
129129 if (!root.sessions.every((value) => typeof value === "string")) return cache;
130130 const models = root.models as readonly string[];