1import { describe, expect, it } from "@effect/vitest";
2
3import {
4 decodeScanCache,
5 dedupeWithinFile,
6 encodeScanCache,
7 makeScanCacheWriter,
8 pruneScanCache,
9 type CachedFile,
10 type ScanCache,
11} from "./usageScanCache.ts";
12import type { UsageRecord } from "@t3tools/provider-core/server/usage";
13
14import { TEST_FORMAT_MAP } from "./usageTestFormats.ts";
15
16function record(overrides: Partial<UsageRecord> = {}): UsageRecord {
17 return {
18 provider: "claude",
19 timestampMs: 1_786_000_000_000,
20 model: "claude-fable-5",
21 sessionId: "session-a",
22 totals: {
23 uncachedInputTokens: 2,
24 cachedInputTokens: 1000,
25 cacheCreationTokens: 10,
26 outputTokens: 50,
27 reasoningTokens: 0,
28 },
29 reportedCostUsd: null,
30 speed: "standard",
31 dedupeKey: "msg_1:",
32 ...overrides,
33 };
34}
35
36function position(overrides: Partial<CachedFile["position"]> = {}): CachedFile["position"] {
37 return {
38 resumeOffset: 120,
39 guardLength: 64,
40 guardHash: 0xdeadbeef,
41 state: null,
42 ...overrides,
43 };
44}
45
46function cacheWith(entries: readonly [string, number, readonly UsageRecord[]][]): ScanCache {
47 const cache: ScanCache = new Map();
48 for (const [path, mtimeMs, records] of entries) {
49 cache.set(path, {
50 size: records.length * 10,
51 mtimeMs,
52 provider: "claude",
53 records,
54 tailRecords: [],
55 position: position(),
56 });
57 }
58 return cache;
59}
60
61describe("scan cache round trip", () => {
62 it("restores records unchanged", () => {
63 const original = cacheWith([
64 [
65 "/a.jsonl",
66 100,
67 [record(), record({ dedupeKey: "msg_2:", model: "claude-opus-5-5", speed: "fast" })],
68 ],
69 ["/b.jsonl", 200, [record({ sessionId: "session-b", reportedCostUsd: 1.5 })]],
70 ]);
71 original.set("/grok.jsonl", {
72 size: 40,
73 mtimeMs: 300,
74 provider: "grok",
75 records: [
76 record({ provider: "grok", model: "grok-4.5-build", dedupeKey: "s:p:grok-4.5-build" }),
77 ],
78 tailRecords: [record({ provider: "grok", model: "grok-4.5-build", dedupeKey: null })],
79 position: position({ resumeOffset: 30, guardLength: 30, guardHash: 123 }),
80 });
81 original.set("/codex.jsonl", {
82 size: 80,
83 mtimeMs: 400,
84 provider: "codex",
85 records: [
86 record({ provider: "codex", model: "gpt-6-astra", dedupeKey: null, speed: "ultrafast" }),
87 ],
88 tailRecords: [],
89 position: position({
90 state: {
91 model: "gpt-6-astra",
92 speed: "ultrafast",
93 sessionId: "session-c",
94 lastUsageSignature: '{"input_tokens":1}',
95 sawSessionMeta: true,
96 suppressingForkCopies: false,
97 forkCopyAnchorMs: 0,
98 },
99 }),
100 });
101
102 const restored = decodeScanCache(
103 JSON.parse(JSON.stringify(encodeScanCache(original))),
104 TEST_FORMAT_MAP,
105 );
106
107 expect(restored.size).toBe(4);
108 expect(restored.get("/a.jsonl")).toEqual(original.get("/a.jsonl"));
109 expect(restored.get("/b.jsonl")).toEqual(original.get("/b.jsonl"));
110 expect(restored.get("/grok.jsonl")).toEqual(original.get("/grok.jsonl"));
111 expect(restored.get("/codex.jsonl")).toEqual(original.get("/codex.jsonl"));
112 });
113
114 it("drops an entry whose persisted parse state is corrupt", () => {
115 // Resuming with a bad reducer state would attach appended usage to the
116 // wrong model or replay fork-copied history; that entry must cold parse.
117 const encoded = encodeScanCache(cacheWith([["/a.jsonl", 100, [record()]]]));
118 const poisoned = {
119 ...encoded,
120 files: {
121 "/a.jsonl": { ...encoded.files["/a.jsonl"]!, cs: { model: 42 } },
122 },
123 };
124
125 expect(
126 decodeScanCache(JSON.parse(JSON.stringify(poisoned)), TEST_FORMAT_MAP).has("/a.jsonl"),
127 ).toBe(false);
128 });
129
130 it("drops an entry whose guard length is outside the supported range", () => {
131 // The guard length sizes a Buffer in the reader; a bogus value would make
132 // every parse of that file fail and silently drop its usage.
133 const encoded = encodeScanCache(cacheWith([["/a.jsonl", 100, [record()]]]));
134 const poisoned = {
135 ...encoded,
136 files: { "/a.jsonl": { ...encoded.files["/a.jsonl"]!, gl: 1e20 } },
137 };
138
139 expect(
140 decodeScanCache(JSON.parse(JSON.stringify(poisoned)), TEST_FORMAT_MAP).has("/a.jsonl"),
141 ).toBe(false);
142 });
143
144 it("drops an entry whose speed is not a known index", () => {
145 const encoded = encodeScanCache(cacheWith([["/a.jsonl", 100, [record({ speed: "fast" })]]]));
146 const row = encoded.files["/a.jsonl"]!.r[0]!;
147 const poisoned = {
148 ...encoded,
149 files: { "/a.jsonl": { ...encoded.files["/a.jsonl"]!, r: [[...row.slice(0, 10), true]] } },
150 };
151
152 expect(
153 decodeScanCache(JSON.parse(JSON.stringify(poisoned)), TEST_FORMAT_MAP).has("/a.jsonl"),
154 ).toBe(false);
155 });
156
157 it("rejects a document from before records carried a speed", () => {
158 const encoded = encodeScanCache(cacheWith([["/a.jsonl", 100, [record()]]]));
159 const previous = { ...encoded, version: 3 };
160
161 expect(decodeScanCache(JSON.parse(JSON.stringify(previous)), TEST_FORMAT_MAP).size).toBe(0);
162 });
163
164 it("rewrites only changed entries and still restores the whole cache", () => {
165 const write = makeScanCacheWriter();
166 const cache = cacheWith([
167 ["/a.jsonl", 100, [record()]],
168 ["/b.jsonl", 200, [record({ sessionId: "session-b" })]],
169 ]);
170 const sources = { "claude\u0000/projects": { dir: "/projects", volumeId: "1:2" } };
171 expect(decodeScanCache(JSON.parse(write(cache, { sources })), TEST_FORMAT_MAP)).toEqual(cache);
172
173 // The replacement adds intern entries; /a's memoised indexes must hold.
174 cache.set("/b.jsonl", {
175 ...cache.get("/b.jsonl")!,
176 size: 30,
177 records: [record({ sessionId: "session-c", model: "claude-opus-5-5", dedupeKey: "msg_3:" })],
178 });
179 const document = JSON.parse(write(cache, { sources }));
180 expect(decodeScanCache(document, TEST_FORMAT_MAP)).toEqual(cache);
181 expect(document.sources).toEqual(sources);
182 });
183
184 it("interns repeated model and session strings", () => {
185 const encoded = encodeScanCache(
186 cacheWith([["/a.jsonl", 100, [record(), record({ dedupeKey: "msg_2:" }), record()]]]),
187 );
188
189 expect(encoded.models).toEqual(["claude-fable-5"]);
190 expect(encoded.sessions).toEqual(["session-a"]);
191 });
192
193 it("treats a corrupt or foreign document as an empty cache", () => {
194 // A bad cache should cost one cold scan, never a broken page.
195 expect(decodeScanCache(null, TEST_FORMAT_MAP).size).toBe(0);
196 expect(decodeScanCache("nonsense", TEST_FORMAT_MAP).size).toBe(0);
197 expect(
198 decodeScanCache({ version: 999, models: [], sessions: [], files: {} }, TEST_FORMAT_MAP).size,
199 ).toBe(0);
200 });
201
202 it("skips malformed file entries but keeps good ones", () => {
203 const encoded = encodeScanCache(cacheWith([["/good.jsonl", 100, [record()]]]));
204 const withJunk = {
205 ...encoded,
206 files: { ...encoded.files, "/bad.jsonl": { s: "nope", m: 1, p: "claude", r: [] } },
207 };
208
209 const restored = decodeScanCache(JSON.parse(JSON.stringify(withJunk)), TEST_FORMAT_MAP);
210 expect([...restored.keys()]).toEqual(["/good.jsonl"]);
211 });
212
213 it("rejects the whole cache when an intern table holds a non-string", () => {
214 // models: [1] would pass the undefined guard, put a number in a record's
215 // model, and crash lookupRate at aggregate time.
216 const encoded = encodeScanCache(cacheWith([["/a.jsonl", 100, [record()]]]));
217 const poisoned = { ...encoded, models: [1] };
218
219 expect(decodeScanCache(JSON.parse(JSON.stringify(poisoned)), TEST_FORMAT_MAP).size).toBe(0);
220 });
221
222 it("drops the whole entry when any row is corrupt, forcing a cold re-parse", () => {
223 // Keeping the surviving rows under the original (size, mtime) would read
224 // as a valid warm hit and the file would never be re-parsed.
225 const encoded = encodeScanCache(
226 cacheWith([["/a.jsonl", 100, [record(), record({ dedupeKey: "msg_2:" })]]]),
227 );
228 const rows = encoded.files["/a.jsonl"]!.r;
229 const poisoned = {
230 ...encoded,
231 files: {
232 "/a.jsonl": {
233 ...encoded.files["/a.jsonl"]!,
234 r: [rows[0]!, [...rows[1]!.slice(0, 3), "not-a-number", ...rows[1]!.slice(4)]],
235 },
236 },
237 };
238
239 const restored = decodeScanCache(JSON.parse(JSON.stringify(poisoned)), TEST_FORMAT_MAP);
240 expect(restored.has("/a.jsonl")).toBe(false);
241 });
242});
243
244describe("pruneScanCache", () => {
245 const retentionCutoffMs = 1000;
246
247 it("drops entries older than retention", () => {
248 const cache = cacheWith([["/old.jsonl", 500, [record()]]]);
249
250 const removed = pruneScanCache(cache, retentionCutoffMs);
251
252 expect(removed).toBe(1);
253 expect(cache.size).toBe(0);
254 });
255
256 it("keeps entries whose file has disappeared", () => {
257 const cache = cacheWith([["/gone.jsonl", 5000, [record()]]]);
258
259 pruneScanCache(cache, retentionCutoffMs);
260
261 expect(cache.size).toBe(1);
262 });
263});
264
265describe("dedupeWithinFile", () => {
266 it("keeps the first record per dedupe key", () => {
267 const kept = dedupeWithinFile([
268 record({ totals: { ...record().totals, outputTokens: 1 } }),
269 record({ totals: { ...record().totals, outputTokens: 999 } }),
270 record({ dedupeKey: "msg_2:" }),
271 ]);
272
273 expect(kept).toHaveLength(2);
274 expect(kept[0]?.totals.outputTokens).toBe(1);
275 });
276
277 it("keeps every record that has no dedupe key", () => {
278 expect(
279 dedupeWithinFile([record({ dedupeKey: null }), record({ dedupeKey: null })]),
280 ).toHaveLength(2);
281 });
282});