1// @effect-diagnostics nodeBuiltinImport:off - resume coverage writes, appends
2// to, and truncates real transcript files byte-exactly, mirroring the reader's
3// own deliberate node:fs usage.
4import * as NodeFSP from "node:fs/promises";
5import * as NodeOS from "node:os";
6import * as NodePath from "node:path";
7
8import { afterEach, assert, beforeEach, describe, it } from "@effect/vitest";
9
10import { TEST_FORMATS } from "./usageTestFormats.ts";
11import { readTranscriptRecords, transcriptLineGate } from "./usageTranscriptReader.ts";
12
13let dir: string;
14
15beforeEach(async () => {
16 dir = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "usage-reader-test-"));
17});
18
19afterEach(async () => {
20 await NodeFSP.rm(dir, { recursive: true, force: true });
21});
22
23function claudeLine(id: number, outputTokens: number): string {
24 return `${JSON.stringify({
25 type: "assistant",
26 timestamp: "2026-08-01T10:00:00Z",
27 requestId: `req_${id}`,
28 sessionId: "session-1",
29 message: {
30 id: `msg_${id}`,
31 model: "claude-fable-5",
32 usage: { input_tokens: 10, output_tokens: outputTokens },
33 },
34 })}\n`;
35}
36
37function codexMetaLine(): string {
38 return `${JSON.stringify({
39 type: "session_meta",
40 timestamp: "2026-08-01T10:00:00Z",
41 payload: { type: "session_meta", id: "codex-session-1" },
42 })}\n`;
43}
44
45function codexModelLine(model: string): string {
46 return `${JSON.stringify({
47 type: "turn_context",
48 timestamp: "2026-08-01T10:00:01Z",
49 payload: { type: "turn_context", model },
50 })}\n`;
51}
52
53function codexTierLine(serviceTier: string): string {
54 return `${JSON.stringify({
55 type: "event_msg",
56 timestamp: "2026-08-01T10:00:01Z",
57 payload: { type: "thread_settings_applied", thread_settings: { service_tier: serviceTier } },
58 })}\n`;
59}
60
61function codexUsageLine(outputTokens: number, secondsOffset: number): string {
62 return `${JSON.stringify({
63 type: "event_msg",
64 timestamp: `2026-08-01T10:00:${String(secondsOffset).padStart(2, "0")}Z`,
65 payload: {
66 type: "token_count",
67 info: { last_token_usage: { input_tokens: 100, output_tokens: outputTokens } },
68 },
69 })}\n`;
70}
71
72describe("readTranscriptRecords resume", () => {
73 it("parses only appended lines when resuming a grown file", async () => {
74 const path = NodePath.join(dir, "claude.jsonl");
75 await NodeFSP.writeFile(path, claudeLine(1, 5) + claudeLine(2, 7));
76 const first = await readTranscriptRecords(path, TEST_FORMATS.claude);
77 assert.isNotNull(first);
78 assert.strictEqual(first.records.length, 2);
79 assert.isFalse(first.resumed);
80
81 await NodeFSP.appendFile(path, claudeLine(3, 11));
82 const second = await readTranscriptRecords(path, TEST_FORMATS.claude, first.position);
83 assert.isNotNull(second);
84 assert.isTrue(second.resumed);
85 assert.strictEqual(second.records.length, 1);
86 assert.strictEqual(second.records[0]?.totals.outputTokens, 11);
87
88 // The stitched result matches a from-scratch parse of the whole file.
89 const full = await readTranscriptRecords(path, TEST_FORMATS.claude);
90 assert.isNotNull(full);
91 assert.deepStrictEqual([...first.records, ...second.records], [...full.records]);
92 });
93
94 it("carries the Codex reducer state across the resume boundary", async () => {
95 const path = NodePath.join(dir, "rollout.jsonl");
96 await NodeFSP.writeFile(
97 path,
98 codexMetaLine() + codexModelLine("gpt-5.2-codex") + codexTierLine("ultrafast"),
99 );
100 const first = await readTranscriptRecords(path, TEST_FORMATS.codex);
101 assert.isNotNull(first);
102 assert.strictEqual(first.records.length, 0);
103
104 // The appended usage event has no turn_context, thread settings, or
105 // session_meta of its own; model, tier, and session must come from the
106 // state captured before the boundary.
107 await NodeFSP.appendFile(path, codexUsageLine(9, 5));
108 const second = await readTranscriptRecords(path, TEST_FORMATS.codex, first.position);
109 assert.isNotNull(second);
110 assert.isTrue(second.resumed);
111 assert.strictEqual(second.records.length, 1);
112 assert.strictEqual(second.records[0]?.model, "gpt-5.2-codex");
113 assert.strictEqual(second.records[0]?.speed, "ultrafast");
114 assert.strictEqual(second.records[0]?.sessionId, "codex-session-1");
115 });
116
117 it("suppresses a Codex duplicate usage event that straddles the boundary", async () => {
118 const path = NodePath.join(dir, "rollout.jsonl");
119 await NodeFSP.writeFile(
120 path,
121 codexMetaLine() + codexModelLine("gpt-5.2-codex") + codexUsageLine(9, 5),
122 );
123 const first = await readTranscriptRecords(path, TEST_FORMATS.codex);
124 assert.isNotNull(first);
125 assert.strictEqual(first.records.length, 1);
126
127 // Codex re-emits an unchanged token_count on stream boundaries; the copy
128 // lands after the resume point and must still be dropped.
129 await NodeFSP.appendFile(path, codexUsageLine(9, 5) + codexUsageLine(21, 8));
130 const second = await readTranscriptRecords(path, TEST_FORMATS.codex, first.position);
131 assert.isNotNull(second);
132 assert.isTrue(second.resumed);
133 assert.deepStrictEqual(
134 second.records.map((record) => record.totals.outputTokens),
135 [21],
136 );
137 });
138
139 it("defers an unterminated trailing line to tailRecords, then consumes it once terminated", async () => {
140 const path = NodePath.join(dir, "claude.jsonl");
141 const unterminated = claudeLine(2, 7).trimEnd();
142 await NodeFSP.writeFile(path, claudeLine(1, 5) + unterminated);
143 const first = await readTranscriptRecords(path, TEST_FORMATS.claude);
144 assert.isNotNull(first);
145 assert.strictEqual(first.records.length, 1);
146 assert.strictEqual(first.tailRecords.length, 1);
147 assert.strictEqual(first.tailRecords[0]?.totals.outputTokens, 7);
148
149 // Completing the line and appending another re-reads from the resume
150 // point, so the once-tail record arrives exactly once as a line record.
151 await NodeFSP.appendFile(path, `\n${claudeLine(3, 11)}`);
152 const second = await readTranscriptRecords(path, TEST_FORMATS.claude, first.position);
153 assert.isNotNull(second);
154 assert.isTrue(second.resumed);
155 assert.deepStrictEqual(
156 second.records.map((record) => record.totals.outputTokens),
157 [7, 11],
158 );
159 assert.strictEqual(second.tailRecords.length, 0);
160 });
161
162 it("re-parses from the start when the guard bytes no longer match", async () => {
163 const path = NodePath.join(dir, "claude.jsonl");
164 await NodeFSP.writeFile(path, claudeLine(1, 5));
165 const first = await readTranscriptRecords(path, TEST_FORMATS.claude);
166 assert.isNotNull(first);
167
168 // Same path, larger size, different content: a replaced file, not growth.
169 await NodeFSP.writeFile(path, claudeLine(4, 13) + claudeLine(5, 17));
170 const second = await readTranscriptRecords(path, TEST_FORMATS.claude, first.position);
171 assert.isNotNull(second);
172 assert.isFalse(second.resumed);
173 assert.deepStrictEqual(
174 second.records.map((record) => record.totals.outputTokens),
175 [13, 17],
176 );
177 });
178
179 it("re-parses from the start when the file shrank below the resume point", async () => {
180 const path = NodePath.join(dir, "claude.jsonl");
181 await NodeFSP.writeFile(path, claudeLine(1, 5) + claudeLine(2, 7));
182 const first = await readTranscriptRecords(path, TEST_FORMATS.claude);
183 assert.isNotNull(first);
184
185 await NodeFSP.writeFile(path, claudeLine(3, 11));
186 const second = await readTranscriptRecords(path, TEST_FORMATS.claude, first.position);
187 assert.isNotNull(second);
188 assert.isFalse(second.resumed);
189 assert.deepStrictEqual(
190 second.records.map((record) => record.totals.outputTokens),
191 [11],
192 );
193 });
194
195 it("parses a line larger than one stream chunk", async () => {
196 // Tool-heavy transcripts carry multi-megabyte single lines; they arrive
197 // split across many chunks and must reassemble into one record.
198 const path = NodePath.join(dir, "claude.jsonl");
199 const bigLine = `${JSON.stringify({
200 type: "assistant",
201 timestamp: "2026-08-01T10:00:00Z",
202 requestId: "req_big",
203 sessionId: "session-1",
204 padding: "x".repeat(512 * 1024),
205 message: {
206 id: "msg_big",
207 model: "claude-fable-5",
208 usage: { input_tokens: 10, output_tokens: 42 },
209 },
210 })}\n`;
211 await NodeFSP.writeFile(path, bigLine + claudeLine(2, 7));
212
213 const parsed = await readTranscriptRecords(path, TEST_FORMATS.claude);
214 assert.isNotNull(parsed);
215 assert.deepStrictEqual(
216 parsed.records.map((record) => record.totals.outputTokens),
217 [42, 7],
218 );
219 });
220
221 it("keeps a CRLF Codex reducer line whose marker straddles a stream chunk", async () => {
222 const path = NodePath.join(dir, "rollout.jsonl");
223 const crlf = (line: string) => line.replace("\n", "\r\n");
224 const modelLine = crlf(codexModelLine("gpt-5.2-codex"));
225 // Multi-byte tool output that fails the gate, sized so the 256 KiB chunk
226 // boundary falls inside the model line's `"turn_context"` marker.
227 const head = crlf(codexMetaLine());
228 const fillerBytes =
229 256 * 1024 - Buffer.byteLength(head) - modelLine.indexOf("turn_context") - 4;
230 const fillerEnvelope = crlf(`${JSON.stringify({ type: "response_item", output: "" })}\n`);
231 const padding = "é".repeat((fillerBytes - Buffer.byteLength(fillerEnvelope)) / 2);
232 const filler = crlf(`${JSON.stringify({ type: "response_item", output: padding })}\n`);
233 const content = head + filler + modelLine + crlf(codexUsageLine(40, 2));
234 await NodeFSP.writeFile(path, content);
235
236 const result = await readTranscriptRecords(path, TEST_FORMATS.codex);
237 assert.isNotNull(result);
238 assert.strictEqual(result.records.length, 1);
239 assert.strictEqual(result.records[0]?.model, "gpt-5.2-codex");
240 assert.strictEqual(result.records[0]?.sessionId, "codex-session-1");
241 assert.strictEqual(result.position.resumeOffset, Buffer.byteLength(content));
242 });
243
244 it("returns null for an unreadable file", async () => {
245 assert.isNull(
246 await readTranscriptRecords(NodePath.join(dir, "missing.jsonl"), TEST_FORMATS.claude),
247 );
248 });
249});
250
251describe("transcriptLineGate", () => {
252 const gates = {
253 one: transcriptLineGate(['"usage"']),
254 several: transcriptLineGate(['"token_count"', '"session_meta"', '"a.b(c)"']),
255 };
256 const markers = { one: ['"usage"'], several: ['"token_count"', '"session_meta"', '"a.b(c)"'] };
257
258 it("passes only lines containing one of its markers", () => {
259 for (const kind of ["one", "several"] as const) {
260 for (const marker of markers[kind]) {
261 assert.isTrue(gates[kind](Buffer.from(`{"payload":{${marker}:1}}`)));
262 // The same text without its quotes, or with one character changed.
263 assert.isFalse(gates[kind](Buffer.from(`{"text":"see ${marker.slice(1, -1)} here"}`)));
264 assert.isFalse(gates[kind](Buffer.from(`{${marker.replace(/[._]|usage/, "x")}:1}`)));
265 }
266 assert.isFalse(gates[kind](Buffer.alloc(0)));
267 }
268 });
269
270 it("matches markers beside multi-byte and malformed UTF-8", () => {
271 const invalid = Buffer.from([0xff, 0xc3]);
272 for (const kind of ["one", "several"] as const) {
273 for (const marker of markers[kind]) {
274 assert.isTrue(gates[kind](Buffer.from(`{"text":"日本語 é",${marker}:1}`)));
275 assert.isTrue(gates[kind](Buffer.concat([invalid, Buffer.from(marker), invalid])));
276 assert.isFalse(gates[kind](Buffer.from(`{"text":"日本語 é ${marker.slice(1, -1)}"}`)));
277 }
278 }
279 });
280});