| author |  maria <maria@kuuro.net> 2026-10-09 22:50:44-03:00 |
| committer |  GitHub <noreply@github.com> 2026-10-09 22:50:44-03:00 |
| log | b87b13d41426418c4f42c6477ce62cfe2a43fbae |
| tree | 0cce475413c80c13f6e9c105bce42df83d8d74ad |
| parent | b744bded016b15b9b5ade552a3e28270ac20176f |
| signature | Signed by PGP key B5690EEEBB952194 |
fix(server): report incomplete transcript usage scans (#15661)
3 files changed, 56 insertions(+), 9 deletions(-)
apps
apps/server/src/usage/UsageService.test.ts+28
| ... | ... | @@ -996,6 +996,34 @@ describe("UsageService", () => { |
| 996 | 996 | }).pipe(Effect.scoped), |
| 997 | 997 | ); |
| 998 | 998 | |
| 999 | it.live.skipIf(HostProcess.Platform.defaultValue() === "win32" || process.getuid?.() === 0)( | |
| 1000 | "reports unreadable transcripts as partial and recovers once they can be read", | |
| 1001 | () => | |
| 1002 | Effect.gen(function* () { | |
| 1003 | const { transcript, settings, home } = yield* setup; | |
| 1004 | const unreadable = NodePath.join(NodePath.dirname(transcript), "other.jsonl"); | |
| 1005 | yield* Effect.promise(async () => { | |
| 1006 | await NodeFSP.writeFile(transcript, claudeLine(1, 5)); | |
| 1007 | await NodeFSP.writeFile(unreadable, claudeLine(2, 7)); | |
| 1008 | await NodeFSP.chmod(unreadable, 0); | |
| 1009 | }); | |
| 1010 | yield* Effect.gen(function* () { | |
| 1011 | const service = yield* UsageService.make; | |
| 1012 | const partial = yield* service.readSummary(WINDOW); | |
| 1013 | assert.strictEqual(totalOutputTokens(partial), 5); | |
| 1014 | assert.strictEqual(partial.sources[0]?.status, "partial"); | |
| 1015 | ||
| 1016 | yield* Effect.promise(() => NodeFSP.chmod(unreadable, 0o600)); | |
| 1017 | const healthy = yield* service.readSummary(WINDOW); | |
| 1018 | assert.strictEqual(totalOutputTokens(healthy), 12); | |
| 1019 | assert.strictEqual(healthy.sources[0]?.status, "ok"); | |
| 1020 | assert.isNull(healthy.sources[0]?.message); | |
| 1021 | }).pipe( | |
| 1022 | Effect.provide(layerService({ prefix: "usage-service-unreadable", home, settings })), | |
| 1023 | ); | |
| 1024 | }).pipe(Effect.scoped), | |
| 1025 | ); | |
| 1026 | ||
| 999 | 1027 | it.live("reprices unchanged transcripts when custom prices are added, edited, or removed", () => |
| 1000 | 1028 | Effect.gen(function* () { |
| 1001 | 1029 | const { transcript, settings, home } = yield* setup; |
apps/server/src/usage/UsageService.ts+17-2
| ... | ... | @@ -519,6 +519,7 @@ export const make = Effect.gen(function* () { |
| 519 | 519 | format: TranscriptUsageFormat<unknown>, |
| 520 | 520 | ): Effect.Effect<{ |
| 521 | 521 | readonly records: readonly UsageRecord[]; |
| 522 | readonly failed?: true; | |
| 522 | 523 | readonly update?: { readonly entry: CachedFile; readonly replaces: CachedFile | undefined }; |
| 523 | 524 | }> => |
| 524 | 525 | Effect.gen(function* () { |
| ... | ... | @@ -554,6 +555,7 @@ export const make = Effect.gen(function* () { |
| 554 | 555 | if (parsed === null) |
| 555 | 556 | return { |
| 556 | 557 | records: cached?.provider === provider ? [...cached.records, ...cached.tailRecords] : [], |
| 558 | failed: true, | |
| 557 | 559 | }; |
| 558 | 560 | |
| 559 | 561 | // Stored already de-duplicated within the file, which is 99% of all |
| ... | ... | @@ -600,7 +602,7 @@ export const make = Effect.gen(function* () { |
| 600 | 602 | .exists(dir) |
| 601 | 603 | .pipe(Effect.catchCause(() => Effect.succeed(false))); |
| 602 | 604 | if (!exists) return { provider, dir, volumeId, files: null } satisfies ScannedDir; |
| 603 | const files = yield* Effect.promise(() => | |
| 605 | const { files, failedPaths } = yield* Effect.promise(() => | |
| 604 | 606 | listTranscriptFiles(dir, windowStartMs, fileName === undefined ? undefined : { fileName }), |
| 605 | 607 | ); |
| 606 | 608 | // A cold parse waits on disk reads, so a few files in flight read |
| ... | ... | @@ -629,7 +631,20 @@ export const make = Effect.gen(function* () { |
| 629 | 631 | } |
| 630 | 632 | return { path, records }; |
| 631 | 633 | }); |
| 632 | return { provider, dir, volumeId, files: parsedFiles } satisfies ScannedDir; | |
| 634 | // Unread files keep their cached usage, but the total may be short. | |
| 635 | const unread = failedPaths + read.filter((file) => file.failed).length; | |
| 636 | return { | |
| 637 | provider, | |
| 638 | dir, | |
| 639 | volumeId, | |
| 640 | files: parsedFiles, | |
| 641 | ...(unread > 0 | |
| 642 | ? { | |
| 643 | status: "partial", | |
| 644 | message: `${unread} transcript path(s) could not be read; usage may be incomplete.`, | |
| 645 | } | |
| 646 | : {}), | |
| 647 | } satisfies ScannedDir; | |
| 633 | 648 | }); |
| 634 | 649 | |
| 635 | 650 | const collectDirs = Effect.fn("UsageService.collectDirs")(function* ( |
apps/server/src/usage/usageTranscriptReader.ts+11-7
| ... | ... | @@ -109,9 +109,9 @@ function fnv1a(buffer: Buffer): number { |
| 109 | 109 | /** |
| 110 | 110 | * Lists `.jsonl` transcripts under `root` last modified at or after `sinceMs`. |
| 111 | 111 | * |
| 112 | * Errors on individual entries are swallowed: session files rotate and get | |
| 113 | * removed while the walk is in flight, and a partial listing is far better than | |
| 114 | * failing the page. | |
| 112 | * Unreadable directories and files are counted in `failedPaths` rather than | |
| 113 | * failing the page, so the source can report incomplete usage. Files that | |
| 114 | * vanish between `readdir` and `stat` are ordinary rotation and not counted. | |
| 115 | 115 | * |
| 116 | 116 | * `fileName` restricts the walk to a single basename (Grok's `updates.jsonl`). |
| 117 | 117 | * Grok sessions also ship multi-megabyte `chat_history` and `events` logs that |
| ... | ... | @@ -126,14 +126,16 @@ export async function listTranscriptFiles( |
| 126 | 126 | root: string, |
| 127 | 127 | sinceMs: number, |
| 128 | 128 | options?: { readonly fileName?: string }, |
| 129 | ): Promise<readonly TranscriptFile[]> { | |
| 129 | ): Promise<{ readonly files: readonly TranscriptFile[]; readonly failedPaths: number }> { | |
| 130 | 130 | const fileName = options?.fileName; |
| 131 | 131 | const candidates: string[] = []; |
| 132 | let failedPaths = 0; | |
| 132 | 133 | const walk = async (dir: string): Promise<void> => { |
| 133 | 134 | let entries; |
| 134 | 135 | try { |
| 135 | 136 | entries = await NodeFSP.readdir(dir, { withFileTypes: true }); |
| 136 | 137 | } catch { |
| 138 | failedPaths += 1; | |
| 137 | 139 | return; |
| 138 | 140 | } |
| 139 | 141 | for (const entry of entries) { |
| ... | ... | @@ -156,15 +158,17 @@ export async function listTranscriptFiles( |
| 156 | 158 | if (stats.mtimeMs >= sinceMs) { |
| 157 | 159 | found[index] = { path, size: stats.size, mtimeMs: stats.mtimeMs }; |
| 158 | 160 | } |
| 159 | } catch { | |
| 160 | // Vanished between readdir and stat. | |
| 161 | } catch (error) { | |
| 162 | if (!(error instanceof Error && "code" in error && error.code === "ENOENT")) { | |
| 163 | failedPaths += 1; | |
| 164 | } | |
| 161 | 165 | } |
| 162 | 166 | } |
| 163 | 167 | }; |
| 164 | 168 | await Promise.all( |
| 165 | 169 | Array.from({ length: Math.min(STAT_CONCURRENCY, candidates.length) }, statQueued), |
| 166 | 170 | ); |
| 167 | return found.filter((file) => file !== undefined); | |
| 171 | return { files: found.filter((file) => file !== undefined), failedPaths }; | |
| 168 | 172 | } |
| 169 | 173 | |
| 170 | 174 | /** |