diff --git a/apps/server/src/usage/UsageService.test.ts b/apps/server/src/usage/UsageService.test.ts index cb9270a7ed3b9570ef3b075b54e0126332e08bb2..393437fcc37b1b7c6a8a84e422827b5769f0eb2b 100644 --- a/apps/server/src/usage/UsageService.test.ts +++ b/apps/server/src/usage/UsageService.test.ts @@ -996,6 +996,34 @@ describe("UsageService", () => { }).pipe(Effect.scoped), ); + it.live.skipIf(HostProcess.Platform.defaultValue() === "win32" || process.getuid?.() === 0)( + "reports unreadable transcripts as partial and recovers once they can be read", + () => + Effect.gen(function* () { + const { transcript, settings, home } = yield* setup; + const unreadable = NodePath.join(NodePath.dirname(transcript), "other.jsonl"); + yield* Effect.promise(async () => { + await NodeFSP.writeFile(transcript, claudeLine(1, 5)); + await NodeFSP.writeFile(unreadable, claudeLine(2, 7)); + await NodeFSP.chmod(unreadable, 0); + }); + yield* Effect.gen(function* () { + const service = yield* UsageService.make; + const partial = yield* service.readSummary(WINDOW); + assert.strictEqual(totalOutputTokens(partial), 5); + assert.strictEqual(partial.sources[0]?.status, "partial"); + + yield* Effect.promise(() => NodeFSP.chmod(unreadable, 0o600)); + const healthy = yield* service.readSummary(WINDOW); + assert.strictEqual(totalOutputTokens(healthy), 12); + assert.strictEqual(healthy.sources[0]?.status, "ok"); + assert.isNull(healthy.sources[0]?.message); + }).pipe( + Effect.provide(layerService({ prefix: "usage-service-unreadable", home, settings })), + ); + }).pipe(Effect.scoped), + ); + it.live("reprices unchanged transcripts when custom prices are added, edited, or removed", () => Effect.gen(function* () { const { transcript, settings, home } = yield* setup; diff --git a/apps/server/src/usage/UsageService.ts b/apps/server/src/usage/UsageService.ts index 3b496d8f4498b34ff9c3c7f94e438fec231dc346..2259c1d9f7730f98ce8fb4cd5664fe4bd75608fb 100644 --- a/apps/server/src/usage/UsageService.ts +++ b/apps/server/src/usage/UsageService.ts @@ -519,6 +519,7 @@ export const make = Effect.gen(function* () { format: TranscriptUsageFormat, ): Effect.Effect<{ readonly records: readonly UsageRecord[]; + readonly failed?: true; readonly update?: { readonly entry: CachedFile; readonly replaces: CachedFile | undefined }; }> => Effect.gen(function* () { @@ -554,6 +555,7 @@ export const make = Effect.gen(function* () { if (parsed === null) return { records: cached?.provider === provider ? [...cached.records, ...cached.tailRecords] : [], + failed: true, }; // Stored already de-duplicated within the file, which is 99% of all @@ -600,7 +602,7 @@ export const make = Effect.gen(function* () { .exists(dir) .pipe(Effect.catchCause(() => Effect.succeed(false))); if (!exists) return { provider, dir, volumeId, files: null } satisfies ScannedDir; - const files = yield* Effect.promise(() => + const { files, failedPaths } = yield* Effect.promise(() => listTranscriptFiles(dir, windowStartMs, fileName === undefined ? undefined : { fileName }), ); // A cold parse waits on disk reads, so a few files in flight read @@ -629,7 +631,20 @@ export const make = Effect.gen(function* () { } return { path, records }; }); - return { provider, dir, volumeId, files: parsedFiles } satisfies ScannedDir; + // Unread files keep their cached usage, but the total may be short. + const unread = failedPaths + read.filter((file) => file.failed).length; + return { + provider, + dir, + volumeId, + files: parsedFiles, + ...(unread > 0 + ? { + status: "partial", + message: `${unread} transcript path(s) could not be read; usage may be incomplete.`, + } + : {}), + } satisfies ScannedDir; }); const collectDirs = Effect.fn("UsageService.collectDirs")(function* ( diff --git a/apps/server/src/usage/usageTranscriptReader.ts b/apps/server/src/usage/usageTranscriptReader.ts index 16684e3a85abd5476bf4f125f81fe7aa3115e7e5..dbf0fd7aeb35f80718071ee8fecfc611f9daaba2 100644 --- a/apps/server/src/usage/usageTranscriptReader.ts +++ b/apps/server/src/usage/usageTranscriptReader.ts @@ -109,9 +109,9 @@ function fnv1a(buffer: Buffer): number { /** * Lists `.jsonl` transcripts under `root` last modified at or after `sinceMs`. * - * Errors on individual entries are swallowed: session files rotate and get - * removed while the walk is in flight, and a partial listing is far better than - * failing the page. + * Unreadable directories and files are counted in `failedPaths` rather than + * failing the page, so the source can report incomplete usage. Files that + * vanish between `readdir` and `stat` are ordinary rotation and not counted. * * `fileName` restricts the walk to a single basename (Grok's `updates.jsonl`). * Grok sessions also ship multi-megabyte `chat_history` and `events` logs that @@ -126,14 +126,16 @@ export async function listTranscriptFiles( root: string, sinceMs: number, options?: { readonly fileName?: string }, -): Promise { +): Promise<{ readonly files: readonly TranscriptFile[]; readonly failedPaths: number }> { const fileName = options?.fileName; const candidates: string[] = []; + let failedPaths = 0; const walk = async (dir: string): Promise => { let entries; try { entries = await NodeFSP.readdir(dir, { withFileTypes: true }); } catch { + failedPaths += 1; return; } for (const entry of entries) { @@ -156,15 +158,17 @@ export async function listTranscriptFiles( if (stats.mtimeMs >= sinceMs) { found[index] = { path, size: stats.size, mtimeMs: stats.mtimeMs }; } - } catch { - // Vanished between readdir and stat. + } catch (error) { + if (!(error instanceof Error && "code" in error && error.code === "ENOENT")) { + failedPaths += 1; + } } } }; await Promise.all( Array.from({ length: Math.min(STAT_CONCURRENCY, candidates.length) }, statQueued), ); - return found.filter((file) => file !== undefined); + return { files: found.filter((file) => file !== undefined), failedPaths }; } /**