| ... | ... | @@ -14,7 +14,6 @@ import { |
| 14 | 14 | import * as HostProcess from "@t3tools/shared/HostProcess"; |
| 15 | 15 | import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; |
| 16 | 16 | import * as Effect from "effect/Effect"; |
| 17 | | import * as Exit from "effect/Exit"; |
| 18 | 17 | import * as FileSystem from "effect/FileSystem"; |
| 19 | 18 | import * as Option from "effect/Option"; |
| 20 | 19 | import * as Path from "effect/Path"; |
| ... | ... | @@ -94,14 +93,29 @@ export interface OpenCodeUsageReadResult { |
| 94 | 93 | const isNotFound = (cause: PlatformError.PlatformError) => cause.reason._tag === "NotFound"; |
| 95 | 94 | const MESSAGE_PAGE_SIZE = 64; |
| 96 | 95 | |
| 96 | /** Whether the path is itself a symlink, without following it. */ |
| 97 | const isSymbolicLink = Effect.fnUntraced(function* (path: string) { |
| 98 | const fileSystem = yield* FileSystem.FileSystem; |
| 99 | return yield* Effect.isSuccess(fileSystem.readLink(path)); |
| 100 | }); |
| 101 | |
| 97 | 102 | /** Stat of a regular file or directory; a symlink reports as itself and is never followed. */ |
| 98 | 103 | const entryInfo = Effect.fn("entryInfo")(function* (path: string) { |
| 99 | 104 | const fileSystem = yield* FileSystem.FileSystem; |
| 100 | | const link = yield* Effect.exit(fileSystem.readLink(path)); |
| 101 | | if (Exit.isSuccess(link)) return { type: "SymbolicLink" as const, mtime: Option.none<Date>() }; |
| 105 | if (yield* isSymbolicLink(path)) |
| 106 | return { type: "SymbolicLink" as const, mtime: Option.none<Date>() }; |
| 102 | 107 | return yield* fileSystem.stat(path); |
| 103 | 108 | }); |
| 104 | 109 | |
| 110 | /** |
| 111 | * Stats run in the libuv thread pool, so a few at a time cut the walk's wall |
| 112 | * time without lengthening any event-loop turn. |
| 113 | */ |
| 114 | const LEGACY_STAT_CONCURRENCY = 4; |
| 115 | |
| 116 | /** Entries stat'd before any is processed, so one huge directory is not held in memory at once. */ |
| 117 | const LEGACY_STAT_BATCH = 256; |
| 118 | |
| 105 | 119 | /** Reads current SQLite and pre-migration JSON stores without modifying either. */ |
| 106 | 120 | export const readOpenCodeUsage = Effect.fn("readOpenCodeUsage")(function* ( |
| 107 | 121 | root: string, |
| ... | ... | @@ -234,34 +248,61 @@ export const readOpenCodeUsage = Effect.fn("readOpenCodeUsage")(function* ( |
| 234 | 248 | while (directories.length > 0) { |
| 235 | 249 | const directory = directories.pop()!; |
| 236 | 250 | yield* Effect.gen(function* () { |
| 237 | | for (const name of yield* fileSystem.readDirectory(directory)) { |
| 238 | | const entry = path.join(directory, name); |
| 239 | | const info = yield* entryInfo(entry).pipe( |
| 240 | | Effect.catchTags({ |
| 241 | | PlatformError: (cause) => |
| 242 | | isNotFound(cause) ? Effect.succeed(null) : Effect.fail(cause), |
| 243 | | }), |
| 251 | const names = yield* fileSystem.readDirectory(directory); |
| 252 | for (let start = 0; start < names.length; start += LEGACY_STAT_BATCH) { |
| 253 | const batch = names.slice(start, start + LEGACY_STAT_BATCH); |
| 254 | const entries = batch.map((name) => path.join(directory, name)); |
| 255 | // `stat` follows symlinks and `readLink` fails, expensively, for |
| 256 | // everything else. Stat a batch, then ask whether an entry is a symlink |
| 257 | // only where the answer changes the result. |
| 258 | const infos = yield* Effect.forEach( |
| 259 | entries, |
| 260 | (entry) => |
| 261 | fileSystem.stat(entry).pipe( |
| 262 | Effect.catchTags({ |
| 263 | // A symlink whose target cannot be stat'd is skipped, not an error. |
| 264 | PlatformError: (cause) => |
| 265 | isNotFound(cause) |
| 266 | ? Effect.succeed(null) |
| 267 | : isSymbolicLink(entry).pipe( |
| 268 | Effect.flatMap((link) => |
| 269 | link ? Effect.succeed(null) : Effect.fail(cause), |
| 270 | ), |
| 271 | ), |
| 272 | }), |
| 273 | Effect.exit, |
| 274 | ), |
| 275 | { concurrency: LEGACY_STAT_CONCURRENCY }, |
| 244 | 276 | ); |
| 245 | | if (info?.type === "Directory") { |
| 246 | | directories.push(entry); |
| 247 | | } else if (info?.type === "File" && name.endsWith(".json")) { |
| 248 | | found = true; |
| 249 | | const id = name.slice(0, -5); |
| 250 | | if (seen.has(`opencode:${id}`)) continue; |
| 251 | | // A message cannot be created after its file was last written, so a |
| 252 | | // file untouched since the window opened holds nothing in range. |
| 253 | | if (Option.exists(info.mtime, (mtime) => mtime.getTime() < sinceMs)) continue; |
| 254 | | const file = { path: entry, records: [] as UsageRecord[] }; |
| 255 | | files.push(file); |
| 256 | | yield* fileSystem.readFileString(entry).pipe( |
| 257 | | Effect.map((source) => append(file.records, parseOpenCodeMessage(source, { id }))), |
| 258 | | Effect.catchTags({ |
| 259 | | PlatformError: (cause) => { |
| 260 | | if (!isNotFound(cause)) error = true; |
| 261 | | return Effect.void; |
| 262 | | }, |
| 263 | | }), |
| 264 | | ); |
| 277 | for (const [index, name] of batch.entries()) { |
| 278 | const entry = entries[index]!; |
| 279 | const info = yield* infos[index]!; |
| 280 | if (info?.type === "Directory") { |
| 281 | if (!(yield* isSymbolicLink(entry))) directories.push(entry); |
| 282 | } else if (info?.type === "File" && name.endsWith(".json")) { |
| 283 | const id = name.slice(0, -5); |
| 284 | // A message cannot be created after its file was last written, so a |
| 285 | // file untouched since the window opened holds nothing in range. |
| 286 | const skip = |
| 287 | seen.has(`opencode:${id}`) || |
| 288 | Option.exists(info.mtime, (mtime) => mtime.getTime() < sinceMs); |
| 289 | // A skipped file only matters as proof that the store exists. |
| 290 | if (found && skip) continue; |
| 291 | if (yield* isSymbolicLink(entry)) continue; |
| 292 | found = true; |
| 293 | if (skip) continue; |
| 294 | const file = { path: entry, records: [] as UsageRecord[] }; |
| 295 | files.push(file); |
| 296 | yield* fileSystem.readFileString(entry).pipe( |
| 297 | Effect.map((source) => append(file.records, parseOpenCodeMessage(source, { id }))), |
| 298 | Effect.catchTags({ |
| 299 | PlatformError: (cause) => { |
| 300 | if (!isNotFound(cause)) error = true; |
| 301 | return Effect.void; |
| 302 | }, |
| 303 | }), |
| 304 | ); |
| 305 | } |
| 265 | 306 | } |
| 266 | 307 | } |
| 267 | 308 | }).pipe( |
Tristan Knight <admin@snappeh.com> 2026-10-11 02:52:10+01:00
GitHub <noreply@github.com> 2026-10-10 18:52:10-07:00