| 1 | /** |
| 2 | * NOTE: documentation is in progress |
| 3 | * |
| 4 | * Clover's `incremental.ts` is the foundation for incremental developer |
| 5 | * tools, such as site generators, linters, and more. A build script is |
| 6 | * written as an orchestration of individual steps, where each builder |
| 7 | * perfectly traces it's inputs and dependencies with other steps. In |
| 8 | * return, `@clo/lib/incremental` is able to watch for and rebuild only |
| 9 | * the steps that change. |
| 10 | * |
| 11 | * Consider a basic website generator that turns Markdown into HTML: |
| 12 | * `lib/incremental/example/markdown-blog.ts` |
| 13 | * |
| 14 | * In the above, the page render steps act individually, only rebuilding the |
| 15 | * post changed and the index. Notably, this will already handle a few fun cases: |
| 16 | * |
| 17 | * - Adding a new file in `posts/` does not rebuild any pages excep the new one. |
| 18 | * - Changing post contents but not frontmatter does not re-run `renderHomepage` |
| 19 | * |
| 20 | * Since return values of steps are cached, [i need to continue writing this] |
| 21 | * |
| 22 | * `@clo/lib/incremental` is implemented as a single file depending only on the |
| 23 | * node.js builtins and other `@clo/lib` libraries. i intend for this codebase |
| 24 | * to be a good introduction to how an incremental system can function. |
| 25 | * |
| 26 | * @module |
| 27 | */ |
| 28 | |
| 29 | // - [x] steps |
| 30 | // - [ ] network inputs |
| 31 | // - [ ] re-validating network inputs |
| 32 | // - [ ] timestamp source (separate file) |
| 33 | // - [ ] trace every file ever + other insane details like runtime info |
| 34 | // - [ ] visualization tools for the build graph (separate file) |
| 35 | |
| 36 | /** |
| 37 | * serialization special cases top-level `Uint8Array`s and `string`s to use more |
| 38 | * effective data storage strategies. other variants use JSON + a reviver, with |
| 39 | * nested `Uint8Array`s stored beside the JSON as raw bytes. |
| 40 | */ |
| 41 | export type Serializable = |
| 42 | | string |
| 43 | | number |
| 44 | | boolean |
| 45 | | null |
| 46 | | undefined |
| 47 | | Uint8Array |
| 48 | | Date |
| 49 | | Set<Serializable> |
| 50 | | Map<Serializable, Serializable> |
| 51 | | Serializable[] |
| 52 | | { [K in string]?: Serializable }; |
| 53 | |
| 54 | /** |
| 55 | * `T`, with `never` wherever it isn't {@linkcode Serializable} or `Also`. an |
| 56 | * interface lacks the index signature to be assigned to `Serializable`, so |
| 57 | * functions that take values check them as `T & ValidateSerializable<T>`. |
| 58 | */ |
| 59 | export type ValidateSerializable<T, Also = never> = T extends Serializable | Also ? T |
| 60 | : T extends Set<infer V> ? Set<ValidateSerializable<V, Also>> |
| 61 | : T extends Map<infer K, infer V> ? Map<ValidateSerializable<K, Also>, ValidateSerializable<V, Also>> |
| 62 | : T extends (...args: never) => unknown ? never |
| 63 | : T extends object ? { [K in keyof T]: ValidateSerializable<T[K], Also> } |
| 64 | : never; |
| 65 | |
| 66 | /** |
| 67 | * for values that a callback returns, which are inferred after the arguments |
| 68 | * are checked. until then they are `unknown`, which passes, so another error |
| 69 | * isn't blamed on the callback. |
| 70 | */ |
| 71 | type CheckSerializable<T, What extends string> = unknown extends T ? unknown |
| 72 | : [T] extends [ValidateSerializable<T, void>] ? unknown |
| 73 | : { [K in `${What} must be serializable`]: ValidateSerializable<T, void> }; |
| 74 | |
| 75 | export interface GraphOptions { |
| 76 | /** file path to a database, or `null` for an in-memory database. */ |
| 77 | db?: Database | string | null; |
| 78 | /** |
| 79 | * relative paths resolve against this, rather than wherever the working |
| 80 | * directory is later. defaults to the working directory. |
| 81 | */ |
| 82 | root?: string; |
| 83 | /** custom data {@linkcode Source}s, besides the rows of {@linkcode trackSqlite}d connections, which every graph checks */ |
| 84 | sources?: Source[]; |
| 85 | /** override the filesystem implementation. (does not cover CommonJS) */ |
| 86 | fs?: Fs; |
| 87 | /** |
| 88 | * key is an extension including the `.`, mapping to a function to load it's |
| 89 | * source code. added to the default JavaScript+Typescript (`.[mc]?[tj]s`), |
| 90 | * `.json`, `.node` loaders, which can't be replaced, since code node.js runs |
| 91 | * is traced with them. a custom loader's code is tracked through the module |
| 92 | * given to {@linkcode lazyLoader}, or else the file that created the graph. |
| 93 | */ |
| 94 | loaders?: Record<string, LoaderFunction>; |
| 95 | /** appended to the defaults. */ |
| 96 | conditions?: string[]; |
| 97 | /** |
| 98 | * modules to load with node.js instead of `node:vm`, shared with the host |
| 99 | * process. like `node_modules`, they never reload, so changing one requires a |
| 100 | * restart. |
| 101 | */ |
| 102 | external?: (file: string) => boolean; |
| 103 | /** |
| 104 | * the most bytes that the values of {@linkcode Graph#lazy} steps take in the |
| 105 | * database. past it, the ones used least recently go. defaults to 64MiB. |
| 106 | */ |
| 107 | lazyBytes?: number; |
| 108 | } |
| 109 | |
| 110 | export interface BuildOptions<T> { |
| 111 | /** |
| 112 | * `changed` is the urls of the inputs whose change caused this build. only |
| 113 | * a build of {@linkcode Graph#incrementalWatch} after its first has any. |
| 114 | */ |
| 115 | buildAndWriteOutput: (topLevel: Ref, changed: readonly string[]) => Promise<T>; |
| 116 | progress?: progress.Ref; |
| 117 | /** stops starting steps, and caching results that finish after an abort */ |
| 118 | signal?: AbortSignal; |
| 119 | /** |
| 120 | * re-run every step whose cached result has errors, reporting any that |
| 121 | * change. this exists to find bugs in this library. |
| 122 | */ |
| 123 | rerunErrors?: boolean; |
| 124 | } |
| 125 | |
| 126 | /** a build from {@linkcode Graph#incrementalWatch}, after the inputs at `changed` changed */ |
| 127 | export type WatchResult<T> = |
| 128 | & ( |
| 129 | | { ok: true; value: T; error?: undefined } |
| 130 | /** `value` is what `buildAndWriteOutput` returned despite the errors, unless it threw */ |
| 131 | | { ok: false; error: BuildError; value?: T } |
| 132 | ) |
| 133 | & { |
| 134 | /** urls of the inputs whose change caused this build, which the first build has none of */ |
| 135 | changed: string[]; |
| 136 | stats: BuildStats; |
| 137 | /** how many inputs are watched for the next build, by protocol, like `{ file: 12, sqlite: 3 }` */ |
| 138 | watching: Watching; |
| 139 | /** |
| 140 | * what is watched, for a log: "Watching 12 files and 3 other sources". the |
| 141 | * first build says so when it found nothing to do: "Build is up to date." |
| 142 | */ |
| 143 | statusLine: string; |
| 144 | }; |
| 145 | |
| 146 | /** what a build did */ |
| 147 | export interface BuildStats { |
| 148 | /** how long the build took, in milliseconds */ |
| 149 | duration: number; |
| 150 | /** steps whose code ran */ |
| 151 | ran: number; |
| 152 | /** steps whose result from an earlier build was used */ |
| 153 | reused: number; |
| 154 | /** output files that were written or copied, as the ones on disk were not what the build wrote last */ |
| 155 | written: number; |
| 156 | /** size of the database that everything is cached in, in bytes */ |
| 157 | cacheBytes: number; |
| 158 | /** size of every output file the build has written, in bytes */ |
| 159 | outputBytes: number; |
| 160 | } |
| 161 | |
| 162 | export interface Watching { |
| 163 | file: number; |
| 164 | /** the inputs of each source that reports changes */ |
| 165 | [protocol: string]: number; |
| 166 | } |
| 167 | |
| 168 | /** what happens to a step of a build, or of {@linkcode Graph#lazy}, for {@linkcode Graph#observe} */ |
| 169 | export type StepEvent = |
| 170 | & { |
| 171 | key: string; |
| 172 | label: string; |
| 173 | /** started by {@linkcode Graph#lazy} */ |
| 174 | lazy: boolean; |
| 175 | } |
| 176 | & ( |
| 177 | | { |
| 178 | /** |
| 179 | * `kept` is trusted from the last build of a watch, without a check. |
| 180 | * `checking` compares what it observed with what is there now, after |
| 181 | * which it is `reused`, or else `running`. |
| 182 | */ |
| 183 | state: "kept" | "checking" | "reused" | "running"; |
| 184 | } |
| 185 | | { |
| 186 | state: "ran" | "failed"; |
| 187 | /** the steps it read this time, by key, the inputs, with what each one was, and the urls of the files it wrote */ |
| 188 | steps: string[]; |
| 189 | inputs: Array<{ url: string; kind: string }>; |
| 190 | outputs: string[]; |
| 191 | } |
| 192 | ); |
| 193 | |
| 194 | const kDepend = Symbol("depend"); |
| 195 | const kImport = Symbol("import"); |
| 196 | const kInput = Symbol("input"); |
| 197 | const kInternal = Symbol("internal"); |
| 198 | const kMapFrame = Symbol("map frame"); |
| 199 | const kNative = Symbol("native"); |
| 200 | const kObserve = Symbol("observe"); |
| 201 | const kOption = Symbol("option"); |
| 202 | const kQuery = Symbol("query"); |
| 203 | const kRun = Symbol("run"); |
| 204 | const kRuntime = Symbol("runtime"); |
| 205 | const kSerialized = Symbol("serialized"); |
| 206 | const kStale = Symbol("stale"); |
| 207 | const kState = Symbol("step"); |
| 208 | const kStepEvent = Symbol("step event"); |
| 209 | |
| 210 | /** |
| 211 | * a web of modules and incremental state. |
| 212 | * this orchestrates {@link incrementalBuild|incremental builds}. |
| 213 | */ |
| 214 | export class Graph { |
| 215 | db: Database; |
| 216 | fs: Fs; |
| 217 | /** relative paths resolve against this */ |
| 218 | root: string; |
| 219 | #loaders: Record<string, LoaderFunction>; |
| 220 | #conditions: string[]; |
| 221 | #external: (file: string) => boolean; |
| 222 | #sources: Map<string, Source>; |
| 223 | /** the file that made the graph, which runs the loaders not from `lazyLoader` */ |
| 224 | #creator: string | null; |
| 225 | /** |
| 226 | * the build in progress, or else the last run that checked the loaded code. |
| 227 | * module instances always match what this run read. |
| 228 | */ |
| 229 | #run: Run | null = null; |
| 230 | /** the entry point, which keeps running its code even when it changes */ |
| 231 | #main: string | null = null; |
| 232 | /** module instances, with the module each was made from, all discarded whenever loaded code changes */ |
| 233 | #modules = new Map<string, { node: Module; instance: Promise<Instance> }>(); |
| 234 | /** every module the graph has made itself, rather than node.js, in any generation */ |
| 235 | #made = new Set<string>(); |
| 236 | /** steps that module instances were made from, and their hashes. replaced with every generation */ |
| 237 | #imported = new Map<string, string>(); |
| 238 | /** what modules observed while running, beyond their static imports */ |
| 239 | #runtime = new Map<string, Runtime>(); |
| 240 | /** source maps of modules in `node:vm`, applied to stack traces */ |
| 241 | #maps = new Map<string, string | module.SourceMap>(); |
| 242 | /** code node.js loaded, which can't be reloaded, by load step key */ |
| 243 | #native = new Map<string, string>(); |
| 244 | /** files loaded with `require`, removed from its cache on reset */ |
| 245 | #required = new Set<string>(); |
| 246 | /** compiled code from v8, reused while a module's code is unchanged */ |
| 247 | #code = new Map<string, { hash: string; data: Uint8Array }>(); |
| 248 | /** inputs a build of a watch checked, trusted while they are watched, until the watcher reports them */ |
| 249 | #trusted: Map<string, Input> | null = null; |
| 250 | /** the last build of a watch, which the next one reuses what it can from */ |
| 251 | #last: Run | null = null; |
| 252 | /** a build is under way, from before it waits for the lock */ |
| 253 | #building = false; |
| 254 | /** the watcher of `incrementalWatch`, which lazy steps watch their inputs with too */ |
| 255 | #watcher: Watcher | null = null; |
| 256 | #lazyBytes: number; |
| 257 | /** lazy steps that are running, by key, which calls for the same step wait for */ |
| 258 | #running = new Map<string, Promise<Result>>(); |
| 259 | /** thrown by `import.meta.hot.dispose` callbacks, reported by the next build */ |
| 260 | #disposeErrors: unknown[] = []; |
| 261 | #observers = new Set<(event: StepEvent) => void>(); |
| 262 | |
| 263 | constructor(options: GraphOptions = {}) { |
| 264 | if (typeof vm.SourceTextModule !== "function") { |
| 265 | throw new Error( |
| 266 | "@clo/lib/incremental requires Node.js to run with --experimental-vm-modules", |
| 267 | ); |
| 268 | } |
| 269 | this.fs = options.fs ?? Fs.default; |
| 270 | // real, like the paths modules resolve to, so step keys and stored urls are relative to it |
| 271 | this.root = realPath(this.fs, path.resolve(options.root ?? process.cwd())); |
| 272 | this.db = options.db instanceof Database |
| 273 | ? options.db |
| 274 | : new Database(options.db == null ? null : path.resolve(this.root, options.db), this.root); |
| 275 | for (const ext of Object.keys(options.loaders ?? {})) { |
| 276 | ASSERT(!Object.hasOwn(defaultLoaders, ext), () => `the ${ext} loader can't be replaced`); |
| 277 | } |
| 278 | this.#loaders = { ...defaultLoaders, ...options.loaders }; |
| 279 | this.#conditions = [...importConditions, ...(options.conditions ?? [])]; |
| 280 | this.#external = options.external ?? (() => false); |
| 281 | this.#sources = new Map([sqliteSource, ...options.sources ?? []].map((source) => [source.protocol, source])); |
| 282 | this.#lazyBytes = options.lazyBytes ?? 64 * 1024 * 1024; |
| 283 | const inline = Object.values(options.loaders ?? {}).some((loader) => !loaderModules.has(loader)); |
| 284 | this.#creator = inline ? frameUrl(stack.capture()[0]) : null; |
| 285 | } |
| 286 | |
| 287 | /** |
| 288 | * runs the provided build pipeline. the pipeline is an orchestration of |
| 289 | * {@linkcode Builder|steps}, async functions that read and process data. by |
| 290 | * keeping all actual reads in these step functions, an incremental re-run |
| 291 | * can quickly skip all of the actual logic. at the end, the pipeline should |
| 292 | * write its outputs with the output APIs in {@linkcode Ref}. a step can write |
| 293 | * files with {@linkcode Build#writeFile}, which a build that reuses the step |
| 294 | * keeps without making them again. |
| 295 | * |
| 296 | * when the pipeline returns, cache entries and outputs the build did not use |
| 297 | * are deleted. if any step has an error, cached or not, a |
| 298 | * {@linkcode BuildError} is thrown after that. |
| 299 | */ |
| 300 | async incrementalBuild<T>(options: BuildOptions<T>): Promise<T> { |
| 301 | const ref = options.progress ?? progress; |
| 302 | using _ = await this.#lock(ref, options.signal); |
| 303 | const run = this.#run = this.#newRun(options); |
| 304 | try { |
| 305 | // loaded code is checked by the build's own run, so a file that changes |
| 306 | // during the build can't make a step record code other than what it ran. |
| 307 | await this.#refresh(run); |
| 308 | const errors = this.#disposeErrors.splice(0) |
| 309 | .map((error) => run.entry(error, "import.meta.hot.dispose")); |
| 310 | let result: { value: T } | null = null; |
| 311 | try { |
| 312 | result = { value: await options.buildAndWriteOutput(new Ref(run, ref), []) }; |
| 313 | } catch (error) { |
| 314 | errors.push(run.entry(error, "buildAndWriteOutput")); |
| 315 | } |
| 316 | await run.settle(); |
| 317 | options.signal?.throwIfAborted(); |
| 318 | errors.unshift(...run.errors()); |
| 319 | if (result) { |
| 320 | const used = new Set<RowId>(); |
| 321 | for (const { id } of run.results.values()) if (id !== null) used.add(id); |
| 322 | const stale = this.db.collect(used, run.outputs); |
| 323 | for (const file of stale.outputs) await removeOutput(file, this.root); |
| 324 | run.unread = stale.inputs; |
| 325 | } |
| 326 | if (errors.length > 0) throw new BuildError(groupErrors(errors), result?.value); |
| 327 | return result!.value; |
| 328 | } finally { |
| 329 | run.close(); |
| 330 | } |
| 331 | } |
| 332 | |
| 333 | /** |
| 334 | * load a module without tracking it. inside a step, use |
| 335 | * {@linkcode Build#import} so the module's code becomes an input. |
| 336 | * |
| 337 | * `main` marks the entry point, setting `import.meta.main`. whoever imports |
| 338 | * it keeps running its code, so changing the entry point or anything it |
| 339 | * imports requires a restart. code that should reload is imported again |
| 340 | * every build, such as `graph.import("generate.ts")` in a watch loop. |
| 341 | */ |
| 342 | async import<T = unknown>(file: string, options: { main?: boolean } = {}): Promise<T> { |
| 343 | const run = await this.#importRun(); |
| 344 | const href = toUrl(file, this.root, this.fs); |
| 345 | if (options.main) this.#main = href; |
| 346 | try { |
| 347 | const { namespace } = await this[kImport](run, href); |
| 348 | if (options.main) this.#pin((await run.internal(closureKey(href))).value() as Module[]); |
| 349 | return namespace as T; |
| 350 | } finally { |
| 351 | // code loaded between the builds of a watch is watched too, even code that failed to load |
| 352 | if (run.closed) this.#watcher?.update((await run.checked()).values(), [], this.#sources); |
| 353 | } |
| 354 | } |
| 355 | |
| 356 | /** |
| 357 | * the build in progress, or else a run that checks loaded code, which |
| 358 | * starts no steps but keeps what the last build left |
| 359 | */ |
| 360 | async #importRun(): Promise<Run> { |
| 361 | if (this.#run && !this.#run.closed) return this.#run; |
| 362 | const run = this.#run = this.#newRun(); |
| 363 | run.closed = true; |
| 364 | await this.#refresh(run); |
| 365 | return run; |
| 366 | } |
| 367 | |
| 368 | /** |
| 369 | * {@linkcode incrementalBuild|builds}, then builds again whenever an input |
| 370 | * of the last build, or a loaded module, may have changed, yielding every |
| 371 | * result with the urls that changed. errors other than a |
| 372 | * {@linkcode BuildError}, such as {@linkcode RestartRequired}, are thrown. |
| 373 | * |
| 374 | * inputs that were watched for all of a build are trusted until the watcher |
| 375 | * reports them. a step that observed nothing else is reused as the last |
| 376 | * build left it, and so is an output, without looking at the database or |
| 377 | * the disk. a value the last build read is the same object in the next. a |
| 378 | * rebuild then costs what the pipeline costs to call, plus the steps that a |
| 379 | * change reaches. |
| 380 | * |
| 381 | * `files: false` is for a process whose code and content only change with a |
| 382 | * restart. the filesystem is not watched, a file is trusted once a build has |
| 383 | * read it, and only the sources that report changes cause another build. |
| 384 | */ |
| 385 | async *incrementalWatch<T>( |
| 386 | options: BuildOptions<T> & { debounce?: number; files?: boolean }, |
| 387 | ): AsyncGenerator<WatchResult<T>, void> { |
| 388 | using watcher = this.#watcher = new Watcher(options.debounce ?? 25, options.files ?? true, this.root); |
| 389 | const trusted = this.#trusted = new Map<string, Input>(); |
| 390 | try { |
| 391 | for (let changed: string[] = [];;) { |
| 392 | const cause = changed; |
| 393 | const start = performance.now(); |
| 394 | let outcome: { ok: true; value: T } | { ok: false; error: BuildError; value?: T }; |
| 395 | try { |
| 396 | const buildAndWriteOutput = (ref: Ref) => options.buildAndWriteOutput(ref, cause); |
| 397 | outcome = { ok: true, value: await this.incrementalBuild({ ...options, buildAndWriteOutput }) }; |
| 398 | } catch (error) { |
| 399 | if (!(error instanceof BuildError)) throw error; |
| 400 | outcome = { ok: false, error, value: error.value as T | undefined }; |
| 401 | } |
| 402 | const run = this.#last = UNWRAP(this.#run); |
| 403 | const stats: BuildStats = { |
| 404 | duration: performance.now() - start, |
| 405 | ran: run.ran, |
| 406 | reused: run.started - run.ran, |
| 407 | written: run.written, |
| 408 | cacheBytes: this.db.size(), |
| 409 | outputBytes: this.db.outputSize(), |
| 410 | }; |
| 411 | |
| 412 | const checked = await run.checked(); |
| 413 | this.#trust(checked); |
| 414 | const watching = watcher.update(checked.values(), run.unread, this.#sources); |
| 415 | for (const href of run.unread) trusted.delete(href); |
| 416 | |
| 417 | const count = (n: number, noun: string) => `${n} ${noun}${n === 1 ? "" : "s"}`; |
| 418 | const sources = Object.values(watching).reduce((sum, n) => sum + n, -watching.file); |
| 419 | const watched = [ |
| 420 | watching.file > 0 && count(watching.file, "file"), |
| 421 | sources > 0 && count(sources, watching.file > 0 ? "other source" : "source"), |
| 422 | ].filter(Boolean).join(" and "); |
| 423 | // the first build is the one that no change caused |
| 424 | const upToDate = cause.length === 0 && outcome.ok && stats.ran === 0 && stats.written === 0; |
| 425 | const statusLine = `${upToDate ? "Build is up to date. " : ""}Watching ${watched || "nothing"}`; |
| 426 | |
| 427 | yield { ...outcome, changed, stats, watching, statusLine }; |
| 428 | changed = await watcher.next(options.signal); |
| 429 | for (const href of changed) trusted.delete(href); |
| 430 | } |
| 431 | } finally { |
| 432 | this.#trusted = null; |
| 433 | this.#last = null; |
| 434 | this.#watcher = null; |
| 435 | } |
| 436 | } |
| 437 | |
| 438 | /** only what was watched for all of a run can't have changed unseen, so the rest is not trusted */ |
| 439 | #trust(checked: Map<string, Input>): void { |
| 440 | const trusted = UNWRAP(this.#trusted); |
| 441 | for (const [href, input] of checked) { |
| 442 | // a graph's options never change |
| 443 | if (!href.startsWith("graph:") && !this.#watcher?.covers(href)) continue; |
| 444 | // a probe only knows the kind, which contents that are still trusted say too |
| 445 | const probe = input.kind === "file" && kinds.has(input.hash); |
| 446 | if (!(probe && trusted.get(href)?.kind === "file")) trusted.set(href, input); |
| 447 | } |
| 448 | } |
| 449 | |
| 450 | /** |
| 451 | * run a step when something asks for it, instead of in a build, like a page |
| 452 | * that is rendered on its first request. it is reused like a step of a |
| 453 | * build, across builds and restarts, until something it read changes, or |
| 454 | * until the values of lazy steps take more than `lazyBytes` and it is the |
| 455 | * one used least recently. what throws is not kept, so the next call runs it |
| 456 | * again. calls while it runs wait for it. steps of a build that it reads are |
| 457 | * given to it as arguments. |
| 458 | */ |
| 459 | lazy<Args extends unknown[], Result>( |
| 460 | fn: Builder<Args, Result> & NoInfer<CheckSerializable<Result, "a step's result">>, |
| 461 | ...args: Args & NoInfer<ValidateSerializable<Args, Step<unknown>>> |
| 462 | ): Step<Result> { |
| 463 | const run = this.#newRun({ lazy: true }); |
| 464 | const reports = this.#watcher?.reports ?? 0; |
| 465 | const { key, label, compute } = begin(run, callerOf(Graph.prototype.lazy), fn, args); |
| 466 | const result = memo(this.#running, key, async () => { |
| 467 | try { |
| 468 | const result = await run.step(key, label, compute); |
| 469 | if (result.id !== null && result.error) { |
| 470 | // a failure can come from outside what the step read, like a browser that did not start |
| 471 | this.db.deleteWithReaders(result.id); |
| 472 | } else if (result.id !== null) { |
| 473 | this.db.markLazyUsed(result.id, performance.timeOrigin + performance.now()); |
| 474 | // only a value that was just made can take more bytes |
| 475 | if (!result.cached) this.db.evict(this.#lazyBytes); |
| 476 | } |
| 477 | if (this.#watcher) { |
| 478 | const checked = await run.checked(); |
| 479 | // what was reported changed while the step ran is checked again by the next one |
| 480 | for (const href of checked.keys()) if (this.#watcher.reportedSince(href, reports)) checked.delete(href); |
| 481 | this.#trust(checked); |
| 482 | this.#watcher.update(checked.values(), [], this.#sources); |
| 483 | } |
| 484 | return result; |
| 485 | } finally { |
| 486 | this.#running.delete(key); |
| 487 | } |
| 488 | }); |
| 489 | return new Step(key, result); |
| 490 | } |
| 491 | |
| 492 | /** hear what happens to every step, as builds and lazy steps go, until disposed */ |
| 493 | observe(observer: (event: StepEvent) => void): Disposable { |
| 494 | this.#observers.add(observer); |
| 495 | return { [Symbol.dispose]: () => void this.#observers.delete(observer) }; |
| 496 | } |
| 497 | |
| 498 | /** @internal */ |
| 499 | [kStepEvent](event: StepEvent): void { |
| 500 | for (const observer of this.#observers) observer(event); |
| 501 | } |
| 502 | |
| 503 | #newRun(options: { signal?: AbortSignal; rerunErrors?: boolean; lazy?: boolean } = {}): Run { |
| 504 | return new Run(this, { |
| 505 | sources: this.#sources, |
| 506 | trusted: this.#trusted, |
| 507 | last: this.#last, |
| 508 | signal: options.signal, |
| 509 | rerunErrors: options.rerunErrors ?? false, |
| 510 | lazy: options.lazy ?? false, |
| 511 | }); |
| 512 | } |
| 513 | |
| 514 | /** other processes using the database wait for this build to finish */ |
| 515 | async #lock(p: progress.Ref, signal?: AbortSignal): Promise<Disposable> { |
| 516 | // the database lets this process take its lock again, so this process keeps its own builds apart |
| 517 | ASSERT(!this.#building, "builds can not overlap"); |
| 518 | this.#building = true; |
| 519 | let node: progress.Node | null = null; |
| 520 | try { |
| 521 | while (!this.db.lock()) { |
| 522 | signal?.throwIfAborted(); |
| 523 | node ??= p.start(`waiting for process ${this.db.locked} to finish building`); |
| 524 | await async.delay(250); |
| 525 | } |
| 526 | } catch (error) { |
| 527 | this.#building = false; |
| 528 | throw error; |
| 529 | } finally { |
| 530 | node?.end(); |
| 531 | } |
| 532 | return { |
| 533 | [Symbol.dispose]: () => { |
| 534 | this.db.unlock(); |
| 535 | this.#building = false; |
| 536 | }, |
| 537 | }; |
| 538 | } |
| 539 | |
| 540 | /** discard module instances when the code they were made from changed */ |
| 541 | async #refresh(run: Run): Promise<void> { |
| 542 | const checking = this.#imported; |
| 543 | // loaders not from `lazyLoader` run the code that made the graph, which |
| 544 | // node.js loaded before any build, so it is pinned before a build reads it. |
| 545 | if (this.#creator && !this.#native.has(loadKey(this.#creator))) { |
| 546 | const closure = await run.internal(closureKey(this.#creator)); |
| 547 | if (!closure.error) this.#pin(closure.value() as Module[]); |
| 548 | } |
| 549 | const changed = async ([key, hash]: [string, string]) => (await run.internal(key)).hash !== hash; |
| 550 | const native = Array.from(this.#native); |
| 551 | const stale = await Promise.all(native.map(changed)); |
| 552 | const restart = native.filter((_, i) => stale[i]).map(([key]) => |
| 553 | url.fileURLToPath(key.slice(5).replace(/^glob:/, "")) |
| 554 | ); |
| 555 | if (restart.length > 0) throw new RestartRequired(restart); |
| 556 | const inputs = Array.from(this.#runtime.values(), (runtime) => Array.from(runtime.inputs.values())); |
| 557 | const reset = await Promise.all([ |
| 558 | ...Array.from(checking, changed), |
| 559 | ...inputs.flat().map(async (input) => !(await run.matches(input.url, input.hash))), |
| 560 | ]); |
| 561 | // unless an import during the check already had to |
| 562 | if (reset.some(Boolean) && checking === this.#imported) await this.#reset(); |
| 563 | } |
| 564 | |
| 565 | /** discard every module instance, so the next import evaluates fresh code */ |
| 566 | async #reset(): Promise<void> { |
| 567 | // before anything waits, so an import that also finds the code outdated meanwhile finds it discarded |
| 568 | const discarded = Array.from(this.#modules.values(), ({ instance }) => instance); |
| 569 | this.#modules.clear(); |
| 570 | this.#imported = new Map(); |
| 571 | this.#runtime.clear(); |
| 572 | this.#maps.clear(); |
| 573 | const { cache } = module.createRequire(import.meta.url); |
| 574 | const purge = (file: string) => { |
| 575 | const entry = cache[file]; |
| 576 | if (!entry || isNodeModules(file)) return; |
| 577 | delete cache[file]; |
| 578 | for (const child of entry.children) purge(child.filename); |
| 579 | }; |
| 580 | for (const file of this.#required) purge(file); |
| 581 | this.#required.clear(); |
| 582 | for (const instance of await Promise.allSettled(discarded)) { |
| 583 | if (instance.status === "rejected") continue; |
| 584 | for (const dispose of instance.value.dispose) { |
| 585 | try { |
| 586 | await dispose({}); |
| 587 | } catch (error) { |
| 588 | this.#disposeErrors.push(error); |
| 589 | } |
| 590 | } |
| 591 | } |
| 592 | } |
| 593 | |
| 594 | /** @internal instantiate and evaluate a module from the current generation */ |
| 595 | async [kImport](run: Run, href: string): Promise<vm.Module> { |
| 596 | const closure = await run.internal(closureKey(href)); |
| 597 | const modules = closure.value() as Module[]; |
| 598 | const nodes = new Map(modules.map((node) => [node.url, node])); |
| 599 | // instances can be older than the code this run read: a lazy run never |
| 600 | // checks them, and a build's check imports loaders before it is done |
| 601 | if (this.#outdated(modules)) await this.#reset(); |
| 602 | this.#imported.set(closureKey(href), closure.hash); |
| 603 | // creating an instance already runs the modules that node.js loads |
| 604 | const loading: Loading = { done: false, [kRun]: run }; |
| 605 | try { |
| 606 | return await running.run(loading, async () => { |
| 607 | // node.js links the modules it loads, so only imports of `node:vm` modules |
| 608 | // that are not linked yet are followed. |
| 609 | const instances = new Map<string, Instance>(); |
| 610 | for (let level = [href]; level.length > 0;) { |
| 611 | const created = await Promise.all( |
| 612 | level.map((dep) => this.#instance(run, UNWRAP(nodes.get(dep)))), |
| 613 | ); |
| 614 | level.forEach((dep, i) => instances.set(dep, created[i]!)); |
| 615 | level = Array.from( |
| 616 | new Set(level.flatMap((dep) => { |
| 617 | if (!UNWRAP(instances.get(dep)).unlinked) return []; |
| 618 | return UNWRAP(nodes.get(dep)).imports.filter((target) => !instances.has(target)); |
| 619 | })), |
| 620 | ); |
| 621 | } |
| 622 | for (const [dep, instance] of instances) { |
| 623 | if (!instance.unlinked) continue; |
| 624 | instance.unlinked = false; |
| 625 | (instance.module as vm.SourceTextModule).linkRequests( |
| 626 | UNWRAP(nodes.get(dep)).imports.map((target) => UNWRAP(instances.get(target)).module), |
| 627 | ); |
| 628 | } |
| 629 | const root = UNWRAP(instances.get(href)).module; |
| 630 | if (root.status === "unlinked") (root as vm.SourceTextModule).instantiate(); |
| 631 | await root.evaluate(); |
| 632 | return root; |
| 633 | }); |
| 634 | } finally { |
| 635 | loading.done = true; |
| 636 | } |
| 637 | } |
| 638 | |
| 639 | #instance(run: Run, node: Module): Promise<Instance> { |
| 640 | const existing = this.#modules.get(node.url); |
| 641 | if (existing) return existing.instance; |
| 642 | const instance = this.#instantiate(run, node); |
| 643 | this.#modules.set(node.url, { node, instance }); |
| 644 | // CommonJS runs as it loads, so one that threw runs again on the next import |
| 645 | instance.catch(() => { |
| 646 | if (this.#modules.get(node.url)?.instance === instance) this.#modules.delete(node.url); |
| 647 | }); |
| 648 | return instance; |
| 649 | } |
| 650 | |
| 651 | /** whether an instance runs other code than `modules` describe, or is linked to other imports */ |
| 652 | #outdated(modules: readonly Module[]): boolean { |
| 653 | return modules.some((node) => { |
| 654 | const made = this.#modules.get(node.url)?.node; |
| 655 | return made !== undefined && (made.hash !== node.hash || !util.isDeepStrictEqual(made.imports, node.imports)); |
| 656 | }); |
| 657 | } |
| 658 | |
| 659 | /** create a module in `node:vm`, or wrap the one node.js has */ |
| 660 | async #instantiate(run: Run, node: Module): Promise<Instance> { |
| 661 | const href = node.url; |
| 662 | if (node.format === "builtin") { |
| 663 | const builtin = UNWRAP( |
| 664 | process.getBuiltinModule(href), |
| 665 | () => `${href} is not a builtin module`, |
| 666 | ) as Record<string, unknown>; |
| 667 | return linked(href, { ...builtin, default: builtin }); |
| 668 | } |
| 669 | const file = url.fileURLToPath(href.replace(/^glob:/, "")); |
| 670 | // node.js can't load the code of a glob, which only the graph has |
| 671 | if ( |
| 672 | node.format === "external" |
| 673 | || (node.format === "js" && href.startsWith("file:") && (href === self || this.#external(file))) |
| 674 | ) { |
| 675 | // node.js loads what the module imports too, so none of it can reload |
| 676 | const closure = node.format === "external" ? null : await run.internal(closureKey(href)); |
| 677 | this.#pin(closure ? closure.value() as Module[] : [node]); |
| 678 | return linked(href, await import(href)); |
| 679 | } |
| 680 | this.#made.add(href); |
| 681 | if (node.format === "cjs") { |
| 682 | ASSERT(this.fs === Fs.default, () => `${file} is CommonJS, which only loads from disk`); |
| 683 | this.#required.add(file); |
| 684 | const require = module.createRequire(file); |
| 685 | // what a module requires is only known once it ran, so it is observed |
| 686 | // like `import.meta.resolve` |
| 687 | const { inputs } = this.#record(href); |
| 688 | const fs = this.#observing(inputs); |
| 689 | const before = new Set(Object.keys(require.cache)); |
| 690 | let exports: unknown; |
| 691 | try { |
| 692 | exports = require(file); |
| 693 | } catch (error) { |
| 694 | // node.js forgets a module that threw, but not what it required first |
| 695 | for (const loaded of Object.keys(require.cache)) { |
| 696 | if (before.has(loaded) || isNodeModules(loaded)) continue; |
| 697 | this.#required.add(loaded); |
| 698 | if (!inputs.has(url.pathToFileURL(loaded).href)) fs.readFileSync(loaded); |
| 699 | } |
| 700 | throw error; |
| 701 | } |
| 702 | const visit = (entry: NodeJS.Module | undefined) => { |
| 703 | for (const child of entry?.children ?? []) { |
| 704 | if (isNodeModules(child.filename) || inputs.has(url.pathToFileURL(child.filename).href)) continue; |
| 705 | fs.readFileSync(child.filename); |
| 706 | visit(child); |
| 707 | } |
| 708 | }; |
| 709 | visit(require.cache[file]); |
| 710 | return linked(href, { ...(isPlainObject(exports) ? exports : {}), default: exports }); |
| 711 | } |
| 712 | const load = await run.internal(loadKey(href)); |
| 713 | const loaded = load.value() as Loaded; |
| 714 | // the step's value is shared between runs, so each instance changes its own copy |
| 715 | if (loaded.format === "serializable") return linked(href, { default: structuredClone(loaded.value) }); |
| 716 | ASSERT(loaded.format === "js"); |
| 717 | if (loaded.map) this.#maps.set(href, loaded.map); |
| 718 | const dispose: Instance["dispose"] = []; |
| 719 | const options: vm.SourceTextModuleOptions = { |
| 720 | identifier: href, |
| 721 | initializeImportMeta: (meta) => this.#meta(meta, href, dispose), |
| 722 | importModuleDynamically: (specifier, _, attributes) => this.#importDynamic(href, specifier, attributes), |
| 723 | }; |
| 724 | const memory = this.#code.get(href); |
| 725 | const cached = memory?.hash === load.hash |
| 726 | ? memory.data |
| 727 | : load.id === null |
| 728 | ? null |
| 729 | : this.db.readCode(load.id); |
| 730 | if (cached) { |
| 731 | try { |
| 732 | const reused = new vm.SourceTextModule(loaded.code, { ...options, cachedData: cached }); |
| 733 | return { module: reused, unlinked: true, dispose }; |
| 734 | } catch (error) { |
| 735 | if (exception.code(error) !== "ERR_VM_MODULE_CACHED_DATA_REJECTED") throw error; |
| 736 | } |
| 737 | } |
| 738 | // v8 can only save compiled code before the module evaluates |
| 739 | const compiled = new vm.SourceTextModule(loaded.code, options) as Compiled; |
| 740 | const data = compiled.createCachedData(); |
| 741 | this.#code.set(href, { hash: load.hash, data }); |
| 742 | if (load.id !== null) this.db.writeCode(load.id, data); |
| 743 | return { module: compiled, unlinked: true, dispose }; |
| 744 | } |
| 745 | |
| 746 | #meta(meta: ImportMeta, href: string, dispose: Instance["dispose"]): void { |
| 747 | const file = url.fileURLToPath(href); |
| 748 | meta.url = href; |
| 749 | meta.filename = file; |
| 750 | meta.dirname = path.dirname(file); |
| 751 | (meta as { main: boolean }).main = href === this.#main; |
| 752 | meta.resolve = (specifier: string) => this.#resolveMeta(href, specifier); |
| 753 | // the dispose half of vite's `import.meta.hot`, see `onDispose` |
| 754 | (meta as { hot?: object }).hot = { |
| 755 | dispose: (fn: (data: object) => unknown) => void dispose.push(fn), |
| 756 | }; |
| 757 | } |
| 758 | |
| 759 | /** `import.meta.resolve`, which has to be synchronous */ |
| 760 | #resolveMeta(from: string, specifier: string): string { |
| 761 | // like node.js, paths and urls resolve without looking at the filesystem |
| 762 | if (pathSpecifier.test(specifier) || specifier.startsWith("file:")) return new URL(specifier, from).href; |
| 763 | const observed = new Map<string, Input>(); |
| 764 | try { |
| 765 | return this.#resolveWith(observed, path.dirname(url.fileURLToPath(from)), specifier); |
| 766 | } finally { |
| 767 | // the module may keep the result in its state |
| 768 | const { inputs } = this.#record(from); |
| 769 | for (const input of observed.values()) inputs.set(input.url, input); |
| 770 | } |
| 771 | } |
| 772 | |
| 773 | async #importDynamic( |
| 774 | from: string, |
| 775 | specifier: string, |
| 776 | attributes: Attributes, |
| 777 | ): Promise<vm.Module> { |
| 778 | const owner = running.getStore(); |
| 779 | // a step imports with its own run, and a module's top level with the run loading it |
| 780 | const live = owner instanceof Build ? Build.active : owner && !owner.done ? owner : undefined; |
| 781 | const run = live?.[kRun] ?? await this.#importRun(); |
| 782 | const key = resolveKey(path.dirname(url.fileURLToPath(from)), specifier); |
| 783 | const resolved = await run.internal(key); |
| 784 | // like a static import, what `import()` loads becomes part of the module |
| 785 | this.#imported.set(key, resolved.hash); |
| 786 | const { steps } = this.#record(from); |
| 787 | steps.add(key); |
| 788 | const target = resolved.value() as string; |
| 789 | checkAttributes(target, { ...attributes }); |
| 790 | steps.add(closureKey(target)); |
| 791 | return this[kImport](run, target); |
| 792 | } |
| 793 | |
| 794 | #record(href: string): Runtime { |
| 795 | return memo(this.#runtime, href, () => ({ inputs: new Map(), steps: new Set() })); |
| 796 | } |
| 797 | |
| 798 | /** @internal the steps that load modules, which run from their key alone */ |
| 799 | [kInternal](run: Run, key: string): Promise<Result> { |
| 800 | const colon = key.indexOf(":"); |
| 801 | const kind = key.slice(0, colon); |
| 802 | const arg = key.slice(colon + 1); |
| 803 | const label = `${kind} ${ |
| 804 | arg.replace(/file:\/\/\S*/, (href) => path.relative(this.root, url.fileURLToPath(href)) || ".") |
| 805 | }`; |
| 806 | switch (kind) { |
| 807 | case "load": |
| 808 | return run.step(key, label, (b) => this.#load(b, arg)); |
| 809 | case "closure": |
| 810 | return run.step(key, label, (b) => this.#closure(b, arg)); |
| 811 | case "package": |
| 812 | return run.step(key, label, (b) => this.#package(b, url.fileURLToPath(arg))); |
| 813 | case "resolve": { |
| 814 | const space = arg.indexOf(" "); |
| 815 | const dir = url.fileURLToPath(arg.slice(0, space)); |
| 816 | return run.step(key, label, (b) => this.#resolve(b, dir, arg.slice(space + 1))); |
| 817 | } |
| 818 | } |
| 819 | throw new Error(`unknown internal step: ${key}`); |
| 820 | } |
| 821 | |
| 822 | /** read and compile a module, and find what it imports */ |
| 823 | async #load(b: Build, href: string): Promise<Loaded> { |
| 824 | if (href.startsWith("glob:")) { |
| 825 | const glob = new URL(href.slice(5)); |
| 826 | return await this.#glob(b, url.fileURLToPath(glob), Number(glob.hash.slice(1))); |
| 827 | } |
| 828 | const file = url.fileURLToPath(href); |
| 829 | const ext = path.extname(file); |
| 830 | try { |
| 831 | if (ext !== ".json" && isNodeModules(file)) { |
| 832 | // packages are loaded by node.js. the version stands in for code that |
| 833 | // isn't traced, like every file the package imports itself. |
| 834 | const pkg = await b.readFile(packageJsonOf(file)).json<{ version?: unknown }>(); |
| 835 | return { |
| 836 | format: "external", |
| 837 | source: await b.readFile(file).hash(), |
| 838 | version: String(pkg.version ?? ""), |
| 839 | data: null, |
| 840 | }; |
| 841 | } |
| 842 | const loader = this.#loaders[ext]; |
| 843 | if (!Object.hasOwn(defaultLoaders, ext)) { |
| 844 | // graphs with another loader for this extension load it differently |
| 845 | b[kObserve](this[kOption](`graph:loader/${encodeURIComponent(ext)}`)); |
| 846 | if (loader && !loaderModules.has(loader) && this.#creator) { |
| 847 | // the loader is code node.js loaded, so it can't reload |
| 848 | const closure = await b[kDepend](closureKey(this.#creator)); |
| 849 | if (!closure.error) this.#pin(closure.value() as Module[]); |
| 850 | } |
| 851 | } |
| 852 | if (!loader) { |
| 853 | throw Object.assign( |
| 854 | new Error(`no loader registered for ${JSON.stringify(ext)} files`), |
| 855 | { code: "ERR_UNKNOWN_FILE_EXTENSION" }, |
| 856 | ); |
| 857 | } |
| 858 | const loaded = await loader({ |
| 859 | file, |
| 860 | source: await b.readFile(file).text(), |
| 861 | build: b, |
| 862 | getPackageMeta: async () => (await b[kDepend](packageKey(path.dirname(file)))).value() as PackageMeta | null, |
| 863 | }); |
| 864 | const data = loaded.data ?? null; |
| 865 | if (loaded.format === "serializable") { |
| 866 | return { format: "serializable", value: loaded.content, data }; |
| 867 | } |
| 868 | if (loaded.format === "cjs") { |
| 869 | return { format: "cjs", source: await b.readFile(file).hash(), data }; |
| 870 | } |
| 871 | const code = loaded.format === "ts" ? stripTypes(loaded.content) : loaded.content; |
| 872 | const { moduleRequests } = new vm.SourceTextModule(code, { identifier: href }); |
| 873 | return { |
| 874 | format: "js", |
| 875 | code, |
| 876 | requests: moduleRequests.map(({ specifier, attributes }) => ({ |
| 877 | specifier, |
| 878 | attributes: { ...attributes }, |
| 879 | })), |
| 880 | map: loaded.map === undefined |
| 881 | ? inlineSourceMap(code) |
| 882 | : typeof loaded.map === "string" |
| 883 | ? loaded.map |
| 884 | : JSON.stringify(loaded.map), |
| 885 | data, |
| 886 | }; |
| 887 | } catch (error) { |
| 888 | exception.rethrowWithMetadata(error, { file }); |
| 889 | } |
| 890 | } |
| 891 | |
| 892 | /** |
| 893 | * the module of a `glob:` specifier, like `glob:./posts/*.md`, whose default |
| 894 | * export has the namespace of every file the pattern matches, sorted, by its |
| 895 | * path from where the wildcards start. the directories it lists are inputs, |
| 896 | * so a file that starts or stops matching changes its code. `depth` counts |
| 897 | * the segments of `pattern` from where the specifier's wildcards start. |
| 898 | */ |
| 899 | async #glob(b: Build, pattern: string, depth: number): Promise<Loaded> { |
| 900 | const segments = pattern.split(path.sep); |
| 901 | const start = segments.length - depth; |
| 902 | if (segments.at(-1) === "**") segments.push("*"); |
| 903 | const base = segments.slice(0, start).join(path.sep) || path.sep; |
| 904 | const files = new Set<string>(); |
| 905 | const walk = async (dir: string, segments: string[]): Promise<void> => { |
| 906 | const [segment, ...rest] = segments; |
| 907 | if (segment === undefined) return; |
| 908 | let entries: string[]; |
| 909 | try { |
| 910 | entries = await b.readDir(dir); |
| 911 | } catch (error) { |
| 912 | // the listing is an input either way, so a directory made later is seen |
| 913 | if (isMissing(error)) return; |
| 914 | throw error; |
| 915 | } |
| 916 | await Promise.all(entries.map(async (entry) => { |
| 917 | const directory = entry.endsWith("/"); |
| 918 | const name = directory ? entry.slice(0, -1) : entry; |
| 919 | if (segment === "**") { |
| 920 | if (directory && !name.startsWith(".")) await walk(path.join(dir, name), segments); |
| 921 | } else if (path.matchesGlob(name, segment)) { |
| 922 | if (rest.length === 0 && !directory) files.add(path.join(dir, name)); |
| 923 | if (rest.length > 0 && directory) await walk(path.join(dir, name), rest); |
| 924 | } |
| 925 | })); |
| 926 | // `**` also matches no directories at all |
| 927 | if (segment === "**") await walk(dir, rest); |
| 928 | }; |
| 929 | await walk(base, segments.slice(start)); |
| 930 | |
| 931 | // the imports are relative to the pattern's directory, like `#closure` |
| 932 | // resolves them, so the code still fits once the project has moved |
| 933 | const requests = Array.from(files).sort().map((file) => { |
| 934 | const relative = path.relative(path.dirname(pattern), file).split(path.sep).map(encodeURIComponent).join("/"); |
| 935 | return { |
| 936 | specifier: relative.startsWith("../") ? relative : `./${relative}`, |
| 937 | attributes: file.endsWith(".json") ? { type: "json" } : {}, |
| 938 | key: path.relative(base, file).replaceAll(path.sep, "/"), |
| 939 | }; |
| 940 | }); |
| 941 | const code = requests.map(({ specifier, attributes }, i) => { |
| 942 | const json = attributes.type ? ` with { type: "json" }` : ""; |
| 943 | return `import * as m${i} from ${JSON.stringify(specifier)}${json};\n`; |
| 944 | }).join("") + `export default { ${requests.map(({ key }, i) => `${JSON.stringify(key)}: m${i}`).join(", ")} };\n`; |
| 945 | return { |
| 946 | format: "js", |
| 947 | code, |
| 948 | requests: requests.map(({ specifier, attributes }) => ({ specifier, attributes })), |
| 949 | map: null, |
| 950 | data: null, |
| 951 | }; |
| 952 | } |
| 953 | |
| 954 | /** walk the static imports of a module, depending on each module's code */ |
| 955 | async #closure(b: Build, root: string): Promise<Module[]> { |
| 956 | const visit = async (href: string): Promise<Module> => { |
| 957 | if (href.startsWith("node:")) { |
| 958 | return { url: href, format: "builtin", imports: [], hash: "", data: null }; |
| 959 | } |
| 960 | const load = await b[kDepend](loadKey(href)); |
| 961 | const loaded = load.value() as Loaded; |
| 962 | // a glob's imports are relative to its pattern |
| 963 | const file = url.fileURLToPath(href.replace(/^glob:/, "")); |
| 964 | const imports = loaded.format !== "js" ? [] : await Promise.all( |
| 965 | loaded.requests.map(async ({ specifier, attributes }) => { |
| 966 | try { |
| 967 | const resolved = await b[kDepend](resolveKey(path.dirname(file), specifier)); |
| 968 | const target = resolved.value() as string; |
| 969 | checkAttributes(target, attributes); |
| 970 | return target; |
| 971 | } catch (error) { |
| 972 | exception.rethrowWithMetadata(error, { file, specifier }); |
| 973 | } |
| 974 | }), |
| 975 | ); |
| 976 | return { url: href, format: loaded.format, imports, hash: load.hash, data: loaded.data }; |
| 977 | }; |
| 978 | const modules: Module[] = []; |
| 979 | const seen = new Set([root]); |
| 980 | for (let level = [root]; level.length > 0;) { |
| 981 | const visited = await Promise.all(level.map(visit)); |
| 982 | modules.push(...visited); |
| 983 | level = []; |
| 984 | for (const dep of visited.flatMap((node) => node.imports)) { |
| 985 | if (seen.has(dep)) continue; |
| 986 | seen.add(dep); |
| 987 | level.push(dep); |
| 988 | } |
| 989 | } |
| 990 | return modules; |
| 991 | } |
| 992 | |
| 993 | #resolve(b: Build, dir: string, specifier: string): string { |
| 994 | const observed = new Map<string, Input>(); |
| 995 | try { |
| 996 | return this.#resolveWith(observed, dir, specifier); |
| 997 | } finally { |
| 998 | for (const input of observed.values()) b[kObserve](input); |
| 999 | } |
| 1000 | } |
| 1001 | |
| 1002 | /** the nearest `package.json`, which decides the format of `.js` files */ |
| 1003 | #package(b: Build, dir: string): PackageMeta | null { |
| 1004 | const observed = new Map<string, Input>(); |
| 1005 | try { |
| 1006 | const resolver = new Resolver(this.#observing(observed), this.#conditions); |
| 1007 | const file = resolver.findClosestPackageJson(dir); |
| 1008 | return file ? resolver.readPackageJsonMeta(file) : null; |
| 1009 | } finally { |
| 1010 | for (const input of observed.values()) b[kObserve](input); |
| 1011 | } |
| 1012 | } |
| 1013 | |
| 1014 | #resolveWith(observed: Map<string, Input>, dir: string, specifier: string): string { |
| 1015 | if (specifier.startsWith("glob:")) { |
| 1016 | // the fragment counts the segments from the specifier's wildcards, so the directories above can have any name |
| 1017 | const pattern = specifier.slice(5); |
| 1018 | const segments = pattern.split("/"); |
| 1019 | const wild = segments.findIndex((segment) => /[*?[{]/.test(segment)); |
| 1020 | const depth = wild === -1 ? 1 : segments.length - wild; |
| 1021 | return `glob:${url.pathToFileURL(path.resolve(dir, pattern)).href}#${depth}`; |
| 1022 | } |
| 1023 | observed.set("graph:conditions", this[kOption]("graph:conditions")); |
| 1024 | const resolver = new Resolver(this.#observing(observed), this.#conditions); |
| 1025 | const { namespace, subPath } = resolver.resolve(specifier, dir, false); |
| 1026 | return namespace === "node" ? `node:${subPath}` : url.pathToFileURL(subPath).href; |
| 1027 | } |
| 1028 | |
| 1029 | /** |
| 1030 | * a filesystem for the resolver that records what it saw. probes only care |
| 1031 | * if a path exists, so they observe a kind, while reads observe contents. |
| 1032 | */ |
| 1033 | #observing(observed: Map<string, Input>): ResolverFs { |
| 1034 | const fs = this.fs; |
| 1035 | const stat = (file: string): Stat | null => { |
| 1036 | try { |
| 1037 | return fs.statSync(file); |
| 1038 | } catch (error) { |
| 1039 | if (isMissing(error)) return null; |
| 1040 | throw error; |
| 1041 | } |
| 1042 | }; |
| 1043 | return { |
| 1044 | statSync(file) { |
| 1045 | const found = stat(file); |
| 1046 | const href = url.pathToFileURL(file).href; |
| 1047 | if (!observed.has(href)) { |
| 1048 | observed.set(href, found ? { url: href, ...found, hash: found.kind } : record(href, "missing")); |
| 1049 | } |
| 1050 | if (!found) { |
| 1051 | throw Object.assign(new Error(`ENOENT: no such file or directory, stat '${file}'`), { |
| 1052 | code: "ENOENT", |
| 1053 | }); |
| 1054 | } |
| 1055 | return found; |
| 1056 | }, |
| 1057 | readFileSync(file) { |
| 1058 | const found = UNWRAP(stat(file)); |
| 1059 | const bytes = fs.readFileSync(file); |
| 1060 | const href = url.pathToFileURL(file).href; |
| 1061 | observed.set(href, { url: href, ...found, hash: sha256(bytes) }); |
| 1062 | return bytes; |
| 1063 | }, |
| 1064 | realPathSync: (file) => fs.realPathSync(file), |
| 1065 | }; |
| 1066 | } |
| 1067 | |
| 1068 | /** @internal what a module observed while running */ |
| 1069 | [kRuntime](href: string): Runtime | undefined { |
| 1070 | return this.#runtime.get(href); |
| 1071 | } |
| 1072 | |
| 1073 | /** |
| 1074 | * @internal options change what the steps that load modules return, so they |
| 1075 | * are inputs of those steps, and graphs with other options don't reuse them. |
| 1076 | */ |
| 1077 | [kOption](href: string): Input { |
| 1078 | const ext = href.startsWith("graph:loader/") ? decodeURIComponent(href.slice(13)) : null; |
| 1079 | const loader = ext === null ? undefined : this.#loaders[ext]; |
| 1080 | const module = loader && loaderModules.get(loader); |
| 1081 | const value = ext === null |
| 1082 | ? this.#conditions |
| 1083 | : loader && [module ? path.relative(this.root, url.fileURLToPath(module)) : null, loader.toString()]; |
| 1084 | return { url: href, kind: "graph", time: 0, size: 0, hash: hashSerializable(value ?? null) }; |
| 1085 | } |
| 1086 | |
| 1087 | /** @internal code node.js loaded can't reload, so changing it needs a restart */ |
| 1088 | [kNative](code: string, modules: readonly Module[]): void { |
| 1089 | if (!this.#made.has(code)) this.#pin(modules); |
| 1090 | } |
| 1091 | |
| 1092 | /** |
| 1093 | * @internal whether code the graph made is older than `modules`, as it is |
| 1094 | * once the graph discarded it, or before the graph has made it again |
| 1095 | */ |
| 1096 | [kStale](code: string, modules: readonly Module[]): boolean { |
| 1097 | if (!this.#made.has(code) || this.#native.has(loadKey(code))) return false; |
| 1098 | return !this.#modules.has(code) || this.#outdated(modules); |
| 1099 | } |
| 1100 | |
| 1101 | /** code that keeps running after it changes, which requires a restart */ |
| 1102 | #pin(modules: readonly Module[]): void { |
| 1103 | for (const node of modules) { |
| 1104 | if (node.format === "builtin" || this.#native.has(loadKey(node.url))) continue; |
| 1105 | this.#native.set(loadKey(node.url), node.hash); |
| 1106 | } |
| 1107 | } |
| 1108 | |
| 1109 | /** @internal point a stack frame at the source a loader compiled */ |
| 1110 | [kMapFrame](frame: stack.Frame): stack.Frame { |
| 1111 | const file = frame.file; |
| 1112 | let map = file ? this.#maps.get(file) : undefined; |
| 1113 | if (!file || !map || !frame.line || !frame.col) return frame; |
| 1114 | if (typeof map === "string") this.#maps.set(file, map = new module.SourceMap(JSON.parse(map))); |
| 1115 | const entry = map.findEntry(frame.line - 1, frame.col - 1); |
| 1116 | if (!("originalSource" in entry) || !entry.originalSource) return frame; |
| 1117 | return { |
| 1118 | fn: frame.fn, |
| 1119 | file: new URL(entry.originalSource, file).href, |
| 1120 | line: entry.originalLine + 1, |
| 1121 | col: entry.originalColumn + 1, |
| 1122 | }; |
| 1123 | } |
| 1124 | } |
| 1125 | |
| 1126 | /** `@types/node` is missing this method of `vm.SourceTextModule` */ |
| 1127 | type Compiled = vm.SourceTextModule & { createCachedData(): Uint8Array }; |
| 1128 | |
| 1129 | /** what a module observed while running, which belongs to every step that depends on it */ |
| 1130 | interface Runtime { |
| 1131 | /** from `import.meta.resolve` and `require` */ |
| 1132 | inputs: Map<string, Input>; |
| 1133 | /** the steps that resolved and loaded its `import()`s */ |
| 1134 | steps: Set<string>; |
| 1135 | } |
| 1136 | |
| 1137 | /** a module created by a {@linkcode Graph} */ |
| 1138 | interface Instance { |
| 1139 | module: vm.Module; |
| 1140 | /** `linkRequests` can only be called once, and `status` does not tell */ |
| 1141 | unlinked: boolean; |
| 1142 | /** from `import.meta.hot.dispose`, called when the generation is discarded */ |
| 1143 | dispose: Array<(data: object) => unknown>; |
| 1144 | } |
| 1145 | |
| 1146 | function linked(href: string, namespace: Record<string, unknown>): Instance { |
| 1147 | return { module: synthetic(href, namespace), unlinked: false, dispose: [] }; |
| 1148 | } |
| 1149 | |
| 1150 | /** wrap values from outside of `node:vm` as a module */ |
| 1151 | function synthetic(identifier: string, namespace: Record<string, unknown>): vm.SyntheticModule { |
| 1152 | const names = Object.keys(namespace); |
| 1153 | return new vm.SyntheticModule(names, function() { |
| 1154 | for (const name of names) this.setExport(name, namespace[name]); |
| 1155 | }, { identifier }); |
| 1156 | } |
| 1157 | |
| 1158 | /** |
| 1159 | * the top level of a build, which starts {@linkcode Step}s and writes outputs. |
| 1160 | * starting a progress node returns a `Ref` for it, so instrumenting a pipeline |
| 1161 | * is free. |
| 1162 | */ |
| 1163 | export class Ref implements progress.Ref { |
| 1164 | /** aborted when the build is */ |
| 1165 | signal: AbortSignal; |
| 1166 | #run: Run; |
| 1167 | #progress: progress.Ref; |
| 1168 | |
| 1169 | /** @internal */ |
| 1170 | constructor(run: Run, progress: progress.Ref) { |
| 1171 | this.signal = run.signal ?? neverAborted; |
| 1172 | this.#run = run; |
| 1173 | this.#progress = progress; |
| 1174 | } |
| 1175 | |
| 1176 | /** |
| 1177 | * creates a new trackable unit of work as a child of this one. |
| 1178 | * when given an estimate, a progress bar is rendered. |
| 1179 | */ |
| 1180 | start = (text: string, opts?: progress.StartOptions): ProgressNode => { |
| 1181 | const node = this.#progress.start(text, opts); |
| 1182 | const ref = new Ref(this.#run, { start: node.start.bind(node) }); |
| 1183 | return Object.assign(ts.mixin(node, ref), { start: ref.start, signal: ref.signal }); |
| 1184 | }; |
| 1185 | |
| 1186 | /** |
| 1187 | * run a deterministic "step function" and cache the result. the cache entry |
| 1188 | * is busted if the given `args` or any referenced {@linkcode Input}s are |
| 1189 | * changed. inside the step, the {@linkcode Build} provides access to the |
| 1190 | * filesystem, other steps, modules, and custom sources. the key can contain |
| 1191 | * unresolved {@linkcode Step} objects to easily tie the steps together, or |
| 1192 | * the steps can be linked via {@linkcode Build#readStep|readStep}. |
| 1193 | * |
| 1194 | * on a cache hit, every {@linkcode Input} the step read is checked again. a |
| 1195 | * step that re-runs but returns an identical value does not cause the steps |
| 1196 | * that read it to re-run. |
| 1197 | * |
| 1198 | * implicitly, the code of the file calling `step`, and every module it |
| 1199 | * imports, is an input, so the implementation of the builder is tracked |
| 1200 | * incrementally. a call site that starts the same step twice in one build is |
| 1201 | * an error, since it would run the same code twice. |
| 1202 | * |
| 1203 | * NOTE: the provided callback MAY capture arbitrary values that are not a part |
| 1204 | * of `key`. this is by design to bypass the cache key when needed. the step's |
| 1205 | * logic MUST be deterministic, and future debugging tools reserve the right to |
| 1206 | * arbitrarily re-call the step function multiple times per build, or even |
| 1207 | * after a build has completed. |
| 1208 | */ |
| 1209 | step<Args extends unknown[], Result>( |
| 1210 | fn: Builder<Args, Result> & NoInfer<CheckSerializable<Result, "a step's result">>, |
| 1211 | // a step is hashed by its key, so it can be passed before it resolves |
| 1212 | ...args: Args & NoInfer<ValidateSerializable<Args, Step<unknown>>> |
| 1213 | ): Step<Result> { |
| 1214 | const run = this.#run; |
| 1215 | ASSERT(!run.closed, "cannot start a step after the build has closed"); |
| 1216 | run.signal?.throwIfAborted(); |
| 1217 | const { key, label, compute } = begin(run, callerOf(Ref.prototype.step), fn, args); |
| 1218 | if (run.steps.has(key)) { |
| 1219 | throw new Error( |
| 1220 | `${label} was started twice with the same arguments, which runs its code twice. ` |
| 1221 | + `pass what differs between the calls as an argument`, |
| 1222 | ); |
| 1223 | } |
| 1224 | return new Step(key, run.step(key, label, compute, this.#progress)); |
| 1225 | } |
| 1226 | |
| 1227 | /** |
| 1228 | * write a file, skipped when the file on disk is what the last build wrote. |
| 1229 | * files that a build stops writing are deleted after it finishes. |
| 1230 | */ |
| 1231 | writeFile(file: string, contents: string | Uint8Array | Step<string | Uint8Array>): Promise<void> { |
| 1232 | return this.#run.output(file, async () => { |
| 1233 | if (!(contents instanceof Step)) { |
| 1234 | return { hash: hashSerializable(contents), write: (temp) => fs.writeFile(temp, contents) }; |
| 1235 | } |
| 1236 | const result = await contents[kState].result; |
| 1237 | return { |
| 1238 | hash: result.hash, |
| 1239 | write: (temp) => fs.writeFile(temp, result.value() as string | Uint8Array), |
| 1240 | }; |
| 1241 | }); |
| 1242 | } |
| 1243 | |
| 1244 | /** copy a file, skipped when neither side changed since the last build */ |
| 1245 | copyFile(file: string, source: string): Promise<void> { |
| 1246 | const run = this.#run; |
| 1247 | const from = path.resolve(run.graph.root, source); |
| 1248 | const href = url.pathToFileURL(from).href; |
| 1249 | return run.output(file, async () => { |
| 1250 | const input = await run.input(href); |
| 1251 | // checking the file may have kept its bytes for a step, which a copy never reads |
| 1252 | run.takeContents(from); |
| 1253 | if (input.kind !== "file") { |
| 1254 | throw Object.assign(new Error(`cannot copy ${from}, which is ${input.kind}`), { |
| 1255 | code: "ENOENT", |
| 1256 | }); |
| 1257 | } |
| 1258 | return { hash: input.hash, source: href, write: (temp) => fs.copyFile(from, temp) }; |
| 1259 | }); |
| 1260 | } |
| 1261 | } |
| 1262 | |
| 1263 | export type ProgressNode = progress.Node & Ref; |
| 1264 | |
| 1265 | export type Builder<Args extends unknown[] = unknown[], Result = Serializable> = ( |
| 1266 | b: Build, |
| 1267 | ...args: Args |
| 1268 | ) => ts.Awaitable<Result>; |
| 1269 | |
| 1270 | /** a step's value, which may still be computing. see {@linkcode Ref#step} */ |
| 1271 | export class Step<T> { |
| 1272 | /** @internal */ |
| 1273 | [kState]: { key: string; result: Promise<Result> }; |
| 1274 | |
| 1275 | /** @internal */ |
| 1276 | constructor(key: string, result: Promise<Result>) { |
| 1277 | this[kState] = { key, result }; |
| 1278 | } |
| 1279 | |
| 1280 | /** |
| 1281 | * reads the step's value. when called in a step, an implicit dependency is |
| 1282 | * assigned. the top level of a module can't read a step. |
| 1283 | */ |
| 1284 | async read(): Promise<ts.DeepReadonly<T>> { |
| 1285 | const build = Build.active; |
| 1286 | if (build) return build.readStep(this); |
| 1287 | return (await this[kState].result).value() as ts.DeepReadonly<T>; |
| 1288 | } |
| 1289 | |
| 1290 | /** |
| 1291 | * the hash of the step's value, or of its error, once it is done. equal |
| 1292 | * values hash the same, so it can stand for the value, like in an etag. a |
| 1293 | * step reads another step's value instead, which tracks it. |
| 1294 | */ |
| 1295 | async hash(): Promise<string> { |
| 1296 | ASSERT(!Build.active, "a step can't read another step's hash, only its value"); |
| 1297 | return (await this[kState].result).hash; |
| 1298 | } |
| 1299 | } |
| 1300 | |
| 1301 | /** |
| 1302 | * thrown by {@linkcode Graph#incrementalBuild} when a build has errors, cached |
| 1303 | * or not, thrown or reported with {@linkcode Build#warn}. errors thrown from |
| 1304 | * the same place are grouped into a nested `AggregateError`, and every error |
| 1305 | * lists the `steps` it came from, if it was `fatal` (thrown, rather than a |
| 1306 | * warning), and if it was `cached`. |
| 1307 | */ |
| 1308 | export class BuildError extends AggregateError { |
| 1309 | /** what `buildAndWriteOutput` returned, unless it threw */ |
| 1310 | declare value: unknown; |
| 1311 | |
| 1312 | constructor(errors: Error[], value: unknown) { |
| 1313 | const entries = errors.flatMap((group) => "steps" in group ? [group] : (group as AggregateError).errors as Error[]); |
| 1314 | const thrown = entries.filter((error) => (error as BuildEntry).fatal).length; |
| 1315 | const counts = [[thrown, "error"], [entries.length - thrown, "warning"]] as const; |
| 1316 | const said = counts.filter(([n]) => n > 0).map(([n, noun]) => `${n} ${noun}${n === 1 ? "" : "s"}`); |
| 1317 | super(errors, `build has ${said.join(" and ")}`); |
| 1318 | this.name = "BuildError"; |
| 1319 | // not enumerable, so printing the error doesn't print the whole site |
| 1320 | Object.defineProperty(this, "value", { value }); |
| 1321 | } |
| 1322 | |
| 1323 | /** @internal each error, said with the steps it came from */ |
| 1324 | [stack.formatSymbol](colors: boolean): string { |
| 1325 | const diagnostics = diagnose(this, undefined); |
| 1326 | const tone = diagnostics.some((diagnostic) => diagnostic.severity === "error") ? ansi.fgRed : ansi.fgYellow; |
| 1327 | const printed = diagnostics.map((diagnostic) => stack.formatDiagnostic(diagnostic, colors)).join("\n"); |
| 1328 | return printed + "\n" + (colors ? ansi.style(tone, this.message) : this.message) + "\n"; |
| 1329 | } |
| 1330 | } |
| 1331 | |
| 1332 | /** an error of a {@linkcode BuildError}, which says where it came from */ |
| 1333 | type BuildEntry = Error & { steps: string[]; fatal: boolean; cached: boolean }; |
| 1334 | |
| 1335 | /** |
| 1336 | * each error of a build laid out like `log/stack` prints it, saying the step |
| 1337 | * it was thrown in, and the errors thrown from the same place as one |
| 1338 | */ |
| 1339 | function diagnose(build: BuildError, root: string | undefined): stack.Diagnostic[] { |
| 1340 | const from = ({ steps, fatal, cached }: BuildEntry) => { |
| 1341 | // the pipeline only throws again what a step threw |
| 1342 | const own = steps.length > 1 ? steps.filter((step) => step !== "buildAndWriteOutput") : steps; |
| 1343 | // a step is named by its function, as where it was started is not in its trace |
| 1344 | const named = own[0]!.startsWith("step at ") ? `the ${own[0]}` : own[0]!.replace(/ at \S+$/, ""); |
| 1345 | const more = own.length > 1 ? ` and ${own.length - 1} other step${own.length === 2 ? "" : "s"}` : ""; |
| 1346 | return `${fatal ? "thrown" : "warned"} in ${named}${more}${cached ? " (cached)" : ""}`; |
| 1347 | }; |
| 1348 | const diagnostics = (build.errors as Error[]).map((group): stack.Diagnostic => { |
| 1349 | const [first, ...similar] = ("steps" in group ? [group] : (group as AggregateError).errors) as BuildEntry[]; |
| 1350 | return { |
| 1351 | ...traced(stack.describe(first, { severity: first!.fatal ? "error" : "warning", root }), from(first!)), |
| 1352 | similar: similar.map((error) => ({ message: error.message, from: from(error) })), |
| 1353 | }; |
| 1354 | }); |
| 1355 | // what stopped the build comes before what only warns |
| 1356 | return diagnostics.sort((a, b) => Number(b.severity === "error") - Number(a.severity === "error")); |
| 1357 | } |
| 1358 | |
| 1359 | /** a diagnostic whose stack is one note, saying where it was thrown */ |
| 1360 | function traced(diagnostic: stack.Diagnostic, text: string): stack.Diagnostic { |
| 1361 | const notes = diagnostic.notes.filter((note) => note.frames.length === 0); |
| 1362 | const frames = diagnostic.notes.flatMap((note) => note.frames); |
| 1363 | return { ...diagnostic, notes: [...notes, { text, location: null, frames }] }; |
| 1364 | } |
| 1365 | |
| 1366 | /** |
| 1367 | * code that node.js loaded changed, which only a new process can run. see |
| 1368 | * {@linkcode supervise}. |
| 1369 | */ |
| 1370 | export class RestartRequired extends Error { |
| 1371 | /** the code that changed */ |
| 1372 | files: string[]; |
| 1373 | |
| 1374 | constructor(files: string[]) { |
| 1375 | super(`restart to load changes to code that can't reload while it runs: ${files.join(", ")}`); |
| 1376 | this.name = "RestartRequired"; |
| 1377 | this.files = files; |
| 1378 | } |
| 1379 | } |
| 1380 | |
| 1381 | /** |
| 1382 | * runs `main` where `node:vm` modules work, starting it again in a new |
| 1383 | * process when it throws {@linkcode RestartRequired}. without |
| 1384 | * `--experimental-vm-modules`, this process stays to run its script again as a |
| 1385 | * child with it, every time the child asks, then exits like the last child. |
| 1386 | */ |
| 1387 | export async function supervise(main: () => Promise<void>): Promise<void> { |
| 1388 | if (process.execArgv.includes("--experimental-vm-modules")) { |
| 1389 | try { |
| 1390 | return await main(); |
| 1391 | } catch (error) { |
| 1392 | if (!(error instanceof RestartRequired)) throw error; |
| 1393 | log.info(error.message); |
| 1394 | log.getDrawLock("long"); |
| 1395 | process.exit(restartCode); |
| 1396 | } |
| 1397 | } |
| 1398 | // the terminal sends ctrl+c to the child too, which decides when to stop |
| 1399 | process.on("SIGINT", () => {}); |
| 1400 | while (true) { |
| 1401 | const child = child_process.spawn(process.execPath, [ |
| 1402 | "--experimental-vm-modules", |
| 1403 | "--no-warnings", |
| 1404 | ...process.execArgv, |
| 1405 | ...process.argv.slice(1), |
| 1406 | ], { stdio: "inherit" }); |
| 1407 | const stop = () => child.kill("SIGTERM"); |
| 1408 | process.on("SIGTERM", stop); |
| 1409 | const [code, signal] = await once(child, "exit") as [number | null, string | null]; |
| 1410 | process.off("SIGTERM", stop); |
| 1411 | if (code !== restartCode) process.exit(code ?? (signal ? 1 : 0)); |
| 1412 | } |
| 1413 | } |
| 1414 | |
| 1415 | /** |
| 1416 | * passed to a {@linkcode Builder}, to read inputs while tracking them. it can't |
| 1417 | * be used after the step returns. |
| 1418 | */ |
| 1419 | export class Build implements progress.Ref { |
| 1420 | /** start a progress node */ |
| 1421 | start: progress.Ref["start"]; |
| 1422 | /** aborted when the build is */ |
| 1423 | signal: AbortSignal; |
| 1424 | #run: Run; |
| 1425 | #trace: Trace; |
| 1426 | #files = new Map<string, File>(); |
| 1427 | |
| 1428 | /** @internal */ |
| 1429 | constructor(run: Run, progress: progress.Ref, trace: Trace) { |
| 1430 | this.start = (text, opts) => progress.start(text, opts); |
| 1431 | this.signal = run.signal ?? neverAborted; |
| 1432 | this.#run = run; |
| 1433 | this.#trace = trace; |
| 1434 | } |
| 1435 | |
| 1436 | /** |
| 1437 | * the step whose code is running, for inputs read where no `Build` was passed |
| 1438 | * along, like a query deep inside a database library. the top level of a |
| 1439 | * module is not a place to read inputs, and reading this there throws. |
| 1440 | */ |
| 1441 | static get active(): Build | undefined { |
| 1442 | const owner = running.getStore(); |
| 1443 | if (owner instanceof Build) return owner.#trace.done ? undefined : owner; |
| 1444 | if (owner === undefined || owner.done) return undefined; |
| 1445 | // a module loads once for all the steps that import it, and only if nothing |
| 1446 | // loaded it before, so what its top level reads can't be an input of any one step. |
| 1447 | throw new Error( |
| 1448 | "a module can't read inputs while it loads, since it loads once for every step that imports it. " |
| 1449 | + "read them in a function that a step calls", |
| 1450 | ); |
| 1451 | } |
| 1452 | |
| 1453 | #assert() { |
| 1454 | ASSERT(!this.#trace.done, "cannot read inputs or write files after step function returns"); |
| 1455 | } |
| 1456 | |
| 1457 | /** opens a file for reading. */ |
| 1458 | readFile(file: string): File { |
| 1459 | this.#assert(); |
| 1460 | const abs = path.resolve(this.#run.graph.root, file); |
| 1461 | let read = this.#files.get(abs); |
| 1462 | if (!read) { |
| 1463 | const opened = read = new File(this.#run, abs, this.#run.takeContents(abs)); |
| 1464 | this.#files.set(abs, opened); |
| 1465 | this.#trace.inputs.set(opened.url, () => opened[kInput]()); |
| 1466 | } |
| 1467 | return read; |
| 1468 | } |
| 1469 | |
| 1470 | /** lists a directory, sorted, with the names of directories ending in `/` */ |
| 1471 | async readDir(dir: string): Promise<string[]> { |
| 1472 | this.#assert(); |
| 1473 | const abs = path.resolve(this.#run.graph.root, dir); |
| 1474 | const listing = this.#run.dir(abs); |
| 1475 | this.#trace.inputs.set(dirUrl(abs), async () => (await listing).input); |
| 1476 | const { entries, error } = await listing; |
| 1477 | if (!entries) throw error; |
| 1478 | return this.#run.turn(() => [...entries]); |
| 1479 | } |
| 1480 | |
| 1481 | /** |
| 1482 | * lists a directory and the directories in it, as paths relative to `dir`, |
| 1483 | * each directory followed by what is in it. |
| 1484 | */ |
| 1485 | async readDirRecursive(dir: string): Promise<string[]> { |
| 1486 | const entries = await this.readDir(dir); |
| 1487 | const nested = await Promise.all( |
| 1488 | entries.map((name) => name.endsWith("/") ? this.readDirRecursive(path.join(dir, name)) : []), |
| 1489 | ); |
| 1490 | return entries.flatMap((name, i) => [name, ...nested[i]!.map((inner) => name + inner)]); |
| 1491 | } |
| 1492 | |
| 1493 | /** read another step's value, re-running this step when it changes */ |
| 1494 | async readStep<T>(step: Step<T>): Promise<ts.DeepReadonly<T>> { |
| 1495 | this.#assert(); |
| 1496 | const { key, result } = step[kState]; |
| 1497 | this.#trace.steps.set(key, result); |
| 1498 | const read = await result; |
| 1499 | return this.#run.turn(() => read.value() as ts.DeepReadonly<T>); |
| 1500 | } |
| 1501 | |
| 1502 | /** load a module, re-running this step when a file in its import graph changes */ |
| 1503 | async import<T = unknown>(file: string): Promise<T> { |
| 1504 | this.#assert(); |
| 1505 | const href = toUrl(file, this.#run.graph.root, this.#run.graph.fs); |
| 1506 | (await this[kDepend](closureKey(href))).value(); |
| 1507 | const module = await this.#run.graph[kImport](this.#run, href); |
| 1508 | return this.#run.turn(() => module.namespace as T); |
| 1509 | } |
| 1510 | |
| 1511 | /** |
| 1512 | * follow the static imports of a module without evaluating it, tracked like |
| 1513 | * {@linkcode import}. this finds things like every `.css` file a page |
| 1514 | * imports, or the data a loader returned about each module. |
| 1515 | */ |
| 1516 | async trackModule(file: string): Promise<readonly Module[]> { |
| 1517 | this.#assert(); |
| 1518 | const href = toUrl(file, this.#run.graph.root, this.#run.graph.fs); |
| 1519 | const closure = await this[kDepend](closureKey(href)); |
| 1520 | return this.#run.turn(() => closure.value() as Module[]); |
| 1521 | } |
| 1522 | |
| 1523 | /** |
| 1524 | * report a problem without stopping the step. it is cached with the step, |
| 1525 | * and fails the build like an error thrown from a step does, but nothing |
| 1526 | * that reads the step fails with it. a warning can point into source files |
| 1527 | * with `file`, `line`, and `col` fields, and `notes`. |
| 1528 | */ |
| 1529 | warn(warning: unknown): void { |
| 1530 | this.#assert(); |
| 1531 | // a warning's trace starts where it was warned, as an error's starts where it was thrown |
| 1532 | const here = new Error(); |
| 1533 | Error.captureStackTrace(here, Build.prototype.warn); |
| 1534 | this.#trace.errors.push({ ...this.#run.serialize(warning), stack: this.#run.serialize(here).stack }); |
| 1535 | } |
| 1536 | |
| 1537 | /** |
| 1538 | * write a file as part of the step. a build that reuses the step keeps the |
| 1539 | * file without running the step, unless the file is not as the step left it. |
| 1540 | */ |
| 1541 | writeFile(file: string, contents: string | Uint8Array): Promise<void> { |
| 1542 | this.#assert(); |
| 1543 | const hash = hashSerializable(contents); |
| 1544 | const written = this.#run.output(file, async () => ({ hash, write: (temp) => fs.writeFile(temp, contents) })); |
| 1545 | this.#trace.outputs.push({ key: path.resolve(this.#run.graph.root, file), hash }); |
| 1546 | return written; |
| 1547 | } |
| 1548 | |
| 1549 | /** @internal the run the step belongs to */ |
| 1550 | get [kRun](): Run { |
| 1551 | return this.#run; |
| 1552 | } |
| 1553 | |
| 1554 | /** @internal depend on a step that loads modules */ |
| 1555 | [kDepend](key: string): Promise<Result> { |
| 1556 | const result = this.#run.internal(key); |
| 1557 | if (!this.#trace.done) this.#trace.steps.set(key, result); |
| 1558 | return result; |
| 1559 | } |
| 1560 | |
| 1561 | /** @internal record an input the step already read by itself */ |
| 1562 | [kObserve](input: Input): void { |
| 1563 | if (this.#trace.done) return; |
| 1564 | const seen = this.#trace.observed.get(input.url); |
| 1565 | if (!seen) { |
| 1566 | // a later build can only check a custom input with its source |
| 1567 | if (!input.url.startsWith("file:") && !input.url.startsWith("graph:")) this.#run.source(input.url); |
| 1568 | this.#trace.observed.set(input.url, input); |
| 1569 | } else if (seen.hash !== input.hash) { |
| 1570 | // the input changed between two reads, so neither can be trusted |
| 1571 | this.#trace.observed.set(input.url, { ...seen, hash: "" }); |
| 1572 | } |
| 1573 | } |
| 1574 | |
| 1575 | /** @internal read a custom source */ |
| 1576 | async [kQuery](href: string): Promise<unknown> { |
| 1577 | this.#assert(); |
| 1578 | const query = this.#run.query(href); |
| 1579 | this.#trace.inputs.set(href, async () => (await query).input); |
| 1580 | const { value } = await query; |
| 1581 | return this.#run.turn(() => value); |
| 1582 | } |
| 1583 | } |
| 1584 | |
| 1585 | /** what a running step has read */ |
| 1586 | interface Trace { |
| 1587 | /** read when the step finishes, by url */ |
| 1588 | inputs: Map<string, () => Promise<Input>>; |
| 1589 | /** inputs the step already read by itself, like the resolver's, by url */ |
| 1590 | observed: Map<string, Input>; |
| 1591 | /** steps, including the ones that load modules, by key */ |
| 1592 | steps: Map<string, Promise<Result>>; |
| 1593 | /** from `Build#warn` */ |
| 1594 | errors: SerializedError[]; |
| 1595 | /** the files the step wrote */ |
| 1596 | outputs: Dependency[]; |
| 1597 | done: boolean; |
| 1598 | } |
| 1599 | |
| 1600 | /** a step's result in a run, from the database or just computed */ |
| 1601 | interface Result { |
| 1602 | /** `null` when the result could not be cached */ |
| 1603 | id: RowId | null; |
| 1604 | label: string; |
| 1605 | /** of the value, or of the error */ |
| 1606 | hash: string; |
| 1607 | /** what the step threw */ |
| 1608 | error: SerializedError | null; |
| 1609 | /** from `Build#warn` */ |
| 1610 | errors: SerializedError[]; |
| 1611 | /** the inputs it observed. none when the result could not be cached */ |
| 1612 | inputs: Dependency[]; |
| 1613 | /** the steps it read, in the order it read them */ |
| 1614 | steps: Dependency[]; |
| 1615 | /** the files it wrote */ |
| 1616 | outputs: Dependency[]; |
| 1617 | cached: boolean; |
| 1618 | /** the step's value, or throws its error */ |
| 1619 | value(): Serializable; |
| 1620 | } |
| 1621 | |
| 1622 | interface RunOptions { |
| 1623 | sources: Map<string, Source>; |
| 1624 | trusted: Map<string, Input> | null; |
| 1625 | /** the last build of a watch */ |
| 1626 | last: Run | null; |
| 1627 | signal: AbortSignal | undefined; |
| 1628 | rerunErrors: boolean; |
| 1629 | /** started by `Graph#lazy` */ |
| 1630 | lazy: boolean; |
| 1631 | } |
| 1632 | |
| 1633 | /** what an output holds once it is on disk */ |
| 1634 | interface Written { |
| 1635 | hash: string; |
| 1636 | source: string | null; |
| 1637 | } |
| 1638 | |
| 1639 | /** an in-progress instance of {@linkcode Graph#incrementalBuild} */ |
| 1640 | class Run { |
| 1641 | graph: Graph; |
| 1642 | db: Database; |
| 1643 | /** {@linkcode Database#dataVersion} when the run started */ |
| 1644 | version: number; |
| 1645 | signal: AbortSignal | undefined; |
| 1646 | rerunErrors: boolean; |
| 1647 | lazy: boolean; |
| 1648 | /** steps started or reused during this run, by key */ |
| 1649 | steps: Map<string, Promise<Result>> = new Map(); |
| 1650 | /** |
| 1651 | * every result this run has, including steps only checked as dependencies. |
| 1652 | * a closed run keeps the ones in the database, without their values. |
| 1653 | */ |
| 1654 | results: Map<string, Result> = new Map(); |
| 1655 | /** pending steps and outputs, awaited before the build finishes */ |
| 1656 | active: Set<Promise<unknown>> = new Set(); |
| 1657 | /** the outputs the build writes, by path. `null` until the file is on disk */ |
| 1658 | outputs: Map<string, Written | null> = new Map(); |
| 1659 | /** |
| 1660 | * the values this run read, by hash. the next build of a watch starts from |
| 1661 | * them, so a value that did not change is the same object. |
| 1662 | */ |
| 1663 | values: Map<string, Serializable> = new Map(); |
| 1664 | /** the code that called `Ref#step`, by url, as the path that step keys are made from */ |
| 1665 | callers: Map<string, string> = new Map(); |
| 1666 | /** urls of the inputs that nothing reads since this build */ |
| 1667 | unread: string[] = []; |
| 1668 | /** for {@linkcode BuildStats}: steps the build started, the ones of them that ran, and outputs written */ |
| 1669 | started = 0; |
| 1670 | ran = 0; |
| 1671 | written = 0; |
| 1672 | /** errors outside of steps, like failing to write an output */ |
| 1673 | failures: ErrorEntry[] = []; |
| 1674 | closed = false; |
| 1675 | #sources: Map<string, Source>; |
| 1676 | #trusted: Map<string, Input> | null; |
| 1677 | /** results, outputs, and values of the last build of a watch, while the database still has them */ |
| 1678 | #kept: Map<string, Result> | null; |
| 1679 | #written: Map<string, Written | null> | null; |
| 1680 | #held: Map<string, Serializable> | null; |
| 1681 | #reused = new Map<string, Promise<Result | null>>(); |
| 1682 | #inputs = new Map<string, Promise<Input>>(); |
| 1683 | #kinds = new Map<string, Promise<string>>(); |
| 1684 | #dirs = new Map<string, Promise<Listing>>(); |
| 1685 | #queries = new Map<string, Promise<Query>>(); |
| 1686 | /** file contents read while checking an input, kept for the step that reads it next */ |
| 1687 | #contents = new Map<string, Promise<Contents>>(); |
| 1688 | /** inputs that steps recorded, for `watch` to trust */ |
| 1689 | #recorded = new Map<string, Input>(); |
| 1690 | /** when the run last let the rest of the process have the thread */ |
| 1691 | #yielded = performance.now(); |
| 1692 | /** settles once the last step waiting for the thread has it */ |
| 1693 | #turn: Promise<void> = Promise.resolve(); |
| 1694 | |
| 1695 | constructor(graph: Graph, options: RunOptions) { |
| 1696 | this.graph = graph; |
| 1697 | this.db = graph.db; |
| 1698 | this.version = graph.db.dataVersion; |
| 1699 | this.signal = options.signal; |
| 1700 | this.rerunErrors = options.rerunErrors; |
| 1701 | this.lazy = options.lazy; |
| 1702 | this.#sources = options.sources; |
| 1703 | this.#trusted = options.trusted; |
| 1704 | // another process that built since may have deleted or replaced what the last build left |
| 1705 | const last = options.last?.version === this.version ? options.last : null; |
| 1706 | this.#kept = last?.results ?? null; |
| 1707 | this.#written = last?.outputs ?? null; |
| 1708 | this.#held = last?.values ?? null; |
| 1709 | } |
| 1710 | |
| 1711 | /** reuse a step if everything it observed is unchanged, or else run `compute` */ |
| 1712 | step( |
| 1713 | key: string, |
| 1714 | label: string, |
| 1715 | compute: (b: Build) => ts.Awaitable<Serializable>, |
| 1716 | progress: progress.Ref = nullProgress, |
| 1717 | ): Promise<Result> { |
| 1718 | let result = this.steps.get(key); |
| 1719 | if (result) return result; |
| 1720 | if (!isInternal(key)) this.started++; |
| 1721 | |
| 1722 | const kept = this.#keep(key); |
| 1723 | if (kept && !(this.rerunErrors && (kept.error || kept.errors.length > 0))) { |
| 1724 | this.#claim(kept); |
| 1725 | this.graph[kStepEvent]({ key, label, lazy: this.lazy, state: "kept" }); |
| 1726 | this.steps.set(key, result = Promise.resolve(kept)); |
| 1727 | return result; |
| 1728 | } |
| 1729 | |
| 1730 | result = this.turn(() => this.#step(key, label, compute, progress)); |
| 1731 | this.steps.set(key, result); |
| 1732 | this.track(result); |
| 1733 | result.then( |
| 1734 | (done) => this.results.set(key, done), |
| 1735 | (error) => this.failures.push(this.entry(error, label)), |
| 1736 | ); |
| 1737 | return result; |
| 1738 | } |
| 1739 | |
| 1740 | /** |
| 1741 | * a step the last build of a watch left, when nothing it observed can have |
| 1742 | * changed since: its inputs are trusted, and the steps it read are kept the |
| 1743 | * same way. a watch reuses most steps like this, without reading the database. |
| 1744 | */ |
| 1745 | #keep(key: string): Result | null { |
| 1746 | const known = this.results.get(key); |
| 1747 | if (known) return known.id === null ? null : known; |
| 1748 | const kept = this.#kept?.get(key); |
| 1749 | if (!kept) return null; |
| 1750 | |
| 1751 | const same = kept.inputs.every((dep) => this.#trusts(dep)) |
| 1752 | && kept.outputs.every((output) => this.#written?.get(output.key)?.hash === output.hash) |
| 1753 | && kept.steps.every((dep) => this.#keep(dep.key)?.hash === dep.hash); |
| 1754 | if (!same) return null; |
| 1755 | |
| 1756 | const result = this.#cached(kept); |
| 1757 | this.results.set(key, result); |
| 1758 | return result; |
| 1759 | } |
| 1760 | |
| 1761 | /** the files of a step this build reuses are its outputs too */ |
| 1762 | #claim(result: Result): void { |
| 1763 | for (const { key: dest, hash } of result.outputs) { |
| 1764 | ASSERT(!this.outputs.has(dest), () => `${dest} is written twice`); |
| 1765 | this.outputs.set(dest, { hash, source: null }); |
| 1766 | } |
| 1767 | } |
| 1768 | |
| 1769 | /** whether an input is trusted to have the hash a step observed. probes compare kinds. */ |
| 1770 | #trusts(dep: Dependency): boolean { |
| 1771 | const trusted = this.#trusted?.get(dep.key); |
| 1772 | if (!trusted) return false; |
| 1773 | if (dep.key.startsWith("file:") && kinds.has(dep.hash)) return trusted.kind === dep.hash; |
| 1774 | return trusted.hash === dep.hash; |
| 1775 | } |
| 1776 | |
| 1777 | async #step( |
| 1778 | key: string, |
| 1779 | label: string, |
| 1780 | compute: (b: Build) => ts.Awaitable<Serializable>, |
| 1781 | progress: progress.Ref, |
| 1782 | ): Promise<Result> { |
| 1783 | this.graph[kStepEvent]({ key, label, lazy: this.lazy, state: "checking" }); |
| 1784 | const cached = await this.#reuse(key); |
| 1785 | if (!cached) return this.#execute(key, label, compute, progress); |
| 1786 | if (!this.rerunErrors || (!cached.error && cached.errors.length === 0)) { |
| 1787 | this.#claim(cached); |
| 1788 | this.graph[kStepEvent]({ key, label, lazy: this.lazy, state: "reused" }); |
| 1789 | return cached; |
| 1790 | } |
| 1791 | const rerun = await this.#execute(key, label, compute, progress); |
| 1792 | if (rerun.hash !== cached.hash || JSON.stringify(rerun.errors) !== JSON.stringify(cached.errors)) { |
| 1793 | const error = new Error(`${label} had a different result when re-run, so not all of its inputs are tracked`); |
| 1794 | this.failures.push(this.entry(error, label)); |
| 1795 | } |
| 1796 | return rerun; |
| 1797 | } |
| 1798 | |
| 1799 | /** |
| 1800 | * start `work` once it is its turn. steps take the thread one after another, |
| 1801 | * and after a few milliseconds of that, the process serves its other work |
| 1802 | * first, like requests for what the last build wrote. a step takes a turn to |
| 1803 | * start, and to resume from a wait that many steps may resume from at once. |
| 1804 | */ |
| 1805 | turn<T>(work: () => T | PromiseLike<T>): Promise<T> { |
| 1806 | const granted = this.#turn = this.#turn.then(async () => { |
| 1807 | if (performance.now() - this.#yielded < 5) return; |
| 1808 | await new Promise((resolve) => setImmediate(resolve)); |
| 1809 | this.#yielded = performance.now(); |
| 1810 | }); |
| 1811 | return granted.then(work); |
| 1812 | } |
| 1813 | |
| 1814 | async #execute( |
| 1815 | key: string, |
| 1816 | label: string, |
| 1817 | compute: (b: Build) => ts.Awaitable<Serializable>, |
| 1818 | progress: progress.Ref, |
| 1819 | ): Promise<Result> { |
| 1820 | const trace: Trace = { |
| 1821 | inputs: new Map(), |
| 1822 | observed: new Map(), |
| 1823 | steps: new Map(), |
| 1824 | errors: [], |
| 1825 | outputs: [], |
| 1826 | done: false, |
| 1827 | }; |
| 1828 | const b = new Build(this, progress, trace); |
| 1829 | if (!isInternal(key)) this.ran++; |
| 1830 | this.graph[kStepEvent]({ key, label, lazy: this.lazy, state: "running" }); |
| 1831 | let outcome: { value: Serializable } | { error: unknown }; |
| 1832 | try { |
| 1833 | outcome = { value: await running.run(b, () => compute(b)) }; |
| 1834 | } catch (error) { |
| 1835 | outcome = { error }; |
| 1836 | } finally { |
| 1837 | trace.done = true; |
| 1838 | } |
| 1839 | return this.#commit(key, label, trace, outcome); |
| 1840 | } |
| 1841 | |
| 1842 | async #commit( |
| 1843 | key: string, |
| 1844 | label: string, |
| 1845 | trace: Trace, |
| 1846 | outcome: { value: Serializable } | { error: unknown }, |
| 1847 | ): Promise<Result> { |
| 1848 | let error = "error" in outcome ? this.serialize(outcome.error) : null; |
| 1849 | let value = "value" in outcome ? outcome.value : undefined; |
| 1850 | let hash: string; |
| 1851 | try { |
| 1852 | hash = error ? hashSerializable("error", error) : hashSerializable(value); |
| 1853 | } catch (cause) { |
| 1854 | // a value that can't be hashed can't be stored either |
| 1855 | error = this.serialize(cause); |
| 1856 | value = undefined; |
| 1857 | hash = hashSerializable("error", error); |
| 1858 | } |
| 1859 | // errors point into files with stacks, locations, and notes. a cached error |
| 1860 | // re-runs when those change, so its lines and columns stay right. |
| 1861 | for (const file of errorFiles(error ? [error, ...trace.errors] : trace.errors, this.graph.root)) { |
| 1862 | const opened = new File(this, file, this.takeContents(file)); |
| 1863 | if (!trace.inputs.has(opened.url)) trace.inputs.set(opened.url, () => opened[kInput]()); |
| 1864 | } |
| 1865 | // what modules observed while running belongs to every step that depends |
| 1866 | // on them. closures `import()` added are swept as the loop reaches them. |
| 1867 | for (const [dep, pending] of trace.steps) { |
| 1868 | if (!dep.startsWith("closure:")) continue; |
| 1869 | const closure = await pending; |
| 1870 | if (closure.error) continue; |
| 1871 | for (const node of closure.value() as Module[]) { |
| 1872 | const runtime = this.graph[kRuntime](node.url); |
| 1873 | if (!runtime) continue; |
| 1874 | for (const input of runtime.inputs.values()) { |
| 1875 | if (!trace.observed.has(input.url)) trace.observed.set(input.url, input); |
| 1876 | } |
| 1877 | for (const other of runtime.steps) { |
| 1878 | if (!trace.steps.has(other)) trace.steps.set(other, this.internal(other)); |
| 1879 | } |
| 1880 | } |
| 1881 | } |
| 1882 | const inputs = await Promise.allSettled(Array.from(trace.inputs.values(), (read) => read())); |
| 1883 | const steps = await Promise.all(trace.steps.values()); |
| 1884 | // a failed read or an uncached dependency means the trace doesn't describe |
| 1885 | // the whole result, and nothing that finished after an abort is trusted. |
| 1886 | const complete = !this.signal?.aborted |
| 1887 | && inputs.every((input) => input.status === "fulfilled") |
| 1888 | && steps.every((step) => step.id !== null); |
| 1889 | let id: RowId | null = null; |
| 1890 | let observed: Input[] = []; |
| 1891 | if (complete) { |
| 1892 | observed = inputs.map((input) => (input as PromiseFulfilledResult<Input>).value) |
| 1893 | .concat(Array.from(trace.observed.values()).filter((input) => !trace.inputs.has(input.url))); |
| 1894 | for (const input of observed) this.#recorded.set(input.url, input); |
| 1895 | id = this.db.writeStep({ |
| 1896 | key, |
| 1897 | label, |
| 1898 | hash, |
| 1899 | error, |
| 1900 | value, |
| 1901 | errors: trace.errors, |
| 1902 | inputs: observed, |
| 1903 | steps, |
| 1904 | outputs: trace.outputs, |
| 1905 | }); |
| 1906 | // what the last build left no longer describes the row |
| 1907 | this.#kept?.delete(key); |
| 1908 | } |
| 1909 | this.graph[kStepEvent]({ |
| 1910 | key, |
| 1911 | label, |
| 1912 | lazy: this.lazy, |
| 1913 | state: error ? "failed" : "ran", |
| 1914 | steps: Array.from(trace.steps.keys()), |
| 1915 | inputs: observed.map(({ url, kind }) => ({ url, kind })), |
| 1916 | outputs: trace.outputs.map((output) => url.pathToFileURL(output.key).href), |
| 1917 | }); |
| 1918 | return { |
| 1919 | id, |
| 1920 | label, |
| 1921 | hash, |
| 1922 | error, |
| 1923 | errors: trace.errors, |
| 1924 | inputs: observed.map((input) => ({ key: input.url, hash: input.hash })), |
| 1925 | steps: Array.from(trace.steps.keys(), (key, i) => ({ key, hash: steps[i]!.hash })), |
| 1926 | outputs: trace.outputs, |
| 1927 | cached: false, |
| 1928 | value: () => { |
| 1929 | if (error) throw reconstruct(error); |
| 1930 | this.values.set(hash, value); |
| 1931 | return value; |
| 1932 | }, |
| 1933 | }; |
| 1934 | } |
| 1935 | |
| 1936 | /** a result in the database, whose value is read once it is asked for */ |
| 1937 | #cached({ id, label, hash, error, errors, inputs, steps, outputs }: Omit<Result, "cached" | "value">): Result { |
| 1938 | return { |
| 1939 | id, |
| 1940 | label, |
| 1941 | hash, |
| 1942 | error, |
| 1943 | errors, |
| 1944 | inputs, |
| 1945 | steps, |
| 1946 | outputs, |
| 1947 | cached: true, |
| 1948 | value: () => { |
| 1949 | if (error) throw reconstruct(error); |
| 1950 | if (!this.values.has(hash)) { |
| 1951 | this.values.set(hash, this.#held?.has(hash) ? this.#held.get(hash) : this.db.readValue(UNWRAP(id))); |
| 1952 | } |
| 1953 | return this.values.get(hash); |
| 1954 | }, |
| 1955 | }; |
| 1956 | } |
| 1957 | |
| 1958 | /** |
| 1959 | * a step's result in the database, when every hash it observed is still |
| 1960 | * current, and the files it wrote are as it left them. what the last build |
| 1961 | * of a watch left says what the database and the disk have. |
| 1962 | */ |
| 1963 | #reuse(key: string): Promise<Result | null> { |
| 1964 | return memo(this.#reused, key, async () => { |
| 1965 | const row = this.#kept?.get(key) ?? this.#stored(key); |
| 1966 | if (!row) return null; |
| 1967 | const result = this.#cached(row); |
| 1968 | // another lazy step can evict the row while this one is checked, so its value is read first |
| 1969 | if (this.lazy && !row.error) result.value(); |
| 1970 | |
| 1971 | const matches = await Promise.all([ |
| 1972 | ...row.inputs.map((dep) => this.#trusts(dep) || this.matches(dep.key, dep.hash)), |
| 1973 | ...row.outputs.map(async ({ key: dest, hash }) => |
| 1974 | this.#written?.get(dest)?.hash === hash || (await this.#onDisk(dest, hash)) !== null |
| 1975 | ), |
| 1976 | ]); |
| 1977 | if (!matches.every(Boolean)) return null; |
| 1978 | // which steps a step reads can depend on what it read before, so they are |
| 1979 | // checked in order, stopping at the first that changed. |
| 1980 | for (const dep of row.steps) { |
| 1981 | if ((await this.#hash(dep.key)) !== dep.hash) return null; |
| 1982 | } |
| 1983 | |
| 1984 | this.results.set(key, result); |
| 1985 | return result; |
| 1986 | }); |
| 1987 | } |
| 1988 | |
| 1989 | /** a step's row in the database, with the hashes it observed */ |
| 1990 | #stored(key: string): Omit<Result, "cached" | "value"> | null { |
| 1991 | const row = this.db.readStep(key); |
| 1992 | return row && { |
| 1993 | id: row.id, |
| 1994 | label: row.label, |
| 1995 | hash: row.hash, |
| 1996 | error: row.type === "e" ? this.db.readValue(row.id) as SerializedError : null, |
| 1997 | errors: row.errors === null ? [] : JSON.parse(row.errors) as SerializedError[], |
| 1998 | ...this.db.readDependencies(row.id), |
| 1999 | }; |
| 2000 | } |
| 2001 | |
| 2002 | /** a step's current hash, or `null` when there is no row to depend on */ |
| 2003 | async #hash(key: string): Promise<string | null> { |
| 2004 | const pending = this.steps.get(key) ?? (isInternal(key) ? this.internal(key) : null); |
| 2005 | // re-running a changed step needs its code, which hasn't started |
| 2006 | const result = await (pending ?? this.#keep(key) ?? this.#reuse(key)); |
| 2007 | return !result || result.id === null ? null : result.hash; |
| 2008 | } |
| 2009 | |
| 2010 | internal(key: string): Promise<Result> { |
| 2011 | return this.steps.get(key) ?? this.graph[kInternal](this, key); |
| 2012 | } |
| 2013 | |
| 2014 | /** whether an input has the hash a step observed. probes compare kinds. */ |
| 2015 | async matches(href: string, hash: string): Promise<boolean> { |
| 2016 | try { |
| 2017 | if (href.startsWith("file:") && kinds.has(hash)) return (await this.kind(href)) === hash; |
| 2018 | return (await this.input(href)).hash === hash; |
| 2019 | } catch { |
| 2020 | return false; |
| 2021 | } |
| 2022 | } |
| 2023 | |
| 2024 | /** the current state of an input, checked at most once per run */ |
| 2025 | input(href: string): Promise<Input> { |
| 2026 | return memo(this.#inputs, href, async () => { |
| 2027 | const trusted = this.#trusted?.get(href); |
| 2028 | if (trusted && !(trusted.kind === "file" && kinds.has(trusted.hash))) return trusted; |
| 2029 | const prev = this.db.readInput(href); |
| 2030 | const next = href.startsWith("graph:") |
| 2031 | ? this.graph[kOption](href) |
| 2032 | : !href.startsWith("file:") |
| 2033 | ? await this.#checkSource(href, prev) |
| 2034 | : href.endsWith("/") |
| 2035 | ? (await this.dir(url.fileURLToPath(href))).input |
| 2036 | : await this.#check(url.fileURLToPath(href), prev); |
| 2037 | this.db.writeInput(next); |
| 2038 | return next; |
| 2039 | }); |
| 2040 | } |
| 2041 | |
| 2042 | /** whether a path is a file, directory, other, or missing */ |
| 2043 | kind(href: string): Promise<string> { |
| 2044 | return memo(this.#kinds, href, async () => { |
| 2045 | const known = this.#trusted?.get(href) ?? await this.#inputs.get(href)?.catch(() => null); |
| 2046 | if (known) return known.kind; |
| 2047 | try { |
| 2048 | return (await this.graph.fs.stat(url.fileURLToPath(href))).kind; |
| 2049 | } catch (error) { |
| 2050 | if (isMissing(error)) return "missing"; |
| 2051 | throw error; |
| 2052 | } |
| 2053 | }); |
| 2054 | } |
| 2055 | |
| 2056 | async #check(file: string, prev: Input | null): Promise<Input> { |
| 2057 | const href = url.pathToFileURL(file).href; |
| 2058 | let stat: Stat; |
| 2059 | try { |
| 2060 | stat = await this.graph.fs.stat(file); |
| 2061 | } catch (error) { |
| 2062 | if (isMissing(error)) return record(href, "missing"); |
| 2063 | throw error; |
| 2064 | } |
| 2065 | if (stat.kind !== "file") return record(href, stat.kind); |
| 2066 | if ( |
| 2067 | prev?.kind === "file" && !kinds.has(prev.hash) && prev.hash !== "" |
| 2068 | && prev.time === stat.time && prev.size === stat.size |
| 2069 | ) { |
| 2070 | return prev; |
| 2071 | } |
| 2072 | // the step that read this file is likely about to re-run, so it gets |
| 2073 | // these bytes instead of reading the file again. |
| 2074 | const contents = readContents(this.graph.fs, file); |
| 2075 | this.#contents.set(file, contents); |
| 2076 | return (await contents).input; |
| 2077 | } |
| 2078 | |
| 2079 | async #checkSource(href: string, prev: Input | null): Promise<Input> { |
| 2080 | const { source, params } = this.source(href); |
| 2081 | if (prev && (await source.unchanged?.(params, prev))) return prev; |
| 2082 | return (await this.query(href)).input; |
| 2083 | } |
| 2084 | |
| 2085 | /** hand bytes read during a check to the step reading them */ |
| 2086 | takeContents(file: string): Promise<Contents> | null { |
| 2087 | const contents = this.#contents.get(file) ?? null; |
| 2088 | this.#contents.delete(file); |
| 2089 | return contents; |
| 2090 | } |
| 2091 | |
| 2092 | dir(dir: string): Promise<Listing> { |
| 2093 | return memo(this.#dirs, dir, async () => { |
| 2094 | const href = dirUrl(dir); |
| 2095 | try { |
| 2096 | const entries = (await this.graph.fs.listDir(dir)).sort(); |
| 2097 | return { |
| 2098 | input: { url: href, kind: "directory", time: 0, size: entries.length, hash: sha256(entries.join("\0")) }, |
| 2099 | entries, |
| 2100 | error: null, |
| 2101 | }; |
| 2102 | } catch (error) { |
| 2103 | // `kind` stats the path with its slash, which is missing when it is a file |
| 2104 | if (!isMissing(error)) throw error; |
| 2105 | return { input: record(href, "missing"), entries: null, error }; |
| 2106 | } |
| 2107 | }); |
| 2108 | } |
| 2109 | |
| 2110 | query(href: string): Promise<Query> { |
| 2111 | return memo(this.#queries, href, async () => { |
| 2112 | const { source, params } = this.source(href); |
| 2113 | const { value, hash } = await source.read(params); |
| 2114 | return { |
| 2115 | input: { url: href, kind: source.protocol, time: 0, size: 0, hash: hash ?? hashSerializable(value) }, |
| 2116 | value, |
| 2117 | }; |
| 2118 | }); |
| 2119 | } |
| 2120 | |
| 2121 | source(href: string): { source: Source; params: ts.JsonValue } { |
| 2122 | const colon = href.indexOf(":"); |
| 2123 | const source = this.#sources.get(href.slice(0, colon)); |
| 2124 | if (!source) throw new Error(`no source is registered for ${href.slice(0, colon + 1)}`); |
| 2125 | return { source, params: JSON.parse(decodeURIComponent(href.slice(colon + 1))) }; |
| 2126 | } |
| 2127 | |
| 2128 | /** write an output, unless the file on disk is what the last build wrote */ |
| 2129 | output( |
| 2130 | file: string, |
| 2131 | prepare: () => Promise<{ hash: string; source?: string; write(temp: string): Promise<void> }>, |
| 2132 | ): Promise<void> { |
| 2133 | ASSERT(!this.closed, "cannot write an output after the build has closed"); |
| 2134 | // what writes a file has to be in a build, which deletes the files it doesn't write |
| 2135 | ASSERT(!this.lazy, "a lazy step can't write files"); |
| 2136 | const dest = path.resolve(this.graph.root, file); |
| 2137 | // a build deletes the files it stops writing, which must be the project's |
| 2138 | ASSERT(isWithin(this.graph.root, dest), () => `${dest} is outside of ${this.graph.root}`); |
| 2139 | ASSERT(!this.outputs.has(dest), () => `${dest} is written twice`); |
| 2140 | this.outputs.set(dest, null); |
| 2141 | const promise = (async () => { |
| 2142 | const { hash, source = null, write } = await prepare(); |
| 2143 | // the last build of a watch left the file as it is on disk |
| 2144 | const known = this.#written?.get(dest); |
| 2145 | if (known?.hash !== hash || known.source !== source) await this.#write(dest, hash, source, write); |
| 2146 | this.outputs.set(dest, { hash, source }); |
| 2147 | })(); |
| 2148 | this.track(promise); |
| 2149 | promise.catch((error) => this.failures.push(this.entry(error, `write ${path.relative(this.graph.root, dest)}`))); |
| 2150 | return promise; |
| 2151 | } |
| 2152 | |
| 2153 | async #write( |
| 2154 | dest: string, |
| 2155 | hash: string, |
| 2156 | source: string | null, |
| 2157 | write: (temp: string) => Promise<void>, |
| 2158 | ): Promise<void> { |
| 2159 | const prev = await this.#onDisk(dest, hash); |
| 2160 | if (prev) { |
| 2161 | if (prev.source !== source) this.db.writeOutput({ ...prev, source }); |
| 2162 | return; |
| 2163 | } |
| 2164 | // written beside the destination and renamed over it, so nothing reads |
| 2165 | // a half-written file. |
| 2166 | const place = async () => { |
| 2167 | await fs.mkdir(path.dirname(dest), { recursive: true }); |
| 2168 | const temp = `${dest}.${process.pid}.${crypto.randomUUID().slice(0, 8)}.tmp`; |
| 2169 | try { |
| 2170 | await write(temp); |
| 2171 | await fs.rename(temp, dest); |
| 2172 | } finally { |
| 2173 | await fs.rm(temp, { force: true }); |
| 2174 | } |
| 2175 | }; |
| 2176 | try { |
| 2177 | await place(); |
| 2178 | } catch (error) { |
| 2179 | // an output this build no longer writes may be in the way, as a file |
| 2180 | // where a directory goes, or as a directory holding old outputs. |
| 2181 | const stale = this.db.outputs().filter((old) => |
| 2182 | !this.outputs.has(old) && (isWithin(old, dest) || isWithin(dest, old)) |
| 2183 | ); |
| 2184 | if (stale.length === 0) throw error; |
| 2185 | for (const old of stale) { |
| 2186 | await removeOutput(old, this.graph.root); |
| 2187 | this.db.deleteOutput(old); |
| 2188 | } |
| 2189 | await place(); |
| 2190 | } |
| 2191 | const { time, size } = await Fs.default.stat(dest); |
| 2192 | this.db.writeOutput({ path: dest, hash, time, size, source }); |
| 2193 | this.written++; |
| 2194 | } |
| 2195 | |
| 2196 | /** an output's row, when the file on disk is the one a build wrote with `hash` */ |
| 2197 | async #onDisk(dest: string, hash: string): Promise<OutputRow | null> { |
| 2198 | const prev = this.db.readOutput(dest); |
| 2199 | if (prev?.hash !== hash) return null; |
| 2200 | const stat = await Fs.default.stat(dest).catch(() => null); |
| 2201 | return stat?.time === prev.time && stat.size === prev.size ? prev : null; |
| 2202 | } |
| 2203 | |
| 2204 | track(promise: Promise<unknown>): void { |
| 2205 | this.active.add(promise); |
| 2206 | const remove = () => void this.active.delete(promise); |
| 2207 | promise.then(remove, remove); |
| 2208 | } |
| 2209 | |
| 2210 | /** |
| 2211 | * no more steps or outputs start. the steps that load modules stay, so |
| 2212 | * modules loaded later match the ones already loaded. |
| 2213 | */ |
| 2214 | close(): void { |
| 2215 | this.closed = true; |
| 2216 | for (const [key, result] of this.results) { |
| 2217 | if (result.id === null) this.results.delete(key); |
| 2218 | else this.results.set(key, this.#cached(result)); |
| 2219 | } |
| 2220 | // what the build before left would otherwise keep every build of a watch alive |
| 2221 | this.#kept = this.#written = this.#held = null; |
| 2222 | this.#contents.clear(); |
| 2223 | for (const key of this.steps.keys()) if (!isInternal(key)) this.steps.delete(key); |
| 2224 | } |
| 2225 | |
| 2226 | /** wait for every step and output, including ones started while waiting */ |
| 2227 | async settle(): Promise<void> { |
| 2228 | while (this.active.size > 0) await Promise.allSettled(this.active); |
| 2229 | } |
| 2230 | |
| 2231 | /** |
| 2232 | * every error of the steps the build started, and of the steps they read. |
| 2233 | * steps that were only checked, like a dependency a step stopped reading, |
| 2234 | * don't count. |
| 2235 | */ |
| 2236 | errors(): ErrorEntry[] { |
| 2237 | const entries: ErrorEntry[] = []; |
| 2238 | const reached = new Set<string>(); |
| 2239 | const visit = (key: string) => { |
| 2240 | const result = this.results.get(key); |
| 2241 | if (!result || reached.has(key)) return; |
| 2242 | reached.add(key); |
| 2243 | // dependencies first, so a step that threw is listed before its readers |
| 2244 | for (const dep of result.steps) visit(dep.key); |
| 2245 | if (result.error) { |
| 2246 | entries.push({ error: result.error, fatal: true, cached: result.cached, step: result.label }); |
| 2247 | } |
| 2248 | for (const error of result.errors) { |
| 2249 | entries.push({ error, fatal: false, cached: result.cached, step: result.label }); |
| 2250 | } |
| 2251 | }; |
| 2252 | for (const key of this.steps.keys()) if (!isInternal(key)) visit(key); |
| 2253 | return [...entries, ...this.failures]; |
| 2254 | } |
| 2255 | |
| 2256 | entry(error: unknown, step: string): ErrorEntry { |
| 2257 | return { error: this.serialize(error), fatal: true, cached: false, step }; |
| 2258 | } |
| 2259 | |
| 2260 | serialize(error: unknown): SerializedError { |
| 2261 | return serializeError(error, (frame) => this.graph[kMapFrame](frame)); |
| 2262 | } |
| 2263 | |
| 2264 | /** inputs this run checked or recorded, for `watch` to trust */ |
| 2265 | async checked(): Promise<Map<string, Input>> { |
| 2266 | const checked = new Map<string, Input>(); |
| 2267 | for (const [href, kind] of this.#kinds) { |
| 2268 | const value = await kind.catch(() => null); |
| 2269 | if (value) checked.set(href, record(href, value)); |
| 2270 | } |
| 2271 | // contents say more than a kind does |
| 2272 | for (const [href, input] of this.#recorded) checked.set(href, input); |
| 2273 | for (const [href, input] of this.#inputs) { |
| 2274 | const value = await input.catch(() => null); |
| 2275 | if (value) checked.set(href, value); |
| 2276 | } |
| 2277 | // a file that changed while a step read it has no hash to trust |
| 2278 | for (const [href, input] of checked) if (input.hash === "") checked.delete(href); |
| 2279 | return checked; |
| 2280 | } |
| 2281 | } |
| 2282 | |
| 2283 | // --- helpers --- |
| 2284 | |
| 2285 | const neverAborted = new AbortController().signal; |
| 2286 | /** a child process exits with this to be started again */ |
| 2287 | const restartCode = 75; |
| 2288 | const nullProgress: progress.Ref = progress.nullNode; |
| 2289 | |
| 2290 | /** what the running code belongs to: a step, or the module whose top level it is */ |
| 2291 | const running = new AsyncLocalStorage<Build | Loading>(); |
| 2292 | |
| 2293 | /** a module's top level, which a callback it leaves behind runs outside of */ |
| 2294 | interface Loading { |
| 2295 | done: boolean; |
| 2296 | [kRun]: Run; |
| 2297 | } |
| 2298 | |
| 2299 | /** kinds of paths, which probes store instead of a content hash */ |
| 2300 | const kinds = new Set(["file", "directory", "other", "missing"]); |
| 2301 | |
| 2302 | /** keys of the steps that load modules, which can run from the key alone */ |
| 2303 | function isInternal(key: string): boolean { |
| 2304 | return /^(load|closure|package|resolve):/.test(key); |
| 2305 | } |
| 2306 | |
| 2307 | const loadKey = (href: string) => `load:${href}`; |
| 2308 | const closureKey = (href: string) => `closure:${href}`; |
| 2309 | const packageKey = (dir: string) => `package:${dirUrl(dir)}`; |
| 2310 | const resolveKey = (dir: string, specifier: string) => `resolve:${dirUrl(dir)} ${specifier}`; |
| 2311 | |
| 2312 | function memo<K, V>(map: Map<K, V>, key: K, create: () => V): V { |
| 2313 | let value = map.get(key); |
| 2314 | if (value === undefined) map.set(key, value = create()); |
| 2315 | return value; |
| 2316 | } |
| 2317 | |
| 2318 | // --- errors --- |
| 2319 | |
| 2320 | /** additional details in source code to highlight. see {@linkcode Build#warn} */ |
| 2321 | export interface Note { |
| 2322 | text: string; |
| 2323 | file: string; |
| 2324 | line: number; |
| 2325 | col: number; |
| 2326 | } |
| 2327 | |
| 2328 | /** errors are serialized and revived across restarts */ |
| 2329 | export type SerializedError = { |
| 2330 | name: string; |
| 2331 | message: string; |
| 2332 | /** source mapped */ |
| 2333 | stack: Array<ValidateSerializable<stack.Frame>>; |
| 2334 | /** extra fields on the error, keeping only json values */ |
| 2335 | meta: { [key: string]: ts.JsonValue }; |
| 2336 | cause: SerializedError | null; |
| 2337 | /** from an `AggregateError` */ |
| 2338 | errors: SerializedError[] | null; |
| 2339 | }; |
| 2340 | |
| 2341 | interface ErrorEntry { |
| 2342 | error: SerializedError; |
| 2343 | /** thrown, rather than reported with `Build#warn` */ |
| 2344 | fatal: boolean; |
| 2345 | /** replayed from the database */ |
| 2346 | cached: boolean; |
| 2347 | /** label of the step it came from */ |
| 2348 | step: string; |
| 2349 | } |
| 2350 | |
| 2351 | function serializeError( |
| 2352 | value: unknown, |
| 2353 | mapFrame: (frame: stack.Frame) => stack.Frame, |
| 2354 | ): SerializedError { |
| 2355 | const error = exception.obj(value); |
| 2356 | const meta: SerializedError["meta"] = {}; |
| 2357 | for (const [key, field] of Object.entries(error)) { |
| 2358 | if (["name", "message", "stack", "cause", "errors"].includes(key)) continue; |
| 2359 | if (isJson(field)) meta[key] = field; |
| 2360 | } |
| 2361 | // a cached error that was caught and thrown again keeps its original frames |
| 2362 | const original = error[kSerialized] as SerializedError | undefined; |
| 2363 | if (original) return { ...original, meta: { ...original.meta, ...meta } }; |
| 2364 | const frames = (stack.parse(error as Error) ?? []).map((frame) => |
| 2365 | mapFrame({ fn: frame.fn ?? null, file: frame.file ?? null, line: frame.line, col: frame.col }) |
| 2366 | ); |
| 2367 | // this file's frames that ran the step say nothing about the error |
| 2368 | const last = frames.findLastIndex(({ file }) => file !== self && file !== url.fileURLToPath(self)); |
| 2369 | if (last >= 0) frames.length = last + 1; |
| 2370 | return { |
| 2371 | name: typeof error.name === "string" ? error.name : "Error", |
| 2372 | message: error.message, |
| 2373 | stack: frames, |
| 2374 | meta, |
| 2375 | cause: error.cause === undefined ? null : serializeError(error.cause, mapFrame), |
| 2376 | errors: value instanceof AggregateError |
| 2377 | ? value.errors.map((inner) => serializeError(inner, mapFrame)) |
| 2378 | : null, |
| 2379 | }; |
| 2380 | } |
| 2381 | |
| 2382 | /** an `Error` again, with the original stack, for code that catches it */ |
| 2383 | function reconstruct(serialized: SerializedError): Error { |
| 2384 | const options = serialized.cause ? { cause: reconstruct(serialized.cause) } : undefined; |
| 2385 | const error = serialized.errors |
| 2386 | ? new AggregateError(serialized.errors.map(reconstruct), serialized.message, options) |
| 2387 | : new Error(serialized.message, options); |
| 2388 | error.name = serialized.name; |
| 2389 | error.stack = [`${serialized.name}: ${serialized.message}`, ...serialized.stack.map(formatFrame)] |
| 2390 | .join("\n"); |
| 2391 | Object.assign(error, serialized.meta); |
| 2392 | Object.defineProperty(error, kSerialized, { value: serialized }); |
| 2393 | return error; |
| 2394 | } |
| 2395 | |
| 2396 | function formatFrame({ fn, file, line, col }: stack.Frame): string { |
| 2397 | const location = `${file ?? "<anonymous>"}${line ? `:${line}${col ? `:${col}` : ""}` : ""}`; |
| 2398 | return fn ? ` at ${fn} (${location})` : ` at ${location}`; |
| 2399 | } |
| 2400 | |
| 2401 | /** files an error points into, through its stack, location, and notes */ |
| 2402 | function errorFiles(errors: SerializedError[], root: string): Set<string> { |
| 2403 | const files = new Set<string>(); |
| 2404 | const visit = (error: SerializedError) => { |
| 2405 | for (const { file } of error.stack) { |
| 2406 | if (file?.startsWith("file:")) files.add(url.fileURLToPath(file)); |
| 2407 | else if (file && path.isAbsolute(file)) files.add(file); |
| 2408 | } |
| 2409 | const { file, notes } = error.meta; |
| 2410 | if (typeof file === "string") files.add(path.resolve(root, file)); |
| 2411 | if (Array.isArray(notes)) { |
| 2412 | for (const note of notes) { |
| 2413 | const file = isPlainObject(note) ? note.file : null; |
| 2414 | if (typeof file === "string") files.add(path.resolve(root, file)); |
| 2415 | } |
| 2416 | } |
| 2417 | if (error.cause) visit(error.cause); |
| 2418 | error.errors?.forEach(visit); |
| 2419 | }; |
| 2420 | errors.forEach(visit); |
| 2421 | return files; |
| 2422 | } |
| 2423 | |
| 2424 | /** identical errors merge, and errors thrown from the same place are grouped */ |
| 2425 | function groupErrors(entries: ErrorEntry[]): Error[] { |
| 2426 | const identical = new Map<string, Omit<ErrorEntry, "step"> & { steps: string[] }>(); |
| 2427 | for (const { error, fatal, cached, step } of entries) { |
| 2428 | const key = hashSerializable(error); |
| 2429 | const existing = identical.get(key); |
| 2430 | if (!existing) { |
| 2431 | identical.set(key, { error, fatal, cached, steps: [step] }); |
| 2432 | continue; |
| 2433 | } |
| 2434 | existing.fatal ||= fatal; |
| 2435 | existing.cached &&= cached; |
| 2436 | if (!existing.steps.includes(step)) existing.steps.push(step); |
| 2437 | } |
| 2438 | const similar = Map.groupBy(identical.values(), ({ error, fatal }) => { |
| 2439 | // an error that says what it is about is only like the ones about the same place |
| 2440 | const { file, line, col } = error.meta; |
| 2441 | const top = error.stack[0]; |
| 2442 | const place = typeof file === "string" |
| 2443 | ? `${file}:${line}:${col}` |
| 2444 | : top |
| 2445 | ? `${top.file}:${top.line}:${top.col}` |
| 2446 | : error.message; |
| 2447 | return `${fatal}\0${error.name}\0${place}`; |
| 2448 | }); |
| 2449 | return Array.from(similar.values(), (group) => { |
| 2450 | // not enumerable, since the BuildError prints them above each error |
| 2451 | const errors = group.map(({ error, fatal, cached, steps }) => |
| 2452 | Object.defineProperties(reconstruct(error), { |
| 2453 | steps: { value: steps }, |
| 2454 | fatal: { value: fatal }, |
| 2455 | cached: { value: cached }, |
| 2456 | }) |
| 2457 | ); |
| 2458 | return errors.length === 1 ? errors[0]! : new AggregateError(errors, `${errors.length} similar errors`); |
| 2459 | }); |
| 2460 | } |
| 2461 | |
| 2462 | // --- inputs --- |
| 2463 | |
| 2464 | /** |
| 2465 | * a file, directory listing, or {@linkcode source|other data source} read |
| 2466 | * within a {@linkcode Builder} function. |
| 2467 | */ |
| 2468 | export interface Input { |
| 2469 | /** global key for the changable object, with enough information to re-query. */ |
| 2470 | url: string; |
| 2471 | /** "file", "directory", "missing", "other", or a source's protocol. */ |
| 2472 | kind: string; |
| 2473 | |
| 2474 | // `time` and `size` are a cheap fingerprint, so unchanged files are not |
| 2475 | // hashed again. a source can treat them as an opaque [number, number] store. |
| 2476 | |
| 2477 | /** time of last object change. */ |
| 2478 | time: number; |
| 2479 | /** byte size or other cheap source-specific magnitude. */ |
| 2480 | size: number; |
| 2481 | /** the hash of the contents. must be unique within your source implementation. */ |
| 2482 | hash: string; |
| 2483 | } |
| 2484 | |
| 2485 | /** a file read, or why it could not be */ |
| 2486 | interface Contents { |
| 2487 | input: Input; |
| 2488 | bytes: Uint8Array | null; |
| 2489 | error: unknown; |
| 2490 | } |
| 2491 | |
| 2492 | interface Listing { |
| 2493 | input: Input; |
| 2494 | entries: string[] | null; |
| 2495 | error: unknown; |
| 2496 | } |
| 2497 | |
| 2498 | interface Query { |
| 2499 | input: Input; |
| 2500 | value: unknown; |
| 2501 | } |
| 2502 | |
| 2503 | function record(href: string, kind: string): Input { |
| 2504 | return { url: href, kind, time: 0, size: 0, hash: kind }; |
| 2505 | } |
| 2506 | |
| 2507 | /** |
| 2508 | * blob-like handle to a file read within a step. the contents are hashed as |
| 2509 | * they are read, so tracking the file never reads it a second time. |
| 2510 | */ |
| 2511 | export class File { |
| 2512 | readonly path: string; |
| 2513 | readonly url: string; |
| 2514 | #run: Run; |
| 2515 | #contents: Promise<Contents> | null; |
| 2516 | /** the state of the file the first read saw */ |
| 2517 | #observed: Input | null = null; |
| 2518 | |
| 2519 | /** @internal */ |
| 2520 | constructor(run: Run, file: string, contents: Promise<Contents> | null = null) { |
| 2521 | this.path = file; |
| 2522 | this.url = url.pathToFileURL(file).href; |
| 2523 | this.#run = run; |
| 2524 | this.#contents = contents && this.#observeContents(contents); |
| 2525 | } |
| 2526 | |
| 2527 | async bytes(): Promise<Uint8Array> { |
| 2528 | const { bytes, error } = await this.#read(); |
| 2529 | if (!bytes) throw error; |
| 2530 | return bytes; |
| 2531 | } |
| 2532 | |
| 2533 | async text(): Promise<string> { |
| 2534 | return new TextDecoder().decode(await this.bytes()); |
| 2535 | } |
| 2536 | |
| 2537 | async json<T = unknown>(): Promise<T> { |
| 2538 | return JSON.parse(await this.text()) as T; |
| 2539 | } |
| 2540 | |
| 2541 | async arrayBuffer(): Promise<ArrayBuffer> { |
| 2542 | const bytes = await this.bytes(); |
| 2543 | return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength) as ArrayBuffer; |
| 2544 | } |
| 2545 | |
| 2546 | /** sha-256 of the contents, as base64url */ |
| 2547 | async hash(): Promise<string> { |
| 2548 | const { input, error } = await this.#read(); |
| 2549 | if (input.kind !== "file") throw error; |
| 2550 | return input.hash; |
| 2551 | } |
| 2552 | |
| 2553 | /** |
| 2554 | * read the file in chunks, without holding all of it. it is hashed as it |
| 2555 | * streams, and opens on the first read, so one that is never read holds |
| 2556 | * nothing open. |
| 2557 | */ |
| 2558 | async stream(options?: { chunkSize?: number }): Promise<ReadableStream<Uint8Array>> { |
| 2559 | const chunkSize = options?.chunkSize ?? 65536; |
| 2560 | ASSERT(chunkSize > 0, "chunkSize must be greater than zero"); |
| 2561 | if (this.#contents) { |
| 2562 | const bytes = await this.bytes(); |
| 2563 | let offset = 0; |
| 2564 | return new ReadableStream<Uint8Array>({ |
| 2565 | pull(controller) { |
| 2566 | if (offset >= bytes.byteLength) return controller.close(); |
| 2567 | controller.enqueue(bytes.slice(offset, offset += chunkSize)); |
| 2568 | }, |
| 2569 | }); |
| 2570 | } |
| 2571 | const fs = this.#run.graph.fs; |
| 2572 | const open = async () => { |
| 2573 | let handle: FileHandle; |
| 2574 | try { |
| 2575 | handle = await fs.open(this.path); |
| 2576 | } catch (error) { |
| 2577 | if (isMissing(error)) this.#observe(record(this.url, "missing")); |
| 2578 | throw error; |
| 2579 | } |
| 2580 | try { |
| 2581 | const stat = await fs.statHandle(handle); |
| 2582 | if (stat.kind !== "file") { |
| 2583 | this.#observe(record(this.url, stat.kind)); |
| 2584 | throw Object.assign(new Error(`expected a file at ${this.path}`), { code: "EISDIR" }); |
| 2585 | } |
| 2586 | return { handle, stat }; |
| 2587 | } catch (error) { |
| 2588 | await fs.closeHandle(handle); |
| 2589 | throw error; |
| 2590 | } |
| 2591 | }; |
| 2592 | let opened: { handle: FileHandle; stat: Stat } | null = null; |
| 2593 | let closed = false; |
| 2594 | const close = async () => { |
| 2595 | if (closed || !opened) return; |
| 2596 | closed = true; |
| 2597 | await fs.closeHandle(opened.handle); |
| 2598 | }; |
| 2599 | const hasher = crypto.createHash("sha256"); |
| 2600 | let offset = 0; |
| 2601 | return new ReadableStream<Uint8Array>({ |
| 2602 | pull: async (controller) => { |
| 2603 | try { |
| 2604 | const { handle, stat } = opened ??= await open(); |
| 2605 | if (offset < stat.size) { |
| 2606 | const chunk = new Uint8Array(Math.min(chunkSize, stat.size - offset)); |
| 2607 | const read = await fs.readHandle(handle, offset, chunk.byteLength, chunk); |
| 2608 | if (read === 0) throw new Error(`unexpected end of file while reading ${this.path}`); |
| 2609 | offset += read; |
| 2610 | hasher.update(chunk.subarray(0, read)); |
| 2611 | controller.enqueue(chunk.subarray(0, read)); |
| 2612 | } |
| 2613 | if (offset >= stat.size) { |
| 2614 | await close(); |
| 2615 | this.#observe({ url: this.url, ...stat, hash: hasher.digest("base64url") }); |
| 2616 | controller.close(); |
| 2617 | } |
| 2618 | } catch (error) { |
| 2619 | await close(); |
| 2620 | controller.error(error); |
| 2621 | } |
| 2622 | }, |
| 2623 | cancel: close, |
| 2624 | }, { highWaterMark: 0 }); |
| 2625 | } |
| 2626 | |
| 2627 | /** @internal the state of the file the step observed */ |
| 2628 | async [kInput](): Promise<Input> { |
| 2629 | if (!this.#observed) await this.#read(); |
| 2630 | return UNWRAP(this.#observed); |
| 2631 | } |
| 2632 | |
| 2633 | #read(): Promise<Contents> { |
| 2634 | return this.#contents ??= this.#observeContents(readContents(this.#run.graph.fs, this.path)); |
| 2635 | } |
| 2636 | |
| 2637 | async #observeContents(contents: Promise<Contents>): Promise<Contents> { |
| 2638 | const result = await contents; |
| 2639 | this.#observe(result.input); |
| 2640 | return this.#run.turn(() => result); |
| 2641 | } |
| 2642 | |
| 2643 | #observe(input: Input): void { |
| 2644 | if (!this.#observed) this.#observed = input; |
| 2645 | // the file changed between two reads, so neither can be trusted |
| 2646 | else if (this.#observed.hash !== input.hash) this.#observed = { ...input, hash: "" }; |
| 2647 | } |
| 2648 | } |
| 2649 | |
| 2650 | /** read a whole file and hash it, or note why it could not be read */ |
| 2651 | async function readContents(fs: Fs, file: string): Promise<Contents> { |
| 2652 | const href = url.pathToFileURL(file).href; |
| 2653 | let handle: FileHandle | null = null; |
| 2654 | try { |
| 2655 | handle = await fs.open(file); |
| 2656 | const stat = await fs.statHandle(handle); |
| 2657 | if (stat.kind !== "file") { |
| 2658 | const error = Object.assign(new Error(`expected a file at ${file}`), { code: "EISDIR" }); |
| 2659 | return { input: record(href, stat.kind), bytes: null, error }; |
| 2660 | } |
| 2661 | const bytes = await readAll(fs, handle, stat.size, file); |
| 2662 | return { input: { url: href, ...stat, hash: sha256(bytes) }, bytes, error: null }; |
| 2663 | } catch (error) { |
| 2664 | if (!isMissing(error)) throw error; |
| 2665 | return { input: record(href, "missing"), bytes: null, error }; |
| 2666 | } finally { |
| 2667 | if (handle !== null) await fs.closeHandle(handle); |
| 2668 | } |
| 2669 | } |
| 2670 | |
| 2671 | async function readAll(fs: Fs, handle: FileHandle, size: number, file: string): Promise<Uint8Array> { |
| 2672 | const bytes = new Uint8Array(size); |
| 2673 | let offset = 0; |
| 2674 | while (offset < size) { |
| 2675 | const read = await fs.readHandle(handle, offset, size - offset, bytes.subarray(offset)); |
| 2676 | if (read === 0) throw new Error(`unexpected end of file while reading ${file}`); |
| 2677 | offset += read; |
| 2678 | } |
| 2679 | return bytes; |
| 2680 | } |
| 2681 | |
| 2682 | /** |
| 2683 | * a custom kind of {@linkcode Input}, such as the result of a database query. |
| 2684 | * register it with {@linkcode GraphOptions.sources}, then read it within steps |
| 2685 | * with `query`. a cached step reads its sources again to check them, unless |
| 2686 | * `unchanged` has a cheaper way to tell. |
| 2687 | * |
| 2688 | * ```ts |
| 2689 | * const posts = incremental.source({ |
| 2690 | * protocol: "posts", |
| 2691 | * read: ({ slug }: { slug: string }) => ({ value: db.get(slug) }), |
| 2692 | * }); |
| 2693 | * const readPost = (b: incremental.Build, slug: string) => posts.query(b, { slug }); |
| 2694 | * ``` |
| 2695 | */ |
| 2696 | export function source<P extends ts.JsonValue, V>( |
| 2697 | definition: SourceDefinition<P, V> & NoInfer<CheckSerializable<V, "a source's value">>, |
| 2698 | ): Source<P, V> { |
| 2699 | ASSERT( |
| 2700 | /^[a-z][a-z0-9+.-]*$/.test(definition.protocol) && !["file", "graph", "root"].includes(definition.protocol), |
| 2701 | () => `invalid source protocol ${JSON.stringify(definition.protocol)}`, |
| 2702 | ); |
| 2703 | const href = (params: P) => `${definition.protocol}:${encodeURIComponent(stableJson(params))}`; |
| 2704 | return { |
| 2705 | ...definition, |
| 2706 | query: (b, params) => b[kQuery](href(params)) as Promise<ts.DeepReadonly<V>>, |
| 2707 | observe: (b, params, hash) => b[kObserve]({ url: href(params), kind: definition.protocol, time: 0, size: 0, hash }), |
| 2708 | }; |
| 2709 | } |
| 2710 | |
| 2711 | export interface SourceDefinition<P, V = unknown> { |
| 2712 | /** names the urls of this source's inputs, like `sqlite` */ |
| 2713 | protocol: string; |
| 2714 | /** read the current value. the hash defaults to a hash of the value. */ |
| 2715 | read(params: P): ts.Awaitable<{ value: V; hash?: string }>; |
| 2716 | /** a cheap check that `input` is still current, skipping `read` */ |
| 2717 | unchanged?(params: P, input: Input): ts.Awaitable<boolean>; |
| 2718 | /** |
| 2719 | * call `onChange` with the ones of these that may have changed, or with |
| 2720 | * nothing when any may have, for {@linkcode Graph#incrementalWatch} |
| 2721 | */ |
| 2722 | watch?(params: P[], onChange: (changed?: P[]) => void): Disposable; |
| 2723 | } |
| 2724 | |
| 2725 | export interface Source<P extends ts.JsonValue = ts.JsonValue, V = unknown> extends SourceDefinition<P, V> { |
| 2726 | /** read the source within a step, tracking it as an input */ |
| 2727 | query(b: Build, params: P): Promise<ts.DeepReadonly<V>>; |
| 2728 | /** |
| 2729 | * track what a step read from the source by itself, for reads that can't go |
| 2730 | * through `query`. `hash` is what `read` would have returned. |
| 2731 | */ |
| 2732 | observe(b: Build, params: P, hash: string): void; |
| 2733 | } |
| 2734 | |
| 2735 | /** one read of a statement, which is all a later build needs to read it again */ |
| 2736 | type SqlQuery = { |
| 2737 | /** path of the database file */ |
| 2738 | db: string; |
| 2739 | sql: string; |
| 2740 | /** as passed to the statement, with named parameters first when there are any */ |
| 2741 | args: SqlArg[]; |
| 2742 | /** from `get`, which reads one row */ |
| 2743 | first: boolean; |
| 2744 | /** rows were arrays, from `setReturnArrays` */ |
| 2745 | arrays: boolean; |
| 2746 | }; |
| 2747 | |
| 2748 | /** parameters keep their type through a url, since integers and reals bind differently */ |
| 2749 | type SqlArg = null | number | string | ["bigint" | "blob", string] | { [name: string]: SqlArg }; |
| 2750 | |
| 2751 | type SqlRow = Record<string, sqlite.SQLOutputValue>; |
| 2752 | |
| 2753 | /** `all`, `get`, or `iterate`, which take named parameters first when there are any */ |
| 2754 | type SqlRead<Result> = ( |
| 2755 | this: sqlite.StatementSync, |
| 2756 | ...args: Array<sqlite.SQLInputValue | Record<string, sqlite.SQLInputValue>> |
| 2757 | ) => Result; |
| 2758 | |
| 2759 | /** |
| 2760 | * what steps read from a {@linkcode trackSqlite}d connection, which every |
| 2761 | * graph checks. queries are checked on a read-only connection of their own, |
| 2762 | * so a check can never write. |
| 2763 | */ |
| 2764 | const sqliteSource = source({ |
| 2765 | protocol: "sqlite", |
| 2766 | read({ db, sql, args, first, arrays }: SqlQuery) { |
| 2767 | let hash: string; |
| 2768 | try { |
| 2769 | if (!sqlVersions.has(db)) sqlVersions.set(db, sqlVersion(db)); |
| 2770 | const statement = sqlChecker(db).sql.prepare(sql); |
| 2771 | // any integer reads as a bigint, where a number throws past 2^53 |
| 2772 | statement.setReadBigInts(true); |
| 2773 | statement.setReturnArrays(arrays); |
| 2774 | statement.setAllowUnknownNamedParameters(true); |
| 2775 | const params = args.map(decodeSqlArg) as sqlite.SQLInputValue[]; |
| 2776 | hash = hashSqlRows(first ? sqlRowsOf(statement.get(...params)) : statement.all(...params)); |
| 2777 | } catch (error) { |
| 2778 | hash = hashSqlFailure(error); |
| 2779 | } |
| 2780 | // steps read rows from their own connection |
| 2781 | return { value: undefined, hash }; |
| 2782 | }, |
| 2783 | watch(queries: SqlQuery[], onChange) { |
| 2784 | const files = Map.groupBy(queries, (query) => query.db); |
| 2785 | const seen = new Map(Array.from(files.keys(), (file) => [file, sqlVersions.get(file)] as const)); |
| 2786 | const poll = () => { |
| 2787 | for (const [file, reads] of files) { |
| 2788 | const prev = seen.get(file); |
| 2789 | const next = sqlVersion(file); |
| 2790 | seen.set(file, next); |
| 2791 | sqlVersions.set(file, next); |
| 2792 | if (prev !== undefined && prev !== next) onChange(reads); |
| 2793 | } |
| 2794 | }; |
| 2795 | poll(); |
| 2796 | const timer = setInterval(poll, 50); |
| 2797 | timer.unref(); |
| 2798 | return { [Symbol.dispose]: () => clearInterval(timer) }; |
| 2799 | }, |
| 2800 | }); |
| 2801 | |
| 2802 | /** |
| 2803 | * make the rows read from `db`, through statements from `prepare`, inputs of |
| 2804 | * the step that reads them, so a rebuild runs again only the steps that read |
| 2805 | * rows which changed. a query that fails is an input too, so the step runs |
| 2806 | * again once it works. reads outside of a step are left alone, and a module of |
| 2807 | * a graph can prepare statements while it loads, but not run them. tracking |
| 2808 | * sits under `prepare`, so it follows the queries of a library built on |
| 2809 | * `node:sqlite` too, like `drizzle({ client: db })`. |
| 2810 | * |
| 2811 | * ```ts |
| 2812 | * const db = incremental.trackSqlite(new DatabaseSync("site.db")); |
| 2813 | * const postsByTag = db.prepare("select title, url from posts where tag = ?"); |
| 2814 | * |
| 2815 | * function renderTag(b: incremental.Build, tag: string): string { |
| 2816 | * return template(postsByTag.all(tag)); |
| 2817 | * } |
| 2818 | * ``` |
| 2819 | */ |
| 2820 | export function trackSqlite<T extends sqlite.DatabaseSync>(db: T): T { |
| 2821 | const location = UNWRAP(db.location(), "a database in memory can't be read again by a later build"); |
| 2822 | // relative when it can be, so that a build's inputs still name it once the project has moved |
| 2823 | const relative = path.relative(process.cwd(), location); |
| 2824 | const file = relative.startsWith("..") ? location : relative; |
| 2825 | const { prepare } = db; |
| 2826 | db.prepare = function(this: T, ...options: Parameters<T["prepare"]>) { |
| 2827 | const [sql] = options; |
| 2828 | const query = (args: unknown[], first: boolean) => ({ db: file, sql, args: args.map(encodeSqlArg), first }); |
| 2829 | let statement: sqlite.StatementSync; |
| 2830 | try { |
| 2831 | statement = prepare.apply(this, options); |
| 2832 | } catch (error) { |
| 2833 | let b: Build | undefined; |
| 2834 | try { |
| 2835 | b = Build.active; |
| 2836 | } catch { |
| 2837 | // preparing reads no rows, so a module may do it while it loads, and fail by itself |
| 2838 | } |
| 2839 | if (b) sqliteSource.observe(b, { ...query([], false), arrays: false }, hashSqlFailure(error)); |
| 2840 | throw error; |
| 2841 | } |
| 2842 | const all = statement.all as SqlRead<SqlRow[]>; |
| 2843 | const get = statement.get as SqlRead<SqlRow | undefined>; |
| 2844 | const iterate = statement.iterate as SqlRead<NodeJS.Iterator<SqlRow>>; |
| 2845 | const allTracked: SqlRead<SqlRow[]> = function(...args) { |
| 2846 | return trackedSqlRead(file, () => query(args, false), () => all.apply(this, args), (rows) => rows); |
| 2847 | }; |
| 2848 | statement.all = allTracked; |
| 2849 | statement.get = function(...args) { |
| 2850 | return trackedSqlRead(file, () => query(args, true), () => get.apply(this, args), sqlRowsOf); |
| 2851 | } satisfies SqlRead<SqlRow | undefined>; |
| 2852 | // a step may stop iterating early, which a later build could not tell |
| 2853 | statement.iterate = function(...args) { |
| 2854 | return Build.active ? allTracked.apply(this, args).values() : iterate.apply(this, args); |
| 2855 | } satisfies SqlRead<NodeJS.Iterator<SqlRow>>; |
| 2856 | return statement; |
| 2857 | }; |
| 2858 | return db; |
| 2859 | } |
| 2860 | |
| 2861 | /** run a read, and inside of a step, make what it saw an input of the step */ |
| 2862 | function trackedSqlRead<T>( |
| 2863 | file: string, |
| 2864 | query: () => Omit<SqlQuery, "arrays">, |
| 2865 | read: () => T, |
| 2866 | toRows: (result: T) => unknown[], |
| 2867 | ): T { |
| 2868 | const b = Build.active; |
| 2869 | if (!b) return read(); |
| 2870 | if (!sqlVersions.has(file)) sqlVersions.set(file, sqlVersion(file)); |
| 2871 | let result: T; |
| 2872 | try { |
| 2873 | result = read(); |
| 2874 | } catch (error) { |
| 2875 | sqliteSource.observe(b, { ...query(), arrays: false }, hashSqlFailure(error)); |
| 2876 | throw error; |
| 2877 | } |
| 2878 | const rows = toRows(result); |
| 2879 | sqliteSource.observe(b, { ...query(), arrays: Array.isArray(rows[0]) }, hashSqlRows(rows)); |
| 2880 | return result; |
| 2881 | } |
| 2882 | |
| 2883 | function sqlRowsOf(row: unknown): unknown[] { |
| 2884 | return row === undefined ? [] : [row]; |
| 2885 | } |
| 2886 | |
| 2887 | /** |
| 2888 | * rows hash as the step saw them, keyed by column or as arrays. integers hash |
| 2889 | * alike as numbers and as bigints, since a check always reads them as bigints. |
| 2890 | */ |
| 2891 | function hashSqlRows(rows: unknown[]): string { |
| 2892 | const json = JSON.stringify(rows, (_, cell: unknown) => { |
| 2893 | if (typeof cell === "bigint") { |
| 2894 | return Number.isSafeInteger(Number(cell)) ? Number(cell) : ["bigint", String(cell)]; |
| 2895 | } |
| 2896 | if (typeof cell === "number" && !Number.isFinite(cell)) return ["real", String(cell)]; |
| 2897 | if (cell instanceof Uint8Array) return ["blob", base64(cell)]; |
| 2898 | return cell; |
| 2899 | }); |
| 2900 | return crypto.createHash("sha256").update(json).digest("base64url"); |
| 2901 | } |
| 2902 | |
| 2903 | /** a query that keeps failing the same way is unchanged */ |
| 2904 | function hashSqlFailure(error: unknown): string { |
| 2905 | return `failed: ${exception.message(error)}`; |
| 2906 | } |
| 2907 | |
| 2908 | function encodeSqlArg(arg: unknown): SqlArg { |
| 2909 | if (typeof arg === "bigint") return ["bigint", String(arg)]; |
| 2910 | if (ArrayBuffer.isView(arg)) return ["blob", base64(arg)]; |
| 2911 | if (arg !== null && typeof arg === "object") { |
| 2912 | return Object.fromEntries(Object.entries(arg).map(([name, value]) => [name, encodeSqlArg(value)])); |
| 2913 | } |
| 2914 | return arg as SqlArg; |
| 2915 | } |
| 2916 | |
| 2917 | function decodeSqlArg(arg: SqlArg): sqlite.SQLInputValue | Record<string, sqlite.SQLInputValue> { |
| 2918 | if (Array.isArray(arg)) return arg[0] === "bigint" ? BigInt(arg[1]) : Buffer.from(arg[1], "base64"); |
| 2919 | if (arg !== null && typeof arg === "object") { |
| 2920 | return Object.fromEntries( |
| 2921 | Object.entries(arg).map(([name, value]) => [name, decodeSqlArg(value) as sqlite.SQLInputValue]), |
| 2922 | ); |
| 2923 | } |
| 2924 | return arg; |
| 2925 | } |
| 2926 | |
| 2927 | function base64(bytes: ArrayBufferView): string { |
| 2928 | return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength).toString("base64"); |
| 2929 | } |
| 2930 | |
| 2931 | /** connections that check queries, by file */ |
| 2932 | const sqlCheckers = new Map<string, { sql: sqlite.DatabaseSync; ino: number }>(); |
| 2933 | |
| 2934 | function sqlChecker(file: string): { sql: sqlite.DatabaseSync; ino: number } { |
| 2935 | // a connection keeps reading the file it opened, even once another replaces it |
| 2936 | const { ino } = fscb.statSync(file); |
| 2937 | let open = sqlCheckers.get(file); |
| 2938 | if (open?.ino !== ino) { |
| 2939 | open?.sql.close(); |
| 2940 | open = { sql: new sqlite.DatabaseSync(file, { readOnly: true }), ino }; |
| 2941 | sqlCheckers.set(file, open); |
| 2942 | } |
| 2943 | return open; |
| 2944 | } |
| 2945 | |
| 2946 | /** each file's version as of the first read or check since a watch saw it, which a new watch starts from */ |
| 2947 | const sqlVersions = new Map<string, string>(); |
| 2948 | |
| 2949 | /** changes when another connection commits to the file, or the file is replaced */ |
| 2950 | function sqlVersion(file: string): string { |
| 2951 | try { |
| 2952 | const { sql, ino } = sqlChecker(file); |
| 2953 | const row = sql.prepare("pragma data_version;").get() as { data_version: number }; |
| 2954 | return `${ino}:${row.data_version}`; |
| 2955 | } catch { |
| 2956 | return "missing"; |
| 2957 | } |
| 2958 | } |
| 2959 | |
| 2960 | export interface FsOptions { |
| 2961 | open(path: string): Promise<FileHandle>; |
| 2962 | statHandle(handle: FileHandle): Promise<Stat>; |
| 2963 | readHandle( |
| 2964 | handle: FileHandle, |
| 2965 | offset: number, |
| 2966 | length: number, |
| 2967 | destination: Uint8Array, |
| 2968 | ): Promise<number>; |
| 2969 | closeHandle(handle: FileHandle): Promise<void>; |
| 2970 | /** names of the entries, with directories ending in `/` */ |
| 2971 | listDir(path: string): Promise<string[]>; |
| 2972 | // module resolution is synchronous, so `import.meta.resolve` can be traced |
| 2973 | statSync(path: string): Stat; |
| 2974 | readFileSync(path: string): Uint8Array; |
| 2975 | realPathSync(path: string): string; |
| 2976 | } |
| 2977 | |
| 2978 | export type FileHandle = ts.Brand<"@clo/lib/incremental/FileHandle"> & number; |
| 2979 | |
| 2980 | /** |
| 2981 | * Clover Incremental's file system abstraction requires raw file handle |
| 2982 | * operations, with the rest of the helpers being overridable from there. |
| 2983 | * |
| 2984 | * `Fs.default` is likely what you want. |
| 2985 | */ |
| 2986 | export class Fs { |
| 2987 | open: FsOptions["open"]; |
| 2988 | statHandle: FsOptions["statHandle"]; |
| 2989 | readHandle: FsOptions["readHandle"]; |
| 2990 | closeHandle: FsOptions["closeHandle"]; |
| 2991 | listDir: FsOptions["listDir"]; |
| 2992 | statSync: FsOptions["statSync"]; |
| 2993 | readFileSync: FsOptions["readFileSync"]; |
| 2994 | realPathSync: FsOptions["realPathSync"]; |
| 2995 | |
| 2996 | constructor(options: FsOptions & Partial<Fs>) { |
| 2997 | this.open = options.open; |
| 2998 | this.statHandle = options.statHandle; |
| 2999 | this.readHandle = options.readHandle; |
| 3000 | this.closeHandle = options.closeHandle; |
| 3001 | this.listDir = options.listDir; |
| 3002 | this.statSync = options.statSync; |
| 3003 | this.readFileSync = options.readFileSync; |
| 3004 | this.realPathSync = options.realPathSync; |
| 3005 | Object.assign(this, options); |
| 3006 | } |
| 3007 | |
| 3008 | /** using `node:fs` */ |
| 3009 | static default: Fs = new Fs({ |
| 3010 | open: (file) => |
| 3011 | new Promise((resolve, reject) => { |
| 3012 | fscb.open(file, "r", (error, fd) => error ? reject(error) : resolve(fd as FileHandle)); |
| 3013 | }), |
| 3014 | statHandle: (handle) => |
| 3015 | new Promise((resolve, reject) => { |
| 3016 | fscb.fstat(handle, (error, st) => error ? reject(error) : resolve(toStat(st))); |
| 3017 | }), |
| 3018 | readHandle: (handle, offset, length, destination) => |
| 3019 | new Promise((resolve, reject) => { |
| 3020 | fscb.read(handle, destination, 0, length, offset, (error, read) => error ? reject(error) : resolve(read)); |
| 3021 | }), |
| 3022 | closeHandle: (handle) => |
| 3023 | new Promise((resolve, reject) => { |
| 3024 | fscb.close(handle, (error) => error ? reject(error) : resolve()); |
| 3025 | }), |
| 3026 | async listDir(dir) { |
| 3027 | const entries = await fs.readdir(dir, { withFileTypes: true }); |
| 3028 | return await Promise.all(entries.map(async (entry) => { |
| 3029 | const isDir = entry.isDirectory() || (entry.isSymbolicLink() |
| 3030 | && await fs.stat(path.join(dir, entry.name)).then((st) => st.isDirectory(), () => false)); |
| 3031 | return isDir ? `${entry.name}/` : entry.name; |
| 3032 | })); |
| 3033 | }, |
| 3034 | statSync: (file) => toStat(fscb.statSync(file)), |
| 3035 | readFileSync: (file) => fscb.readFileSync(file), |
| 3036 | realPathSync: (file) => fscb.realpathSync.native(file), |
| 3037 | stat: async (file) => toStat(await fs.stat(file)), |
| 3038 | }); |
| 3039 | |
| 3040 | async stat(path: string): Promise<Stat> { |
| 3041 | const handle = await this.open(path); |
| 3042 | try { |
| 3043 | return await this.statHandle(handle); |
| 3044 | } finally { |
| 3045 | await this.closeHandle(handle); |
| 3046 | } |
| 3047 | } |
| 3048 | } |
| 3049 | |
| 3050 | export interface Stat { |
| 3051 | kind: "file" | "directory" | "other"; |
| 3052 | size: number; |
| 3053 | /** time of last object change. on disk, that is the ctime. */ |
| 3054 | time: number; |
| 3055 | } |
| 3056 | |
| 3057 | function toStat(st: fscb.Stats): Stat { |
| 3058 | return { |
| 3059 | kind: st.isFile() ? "file" : st.isDirectory() ? "directory" : "other", |
| 3060 | size: st.size, |
| 3061 | time: st.ctimeMs, |
| 3062 | }; |
| 3063 | } |
| 3064 | |
| 3065 | function isMissing(error: unknown): boolean { |
| 3066 | const code = exception.code(error); |
| 3067 | return code === "ENOENT" || code === "ENOTDIR"; |
| 3068 | } |
| 3069 | |
| 3070 | // --- modules --- |
| 3071 | |
| 3072 | /** context to a {@linkcode LoaderFunction} */ |
| 3073 | export interface LoaderInput { |
| 3074 | /** absolute path */ |
| 3075 | file: string; |
| 3076 | /** contents of `file` */ |
| 3077 | source: string; |
| 3078 | /** for tracking anything else the loader depends on, like a compiler's `package.json` */ |
| 3079 | build: Build; |
| 3080 | /** read the nearest `package.json`, to choose a module format */ |
| 3081 | getPackageMeta(): Promise<PackageMeta | null>; |
| 3082 | } |
| 3083 | |
| 3084 | /** |
| 3085 | * `js` and `ts` are ES modules. `cjs` loads with `require`, from disk. |
| 3086 | * `serializable` content becomes the default export. `data` is returned from |
| 3087 | * {@linkcode Build#trackModule}, and `map` is a source map for stack traces, |
| 3088 | * read from an inline `sourceMappingURL` comment when missing. |
| 3089 | */ |
| 3090 | export type LoaderResult = |
| 3091 | | { format: "js" | "ts"; content: string; map?: string | object; data?: Serializable } |
| 3092 | | { format: "cjs"; data?: Serializable } |
| 3093 | | { format: "serializable"; content: Serializable; data?: Serializable }; |
| 3094 | |
| 3095 | export type LoaderFunction = (input: LoaderInput) => ts.Awaitable<LoaderResult>; |
| 3096 | |
| 3097 | /** a module in an import graph, from {@linkcode Build#trackModule} */ |
| 3098 | export type Module = { |
| 3099 | url: string; |
| 3100 | /** `external` modules are in `node_modules`, and loaded by node.js */ |
| 3101 | format: "js" | "cjs" | "external" | "serializable" | "builtin"; |
| 3102 | /** urls of the static imports, in order */ |
| 3103 | imports: string[]; |
| 3104 | /** changes whenever the module's code does */ |
| 3105 | hash: string; |
| 3106 | /** from the {@linkcode LoaderResult} */ |
| 3107 | data: Serializable; |
| 3108 | }; |
| 3109 | |
| 3110 | /** the value of a step that loads a module */ |
| 3111 | type Loaded = |
| 3112 | | { |
| 3113 | format: "js"; |
| 3114 | code: string; |
| 3115 | requests: Array<{ specifier: string; attributes: Attributes }>; |
| 3116 | map: string | null; |
| 3117 | data: Serializable; |
| 3118 | } |
| 3119 | | { format: "cjs"; source: string; data: Serializable } |
| 3120 | | { format: "external"; source: string; version: string; data: Serializable } |
| 3121 | | { format: "serializable"; value: Serializable; data: Serializable }; |
| 3122 | |
| 3123 | type Attributes = { [key: string]: string | undefined }; |
| 3124 | |
| 3125 | /** the module each loader from {@linkcode lazyLoader} imports */ |
| 3126 | const loaderModules = new WeakMap<LoaderFunction, string>(); |
| 3127 | |
| 3128 | /** |
| 3129 | * a loader that is the default export of another module, which the graph |
| 3130 | * imports on first use. that module and what it imports are tracked as the |
| 3131 | * loader's code, and reload like any other module of the graph. |
| 3132 | */ |
| 3133 | export function lazyLoader(specifier: string | URL): LoaderFunction { |
| 3134 | const href = toUrl(String(specifier), process.cwd(), Fs.default); |
| 3135 | const loader: LoaderFunction = async (input) => |
| 3136 | (await input.build.import<{ default: LoaderFunction }>(href)).default(input); |
| 3137 | loaderModules.set(loader, href); |
| 3138 | return loader; |
| 3139 | } |
| 3140 | |
| 3141 | /** |
| 3142 | * run `fn` when the module is discarded, so its listeners and timers do not |
| 3143 | * leak into the next generation. this is the dispose half of vite's |
| 3144 | * `import.meta.hot`, and does nothing where modules are never discarded. |
| 3145 | * |
| 3146 | * ```ts |
| 3147 | * const timer = setInterval(poll, 1000); |
| 3148 | * incremental.onDispose(import.meta, () => clearInterval(timer)); |
| 3149 | * ``` |
| 3150 | */ |
| 3151 | export function onDispose(meta: ImportMeta, fn: () => void): void { |
| 3152 | (meta as { hot?: { dispose(fn: () => void): void } }).hot?.dispose(fn); |
| 3153 | } |
| 3154 | |
| 3155 | const defaultLoaders: Record<string, LoaderFunction> = { |
| 3156 | ".js": async ({ source, getPackageMeta }) => { |
| 3157 | const type = (await getPackageMeta())?.type; |
| 3158 | // like node.js, files without a `"type"` are CommonJS unless they use module syntax |
| 3159 | return type === "module" || (type == null && containsModuleSyntax(source)) |
| 3160 | ? { format: "js", content: source } |
| 3161 | : { format: "cjs" }; |
| 3162 | }, |
| 3163 | ".ts": async ({ source, getPackageMeta }) => { |
| 3164 | const type = (await getPackageMeta())?.type; |
| 3165 | return type === "module" || (type == null && containsModuleSyntax(stripTypes(source))) |
| 3166 | ? { format: "ts", content: source } |
| 3167 | : { format: "cjs" }; |
| 3168 | }, |
| 3169 | ".mjs": ({ source }) => ({ format: "js", content: source }), |
| 3170 | ".mts": ({ source }) => ({ format: "ts", content: source }), |
| 3171 | ".cjs": () => ({ format: "cjs" }), |
| 3172 | ".cts": () => ({ format: "cjs" }), |
| 3173 | ".node": () => ({ format: "cjs" }), |
| 3174 | ".json": ({ source }) => ({ format: "serializable", content: JSON.parse(source) as Serializable }), |
| 3175 | }; |
| 3176 | |
| 3177 | function containsModuleSyntax(source: string): boolean { |
| 3178 | try { |
| 3179 | vm.compileFunction(source, ["exports", "require", "module", "__filename", "__dirname"]); |
| 3180 | return false; |
| 3181 | } catch (error) { |
| 3182 | return error instanceof SyntaxError |
| 3183 | && /\b(import|export)\b|import\.meta|top level bodies of modules/.test(error.message); |
| 3184 | } |
| 3185 | } |
| 3186 | |
| 3187 | /** like node.js, json is imported with `type: "json"`, and nothing else is */ |
| 3188 | function checkAttributes(target: string, attributes: Attributes): void { |
| 3189 | for (const [key, value] of Object.entries(attributes)) { |
| 3190 | if (value === undefined || (key === "type" && value === "json")) continue; |
| 3191 | exception.rethrowWithMetadata( |
| 3192 | new Error(`import attribute "${key}" with value "${value}" is not supported`), |
| 3193 | { code: "ERR_IMPORT_ATTRIBUTE_UNSUPPORTED" }, |
| 3194 | ); |
| 3195 | } |
| 3196 | // a glob of json files is still code |
| 3197 | const json = target.endsWith(".json") && !target.startsWith("glob:"); |
| 3198 | if (json && attributes.type !== "json") { |
| 3199 | exception.rethrowWithMetadata( |
| 3200 | new Error(`module "${target}" needs an import attribute of "type: json"`), |
| 3201 | { code: "ERR_IMPORT_ATTRIBUTE_MISSING" }, |
| 3202 | ); |
| 3203 | } |
| 3204 | if (!json && attributes.type === "json") { |
| 3205 | exception.rethrowWithMetadata( |
| 3206 | new Error(`module "${target}" is not of type "json"`), |
| 3207 | { code: "ERR_IMPORT_ATTRIBUTE_TYPE_INCOMPATIBLE" }, |
| 3208 | ); |
| 3209 | } |
| 3210 | } |
| 3211 | |
| 3212 | /** `module.stripTypeScriptTypes`, whose syntax errors say where only in the first lines of their stack */ |
| 3213 | function stripTypes(source: string): string { |
| 3214 | try { |
| 3215 | return module.stripTypeScriptTypes(source); |
| 3216 | } catch (error) { |
| 3217 | const [, line, indent, carets] = /^:(\d+)\n[^]*?\n( *)(\^+)\n/.exec((error as Error).stack ?? "") ?? []; |
| 3218 | if (!line) throw error; |
| 3219 | Object.assign(error as Error, { line: Number(line), col: indent!.length + 1, width: carets!.length }); |
| 3220 | // what is left of the stack is node's own |
| 3221 | (error as Error).stack = String(error); |
| 3222 | throw error; |
| 3223 | } |
| 3224 | } |
| 3225 | |
| 3226 | function inlineSourceMap(code: string): string | null { |
| 3227 | const match = code.match( |
| 3228 | /\/\/[#@] sourceMappingURL=data:application\/json(?:;charset=[^;,]+)?;base64,([A-Za-z0-9+/=]+)\s*$/, |
| 3229 | ); |
| 3230 | return match ? Buffer.from(match[1]!, "base64").toString("utf-8") : null; |
| 3231 | } |
| 3232 | |
| 3233 | function isNodeModules(file: string): boolean { |
| 3234 | return file.includes(`${path.sep}node_modules${path.sep}`); |
| 3235 | } |
| 3236 | |
| 3237 | /** the `package.json` at the root of the package a file is in */ |
| 3238 | function packageJsonOf(file: string): string { |
| 3239 | const marker = `${path.sep}node_modules${path.sep}`; |
| 3240 | const start = file.lastIndexOf(marker) + marker.length; |
| 3241 | const parts = file.slice(start).split(path.sep); |
| 3242 | const name = parts[0]!.startsWith("@") ? parts.slice(0, 2) : parts.slice(0, 1); |
| 3243 | return path.join(file.slice(0, start), ...name, "package.json"); |
| 3244 | } |
| 3245 | |
| 3246 | /** accepts paths relative to `root`, absolute paths, and file urls, giving the url of the real path like imports do */ |
| 3247 | function toUrl(file: string, root: string, fs: Fs): string { |
| 3248 | const abs = file.startsWith("file:") ? url.fileURLToPath(file) : path.resolve(root, file); |
| 3249 | return url.pathToFileURL(realPath(fs, abs)).href; |
| 3250 | } |
| 3251 | |
| 3252 | /** a missing path stays as it is, so reading it records where it was looked for */ |
| 3253 | function realPath(fs: Fs, file: string): string { |
| 3254 | try { |
| 3255 | return fs.realPathSync(file); |
| 3256 | } catch (error) { |
| 3257 | if (!isMissing(error)) throw error; |
| 3258 | return file; |
| 3259 | } |
| 3260 | } |
| 3261 | |
| 3262 | function dirUrl(dir: string): string { |
| 3263 | return url.pathToFileURL(path.join(dir, "/")).href; |
| 3264 | } |
| 3265 | |
| 3266 | /** |
| 3267 | * a step's key, from where it was started, its code, and its arguments, and |
| 3268 | * its work, which depends on the code that started it. for `Ref#step` and |
| 3269 | * `Graph#lazy`. |
| 3270 | */ |
| 3271 | function begin<Args extends unknown[]>( |
| 3272 | run: Run, |
| 3273 | caller: stack.Frame | undefined, |
| 3274 | fn: Builder<Args, unknown>, |
| 3275 | args: Args, |
| 3276 | ) { |
| 3277 | const code = frameUrl(caller); |
| 3278 | const where = !code ? "?" : memo(run.callers, code, () => { |
| 3279 | // the code that starts a step keeps running after it changes, unless the |
| 3280 | // graph reloads it, so it is pinned even when the step is reused |
| 3281 | run.internal(closureKey(code)).then((closure) => { |
| 3282 | if (!closure.error) run.graph[kNative](code, closure.value() as Module[]); |
| 3283 | }, () => {}); |
| 3284 | // relative, so the key is the same wherever the project is |
| 3285 | return path.relative(run.graph.root, url.fileURLToPath(code)); |
| 3286 | }); |
| 3287 | const key = hashSerializable(where, caller?.line ?? null, caller?.col ?? null, fn.toString(), ...args); |
| 3288 | const label = `${fn.name || "step"} at ${where}:${caller?.line}`; |
| 3289 | const compute = async (b: Build) => { |
| 3290 | if (code) { |
| 3291 | const closure = await b[kDepend](closureKey(code)); |
| 3292 | // the code that runs is not what the step records, so the next check runs it again |
| 3293 | if (!closure.error && run.graph[kStale](code, closure.value() as Module[])) { |
| 3294 | b[kObserve]({ url: code, kind: "file", time: 0, size: 0, hash: "" }); |
| 3295 | } |
| 3296 | } |
| 3297 | // a step that only reports errors returns nothing, which is stored as undefined |
| 3298 | return await run.turn(() => fn(b, ...args)) as Serializable; |
| 3299 | }; |
| 3300 | return { key, label, compute }; |
| 3301 | } |
| 3302 | |
| 3303 | /** the frame that called `fn`, which costs much less than a whole stack does */ |
| 3304 | function callerOf(fn: (...args: never[]) => unknown): stack.Frame | undefined { |
| 3305 | const { prepareStackTrace, stackTraceLimit } = Error; |
| 3306 | const holder: { stack?: NodeJS.CallSite[] } = {}; |
| 3307 | try { |
| 3308 | Error.prepareStackTrace = (_, sites) => sites; |
| 3309 | Error.stackTraceLimit = 1; |
| 3310 | Error.captureStackTrace(holder, fn); |
| 3311 | const site = holder.stack?.[0]; |
| 3312 | return site && { |
| 3313 | fn: site.getFunctionName(), |
| 3314 | file: site.getFileName() ?? null, |
| 3315 | line: site.getLineNumber(), |
| 3316 | col: site.getColumnNumber(), |
| 3317 | }; |
| 3318 | } finally { |
| 3319 | Error.prepareStackTrace = prepareStackTrace; |
| 3320 | Error.stackTraceLimit = stackTraceLimit; |
| 3321 | } |
| 3322 | } |
| 3323 | |
| 3324 | /** the file a stack frame points into, as a url */ |
| 3325 | function frameUrl(frame: stack.Frame | undefined): string | null { |
| 3326 | const file = frame?.file; |
| 3327 | if (!file) return null; |
| 3328 | if (file.startsWith("file:")) return file; |
| 3329 | return path.isAbsolute(file) ? url.pathToFileURL(file).href : null; |
| 3330 | } |
| 3331 | |
| 3332 | /** the module this file is, which loaded code shares instead of copying */ |
| 3333 | const self = import.meta.url; |
| 3334 | |
| 3335 | // --- module resolution --- |
| 3336 | |
| 3337 | export type PackageMeta = { |
| 3338 | type: "module" | "commonjs" | null; |
| 3339 | name: string | null; |
| 3340 | imports: Serializable; |
| 3341 | exports: Serializable; |
| 3342 | }; |
| 3343 | |
| 3344 | /** what the resolver reads, which is synchronous for `import.meta.resolve` */ |
| 3345 | export type ResolverFs = Pick<Fs, "statSync" | "readFileSync" | "realPathSync">; |
| 3346 | |
| 3347 | export interface ResolveOptions { |
| 3348 | specifier: string; |
| 3349 | resolveDir: string; |
| 3350 | /** override the filesystem implementation */ |
| 3351 | fs?: ResolverFs; |
| 3352 | /** |
| 3353 | * resolve like `require`, with its conditions, extensions, and directory indexes |
| 3354 | * @default false |
| 3355 | */ |
| 3356 | sloppy?: boolean; |
| 3357 | /** replaces node.js's conditions for `import`, or for `require` when sloppy */ |
| 3358 | conditions?: string[]; |
| 3359 | /** caller-owned shared package.json metadata cache */ |
| 3360 | packageJsonCache?: Map<string, PackageMeta>; |
| 3361 | } |
| 3362 | |
| 3363 | export type Resolution = |
| 3364 | & { |
| 3365 | /** the files that could have affected this resolution */ |
| 3366 | files: string[]; |
| 3367 | } |
| 3368 | & ( |
| 3369 | | { |
| 3370 | type: "success"; |
| 3371 | namespace: string; |
| 3372 | subPath: string; |
| 3373 | error?: undefined; |
| 3374 | } |
| 3375 | | { |
| 3376 | type: "error"; |
| 3377 | error: unknown; |
| 3378 | namespace?: undefined; |
| 3379 | subPath?: undefined; |
| 3380 | } |
| 3381 | ); |
| 3382 | |
| 3383 | /** like node.js, `default` always matches */ |
| 3384 | const importConditions = ["node", "import", "module-sync", "node-addons"]; |
| 3385 | const requireConditions = ["node", "require", "module-sync", "node-addons"]; |
| 3386 | const sloppyExtensions = [".js", ".json", ".node"]; |
| 3387 | /** like node.js, `/`, `./`, `../`, `.`, and `..` start paths instead of package names */ |
| 3388 | const pathSpecifier = /^(\/|\.\.?(\/|$))/; |
| 3389 | /** `.`, `..`, and `node_modules` path segments, percent-encoded or not */ |
| 3390 | const invalidSegment = |
| 3391 | /(^|[\\/])((\.|%2e){1,2}|(n|%[46]e)(o|%[46]f)(d|%[46]4)(e|%[46]5)(_|%5f)(m|%[46]d)(o|%[46]f)(d|%[46]4)(u|%[57]5)(l|%[46]c)(e|%[46]5)(s|%[57]3))([\\/]|$)/i; |
| 3392 | |
| 3393 | /** |
| 3394 | * perform filesystem-based module resolution of paths, `file:` urls, packages, |
| 3395 | * and builtins, like node.js resolves `import`, or `require` when sloppy. |
| 3396 | * the difference between this and something like `require.resolve` is it |
| 3397 | * returns the full list of paths that could affect the outcome of the |
| 3398 | * resolution. |
| 3399 | */ |
| 3400 | export function resolve({ |
| 3401 | specifier, |
| 3402 | resolveDir, |
| 3403 | fs = Fs.default, |
| 3404 | sloppy = false, |
| 3405 | conditions = sloppy ? requireConditions : importConditions, |
| 3406 | packageJsonCache = new Map<string, PackageMeta>(), |
| 3407 | }: ResolveOptions): Resolution { |
| 3408 | const r = new Resolver(fs, conditions, packageJsonCache); |
| 3409 | try { |
| 3410 | const { namespace, subPath } = r.resolve(specifier, resolveDir, sloppy); |
| 3411 | return { type: "success", namespace, subPath, files: Array.from(r.files) }; |
| 3412 | } catch (error) { |
| 3413 | return { type: "error", error, files: Array.from(r.files) }; |
| 3414 | } |
| 3415 | } |
| 3416 | |
| 3417 | /** sanity class to track file read states during module resolution */ |
| 3418 | class Resolver { |
| 3419 | fs: ResolverFs; |
| 3420 | files = new Set<string>(); |
| 3421 | conditions: string[]; |
| 3422 | packageJsonCache: Map<string, PackageMeta>; |
| 3423 | |
| 3424 | constructor( |
| 3425 | fs: ResolverFs, |
| 3426 | conditions: string[], |
| 3427 | packageJsonCache = new Map<string, PackageMeta>(), |
| 3428 | ) { |
| 3429 | this.fs = fs; |
| 3430 | this.conditions = conditions; |
| 3431 | this.packageJsonCache = packageJsonCache; |
| 3432 | } |
| 3433 | |
| 3434 | resolve( |
| 3435 | specifier: string, |
| 3436 | resolveDir: string, |
| 3437 | sloppy: boolean, |
| 3438 | ): { namespace: string; subPath: string } { |
| 3439 | const isNodeBuiltin = specifier.startsWith("node:"); |
| 3440 | if (isNodeBuiltin || module.builtinModules.includes(specifier)) { |
| 3441 | return { |
| 3442 | namespace: "node", |
| 3443 | subPath: isNodeBuiltin ? specifier.slice(5) : specifier, |
| 3444 | }; |
| 3445 | } |
| 3446 | |
| 3447 | if (pathSpecifier.test(specifier) || (!sloppy && specifier.startsWith("file:"))) { |
| 3448 | // es modules import urls, which can be percent-encoded |
| 3449 | const abs = sloppy |
| 3450 | ? path.resolve(resolveDir, specifier) |
| 3451 | : url.fileURLToPath(new URL(specifier, url.pathToFileURL(path.join(resolveDir, "/")))); |
| 3452 | const result = sloppy |
| 3453 | ? this.resolvePathSloppy(abs) |
| 3454 | : this.resolvePathExact(abs); |
| 3455 | return { namespace: "file", subPath: this.fs.realPathSync(result) }; |
| 3456 | } |
| 3457 | |
| 3458 | if (specifier.startsWith("#")) { |
| 3459 | const packageJsonPath = this.findClosestPackageJson(resolveDir); |
| 3460 | if (!packageJsonPath) { |
| 3461 | this.throwWithCode( |
| 3462 | `Cannot find package.json for import ${JSON.stringify(specifier)} from ${resolveDir}`, |
| 3463 | "ERR_PACKAGE_IMPORT_NOT_DEFINED", |
| 3464 | ); |
| 3465 | } |
| 3466 | const meta = this.readPackageJsonMeta(packageJsonPath); |
| 3467 | const target = this.resolvePackageImportCondition( |
| 3468 | meta.imports, |
| 3469 | specifier, |
| 3470 | ); |
| 3471 | if (!target) { |
| 3472 | this.throwWithCode( |
| 3473 | `Package import specifier ${JSON.stringify(specifier)} is not defined in ${packageJsonPath}`, |
| 3474 | "ERR_PACKAGE_IMPORT_NOT_DEFINED", |
| 3475 | ); |
| 3476 | } |
| 3477 | const packageDir = path.dirname(packageJsonPath); |
| 3478 | // like node.js, `imports` can map to packages and builtins |
| 3479 | if (!target.startsWith("./")) return this.resolve(target, packageDir, sloppy); |
| 3480 | return { |
| 3481 | namespace: "file", |
| 3482 | subPath: this.fs.realPathSync(this.resolvePackageTarget(packageDir, target)), |
| 3483 | }; |
| 3484 | } |
| 3485 | |
| 3486 | if (!this.isBareSpecifier(specifier)) { |
| 3487 | this.throwWithCode( |
| 3488 | `Invalid module specifier ${JSON.stringify(specifier)}`, |
| 3489 | "ERR_INVALID_MODULE_SPECIFIER", |
| 3490 | ); |
| 3491 | } |
| 3492 | |
| 3493 | const resolved = this.resolveBareSpecifier( |
| 3494 | specifier, |
| 3495 | resolveDir, |
| 3496 | sloppy, |
| 3497 | ); |
| 3498 | return { namespace: "file", subPath: this.fs.realPathSync(resolved) }; |
| 3499 | } |
| 3500 | |
| 3501 | stat(file: string): Stat | null { |
| 3502 | this.files.add(file); |
| 3503 | try { |
| 3504 | return this.fs.statSync(file); |
| 3505 | } catch (error) { |
| 3506 | if (isMissing(error)) return null; |
| 3507 | throw error; |
| 3508 | } |
| 3509 | } |
| 3510 | |
| 3511 | readJsonObject(file: string): Record<string, unknown> { |
| 3512 | this.files.add(file); |
| 3513 | return JSON.parse(new TextDecoder().decode(this.fs.readFileSync(file))) as Record< |
| 3514 | string, |
| 3515 | unknown |
| 3516 | >; |
| 3517 | } |
| 3518 | |
| 3519 | readPackageJsonMeta(file: string): PackageMeta { |
| 3520 | this.files.add(file); |
| 3521 | const cached = this.packageJsonCache.get(file); |
| 3522 | if (cached) return cached; |
| 3523 | |
| 3524 | const json = this.readJsonObject(file); |
| 3525 | const meta: PackageMeta = { |
| 3526 | type: json.type === "module" || json.type === "commonjs" |
| 3527 | ? json.type |
| 3528 | : null, |
| 3529 | name: typeof json.name === "string" ? json.name : null, |
| 3530 | imports: json.imports as Serializable, |
| 3531 | exports: json.exports as Serializable, |
| 3532 | }; |
| 3533 | this.packageJsonCache.set(file, meta); |
| 3534 | return meta; |
| 3535 | } |
| 3536 | |
| 3537 | resolvePathExact(abs: string): string { |
| 3538 | const stat = this.stat(abs); |
| 3539 | if (!stat) { |
| 3540 | this.throwWithCode( |
| 3541 | `Cannot find module ${JSON.stringify(abs)}`, |
| 3542 | "ERR_MODULE_NOT_FOUND", |
| 3543 | ); |
| 3544 | } |
| 3545 | if (stat.kind === "directory") { |
| 3546 | this.throwWithCode( |
| 3547 | `Directory import ${JSON.stringify(abs)} is not supported`, |
| 3548 | "ERR_UNSUPPORTED_DIR_IMPORT", |
| 3549 | ); |
| 3550 | } |
| 3551 | if (stat.kind !== "file") { |
| 3552 | this.throwWithCode( |
| 3553 | `Cannot load module ${JSON.stringify(abs)}`, |
| 3554 | "ERR_MODULE_NOT_FOUND", |
| 3555 | ); |
| 3556 | } |
| 3557 | return abs; |
| 3558 | } |
| 3559 | |
| 3560 | resolvePathSloppy(abs: string): string { |
| 3561 | for (const candidate of [abs, ...sloppyExtensions.map((ext) => abs + ext)]) { |
| 3562 | if (this.stat(candidate)?.kind === "file") return candidate; |
| 3563 | } |
| 3564 | |
| 3565 | // like node.js, `require` reads only the `main` of a directory's package.json |
| 3566 | const packageJson = path.join(abs, "package.json"); |
| 3567 | const main = this.stat(packageJson)?.kind === "file" ? this.readJsonObject(packageJson).main : null; |
| 3568 | return this.resolveMain(abs, main); |
| 3569 | } |
| 3570 | |
| 3571 | /** like node.js, `main` can leave off its extension or be a directory, and falls back to the index */ |
| 3572 | resolveMain(dir: string, main: unknown): string { |
| 3573 | const mains = typeof main === "string" && main |
| 3574 | ? [ |
| 3575 | main, |
| 3576 | ...sloppyExtensions.map((ext) => main + ext), |
| 3577 | ...sloppyExtensions.map((ext) => path.join(main, `index${ext}`)), |
| 3578 | ] |
| 3579 | : []; |
| 3580 | for (const candidate of [...mains, ...sloppyExtensions.map((ext) => `index${ext}`)]) { |
| 3581 | const file = path.resolve(dir, candidate); |
| 3582 | if (this.stat(file)?.kind === "file") return file; |
| 3583 | } |
| 3584 | |
| 3585 | this.throwWithCode( |
| 3586 | `Cannot find module ${JSON.stringify(dir)}`, |
| 3587 | "ERR_MODULE_NOT_FOUND", |
| 3588 | ); |
| 3589 | } |
| 3590 | |
| 3591 | findClosestPackageJson(startDir: string): string | null { |
| 3592 | for (const dir of walkUpDirectories(startDir)) { |
| 3593 | const candidate = path.join(dir, "package.json"); |
| 3594 | const stat = this.stat(candidate); |
| 3595 | if (stat?.kind === "file") return candidate; |
| 3596 | } |
| 3597 | return null; |
| 3598 | } |
| 3599 | |
| 3600 | resolveBareSpecifier( |
| 3601 | specifier: string, |
| 3602 | resolveDir: string, |
| 3603 | sloppy: boolean, |
| 3604 | ): string { |
| 3605 | const { packageName, packageSubpath } = this.parseBareSpecifier(specifier); |
| 3606 | const subpath = packageSubpath ? `./${packageSubpath}` : "."; |
| 3607 | |
| 3608 | // like node.js, a package with `exports` can import itself by its name |
| 3609 | const selfJsonPath = this.findClosestPackageJson(resolveDir); |
| 3610 | if (selfJsonPath) { |
| 3611 | const self = this.readPackageJsonMeta(selfJsonPath); |
| 3612 | if (self.name === packageName && self.exports != null) { |
| 3613 | return this.resolvePackageExports(path.dirname(selfJsonPath), self.exports, subpath); |
| 3614 | } |
| 3615 | } |
| 3616 | |
| 3617 | for (const dir of walkUpDirectories(resolveDir)) { |
| 3618 | // like node.js, a package is a directory, with or without a package.json |
| 3619 | const packageDir = path.join(dir, "node_modules", packageName); |
| 3620 | if (this.stat(packageDir)?.kind !== "directory") continue; |
| 3621 | const packageJsonPath = path.join(packageDir, "package.json"); |
| 3622 | const hasPackageJson = this.stat(packageJsonPath)?.kind === "file"; |
| 3623 | const meta = hasPackageJson ? this.readPackageJsonMeta(packageJsonPath) : null; |
| 3624 | |
| 3625 | if (meta?.exports != null) { |
| 3626 | return this.resolvePackageExports(packageDir, meta.exports, subpath); |
| 3627 | } |
| 3628 | |
| 3629 | if (subpath !== ".") { |
| 3630 | // es modules import urls, which can be percent-encoded |
| 3631 | return sloppy |
| 3632 | ? this.resolvePathSloppy(path.join(packageDir, packageSubpath)) |
| 3633 | : this.resolvePathExact(url.fileURLToPath(new URL(subpath, dirUrl(packageDir)))); |
| 3634 | } |
| 3635 | |
| 3636 | const json = hasPackageJson ? this.readJsonObject(packageJsonPath) : {}; |
| 3637 | return this.resolveMain(packageDir, json.main); |
| 3638 | } |
| 3639 | |
| 3640 | this.throwWithCode( |
| 3641 | `Cannot find package ${JSON.stringify(packageName)}`, |
| 3642 | "ERR_MODULE_NOT_FOUND", |
| 3643 | ); |
| 3644 | } |
| 3645 | |
| 3646 | resolvePackageExports(packageDir: string, exports: Serializable, subpath: string): string { |
| 3647 | const target = this.resolvePackageImportCondition(exports, subpath); |
| 3648 | if (!target) { |
| 3649 | this.throwWithCode( |
| 3650 | `Package subpath ${JSON.stringify(subpath)} is not exported by ${packageDir}`, |
| 3651 | "ERR_PACKAGE_PATH_NOT_EXPORTED", |
| 3652 | ); |
| 3653 | } |
| 3654 | return this.resolvePackageTarget(packageDir, target); |
| 3655 | } |
| 3656 | |
| 3657 | /** like node.js, targets are exact urls, even for `require` */ |
| 3658 | resolvePackageTarget(packageDir: string, target: string): string { |
| 3659 | return this.resolvePathExact(url.fileURLToPath(new URL(target, dirUrl(packageDir)))); |
| 3660 | } |
| 3661 | |
| 3662 | isBareSpecifier(specifier: string): boolean { |
| 3663 | return ( |
| 3664 | !specifier.startsWith(".") |
| 3665 | && !specifier.startsWith("/") |
| 3666 | && !specifier.startsWith("#") |
| 3667 | ); |
| 3668 | } |
| 3669 | |
| 3670 | parseBareSpecifier(specifier: string): { |
| 3671 | packageName: string; |
| 3672 | packageSubpath: string; |
| 3673 | } { |
| 3674 | const parts = specifier.split("/"); |
| 3675 | const packageName = specifier.startsWith("@") |
| 3676 | ? parts.slice(0, 2).join("/") |
| 3677 | : (parts[0] ?? ""); |
| 3678 | const packageSubpath = specifier |
| 3679 | .slice(packageName.length) |
| 3680 | .replace(/^\/+/, ""); |
| 3681 | return { packageName, packageSubpath }; |
| 3682 | } |
| 3683 | |
| 3684 | resolvePackageImportCondition( |
| 3685 | map: unknown, |
| 3686 | specifier: string, |
| 3687 | ): string | null { |
| 3688 | if (map !== null && typeof map === "object" && !Array.isArray(map)) { |
| 3689 | const keys = Object.keys(map as object); |
| 3690 | const hasPathKeys = keys.some( |
| 3691 | (k) => k.startsWith(".") || k.startsWith("#"), |
| 3692 | ); |
| 3693 | if (hasPathKeys) { |
| 3694 | return this.resolveSubpathMap( |
| 3695 | map as Record<string, unknown>, |
| 3696 | specifier, |
| 3697 | ); |
| 3698 | } |
| 3699 | } |
| 3700 | // a target without subpaths is what the package itself exports |
| 3701 | return specifier === "." ? this.resolveTarget(map, null, false) : null; |
| 3702 | } |
| 3703 | |
| 3704 | private resolveSubpathMap( |
| 3705 | matchObj: Record<string, unknown>, |
| 3706 | matchKey: string, |
| 3707 | ): string | null { |
| 3708 | // like node.js, `imports` can map to other packages, and `exports` can't |
| 3709 | const imports = matchKey.startsWith("#"); |
| 3710 | if (Object.hasOwn(matchObj, matchKey)) { |
| 3711 | return this.resolveTarget(matchObj[matchKey], null, imports); |
| 3712 | } |
| 3713 | |
| 3714 | // like node.js, the longest base wins, then the longest key, and keys have one `*` |
| 3715 | const patternKeys = Object.keys(matchObj) |
| 3716 | .filter((k) => k.includes("*") && k.indexOf("*") === k.lastIndexOf("*")) |
| 3717 | .sort((a, b) => b.indexOf("*") - a.indexOf("*") || b.length - a.length); |
| 3718 | |
| 3719 | for (const expansionKey of patternKeys) { |
| 3720 | const starIdx = expansionKey.indexOf("*"); |
| 3721 | const patternBase = expansionKey.slice(0, starIdx); |
| 3722 | const patternTrailer = expansionKey.slice(starIdx + 1); |
| 3723 | |
| 3724 | if (!matchKey.startsWith(patternBase)) continue; |
| 3725 | if (matchKey.length < expansionKey.length) continue; |
| 3726 | if (patternTrailer.length > 0 && !matchKey.endsWith(patternTrailer)) { |
| 3727 | continue; |
| 3728 | } |
| 3729 | |
| 3730 | const patternMatch = matchKey.slice( |
| 3731 | patternBase.length, |
| 3732 | matchKey.length - patternTrailer.length, |
| 3733 | ); |
| 3734 | return this.resolveTarget(matchObj[expansionKey], patternMatch, imports); |
| 3735 | } |
| 3736 | |
| 3737 | return null; |
| 3738 | } |
| 3739 | |
| 3740 | private resolveTarget( |
| 3741 | target: unknown, |
| 3742 | patternMatch: string | null, |
| 3743 | imports: boolean, |
| 3744 | ): string | null { |
| 3745 | if (typeof target === "string") { |
| 3746 | const expanded = patternMatch !== null |
| 3747 | ? target.replaceAll("*", patternMatch) |
| 3748 | : target; |
| 3749 | if (imports && this.isBareSpecifier(target) && !URL.canParse(target)) { |
| 3750 | return expanded; |
| 3751 | } |
| 3752 | // like node.js, a target can't leave its package or enter `node_modules` |
| 3753 | if (!target.startsWith("./") || invalidSegment.test(target.slice(2))) { |
| 3754 | this.throwWithCode( |
| 3755 | `invalid target: "${target}"`, |
| 3756 | "ERR_INVALID_PACKAGE_TARGET", |
| 3757 | ); |
| 3758 | } |
| 3759 | if (patternMatch !== null && invalidSegment.test(patternMatch)) { |
| 3760 | this.throwWithCode( |
| 3761 | `invalid subpath: "${patternMatch}" in target "${target}"`, |
| 3762 | "ERR_INVALID_MODULE_SPECIFIER", |
| 3763 | ); |
| 3764 | } |
| 3765 | return expanded; |
| 3766 | } |
| 3767 | |
| 3768 | if (Array.isArray(target)) { |
| 3769 | for (const item of target) { |
| 3770 | try { |
| 3771 | const result = this.resolveTarget(item, patternMatch, imports); |
| 3772 | if (result !== null) return result; |
| 3773 | } catch { |
| 3774 | // |
| 3775 | } |
| 3776 | } |
| 3777 | return null; |
| 3778 | } |
| 3779 | |
| 3780 | if (target !== null && typeof target === "object") { |
| 3781 | for ( |
| 3782 | const [key, value] of Object.entries( |
| 3783 | target as Record<string, unknown>, |
| 3784 | ) |
| 3785 | ) { |
| 3786 | if (key === "default" || this.conditions.includes(key)) { |
| 3787 | const result = this.resolveTarget(value, patternMatch, imports); |
| 3788 | if (result !== null) return result; |
| 3789 | } |
| 3790 | } |
| 3791 | } |
| 3792 | |
| 3793 | return null; |
| 3794 | } |
| 3795 | |
| 3796 | private throwWithCode(message: string, code: node.ErrorCode): never { |
| 3797 | exception.rethrowWithMetadata(new Error(message), { code }); |
| 3798 | } |
| 3799 | } |
| 3800 | |
| 3801 | function* walkUpDirectories(startDir: string): Generator<string> { |
| 3802 | let current = path.resolve(startDir); |
| 3803 | while (true) { |
| 3804 | yield current; |
| 3805 | const parent = path.dirname(current); |
| 3806 | if (parent === current) break; |
| 3807 | current = parent; |
| 3808 | } |
| 3809 | } |
| 3810 | |
| 3811 | // --- watching --- |
| 3812 | |
| 3813 | /** watches the inputs of finished builds, see {@linkcode Graph#watch} */ |
| 3814 | class Watcher implements Disposable { |
| 3815 | #debounce: number; |
| 3816 | /** whether files are watched, or trusted to never change */ |
| 3817 | #files: boolean; |
| 3818 | #roots = new Map<string, fscb.FSWatcher>(); |
| 3819 | /** kinds of the watched inputs, by url */ |
| 3820 | #inputs = new Map<string, string>(); |
| 3821 | #watching: Watching = { file: 0 }; |
| 3822 | /** urls of the watched files, by path */ |
| 3823 | #paths = new Map<string, string[]>(); |
| 3824 | /** the watched paths in order, where the ones in a directory are next to each other */ |
| 3825 | #sorted: string[] = []; |
| 3826 | #sources: Disposable[] = []; |
| 3827 | /** urls of custom inputs whose source can report changes */ |
| 3828 | #subscribed = new Set<string>(); |
| 3829 | /** the directories that missing paths would be made in, watched by themselves */ |
| 3830 | #shallow = new Map<string, fscb.FSWatcher>(); |
| 3831 | #changed = new Set<string>(); |
| 3832 | #timer: ts.Timer | null = null; |
| 3833 | #wake: (() => void) | null = null; |
| 3834 | /** the project's directory, which the missing paths that are watched are in */ |
| 3835 | #root: string; |
| 3836 | /** how many times changes were reported, and when each url last was */ |
| 3837 | reports = 0; |
| 3838 | #reported = new Map<string, number>(); |
| 3839 | |
| 3840 | constructor(debounce: number, files: boolean, root: string) { |
| 3841 | this.#debounce = debounce; |
| 3842 | this.#files = files; |
| 3843 | this.#root = root; |
| 3844 | } |
| 3845 | |
| 3846 | /** |
| 3847 | * watch the inputs a build checked, and not the ones it left unread, and |
| 3848 | * count them. a build that read the same inputs as the last costs nothing. |
| 3849 | */ |
| 3850 | update(checked: Iterable<Input>, unread: string[], sources: Map<string, Source>): Watching { |
| 3851 | let moved = false; |
| 3852 | for (const input of checked) { |
| 3853 | if (this.#inputs.get(input.url) === input.kind) continue; |
| 3854 | this.#inputs.set(input.url, input.kind); |
| 3855 | moved = true; |
| 3856 | } |
| 3857 | for (const href of unread) moved = this.#inputs.delete(href) || moved; |
| 3858 | if (!moved) return this.#watching; |
| 3859 | |
| 3860 | this.#paths.clear(); |
| 3861 | const dirs = new Set<string>(); |
| 3862 | const missing = new Set<string>(); |
| 3863 | const custom = new Map<string, string[]>(); |
| 3864 | let files = 0; |
| 3865 | for (const [href, kind] of this.#inputs) { |
| 3866 | if (!href.startsWith("file:")) { |
| 3867 | const protocol = href.slice(0, href.indexOf(":")); |
| 3868 | memo(custom, protocol, () => []).push(href); |
| 3869 | continue; |
| 3870 | } |
| 3871 | if (!this.#files) continue; |
| 3872 | if (kind === "file") files++; |
| 3873 | const file = path.resolve(url.fileURLToPath(href)); |
| 3874 | memo(this.#paths, file, () => []).push(href); |
| 3875 | if (kind !== "missing") { |
| 3876 | dirs.add(href.endsWith("/") ? file : path.dirname(file)); |
| 3877 | } else if (isWithin(this.#root, file)) { |
| 3878 | // what makes a missing path makes something in the nearest directory that exists. a probe for a |
| 3879 | // package walks up to the filesystem's root, so only the project's missing paths are watched |
| 3880 | let dir = path.dirname(file); |
| 3881 | while (!fscb.existsSync(dir)) dir = path.dirname(dir); |
| 3882 | missing.add(dir); |
| 3883 | } |
| 3884 | } |
| 3885 | this.#sorted = Array.from(this.#paths.keys()).sort(); |
| 3886 | const roots = Array.from(dirs) |
| 3887 | .sort((a, b) => a.length - b.length) |
| 3888 | .filter((dir, i, all) => !all.slice(0, i).some((root) => isWithin(root, dir))); |
| 3889 | const shallow = Array.from(missing).filter((dir) => !roots.some((root) => isWithin(root, dir))); |
| 3890 | for (const [watched, recursive] of [[this.#roots, roots], [this.#shallow, shallow]] as const) { |
| 3891 | for (const [dir, watcher] of watched) { |
| 3892 | if (recursive.includes(dir)) continue; |
| 3893 | watcher.close(); |
| 3894 | watched.delete(dir); |
| 3895 | } |
| 3896 | for (const dir of recursive) { |
| 3897 | if (watched.has(dir)) continue; |
| 3898 | const watcher = fscb.watch(dir, { recursive: watched === this.#roots }, (_, name) => { |
| 3899 | if (name) this.#event(path.join(dir, name.toString())); |
| 3900 | }); |
| 3901 | watcher.on("error", () => { |
| 3902 | // a watcher that failed hears nothing more, so what it covered may have changed, and the |
| 3903 | // next update watches everything again |
| 3904 | watched.delete(dir); |
| 3905 | this.#inputs.clear(); |
| 3906 | this.#notify(Array.from(this.#paths).filter(([file]) => isWithin(dir, file)).flatMap(([, hrefs]) => hrefs)); |
| 3907 | }); |
| 3908 | watched.set(dir, watcher); |
| 3909 | } |
| 3910 | } |
| 3911 | for (const subscription of this.#sources) subscription[Symbol.dispose](); |
| 3912 | this.#subscribed.clear(); |
| 3913 | this.#sources = Array.from(custom).flatMap(([protocol, urls]) => { |
| 3914 | const source = sources.get(protocol); |
| 3915 | if (!source?.watch) return []; |
| 3916 | for (const href of urls) this.#subscribed.add(href); |
| 3917 | // a source may report params equal to the ones it was given, rather than the same objects |
| 3918 | const hrefs = new Map(urls.map((href) => [decodeURIComponent(href.slice(protocol.length + 1)), href])); |
| 3919 | const notify = (changed?: ts.JsonValue[]) => |
| 3920 | this.#notify(changed?.map((params) => UNWRAP(hrefs.get(stableJson(params)))) ?? urls); |
| 3921 | return [source.watch(Array.from(hrefs.keys(), (json) => JSON.parse(json) as ts.JsonValue), notify)]; |
| 3922 | }); |
| 3923 | |
| 3924 | const watching: Watching = { file: files }; |
| 3925 | for (const href of this.#subscribed) { |
| 3926 | const protocol = href.slice(0, href.indexOf(":")); |
| 3927 | watching[protocol] = (watching[protocol] ?? 0) + 1; |
| 3928 | } |
| 3929 | return this.#watching = watching; |
| 3930 | } |
| 3931 | |
| 3932 | /** whether a change to an input would have been reported since the last `update` */ |
| 3933 | covers(href: string): boolean { |
| 3934 | if (!href.startsWith("file:")) return this.#subscribed.has(href); |
| 3935 | if (!this.#files) return true; |
| 3936 | if (!this.#inputs.has(href)) return false; |
| 3937 | const file = url.fileURLToPath(href); |
| 3938 | return [...this.#roots.keys(), ...this.#shallow.keys()].some((root) => isWithin(root, file)); |
| 3939 | } |
| 3940 | |
| 3941 | /** whether `href` was reported changed since `reports` was read */ |
| 3942 | reportedSince(href: string, reports: number): boolean { |
| 3943 | return (this.#reported.get(href) ?? 0) > reports; |
| 3944 | } |
| 3945 | |
| 3946 | #event(file: string): void { |
| 3947 | // a path changes itself, everything in it, and the listing of its directory |
| 3948 | const urls = [ |
| 3949 | ...this.#paths.get(file) ?? [], |
| 3950 | ...this.#paths.get(path.dirname(file))?.filter((href) => href.endsWith("/")) ?? [], |
| 3951 | ]; |
| 3952 | |
| 3953 | // a build writes its outputs where it is watched, so an event can't look at every path |
| 3954 | const within = file + path.sep; |
| 3955 | let low = 0; |
| 3956 | for (let high = this.#sorted.length; low < high;) { |
| 3957 | const middle = (low + high) >> 1; |
| 3958 | if (this.#sorted[middle]! < within) low = middle + 1; |
| 3959 | else high = middle; |
| 3960 | } |
| 3961 | for (let i = low; this.#sorted[i]?.startsWith(within); i++) urls.push(...UNWRAP(this.#paths.get(this.#sorted[i]!))); |
| 3962 | |
| 3963 | if (urls.length > 0) this.#notify(urls); |
| 3964 | } |
| 3965 | |
| 3966 | #notify(urls: string[]): void { |
| 3967 | this.reports++; |
| 3968 | for (const href of urls) { |
| 3969 | this.#changed.add(href); |
| 3970 | this.#reported.set(href, this.reports); |
| 3971 | } |
| 3972 | if (this.#timer) clearTimeout(this.#timer); |
| 3973 | this.#timer = setTimeout(() => { |
| 3974 | this.#timer = null; |
| 3975 | this.#wake?.(); |
| 3976 | }, this.#debounce); |
| 3977 | } |
| 3978 | |
| 3979 | /** wait for changes, once events stop arriving, or until `signal` aborts */ |
| 3980 | async next(signal?: AbortSignal): Promise<string[]> { |
| 3981 | const abort = () => this.#wake?.(); |
| 3982 | signal?.addEventListener("abort", abort); |
| 3983 | try { |
| 3984 | while (this.#changed.size === 0 || this.#timer) { |
| 3985 | signal?.throwIfAborted(); |
| 3986 | await new Promise<void>((resolve) => this.#wake = resolve); |
| 3987 | } |
| 3988 | } finally { |
| 3989 | signal?.removeEventListener("abort", abort); |
| 3990 | } |
| 3991 | this.#wake = null; |
| 3992 | const changed = Array.from(this.#changed); |
| 3993 | this.#changed.clear(); |
| 3994 | return changed; |
| 3995 | } |
| 3996 | |
| 3997 | [Symbol.dispose](): void { |
| 3998 | for (const watcher of [...this.#roots.values(), ...this.#shallow.values()]) watcher.close(); |
| 3999 | for (const subscription of this.#sources) subscription[Symbol.dispose](); |
| 4000 | if (this.#timer) clearTimeout(this.#timer); |
| 4001 | } |
| 4002 | } |
| 4003 | |
| 4004 | function isWithin(root: string, dir: string): boolean { |
| 4005 | return dir === root || dir.startsWith(root.endsWith(path.sep) ? root : root + path.sep); |
| 4006 | } |
| 4007 | |
| 4008 | /** delete an output, and the directories it leaves empty */ |
| 4009 | async function removeOutput(file: string, root: string): Promise<void> { |
| 4010 | try { |
| 4011 | await fs.rm(file); |
| 4012 | } catch (error) { |
| 4013 | // a newer output may have replaced a directory it was in with a file |
| 4014 | if (!isMissing(error)) throw error; |
| 4015 | } |
| 4016 | for (let dir = path.dirname(file); isWithin(root, dir) && dir !== root; dir = path.dirname(dir)) { |
| 4017 | try { |
| 4018 | await fs.rmdir(dir); |
| 4019 | } catch { |
| 4020 | break; // not empty |
| 4021 | } |
| 4022 | } |
| 4023 | } |
| 4024 | |
| 4025 | // --- database & serialization --- |
| 4026 | export type RowId = number | bigint; |
| 4027 | |
| 4028 | interface StepRow { |
| 4029 | id: RowId; |
| 4030 | key: string; |
| 4031 | label: string; |
| 4032 | hash: string; |
| 4033 | type: "s" | "b" | "j" | "e"; |
| 4034 | errors: string | null; |
| 4035 | } |
| 4036 | |
| 4037 | /** an input url or step key and the hash that was observed, or a file a step wrote and its hash */ |
| 4038 | interface Dependency { |
| 4039 | key: string; |
| 4040 | hash: string; |
| 4041 | } |
| 4042 | |
| 4043 | interface OutputRow { |
| 4044 | path: string; |
| 4045 | hash: string; |
| 4046 | time: number; |
| 4047 | size: number; |
| 4048 | /** the input a copy was made from */ |
| 4049 | source: string | null; |
| 4050 | } |
| 4051 | |
| 4052 | /** dangerous trust-me-bro types to quickly add some type safety to sqlite */ |
| 4053 | type TypedSqliteStatement< |
| 4054 | Input extends object, |
| 4055 | Output extends object = never, |
| 4056 | > = Omit<sqlite.StatementSync, "run" | "get" | "all" | "iterate"> & { |
| 4057 | run(input: Input): sqlite.StatementResultingChanges; |
| 4058 | get(input: Input): Output | undefined; |
| 4059 | all(input: Input): Output[]; |
| 4060 | iterate(input: Input): IterableIterator<Output>; |
| 4061 | }; |
| 4062 | |
| 4063 | function prepareQueries(sql: sqlite.DatabaseSync) { |
| 4064 | const prepare = <I extends object, O extends object = never>(query: string) => |
| 4065 | sql.prepare(query) as unknown as TypedSqliteStatement<I, O>; |
| 4066 | type None = Record<string, never>; |
| 4067 | const inputColumns = "url, kind, time, size, hash"; |
| 4068 | return { |
| 4069 | readLock: prepare<None, { lock: number | null }>("select lock from incremental;"), |
| 4070 | dataVersion: prepare<None, { data_version: number }>("pragma data_version;"), |
| 4071 | swapLock: prepare<{ next: number | null; prev: number | null }>( |
| 4072 | "update incremental set lock = $next where lock is $prev;", |
| 4073 | ), |
| 4074 | readInputs: prepare<None, Input>(`select ${inputColumns} from inputs;`), |
| 4075 | readInput: prepare<{ url: string }, Input>( |
| 4076 | `select ${inputColumns} from inputs where url = $url;`, |
| 4077 | ), |
| 4078 | writeInput: prepare<Input, { id: RowId }>( |
| 4079 | `insert into inputs (${inputColumns}) values ($url, $kind, $time, $size, $hash) ` |
| 4080 | + "on conflict (url) do update set kind = excluded.kind, time = excluded.time, " |
| 4081 | + "size = excluded.size, hash = excluded.hash returning id;", |
| 4082 | ), |
| 4083 | observeInput: prepare<Input, { id: RowId }>( |
| 4084 | `insert into inputs (${inputColumns}) values ($url, $kind, $time, $size, $hash) ` |
| 4085 | + "on conflict (url) do update set url = excluded.url returning id;", |
| 4086 | ), |
| 4087 | readStep: prepare<{ key: string }, StepRow>( |
| 4088 | "select id, key, label, hash, type, errors from steps where key = $key;", |
| 4089 | ), |
| 4090 | readValue: prepare<{ id: RowId }, { key: string; type: StepRow["type"]; value: string | Uint8Array }>( |
| 4091 | "select key, type, value from steps where id = $id;", |
| 4092 | ), |
| 4093 | writeStep: prepare<Omit<StepRow, "id"> & { value: string | Uint8Array }, { id: RowId }>( |
| 4094 | "insert into steps (key, label, hash, type, errors, value) " |
| 4095 | + "values ($key, $label, $hash, $type, $errors, $value) " |
| 4096 | + "on conflict (key) do update set label = excluded.label, hash = excluded.hash, " |
| 4097 | + "type = excluded.type, errors = excluded.errors, value = excluded.value returning id;", |
| 4098 | ), |
| 4099 | keepStep: prepare<{ id: RowId; label: string; errors: string | null }>( |
| 4100 | "update steps set label = $label, errors = $errors where id = $id;", |
| 4101 | ), |
| 4102 | countSteps: prepare<None, { count: number }>("select count(*) as count from steps;"), |
| 4103 | // a lazy step that read a step whose value changed since is not right anymore, nor what read it |
| 4104 | deleteStaleLazy: prepare<None>( |
| 4105 | "with recursive doomed(id) as (select lazy.step from lazy join step_steps on step_steps.step = lazy.step " |
| 4106 | + "join steps on steps.id = step_steps.dependency where step_steps.hash != steps.hash " |
| 4107 | + "union select step_steps.step from step_steps join doomed on step_steps.dependency = doomed.id) " |
| 4108 | + "delete from steps where id in (select id from doomed);", |
| 4109 | ), |
| 4110 | // lazy steps stay, and so does what they read |
| 4111 | deleteOtherSteps: prepare<{ ids: string }>( |
| 4112 | "with recursive kept(id) as (select value from json_each($ids) union select step from lazy " |
| 4113 | + "union select step_steps.dependency from step_steps join kept on step_steps.step = kept.id) " |
| 4114 | + "delete from steps where id not in (select id from kept);", |
| 4115 | ), |
| 4116 | markLazyUsed: prepare<{ step: RowId; used: number }>( |
| 4117 | "insert into lazy (step, used, size) select id, $used, octet_length(value) " |
| 4118 | + "+ coalesce((select sum(octet_length(content)) from blobs where step = id), 0) " |
| 4119 | + "from steps where id = $step " |
| 4120 | + "on conflict (step) do update set used = excluded.used, size = excluded.size;", |
| 4121 | ), |
| 4122 | lazyBytes: prepare<None, { size: number }>("select coalesce(sum(size), 0) as size from lazy;"), |
| 4123 | oldestLazy: prepare<None, { step: RowId }>("select step from lazy order by used limit 1;"), |
| 4124 | // the steps that read a step are only right with it |
| 4125 | deleteWithReaders: prepare<{ step: RowId }>( |
| 4126 | "with recursive doomed(id) as (select $step union select step_steps.step from step_steps " |
| 4127 | + "join doomed on step_steps.dependency = doomed.id) delete from steps where id in (select id from doomed);", |
| 4128 | ), |
| 4129 | readGraphSteps: prepare< |
| 4130 | None, |
| 4131 | { id: RowId; key: string; label: string; type: StepRow["type"]; lazy: number; bytes: number } |
| 4132 | >( |
| 4133 | "select id, key, label, type, exists (select 1 from lazy where step = id) as lazy, octet_length(value) " |
| 4134 | + "+ coalesce((select sum(octet_length(content)) from blobs where step = id), 0) " |
| 4135 | + "+ coalesce((select octet_length(data) from code where step = id), 0) as bytes from steps;", |
| 4136 | ), |
| 4137 | readGraphStepSteps: prepare<None, { step: RowId; key: string }>( |
| 4138 | "select step, key from step_steps join steps on steps.id = step_steps.dependency;", |
| 4139 | ), |
| 4140 | readGraphStepInputs: prepare<None, { step: RowId; url: string }>( |
| 4141 | "select step, url from step_inputs join inputs on inputs.id = step_inputs.input;", |
| 4142 | ), |
| 4143 | readGraphStepOutputs: prepare<None, { step: RowId; url: string }>("select step, path as url from step_outputs;"), |
| 4144 | readGraphInputs: prepare<None, { url: string; kind: string; size: number }>("select url, kind, size from inputs;"), |
| 4145 | readBlobs: prepare<{ step: RowId }, { content: Uint8Array }>( |
| 4146 | "select content from blobs where step = $step order by idx;", |
| 4147 | ), |
| 4148 | deleteBlobs: prepare<{ step: RowId }>("delete from blobs where step = $step;"), |
| 4149 | insertBlob: prepare<{ step: RowId; idx: number; content: Uint8Array }>( |
| 4150 | "insert into blobs (step, idx, content) values ($step, $idx, $content);", |
| 4151 | ), |
| 4152 | readCode: prepare<{ step: RowId }, { data: Uint8Array }>( |
| 4153 | "select data from code where step = $step;", |
| 4154 | ), |
| 4155 | writeCode: prepare<{ step: RowId; data: Uint8Array }>( |
| 4156 | "insert into code (step, data) values ($step, $data) " |
| 4157 | + "on conflict (step) do update set data = excluded.data;", |
| 4158 | ), |
| 4159 | deleteCode: prepare<{ step: RowId }>("delete from code where step = $step;"), |
| 4160 | readStepInputs: prepare<{ step: RowId }, Dependency>( |
| 4161 | "select inputs.url as key, step_inputs.hash from step_inputs " |
| 4162 | + "join inputs on inputs.id = step_inputs.input where step_inputs.step = $step;", |
| 4163 | ), |
| 4164 | readStepSteps: prepare<{ step: RowId }, Dependency>( |
| 4165 | "select steps.key, step_steps.hash from step_steps " |
| 4166 | + "join steps on steps.id = step_steps.dependency where step_steps.step = $step " |
| 4167 | + "order by step_steps.rowid;", |
| 4168 | ), |
| 4169 | readStepOutputs: prepare<{ step: RowId }, Dependency>( |
| 4170 | "select path as key, hash from step_outputs where step = $step;", |
| 4171 | ), |
| 4172 | deleteStepInputs: prepare<{ step: RowId }>("delete from step_inputs where step = $step;"), |
| 4173 | deleteStepSteps: prepare<{ step: RowId }>("delete from step_steps where step = $step;"), |
| 4174 | deleteStepOutputs: prepare<{ step: RowId }>("delete from step_outputs where step = $step;"), |
| 4175 | insertStepInput: prepare<{ step: RowId; input: RowId; hash: string }>( |
| 4176 | "insert into step_inputs (step, input, hash) values ($step, $input, $hash);", |
| 4177 | ), |
| 4178 | insertStepStep: prepare<{ step: RowId; dependency: RowId; hash: string }>( |
| 4179 | "insert into step_steps (step, dependency, hash) values ($step, $dependency, $hash);", |
| 4180 | ), |
| 4181 | insertStepOutput: prepare<{ step: RowId; path: string; hash: string }>( |
| 4182 | "insert into step_outputs (step, path, hash) values ($step, $path, $hash);", |
| 4183 | ), |
| 4184 | readOutput: prepare<{ path: string }, OutputRow>( |
| 4185 | "select path, hash, time, size, source from outputs where path = $path;", |
| 4186 | ), |
| 4187 | writeOutput: prepare<OutputRow>( |
| 4188 | "insert into outputs (path, hash, time, size, source) " |
| 4189 | + "values ($path, $hash, $time, $size, $source) " |
| 4190 | + "on conflict (path) do update set hash = excluded.hash, time = excluded.time, " |
| 4191 | + "size = excluded.size, source = excluded.source;", |
| 4192 | ), |
| 4193 | countOutputs: prepare<None, { count: number }>("select count(*) as count from outputs;"), |
| 4194 | readOutputs: prepare<None, { path: string }>("select path from outputs;"), |
| 4195 | outputSize: prepare<None, { size: number }>("select coalesce(sum(size), 0) as size from outputs;"), |
| 4196 | size: prepare<None, { size: number }>( |
| 4197 | "select page_count * page_size as size from pragma_page_count(), pragma_page_size();", |
| 4198 | ), |
| 4199 | deleteOutput: prepare<{ path: string }>("delete from outputs where path = $path;"), |
| 4200 | deleteStaleInputs: prepare<None, { url: string }>( |
| 4201 | "delete from inputs where id not in (select input from step_inputs) " |
| 4202 | + "and url not in (select source from outputs where source is not null) returning url;", |
| 4203 | ), |
| 4204 | }; |
| 4205 | } |
| 4206 | |
| 4207 | /** backing storage for incremental artifacts built on `node:sqlite` */ |
| 4208 | export class Database { |
| 4209 | sql: sqlite.DatabaseSync; |
| 4210 | #query: ReturnType<typeof prepareQueries>; |
| 4211 | /** |
| 4212 | * the url of the project's directory. urls and paths under it are stored |
| 4213 | * relative to it, so the database can move along with the project. |
| 4214 | */ |
| 4215 | #root: string; |
| 4216 | |
| 4217 | /** |
| 4218 | * the database stores this hash, nuking everything when it changes. it |
| 4219 | * covers this whole file, since any of it can change what is cached, and the |
| 4220 | * node.js version, since it compiles typescript and caches v8 code. |
| 4221 | */ |
| 4222 | static #version = crypto.createHash("sha256") |
| 4223 | .update(fscb.readFileSync(url.fileURLToPath(import.meta.url))) |
| 4224 | .update(process.version) |
| 4225 | .digest("base64url"); |
| 4226 | |
| 4227 | constructor(file: string | null, root: string = process.cwd()) { |
| 4228 | ASSERT(!file?.startsWith(":")); |
| 4229 | this.#root = dirUrl(root); |
| 4230 | if (file) fscb.mkdirSync(path.dirname(file), { recursive: true }); |
| 4231 | this.sql = new sqlite.DatabaseSync(file ?? ":memory:"); |
| 4232 | this.sql.exec( |
| 4233 | "pragma journal_mode = wal; pragma synchronous = normal; " |
| 4234 | + "pragma busy_timeout = 10000; pragma foreign_keys = on;", |
| 4235 | ); |
| 4236 | const version = () => { |
| 4237 | try { |
| 4238 | return (this.sql.prepare("select version from incremental;").get() as { version: string } | undefined) |
| 4239 | ?.version; |
| 4240 | } catch { |
| 4241 | // a new database |
| 4242 | } |
| 4243 | }; |
| 4244 | // the schema is replaced in one write transaction, so a process that stops midway leaves the old one, |
| 4245 | // and of two processes that start together, the second sees the first's |
| 4246 | const stale = version() !== Database.#version; |
| 4247 | if (stale) this.sql.exec("pragma foreign_keys = off; begin immediate;"); |
| 4248 | if (stale && version() !== Database.#version) { |
| 4249 | const tables = this.sql.prepare(` |
| 4250 | select name |
| 4251 | from sqlite_schema |
| 4252 | where type = 'table' |
| 4253 | and name not like 'sqlite_%'; |
| 4254 | `).all() as { name: string }[]; |
| 4255 | this.sql.exec([ |
| 4256 | // destroy an existing schema |
| 4257 | ...tables.map((table) => `drop table ${table.name};`), |
| 4258 | |
| 4259 | // global state in a single row |
| 4260 | "create table incremental (", |
| 4261 | /**/ "version text not null,", |
| 4262 | /**/ "lock integer", // PID of the process owning the lock |
| 4263 | ") strict;", |
| 4264 | `insert into incremental (version) values ('${Database.#version}');`, |
| 4265 | |
| 4266 | // the last known state of each input. this is a memo for cheap checks, |
| 4267 | // never what decides if a step is fresh. `step_inputs` decides that. |
| 4268 | // an input is kept while a step or a copied output refers to it, and |
| 4269 | // watching is watching these. |
| 4270 | "create table inputs (", |
| 4271 | /**/ "id integer primary key,", |
| 4272 | /**/ "url text unique not null,", |
| 4273 | /**/ "kind text not null,", |
| 4274 | /**/ "time real not null,", |
| 4275 | /**/ "size integer not null,", |
| 4276 | /**/ "hash text not null", |
| 4277 | ") strict;", |
| 4278 | |
| 4279 | // a step's result is its value, or the error it threw ('e'), so |
| 4280 | // failures are cached too. `hash` covers either, and is what the steps |
| 4281 | // that read it compare. `key` hashes the call site and arguments, or |
| 4282 | // is `kind:arg` for the steps that load modules. `errors` is a json |
| 4283 | // array of the step's non-fatal errors. `value` is last, so reading |
| 4284 | // the other columns skips it. |
| 4285 | "create table steps (", |
| 4286 | /**/ "id integer primary key,", |
| 4287 | /**/ "key text unique not null,", |
| 4288 | /**/ "label text not null,", |
| 4289 | /**/ "hash text not null,", |
| 4290 | /**/ "type text not null check (type in ('s', 'b', 'j', 'e')),", |
| 4291 | /**/ "errors text,", |
| 4292 | /**/ "value any not null", |
| 4293 | ") strict;", |
| 4294 | |
| 4295 | // bytes nested anywhere in a 'j' value, kept raw instead of encoding |
| 4296 | // them into the json, which holds { $clover: "bytes", index }. |
| 4297 | "create table blobs (", |
| 4298 | /**/ "step integer not null references steps(id) on delete cascade,", |
| 4299 | /**/ "idx integer not null,", |
| 4300 | /**/ "content blob not null,", |
| 4301 | /**/ "primary key (step, idx)", |
| 4302 | ") strict;", |
| 4303 | |
| 4304 | // what each step observed. a step is fresh when every hash here is |
| 4305 | // still current. resolution probes only care if a path exists, so |
| 4306 | // they store a kind ("file", "missing", ...) instead of a content hash. |
| 4307 | "create table step_inputs (", |
| 4308 | /**/ "step integer not null references steps(id) on delete cascade,", |
| 4309 | /**/ "input integer not null references inputs(id),", |
| 4310 | /**/ "hash text not null,", |
| 4311 | /**/ "primary key (step, input)", |
| 4312 | ") strict;", |
| 4313 | "create table step_steps (", |
| 4314 | /**/ "step integer not null references steps(id) on delete cascade,", |
| 4315 | /**/ "dependency integer not null references steps(id),", |
| 4316 | /**/ "hash text not null,", |
| 4317 | /**/ "primary key (step, dependency)", |
| 4318 | ") strict;", |
| 4319 | |
| 4320 | // the files each step wrote. a build that reuses a step keeps them, |
| 4321 | // and runs the step again when one is not as the step left it. |
| 4322 | "create table step_outputs (", |
| 4323 | /**/ "step integer not null references steps(id) on delete cascade,", |
| 4324 | /**/ "path text not null,", |
| 4325 | /**/ "hash text not null,", |
| 4326 | /**/ "primary key (step, path)", |
| 4327 | ") strict;", |
| 4328 | |
| 4329 | // files written by builds. a build checks them against the disk, |
| 4330 | // unless it is a later build of a watch, and they are deleted once a |
| 4331 | // build stops writing them. `source` is the input url a copy was made |
| 4332 | // from. |
| 4333 | "create table outputs (", |
| 4334 | /**/ "path text primary key,", |
| 4335 | /**/ "hash text not null,", |
| 4336 | /**/ "time real not null,", |
| 4337 | /**/ "size integer not null,", |
| 4338 | /**/ "source text", |
| 4339 | ") strict;", |
| 4340 | |
| 4341 | // steps that ran at request time, from `Graph#lazy`. a build keeps |
| 4342 | // them and what they read, though it did not use them, unless it |
| 4343 | // changed a step they read. they go, the ones used least recently |
| 4344 | // first, once their values take more bytes than the graph allows. |
| 4345 | "create table lazy (", |
| 4346 | /**/ "step integer primary key references steps(id) on delete cascade,", |
| 4347 | /**/ "used real not null,", |
| 4348 | /**/ "size integer not null", |
| 4349 | ") strict;", |
| 4350 | "create index lazy_used on lazy (used);", |
| 4351 | |
| 4352 | // v8's compiled code for the module a load step produced. |
| 4353 | "create table code (", |
| 4354 | /**/ "step integer primary key references steps(id) on delete cascade,", |
| 4355 | /**/ "data blob not null", |
| 4356 | ") strict;", |
| 4357 | |
| 4358 | "create index step_inputs_input on step_inputs (input);", |
| 4359 | "create index step_steps_dependency on step_steps (dependency);", |
| 4360 | ].join("")); |
| 4361 | } |
| 4362 | if (stale) this.sql.exec("commit; pragma foreign_keys = on;"); |
| 4363 | this.#query = prepareQueries(this.sql); |
| 4364 | } |
| 4365 | |
| 4366 | /** changes whenever another process writes to the database */ |
| 4367 | get dataVersion(): number { |
| 4368 | return UNWRAP(this.#query.dataVersion.get({})).data_version; |
| 4369 | } |
| 4370 | |
| 4371 | /** the pid of the process holding the build lock */ |
| 4372 | get locked(): number | null { |
| 4373 | return UNWRAP(this.#query.readLock.get({})).lock; |
| 4374 | } |
| 4375 | |
| 4376 | /** take the build lock, if it is free or its process is gone */ |
| 4377 | lock(): boolean { |
| 4378 | const holder = this.locked; |
| 4379 | if (holder !== null && holder !== process.pid && isAlive(holder)) return false; |
| 4380 | return Number(this.#query.swapLock.run({ next: process.pid, prev: holder }).changes) === 1; |
| 4381 | } |
| 4382 | |
| 4383 | unlock(): void { |
| 4384 | this.#query.swapLock.run({ next: null, prev: process.pid }); |
| 4385 | } |
| 4386 | |
| 4387 | /** a url as it is stored, which is `root:` and the rest when it is under the root */ |
| 4388 | #stored(href: string): string { |
| 4389 | if (href.startsWith("glob:")) return "glob:" + this.#stored(href.slice(5)); |
| 4390 | return href.startsWith(this.#root) ? "root:" + href.slice(this.#root.length) : href; |
| 4391 | } |
| 4392 | |
| 4393 | #restored(href: string): string { |
| 4394 | if (href.startsWith("glob:")) return "glob:" + this.#restored(href.slice(5)); |
| 4395 | return href.startsWith("root:") ? this.#root + href.slice(5) : href; |
| 4396 | } |
| 4397 | |
| 4398 | /** the keys of the steps that load modules start with a kind, and then a url */ |
| 4399 | #key(key: string, move: (href: string) => string): string { |
| 4400 | const colon = key.indexOf(":") + 1; |
| 4401 | return isInternal(key) ? key.slice(0, colon) + move(key.slice(colon)) : key; |
| 4402 | } |
| 4403 | |
| 4404 | /** the values of the steps that find modules are the urls of the modules */ |
| 4405 | #value(key: string, value: Serializable, move: (href: string) => string): Serializable { |
| 4406 | if (key.startsWith("resolve:")) return move(value as string); |
| 4407 | if (!key.startsWith("closure:")) return value; |
| 4408 | return (value as Module[]).map((node) => ({ ...node, url: move(node.url), imports: node.imports.map(move) })); |
| 4409 | } |
| 4410 | |
| 4411 | /** the state of every input, as of the last time it was checked */ |
| 4412 | inputs(): Input[] { |
| 4413 | return this.#query.readInputs.all({}).map((input) => ({ ...input, url: this.#restored(input.url) })); |
| 4414 | } |
| 4415 | |
| 4416 | readInput(url: string): Input | null { |
| 4417 | const input = this.#query.readInput.get({ url: this.#stored(url) }); |
| 4418 | return input ? { ...input, url } : null; |
| 4419 | } |
| 4420 | |
| 4421 | writeInput(input: Input): RowId { |
| 4422 | validateInput(input); |
| 4423 | return UNWRAP(this.#query.writeInput.get({ ...input, url: this.#stored(input.url) })).id; |
| 4424 | } |
| 4425 | |
| 4426 | readStep(key: string): StepRow | null { |
| 4427 | const row = this.#query.readStep.get({ key: this.#key(key, (href) => this.#stored(href)) }); |
| 4428 | return row ? { ...row, key } : null; |
| 4429 | } |
| 4430 | |
| 4431 | /** the hashes a step observed when it ran, and of the files it wrote */ |
| 4432 | readDependencies(step: RowId): { inputs: Dependency[]; steps: Dependency[]; outputs: Dependency[] } { |
| 4433 | const restore = (href: string) => this.#restored(href); |
| 4434 | return { |
| 4435 | inputs: this.#query.readStepInputs.all({ step }).map((dep) => ({ ...dep, key: restore(dep.key) })), |
| 4436 | steps: this.#query.readStepSteps.all({ step }).map((dep) => ({ ...dep, key: this.#key(dep.key, restore) })), |
| 4437 | outputs: this.#query.readStepOutputs.all({ step }) |
| 4438 | .map((output) => ({ ...output, key: url.fileURLToPath(restore(output.key)) })), |
| 4439 | }; |
| 4440 | } |
| 4441 | |
| 4442 | /** every step, with the size of what it holds, what it read, and what it wrote, and every input, for tools that draw the graph */ |
| 4443 | readGraph(): { |
| 4444 | steps: Array<{ |
| 4445 | key: string; |
| 4446 | label: string; |
| 4447 | failed: boolean; |
| 4448 | lazy: boolean; |
| 4449 | bytes: number; |
| 4450 | steps: string[]; |
| 4451 | inputs: string[]; |
| 4452 | outputs: string[]; |
| 4453 | }>; |
| 4454 | inputs: Array<{ url: string; kind: string; size: number }>; |
| 4455 | } { |
| 4456 | const restore = (href: string) => this.#restored(href); |
| 4457 | const steps = new Map( |
| 4458 | this.#query.readGraphSteps.all({}).map((row) => [row.id, { |
| 4459 | key: this.#key(row.key, restore), |
| 4460 | label: row.label, |
| 4461 | failed: row.type === "e", |
| 4462 | lazy: row.lazy === 1, |
| 4463 | bytes: row.bytes, |
| 4464 | steps: [] as string[], |
| 4465 | inputs: [] as string[], |
| 4466 | outputs: [] as string[], |
| 4467 | }]), |
| 4468 | ); |
| 4469 | for (const { step, key } of this.#query.readGraphStepSteps.all({})) { |
| 4470 | UNWRAP(steps.get(step)).steps.push(this.#key(key, restore)); |
| 4471 | } |
| 4472 | for (const { step, url } of this.#query.readGraphStepInputs.all({})) { |
| 4473 | UNWRAP(steps.get(step)).inputs.push(restore(url)); |
| 4474 | } |
| 4475 | for (const { step, url } of this.#query.readGraphStepOutputs.all({})) { |
| 4476 | UNWRAP(steps.get(step)).outputs.push(restore(url)); |
| 4477 | } |
| 4478 | const inputs = this.#query.readGraphInputs.all({}).map((input) => ({ ...input, url: restore(input.url) })); |
| 4479 | return { steps: Array.from(steps.values()), inputs }; |
| 4480 | } |
| 4481 | |
| 4482 | /** a step's value, or for a step that threw, its serialized error */ |
| 4483 | readValue(step: RowId): Serializable { |
| 4484 | const row = UNWRAP(this.#query.readValue.get({ id: step }), () => `missing step ${step}`); |
| 4485 | if (row.type === "b") return row.value; |
| 4486 | if (row.type === "s") return this.#value(row.key, row.value, (href) => this.#restored(href)); |
| 4487 | const blobs = row.type === "j" |
| 4488 | ? this.#query.readBlobs.all({ step }).map((blob) => blob.content) |
| 4489 | : []; |
| 4490 | const value = fromJson(JSON.parse(row.value as string), blobs); |
| 4491 | return row.type === "j" ? this.#value(row.key, value, (href) => this.#restored(href)) : value; |
| 4492 | } |
| 4493 | |
| 4494 | /** store a step's result, every hash it observed, and the files it wrote */ |
| 4495 | writeStep(step: { |
| 4496 | key: string; |
| 4497 | label: string; |
| 4498 | hash: string; |
| 4499 | error: SerializedError | null; |
| 4500 | value: Serializable; |
| 4501 | errors: SerializedError[]; |
| 4502 | inputs: Input[]; |
| 4503 | steps: Array<{ id: RowId | null; hash: string }>; |
| 4504 | outputs: Dependency[]; |
| 4505 | }): RowId { |
| 4506 | const errors = step.errors.length > 0 ? JSON.stringify(step.errors) : null; |
| 4507 | const store = (href: string) => this.#stored(href); |
| 4508 | const key = this.#key(step.key, store); |
| 4509 | return this.#transaction(() => { |
| 4510 | const prev = this.#query.readStep.get({ key }); |
| 4511 | let id: RowId; |
| 4512 | if (prev?.hash === step.hash) { |
| 4513 | // an identical result keeps its value and blobs, which can be large |
| 4514 | id = prev.id; |
| 4515 | this.#query.keepStep.run({ id, label: step.label, errors }); |
| 4516 | } else { |
| 4517 | const { type, value, blobs } = step.error |
| 4518 | ? { type: "e" as const, value: JSON.stringify(step.error), blobs: [] } |
| 4519 | : serialize(this.#value(step.key, step.value, store)); |
| 4520 | id = UNWRAP(this.#query.writeStep.get({ |
| 4521 | key, |
| 4522 | label: step.label, |
| 4523 | hash: step.hash, |
| 4524 | type, |
| 4525 | errors, |
| 4526 | value, |
| 4527 | })).id; |
| 4528 | this.#query.deleteBlobs.run({ step: id }); |
| 4529 | blobs.forEach((content, idx) => this.#query.insertBlob.run({ step: id, idx, content })); |
| 4530 | this.#query.deleteCode.run({ step: id }); |
| 4531 | } |
| 4532 | this.#query.deleteStepInputs.run({ step: id }); |
| 4533 | this.#query.deleteStepSteps.run({ step: id }); |
| 4534 | this.#query.deleteStepOutputs.run({ step: id }); |
| 4535 | for (const input of step.inputs) { |
| 4536 | validateInput(input); |
| 4537 | // a probe that saw only a kind can't replace a known content hash |
| 4538 | const probe = input.kind === "file" && kinds.has(input.hash); |
| 4539 | const write = probe ? this.#query.observeInput : this.#query.writeInput; |
| 4540 | const { id: inputId } = UNWRAP(write.get({ ...input, url: store(input.url) })); |
| 4541 | this.#query.insertStepInput.run({ step: id, input: inputId, hash: input.hash }); |
| 4542 | } |
| 4543 | for (const dependency of step.steps) { |
| 4544 | this.#query.insertStepStep.run({ |
| 4545 | step: id, |
| 4546 | dependency: UNWRAP(dependency.id), |
| 4547 | hash: dependency.hash, |
| 4548 | }); |
| 4549 | } |
| 4550 | for (const output of step.outputs) { |
| 4551 | this.#query.insertStepOutput.run({ step: id, path: this.#outputUrl(output.key), hash: output.hash }); |
| 4552 | } |
| 4553 | return id; |
| 4554 | }); |
| 4555 | } |
| 4556 | |
| 4557 | readCode(step: RowId): Uint8Array | null { |
| 4558 | return this.#query.readCode.get({ step })?.data ?? null; |
| 4559 | } |
| 4560 | |
| 4561 | writeCode(step: RowId, data: Uint8Array): void { |
| 4562 | this.#query.writeCode.run({ step, data }); |
| 4563 | } |
| 4564 | |
| 4565 | /** outputs are stored by the url of their path, like inputs are */ |
| 4566 | #outputUrl(file: string): string { |
| 4567 | return this.#stored(url.pathToFileURL(file).href); |
| 4568 | } |
| 4569 | |
| 4570 | readOutput(file: string): OutputRow | null { |
| 4571 | const row = this.#query.readOutput.get({ path: this.#outputUrl(file) }); |
| 4572 | return row ? { ...row, path: file, source: row.source && this.#restored(row.source) } : null; |
| 4573 | } |
| 4574 | |
| 4575 | writeOutput(output: OutputRow): void { |
| 4576 | const source = output.source && this.#stored(output.source); |
| 4577 | this.#query.writeOutput.run({ ...output, path: this.#outputUrl(output.path), source }); |
| 4578 | } |
| 4579 | |
| 4580 | deleteOutput(file: string): void { |
| 4581 | this.#query.deleteOutput.run({ path: this.#outputUrl(file) }); |
| 4582 | } |
| 4583 | |
| 4584 | /** a lazy step was used, at `used` milliseconds since the epoch */ |
| 4585 | markLazyUsed(step: RowId, used: number): void { |
| 4586 | this.#query.markLazyUsed.run({ step, used }); |
| 4587 | } |
| 4588 | |
| 4589 | /** |
| 4590 | * delete the lazy steps used least recently, and the steps that read them, |
| 4591 | * until the values of lazy steps take at most `bytes` |
| 4592 | */ |
| 4593 | evict(bytes: number): void { |
| 4594 | while (UNWRAP(this.#query.lazyBytes.get({})).size > bytes) { |
| 4595 | this.deleteWithReaders(UNWRAP(this.#query.oldestLazy.get({})).step); |
| 4596 | } |
| 4597 | } |
| 4598 | |
| 4599 | /** delete a step, and the steps that read it */ |
| 4600 | deleteWithReaders(step: RowId): void { |
| 4601 | this.#query.deleteWithReaders.run({ step }); |
| 4602 | } |
| 4603 | |
| 4604 | /** bytes of every output a build wrote */ |
| 4605 | outputSize(): number { |
| 4606 | return UNWRAP(this.#query.outputSize.get({})).size; |
| 4607 | } |
| 4608 | |
| 4609 | /** bytes of the database */ |
| 4610 | size(): number { |
| 4611 | return UNWRAP(this.#query.size.get({})).size; |
| 4612 | } |
| 4613 | |
| 4614 | /** the path of every output */ |
| 4615 | outputs(): string[] { |
| 4616 | return this.#query.readOutputs.all({}).map((row) => url.fileURLToPath(this.#restored(row.path))); |
| 4617 | } |
| 4618 | |
| 4619 | /** |
| 4620 | * after a build, delete the steps and outputs it didn't use, the lazy steps |
| 4621 | * that read a step it changed, and the inputs nothing reads anymore. returns |
| 4622 | * the outputs to remove from disk, and the urls of those inputs. |
| 4623 | */ |
| 4624 | collect(steps: Set<RowId>, outputs: ReadonlyMap<string, unknown>): { outputs: string[]; inputs: string[] } { |
| 4625 | return this.#transaction(() => { |
| 4626 | // every output a build has on disk is in the database, so only a higher count has more to look for |
| 4627 | const written = Array.from(outputs.values()).filter((output) => output !== null).length; |
| 4628 | const stale = UNWRAP(this.#query.countOutputs.get({})).count === written |
| 4629 | ? [] |
| 4630 | : this.outputs().filter((file) => !outputs.has(file)); |
| 4631 | for (const file of stale) this.deleteOutput(file); |
| 4632 | |
| 4633 | this.#query.deleteStaleLazy.run({}); |
| 4634 | if (UNWRAP(this.#query.countSteps.get({})).count !== steps.size) { |
| 4635 | this.#query.deleteOtherSteps.run({ ids: JSON.stringify(Array.from(steps, Number)) }); |
| 4636 | } |
| 4637 | |
| 4638 | const inputs = this.#query.deleteStaleInputs.all({}).map((row) => this.#restored(row.url)); |
| 4639 | return { outputs: stale, inputs }; |
| 4640 | }); |
| 4641 | } |
| 4642 | |
| 4643 | #transaction<T>(fn: () => T): T { |
| 4644 | // a transaction that reads and then writes takes the write lock first, since in wal mode another |
| 4645 | // process's commit in between fails it without waiting |
| 4646 | this.sql.exec("begin immediate;"); |
| 4647 | try { |
| 4648 | const result = fn(); |
| 4649 | this.sql.exec("commit;"); |
| 4650 | return result; |
| 4651 | } catch (error) { |
| 4652 | this.sql.exec("rollback;"); |
| 4653 | throw error; |
| 4654 | } |
| 4655 | } |
| 4656 | } |
| 4657 | |
| 4658 | function isAlive(pid: number): boolean { |
| 4659 | try { |
| 4660 | process.kill(pid, 0); |
| 4661 | return true; |
| 4662 | } catch (error) { |
| 4663 | return exception.code(error) === "EPERM"; |
| 4664 | } |
| 4665 | } |
| 4666 | |
| 4667 | function validateInput(input: Input): void { |
| 4668 | let parsed: URL; |
| 4669 | try { |
| 4670 | parsed = new URL(input.url); |
| 4671 | } catch { |
| 4672 | throw new Error(`input url must be a complete URL: ${input.url}`); |
| 4673 | } |
| 4674 | if (parsed.href !== input.url) { |
| 4675 | throw new Error(`input url must be canonical: ${input.url}`); |
| 4676 | } |
| 4677 | if (!Number.isFinite(input.time)) { |
| 4678 | throw new Error(`input time must be a finite number: ${input.url}`); |
| 4679 | } |
| 4680 | if (!Number.isFinite(input.size)) { |
| 4681 | throw new Error(`input size must be a finite number: ${input.url}`); |
| 4682 | } |
| 4683 | } |
| 4684 | |
| 4685 | /** strings and bytes are stored as is, and anything else as json with bytes beside it */ |
| 4686 | function serialize(data: Serializable): { |
| 4687 | type: "s" | "b" | "j"; |
| 4688 | value: string | Uint8Array; |
| 4689 | blobs: Uint8Array[]; |
| 4690 | } { |
| 4691 | if (typeof data === "string") return { type: "s", value: data, blobs: [] }; |
| 4692 | if (data instanceof Uint8Array) return { type: "b", value: data, blobs: [] }; |
| 4693 | const blobs: Uint8Array[] = []; |
| 4694 | return { type: "j", value: JSON.stringify(toJson(data, blobs)), blobs }; |
| 4695 | } |
| 4696 | |
| 4697 | function toJson(data: Serializable, blobs: Uint8Array[]): unknown { |
| 4698 | if ( |
| 4699 | data === null |
| 4700 | || typeof data === "string" |
| 4701 | || typeof data === "boolean" |
| 4702 | ) { |
| 4703 | return data; |
| 4704 | } |
| 4705 | if (typeof data === "number") { |
| 4706 | return Number.isFinite(data) && !Object.is(data, -0) |
| 4707 | ? data |
| 4708 | : { $clover: "number", value: Object.is(data, -0) ? "-0" : String(data) }; |
| 4709 | } |
| 4710 | if (typeof data === "undefined") { |
| 4711 | return { $clover: "undefined" }; |
| 4712 | } |
| 4713 | if (data instanceof Uint8Array) { |
| 4714 | return { $clover: "bytes", index: blobs.push(data) - 1 }; |
| 4715 | } |
| 4716 | if (data instanceof Date) { |
| 4717 | return { $clover: "date", time: data.getTime() }; |
| 4718 | } |
| 4719 | if (data instanceof Set) { |
| 4720 | return { $clover: "set", entries: Array.from(data, (value) => toJson(value, blobs)) }; |
| 4721 | } |
| 4722 | if (data instanceof Map) { |
| 4723 | return { |
| 4724 | $clover: "map", |
| 4725 | entries: Array.from(data, ([key, value]) => [toJson(key, blobs), toJson(value, blobs)]), |
| 4726 | }; |
| 4727 | } |
| 4728 | if (Array.isArray(data)) { |
| 4729 | return data.map((value) => toJson(value, blobs)); |
| 4730 | } |
| 4731 | // `fromEntries` defines keys, so `__proto__` stays a key rather than a prototype |
| 4732 | const out = Object.fromEntries(Object.entries(data).map(([key, value]) => [key, toJson(value, blobs)])); |
| 4733 | if (Object.hasOwn(out, "$clover")) { |
| 4734 | return { $clover: "escaped", value: out }; |
| 4735 | } |
| 4736 | return out; |
| 4737 | } |
| 4738 | |
| 4739 | function fromJson(data: unknown, blobs: Uint8Array[]): Serializable { |
| 4740 | if ( |
| 4741 | data === null |
| 4742 | || typeof data === "string" |
| 4743 | || typeof data === "boolean" |
| 4744 | || typeof data === "number" |
| 4745 | ) { |
| 4746 | return data; |
| 4747 | } |
| 4748 | if (Array.isArray(data)) { |
| 4749 | return data.map((value) => fromJson(value, blobs)); |
| 4750 | } |
| 4751 | if (!data || typeof data !== "object") { |
| 4752 | throw new Error(`unsupported serialized value: ${String(data)}`); |
| 4753 | } |
| 4754 | |
| 4755 | const object = data as Record<string, unknown>; |
| 4756 | const entries = (value: unknown): Serializable => |
| 4757 | Object.fromEntries(Object.entries(value as object).map(([key, entry]) => [key, fromJson(entry, blobs)])); |
| 4758 | switch (object.$clover) { |
| 4759 | case undefined: |
| 4760 | return entries(object); |
| 4761 | case "number": |
| 4762 | return Number(object.value); |
| 4763 | case "undefined": |
| 4764 | return undefined; |
| 4765 | case "bytes": |
| 4766 | return UNWRAP(blobs[object.index as number], "missing blob"); |
| 4767 | case "date": |
| 4768 | return new Date(object.time as number); |
| 4769 | case "set": |
| 4770 | return new Set((object.entries as unknown[]).map((value) => fromJson(value, blobs))); |
| 4771 | case "map": |
| 4772 | return new Map( |
| 4773 | (object.entries as [unknown, unknown][]).map(([key, value]) => [ |
| 4774 | fromJson(key, blobs), |
| 4775 | fromJson(value, blobs), |
| 4776 | ]), |
| 4777 | ); |
| 4778 | case "escaped": |
| 4779 | return entries(object.value); |
| 4780 | default: |
| 4781 | throw new Error(`unknown serialized marker: ${String(object.$clover)}`); |
| 4782 | } |
| 4783 | } |
| 4784 | |
| 4785 | function hashSerializable(...values: unknown[]): string { |
| 4786 | const hash = crypto.createHash("sha256"); |
| 4787 | writeSerializableToHasher(hash, values); |
| 4788 | return hash.digest("base64url"); |
| 4789 | } |
| 4790 | |
| 4791 | /** throws for values that would hash ambiguously, instead of colliding */ |
| 4792 | function writeSerializableToHasher( |
| 4793 | hash: crypto.Hash, |
| 4794 | value: unknown, |
| 4795 | ): void { |
| 4796 | if (value === null) { |
| 4797 | hash.update("n;"); |
| 4798 | return; |
| 4799 | } |
| 4800 | if (typeof value === "undefined") { |
| 4801 | hash.update("u;"); |
| 4802 | return; |
| 4803 | } |
| 4804 | if (typeof value === "boolean") { |
| 4805 | hash.update(value ? "t;" : "f;"); |
| 4806 | return; |
| 4807 | } |
| 4808 | if (typeof value === "number") { |
| 4809 | writeStringToHasher(hash, "#", Object.is(value, -0) ? "-0" : String(value)); |
| 4810 | return; |
| 4811 | } |
| 4812 | if (typeof value === "string") { |
| 4813 | writeStringToHasher(hash, "s", value); |
| 4814 | return; |
| 4815 | } |
| 4816 | if (typeof value !== "object") { |
| 4817 | throw new TypeError(`a ${typeof value} can not be hashed or serialized`); |
| 4818 | } |
| 4819 | if (value instanceof Uint8Array) { |
| 4820 | hash.update(`b${value.byteLength}:`); |
| 4821 | hash.update(value); |
| 4822 | hash.update(";"); |
| 4823 | return; |
| 4824 | } |
| 4825 | if (value instanceof Date) { |
| 4826 | writeStringToHasher(hash, "d", String(value.getTime())); |
| 4827 | return; |
| 4828 | } |
| 4829 | if (value instanceof Set) { |
| 4830 | hash.update(`e${value.size}:[`); |
| 4831 | for (const entry of value) writeSerializableToHasher(hash, entry); |
| 4832 | hash.update("]"); |
| 4833 | return; |
| 4834 | } |
| 4835 | if (value instanceof Map) { |
| 4836 | hash.update(`m${value.size}:{`); |
| 4837 | for (const [key, entry] of value) { |
| 4838 | writeSerializableToHasher(hash, key); |
| 4839 | writeSerializableToHasher(hash, entry); |
| 4840 | } |
| 4841 | hash.update("}"); |
| 4842 | return; |
| 4843 | } |
| 4844 | if (Array.isArray(value)) { |
| 4845 | hash.update(`a${value.length}:[`); |
| 4846 | for (const entry of value) writeSerializableToHasher(hash, entry); |
| 4847 | hash.update("]"); |
| 4848 | return; |
| 4849 | } |
| 4850 | if (value instanceof Step) { |
| 4851 | hash.update("S" + value[kState].key + ";"); |
| 4852 | return; |
| 4853 | } |
| 4854 | if (!isPlainObject(value)) { |
| 4855 | const name = (value as object).constructor?.name ?? "non-plain object"; |
| 4856 | throw new TypeError(`a ${name} can not be hashed or serialized, only plain objects`); |
| 4857 | } |
| 4858 | const entries = Object.entries(value).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0); |
| 4859 | hash.update(`o${entries.length}:{`); |
| 4860 | for (const [key, entry] of entries) { |
| 4861 | writeStringToHasher(hash, "k", key); |
| 4862 | writeSerializableToHasher(hash, entry); |
| 4863 | } |
| 4864 | hash.update("}"); |
| 4865 | } |
| 4866 | |
| 4867 | function writeStringToHasher( |
| 4868 | hash: crypto.Hash, |
| 4869 | tag: string, |
| 4870 | value: string, |
| 4871 | ): void { |
| 4872 | const byteLength = Buffer.byteLength(value); |
| 4873 | hash.update(`${tag}${byteLength}:`); |
| 4874 | hash.update(value); |
| 4875 | hash.update(";"); |
| 4876 | } |
| 4877 | |
| 4878 | function sha256(data: string | Uint8Array): string { |
| 4879 | return crypto.createHash("sha256").update(data).digest("base64url"); |
| 4880 | } |
| 4881 | |
| 4882 | function isPlainObject(value: unknown): value is Record<string, unknown> { |
| 4883 | if (value === null || typeof value !== "object") return false; |
| 4884 | const proto = Object.getPrototypeOf(value); |
| 4885 | return proto === Object.prototype || proto === null; |
| 4886 | } |
| 4887 | |
| 4888 | function isJson(value: unknown): value is ts.JsonValue { |
| 4889 | if (value === null || typeof value === "string" || typeof value === "boolean") return true; |
| 4890 | if (typeof value === "number") return Number.isFinite(value); |
| 4891 | if (Array.isArray(value)) return value.every(isJson); |
| 4892 | return isPlainObject(value) && Object.values(value).every(isJson); |
| 4893 | } |
| 4894 | |
| 4895 | /** JSON with sorted keys, so equal params make equal urls */ |
| 4896 | function stableJson(value: ts.JsonValue): string { |
| 4897 | return JSON.stringify(value, (_, inner) => |
| 4898 | isPlainObject(inner) |
| 4899 | ? Object.fromEntries(Object.entries(inner).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0)) |
| 4900 | : inner); |
| 4901 | } |
| 4902 | |
| 4903 | // -- serving builds, with a debugger for the pages this machine asks for -- |
| 4904 | |
| 4905 | /** what the {@linkcode Debugger} tells a page, which only ever answers by leaving */ |
| 4906 | export type DebuggerMessage = |
| 4907 | | ["building"] |
| 4908 | | ["status", BuildStatus] |
| 4909 | | ["progress", progress.StreamEvent] |
| 4910 | | ["steps", StepEvent[]]; |
| 4911 | |
| 4912 | /** what `/.clover/graph` answers with, where `root` is the url that relative paths start from */ |
| 4913 | export type GraphSnapshot = ReturnType<Database["readGraph"]> & { root: string }; |
| 4914 | |
| 4915 | /** what the last build did */ |
| 4916 | export interface BuildStatus { |
| 4917 | /** `warnings` are errors that the build is served with, and `failed` ones left nothing to serve */ |
| 4918 | state: "ok" | "warnings" | "failed"; |
| 4919 | /** an earlier build is served, which a failed one leaves serving */ |
| 4920 | earlier: boolean; |
| 4921 | /** paths in the project of the inputs whose change caused the build, a directory's ending in `/` */ |
| 4922 | changed: string[]; |
| 4923 | stats: BuildStats; |
| 4924 | /** when the build was served, in milliseconds since the unix epoch */ |
| 4925 | at: number; |
| 4926 | statusLine: string; |
| 4927 | /** the errors and warnings, laid out like the terminal prints them, which a page draws */ |
| 4928 | errors: stack.Diagnostic[]; |
| 4929 | /** |
| 4930 | * the token at each character of the javascript in the errors' code, by |
| 4931 | * `file:line`: the first letter of a keyword, literal, number, string, |
| 4932 | * regexp, or comment, and a space for the rest |
| 4933 | */ |
| 4934 | tokens: Record<string, string>; |
| 4935 | } |
| 4936 | |
| 4937 | /** the header that asks for the hash of a page instead of the page */ |
| 4938 | const pageHeader = "clover-page"; |
| 4939 | |
| 4940 | const libPath = "/.clover/lib/"; |
| 4941 | const socketPath = "/.clover/socket"; |
| 4942 | const graphPath = "/.clover/graph"; |
| 4943 | /** the visualization alone, which can be watched before anything else is served */ |
| 4944 | const homePath = "/.clover"; |
| 4945 | const libDir = url.fileURLToPath(new URL("./", import.meta.url)); |
| 4946 | |
| 4947 | /** |
| 4948 | * the server half of `incremental/visualization.ts`, which a page shows when |
| 4949 | * this machine requests it: the status of the last build, its progress, its |
| 4950 | * errors, and the graph. a page reloads when a build changes what it was |
| 4951 | * served, and until a build is served, a page shows the graph and progress of |
| 4952 | * the build instead. requests through a proxy, or from other machines, are |
| 4953 | * served as if this was not here. {@linkcode serve} sets one up with a server. |
| 4954 | * |
| 4955 | * ```ts |
| 4956 | * const debug = new Debugger(graph); |
| 4957 | * const server = await http.serve({ |
| 4958 | * respond: debug.respond(respond), |
| 4959 | * upgrade: debug.upgrade, |
| 4960 | * shouldUpgrade: debug.shouldUpgrade, |
| 4961 | * port, |
| 4962 | * }); |
| 4963 | * for await (const result of graph.incrementalWatch({ progress: debug.progress, buildAndWriteOutput })) { |
| 4964 | * // ...once `respond` serves what the build made |
| 4965 | * debug.update(result); |
| 4966 | * } |
| 4967 | * ``` |
| 4968 | */ |
| 4969 | export class Debugger { |
| 4970 | /** where builds report their progress, for every page, and for the terminal unless the root was given */ |
| 4971 | readonly progress: progress.Root; |
| 4972 | #graph: Graph; |
| 4973 | #status: BuildStatus | null = null; |
| 4974 | /** a build was served, which pages are from until another is */ |
| 4975 | #served = false; |
| 4976 | #pages = new Set<(message: DebuggerMessage) => void>(); |
| 4977 | /** what happened to steps since the last frame */ |
| 4978 | #steps: StepEvent[] = []; |
| 4979 | #subscriptions = new DisposableStack(); |
| 4980 | |
| 4981 | constructor(graph: Graph, root?: progress.Root) { |
| 4982 | this.#graph = graph; |
| 4983 | this.progress = root ?? new progress.Root(); |
| 4984 | if (!root) this.#subscriptions.use(progress.attachToScreen(this.progress, log)); |
| 4985 | // a build starts its progress at the top |
| 4986 | this.#subscriptions.use(this.progress.on("node-start", (_, parent) => { |
| 4987 | if (!parent) { for (const send of this.#pages) send(["building"]); } |
| 4988 | })); |
| 4989 | this.#subscriptions.use(graph.observe((event) => { |
| 4990 | if (this.#pages.size === 0 || this.#steps.push(event) > 1) return; |
| 4991 | setTimeout(() => { |
| 4992 | const steps = this.#steps.splice(0); |
| 4993 | for (const send of this.#pages) send(["steps", steps]); |
| 4994 | }, 1000 / 30); |
| 4995 | })); |
| 4996 | } |
| 4997 | |
| 4998 | /** stop hearing of builds, and drawing them on the terminal */ |
| 4999 | [Symbol.dispose](): void { |
| 5000 | this.#subscriptions.dispose(); |
| 5001 | } |
| 5002 | |
| 5003 | /** |
| 5004 | * tell every page what a build did, once it is what `respond` serves. |
| 5005 | * `failure` is why what it made can't be served, like a handler that threw |
| 5006 | * as it loaded. |
| 5007 | */ |
| 5008 | update(result: WatchResult<unknown>, failure?: unknown): void { |
| 5009 | const served = failure === undefined && (result.ok || result.value !== undefined); |
| 5010 | const errors = result.ok ? [] : diagnose(result.error, this.#graph.root); |
| 5011 | if (failure !== undefined) { |
| 5012 | errors.unshift(traced(stack.describe(failure, { root: this.#graph.root }), "thrown while the handler loaded")); |
| 5013 | } |
| 5014 | const status: BuildStatus = this.#status = { |
| 5015 | state: !served ? "failed" : result.ok ? "ok" : "warnings", |
| 5016 | earlier: this.#served, |
| 5017 | changed: result.changed.map((href) => |
| 5018 | href.startsWith("file:") |
| 5019 | ? path.relative(this.#graph.root, url.fileURLToPath(href)) + (href.endsWith("/") ? "/" : "") |
| 5020 | : href |
| 5021 | ), |
| 5022 | stats: result.stats, |
| 5023 | at: Date.now(), |
| 5024 | statusLine: result.statusLine, |
| 5025 | errors, |
| 5026 | tokens: highlight(errors), |
| 5027 | }; |
| 5028 | this.#served ||= served; |
| 5029 | for (const send of this.#pages) send(["status", status]); |
| 5030 | } |
| 5031 | |
| 5032 | /** |
| 5033 | * `inner`, with the visualization in the pages that this machine requests, |
| 5034 | * and its code at `/.clover/lib/`, which is this library's |
| 5035 | */ |
| 5036 | respond(inner: http.ServeOptions["respond"]): http.ServeOptions["respond"] { |
| 5037 | return async (request, context) => { |
| 5038 | if (!isLocal(request, context)) return inner(request, context); |
| 5039 | let { pathname } = new URL(request.url); |
| 5040 | pathname = pathname.replace(/\/+$/, "") || "/"; |
| 5041 | if (pathname.startsWith(libPath)) return serveLib(pathname.slice(libPath.length)); |
| 5042 | if (pathname === homePath) return graphPage(""); |
| 5043 | if (pathname === graphPath) { |
| 5044 | const snapshot: GraphSnapshot = { |
| 5045 | root: url.pathToFileURL(this.#graph.root + "/").href, |
| 5046 | ...this.#graph.db.readGraph(), |
| 5047 | }; |
| 5048 | return Response.json(snapshot, { headers: { "cache-control": "no-store" } }); |
| 5049 | } |
| 5050 | const check = request.headers.has(pageHeader); |
| 5051 | const navigated = request.headers.get("sec-fetch-dest") === "document"; |
| 5052 | // until a build is served, a page shows that build, and reloads once there is something to serve |
| 5053 | if (navigated && request.method === "GET" && this.#status === null) return graphPage("?page=&graph"); |
| 5054 | if (!check && !navigated) return inner(request, context); |
| 5055 | |
| 5056 | // a page is read whole, which a compressed one can't be |
| 5057 | request.headers.delete("accept-encoding"); |
| 5058 | const response = await inner(request, context); |
| 5059 | const html = response.headers.get("content-type")?.startsWith("text/html") |
| 5060 | && !response.headers.has("content-encoding"); |
| 5061 | if (!check && !(html && response.body)) return response; |
| 5062 | |
| 5063 | const body = new Uint8Array(await response.arrayBuffer()); |
| 5064 | const page = crypto.createHash("sha1").update(body).digest("base64url"); |
| 5065 | if (check) return new Response(null, { status: 204, headers: { [pageHeader]: page } }); |
| 5066 | |
| 5067 | // a posted page can't be asked for again, so it has nothing to compare with |
| 5068 | const script = visualization(request.method === "GET" ? `?page=${page}` : ""); |
| 5069 | const headers = new Headers(response.headers); |
| 5070 | headers.delete("content-length"); |
| 5071 | return new Response(Buffer.concat([body, Buffer.from(script)]), { |
| 5072 | status: response.status, |
| 5073 | statusText: response.statusText, |
| 5074 | headers, |
| 5075 | }); |
| 5076 | }; |
| 5077 | } |
| 5078 | |
| 5079 | /** for `http.serve`, which requests to switch protocols are for `upgrade` */ |
| 5080 | shouldUpgrade = (request: Request): boolean => new URL(request.url).pathname === socketPath; |
| 5081 | |
| 5082 | /** for `http.serve`, the websocket that pages follow builds through */ |
| 5083 | upgrade = (request: Request, socket: stream.Duplex, context: http.ServeContext): void => { |
| 5084 | const key = request.headers.get("sec-websocket-key"); |
| 5085 | const origin = request.headers.get("origin"); |
| 5086 | if ( |
| 5087 | !this.shouldUpgrade(request) || !key || !isLocal(request, context) |
| 5088 | // any page in the browser can open a websocket here, but only this server's may read it |
| 5089 | || (origin !== null && origin !== `http://${request.headers.get("host")}`) |
| 5090 | ) { |
| 5091 | return void socket.end("HTTP/1.1 403 Forbidden\r\nconnection: close\r\n\r\n"); |
| 5092 | } |
| 5093 | const accept = crypto.createHash("sha1").update(key + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11").digest("base64"); |
| 5094 | socket.write( |
| 5095 | "HTTP/1.1 101 Switching Protocols\r\nupgrade: websocket\r\nconnection: Upgrade\r\n" |
| 5096 | + `sec-websocket-accept: ${accept}\r\n\r\n`, |
| 5097 | ); |
| 5098 | |
| 5099 | const send = (message: DebuggerMessage) => void socket.write(frame(1, Buffer.from(JSON.stringify(message)))); |
| 5100 | const stop = new AbortController(); |
| 5101 | this.#pages.add(send); |
| 5102 | socket.once("close", () => { |
| 5103 | this.#pages.delete(send); |
| 5104 | stop.abort(); |
| 5105 | }); |
| 5106 | if (this.#status) send(["status", this.#status]); |
| 5107 | if (this.progress.active.length > 0) send(["building"]); |
| 5108 | void progress.encodeEventStream(this.progress) |
| 5109 | .pipeTo(new WritableStream({ write: (event) => send(["progress", event]) }), { signal: stop.signal }) |
| 5110 | .catch(() => {}); |
| 5111 | |
| 5112 | // a page sends nothing but a close, which is answered before the connection ends |
| 5113 | let read = Buffer.alloc(0); |
| 5114 | socket.on("data", (chunk: Buffer) => { |
| 5115 | read = Buffer.concat([read, chunk]); |
| 5116 | while (read.length >= 2) { |
| 5117 | const size = read[1]! & 0x7f; |
| 5118 | const start = size === 126 ? 4 : size === 127 ? 10 : 2; |
| 5119 | if (read.length < start) return; |
| 5120 | const length = size === 126 ? read.readUInt16BE(2) : size === 127 ? Number(read.readBigUInt64BE(2)) : size; |
| 5121 | const end = start + (read[1]! & 0x80 ? 4 : 0) + length; |
| 5122 | if (read.length < end) return; |
| 5123 | if ((read[0]! & 0x0f) === 8) return void socket.end(frame(8, Buffer.alloc(0))); |
| 5124 | read = read.subarray(end); |
| 5125 | } |
| 5126 | }); |
| 5127 | }; |
| 5128 | } |
| 5129 | |
| 5130 | /** see {@linkcode BuildStatus.tokens} */ |
| 5131 | function highlight(diagnostics: stack.Diagnostic[]): Record<string, string> { |
| 5132 | const tokens: Record<string, string> = {}; |
| 5133 | const lex = (place: stack.Excerpt | null) => { |
| 5134 | if (!place || !/\.[cm]?[jt]sx?$/.test(place.file ?? "")) return; |
| 5135 | // lines from the middle of a doc comment are lexed as the comment they are in |
| 5136 | const opened = /^\s*\*/.test(place.code[0] ?? "") ? "/*" : ""; |
| 5137 | let kinds = ""; |
| 5138 | for (const { end, kind } of repl.tokenize(repl.javascriptTokenizer, opened + place.code.join("\n"))) { |
| 5139 | kinds += (colored.has(kind) ? kind[0]! : " ").repeat(end - kinds.length); |
| 5140 | } |
| 5141 | let at = opened.length; |
| 5142 | place.code.forEach((line, i) => { |
| 5143 | tokens[`${place.file}:${place.first + i}`] = kinds.slice(at, at + line.length); |
| 5144 | at += line.length + 1; |
| 5145 | }); |
| 5146 | }; |
| 5147 | const visit = (diagnostic: stack.Diagnostic) => { |
| 5148 | lex(diagnostic.location); |
| 5149 | for (const note of diagnostic.notes) { |
| 5150 | lex(note.location); |
| 5151 | note.frames.forEach(lex); |
| 5152 | } |
| 5153 | if (diagnostic.cause) visit(diagnostic.cause); |
| 5154 | diagnostic.errors.forEach(visit); |
| 5155 | }; |
| 5156 | diagnostics.forEach(visit); |
| 5157 | return tokens; |
| 5158 | } |
| 5159 | |
| 5160 | /** |
| 5161 | * the kinds of token that are colored, while names and punctuation are not. |
| 5162 | * a few lines without the rest of their file have brackets that seem to |
| 5163 | * mismatch, so what the lexer calls invalid is not |
| 5164 | */ |
| 5165 | const colored = new Set(["keyword", "literal", "number", "string", "regexp", "comment"]); |
| 5166 | |
| 5167 | function visualization(query: string) { |
| 5168 | return `<script type=module src=${libPath}incremental/visualization.ts${query}></script>`; |
| 5169 | } |
| 5170 | |
| 5171 | /** the visualization alone, which shows the graph full screen */ |
| 5172 | function graphPage(query: string): Response { |
| 5173 | const page = `<!doctype html><title>build</title><body style="margin:0;background:#000">${visualization(query)}`; |
| 5174 | return new Response(page, { headers: { "content-type": "text/html;charset=utf-8" } }); |
| 5175 | } |
| 5176 | |
| 5177 | /** |
| 5178 | * from this machine, not through a proxy, and to a loopback name, since any |
| 5179 | * site can point a name of its own here and read what its pages are served |
| 5180 | */ |
| 5181 | function isLocal(request: Request, { remoteAddress = "" }: http.ServeContext): boolean { |
| 5182 | return /^(127\.|::1$|::ffff:127\.)/.test(remoteAddress) |
| 5183 | && /^(localhost|127\.0\.0\.1|\[::1\])(:\d+)?$/.test(request.headers.get("host") ?? "") |
| 5184 | && !Array.from(request.headers.keys()).some((name) => name === "forwarded" || name.startsWith("x-forwarded-")); |
| 5185 | } |
| 5186 | |
| 5187 | /** a file of this library as javascript. the path of a parsed url has no `..` left in it */ |
| 5188 | async function serveLib(file: string): Promise<Response> { |
| 5189 | const source = file === "safari.js" ? safari : file.endsWith(".ts") |
| 5190 | ? await fs.readFile(libDir + file, "utf8").catch((error) => { |
| 5191 | if (exception.code(error) === "ENOENT") return null; |
| 5192 | throw error; |
| 5193 | }) |
| 5194 | : null; |
| 5195 | if (source === null) return new Response(null, { status: 404 }); |
| 5196 | return new Response(forSafari(file, module.stripTypeScriptTypes(source)), { |
| 5197 | headers: { "content-type": "text/javascript;charset=utf-8", "cache-control": "no-cache" }, |
| 5198 | }); |
| 5199 | } |
| 5200 | |
| 5201 | /** |
| 5202 | * a stupid hack, but better than giving up `using` everywhere: safari has no |
| 5203 | * explicit resource management. a page needs neither `log.ts` nor the one |
| 5204 | * `using` in `progress.ts`, so they are stubbed, and the globals the rest |
| 5205 | * touches are polyfilled by an import that runs before the others. |
| 5206 | */ |
| 5207 | function forSafari(file: string, code: string): string { |
| 5208 | if (file === "log.ts") return "export const headlessScope = (writeMessage) => ({ writeMessage });\n"; |
| 5209 | if (file === "progress.ts") { |
| 5210 | return code.replace(/^function writeStreamEvent\b[\s\S]*?^\}$/m, "function writeStreamEvent() {}"); |
| 5211 | } |
| 5212 | return file === "incremental/visualization.ts" ? `import "../safari.js";\n${code}` : code; |
| 5213 | } |
| 5214 | |
| 5215 | const safari = ` |
| 5216 | Symbol.dispose ??= Symbol.for("Symbol.dispose"); |
| 5217 | Symbol.asyncDispose ??= Symbol.for("Symbol.asyncDispose"); |
| 5218 | globalThis.DisposableStack ??= class DisposableStack { |
| 5219 | #fns = []; |
| 5220 | use(value) { if (value != null) this.#fns.push(() => value[Symbol.dispose]()); return value; } |
| 5221 | defer(fn) { this.#fns.push(fn); } |
| 5222 | dispose() { for (const fn of this.#fns.splice(0).reverse()) fn(); } |
| 5223 | [Symbol.dispose]() { this.dispose(); } |
| 5224 | }; |
| 5225 | `; |
| 5226 | |
| 5227 | /** a websocket frame from a server, which is not masked */ |
| 5228 | function frame(opcode: number, payload: Buffer): Buffer { |
| 5229 | const { length } = payload; |
| 5230 | const header = Buffer.alloc(length < 126 ? 2 : length < 65536 ? 4 : 10); |
| 5231 | header[0] = 0x80 | opcode; |
| 5232 | if (length < 126) { |
| 5233 | header[1] = length; |
| 5234 | } else if (length < 65536) { |
| 5235 | header[1] = 126; |
| 5236 | header.writeUInt16BE(length, 2); |
| 5237 | } else { |
| 5238 | header[1] = 127; |
| 5239 | header.writeBigUInt64BE(BigInt(length), 2); |
| 5240 | } |
| 5241 | return Buffer.concat([header, payload]); |
| 5242 | } |
| 5243 | |
| 5244 | export interface ServeOptions<T> { |
| 5245 | graph: Graph; |
| 5246 | buildAndWriteOutput: BuildOptions<T>["buildAndWriteOutput"]; |
| 5247 | /** |
| 5248 | * `false` is for a process whose files only change with a restart, like a |
| 5249 | * deploy: they are read once and not watched. see {@linkcode Graph#incrementalWatch} |
| 5250 | */ |
| 5251 | watchFilesystem?: boolean; |
| 5252 | /** defaults to the `PORT` environment variable, or else 3000 */ |
| 5253 | port?: number; |
| 5254 | /** |
| 5255 | * where the url, and each build's errors and status, are said. the global |
| 5256 | * log otherwise, which owns a terminal, so each build's report replaces the |
| 5257 | * last there. |
| 5258 | */ |
| 5259 | log?: log.Scope; |
| 5260 | /** where builds report their progress, for every page. a root drawn on the terminal otherwise */ |
| 5261 | progress?: progress.Root; |
| 5262 | } |
| 5263 | |
| 5264 | type Respond = (request: Request) => Response | Promise<Response>; |
| 5265 | |
| 5266 | export interface Server<T> { |
| 5267 | readonly url: string; |
| 5268 | readonly graph: Graph; |
| 5269 | /** where builds report their progress, and so can what a handler loads */ |
| 5270 | readonly progress: progress.Ref; |
| 5271 | /** |
| 5272 | * serve requests with `respond` from now on. a promise of it serves once it |
| 5273 | * resolves. one that rejects leaves the last handler serving, and pages hear |
| 5274 | * why with the build that `watch` gave. |
| 5275 | */ |
| 5276 | setHandler(respond: Respond | Promise<Respond>): void; |
| 5277 | /** |
| 5278 | * {@linkcode Graph#incrementalWatch}, saying each build's errors and status. |
| 5279 | * pages hear of a build once the next one is asked for, so set what serves |
| 5280 | * it first. |
| 5281 | */ |
| 5282 | watch(): AsyncGenerator<WatchResult<T>, void>; |
| 5283 | close(): Promise<void>; |
| 5284 | [Symbol.asyncDispose](): Promise<void>; |
| 5285 | } |
| 5286 | |
| 5287 | /** |
| 5288 | * serve what `graph` builds, with the {@linkcode Debugger} while files are |
| 5289 | * watched. the server listens at once, and a request waits for the first |
| 5290 | * `setHandler`, or for a build that left nothing to serve. |
| 5291 | * |
| 5292 | * ```ts |
| 5293 | * const server = await incremental.serve({ graph, buildAndWriteOutput }); |
| 5294 | * for await (const result of server.watch()) { |
| 5295 | * if (result.value) server.setHandler(loadHandler(result.value)); |
| 5296 | * } |
| 5297 | * ``` |
| 5298 | */ |
| 5299 | export async function serve<T>(options: ServeOptions<T>): Promise<Server<T>> { |
| 5300 | const { graph, buildAndWriteOutput, watchFilesystem = true, port = Number(process.env.PORT ?? 3000) } = options; |
| 5301 | const scope = options.log ?? log; |
| 5302 | // a process whose files only change with a restart, like a deploy's, has no one at its pages to debug |
| 5303 | const debug = watchFilesystem ? new Debugger(graph, options.progress) : null; |
| 5304 | const builds = debug?.progress ?? options.progress ?? progress; |
| 5305 | let handler: Respond | null = null; |
| 5306 | /** a handler that `setHandler` was given and is still loading, which settles with why it failed */ |
| 5307 | let loading: Promise<unknown> | null = null; |
| 5308 | const ready = Promise.withResolvers<void>(); |
| 5309 | const answer = async (request: Request) => { |
| 5310 | await ready.promise; |
| 5311 | if (handler) return handler(request); |
| 5312 | // a page, so that a browser on this machine shows why |
| 5313 | return new Response("<!doctype html><title>build failed</title><p>the site did not build.", { |
| 5314 | status: 503, |
| 5315 | headers: { "content-type": "text/html;charset=utf8" }, |
| 5316 | }); |
| 5317 | }; |
| 5318 | const server = await http.serve({ |
| 5319 | respond: debug?.respond(answer) ?? answer, |
| 5320 | upgrade: debug?.upgrade, |
| 5321 | shouldUpgrade: debug?.shouldUpgrade, |
| 5322 | port, |
| 5323 | }); |
| 5324 | scope.info(`-> ${server.url}`); |
| 5325 | const clear = !options.log && process.stdout.isTTY; |
| 5326 | const close = async () => { |
| 5327 | await server.close(); |
| 5328 | debug?.[Symbol.dispose](); |
| 5329 | }; |
| 5330 | return { |
| 5331 | url: server.url, |
| 5332 | graph, |
| 5333 | progress: builds, |
| 5334 | setHandler(respond) { |
| 5335 | loading = Promise.resolve(respond).then((loaded) => { |
| 5336 | handler = loaded; |
| 5337 | ready.resolve(); |
| 5338 | }, (error: unknown) => error ?? new Error("the handler did not load")); |
| 5339 | }, |
| 5340 | async *watch() { |
| 5341 | let built = false; |
| 5342 | for await ( |
| 5343 | const result of graph.incrementalWatch({ |
| 5344 | progress: builds, |
| 5345 | buildAndWriteOutput, |
| 5346 | files: watchFilesystem, |
| 5347 | }) |
| 5348 | ) { |
| 5349 | if (clear && built) { |
| 5350 | log.writeOutput(ansi.cursorHome + ansi.clearToEndOfScreen); |
| 5351 | scope.info(`-> ${server.url}`); |
| 5352 | } |
| 5353 | built = true; |
| 5354 | // errors stay listed until they are fixed, even when cached |
| 5355 | if (!result.ok) scope.error(result.error); |
| 5356 | scope.info(result.statusLine); |
| 5357 | // a build that left nothing to serve has nothing to wait for |
| 5358 | if (!result.ok && result.value === undefined) ready.resolve(); |
| 5359 | yield result; |
| 5360 | const failure = await loading ?? undefined; |
| 5361 | loading = null; |
| 5362 | if (failure !== undefined) scope.error(failure); |
| 5363 | // a page compares itself with what is served, so it hears of a build once that is this one |
| 5364 | debug?.update(result, failure); |
| 5365 | ready.resolve(); |
| 5366 | } |
| 5367 | }, |
| 5368 | close, |
| 5369 | [Symbol.asyncDispose]: close, |
| 5370 | }; |
| 5371 | } |
| 5372 | |
| 5373 | import { ASSERT, UNWRAP } from "./assert.ts"; |
| 5374 | import * as async from "./async.ts"; |
| 5375 | import * as exception from "./exception.ts"; |
| 5376 | import * as http from "./http.ts"; |
| 5377 | import * as log from "./log.ts"; |
| 5378 | import * as stack from "./log/stack.ts"; |
| 5379 | import type * as node from "./node.ts"; |
| 5380 | import * as progress from "./progress.ts"; |
| 5381 | import * as repl from "./repl.ts"; |
| 5382 | import * as ansi from "./string/ansi.ts"; |
| 5383 | import * as ts from "./ts.ts"; |
| 5384 | |
| 5385 | import { AsyncLocalStorage } from "node:async_hooks"; |
| 5386 | import * as child_process from "node:child_process"; |
| 5387 | import * as crypto from "node:crypto"; |
| 5388 | import { once } from "node:events"; |
| 5389 | import * as fscb from "node:fs"; |
| 5390 | import * as fs from "node:fs/promises"; |
| 5391 | import * as module from "node:module"; |
| 5392 | import * as path from "node:path"; |
| 5393 | import process from "node:process"; |
| 5394 | import * as sqlite from "node:sqlite"; |
| 5395 | import type * as stream from "node:stream"; |
| 5396 | import * as url from "node:url"; |
| 5397 | import * as util from "node:util"; |
| 5398 | import * as vm from "node:vm"; |