1test("readFile reads text, bytes, streams, and hashes, recording one input", async () => {
2 await using p = await project({ "source.txt": "hello" });
3 const graph = p.graph();
4 const result = await build(graph, (ref) =>
5 ref.step(async (b) => {
6 const file = b.readFile("source.txt");
7 assert.equal(b.readFile(p.path("source.txt")), file);
8 const stream = await readStream(await file.stream({ chunkSize: 2 }));
9 return {
10 text: await file.text(),
11 bytes: (await file.bytes()).byteLength,
12 json: await b.readFile("source.json").json<string>().catch((error) => String(error.code)),
13 stream,
14 hash: await file.hash(),
15 };
16 }).read());
17 assert.deepEqual(result, {
18 text: "hello",
19 bytes: 5,
20 json: "ENOENT",
21 stream: "hello",
22 hash: sha256("hello"),
23 });
24 const inputs = graph.db.inputs().filter((input) => input.url.endsWith(".txt"));
25 assert.deepEqual(inputs.map(({ url, kind, size, hash }) => ({ url, kind, size, hash })), [
26 { url: url.pathToFileURL(p.path("source.txt")).href, kind: "file", size: 5, hash: sha256("hello") },
27 ]);
28});
29
30test("a stream hashes as it reads, and a cancelled one is hashed when the step ends", async () => {
31 await using p = await project({ "source.txt": "hello" });
32 let runs = 0;
33 const graph = p.graph();
34 const site = () =>
35 build(graph, (ref) =>
36 ref.step(async (b) => {
37 runs++;
38 const file = b.readFile("source.txt");
39 const reader = (await file.stream({ chunkSize: 2 })).getReader();
40 const first = await reader.read();
41 await reader.cancel();
42 const [whole, hash] = await Promise.all([
43 readStream(await b.readFile("source.txt").stream()),
44 file.hash(),
45 ]);
46 return { first: new TextDecoder().decode(first.value), whole, hash };
47 }).read());
48 assert.deepEqual(await site(), { first: "he", whole: "hello", hash: sha256("hello") });
49 await site();
50 assert.equal(runs, 1);
51
52 await p.write("source.txt", "world");
53 assert.deepEqual(await site(), { first: "wo", whole: "world", hash: sha256("world") });
54 assert.equal(runs, 2);
55});
56
57test("a stream opens its file on the first read, so one that is never read holds nothing open", async () => {
58 await using p = await project({ "source.txt": "hello" });
59 const disk = incremental.Fs.default;
60 let open = 0;
61 const graph = new incremental.Graph({
62 db: null,
63 root: p.dir,
64 fs: new incremental.Fs({
65 ...disk,
66 open: (file) => (open++, disk.open(file)),
67 closeHandle: (handle) => (open--, disk.closeHandle(handle)),
68 }),
69 });
70 const unread = build(graph, (ref) =>
71 ref.step(async (b) => {
72 await b.readFile("source.txt").stream();
73 throw new Error("stopped before reading");
74 }).read());
75 await assert.rejects(unread, incremental.BuildError);
76 assert.equal(open, 0);
77});
78
79test("a symlink is tracked by the path that was read", async () => {
80 await using p = await project({ "a.txt": "a", "b.txt": "b" });
81 await fs.symlink("a.txt", p.path("link.txt"));
82 let runs = 0;
83 const graph = p.graph();
84 const site = () =>
85 build(graph, (ref) =>
86 ref.step(async (b) => {
87 runs++;
88 return await b.readFile("link.txt").text();
89 }).read());
90 assert.equal(await site(), "a");
91 assert.equal(await site(), "a");
92 assert.equal(runs, 1);
93
94 await fs.rm(p.path("link.txt"));
95 await fs.symlink("b.txt", p.path("link.txt"));
96 assert.equal(await site(), "b");
97 assert.equal(runs, 2);
98});
99
100test("a missing file is an input, so creating it re-runs the step", async () => {
101 await using p = await project();
102 let runs = 0;
103 const graph = p.graph();
104 const site = () =>
105 build(graph, (ref) =>
106 ref.step(async (b) => {
107 runs++;
108 return await b.readFile("optional.txt").text().catch((error) => error.code);
109 }).read());
110 assert.equal(await site(), "ENOENT");
111 assert.equal(await site(), "ENOENT");
112 assert.equal(runs, 1);
113 const input = graph.db.inputs().find((input) => input.url.endsWith("/optional.txt"));
114 assert.deepEqual({ kind: input?.kind, hash: input?.hash }, { kind: "missing", hash: "missing" });
115
116 await p.write("optional.txt", "now here");
117 assert.equal(await site(), "now here");
118 assert.equal(runs, 2);
119});
120
121test("a directory read as a file is tracked by its kind", async () => {
122 await using p = await project();
123 await fs.mkdir(p.path("optional"));
124 let runs = 0;
125 const graph = p.graph();
126 const site = () =>
127 build(graph, (ref) =>
128 ref.step(async (b) => {
129 runs++;
130 return await b.readFile("optional").text().catch((error) => error.code);
131 }).read());
132 assert.equal(await site(), "EISDIR");
133 assert.equal(await site(), "EISDIR");
134 assert.equal(runs, 1);
135
136 await fs.rm(p.path("optional"), { recursive: true });
137 await p.write("optional", "now a file");
138 assert.equal(await site(), "now a file");
139 assert.equal(runs, 2);
140});
141
142/** a project in a temporary directory, removed once disposed */
143async function project(files: Record<string, string> = {}) {
144 const dir = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "incremental-")));
145 const write = async (name: string, contents: string) => {
146 await fs.mkdir(path.dirname(path.join(dir, name)), { recursive: true });
147 await fs.writeFile(path.join(dir, name), contents);
148 };
149 for (const [name, contents] of Object.entries(files)) await write(name, contents);
150 return {
151 dir,
152 write,
153 path: (name: string) => path.join(dir, name),
154 /** a graph on the project's database, like a new process would make */
155 graph: () => new incremental.Graph({ db: ".clover/incremental.db", root: dir }),
156 [Symbol.asyncDispose]: () => fs.rm(dir, { recursive: true, force: true }),
157 };
158}
159
160function build<T>(
161 graph: incremental.Graph,
162 buildAndWriteOutput: (ref: incremental.Ref) => Promise<T>,
163): Promise<T> {
164 return graph.incrementalBuild({ progress: new progress.Root(), buildAndWriteOutput });
165}
166
167async function readStream(stream: ReadableStream<Uint8Array>): Promise<string> {
168 const chunks: Uint8Array[] = [];
169 for await (const chunk of stream) chunks.push(chunk);
170 return new TextDecoder().decode(Buffer.concat(chunks));
171}
172
173function sha256(content: string): string {
174 return crypto.createHash("sha256").update(content).digest("base64url");
175}
176
177import assert from "node:assert/strict";
178import * as crypto from "node:crypto";
179import * as fs from "node:fs/promises";
180import * as os from "node:os";
181import * as path from "node:path";
182import * as url from "node:url";
183import { test } from "vitest";
184import * as incremental from "../incremental.ts";
185import * as progress from "../progress.ts";