1// ## queries
2test("a step is reused until rows it read change, across restarts", async () => {
3 await using p = await project();
4 const db = p.open("site.db");
5 db.exec("create table posts (id integer primary key, tag text, title text);");
6 db.exec("insert into posts (tag, title) values ('a', 'one'), ('b', 'other');");
7 const byTag = db.prepare("select id, title from posts where tag = ? order by id");
8 let runs = 0;
9 const titles = (graph: incremental.Graph) =>
10 build(graph, (ref) =>
11 ref.step((_, tag) => {
12 runs++;
13 return byTag.all(tag).map((row) => `${row.id} ${row.title}`);
14 }, "a").read());
15 const graph = p.graph();
16 assert.deepEqual(await titles(graph), ["1 one"]);
17 assert.deepEqual(await titles(graph), ["1 one"]);
18 assert.deepEqual(await titles(p.graph()), ["1 one"]);
19 assert.equal(runs, 1);
20
21 db.exec("update posts set title = 'changed' where tag = 'b';");
22 assert.deepEqual(await titles(graph), ["1 one"]);
23 assert.equal(runs, 1);
24
25 db.exec("insert into posts (tag, title) values ('a', 'two');");
26 assert.deepEqual(await titles(graph), ["1 one", "3 two"]);
27 assert.equal(runs, 2);
28});
29
30test("a step that read one row with `get` is reused when other rows change", async () => {
31 await using p = await project();
32 const db = p.open("site.db");
33 db.exec("create table posts (id integer primary key, title text); insert into posts (title) values ('one');");
34 const latest = db.prepare("select title from posts order by id limit 5");
35 let runs = 0;
36 const graph = p.graph();
37 const first = () =>
38 build(graph, (ref) =>
39 ref.step(() => {
40 runs++;
41 return String(latest.get()?.title);
42 }).read());
43 assert.equal(await first(), "one");
44
45 db.exec("insert into posts (title) values ('two');");
46 assert.equal(await first(), "one");
47 assert.equal(runs, 1);
48
49 db.exec("update posts set title = 'first' where id = 1;");
50 assert.equal(await first(), "first");
51 assert.equal(runs, 2);
52});
53
54test("a query that fails is an input, so its step runs again once it works", async () => {
55 await using p = await project();
56 const db = p.open("site.db");
57 let runs = 0;
58 const graph = p.graph();
59 const drafts = () =>
60 build(graph, (ref) =>
61 ref.step(() => {
62 runs++;
63 return db.prepare("select title from drafts").all().length;
64 }).read());
65 await assert.rejects(drafts(), /build has 1 error/);
66 await assert.rejects(drafts(), /build has 1 error/);
67 assert.equal(runs, 1);
68
69 db.exec("create table drafts (title text);");
70 assert.equal(await drafts(), 0);
71 assert.equal(runs, 2);
72});
73
74test("rows read as arrays keep the columns that share a name", async () => {
75 await using p = await project();
76 const db = p.open("site.db");
77 db.exec("create table posts (id integer primary key, title text, parent integer);");
78 db.exec("insert into posts (title, parent) values ('root', null), ('reply', 1);");
79 const thread = db.prepare(
80 "select parent.title, post.title from posts post join posts parent on parent.id = post.parent",
81 );
82 thread.setReturnArrays(true);
83 let runs = 0;
84 const graph = p.graph();
85 const threads = () =>
86 build(graph, (ref) =>
87 ref.step(() => {
88 runs++;
89 return thread.all().map((row) => Object.values(row).join(" > "));
90 }).read());
91 assert.deepEqual(await threads(), ["root > reply"]);
92 assert.deepEqual(await threads(), ["root > reply"]);
93 assert.equal(runs, 1);
94
95 db.exec("update posts set title = 'renamed' where id = 1;");
96 assert.deepEqual(await threads(), ["renamed > reply"]);
97 assert.equal(runs, 2);
98});
99
100test("parameters and integers of any size are read again as the step read them", async () => {
101 await using p = await project();
102 const db = p.open("site.db");
103 // past 2^53, where a number would name another integer
104 const size = 2n ** 60n + 1n;
105 db.exec("create table files (hash blob, size integer, name text);");
106 db.prepare("insert into files values (?, ?, ?)").run(new Uint8Array([1, 2]), size, "big");
107 const bySize = db.prepare("select hash, size from files where hash = ? and size = $size and name = :name");
108 bySize.setReadBigInts(true);
109 let runs = 0;
110 const graph = p.graph();
111 const sizes = () =>
112 build(graph, (ref) =>
113 ref.step(() => {
114 runs++;
115 return bySize.all({ size, name: "big" }, new Uint8Array([1, 2])).map((row) => String(row.size));
116 }).read());
117 assert.deepEqual(await sizes(), [String(size)]);
118 assert.deepEqual(await sizes(), [String(size)]);
119 assert.equal(runs, 1);
120
121 db.exec("update files set size = size + 1;");
122 assert.deepEqual(await sizes(), []);
123 assert.equal(runs, 2);
124});
125
126test("iterating in a step reads every row, and reads outside of a step are left alone", async () => {
127 await using p = await project();
128 const db = p.open("site.db");
129 db.exec("create table posts (title text); insert into posts values ('one'), ('two');");
130 const all = db.prepare("select title from posts order by rowid");
131 assert.deepEqual(all.iterate().next().value, { __proto__: null, title: "one" });
132 let runs = 0;
133 const graph = p.graph();
134 const first = () =>
135 build(graph, (ref) =>
136 ref.step(() => {
137 runs++;
138 return String(all.iterate().next().value?.title);
139 }).read());
140 assert.equal(await first(), "one");
141
142 db.exec("update posts set title = 'second' where title = 'two';");
143 assert.equal(await first(), "one");
144 assert.equal(runs, 2);
145});
146
147test("a database file that was replaced is read again", async () => {
148 await using p = await project();
149 // a write-ahead log left beside a replaced file would be read as part of the new one
150 let db = p.open("site.db", "delete");
151 db.exec("create table posts (title text); insert into posts values ('old');");
152 let runs = 0;
153 const graph = p.graph();
154 const titles = () =>
155 build(graph, (ref) =>
156 ref.step(() => {
157 runs++;
158 return db.prepare("select title from posts").all().map((row) => String(row.title));
159 }).read());
160 assert.deepEqual(await titles(), ["old"]);
161 assert.deepEqual(await titles(), ["old"]);
162
163 const next = p.open("next.db", "delete");
164 next.exec("create table posts (title text); insert into posts values ('new');");
165 next.close();
166 db.close();
167 await fs.rename(p.path("next.db"), p.path("site.db"));
168 db = p.open("site.db", "delete");
169 assert.deepEqual(await titles(), ["new"]);
170 assert.deepEqual(await titles(), ["new"]);
171 assert.equal(runs, 2);
172});
173
174test("a module of a graph can't query while it loads", async () => {
175 await using p = await project();
176 p.open("site.db").exec("create table posts (title text); insert into posts values ('one');");
177 const lib = JSON.stringify(url.fileURLToPath(new URL("../incremental.ts", import.meta.url)));
178 const model = (query: string) =>
179 [
180 `import * as incremental from ${lib};`,
181 "import { DatabaseSync } from 'node:sqlite';",
182 `const db = incremental.trackSqlite(new DatabaseSync(${JSON.stringify(p.path("site.db"))}));`,
183 "const titles = db.prepare('select title from posts');",
184 query,
185 ].join("\n");
186 await fs.writeFile(p.path("package.json"), JSON.stringify({ type: "module" }));
187 await fs.writeFile(p.path("eager.ts"), model("export const all = titles.all();"));
188 await fs.writeFile(p.path("lazy.ts"), model("export const all = () => titles.all();"));
189 const graph = p.graph();
190 const titles = (file: string) =>
191 build(graph, (ref) =>
192 ref.step(async (b, file) => {
193 const { all } = await b.import<{ all: unknown }>(file);
194 return (typeof all === "function" ? all() : all).map((row: { title: string }) => row.title);
195 }, file).read());
196 const error = await titles("eager.ts").catch((error) => error);
197 assert.match(error.errors[0].message, /a module can't read inputs while it loads/);
198 assert.deepEqual(await titles("lazy.ts"), ["one"]);
199});
200
201test("a database in memory can't be tracked", () => {
202 assert.throws(() => incremental.trackSqlite(new DatabaseSync(":memory:")), /can't be read again/);
203});
204
205// ## watching
206test("watching rebuilds when a database changes, checking only what was read from it", async () => {
207 await using p = await project();
208 const posts = p.open("posts.db");
209 const files = p.open("files.db");
210 posts.exec("create table posts (title text); insert into posts values ('one');");
211 files.exec("create table files (name text); insert into files values ('a.txt');");
212 const results = p.graph().incrementalWatch({
213 progress: new progress.Root(),
214 debounce: 10,
215 buildAndWriteOutput: async (ref) => {
216 const titles = ref.step(() => posts.prepare("select title from posts").all().map((row) => String(row.title)));
217 const names = ref.step(() => files.prepare("select name from files").all().map((row) => String(row.name)));
218 return [...await titles.read(), ...await names.read()].join(" ");
219 },
220 });
221 /** the first build after a database changed, and the files of the queries that were checked for it */
222 const changed = async () => {
223 for (let next = await results.next(); !next.done; next = await results.next()) {
224 const queries = next.value.changed.filter((href) => href.startsWith("sqlite:"));
225 if (queries.length === 0) continue;
226 if (!next.value.ok) throw next.value.error;
227 const read = queries.map((href) => path.basename(JSON.parse(decodeURIComponent(href.slice(7))).db));
228 return { value: next.value.value, read };
229 }
230 throw new Error("stopped watching");
231 };
232 const { value: first } = await results.next();
233 assert.ok(first?.ok);
234 assert.equal(first.value, "one a.txt");
235 // each query is one more thing that is watched
236 assert.equal(first.watching.sqlite, 2);
237 assert.match(first.statusLine, /^Watching \d+ files and 2 other sources$/);
238
239 // from this process, on the connection steps read from
240 posts.exec("insert into posts values ('two');");
241 assert.deepEqual(await changed(), { value: "one two a.txt", read: ["posts.db"] });
242
243 // from another process
244 const script = `new (require("node:sqlite").DatabaseSync)(process.argv[1]).exec("update files set name = 'b.txt'")`;
245 child_process.execFileSync(process.execPath, ["-e", script, p.path("files.db")]);
246 assert.deepEqual(await changed(), { value: "one two b.txt", read: ["files.db"] });
247 await results.return();
248});
249
250test("a watch sees a commit made after its build read the rows, before it began watching", async () => {
251 await using p = await project();
252 const db = p.open("site.db");
253 db.exec("create table posts (title text); insert into posts values ('one');");
254 const titles = db.prepare("select title from posts");
255 let commit = true;
256 const results = p.graph().incrementalWatch({
257 progress: new progress.Root(),
258 debounce: 10,
259 files: false,
260 buildAndWriteOutput: async (ref) => {
261 const read = await ref.step(() => titles.all().map((row) => String(row.title)).join(" ")).read();
262 // a build ends, and its watch begins, after this
263 if (commit) commit = false, db.exec("insert into posts values ('two');");
264 return read;
265 },
266 });
267 assert.equal((await results.next()).value?.value, "one");
268 assert.equal((await results.next()).value?.value, "one two");
269 await results.return();
270});
271
272test("two watches of one database each see its commits", async () => {
273 await using p = await project();
274 const db = p.open("site.db");
275 db.exec("create table posts (title text); insert into posts values ('one');");
276 const titles = db.prepare("select title from posts");
277 const watch = async (name: string) => {
278 await fs.mkdir(p.path(name));
279 return new incremental.Graph({ db: p.path(`${name}/incremental.db`), root: p.path(name) }).incrementalWatch({
280 progress: new progress.Root(),
281 debounce: 10,
282 files: false,
283 buildAndWriteOutput: (ref) => ref.step(() => titles.all().map((row) => String(row.title)).join(" ")).read(),
284 });
285 };
286 const [a, b] = [await watch("a"), await watch("b")];
287 await Promise.all([a.next(), b.next()]);
288 db.exec("insert into posts values ('two');");
289 const [left, right] = await Promise.all([a.next(), b.next()]);
290 assert.deepEqual([left.value?.value, right.value?.value], ["one two", "one two"]);
291 await Promise.all([a.return(), b.return()]);
292});
293
294/** a project in a temporary directory, removed once disposed */
295async function project() {
296 const dir = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "incremental-")));
297 const opened: DatabaseSync[] = [];
298 return {
299 path: (name: string) => path.join(dir, name),
300 /** a graph on the project's database, like a new process would make */
301 graph: () => new incremental.Graph({ db: ".clover/incremental.db", root: dir }),
302 /** a connection that steps read from */
303 open(name: string, journal = "wal"): DatabaseSync {
304 const db = incremental.trackSqlite(new DatabaseSync(path.join(dir, name)));
305 db.exec(`pragma journal_mode = ${journal};`);
306 opened.push(db);
307 return db;
308 },
309 async [Symbol.asyncDispose]() {
310 for (const db of opened) if (db.isOpen) db.close();
311 await fs.rm(dir, { recursive: true, force: true });
312 },
313 };
314}
315
316function build<T>(
317 graph: incremental.Graph,
318 buildAndWriteOutput: (ref: incremental.Ref) => Promise<T>,
319): Promise<T> {
320 return graph.incrementalBuild({ progress: new progress.Root(), buildAndWriteOutput });
321}
322
323import assert from "node:assert/strict";
324import * as child_process from "node:child_process";
325import * as fs from "node:fs/promises";
326import * as os from "node:os";
327import * as path from "node:path";
328import { DatabaseSync } from "node:sqlite";
329import * as url from "node:url";
330import { test } from "vitest";
331import * as incremental from "../incremental.ts";
332import * as progress from "../progress.ts";