1/**
2 * build script for paperclover.net. the root entry point for the site.
3 * @module
4 */
5const root = process.cwd();
6const out = ".clover/o";
7const globalCss = "src/global.css";
8const toRel = (str: string) => path.relative(root, str).replaceAll("\\", "/");
9
10export async function main(graph: incremental.Graph) {
11 if (process.argv.includes("--once")) {
12 await graph.incrementalBuild({ buildAndWriteOutput });
13 return;
14 }
15
16 // in production nothing but a deploy changes the files, so they are read once and not watched
17 const server = await incremental.serve({
18 graph,
19 buildAndWriteOutput,
20 watchFilesystem: !process.argv.includes("--prod"),
21 });
22 // requests in flight finish before the process leaves
23 const stop = () => void server.close().then(() => process.exit(0));
24 process.on("SIGINT", stop);
25 process.on("SIGTERM", stop);
26 for await (const result of server.watch()) {
27 if (result.value) server.setHandler(loadBackend(server, result.value));
28 }
29}
30
31type Respond = (request: Request) => Response | Promise<Response>;
32
33/** hand a build to the code that serves it, loaded by the graph so that it reloads with its code */
34async function loadBackend({ graph, progress }: incremental.Server<Output>, output: Output): Promise<Respond> {
35 using _ = progress.start("load backend");
36 const sitegen = await graph.import<typeof import("../framework/lib/sitegen.ts")>("framework/lib/sitegen.ts");
37 sitegen.setGraph(graph);
38 const assets = await graph.import<typeof import("../framework/lib/assets.ts")>("framework/lib/assets.ts");
39 await assets.load(output.assets, `${out}/asset`);
40
41 // views are loaded before any request needs one, so serving never waits on the graph
42 const modules: Parameters<typeof view.load>[1] = {};
43 for (const { id, file } of Object.values(output.views)) modules[id] = await graph.import(file);
44 const view = await graph.import<typeof import("../framework/lib/view.ts")>("framework/lib/view.ts");
45 view.load(output, modules);
46
47 const backend = await graph.import<{ default: { fetch: Respond } }>("src/backend.ts");
48 return backend.default.fetch;
49}
50
51export interface Output {
52 /** by pathname. the bytes are in `.clover/o/asset`, by hash, also as `.gz` and `.zst` */
53 assets: Record<string, Asset>;
54 /** pages rendered per request, by id */
55 views: Record<string, ts.DeepReadonly<PreparedView>>;
56 /** the step that bundled the client scripts a page can include, by script id */
57 scripts: incremental.Step<{ scripts: Record<string, string> }>;
58 /** the steps that bundled stylesheets, by the style key of a page or view */
59 styles: Record<string, incremental.Step<string>>;
60 /** what every page's metadata starts from */
61 meta: SiteMeta;
62}
63
64export interface Asset {
65 hash: string;
66 headers: Record<string, string>;
67}
68
69export async function buildAndWriteOutput(ref: incremental.Ref): Promise<Output> {
70 using incr = ref.start("sitegen!");
71
72 const site = await incr.step(async (b) => {
73 const { siteSections, fonts, meta } = await b.import<typeof import("./site.ts")>("./src/site.ts");
74 // relative paths, so what a build caches still fits once the project has moved
75 const sections = siteSections.map(({ root, base = "/" }) => ({ root: toRel(root), base }));
76 return { sections, fonts, meta };
77 }).read();
78 const files = await discoverAllFiles(incr, site.sections);
79
80 const aliasing = Array.from(Map.groupBy([...files.staticFiles, ...files.pages], (item) => item.id))
81 .filter(([, items]) => items.length > 1);
82 if (aliasing.length > 0) {
83 throw new Error(
84 `source files alias:\n`
85 + aliasing.map(([id, items]) => ` - ${id}\n` + items.map((item) => " - " + item.file).join("\n")).join("\n"),
86 );
87 }
88
89 // -- perform build-time rendering --
90 const fontFiles = site.fonts.flatMap((font) => {
91 const source = incr.step(fonts.fetchFont, font.name, font.sources);
92 return font.subsets.map((subset) => incr.step(fonts.subsetFont, source, subset, site.meta.base));
93 });
94 const staticFiles = files.staticFiles.map((item) => incr.step((b, file) => b.readFile(file).bytes(), item.file));
95 const pages = files.pages.map((item) => incr.step(preparePage, item, site.meta));
96 const views = await Promise.all(files.views.map((item) => incr.step(prepareView, item).read()));
97 const prepared = await Promise.all(pages.map((page) => page.read()));
98 const routes = [...views, ...prepared];
99
100 // -- page resources --
101 const bundleScripts = incr.step(bundle.bundleClientJavaScript, {
102 clientRefs: Array.from(new Set(routes.flatMap((route) => route.clientRefs))).sort(),
103 publicScripts: files.scripts.map((item) => getScriptId(path.resolve(root, item.file))).sort(),
104 });
105 const styles = new Map<string, incremental.Step<string>>();
106 for (const { styleKey, cssImports, theme } of routes) {
107 if (styles.has(styleKey)) continue;
108 styles.set(styleKey, incr.step(css.bundleCssFiles, { cssImports: [...cssImports], theme }));
109 }
110
111 // -- assemble page assets --
112 const documents = pages.map((page, i) =>
113 incr.step(assemblePage, page, UNWRAP(styles.get(prepared[i]!.styleKey)), bundleScripts)
114 );
115
116 // -- write assets, once for all the pathnames with the same bytes --
117 // each asset's bytes are a step of their own, so an asset is hashed and
118 // written only when its bytes change. assets are listed in the order the
119 // files were found, so the same site lists them the same way.
120 const { assets: scriptAssets } = await bundleScripts.read();
121 const fontAssets = await Promise.all(fontFiles.map((font) => font.read()));
122 const html = { "content-type": "text/html;charset=utf8" };
123 const sources: Array<{ pathname: string; bytes: incremental.Step<string | Uint8Array>; headers?: typeof html }> = [
124 ...Object.keys(scriptAssets).map((pathname) => ({
125 pathname,
126 bytes: incr.step(scriptAsset, bundleScripts, pathname),
127 })),
128 ...fontFiles.flatMap((font, i) =>
129 Object.keys(fontAssets[i]!).map((pathname) => ({ pathname, bytes: incr.step(fontAsset, font, pathname) }))
130 ),
131 ...files.staticFiles.map((item, i) => ({ pathname: item.id, bytes: staticFiles[i]! })),
132 ...files.pages.map((item, i) => ({ pathname: item.id, bytes: documents[i]!, headers: html })),
133 ];
134 const hashes = await Promise.all(sources.map(({ bytes }) => incr.step(hashAsset, bytes).read()));
135 const assets: Output["assets"] = {};
136 const writes = new Map<string, Promise<unknown>>();
137 sources.forEach(({ pathname, bytes, headers }, i) => {
138 ASSERT(pathname.startsWith("/") && !(pathname in assets), () => `asset ${pathname} is written twice`);
139 const hash = hashes[i]!;
140 assets[pathname] = {
141 hash,
142 headers: { "content-type": mime.for(pathname), ...headers, etag: JSON.stringify(hash) },
143 };
144 if (!writes.has(hash)) writes.set(hash, incr.step(writeAsset, hash, bytes).read());
145 });
146 await Promise.all(writes.values());
147 await incr.writeFile(`${out}/asset.json`, JSON.stringify(assets));
148
149 // -- types of what a backend can ask for by name --
150 const union = (keys: string[]) => keys.map((key) => JSON.stringify(key)).join(" | ") || "never";
151 const viewTypes = views.map((view) =>
152 ` ${JSON.stringify(view.id)}: typeof import(${JSON.stringify("../../" + view.file)});`
153 );
154 await Promise.all([
155 incr.writeFile(".clover/ts/asset.d.ts", `export type AssetKey = ${union(Object.keys(assets))};\n`),
156 incr.writeFile(".clover/ts/view.d.ts", `export interface RegisteredViews {\n${viewTypes.join("\n")}\n}\n`),
157 ]);
158
159 return {
160 assets,
161 views: Object.fromEntries(views.map((view) => [view.id, view])),
162 scripts: bundleScripts,
163 styles: Object.fromEntries(styles),
164 meta: site.meta,
165 };
166}
167
168async function discoverAllFiles(
169 ref: incremental.Ref,
170 sections: readonly Readonly<sg.Section>[],
171) {
172 using node = ref.start("discover files");
173 const found = await Promise.all(
174 sections.map(({ root, base = "/" }) =>
175 node.step(scanSiteSection, {
176 root,
177 base: base.endsWith("/") ? base : base + "/",
178 }).read()
179 ),
180 );
181 return {
182 staticFiles: found.flatMap((section) => section.staticFiles),
183 pages: found.flatMap((section) => section.pages),
184 views: found.flatMap((section) => section.views),
185 scripts: found.flatMap((section) => section.scripts),
186 };
187}
188
189export async function scanSiteSection(b: incremental.Build, section: Readonly<sg.Section>) {
190 using _ = b.start(section.root);
191 // Static files are compressed and served as-is.
192 // - "{section}/static/*.png"
193 let staticFiles: sg.FileItem[] = [];
194 // Pages are rendered then served as static files.
195 // - "{section}/pages/*.
196 let pages: sg.FileItem[] = [];
197 // Views are dynamically rendered pages called via backend code.
198 // - "{section}/views/*.tsx"
199 let views: sg.FileItem[] = [];
200 // Public scripts are bundled for the client as static assets under "/js/[...]"
201 // This is used for the file viewer's canvases.
202 // Note that '.client.ts' can be placed anywhere in the file structure.
203 // - "{section}/scripts/*.client.ts"
204 let scripts: sg.FileItem[] = [];
205
206 const sectionPath = (...sub: string[]) => path.join(section.root, ...sub);
207 const rootPrefix = section.root === "src"
208 ? ""
209 : path.relative("src", section.root) + "/";
210 const kinds = [
211 {
212 dir: sectionPath("pages"),
213 list: pages,
214 prefix: section.base,
215 include: [".marko", ".mdo"],
216 exclude: [".client.ts"],
217 page: true,
218 },
219 {
220 dir: sectionPath("static"),
221 list: staticFiles,
222 prefix: section.base,
223 ext: true,
224 page: true,
225 },
226 {
227 dir: sectionPath("scripts"),
228 list: scripts,
229 prefix: rootPrefix,
230 include: [".client.ts"],
231 page: false,
232 },
233 {
234 dir: sectionPath("views"),
235 list: views,
236 prefix: rootPrefix,
237 include: [".marko", ".mdo"],
238 exclude: [".client.ts"],
239 page: false,
240 },
241 ];
242 for (const kind of kinds) {
243 const {
244 dir,
245 list,
246 prefix,
247 include = [""],
248 exclude = [],
249 ext = false,
250 } = kind;
251
252 let items;
253 try {
254 items = await b.readDirRecursive(dir);
255 } catch (err) {
256 if (exception.code(err) === "ENOENT") continue;
257 throw err;
258 }
259 for (const subPath of items) {
260 const file = path.join(dir, subPath);
261 if (file.endsWith("/")) continue;
262 if (!include.some((e) => subPath.endsWith(e))) continue;
263 if (exclude.some((e) => subPath.endsWith(e))) continue;
264 const trim = ext
265 ? subPath
266 : subPath.slice(0, -path.extname(subPath).length).replaceAll(".", "/");
267 let id = prefix + trim.replaceAll("\\", "/");
268 if (kind.page && id.endsWith("/index")) {
269 id = id.slice(0, -"/index".length) || "/";
270 }
271 list.push({ id, file: toRel(file) });
272 }
273 }
274
275 return { staticFiles, pages, views, scripts };
276}
277
278type Route = sg.ViewExports & { layout?: { default?: unknown; theme?: css.Theme } };
279// what a step can hold of a template: no url, and no function
280type SiteMeta = Omit<meta.Template, "base" | "titleTemplate"> & { base: string };
281
282/** render a page that is the same for every request */
283export async function preparePage(b: incremental.Build, item: Readonly<sg.FileItem>, template: SiteMeta) {
284 using _ = b.start(item.file);
285 try {
286 const { default: Page, meta: metadata, theme: pageTheme, layout } = await b.import<Route>(item.file);
287 if (!Page) throw new Error("Page is missing a 'default' export.");
288 if (!metadata) throw new Error("Page is missing 'meta' export with a title.");
289
290 const theme: css.Theme = { ...css.defaultTheme, ...layout?.theme ?? pageTheme };
291 const cssImports = await findCssImports(b, item.file);
292
293 const renderedMeta = meta.toHtml({ ...template, base: new URL(template.base) }, {
294 canonical: item.id,
295 ...typeof metadata === "function" ? await metadata({ ssr: true }) : metadata,
296 });
297 if (!renderedMeta.includes("<title>")) {
298 throw new Error("Page is missing 'meta.title'. " + "All pages need a title tag.");
299 }
300
301 // components add the scripts they need while rendering. marko would pick a
302 // random id for the render, and a step has to return the same html every time
303 const cloverScripts = new Set<string>();
304 const $global = { renderId: "_" };
305 const html = String(await SitegenWrap.render({ $global, cloverScripts, page: Page, attrs: {} }));
306
307 return {
308 id: item.id,
309 file: item.file,
310 html,
311 meta: renderedMeta,
312 cssImports,
313 theme,
314 styleKey: css.styleKey(cssImports, theme),
315 clientRefs: Array.from(cloverScripts),
316 };
317 } catch (e) {
318 exception.rethrowWithMetadata(e, { page: item.id });
319 }
320}
321
322/** what a server needs beside a view's module to render it per request */
323export async function prepareView(b: incremental.Build, item: Readonly<sg.FileItem>) {
324 using _ = b.start(item.file);
325 const module = await b.import<Partial<Route>>(item.file);
326 if (!module.meta) throw new Error(`View is missing 'export const meta'`);
327 if (!module.default) throw new Error(`View is missing a default export.`);
328 const theme: css.Theme = { ...css.defaultTheme, ...module.layout?.theme ?? module.theme };
329 const cssImports = await findCssImports(b, item.file);
330 // a view renders with props no build knows, so it gets every script it could include
331 const clientRefs = (await b.trackModule(item.file)).flatMap((module) =>
332 (module.data as { scannedClientRefs?: string[] } | null)?.scannedClientRefs ?? []
333 );
334 return {
335 id: item.id,
336 file: item.file,
337 cssImports,
338 theme,
339 clientRefs: Array.from(new Set(clientRefs)),
340 hasLayout: !!module.layout?.default,
341 styleKey: css.styleKey(cssImports, theme),
342 };
343}
344export type PreparedView = Awaited<ReturnType<typeof prepareView>>;
345
346/** the stylesheets a module imports, in import order, after the one every page has */
347async function findCssImports(b: incremental.Build, file: string): Promise<string[]> {
348 const modules = await b.trackModule(file);
349 const imports = new Map(modules.map((module) => [module.url, module.imports]));
350 const found = new Set([globalCss]);
351 const seen = new Set<string>();
352 const visit = (href: string) => {
353 if (seen.has(href)) return;
354 seen.add(href);
355 if (href.endsWith(".css")) return void found.add(toRel(url.fileURLToPath(href)));
356 imports.get(href)?.forEach(visit);
357 };
358 visit(UNWRAP(modules[0]).url);
359 return Array.from(found);
360}
361
362/** the whole html document of a page, with its styles and scripts inline */
363export async function assemblePage(
364 b: incremental.Build,
365 page: incremental.Step<Awaited<ReturnType<typeof preparePage>>>,
366 style: incremental.Step<string>,
367 scripts: incremental.Step<{ scripts: Record<string, string> }>,
368) {
369 const { html, meta, clientRefs } = await b.readStep(page);
370 const scriptMap = (await b.readStep(scripts)).scripts;
371 return sg.wrapDocument({
372 body: html,
373 head: meta,
374 inlineCss: await b.readStep(style),
375 scripts: sg.joinScripts(clientRefs.map((ref) => UNWRAP(scriptMap[ref], `Missing script ${ref}`))),
376 });
377}
378
379/** one of the scripts the bundle made, so writing it follows that script alone */
380async function scriptAsset(
381 b: incremental.Build,
382 bundle: incremental.Step<{ assets: Record<string, string> }>,
383 pathname: string,
384) {
385 return UNWRAP((await b.readStep(bundle)).assets[pathname]);
386}
387
388/** one of the files of a font subset */
389async function fontAsset(b: incremental.Build, font: incremental.Step<Record<string, Uint8Array>>, pathname: string) {
390 return UNWRAP((await b.readStep(font))[pathname]);
391}
392
393/** the hash that an asset's file is named by */
394async function hashAsset(b: incremental.Build, bytes: incremental.Step<string | Uint8Array>) {
395 return crypto.createHash("sha1").update(await b.readStep(bytes)).digest("hex");
396}
397
398/** an asset's file and its compressed copies, named by its hash */
399async function writeAsset(b: incremental.Build, hash: string, bytes: incremental.Step<string | Uint8Array>) {
400 const content = await b.readStep(bytes);
401 const [gzipped, zstded] = await Promise.all([gzip(content), zstd(content)]);
402 await Promise.all([
403 b.writeFile(`${out}/asset/${hash}`, content),
404 b.writeFile(`${out}/asset/${hash}.gz`, gzipped),
405 b.writeFile(`${out}/asset/${hash}.zst`, zstded),
406 ]);
407}
408
409const gzip = util.promisify(zlib.gzip);
410const zstd = util.promisify(zlib.zstdCompress);
411
412import * as sg from "#sitegen";
413import { ASSERT, UNWRAP } from "@clo/lib/assert";
414import * as exception from "@clo/lib/exception";
415import * as incremental from "@clo/lib/incremental";
416import * as meta from "@clo/lib/meta";
417import * as mime from "@clo/lib/mime";
418import type * as ts from "@clo/lib/ts";
419import * as crypto from "node:crypto";
420import path from "node:path";
421import * as url from "node:url";
422import * as util from "node:util";
423import * as zlib from "node:zlib";
424import * as bundle from "../framework/bundle.ts";
425import * as css from "../framework/css.ts";
426import * as fonts from "../framework/font.ts";
427import { getScriptId } from "../framework/marko.ts";
428import SitegenWrap from "../framework/sitegen-wrap.marko";