1// Run one full scan of the file store from this machine, rendering progress
2// in the terminal. In production the indexer service inside the source of
3// truth server does this continuously; this wrapper exists for development
4// (and the nostalgia of watching it go).
5//
6// CLOVER_FILE_RAW=/tmp/store CLOVER_FILE_DERIVED=/tmp/derived \
7// CLOVER_DB=.clover node run file-scan
8export async function main() {
9 const start = performance.now();
10 using _ = log.startWidget({
11 format: ({ now }) => `paper clover's file scanner [${((now - start) / 1000).toFixed(1)}s]`,
12 });
13
14 const scanner = new Scanner({
15 root: Path.resolve(rawFileRoot),
16 progress: progress.globalRoot,
17 settleMs: Number(process.env.CLOVER_SETTLE_MS ?? 10_000),
18 });
19 await scanner.sweep();
20 await scanner.waitIdle();
21 dirmeta.run(progress.globalRoot);
22
23 const orphaned = derived.findOrphanedRoots();
24 for (const orphan of orphaned) {
25 console.info("delete orphaned " + orphan.key);
26 await derived.deleteRootFiles(orphan);
27 derived.deleteRoot(orphan);
28 }
29 await derived.cleanAbandonedTmp();
30
31 console.info(
32 "Updated file viewer index in \x1b[1m"
33 + ((performance.now() - start) / 1000).toFixed(1)
34 + "s\x1b[0m",
35 );
36
37 const { duration, count } = mediaFiles.db.get<{ count: number; duration: number }>(
38 sql`select count(*) as count, sum(duration) as duration from media_files`,
39 );
40 const { derivedSize, derivedCount } = mediaFiles.db.get<{ derivedCount: number; derivedSize: number }>(
41 sql`select count(*) as derivedCount, sum(size) as derivedSize from derived_files`,
42 );
43 const { failed } = mediaFiles.db.get<{ failed: number }>(
44 sql`select count(*) as failed from file_processors where status = 2`,
45 );
46 const canonicalSize = UNWRAP(mediaFiles.getByPath("/")).size;
47
48 console.info();
49 console.info(
50 "Global Stats:\n"
51 + `- File Count: \x1b[1m${count}\x1b[0m\n`
52 + `- Derived Count: \x1b[1m${derivedCount}\x1b[0m\n`
53 + `- Media Duration: \x1b[1m${string.formatDurationLetters(duration)}\x1b[0m\n`
54 + `- Canonical Size: \x1b[1m${string.formatByteSize(canonicalSize)}\x1b[0m\n`
55 + `- Derived Size: \x1b[1m${string.formatByteSize(derivedSize)}\x1b[0m\n`
56 + (failed > 0
57 ? `- \x1b[31mFailed Processors: ${failed}\x1b[0m (select * from file_processors where status = 2)\n`
58 : ""),
59 );
60}
61
62import * as mediaFiles from "#src/file-viewer/models/media-files.ts";
63import * as log from "@clo/lib/log";
64import * as progress from "@clo/lib/progress";
65import * as string from "@clo/lib/string";
66import { sql } from "drizzle-orm";
67
68import { Path } from "#sitegen/path";
69import { UNWRAP } from "@clo/lib/assert";
70
71import * as dirmeta from "#src/file-viewer/indexer/dirmeta.ts";
72import { Scanner } from "#src/file-viewer/indexer/scan.ts";
73import * as derived from "#src/file-viewer/models/derived.ts";
74import { rawFileRoot } from "../paths.ts";