1// @effect-diagnostics nodeBuiltinImport:off
2// @effect-diagnostics globalTimers:off
3// The launcher supervises the server child for the boot service and must keep
4// working across server versions, so it stays on Node built-ins with no Effect
5// runtime: it is the one part of the executable that cannot depend on the
6// rest of it being loadable.
7import * as NodeChildProcess from "node:child_process";
8import * as NodeCrypto from "node:crypto";
9import * as NodeFS from "node:fs";
10import * as NodeFSP from "node:fs/promises";
11import * as NodePath from "node:path";
12
13import type {
14 PendingServiceUpdate,
15 ServiceLauncherChildMessage,
16 ServiceLauncherContext,
17 ServiceLauncherParentMessage,
18 ServiceState,
19 ServiceUpdateRecord,
20} from "./cloud/serviceProtocol.ts";
21import {
22 compareExactServiceVersions,
23 decodeServiceLauncherChildMessage,
24 isExactServiceVersion,
25 parseServiceState,
26 SERVICE_LAUNCHER_CONTEXT_ENV,
27 SERVICE_LAUNCHER_PROTOCOL,
28 SERVICE_STATE_FILE,
29 SERVICE_RESTART_PENDING_FILE,
30 SERVICE_STOP_MARKER_FILE,
31} from "./cloud/serviceProtocol.ts";
32
33const HANDOFF_DELAY_MS = 2_000;
34/**
35 * How long a trial has to report prepared. Migrations and the open-time
36 * `PRAGMA optimize` count toward it and scale with the database: tens of GB on
37 * a cold page cache take minutes. They run synchronously on the child's main
38 * thread, so it cannot report progress meanwhile. A missed window rolls back
39 * the database, so the next attempt pays the same cost. Crashes are caught at
40 * exit; this only bounds a trial that is alive but stuck.
41 */
42export const PREPARED_TIMEOUT_MS = 30 * 60_000;
43const TERMINATE_GRACE_MS = 5_000;
44// Matches SERVER_EXIT_CODE_STATE_DIR_OWNED in contracts. The standalone
45// launcher stays independent of the Effect-backed contracts package.
46const STATE_DIR_OWNED_EXIT_CODE = 78;
47
48type TerminalStatus = "committed" | "rolled-back" | "failed";
49type ChildRole = "active" | "trial";
50
51interface ManagedChild {
52 readonly version: string;
53 role: ChildRole;
54 readonly process: NodeChildProcess.ChildProcess;
55}
56
57// Mirrors pinnedRuntimePaths: a runtime is an unpacked release archive whose
58// executable runs on its own. Kept inline so this file stays on Node
59// built-ins only.
60const runtimePaths = (baseDir: string, version: string) => {
61 const versionDir = NodePath.join(baseDir, "runtime", "versions", version);
62 // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime.
63 const executableName = process.platform === "win32" ? "t3.exe" : "t3";
64 return {
65 versionDir,
66 entryPath: NodePath.join(versionDir, executableName),
67 sentinelPath: NodePath.join(versionDir, ".install-complete"),
68 };
69};
70
71const runtimeSpawnArguments = (paths: ReturnType<typeof runtimePaths>) => ({
72 command: paths.entryPath,
73 args: ["serve"],
74});
75
76/** SQLite persists across the main file plus its WAL and shared-memory sidecars. */
77const DB_FILE_SUFFIXES = ["", "-wal", "-shm"] as const;
78const RESTORE_MARKER = ".restore-pending";
79
80const databaseBackupDir = (baseDir: string, updateId: string) =>
81 NodePath.join(baseDir, "runtime", "db-backup", updateId);
82
83const databaseBackupFile = (backupDir: string, suffix: (typeof DB_FILE_SUFFIXES)[number]) =>
84 NodePath.join(backupDir, suffix === "" ? "database" : `database${suffix}`);
85
86async function pathExists(target: string): Promise<boolean> {
87 try {
88 await NodeFSP.access(target);
89 return true;
90 } catch (cause) {
91 if (cause instanceof Error && "code" in cause && cause.code === "ENOENT") return false;
92 throw cause;
93 }
94}
95
96// Opened read-write: Windows refuses to flush a handle without write access.
97async function syncFile(filePath: string): Promise<void> {
98 const handle = await NodeFSP.open(filePath, "r+");
99 try {
100 await handle.sync();
101 } finally {
102 await handle.close();
103 }
104}
105
106// Flushes a directory entry so a rename into it survives power loss. Windows
107// has no directory fsync: the handle opens but sync fails with EPERM, and
108// NTFS journals the rename on its own.
109async function syncDirectory(directory: string): Promise<void> {
110 const handle = await NodeFSP.open(directory, "r");
111 try {
112 await handle.sync();
113 } catch (error) {
114 if ((error as NodeJS.ErrnoException).code !== "EPERM") throw error;
115 } finally {
116 await handle.close();
117 }
118}
119
120function runCp(args: ReadonlyArray<string>): Promise<boolean> {
121 return new Promise((resolve) => {
122 NodeChildProcess.execFile("cp", args, (error) => resolve(error === null));
123 });
124}
125
126/**
127 * Copies a database file as a copy-on-write clone where the filesystem has
128 * them (APFS via `cp -c`, btrfs/XFS reflinks via `cp --reflink=auto`), which
129 * takes no extra space and finishes instantly for a database of tens of GB.
130 * Node's own clone flag is not used: on macOS `fs.copyFile` falls back to a
131 * full copy. Anywhere cloning is unavailable, or `cp` fails, this is a
132 * regular copy.
133 */
134async function copyDatabaseFile(source: string, destination: string): Promise<void> {
135 // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime.
136 const platform = process.platform;
137 // `cp -c` replaces the destination with a new file, which would cut a
138 // hard-linked database off from its other links; copy into it instead.
139 const hardLinked =
140 platform === "darwin" &&
141 ((await NodeFSP.stat(destination).catch(() => undefined))?.nlink ?? 1) > 1;
142 const args =
143 platform === "darwin" && !hardLinked
144 ? ["-c", source, destination]
145 : platform === "linux"
146 ? ["--reflink=auto", source, destination]
147 : undefined;
148 if (args === undefined || !(await runCp(args))) await NodeFSP.copyFile(source, destination);
149}
150
151const GIB = 1024 ** 3;
152/**
153 * Clones share blocks, but the trial server diverges from them as it writes,
154 * migrations included, so a cloned backup still needs room for that churn.
155 */
156const CLONED_BACKUP_MIN_HEADROOM_BYTES = GIB;
157const CLONED_BACKUP_HEADROOM_FRACTION = 0.1;
158
159/**
160 * Stats of the database and its existing sidecars, following symlinks. A main
161 * file that cannot be read is an error.
162 */
163async function statDatabaseFiles(dbPath: string): Promise<ReadonlyArray<NodeFS.Stats>> {
164 const stats: Array<NodeFS.Stats> = [];
165 for (const suffix of DB_FILE_SUFFIXES) {
166 const stat = await NodeFSP.stat(`${dbPath}${suffix}`).catch((cause: unknown) => {
167 if (suffix !== "" && (cause as NodeJS.ErrnoException).code === "ENOENT") return undefined;
168 throw cause;
169 });
170 if (stat !== undefined) stats.push(stat);
171 }
172 return stats;
173}
174
175/**
176 * Whether `copyDatabaseFile` will clone the database into the backup
177 * directory rather than copy it.
178 * - macOS: `cp -c` clones within one APFS volume and silently copies
179 * anywhere else, so every existing database file must share the backup
180 * directory's device, whose filesystem type must match the system volume's,
181 * which is always APFS.
182 * - Linux: clone each existing database file with `cp --reflink=always`,
183 * which fails instead of copying when a file cannot be reflinked (another
184 * filesystem, or a btrfs file marked No_COW), then discard the clones.
185 * - Elsewhere: never.
186 */
187async function backupCanClone(dbPath: string, backupRoot: string): Promise<boolean> {
188 // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime.
189 const platform = process.platform;
190 if (platform === "darwin") {
191 const [backupDirectory, backupFs, systemFs] = await Promise.all([
192 NodeFSP.stat(backupRoot),
193 NodeFSP.statfs(backupRoot),
194 NodeFSP.statfs("/"),
195 ]);
196 if (backupFs.type !== systemFs.type) return false;
197 // A file linked from another volume is caught because `stat` follows
198 // symlinks. A hard-linked file is restored by a full copy, so it needs the
199 // full space.
200 return (await statDatabaseFiles(dbPath)).every(
201 (source) => source.dev === backupDirectory.dev && source.nlink === 1,
202 );
203 }
204 if (platform !== "linux") return false;
205 const probe = NodePath.join(backupRoot, `.clone-probe-${NodeCrypto.randomUUID()}`);
206 try {
207 for (const suffix of DB_FILE_SUFFIXES) {
208 const source = `${dbPath}${suffix}`;
209 if (suffix !== "" && !(await pathExists(source))) continue;
210 if (!(await runCp(["--reflink=always", source, `${probe}${suffix}`]))) return false;
211 }
212 return true;
213 } finally {
214 for (const suffix of DB_FILE_SUFFIXES) {
215 await NodeFSP.rm(`${probe}${suffix}`, { force: true }).catch(() => undefined);
216 }
217 }
218}
219
220export interface BackupSpaceProbe {
221 readonly readFreeBytes: (directory: string) => Promise<number>;
222 readonly canClone: (dbPath: string, backupRoot: string) => Promise<boolean>;
223}
224
225const filesystemBackupSpaceProbe: BackupSpaceProbe = {
226 readFreeBytes: async (directory) => {
227 const stats = await NodeFSP.statfs(directory);
228 return stats.bavail * stats.bsize;
229 },
230 canClone: backupCanClone,
231};
232
233/**
234 * Returns a client-visible reason when the pre-update database backup cannot
235 * fit, or undefined when it can. Runs before the server is stopped so a full
236 * disk refuses the update instead of causing an outage and a cold restart.
237 * Throws when the database or the filesystem cannot be inspected.
238 */
239export async function databaseBackupBlocker(
240 baseDir: string,
241 dbPath: string,
242 probe: Partial<BackupSpaceProbe> = {},
243): Promise<string | undefined> {
244 const { readFreeBytes, canClone } = { ...filesystemBackupSpaceProbe, ...probe };
245 const backupRoot = NodePath.join(baseDir, "runtime", "db-backup");
246 await NodeFSP.mkdir(backupRoot, { recursive: true, mode: 0o700 });
247 const [files, free] = await Promise.all([statDatabaseFiles(dbPath), readFreeBytes(backupRoot)]);
248 const size = files.reduce((total, file) => total + file.size, 0);
249 const fullCopyBytes = size * 1.05;
250 if (free >= fullCopyBytes) return undefined;
251 const clonedBytes = Math.max(
252 CLONED_BACKUP_MIN_HEADROOM_BYTES,
253 size * CLONED_BACKUP_HEADROOM_FRACTION,
254 );
255 const needed = (await canClone(dbPath, backupRoot)) ? clonedBytes : fullCopyBytes;
256 if (free >= needed) return undefined;
257 const gb = (bytes: number) => (bytes / GIB).toFixed(1);
258 return `Not enough free disk space to back up the database before updating: ${gb(needed)} GB needed, ${gb(free)} GB free. Free up disk space and try again. The server was not stopped.`;
259}
260
261/**
262 * Snapshots the database once per update before the first trial. A completed
263 * backup is never overwritten because a restarted launcher may be looking at
264 * database writes from an earlier attempt by the same trial.
265 */
266async function backupDatabaseOnce(baseDir: string, pending: PendingServiceUpdate): Promise<void> {
267 const backupDir = databaseBackupDir(baseDir, pending.id);
268 if (await pathExists(backupDir)) return;
269
270 const stagingDir = `${backupDir}.staging`;
271 await NodeFSP.rm(stagingDir, { recursive: true, force: true });
272 await NodeFSP.mkdir(stagingDir, { recursive: true, mode: 0o700 });
273 try {
274 for (const suffix of DB_FILE_SUFFIXES) {
275 const source = `${pending.dbPath}${suffix}`;
276 if (suffix !== "" && !(await pathExists(source))) continue;
277 const destination = databaseBackupFile(stagingDir, suffix);
278 await copyDatabaseFile(source, destination);
279 await syncFile(destination);
280 }
281 await NodeFSP.rename(stagingDir, backupDir);
282 await syncDirectory(NodePath.dirname(backupDir));
283 } catch (cause) {
284 await NodeFSP.rm(stagingDir, { recursive: true, force: true }).catch(() => undefined);
285 throw cause;
286 }
287}
288
289const restoreMarkerPath = (baseDir: string, updateId: string) =>
290 NodePath.join(databaseBackupDir(baseDir, updateId), RESTORE_MARKER);
291
292const databaseRestorePending = (baseDir: string, pending: PendingServiceUpdate) =>
293 pathExists(restoreMarkerPath(baseDir, pending.id));
294
295/** Mark rollback before changing live files so launcher recovery cannot boot a partial restore. */
296async function markDatabaseRestorePending(backupDir: string): Promise<void> {
297 const markerPath = NodePath.join(backupDir, RESTORE_MARKER);
298 if (!(await pathExists(markerPath))) {
299 const handle = await NodeFSP.open(markerPath, "wx", 0o600);
300 try {
301 await handle.sync();
302 } finally {
303 await handle.close();
304 }
305 await syncDirectory(backupDir);
306 }
307}
308
309/**
310 * The file a restore writes for `path`: where its symlinks lead, even when the
311 * last target is missing, as after a crash between `cp -c` removing a file
312 * and cloning over it.
313 */
314async function restoreTarget(path: string): Promise<string> {
315 let current = path;
316 for (let hops = 0; hops < 40; hops++) {
317 try {
318 return await NodeFSP.realpath(current);
319 } catch (cause) {
320 if ((cause as NodeJS.ErrnoException).code !== "ENOENT") throw cause;
321 }
322 const link = await NodeFSP.readlink(current).catch((cause: unknown) => {
323 const code = (cause as NodeJS.ErrnoException).code;
324 if (code === "ENOENT" || code === "EINVAL") return undefined;
325 throw cause;
326 });
327 if (link === undefined) return current;
328 // Joined without normalizing: a `..` in the link must follow directory links
329 // the way the filesystem does, not cancel a path segment.
330 current = NodePath.isAbsolute(link)
331 ? link
332 : `${await NodeFSP.realpath(NodePath.dirname(current))}${NodePath.sep}${link}`;
333 }
334 throw new Error(`Too many symbolic links resolving ${path}.`);
335}
336
337/** Restore is retryable after any process crash while the backup directory remains. */
338async function restoreDatabaseBackup(
339 baseDir: string,
340 pending: PendingServiceUpdate,
341): Promise<void> {
342 const backupDir = databaseBackupDir(baseDir, pending.id);
343 if (!(await pathExists(backupDir))) return;
344
345 await markDatabaseRestorePending(backupDir);
346 const directories = new Set([NodePath.dirname(pending.dbPath)]);
347 for (const suffix of DB_FILE_SUFFIXES) {
348 const livePath = `${pending.dbPath}${suffix}`;
349 const source = databaseBackupFile(backupDir, suffix);
350 if (await pathExists(source)) {
351 // A clone replaces its destination, so restore into the file a symlinked
352 // database resolves to and keep the link itself.
353 const target = await restoreTarget(livePath);
354 await copyDatabaseFile(source, target);
355 await syncFile(target);
356 directories.add(NodePath.dirname(target));
357 } else {
358 await NodeFSP.rm(livePath, { force: true });
359 }
360 }
361 for (const directory of directories) await syncDirectory(directory);
362}
363
364async function discardDatabaseBackup(baseDir: string, updateId: string): Promise<void> {
365 const backupDir = databaseBackupDir(baseDir, updateId);
366 if (!(await pathExists(backupDir))) return;
367 await NodeFSP.rm(backupDir, { recursive: true, force: true });
368 await syncDirectory(NodePath.dirname(backupDir));
369}
370
371export async function readServiceState(filePath: string): Promise<ServiceState> {
372 const contents = await NodeFSP.readFile(filePath, "utf8");
373 const state = parseServiceState(contents);
374 if (state === undefined) throw new Error("Service state is invalid or unsupported.");
375 return state;
376}
377
378/** Durable same-directory replacement used for every runtime state transition. */
379export async function writeServiceState(filePath: string, state: ServiceState): Promise<void> {
380 const directory = NodePath.dirname(filePath);
381 await NodeFSP.mkdir(directory, { recursive: true, mode: 0o700 });
382 const tempPath = NodePath.join(
383 directory,
384 `.${NodePath.basename(filePath)}.${process.pid}.${NodeCrypto.randomUUID()}`,
385 );
386 let handle: NodeFSP.FileHandle | undefined;
387 try {
388 handle = await NodeFSP.open(tempPath, "wx", 0o600);
389 await handle.writeFile(`${JSON.stringify(state, null, 2)}\n`, "utf8");
390 await handle.sync();
391 await handle.close();
392 handle = undefined;
393 await NodeFSP.rename(tempPath, filePath);
394 await syncDirectory(directory);
395 } finally {
396 await handle?.close().catch(() => undefined);
397 await NodeFSP.rm(tempPath, { force: true }).catch(() => undefined);
398 }
399}
400
401async function runtimeExists(baseDir: string, version: string): Promise<boolean> {
402 const paths = runtimePaths(baseDir, version);
403 try {
404 const [entry, sentinel] = await Promise.all([
405 NodeFSP.stat(paths.entryPath),
406 NodeFSP.readFile(paths.sentinelPath, "utf8"),
407 ]);
408 return entry.isFile() && sentinel.trim() === version;
409 } catch {
410 return false;
411 }
412}
413
414function terminalUpdate<S extends TerminalStatus>(input: {
415 readonly pending: PendingServiceUpdate;
416 readonly status: S;
417 readonly reason?: string;
418}): Exclude<ServiceUpdateRecord, PendingServiceUpdate> & { readonly status: S } {
419 return {
420 id: input.pending.id,
421 fromVersion: input.pending.fromVersion,
422 targetVersion: input.pending.targetVersion,
423 status: input.status,
424 ...(input.reason === undefined ? {} : { reason: input.reason }),
425 };
426}
427
428function sendMessage(
429 child: NodeChildProcess.ChildProcess,
430 message: ServiceLauncherParentMessage,
431): Promise<void> {
432 return new Promise((resolve, reject) => {
433 if (!child.connected || child.send === undefined) {
434 reject(new Error("service child IPC is disconnected."));
435 return;
436 }
437 child.send(message, (error) => (error === null ? resolve() : reject(error)));
438 });
439}
440
441function waitForExit(child: NodeChildProcess.ChildProcess): Promise<void> {
442 if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve();
443 return new Promise((resolve) => child.once("exit", () => resolve()));
444}
445
446async function terminateChild(
447 child: NodeChildProcess.ChildProcess,
448 signal: NodeJS.Signals = "SIGTERM",
449): Promise<void> {
450 if (child.exitCode !== null || child.signalCode !== null) return;
451 child.kill(signal);
452 const force = setTimeout(() => child.kill("SIGKILL"), TERMINATE_GRACE_MS);
453 try {
454 await waitForExit(child);
455 } finally {
456 clearTimeout(force);
457 }
458}
459
460const stopMarkerPath = (baseDir: string) =>
461 NodePath.join(baseDir, "runtime", SERVICE_STOP_MARKER_FILE);
462const restartPendingPath = (baseDir: string) =>
463 NodePath.join(baseDir, "runtime", SERVICE_RESTART_PENDING_FILE);
464
465export class Launcher {
466 readonly #baseDir: string;
467 readonly #statePath: string;
468 #state: ServiceState;
469 #child: ManagedChild | null = null;
470 #timer: NodeJS.Timeout | undefined;
471 #transitions: Promise<void> = Promise.resolve();
472 #stopRequested = false;
473 #stopping = false;
474 #done = false;
475 readonly #completion = Promise.withResolvers<void>();
476 readonly #backupSpaceProbe: Partial<BackupSpaceProbe> | undefined;
477
478 constructor(
479 baseDir: string,
480 state: ServiceState,
481 options: { readonly backupSpaceProbe?: Partial<BackupSpaceProbe> } = {},
482 ) {
483 this.#baseDir = baseDir;
484 this.#backupSpaceProbe = options.backupSpaceProbe;
485 this.#statePath = NodePath.join(baseDir, "runtime", SERVICE_STATE_FILE);
486 this.#state = state;
487 }
488
489 async run(): Promise<void> {
490 const onSigterm = () => void this.stop("SIGTERM");
491 const onSigint = () => void this.stop("SIGINT");
492 process.once("SIGTERM", onSigterm);
493 process.once("SIGINT", onSigint);
494 try {
495 this.#enqueue(() => this.#recover());
496 await this.#completion.promise;
497 } finally {
498 process.off("SIGTERM", onSigterm);
499 process.off("SIGINT", onSigint);
500 }
501 }
502
503 #enqueue(transition: () => Promise<void>): void {
504 this.#transitions = this.#transitions
505 .then(transition, transition)
506 .catch((cause: unknown) =>
507 this.#fatal(cause instanceof Error ? cause : new Error(String(cause))),
508 );
509 }
510
511 async #fatal(error: Error): Promise<void> {
512 if (this.#done) return;
513 this.#done = true;
514 this.#stopping = true;
515 this.#clearTimer();
516 const child = this.#child?.process;
517 this.#child = null;
518 if (child !== undefined) await terminateChild(child);
519 this.#completion.reject(error);
520 }
521
522 async stop(signal: NodeJS.Signals): Promise<void> {
523 // This must happen synchronously at signal receipt. A queued update
524 // transition may already be terminating the active child, and that child
525 // needs to see the marker in its shutdown finalizer. KillMode=mixed also
526 // ensures systemd signals the launcher before the rest of the cgroup, and
527 // launchd signals only the job's main process (this launcher), so the
528 // marker lands before the child sees any signal on both platforms.
529 try {
530 NodeFS.writeFileSync(stopMarkerPath(this.#baseDir), "", { mode: 0o600 });
531 } catch {
532 // Err toward keeping the tunnel; the next link or unlink reconciles it.
533 }
534 if (this.#stopRequested || this.#stopping) {
535 await this.#completion.promise.catch(() => undefined);
536 return;
537 }
538 this.#stopRequested = true;
539 this.#clearTimer();
540 this.#enqueue(async () => {
541 // Let an update transition already in progress start its replacement
542 // before this queued stop tears it down. That replacement owns the
543 // pre-activation tunnel cleanup path and observes the marker above.
544 this.#stopping = true;
545 const child = this.#child?.process;
546 this.#child = null;
547 if (child !== undefined) await terminateChild(child, signal);
548 this.#done = true;
549 this.#completion.resolve();
550 });
551 await this.#completion.promise.catch(() => undefined);
552 }
553
554 #clearTimer(): void {
555 clearTimeout(this.#timer);
556 this.#timer = undefined;
557 }
558
559 async #recover(): Promise<void> {
560 // A fresh launcher means servers are running again: any stop marker from
561 // a previous explicit stop is stale and must not make a future update
562 // handoff release its tunnel. A restart deferred by `t3 update` is done
563 // no matter who restarted the service, but only once this launcher is
564 // the version the marker waits for: a launcher that came up between the
565 // CLI writing the marker and writing the new state still runs the old
566 // version, and the marker has to outlive it.
567 await NodeFSP.rm(stopMarkerPath(this.#baseDir), { force: true }).catch(() => undefined);
568 const restartPending = restartPendingPath(this.#baseDir);
569 const awaitedVersion = await NodeFSP.readFile(restartPending, "utf8").catch(() => undefined);
570 if (awaitedVersion?.trim() === this.#state.activeVersion) {
571 await NodeFSP.rm(restartPending, { force: true }).catch(() => undefined);
572 }
573 const update = this.#state.update;
574 if (update?.status !== "pending") {
575 if (update !== undefined) {
576 await discardDatabaseBackup(this.#baseDir, update.id).catch(() => undefined);
577 }
578 await this.#startChild(this.#state.activeVersion, "active", update);
579 return;
580 }
581 if (await databaseRestorePending(this.#baseDir, update)) {
582 await this.#returnToPrevious(update, "failed", "rollback-interrupted");
583 return;
584 }
585 if (!(await runtimeExists(this.#baseDir, update.targetVersion))) {
586 await this.#returnToPrevious(update, "failed", "target-runtime-missing");
587 return;
588 }
589 await this.#startTrial(update);
590 }
591
592 async #startTrial(pending: PendingServiceUpdate): Promise<void> {
593 // The previous child is dead here, so all three SQLite files are quiescent.
594 try {
595 await backupDatabaseOnce(this.#baseDir, pending);
596 } catch {
597 await this.#returnToPrevious(pending, "failed", "db-backup-failed");
598 return;
599 }
600 try {
601 await this.#startChild(pending.targetVersion, "trial", pending);
602 } catch {
603 await this.#returnToPrevious(pending, "failed", "candidate-start-failed");
604 }
605 }
606
607 async #startChild(version: string, role: ChildRole, update?: ServiceUpdateRecord): Promise<void> {
608 if (this.#stopping) return;
609 if (!(await runtimeExists(this.#baseDir, version))) {
610 throw new Error(`Selected t3@${version} runtime is missing or incomplete.`);
611 }
612 if (this.#stopping) return;
613 const paths = runtimePaths(this.#baseDir, version);
614 const context: ServiceLauncherContext = {
615 protocol: SERVICE_LAUNCHER_PROTOCOL,
616 childVersion: version,
617 ...(update === undefined ? {} : { update }),
618 };
619 const spawnArguments = runtimeSpawnArguments(paths);
620 const child = NodeChildProcess.spawn(spawnArguments.command, spawnArguments.args, {
621 env: { ...process.env, [SERVICE_LAUNCHER_CONTEXT_ENV]: JSON.stringify(context) },
622 stdio: ["inherit", "inherit", "inherit", "ipc"],
623 });
624 await new Promise<void>((resolve, reject) => {
625 const onError = (error: Error) => reject(error);
626 child.once("error", onError);
627 child.once("spawn", () => {
628 child.removeListener("error", onError);
629 child.on("error", (error) => this.#enqueue(() => Promise.reject(error)));
630 resolve();
631 });
632 });
633 if (this.#stopping) {
634 await terminateChild(child);
635 return;
636 }
637
638 const managed: ManagedChild = {
639 version,
640 role,
641 process: child,
642 };
643 this.#child = managed;
644 child.on("message", (value) => {
645 const message = decodeServiceLauncherChildMessage(value);
646 if (message !== undefined) this.#enqueue(() => this.#handleMessage(managed, message));
647 });
648 child.once("exit", (code, signal) =>
649 this.#enqueue(() => this.#handleExit(managed, code, signal)),
650 );
651
652 if (role === "trial") {
653 this.#timer = setTimeout(
654 () => this.#enqueue(() => this.#handlePreparedTimeout(managed)),
655 PREPARED_TIMEOUT_MS,
656 );
657 }
658 }
659
660 async #handleMessage(child: ManagedChild, message: ServiceLauncherChildMessage): Promise<void> {
661 if (this.#child !== child || this.#stopping) return;
662 if (message.type === "request-update") {
663 await this.#handleUpdateRequest(child, message);
664 return;
665 }
666 await this.#handlePrepared(child, message.updateId);
667 }
668
669 async #handleUpdateRequest(
670 child: ManagedChild,
671 message: Extract<ServiceLauncherChildMessage, { readonly type: "request-update" }>,
672 ): Promise<void> {
673 const reject = (reason: string) =>
674 sendMessage(child.process, { type: "update-rejected", reason });
675 if (child.role !== "active") {
676 await reject("Only the active server can request an update.");
677 return;
678 }
679 if (child.version !== this.#state.activeVersion) {
680 await reject("The requesting server is not the selected active version.");
681 return;
682 }
683 if (this.#state.update?.status === "pending") {
684 await reject("Another server update is already pending.");
685 return;
686 }
687 if (!isExactServiceVersion(message.targetVersion)) {
688 await reject("The requested target is not an exact version.");
689 return;
690 }
691 if (compareExactServiceVersions(message.targetVersion, child.version) <= 0) {
692 await reject("Remote updates must select a newer server version.");
693 return;
694 }
695 if (!NodePath.isAbsolute(message.dbPath)) {
696 await reject("The requested database path is not absolute.");
697 return;
698 }
699 if (!(await runtimeExists(this.#baseDir, message.targetVersion))) {
700 await reject("The requested target runtime is missing or incomplete.");
701 return;
702 }
703 // A check that cannot run refuses the update too: the alternative is
704 // stopping the server without knowing whether the backup fits.
705 const backupBlocker = await databaseBackupBlocker(
706 this.#baseDir,
707 message.dbPath,
708 this.#backupSpaceProbe,
709 ).catch(
710 (error: unknown) =>
711 `Could not check that the database backup fits before updating: ${error instanceof Error ? error.message : String(error)}. The server was not stopped.`,
712 );
713 // The child may have exited during the check. Its queued exit transition
714 // reports that; answering it or recording a pending update would not.
715 if (!child.process.connected) return;
716 if (backupBlocker !== undefined) {
717 await reject(backupBlocker);
718 return;
719 }
720
721 const pending: PendingServiceUpdate = {
722 id: NodeCrypto.randomUUID(),
723 fromVersion: child.version,
724 targetVersion: message.targetVersion,
725 dbPath: message.dbPath,
726 status: "pending",
727 };
728 const next: ServiceState = { ...this.#state, update: pending };
729 await writeServiceState(this.#statePath, next);
730 this.#state = next;
731 await sendMessage(child.process, { type: "update-accepted", updateId: pending.id });
732 this.#timer = setTimeout(() => this.#enqueue(() => this.#beginTrial(child)), HANDOFF_DELAY_MS);
733 }
734
735 async #beginTrial(child: ManagedChild): Promise<void> {
736 const pending = this.#state.update;
737 if (this.#child !== child || child.role !== "active" || pending?.status !== "pending") {
738 return;
739 }
740 this.#timer = undefined;
741 this.#child = null;
742 await terminateChild(child.process);
743 await this.#startTrial(pending);
744 }
745
746 async #handlePrepared(child: ManagedChild, updateId: string): Promise<void> {
747 const pending = this.#state.update;
748 if (
749 child.role !== "trial" ||
750 pending?.status !== "pending" ||
751 pending.id !== updateId ||
752 pending.targetVersion !== child.version
753 ) {
754 if (child.role === "trial" && pending?.status === "pending") {
755 await this.#returnToPrevious(pending, "rolled-back", "invalid-prepared", child);
756 return;
757 }
758 throw new Error("Trial child reported prepared for an unexpected update.");
759 }
760 this.#clearTimer();
761 const committed = terminalUpdate({ pending, status: "committed" });
762 const next: ServiceState = {
763 ...this.#state,
764 activeVersion: pending.targetVersion,
765 update: committed,
766 };
767 await writeServiceState(this.#statePath, next);
768 this.#state = next;
769 child.role = "active";
770 await discardDatabaseBackup(this.#baseDir, committed.id).catch(() => undefined);
771 await sendMessage(child.process, { type: "committed", updateId: committed.id });
772 }
773
774 async #handlePreparedTimeout(child: ManagedChild): Promise<void> {
775 const pending = this.#state.update;
776 if (this.#child !== child || child.role !== "trial" || pending?.status !== "pending") {
777 return;
778 }
779 this.#timer = undefined;
780 await this.#returnToPrevious(pending, "rolled-back", "prepared-timeout", child);
781 }
782
783 async #handleExit(
784 child: ManagedChild,
785 code: number | null,
786 signal: NodeJS.Signals | null,
787 ): Promise<void> {
788 if (this.#child !== child || this.#stopping) return;
789 this.#child = null;
790 if (code === STATE_DIR_OWNED_EXIT_CODE) {
791 await this.#idleWhileStateDirOwned();
792 return;
793 }
794 if (child.role === "trial") {
795 this.#clearTimer();
796 const pending = this.#state.update;
797 if (pending?.status !== "pending") {
798 throw new Error("Trial child exited without matching pending state.");
799 }
800 await this.#returnToPrevious(
801 pending,
802 "rolled-back",
803 `candidate-exited:${String(code ?? signal ?? "unknown")}`,
804 );
805 return;
806 }
807
808 this.#clearTimer();
809 const pending = this.#state.update;
810 if (pending?.status === "pending") {
811 await this.#startTrial(pending);
812 return;
813 }
814 throw new Error(`Active child exited unexpectedly (${String(code ?? signal ?? "unknown")}).`);
815 }
816
817 /**
818 * Another server owns the T3 home. Exiting would make systemd or launchd
819 * start another refused server, so stay idle until stopped or restarted.
820 * A pending update fails without restoring its backup: the other owner may
821 * have written to the database since.
822 */
823 async #idleWhileStateDirOwned(): Promise<void> {
824 this.#clearTimer();
825 const pending = this.#state.update;
826 if (pending?.status === "pending") {
827 const next: ServiceState = {
828 ...this.#state,
829 activeVersion: pending.fromVersion,
830 update: terminalUpdate({ pending, status: "failed", reason: "state-dir-owned" }),
831 };
832 await writeServiceState(this.#statePath, next);
833 this.#state = next;
834 await discardDatabaseBackup(this.#baseDir, pending.id).catch(() => undefined);
835 }
836 // Signal listeners alone do not keep Node alive.
837 this.#timer = setInterval(() => {}, 2_147_483_647);
838 process.stderr.write(
839 "[service-launcher] Another T3 Code server owns this T3 home. Stop it, then restart this service.\n",
840 );
841 }
842
843 async #returnToPrevious(
844 pending: PendingServiceUpdate,
845 status: "rolled-back" | "failed",
846 reason: string,
847 child?: ManagedChild,
848 ): Promise<void> {
849 if (child !== undefined) {
850 this.#child = null;
851 await terminateChild(child.process);
852 }
853 await restoreDatabaseBackup(this.#baseDir, pending);
854 const outcome = terminalUpdate({ pending, status, reason });
855 const next: ServiceState = {
856 ...this.#state,
857 activeVersion: pending.fromVersion,
858 update: outcome,
859 };
860 await writeServiceState(this.#statePath, next);
861 this.#state = next;
862 await discardDatabaseBackup(this.#baseDir, pending.id).catch(() => undefined);
863 await this.#startChild(next.activeVersion, "active", outcome);
864 }
865}
866
867export async function main(): Promise<void> {
868 const baseDir = process.env.T3CODE_HOME?.trim();
869 if (baseDir === undefined || baseDir === "") {
870 throw new Error("T3CODE_HOME is required by the T3 Code service launcher.");
871 }
872 const statePath = NodePath.join(baseDir, "runtime", SERVICE_STATE_FILE);
873 const state = await readServiceState(statePath);
874 await new Launcher(baseDir, state).run();
875}