authorgravatar for me@t3.ggTheo Browne <me@t3.gg> 2026-08-04 18:41:56-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-08-04 18:41:56-07:00
log7b38fb5c6b0fb46a7dadaf33e1739726be3dc104
tree2e4529d1a70ec9314281d6e8c51a493291ebb013
parent7537adc30f41059afcfeb8bd20368cb0acfce293
signature Signed by PGP key B5690EEEBB952194

fix(server): allow remote updates with database migrations (#5374)

17 files changed, 437 insertions(+), 157 deletions(-)

apps/server/src/cli/connect.test.ts+10
...@@ -10,6 +10,7 @@ import * as Option from "effect/Option";...@@ -10,6 +10,7 @@ import * as Option from "effect/Option";
10import * as References from "effect/References";10import * as References from "effect/References";
11import * as Terminal from "effect/Terminal";11import * as Terminal from "effect/Terminal";
1212
13import * as BootService from "../cloud/bootService.ts";
13import {14import {
14 acquireRelayClientForLink,15 acquireRelayClientForLink,
15 formatHeadlessAuthorizationPrompt,16 formatHeadlessAuthorizationPrompt,
...@@ -63,6 +64,15 @@ it.effect("treats cancelling optional background setup as a successful skip", ()...@@ -63,6 +64,15 @@ it.effect("treats cancelling optional background setup as a successful skip", ()
63 }),64 }),
64);65);
6566
67it.effect("keeps a successful connection when a remote service update is pending", () =>
68 Effect.gen(function* () {
69 const result = yield* recoverServiceOnboardingOffer(
70 Effect.fail(new BootService.BootServiceUpdatePendingError()),
71 );
72 assert.isFalse(result);
73 }),
74);
75
66it.effect("does not install the relay client when the user declines the managed download", () =>76it.effect("does not install the relay client when the user declines the managed download", () =>
67 Effect.gen(function* () {77 Effect.gen(function* () {
68 let installCalls = 0;78 let installCalls = 0;
apps/server/src/cli/service.ts+2
...@@ -185,6 +185,8 @@ export const recoverServiceOnboardingOffer = <R>(...@@ -185,6 +185,8 @@ export const recoverServiceOnboardingOffer = <R>(
185 Console.warn(`Background setup did not finish: ${error.message}`).pipe(Effect.as(false)),185 Console.warn(`Background setup did not finish: ${error.message}`).pipe(Effect.as(false)),
186 BootServiceInstallError: (error) =>186 BootServiceInstallError: (error) =>
187 Console.warn(`Background setup did not finish: ${error.message}`).pipe(Effect.as(false)),187 Console.warn(`Background setup did not finish: ${error.message}`).pipe(Effect.as(false)),
188 BootServiceUpdatePendingError: (error) =>
189 Console.warn(`Background setup did not finish: ${error.message}`).pipe(Effect.as(false)),
188 }),190 }),
189 );191 );
190192
apps/server/src/cloud/bootService.test.ts+46-6
...@@ -15,7 +15,11 @@ import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawne...@@ -15,7 +15,11 @@ import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawne
15import * as ProcessRunner from "../processRunner.ts";15import * as ProcessRunner from "../processRunner.ts";
16import * as BootService from "./bootService.ts";16import * as BootService from "./bootService.ts";
17import { pinnedRuntimePaths } from "./pinnedRuntime.ts";17import { pinnedRuntimePaths } from "./pinnedRuntime.ts";
18import { parseServiceState } from "./serviceProtocol.ts";18import {
19 parseServiceState,
20 SERVICE_LAUNCHER_PROTOCOL,
21 serviceStateHasPendingUpdate,
22} from "./serviceProtocol.ts";
1923
20it("keeps systemd pinned to the stable launcher rather than a versioned server", () => {24it("keeps systemd pinned to the stable launcher rather than a versioned server", () => {
21 const unit = BootService.renderBootServiceUnit({25 const unit = BootService.renderBootServiceUnit({
...@@ -97,15 +101,24 @@ it.layer(NodeServices.layer)("boot service install", (it) => {...@@ -97,15 +101,24 @@ it.layer(NodeServices.layer)("boot service install", (it) => {
97 const plan = yield* service.install;101 const plan = yield* service.install;
98102
99 expect(parseServiceState(yield* fs.readFileString(statePath))).toEqual({103 expect(parseServiceState(yield* fs.readFileString(statePath))).toEqual({
100 protocol: 1,104 protocol: SERVICE_LAUNCHER_PROTOCOL,
101 activeVersion: "1.2.3",105 activeVersion: "1.2.3",
102 });106 });
103 expect(yield* fs.readFileString(plan.launcherPath)).toBe("export {};\n");107 expect(yield* fs.readFileString(plan.launcherPath)).toBe("export {};\n");
104 expect((yield* service.status).current).toBe(true);108 expect((yield* service.status).current).toBe(true);
105 yield* fs.writeFileString(109 // @effect-diagnostics-next-line preferSchemaOverJson:off - fixed launcher-owned test document.
106 statePath,110 const pendingState = JSON.stringify({
107 '{"protocol":1,"activeVersion":"1.2.3","update":{"id":"u","fromVersion":"1.2.3","targetVersion":"1.2.4","status":"pending"}}',111 protocol: SERVICE_LAUNCHER_PROTOCOL,
108 );112 activeVersion: "1.2.3",
113 update: {
114 id: "u",
115 fromVersion: "1.2.3",
116 targetVersion: "1.2.4",
117 dbPath: "/tmp/state.sqlite",
118 status: "pending",
119 },
120 });
121 yield* fs.writeFileString(statePath, pendingState);
109 expect((yield* service.status).current).toBe(false);122 expect((yield* service.status).current).toBe(false);
110 expect(yield* service.uninstall).toBe(true);123 expect(yield* service.uninstall).toBe(true);
111 expect((yield* service.status).installed).toBe(false);124 expect((yield* service.status).installed).toBe(false);
...@@ -141,6 +154,33 @@ it.layer(NodeServices.layer)("boot service install", (it) => {...@@ -141,6 +154,33 @@ it.layer(NodeServices.layer)("boot service install", (it) => {
141 }),154 }),
142 );155 );
143156
157 it.effect("restarts without overwriting a pending remote update", () =>
158 Effect.gen(function* () {
159 const { service, fs, statePath, commands } = yield* makeHarness();
160 yield* service.install;
161 // @effect-diagnostics-next-line preferSchemaOverJson:off - fixed launcher-owned test document.
162 const pendingState = JSON.stringify({
163 protocol: SERVICE_LAUNCHER_PROTOCOL - 1,
164 activeVersion: "1.2.3",
165 update: {
166 id: "remote-update",
167 fromVersion: "1.2.3",
168 targetVersion: "1.2.4",
169 status: "pending",
170 },
171 });
172 yield* fs.writeFileString(statePath, pendingState);
173 commands.length = 0;
174
175 expect((yield* service.install.pipe(Effect.flip))._tag).toBe("BootServiceUpdatePendingError");
176 expect(serviceStateHasPendingUpdate(yield* fs.readFileString(statePath))).toBe(true);
177 expect(commands.filter((command) => command.startsWith("systemctl "))).toEqual([
178 "systemctl --user stop t3code.service",
179 "systemctl --user restart t3code.service",
180 ]);
181 }),
182 );
183
144 it.effect("fails closed off Linux", () =>184 it.effect("fails closed off Linux", () =>
145 Effect.gen(function* () {185 Effect.gen(function* () {
146 const { service } = yield* makeHarness("darwin");186 const { service } = yield* makeHarness("darwin");
apps/server/src/cloud/bootService.ts+21-1
...@@ -21,6 +21,7 @@ import {...@@ -21,6 +21,7 @@ import {
21 SERVICE_LAUNCHER_PROTOCOL,21 SERVICE_LAUNCHER_PROTOCOL,
22 SERVICE_STATE_FILE,22 SERVICE_STATE_FILE,
23 parseServiceState,23 parseServiceState,
24 serviceStateHasPendingUpdate,
24 type ServiceState,25 type ServiceState,
25} from "./serviceProtocol.ts";26} from "./serviceProtocol.ts";
2627
...@@ -110,10 +111,20 @@ export class BootServiceInstallError extends Schema.TaggedErrorClass<BootService...@@ -110,10 +111,20 @@ export class BootServiceInstallError extends Schema.TaggedErrorClass<BootService
110 }111 }
111}112}
112113
114export class BootServiceUpdatePendingError extends Schema.TaggedErrorClass<BootServiceUpdatePendingError>()(
115 "BootServiceUpdatePendingError",
116 {},
117) {
118 override get message(): string {
119 return "A remote server update is still pending. Wait for it to finish, then retry.";
120 }
121}
122
113export type BootServiceError =123export type BootServiceError =
114 | BootServiceUnsupportedError124 | BootServiceUnsupportedError
115 | BootServiceCommandError125 | BootServiceCommandError
116 | BootServiceInstallError;126 | BootServiceInstallError
127 | BootServiceUpdatePendingError;
117128
118export interface BootServiceStatus {129export interface BootServiceStatus {
119 readonly supported: boolean;130 readonly supported: boolean;
...@@ -288,6 +299,15 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: {...@@ -288,6 +299,15 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: {
288 }299 }
289300
290 yield* Effect.gen(function* () {301 yield* Effect.gen(function* () {
302 if (installed) {
303 const previousStateText = yield* fs.readFileString(statePath).pipe(Effect.option);
304 if (
305 Option.isSome(previousStateText) &&
306 serviceStateHasPendingUpdate(previousStateText.value)
307 ) {
308 return yield* new BootServiceUpdatePendingError();
309 }
310 }
291 yield* fs311 yield* fs
292 .makeDirectory(unitDir, { recursive: true })312 .makeDirectory(unitDir, { recursive: true })
293 .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause })));313 .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause })));
apps/server/src/cloud/selfUpdate.test.ts+6-2
...@@ -11,6 +11,7 @@ import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawne...@@ -11,6 +11,7 @@ import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawne
11import * as ServerConfig from "../config.ts";11import * as ServerConfig from "../config.ts";
12import * as ProcessRunner from "../processRunner.ts";12import * as ProcessRunner from "../processRunner.ts";
13import * as ServiceLauncherClient from "./serviceLauncherClient.ts";13import * as ServiceLauncherClient from "./serviceLauncherClient.ts";
14import { SERVICE_LAUNCHER_PROTOCOL } from "./serviceProtocol.ts";
14import * as ServerSelfUpdate from "./selfUpdate.ts";15import * as ServerSelfUpdate from "./selfUpdate.ts";
1516
16interface HarnessOptions {17interface HarnessOptions {
...@@ -50,7 +51,11 @@ const makeHarness = Effect.fn("test.make_self_update_harness")(function* (...@@ -50,7 +51,11 @@ const makeHarness = Effect.fn("test.make_self_update_harness")(function* (
50 const result =51 const result =
51 options.preflight === "blocked"52 options.preflight === "blocked"
52 ? { status: "blocked", version: "1.1.0", reason: "local update required" }53 ? { status: "blocked", version: "1.1.0", reason: "local update required" }
53 : { status: "ready", version: "1.1.0", launcherProtocol: 1 };54 : {
55 status: "ready",
56 version: "1.1.0",
57 launcherProtocol: SERVICE_LAUNCHER_PROTOCOL,
58 };
54 return {59 return {
55 // @effect-diagnostics-next-line preferSchemaOverJson:off - fake child-process stdout.60 // @effect-diagnostics-next-line preferSchemaOverJson:off - fake child-process stdout.
56 stdout: JSON.stringify(result),61 stdout: JSON.stringify(result),
...@@ -64,7 +69,6 @@ const makeHarness = Effect.fn("test.make_self_update_harness")(function* (...@@ -64,7 +69,6 @@ const makeHarness = Effect.fn("test.make_self_update_harness")(function* (
64 });69 });
65 const launcher = ServiceLauncherClient.ServiceLauncherClient.of({70 const launcher = ServiceLauncherClient.ServiceLauncherClient.of({
66 managed: options.managed ?? true,71 managed: options.managed ?? true,
67 trial: false,
68 requestUpdate:72 requestUpdate:
69 options.requestUpdate ??73 options.requestUpdate ??
70 (() =>74 (() =>
apps/server/src/cloud/selfUpdate.ts+1-1
...@@ -170,7 +170,7 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* () {...@@ -170,7 +170,7 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* () {
170170
171 yield* reportProgress("installing");171 yield* reportProgress("installing");
172 const updateId = yield* launcher172 const updateId = yield* launcher
173 .requestUpdate({ targetVersion })173 .requestUpdate({ targetVersion, dbPath: serverConfig.dbPath })
174 .pipe(174 .pipe(
175 Effect.mapError((error) =>175 Effect.mapError((error) =>
176 failWith(176 failWith(
apps/server/src/cloud/serviceLauncherClient.test.ts+18-10
...@@ -5,6 +5,7 @@ import * as Fiber from "effect/Fiber";...@@ -5,6 +5,7 @@ import * as Fiber from "effect/Fiber";
55
6import {6import {
7 SERVICE_LAUNCHER_CONTEXT_ENV,7 SERVICE_LAUNCHER_CONTEXT_ENV,
8 SERVICE_LAUNCHER_PROTOCOL,
8 type ServiceLauncherChildMessage,9 type ServiceLauncherChildMessage,
9 type ServiceLauncherParentMessage,10 type ServiceLauncherParentMessage,
10} from "./serviceProtocol.ts";11} from "./serviceProtocol.ts";
...@@ -53,10 +54,11 @@ it.effect("waits for the launcher to durably commit the trial update ID", () =>...@@ -53,10 +54,11 @@ it.effect("waits for the launcher to durably commit the trial update ID", () =>
53 id: "update-1",54 id: "update-1",
54 fromVersion: "1.0.0",55 fromVersion: "1.0.0",
55 targetVersion: "1.1.0",56 targetVersion: "1.1.0",
57 dbPath: "/tmp/state.sqlite",
56 status: "pending" as const,58 status: "pending" as const,
57 };59 };
58 const host = new FakeLauncherProcess({60 const host = new FakeLauncherProcess({
59 protocol: 1,61 protocol: SERVICE_LAUNCHER_PROTOCOL,
60 childVersion: "1.1.0",62 childVersion: "1.1.0",
61 update: pending,63 update: pending,
62 });64 });
...@@ -79,13 +81,14 @@ it.effect("waits for the launcher to durably commit the trial update ID", () =>...@@ -79,13 +81,14 @@ it.effect("waits for the launcher to durably commit the trial update ID", () =>
79it.effect("returns the launcher-generated ID only after update acceptance", () =>81it.effect("returns the launcher-generated ID only after update acceptance", () =>
80 Effect.gen(function* () {82 Effect.gen(function* () {
81 const host = new FakeLauncherProcess({83 const host = new FakeLauncherProcess({
82 protocol: 1,84 protocol: SERVICE_LAUNCHER_PROTOCOL,
83 childVersion: "1.0.0",85 childVersion: "1.0.0",
84 });86 });
85 const client = yield* makeClient(host, "1.0.0");87 const client = yield* makeClient(host, "1.0.0");
86 const requested = yield* Effect.forkChild(client.requestUpdate({ targetVersion: "1.1.0" }), {88 const requested = yield* Effect.forkChild(
87 startImmediately: true,89 client.requestUpdate({ targetVersion: "1.1.0", dbPath: "/tmp/state.sqlite" }),
88 });90 { startImmediately: true },
91 );
89 yield* Effect.yieldNow;92 yield* Effect.yieldNow;
90 host.emit({93 host.emit({
91 type: "update-accepted",94 type: "update-accepted",
...@@ -97,11 +100,15 @@ it.effect("returns the launcher-generated ID only after update acceptance", () =...@@ -97,11 +100,15 @@ it.effect("returns the launcher-generated ID only after update acceptance", () =
97100
98it.effect("preserves a launcher rejection as a distinct error", () =>101it.effect("preserves a launcher rejection as a distinct error", () =>
99 Effect.gen(function* () {102 Effect.gen(function* () {
100 const host = new FakeLauncherProcess({ protocol: 1, childVersion: "1.0.0" });103 const host = new FakeLauncherProcess({
101 const client = yield* makeClient(host, "1.0.0");104 protocol: SERVICE_LAUNCHER_PROTOCOL,
102 const requested = yield* Effect.forkChild(client.requestUpdate({ targetVersion: "1.1.0" }), {105 childVersion: "1.0.0",
103 startImmediately: true,
104 });106 });
107 const client = yield* makeClient(host, "1.0.0");
108 const requested = yield* Effect.forkChild(
109 client.requestUpdate({ targetVersion: "1.1.0", dbPath: "/tmp/state.sqlite" }),
110 { startImmediately: true },
111 );
105 yield* Effect.yieldNow;112 yield* Effect.yieldNow;
106 host.emit({ type: "update-rejected", reason: "requires local update" });113 host.emit({ type: "update-rejected", reason: "requires local update" });
107 expect(yield* Fiber.join(requested).pipe(Effect.flip)).toMatchObject({114 expect(yield* Fiber.join(requested).pipe(Effect.flip)).toMatchObject({
...@@ -115,12 +122,13 @@ it.effect("preserves a launcher rejection as a distinct error", () =>...@@ -115,12 +122,13 @@ it.effect("preserves a launcher rejection as a distinct error", () =>
115it.effect("rejects contradictory trial context instead of leaving activation closed", () =>122it.effect("rejects contradictory trial context instead of leaving activation closed", () =>
116 Effect.gen(function* () {123 Effect.gen(function* () {
117 const host = new FakeLauncherProcess({124 const host = new FakeLauncherProcess({
118 protocol: 1,125 protocol: SERVICE_LAUNCHER_PROTOCOL,
119 childVersion: "1.1.0",126 childVersion: "1.1.0",
120 update: {127 update: {
121 id: "update-1",128 id: "update-1",
122 fromVersion: "1.0.0",129 fromVersion: "1.0.0",
123 targetVersion: "1.2.0",130 targetVersion: "1.2.0",
131 dbPath: "/tmp/state.sqlite",
124 status: "pending",132 status: "pending",
125 },133 },
126 });134 });
apps/server/src/cloud/serviceLauncherClient.ts+10-6
...@@ -100,9 +100,9 @@ export class ServiceLauncherClient extends Context.Service<...@@ -100,9 +100,9 @@ export class ServiceLauncherClient extends Context.Service<
100 ServiceLauncherClient,100 ServiceLauncherClient,
101 {101 {
102 readonly managed: boolean;102 readonly managed: boolean;
103 readonly trial: boolean;
104 readonly requestUpdate: (input: {103 readonly requestUpdate: (input: {
105 readonly targetVersion: string;104 readonly targetVersion: string;
105 readonly dbPath: string;
106 }) => Effect.Effect<string, ServiceLauncherClientError | ServiceLauncherRejectedError>;106 }) => Effect.Effect<string, ServiceLauncherClientError | ServiceLauncherRejectedError>;
107 readonly prepareTrial: Effect.Effect<107 readonly prepareTrial: Effect.Effect<
108 ServerSelfUpdateOutcome | undefined,108 ServerSelfUpdateOutcome | undefined,
...@@ -137,8 +137,8 @@ const resolveStartup = Effect.fn("cloud.service_launcher_client.resolve_startup"...@@ -137,8 +137,8 @@ const resolveStartup = Effect.fn("cloud.service_launcher_client.resolve_startup"
137137
138export const resolveServiceLauncherMode = Effect.fn("cloud.service_launcher_client.resolve_mode")(138export const resolveServiceLauncherMode = Effect.fn("cloud.service_launcher_client.resolve_mode")(
139 function* () {139 function* () {
140 const { context, managed } = yield* resolveStartup();140 const { managed } = yield* resolveStartup();
141 return { managed, trial: context?.update?.status === "pending" };141 return { managed };
142 },142 },
143);143);
144144
...@@ -199,7 +199,7 @@ export const make = Effect.fn("cloud.service_launcher_client.make")(function* (o...@@ -199,7 +199,7 @@ export const make = Effect.fn("cloud.service_launcher_client.make")(function* (o
199 }),199 }),
200 );200 );
201201
202 const requestUpdate = (input: { readonly targetVersion: string }) =>202 const requestUpdate = (input: { readonly targetVersion: string; readonly dbPath: string }) =>
203 exchange(203 exchange(
204 { type: "request-update", ...input },204 { type: "request-update", ...input },
205 (reply) => reply.type === "update-accepted" || reply.type === "update-rejected",205 (reply) => reply.type === "update-accepted" || reply.type === "update-rejected",
...@@ -233,14 +233,18 @@ export const make = Effect.fn("cloud.service_launcher_client.make")(function* (o...@@ -233,14 +233,18 @@ export const make = Effect.fn("cloud.service_launcher_client.make")(function* (o
233 if (reply.type !== "committed") {233 if (reply.type !== "committed") {
234 return Effect.die("service launcher returned an impossible prepared response");234 return Effect.die("service launcher returned an impossible prepared response");
235 }235 }
236 return Effect.succeed({ ...pending, status: "committed" as const });236 return Effect.succeed({
237 id: pending.id,
238 fromVersion: pending.fromVersion,
239 targetVersion: pending.targetVersion,
240 status: "committed" as const,
241 });
237 }),242 }),
238 )243 )
239 : Effect.succeed(outcome);244 : Effect.succeed(outcome);
240245
241 return ServiceLauncherClient.of({246 return ServiceLauncherClient.of({
242 managed,247 managed,
243 trial: pending !== undefined,
244 requestUpdate,248 requestUpdate,
245 prepareTrial,249 prepareTrial,
246 });250 });
apps/server/src/cloud/servicePreflight.test.ts+19-40
...@@ -1,47 +1,26 @@...@@ -1,47 +1,26 @@
1// @effect-diagnostics nodeBuiltinImport:off
2import * as NodeServices from "@effect/platform-node/NodeServices";
3import { expect, it } from "@effect/vitest";1import { expect, it } from "@effect/vitest";
4import * as Effect from "effect/Effect";
5import * as FileSystem from "effect/FileSystem";
6import * as Path from "effect/Path";
7import * as NodeSqlite from "node:sqlite";
82
9import { migrationManifest } from "../persistence/Migrations.ts";
10import { runServicePreflight } from "./servicePreflight.ts";3import { runServicePreflight } from "./servicePreflight.ts";
4import { SERVICE_LAUNCHER_PROTOCOL } from "./serviceProtocol.ts";
115
12it.layer(NodeServices.layer)("service update preflight", (it) => {6it("requires the database-snapshot launcher protocol", () => {
13 it.effect("requires exact migration-manifest equality without mutating the database", () =>7 expect(
14 Effect.gen(function* () {8 runServicePreflight({
15 const fs = yield* FileSystem.FileSystem;9 databasePath: "/missing/state.sqlite",
16 const path = yield* Path.Path;10 launcherProtocol: SERVICE_LAUNCHER_PROTOCOL - 1,
17 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-preflight-test-" });11 version: "1.2.3",
18 const databasePath = path.join(root, "state.sqlite");12 }),
19 const database = new NodeSqlite.DatabaseSync(databasePath);13 ).toMatchObject({ status: "blocked", version: "1.2.3" });
20 database.exec("CREATE TABLE effect_sql_migrations (migration_id INTEGER, name TEXT)");
21 const insert = database.prepare(
22 "INSERT INTO effect_sql_migrations (migration_id, name) VALUES (?, ?)",
23 );
24 for (const [id, name] of migrationManifest) insert.run(id, name);
25 database.close();
26
27 expect(runServicePreflight({ databasePath, launcherProtocol: 1, version: "1.2.3" })).toEqual({
28 status: "ready",
29 version: "1.2.3",
30 launcherProtocol: 1,
31 });
3214
33 const changed = new NodeSqlite.DatabaseSync(databasePath);15 expect(
34 changed.exec("DELETE FROM effect_sql_migrations WHERE migration_id = 35");16 runServicePreflight({
35 changed.close();17 databasePath: "/missing/state.sqlite",
36 const blocked = runServicePreflight({18 launcherProtocol: SERVICE_LAUNCHER_PROTOCOL,
37 databasePath,19 version: "1.2.3",
38 launcherProtocol: 1,
39 version: "1.2.3",
40 });
41 expect(blocked.status).toBe("blocked");
42 if (blocked.status === "blocked") {
43 expect(blocked.reason).toContain("npx t3@1.2.3 service update");
44 }
45 }),20 }),
46 );21 ).toEqual({
22 status: "ready",
23 version: "1.2.3",
24 launcherProtocol: SERVICE_LAUNCHER_PROTOCOL,
25 });
47});26});
apps/server/src/cloud/servicePreflight.ts+1-38
...@@ -1,7 +1,4 @@...@@ -1,7 +1,4 @@
1import * as NodeSqlite from "node:sqlite";
2
3import packageJson from "../../package.json" with { type: "json" };1import packageJson from "../../package.json" with { type: "json" };
4import { migrationManifest } from "../persistence/Migrations.ts";
5import { SERVICE_LAUNCHER_PROTOCOL } from "./serviceProtocol.ts";2import { SERVICE_LAUNCHER_PROTOCOL } from "./serviceProtocol.ts";
63
7export type ServicePreflightResult =4export type ServicePreflightResult =
...@@ -16,20 +13,8 @@ export type ServicePreflightResult =...@@ -16,20 +13,8 @@ export type ServicePreflightResult =
16 readonly reason: string;13 readonly reason: string;
17 };14 };
1815
19const localUpdateReason = (version: string) =>
20 `This version includes a database update and cannot be installed remotely. Run \`npx t3@${version} service update\` on the server machine.`;
21
22const isMigrationRow = (
23 value: unknown,
24): value is { readonly migration_id: number; readonly name: string } =>
25 typeof value === "object" &&
26 value !== null &&
27 "migration_id" in value &&
28 typeof value.migration_id === "number" &&
29 "name" in value &&
30 typeof value.name === "string";
31
32export function runServicePreflight(input: {16export function runServicePreflight(input: {
17 /** Older servers always pass this flag when invoking a staged preflight. */
33 readonly databasePath: string;18 readonly databasePath: string;
34 readonly launcherProtocol: number;19 readonly launcherProtocol: number;
35 readonly version?: string;20 readonly version?: string;
...@@ -44,28 +29,6 @@ export function runServicePreflight(input: {...@@ -44,28 +29,6 @@ export function runServicePreflight(input: {
44 };29 };
45 }30 }
4631
47 try {
48 const database = new NodeSqlite.DatabaseSync(input.databasePath, { readOnly: true });
49 try {
50 const rows: ReadonlyArray<unknown> = database
51 .prepare("SELECT migration_id, name FROM effect_sql_migrations ORDER BY migration_id")
52 .all();
53 const exact =
54 rows.length === migrationManifest.length &&
55 rows.every((row, index) => {
56 const expected = migrationManifest[index];
57 return (
58 isMigrationRow(row) && row.migration_id === expected?.[0] && row.name === expected?.[1]
59 );
60 });
61 if (!exact) return { status: "blocked", version, reason: localUpdateReason(version) };
62 } finally {
63 database.close();
64 }
65 } catch {
66 return { status: "blocked", version, reason: localUpdateReason(version) };
67 }
68
69 return { status: "ready", version, launcherProtocol: SERVICE_LAUNCHER_PROTOCOL };32 return { status: "ready", version, launcherProtocol: SERVICE_LAUNCHER_PROTOCOL };
70}33}
7134
apps/server/src/cloud/serviceProtocol.ts+23-4
...@@ -1,6 +1,7 @@...@@ -1,6 +1,7 @@
1import type { ServerSelfUpdateOutcome } from "@t3tools/contracts";1import type { ServerSelfUpdateOutcome } from "@t3tools/contracts";
22
3export const SERVICE_LAUNCHER_PROTOCOL = 1 as const;3/** Protocol 2 snapshots SQLite before trials so migrations can be rolled back safely. */
4export const SERVICE_LAUNCHER_PROTOCOL = 2 as const;
4export const SERVICE_LAUNCHER_CONTEXT_ENV = "T3_SERVICE_LAUNCHER_CONTEXT";5export const SERVICE_LAUNCHER_CONTEXT_ENV = "T3_SERVICE_LAUNCHER_CONTEXT";
5export const SERVICE_LAUNCHER_FILE = "service-launcher.mjs";6export const SERVICE_LAUNCHER_FILE = "service-launcher.mjs";
6export const SERVICE_STATE_FILE = "service-state.json";7export const SERVICE_STATE_FILE = "service-state.json";
...@@ -9,6 +10,7 @@ export interface PendingServiceUpdate {...@@ -9,6 +10,7 @@ export interface PendingServiceUpdate {
9 readonly id: string;10 readonly id: string;
10 readonly fromVersion: string;11 readonly fromVersion: string;
11 readonly targetVersion: string;12 readonly targetVersion: string;
13 readonly dbPath: string;
12 readonly status: "pending";14 readonly status: "pending";
13}15}
1416
...@@ -31,6 +33,7 @@ export type ServiceLauncherChildMessage =...@@ -31,6 +33,7 @@ export type ServiceLauncherChildMessage =
31 | {33 | {
32 readonly type: "request-update";34 readonly type: "request-update";
33 readonly targetVersion: string;35 readonly targetVersion: string;
36 readonly dbPath: string;
34 }37 }
35 | {38 | {
36 readonly type: "prepared";39 readonly type: "prepared";
...@@ -78,7 +81,9 @@ export function decodeServiceUpdate(value: unknown): ServiceUpdateRecord | undef...@@ -78,7 +81,9 @@ export function decodeServiceUpdate(value: unknown): ServiceUpdateRecord | undef
78 return undefined;81 return undefined;
79 }82 }
80 if (status === "pending") {83 if (status === "pending") {
81 return { id, fromVersion, targetVersion, status };84 return typeof value.dbPath === "string" && value.dbPath.trim() !== ""
85 ? { id, fromVersion, targetVersion, dbPath: value.dbPath, status }
86 : undefined;
82 }87 }
83 if (88 if (
84 (status === "committed" || status === "rolled-back" || status === "failed") &&89 (status === "committed" || status === "rolled-back" || status === "failed") &&
...@@ -165,6 +170,16 @@ export function parseServiceState(value: string): ServiceState | undefined {...@@ -165,6 +170,16 @@ export function parseServiceState(value: string): ServiceState | undefined {
165 }170 }
166}171}
167172
173/** Detects an in-flight update across launcher protocol versions before replacing its state. */
174export function serviceStateHasPendingUpdate(value: string): boolean {
175 try {
176 const parsed: unknown = JSON.parse(value);
177 return isRecord(parsed) && isRecord(parsed.update) && parsed.update.status === "pending";
178 } catch {
179 return false;
180 }
181}
182
168export function decodeServiceLauncherContext(value: string): ServiceLauncherContext | undefined {183export function decodeServiceLauncherContext(value: string): ServiceLauncherContext | undefined {
169 let parsed: unknown;184 let parsed: unknown;
170 try {185 try {
...@@ -202,8 +217,12 @@ export function decodeServiceLauncherChildMessage(...@@ -202,8 +217,12 @@ export function decodeServiceLauncherChildMessage(
202 value: unknown,217 value: unknown,
203): ServiceLauncherChildMessage | undefined {218): ServiceLauncherChildMessage | undefined {
204 if (!isRecord(value)) return undefined;219 if (!isRecord(value)) return undefined;
205 if (value.type === "request-update" && typeof value.targetVersion === "string") {220 if (
206 return { type: value.type, targetVersion: value.targetVersion };221 value.type === "request-update" &&
222 typeof value.targetVersion === "string" &&
223 typeof value.dbPath === "string"
224 ) {
225 return { type: value.type, targetVersion: value.targetVersion, dbPath: value.dbPath };
207 }226 }
208 return value.type === "prepared" && typeof value.updateId === "string"227 return value.type === "prepared" && typeof value.updateId === "string"
209 ? { type: value.type, updateId: value.updateId }228 ? { type: value.type, updateId: value.updateId }
apps/server/src/persistence/Layers/Sqlite.ts+11-17
...@@ -7,7 +7,6 @@ import type { SqlError } from "effect/unstable/sql/SqlError";...@@ -7,7 +7,6 @@ import type { SqlError } from "effect/unstable/sql/SqlError";
77
8import { runMigrations } from "../Migrations.ts";8import { runMigrations } from "../Migrations.ts";
9import { ServerConfig } from "../../config.ts";9import { ServerConfig } from "../../config.ts";
10import * as ServiceLauncherClient from "../../cloud/serviceLauncherClient.ts";
1110
12type RuntimeSqliteLayerConfig = {11type RuntimeSqliteLayerConfig = {
13 readonly filename: string;12 readonly filename: string;
...@@ -31,28 +30,24 @@ const makeRuntimeSqliteLayer = Effect.fn("makeRuntimeSqliteLayer")(function* (...@@ -31,28 +30,24 @@ const makeRuntimeSqliteLayer = Effect.fn("makeRuntimeSqliteLayer")(function* (
31 return clientModule.layer(config);30 return clientModule.layer(config);
32}, Layer.unwrap);31}, Layer.unwrap);
3332
34const setup = (trial: boolean) =>33const setup = Layer.effectDiscard(
35 Layer.effectDiscard(34 Effect.gen(function* () {
36 Effect.gen(function* () {35 const sql = yield* SqlClient.SqlClient;
37 const sql = yield* SqlClient.SqlClient;36 yield* sql`PRAGMA foreign_keys = ON;`;
38 yield* sql`PRAGMA foreign_keys = ON;`;37 yield* sql`PRAGMA journal_mode = WAL;`;
39 if (!trial) {38 yield* runMigrations();
40 yield* sql`PRAGMA journal_mode = WAL;`;39 }),
41 yield* runMigrations();40);
42 }
43 }),
44 );
4541
46export const makeSqlitePersistenceLive = Effect.fn("makeSqlitePersistenceLive")(function* (42export const makeSqlitePersistenceLive = Effect.fn("makeSqlitePersistenceLive")(function* (
47 dbPath: string,43 dbPath: string,
48 options?: { readonly trial?: boolean },
49) {44) {
50 const fs = yield* FileSystem.FileSystem;45 const fs = yield* FileSystem.FileSystem;
51 const path = yield* Path.Path;46 const path = yield* Path.Path;
52 yield* fs.makeDirectory(path.dirname(dbPath), { recursive: true });47 yield* fs.makeDirectory(path.dirname(dbPath), { recursive: true });
5348
54 return Layer.provideMerge(49 return Layer.provideMerge(
55 setup(options?.trial === true),50 setup,
56 makeRuntimeSqliteLayer({51 makeRuntimeSqliteLayer({
57 filename: dbPath,52 filename: dbPath,
58 spanAttributes: {53 spanAttributes: {
...@@ -64,14 +59,13 @@ export const makeSqlitePersistenceLive = Effect.fn("makeSqlitePersistenceLive")(...@@ -64,14 +59,13 @@ export const makeSqlitePersistenceLive = Effect.fn("makeSqlitePersistenceLive")(
64}, Layer.unwrap);59}, Layer.unwrap);
6560
66export const SqlitePersistenceMemory = Layer.provideMerge(61export const SqlitePersistenceMemory = Layer.provideMerge(
67 setup(false),62 setup,
68 makeRuntimeSqliteLayer({ filename: ":memory:" }),63 makeRuntimeSqliteLayer({ filename: ":memory:" }),
69);64);
7065
71export const layerConfig = Layer.unwrap(66export const layerConfig = Layer.unwrap(
72 Effect.gen(function* () {67 Effect.gen(function* () {
73 const { dbPath } = yield* ServerConfig;68 const { dbPath } = yield* ServerConfig;
74 const launcher = yield* ServiceLauncherClient.resolveServiceLauncherMode();69 return makeSqlitePersistenceLive(dbPath);
75 return makeSqlitePersistenceLive(dbPath, { trial: launcher.trial });
76 }),70 }),
77);71);
apps/server/src/serviceLauncher.test.ts+95-8
...@@ -9,6 +9,7 @@ import {...@@ -9,6 +9,7 @@ import {
9 compareExactServiceVersions,9 compareExactServiceVersions,
10 decodeServiceState,10 decodeServiceState,
11 isExactServiceVersion,11 isExactServiceVersion,
12 SERVICE_LAUNCHER_PROTOCOL,
12} from "./cloud/serviceProtocol.ts";13} from "./cloud/serviceProtocol.ts";
1314
14it("accepts only exact semantic versions", () => {15it("accepts only exact semantic versions", () => {
...@@ -33,12 +34,13 @@ it("orders exact semantic versions without treating build metadata as precedence...@@ -33,12 +34,13 @@ it("orders exact semantic versions without treating build metadata as precedence
33it("rejects contradictory service state", () => {34it("rejects contradictory service state", () => {
34 assert.isUndefined(35 assert.isUndefined(
35 decodeServiceState({36 decodeServiceState({
36 protocol: 1,37 protocol: SERVICE_LAUNCHER_PROTOCOL,
37 activeVersion: "0.0.31",38 activeVersion: "0.0.31",
38 update: {39 update: {
39 id: "update-1",40 id: "update-1",
40 fromVersion: "0.0.30",41 fromVersion: "0.0.30",
41 targetVersion: "0.0.32",42 targetVersion: "0.0.32",
43 dbPath: "/tmp/state.sqlite",
42 status: "pending",44 status: "pending",
43 },45 },
44 }),46 }),
...@@ -46,12 +48,26 @@ it("rejects contradictory service state", () => {...@@ -46,12 +48,26 @@ it("rejects contradictory service state", () => {
4648
47 assert.isUndefined(49 assert.isUndefined(
48 decodeServiceState({50 decodeServiceState({
49 protocol: 1,51 protocol: SERVICE_LAUNCHER_PROTOCOL,
52 activeVersion: "1.0.0",
53 update: {
54 id: "update-3",
55 fromVersion: "1.0.0",
56 targetVersion: "1.1.0",
57 status: "pending",
58 },
59 }),
60 );
61
62 assert.isUndefined(
63 decodeServiceState({
64 protocol: SERVICE_LAUNCHER_PROTOCOL,
50 activeVersion: "1.0.0",65 activeVersion: "1.0.0",
51 update: {66 update: {
52 id: "update-2",67 id: "update-2",
53 fromVersion: "1.0.0",68 fromVersion: "1.0.0",
54 targetVersion: "0.9.0",69 targetVersion: "0.9.0",
70 dbPath: "/tmp/state.sqlite",
55 status: "pending",71 status: "pending",
56 },72 },
57 }),73 }),
...@@ -66,7 +82,7 @@ it.layer(NodeServices.layer)("service state persistence", (it) => {...@@ -66,7 +82,7 @@ it.layer(NodeServices.layer)("service state persistence", (it) => {
66 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-test-" });82 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-test-" });
67 const statePath = path.join(root, "runtime", "service-state.json");83 const statePath = path.join(root, "runtime", "service-state.json");
68 const state = {84 const state = {
69 protocol: 1,85 protocol: SERVICE_LAUNCHER_PROTOCOL,
70 activeVersion: "0.0.31",86 activeVersion: "0.0.31",
71 } as const;87 } as const;
7288
...@@ -88,7 +104,7 @@ it.layer(NodeServices.layer)("service state persistence", (it) => {...@@ -88,7 +104,7 @@ it.layer(NodeServices.layer)("service state persistence", (it) => {
88 yield* fs.writeFileString(path.join(versionDir, ".install-complete"), "1.0.0\n");104 yield* fs.writeFileString(path.join(versionDir, ".install-complete"), "1.0.0\n");
89 yield* Effect.promise(() =>105 yield* Effect.promise(() =>
90 writeServiceState(statePath, {106 writeServiceState(statePath, {
91 protocol: 1,107 protocol: SERVICE_LAUNCHER_PROTOCOL,
92 activeVersion: "1.0.0",108 activeVersion: "1.0.0",
93 }),109 }),
94 );110 );
...@@ -106,6 +122,11 @@ it.layer(NodeServices.layer)("service state persistence", (it) => {...@@ -106,6 +122,11 @@ it.layer(NodeServices.layer)("service state persistence", (it) => {
106 const path = yield* Path.Path;122 const path = yield* Path.Path;
107 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-flow-" });123 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-flow-" });
108 const statePath = path.join(root, "runtime", "service-state.json");124 const statePath = path.join(root, "runtime", "service-state.json");
125 const databasePath = path.join(root, "userdata", "state.sqlite");
126 yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true });
127 yield* fs.writeFileString(databasePath, "before trial");
128 // @effect-diagnostics-next-line preferSchemaOverJson:off - embeds a path in fake child source.
129 const encodedDatabasePath = JSON.stringify(databasePath);
109 const childSource = `130 const childSource = `
110const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT);131const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT);
111if (context.update?.status === "pending") {132if (context.update?.status === "pending") {
...@@ -114,7 +135,7 @@ if (context.update?.status === "pending") {...@@ -114,7 +135,7 @@ if (context.update?.status === "pending") {
114 if (message.type === "committed") process.exit(0);135 if (message.type === "committed") process.exit(0);
115 });136 });
116} else if (context.update === undefined) {137} else if (context.update === undefined) {
117 process.send({ type: "request-update", targetVersion: "1.1.0" });138 process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: ${encodedDatabasePath} });
118 setInterval(() => {}, 1_000);139 setInterval(() => {}, 1_000);
119} else {140} else {
120 process.exit(0);141 process.exit(0);
...@@ -129,7 +150,7 @@ if (context.update?.status === "pending") {...@@ -129,7 +150,7 @@ if (context.update?.status === "pending") {
129 }150 }
130 yield* Effect.promise(() =>151 yield* Effect.promise(() =>
131 writeServiceState(statePath, {152 writeServiceState(statePath, {
132 protocol: 1,153 protocol: SERVICE_LAUNCHER_PROTOCOL,
133 activeVersion: "1.0.0",154 activeVersion: "1.0.0",
134 }),155 }),
135 );156 );
...@@ -154,12 +175,17 @@ if (context.update?.status === "pending") {...@@ -154,12 +175,17 @@ if (context.update?.status === "pending") {
154 const path = yield* Path.Path;175 const path = yield* Path.Path;
155 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-rollback-" });176 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-rollback-" });
156 const statePath = path.join(root, "runtime", "service-state.json");177 const statePath = path.join(root, "runtime", "service-state.json");
178 const databasePath = path.join(root, "userdata", "state.sqlite");
179 yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true });
180 yield* fs.writeFileString(databasePath, "before trial");
181 // @effect-diagnostics-next-line preferSchemaOverJson:off - embeds a path in fake child source.
182 const encodedDatabasePath = JSON.stringify(databasePath);
157 const childSource = `183 const childSource = `
158const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT);184const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT);
159if (context.update?.status === "pending") {185if (context.update?.status === "pending") {
160 process.send({ type: "prepared", updateId: "wrong-update" });186 process.send({ type: "prepared", updateId: "wrong-update" });
161} else if (context.update === undefined) {187} else if (context.update === undefined) {
162 process.send({ type: "request-update", targetVersion: "1.1.0" });188 process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: ${encodedDatabasePath} });
163 setInterval(() => {}, 1_000);189 setInterval(() => {}, 1_000);
164} else {190} else {
165 process.exit(0);191 process.exit(0);
...@@ -174,7 +200,7 @@ if (context.update?.status === "pending") {...@@ -174,7 +200,7 @@ if (context.update?.status === "pending") {
174 }200 }
175 yield* Effect.promise(() =>201 yield* Effect.promise(() =>
176 writeServiceState(statePath, {202 writeServiceState(statePath, {
177 protocol: 1,203 protocol: SERVICE_LAUNCHER_PROTOCOL,
178 activeVersion: "1.0.0",204 activeVersion: "1.0.0",
179 }),205 }),
180 );206 );
...@@ -196,4 +222,65 @@ if (context.update?.status === "pending") {...@@ -196,4 +222,65 @@ if (context.update?.status === "pending") {
196 );222 );
197 }),223 }),
198 );224 );
225
226 it.effect("restores the database when a migrating trial exits", () =>
227 Effect.gen(function* () {
228 const fs = yield* FileSystem.FileSystem;
229 const path = yield* Path.Path;
230 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-db-" });
231 const statePath = path.join(root, "runtime", "service-state.json");
232 const databasePath = path.join(root, "userdata", "state.sqlite");
233 const original = "database before migration";
234 yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true });
235 yield* fs.writeFileString(databasePath, original);
236 // @effect-diagnostics-next-line preferSchemaOverJson:off - embeds a path in fake child source.
237 const encodedDatabasePath = JSON.stringify(databasePath);
238 const childSource = `
239import { writeFileSync } from "node:fs";
240const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT);
241if (context.update?.status === "pending") {
242 writeFileSync(context.update.dbPath, "database after migration");
243 writeFileSync(context.update.dbPath + "-wal", "trial wal");
244 writeFileSync(context.update.dbPath + "-shm", "trial shm");
245 process.exit(1);
246} else if (context.update === undefined) {
247 process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: ${encodedDatabasePath} });
248 setInterval(() => {}, 1_000);
249} else {
250 process.exit(0);
251}
252`;
253 for (const version of ["1.0.0", "1.1.0"]) {
254 const versionDir = path.join(root, "runtime", "versions", version);
255 const entryPath = path.join(versionDir, "node_modules", "t3", "dist", "bin.mjs");
256 yield* fs.makeDirectory(path.dirname(entryPath), { recursive: true });
257 yield* fs.writeFileString(entryPath, childSource);
258 yield* fs.writeFileString(path.join(versionDir, ".install-complete"), `${version}\n`);
259 }
260 yield* Effect.promise(() =>
261 writeServiceState(statePath, {
262 protocol: SERVICE_LAUNCHER_PROTOCOL,
263 activeVersion: "1.0.0",
264 }),
265 );
266
267 const launcher = new Launcher(root, yield* Effect.promise(() => readServiceState(statePath)));
268 yield* Effect.promise(() =>
269 launcher.run().then(
270 () => Promise.reject(new Error("launcher unexpectedly completed")),
271 () => Promise.resolve(),
272 ),
273 );
274
275 const state = yield* Effect.promise(() => readServiceState(statePath));
276 assert.equal(state.activeVersion, "1.0.0");
277 assert.equal(state.update?.status, "rolled-back");
278 assert.equal(yield* fs.readFileString(databasePath), original);
279 assert.isFalse(yield* fs.exists(`${databasePath}-wal`));
280 assert.isFalse(yield* fs.exists(`${databasePath}-shm`));
281 const updateId = state.update?.id;
282 assert.isDefined(updateId);
283 assert.isFalse(yield* fs.exists(path.join(root, "runtime", "db-backup", updateId)));
284 }),
285 );
199});286});
apps/server/src/serviceLauncher.ts+141-4
...@@ -48,6 +48,121 @@ const runtimePaths = (baseDir: string, version: string) => {...@@ -48,6 +48,121 @@ const runtimePaths = (baseDir: string, version: string) => {
48 };48 };
49};49};
5050
51/** SQLite persists across the main file plus its WAL and shared-memory sidecars. */
52const DB_FILE_SUFFIXES = ["", "-wal", "-shm"] as const;
53const RESTORE_MARKER = ".restore-pending";
54
55const databaseBackupDir = (baseDir: string, updateId: string) =>
56 NodePath.join(baseDir, "runtime", "db-backup", updateId);
57
58const databaseBackupFile = (backupDir: string, suffix: (typeof DB_FILE_SUFFIXES)[number]) =>
59 NodePath.join(backupDir, suffix === "" ? "database" : `database${suffix}`);
60
61async function pathExists(target: string): Promise<boolean> {
62 try {
63 await NodeFSP.access(target);
64 return true;
65 } catch (cause) {
66 if (cause instanceof Error && "code" in cause && cause.code === "ENOENT") return false;
67 throw cause;
68 }
69}
70
71async function syncFile(filePath: string): Promise<void> {
72 const handle = await NodeFSP.open(filePath, "r");
73 try {
74 await handle.sync();
75 } finally {
76 await handle.close();
77 }
78}
79
80async function syncDirectory(directory: string): Promise<void> {
81 const handle = await NodeFSP.open(directory, "r");
82 try {
83 await handle.sync();
84 } finally {
85 await handle.close();
86 }
87}
88
89/**
90 * Snapshots the database once per update before the first trial. A completed
91 * backup is never overwritten because a restarted launcher may be looking at
92 * database writes from an earlier attempt by the same trial.
93 */
94async function backupDatabaseOnce(baseDir: string, pending: PendingServiceUpdate): Promise<void> {
95 const backupDir = databaseBackupDir(baseDir, pending.id);
96 if (await pathExists(backupDir)) return;
97
98 const stagingDir = `${backupDir}.staging`;
99 await NodeFSP.rm(stagingDir, { recursive: true, force: true });
100 await NodeFSP.mkdir(stagingDir, { recursive: true, mode: 0o700 });
101 try {
102 for (const suffix of DB_FILE_SUFFIXES) {
103 const source = `${pending.dbPath}${suffix}`;
104 if (suffix !== "" && !(await pathExists(source))) continue;
105 const destination = databaseBackupFile(stagingDir, suffix);
106 await NodeFSP.copyFile(source, destination);
107 await syncFile(destination);
108 }
109 await NodeFSP.rename(stagingDir, backupDir);
110 await syncDirectory(NodePath.dirname(backupDir));
111 } catch (cause) {
112 await NodeFSP.rm(stagingDir, { recursive: true, force: true }).catch(() => undefined);
113 throw cause;
114 }
115}
116
117const restoreMarkerPath = (baseDir: string, updateId: string) =>
118 NodePath.join(databaseBackupDir(baseDir, updateId), RESTORE_MARKER);
119
120const databaseRestorePending = (baseDir: string, pending: PendingServiceUpdate) =>
121 pathExists(restoreMarkerPath(baseDir, pending.id));
122
123/** Mark rollback before changing live files so launcher recovery cannot boot a partial restore. */
124async function markDatabaseRestorePending(backupDir: string): Promise<void> {
125 const markerPath = NodePath.join(backupDir, RESTORE_MARKER);
126 if (!(await pathExists(markerPath))) {
127 const handle = await NodeFSP.open(markerPath, "wx", 0o600);
128 try {
129 await handle.sync();
130 } finally {
131 await handle.close();
132 }
133 await syncDirectory(backupDir);
134 }
135}
136
137/** Restore is retryable after any process crash while the backup directory remains. */
138async function restoreDatabaseBackup(
139 baseDir: string,
140 pending: PendingServiceUpdate,
141): Promise<void> {
142 const backupDir = databaseBackupDir(baseDir, pending.id);
143 if (!(await pathExists(backupDir))) return;
144
145 await markDatabaseRestorePending(backupDir);
146 for (const suffix of DB_FILE_SUFFIXES) {
147 const target = `${pending.dbPath}${suffix}`;
148 const source = databaseBackupFile(backupDir, suffix);
149 if (await pathExists(source)) {
150 await NodeFSP.copyFile(source, target);
151 await syncFile(target);
152 } else {
153 await NodeFSP.rm(target, { force: true });
154 }
155 }
156 await syncDirectory(NodePath.dirname(pending.dbPath));
157}
158
159async function discardDatabaseBackup(baseDir: string, updateId: string): Promise<void> {
160 const backupDir = databaseBackupDir(baseDir, updateId);
161 if (!(await pathExists(backupDir))) return;
162 await NodeFSP.rm(backupDir, { recursive: true, force: true });
163 await syncDirectory(NodePath.dirname(backupDir));
164}
165
51export async function readServiceState(filePath: string): Promise<ServiceState> {166export async function readServiceState(filePath: string): Promise<ServiceState> {
52 const contents = await NodeFSP.readFile(filePath, "utf8");167 const contents = await NodeFSP.readFile(filePath, "utf8");
53 const state = parseServiceState(contents);168 const state = parseServiceState(contents);
...@@ -217,9 +332,16 @@ export class Launcher {...@@ -217,9 +332,16 @@ export class Launcher {
217 async #recover(): Promise<void> {332 async #recover(): Promise<void> {
218 const update = this.#state.update;333 const update = this.#state.update;
219 if (update?.status !== "pending") {334 if (update?.status !== "pending") {
335 if (update !== undefined) {
336 await discardDatabaseBackup(this.#baseDir, update.id).catch(() => undefined);
337 }
220 await this.#startChild(this.#state.activeVersion, "active", update);338 await this.#startChild(this.#state.activeVersion, "active", update);
221 return;339 return;
222 }340 }
341 if (await databaseRestorePending(this.#baseDir, update)) {
342 await this.#returnToPrevious(update, "failed", "rollback-interrupted");
343 return;
344 }
223 if (!(await runtimeExists(this.#baseDir, update.targetVersion))) {345 if (!(await runtimeExists(this.#baseDir, update.targetVersion))) {
224 await this.#returnToPrevious(update, "failed", "target-runtime-missing");346 await this.#returnToPrevious(update, "failed", "target-runtime-missing");
225 return;347 return;
...@@ -228,6 +350,13 @@ export class Launcher {...@@ -228,6 +350,13 @@ export class Launcher {
228 }350 }
229351
230 async #startTrial(pending: PendingServiceUpdate): Promise<void> {352 async #startTrial(pending: PendingServiceUpdate): Promise<void> {
353 // The previous child is dead here, so all three SQLite files are quiescent.
354 try {
355 await backupDatabaseOnce(this.#baseDir, pending);
356 } catch {
357 await this.#returnToPrevious(pending, "failed", "db-backup-failed");
358 return;
359 }
231 try {360 try {
232 await this.#startChild(pending.targetVersion, "trial", pending);361 await this.#startChild(pending.targetVersion, "trial", pending);
233 } catch {362 } catch {
...@@ -322,6 +451,10 @@ export class Launcher {...@@ -322,6 +451,10 @@ export class Launcher {
322 await reject("Remote updates must select a newer server version.");451 await reject("Remote updates must select a newer server version.");
323 return;452 return;
324 }453 }
454 if (!NodePath.isAbsolute(message.dbPath)) {
455 await reject("The requested database path is not absolute.");
456 return;
457 }
325 if (!(await runtimeExists(this.#baseDir, message.targetVersion))) {458 if (!(await runtimeExists(this.#baseDir, message.targetVersion))) {
326 await reject("The requested target runtime is missing or incomplete.");459 await reject("The requested target runtime is missing or incomplete.");
327 return;460 return;
...@@ -331,6 +464,7 @@ export class Launcher {...@@ -331,6 +464,7 @@ export class Launcher {
331 id: NodeCrypto.randomUUID(),464 id: NodeCrypto.randomUUID(),
332 fromVersion: child.version,465 fromVersion: child.version,
333 targetVersion: message.targetVersion,466 targetVersion: message.targetVersion,
467 dbPath: message.dbPath,
334 status: "pending",468 status: "pending",
335 };469 };
336 const next: ServiceState = { ...this.#state, update: pending };470 const next: ServiceState = { ...this.#state, update: pending };
...@@ -375,6 +509,7 @@ export class Launcher {...@@ -375,6 +509,7 @@ export class Launcher {
375 await writeServiceState(this.#statePath, next);509 await writeServiceState(this.#statePath, next);
376 this.#state = next;510 this.#state = next;
377 child.role = "active";511 child.role = "active";
512 await discardDatabaseBackup(this.#baseDir, committed.id).catch(() => undefined);
378 await sendMessage(child.process, { type: "committed", updateId: committed.id });513 await sendMessage(child.process, { type: "committed", updateId: committed.id });
379 }514 }
380515
...@@ -423,6 +558,11 @@ export class Launcher {...@@ -423,6 +558,11 @@ export class Launcher {
423 reason: string,558 reason: string,
424 child?: ManagedChild,559 child?: ManagedChild,
425 ): Promise<void> {560 ): Promise<void> {
561 if (child !== undefined) {
562 this.#child = null;
563 await terminateChild(child.process);
564 }
565 await restoreDatabaseBackup(this.#baseDir, pending);
426 const outcome = terminalUpdate({ pending, status, reason });566 const outcome = terminalUpdate({ pending, status, reason });
427 const next: ServiceState = {567 const next: ServiceState = {
428 ...this.#state,568 ...this.#state,
...@@ -431,10 +571,7 @@ export class Launcher {...@@ -431,10 +571,7 @@ export class Launcher {
431 };571 };
432 await writeServiceState(this.#statePath, next);572 await writeServiceState(this.#statePath, next);
433 this.#state = next;573 this.#state = next;
434 if (child !== undefined) {574 await discardDatabaseBackup(this.#baseDir, pending.id).catch(() => undefined);
435 this.#child = null;
436 await terminateChild(child.process);
437 }
438 await this.#startChild(next.activeVersion, "active", outcome);575 await this.#startChild(next.activeVersion, "active", outcome);
439 }576 }
440}577}
docs/internals/server-updates.md+25-15
...@@ -29,39 +29,49 @@ Every write uses same-directory replacement plus file and directory fsync....@@ -29,39 +29,49 @@ Every write uses same-directory replacement plus file and directory fsync.
29## Remote Update29## Remote Update
3030
311. The active server installs `t3@<target>` into a unique staging directory.311. The active server installs `t3@<target>` into a unique staging directory.
322. The target runs `__service-preflight` against the active database in read-only mode.322. The target runs `__service-preflight` and verifies that the stable launcher supports its update
33 protocol.
333. The staging directory is renamed to its immutable version path only after preflight succeeds.343. The staging directory is renamed to its immutable version path only after preflight succeeds.
344. The active child sends `request-update`. The launcher validates the child and target, writes354. The active child sends `request-update`. The launcher validates the child and target, writes
35 pending state, generates the update ID, then replies `update-accepted`.36 pending state, generates the update ID, then replies `update-accepted`.
365. After a short response-flush grace period, the launcher stops the active child.375. After a short response-flush grace period, the launcher stops the active child.
376. The launcher starts the target as a trial and gives it the pending update over IPC.386. With SQLite quiescent, the launcher snapshots the database, WAL, and shared-memory files.
387. The trial acquires dependencies, binds HTTP, starts every long-running root fiber, and verifies397. The launcher starts the target as a trial and gives it the pending update over IPC.
39 that each root is parked at the activation gate. It does not run migrations.408. The trial runs migrations, acquires dependencies, binds HTTP, starts every long-running root
408. The trial sends `prepared`. The launcher durably commits B before replying `committed`.41 fiber, and verifies that each root is parked at the activation gate.
419. The child opens the existing activation gate, accepts commands, and publishes lifecycle ready429. The trial sends `prepared`. The launcher durably commits B, deletes the snapshot, then replies
42 with the terminal update outcome.43 `committed`.
4410. The child opens the existing activation gate, accepts commands, and publishes lifecycle ready
45 with the terminal update outcome.
4346
44Post-commit startup does not call service `start`, `initialize`, `connect`, `load`, or `acquire`47Post-commit startup does not call service `start`, `initialize`, `connect`, `load`, or `acquire`
45operations. It only opens prepared gates and publishes prepared lifecycle state.48operations. It only opens prepared gates and publishes prepared lifecycle state.
4649
47The launcher serializes child exits, IPC messages, and timers. A trial must report prepared within50The launcher serializes child exits, IPC messages, and timers. A trial must report prepared within
48120 seconds. If the trial exits or times out before prepared, the launcher records rollback and51120 seconds. If the trial exits or times out before prepared, the launcher stops it, restores the
49starts A. After commit, B is active and normal systemd restart policy applies.52snapshot, records rollback, and starts A. A durable restore marker makes an interrupted restore
53resume before either version can boot. After commit, B is active and normal systemd restart policy
54applies.
5055
51## Migration Boundary56## Database Rollback
5257
53Remote update preflight requires exact equality between the database's applied `(id, name)` rows58The launcher snapshots `state.sqlite`, `state.sqlite-wal`, and `state.sqlite-shm` after the old
54and the target release's migration manifest. A target with a missing, additional, renamed, or59server stops and before the trial starts. This makes trial migrations and writes reversible without
55unknown migration is blocked. Remote updates never migrate or downgrade a database.60requiring down migrations. The snapshot is retained across launcher restarts and is removed only
61after commit or after both restore and the terminal rollback state are durable.
5662
57This deliberately means any release containing a migration requires a local service update:63The protocol version is part of the safety boundary. A target that requires database snapshots is
64blocked when the installed launcher is too old. Upgrade the launcher once with:
5865
59```sh66```sh
60npx t3@<version> service update67npx t3@<version> service update
61```68```
6269
63The local command stops the unit, selects the new launcher and exact runtime, then restarts the70The local command stops the unit, selects the new launcher and exact runtime, then restarts the
64service. Its normal server startup may run migrations.71service. Later releases, including releases with migrations, can use the remote trial path.
72
73Snapshots briefly require enough free disk for another copy of the SQLite files. Attachments and
74other files under the state directory are outside this rollback boundary.
6575
66## Client Correlation76## Client Correlation
6777
docs/user/background-service.md+5-2
...@@ -30,10 +30,13 @@ npx t3@latest service uninstall...@@ -30,10 +30,13 @@ npx t3@latest service uninstall
30```30```
3131
32Updating restarts T3 Code briefly. Let active agent work and terminal commands finish first.32Updating restarts T3 Code briefly. Let active agent work and terminal commands finish first.
33If a remote update is already in progress, wait for it to finish before retrying a local update.
3334
34The systemd unit runs a small stable launcher. Exact T3 Code versions are installed separately, so35The systemd unit runs a small stable launcher. Exact T3 Code versions are installed separately, so
35a failed remote candidate can return to the previous version without rewriting the unit. Releases36a failed remote candidate can return to the previous version without rewriting the unit. The
36that change the database must be installed with the local `service update` command above.37launcher snapshots the database before a remote candidate starts, so database updates roll back
38with the server version. An older launcher may require one local `service update` before this is
39available.
3740
38## Using It with T3 Connect41## Using It with T3 Connect
3942
docs/user/updating.md+3-3
...@@ -31,9 +31,9 @@ The update does not remove saved threads, settings, or project files....@@ -31,9 +31,9 @@ The update does not remove saved threads, settings, or project files.
31The available action depends on how that server was started. T3 Code does not update connected31The available action depends on how that server was started. T3 Code does not update connected
32servers silently in the background.32servers silently in the background.
3333
34If the requested version includes a database update, remote installation stops before restart and34An older background-service launcher may ask you to run the exact
35asks you to run the exact `npx t3@<version> service update` command on the server machine. This is35`npx t3@<version> service update` command on the server machine. That one local update installs the
36an intentional rollback-safety boundary.36rollback support needed for later remote updates, including versions that change the database.
3737
38After selecting **Update server**, the warning becomes a three-step progress rail:38After selecting **Update server**, the warning becomes a three-step progress rail:
39**Download**, **Install**, and **Resume**. The same progress appears in the conversation and in39**Download**, **Install**, and **Resume**. The same progress appears in the conversation and in