diff --git a/apps/server/src/cli/connect.test.ts b/apps/server/src/cli/connect.test.ts index cc52a50fe3622033ba64531662659d59288bca50..1e0c88c24e84d7fe57ac325ee90b62578ddc71cd 100644 --- a/apps/server/src/cli/connect.test.ts +++ b/apps/server/src/cli/connect.test.ts @@ -10,6 +10,7 @@ import * as Option from "effect/Option"; import * as References from "effect/References"; import * as Terminal from "effect/Terminal"; +import * as BootService from "../cloud/bootService.ts"; import { acquireRelayClientForLink, formatHeadlessAuthorizationPrompt, @@ -63,6 +64,15 @@ it.effect("treats cancelling optional background setup as a successful skip", () }), ); +it.effect("keeps a successful connection when a remote service update is pending", () => + Effect.gen(function* () { + const result = yield* recoverServiceOnboardingOffer( + Effect.fail(new BootService.BootServiceUpdatePendingError()), + ); + assert.isFalse(result); + }), +); + it.effect("does not install the relay client when the user declines the managed download", () => Effect.gen(function* () { let installCalls = 0; diff --git a/apps/server/src/cli/service.ts b/apps/server/src/cli/service.ts index bd846eeee345a359e2ad6ebb5879c3867a21403d..d55b270f18313c9518f648d019e4685b43270d0d 100644 --- a/apps/server/src/cli/service.ts +++ b/apps/server/src/cli/service.ts @@ -185,6 +185,8 @@ export const recoverServiceOnboardingOffer = ( Console.warn(`Background setup did not finish: ${error.message}`).pipe(Effect.as(false)), BootServiceInstallError: (error) => Console.warn(`Background setup did not finish: ${error.message}`).pipe(Effect.as(false)), + BootServiceUpdatePendingError: (error) => + Console.warn(`Background setup did not finish: ${error.message}`).pipe(Effect.as(false)), }), ); diff --git a/apps/server/src/cloud/bootService.test.ts b/apps/server/src/cloud/bootService.test.ts index 9af69eb17926e5c9121155cf5a416a87ced28589..a9ae3f49f70b94f89ec377c35fbef9f53fcc218d 100644 --- a/apps/server/src/cloud/bootService.test.ts +++ b/apps/server/src/cloud/bootService.test.ts @@ -15,7 +15,11 @@ import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawne import * as ProcessRunner from "../processRunner.ts"; import * as BootService from "./bootService.ts"; import { pinnedRuntimePaths } from "./pinnedRuntime.ts"; -import { parseServiceState } from "./serviceProtocol.ts"; +import { + parseServiceState, + SERVICE_LAUNCHER_PROTOCOL, + serviceStateHasPendingUpdate, +} from "./serviceProtocol.ts"; it("keeps systemd pinned to the stable launcher rather than a versioned server", () => { const unit = BootService.renderBootServiceUnit({ @@ -97,15 +101,24 @@ it.layer(NodeServices.layer)("boot service install", (it) => { const plan = yield* service.install; expect(parseServiceState(yield* fs.readFileString(statePath))).toEqual({ - protocol: 1, + protocol: SERVICE_LAUNCHER_PROTOCOL, activeVersion: "1.2.3", }); expect(yield* fs.readFileString(plan.launcherPath)).toBe("export {};\n"); expect((yield* service.status).current).toBe(true); - yield* fs.writeFileString( - statePath, - '{"protocol":1,"activeVersion":"1.2.3","update":{"id":"u","fromVersion":"1.2.3","targetVersion":"1.2.4","status":"pending"}}', - ); + // @effect-diagnostics-next-line preferSchemaOverJson:off - fixed launcher-owned test document. + const pendingState = JSON.stringify({ + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.2.3", + update: { + id: "u", + fromVersion: "1.2.3", + targetVersion: "1.2.4", + dbPath: "/tmp/state.sqlite", + status: "pending", + }, + }); + yield* fs.writeFileString(statePath, pendingState); expect((yield* service.status).current).toBe(false); expect(yield* service.uninstall).toBe(true); expect((yield* service.status).installed).toBe(false); @@ -141,6 +154,33 @@ it.layer(NodeServices.layer)("boot service install", (it) => { }), ); + it.effect("restarts without overwriting a pending remote update", () => + Effect.gen(function* () { + const { service, fs, statePath, commands } = yield* makeHarness(); + yield* service.install; + // @effect-diagnostics-next-line preferSchemaOverJson:off - fixed launcher-owned test document. + const pendingState = JSON.stringify({ + protocol: SERVICE_LAUNCHER_PROTOCOL - 1, + activeVersion: "1.2.3", + update: { + id: "remote-update", + fromVersion: "1.2.3", + targetVersion: "1.2.4", + status: "pending", + }, + }); + yield* fs.writeFileString(statePath, pendingState); + commands.length = 0; + + expect((yield* service.install.pipe(Effect.flip))._tag).toBe("BootServiceUpdatePendingError"); + expect(serviceStateHasPendingUpdate(yield* fs.readFileString(statePath))).toBe(true); + expect(commands.filter((command) => command.startsWith("systemctl "))).toEqual([ + "systemctl --user stop t3code.service", + "systemctl --user restart t3code.service", + ]); + }), + ); + it.effect("fails closed off Linux", () => Effect.gen(function* () { const { service } = yield* makeHarness("darwin"); diff --git a/apps/server/src/cloud/bootService.ts b/apps/server/src/cloud/bootService.ts index 9a8481b11b511d33d4a9cdc9350a361a8482a57e..ec110cb8b6a8e06fa5fa8b3ced95362c99d762ee 100644 --- a/apps/server/src/cloud/bootService.ts +++ b/apps/server/src/cloud/bootService.ts @@ -21,6 +21,7 @@ import { SERVICE_LAUNCHER_PROTOCOL, SERVICE_STATE_FILE, parseServiceState, + serviceStateHasPendingUpdate, type ServiceState, } from "./serviceProtocol.ts"; @@ -110,10 +111,20 @@ export class BootServiceInstallError extends Schema.TaggedErrorClass()( + "BootServiceUpdatePendingError", + {}, +) { + override get message(): string { + return "A remote server update is still pending. Wait for it to finish, then retry."; + } +} + export type BootServiceError = | BootServiceUnsupportedError | BootServiceCommandError - | BootServiceInstallError; + | BootServiceInstallError + | BootServiceUpdatePendingError; export interface BootServiceStatus { readonly supported: boolean; @@ -288,6 +299,15 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { } yield* Effect.gen(function* () { + if (installed) { + const previousStateText = yield* fs.readFileString(statePath).pipe(Effect.option); + if ( + Option.isSome(previousStateText) && + serviceStateHasPendingUpdate(previousStateText.value) + ) { + return yield* new BootServiceUpdatePendingError(); + } + } yield* fs .makeDirectory(unitDir, { recursive: true }) .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); diff --git a/apps/server/src/cloud/selfUpdate.test.ts b/apps/server/src/cloud/selfUpdate.test.ts index 6fe1d5a4a3c3a4a6ad54ca5b05b3edad9663779f..276ee037773c03c746d145736ce5814ff4039e4a 100644 --- a/apps/server/src/cloud/selfUpdate.test.ts +++ b/apps/server/src/cloud/selfUpdate.test.ts @@ -11,6 +11,7 @@ import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawne import * as ServerConfig from "../config.ts"; import * as ProcessRunner from "../processRunner.ts"; import * as ServiceLauncherClient from "./serviceLauncherClient.ts"; +import { SERVICE_LAUNCHER_PROTOCOL } from "./serviceProtocol.ts"; import * as ServerSelfUpdate from "./selfUpdate.ts"; interface HarnessOptions { @@ -50,7 +51,11 @@ const makeHarness = Effect.fn("test.make_self_update_harness")(function* ( const result = options.preflight === "blocked" ? { status: "blocked", version: "1.1.0", reason: "local update required" } - : { status: "ready", version: "1.1.0", launcherProtocol: 1 }; + : { + status: "ready", + version: "1.1.0", + launcherProtocol: SERVICE_LAUNCHER_PROTOCOL, + }; return { // @effect-diagnostics-next-line preferSchemaOverJson:off - fake child-process stdout. stdout: JSON.stringify(result), @@ -64,7 +69,6 @@ const makeHarness = Effect.fn("test.make_self_update_harness")(function* ( }); const launcher = ServiceLauncherClient.ServiceLauncherClient.of({ managed: options.managed ?? true, - trial: false, requestUpdate: options.requestUpdate ?? (() => diff --git a/apps/server/src/cloud/selfUpdate.ts b/apps/server/src/cloud/selfUpdate.ts index 58bb84117309c48b7277e212e7bfa7da44e018d4..015fd557d3bd75e2134aa534c8c0def8d2fd3b08 100644 --- a/apps/server/src/cloud/selfUpdate.ts +++ b/apps/server/src/cloud/selfUpdate.ts @@ -170,7 +170,7 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* () { yield* reportProgress("installing"); const updateId = yield* launcher - .requestUpdate({ targetVersion }) + .requestUpdate({ targetVersion, dbPath: serverConfig.dbPath }) .pipe( Effect.mapError((error) => failWith( diff --git a/apps/server/src/cloud/serviceLauncherClient.test.ts b/apps/server/src/cloud/serviceLauncherClient.test.ts index 6b9e926ff8db4276b0a7eddc276897a095bb2e40..eab0935419a07b5af63d8ea68990cfe5f0eaaea5 100644 --- a/apps/server/src/cloud/serviceLauncherClient.test.ts +++ b/apps/server/src/cloud/serviceLauncherClient.test.ts @@ -5,6 +5,7 @@ import * as Fiber from "effect/Fiber"; import { SERVICE_LAUNCHER_CONTEXT_ENV, + SERVICE_LAUNCHER_PROTOCOL, type ServiceLauncherChildMessage, type ServiceLauncherParentMessage, } from "./serviceProtocol.ts"; @@ -53,10 +54,11 @@ it.effect("waits for the launcher to durably commit the trial update ID", () => id: "update-1", fromVersion: "1.0.0", targetVersion: "1.1.0", + dbPath: "/tmp/state.sqlite", status: "pending" as const, }; const host = new FakeLauncherProcess({ - protocol: 1, + protocol: SERVICE_LAUNCHER_PROTOCOL, childVersion: "1.1.0", update: pending, }); @@ -79,13 +81,14 @@ it.effect("waits for the launcher to durably commit the trial update ID", () => it.effect("returns the launcher-generated ID only after update acceptance", () => Effect.gen(function* () { const host = new FakeLauncherProcess({ - protocol: 1, + protocol: SERVICE_LAUNCHER_PROTOCOL, childVersion: "1.0.0", }); const client = yield* makeClient(host, "1.0.0"); - const requested = yield* Effect.forkChild(client.requestUpdate({ targetVersion: "1.1.0" }), { - startImmediately: true, - }); + const requested = yield* Effect.forkChild( + client.requestUpdate({ targetVersion: "1.1.0", dbPath: "/tmp/state.sqlite" }), + { startImmediately: true }, + ); yield* Effect.yieldNow; host.emit({ type: "update-accepted", @@ -97,11 +100,15 @@ it.effect("returns the launcher-generated ID only after update acceptance", () = it.effect("preserves a launcher rejection as a distinct error", () => Effect.gen(function* () { - const host = new FakeLauncherProcess({ protocol: 1, childVersion: "1.0.0" }); - const client = yield* makeClient(host, "1.0.0"); - const requested = yield* Effect.forkChild(client.requestUpdate({ targetVersion: "1.1.0" }), { - startImmediately: true, + const host = new FakeLauncherProcess({ + protocol: SERVICE_LAUNCHER_PROTOCOL, + childVersion: "1.0.0", }); + const client = yield* makeClient(host, "1.0.0"); + const requested = yield* Effect.forkChild( + client.requestUpdate({ targetVersion: "1.1.0", dbPath: "/tmp/state.sqlite" }), + { startImmediately: true }, + ); yield* Effect.yieldNow; host.emit({ type: "update-rejected", reason: "requires local update" }); expect(yield* Fiber.join(requested).pipe(Effect.flip)).toMatchObject({ @@ -115,12 +122,13 @@ it.effect("preserves a launcher rejection as a distinct error", () => it.effect("rejects contradictory trial context instead of leaving activation closed", () => Effect.gen(function* () { const host = new FakeLauncherProcess({ - protocol: 1, + protocol: SERVICE_LAUNCHER_PROTOCOL, childVersion: "1.1.0", update: { id: "update-1", fromVersion: "1.0.0", targetVersion: "1.2.0", + dbPath: "/tmp/state.sqlite", status: "pending", }, }); diff --git a/apps/server/src/cloud/serviceLauncherClient.ts b/apps/server/src/cloud/serviceLauncherClient.ts index 760642c29f5971f57c9358f9dc85885db9cceea8..970c72cee72152411ee144b2fe033e2b49c4e564 100644 --- a/apps/server/src/cloud/serviceLauncherClient.ts +++ b/apps/server/src/cloud/serviceLauncherClient.ts @@ -100,9 +100,9 @@ export class ServiceLauncherClient extends Context.Service< ServiceLauncherClient, { readonly managed: boolean; - readonly trial: boolean; readonly requestUpdate: (input: { readonly targetVersion: string; + readonly dbPath: string; }) => Effect.Effect; readonly prepareTrial: Effect.Effect< ServerSelfUpdateOutcome | undefined, @@ -137,8 +137,8 @@ const resolveStartup = Effect.fn("cloud.service_launcher_client.resolve_startup" export const resolveServiceLauncherMode = Effect.fn("cloud.service_launcher_client.resolve_mode")( function* () { - const { context, managed } = yield* resolveStartup(); - return { managed, trial: context?.update?.status === "pending" }; + const { managed } = yield* resolveStartup(); + return { managed }; }, ); @@ -199,7 +199,7 @@ export const make = Effect.fn("cloud.service_launcher_client.make")(function* (o }), ); - const requestUpdate = (input: { readonly targetVersion: string }) => + const requestUpdate = (input: { readonly targetVersion: string; readonly dbPath: string }) => exchange( { type: "request-update", ...input }, (reply) => reply.type === "update-accepted" || reply.type === "update-rejected", @@ -233,14 +233,18 @@ export const make = Effect.fn("cloud.service_launcher_client.make")(function* (o if (reply.type !== "committed") { return Effect.die("service launcher returned an impossible prepared response"); } - return Effect.succeed({ ...pending, status: "committed" as const }); + return Effect.succeed({ + id: pending.id, + fromVersion: pending.fromVersion, + targetVersion: pending.targetVersion, + status: "committed" as const, + }); }), ) : Effect.succeed(outcome); return ServiceLauncherClient.of({ managed, - trial: pending !== undefined, requestUpdate, prepareTrial, }); diff --git a/apps/server/src/cloud/servicePreflight.test.ts b/apps/server/src/cloud/servicePreflight.test.ts index d2ce6db8de841a0dc9f34840c67cb8ac2378fbe1..2eb2e02015d0269894ed10eda5adb03b4fc3bb16 100644 --- a/apps/server/src/cloud/servicePreflight.test.ts +++ b/apps/server/src/cloud/servicePreflight.test.ts @@ -1,47 +1,26 @@ -// @effect-diagnostics nodeBuiltinImport:off -import * as NodeServices from "@effect/platform-node/NodeServices"; import { expect, it } from "@effect/vitest"; -import * as Effect from "effect/Effect"; -import * as FileSystem from "effect/FileSystem"; -import * as Path from "effect/Path"; -import * as NodeSqlite from "node:sqlite"; -import { migrationManifest } from "../persistence/Migrations.ts"; import { runServicePreflight } from "./servicePreflight.ts"; +import { SERVICE_LAUNCHER_PROTOCOL } from "./serviceProtocol.ts"; -it.layer(NodeServices.layer)("service update preflight", (it) => { - it.effect("requires exact migration-manifest equality without mutating the database", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-preflight-test-" }); - const databasePath = path.join(root, "state.sqlite"); - const database = new NodeSqlite.DatabaseSync(databasePath); - database.exec("CREATE TABLE effect_sql_migrations (migration_id INTEGER, name TEXT)"); - const insert = database.prepare( - "INSERT INTO effect_sql_migrations (migration_id, name) VALUES (?, ?)", - ); - for (const [id, name] of migrationManifest) insert.run(id, name); - database.close(); - - expect(runServicePreflight({ databasePath, launcherProtocol: 1, version: "1.2.3" })).toEqual({ - status: "ready", - version: "1.2.3", - launcherProtocol: 1, - }); +it("requires the database-snapshot launcher protocol", () => { + expect( + runServicePreflight({ + databasePath: "/missing/state.sqlite", + launcherProtocol: SERVICE_LAUNCHER_PROTOCOL - 1, + version: "1.2.3", + }), + ).toMatchObject({ status: "blocked", version: "1.2.3" }); - const changed = new NodeSqlite.DatabaseSync(databasePath); - changed.exec("DELETE FROM effect_sql_migrations WHERE migration_id = 35"); - changed.close(); - const blocked = runServicePreflight({ - databasePath, - launcherProtocol: 1, - version: "1.2.3", - }); - expect(blocked.status).toBe("blocked"); - if (blocked.status === "blocked") { - expect(blocked.reason).toContain("npx t3@1.2.3 service update"); - } + expect( + runServicePreflight({ + databasePath: "/missing/state.sqlite", + launcherProtocol: SERVICE_LAUNCHER_PROTOCOL, + version: "1.2.3", }), - ); + ).toEqual({ + status: "ready", + version: "1.2.3", + launcherProtocol: SERVICE_LAUNCHER_PROTOCOL, + }); }); diff --git a/apps/server/src/cloud/servicePreflight.ts b/apps/server/src/cloud/servicePreflight.ts index 1843e16388170d159a1444e2d16d274f17f68081..ee0f972baa3fc9b2296740f8d3533f3ae4cced35 100644 --- a/apps/server/src/cloud/servicePreflight.ts +++ b/apps/server/src/cloud/servicePreflight.ts @@ -1,7 +1,4 @@ -import * as NodeSqlite from "node:sqlite"; - import packageJson from "../../package.json" with { type: "json" }; -import { migrationManifest } from "../persistence/Migrations.ts"; import { SERVICE_LAUNCHER_PROTOCOL } from "./serviceProtocol.ts"; export type ServicePreflightResult = @@ -16,20 +13,8 @@ export type ServicePreflightResult = readonly reason: string; }; -const localUpdateReason = (version: string) => - `This version includes a database update and cannot be installed remotely. Run \`npx t3@${version} service update\` on the server machine.`; - -const isMigrationRow = ( - value: unknown, -): value is { readonly migration_id: number; readonly name: string } => - typeof value === "object" && - value !== null && - "migration_id" in value && - typeof value.migration_id === "number" && - "name" in value && - typeof value.name === "string"; - export function runServicePreflight(input: { + /** Older servers always pass this flag when invoking a staged preflight. */ readonly databasePath: string; readonly launcherProtocol: number; readonly version?: string; @@ -44,28 +29,6 @@ export function runServicePreflight(input: { }; } - try { - const database = new NodeSqlite.DatabaseSync(input.databasePath, { readOnly: true }); - try { - const rows: ReadonlyArray = database - .prepare("SELECT migration_id, name FROM effect_sql_migrations ORDER BY migration_id") - .all(); - const exact = - rows.length === migrationManifest.length && - rows.every((row, index) => { - const expected = migrationManifest[index]; - return ( - isMigrationRow(row) && row.migration_id === expected?.[0] && row.name === expected?.[1] - ); - }); - if (!exact) return { status: "blocked", version, reason: localUpdateReason(version) }; - } finally { - database.close(); - } - } catch { - return { status: "blocked", version, reason: localUpdateReason(version) }; - } - return { status: "ready", version, launcherProtocol: SERVICE_LAUNCHER_PROTOCOL }; } diff --git a/apps/server/src/cloud/serviceProtocol.ts b/apps/server/src/cloud/serviceProtocol.ts index 921bc1447ed2b23352bcc2e99a2f01b7969427ac..ebc5d15d54f00f7178af74849b471a2bbca91ae1 100644 --- a/apps/server/src/cloud/serviceProtocol.ts +++ b/apps/server/src/cloud/serviceProtocol.ts @@ -1,6 +1,7 @@ import type { ServerSelfUpdateOutcome } from "@t3tools/contracts"; -export const SERVICE_LAUNCHER_PROTOCOL = 1 as const; +/** Protocol 2 snapshots SQLite before trials so migrations can be rolled back safely. */ +export const SERVICE_LAUNCHER_PROTOCOL = 2 as const; export const SERVICE_LAUNCHER_CONTEXT_ENV = "T3_SERVICE_LAUNCHER_CONTEXT"; export const SERVICE_LAUNCHER_FILE = "service-launcher.mjs"; export const SERVICE_STATE_FILE = "service-state.json"; @@ -9,6 +10,7 @@ export interface PendingServiceUpdate { readonly id: string; readonly fromVersion: string; readonly targetVersion: string; + readonly dbPath: string; readonly status: "pending"; } @@ -31,6 +33,7 @@ export type ServiceLauncherChildMessage = | { readonly type: "request-update"; readonly targetVersion: string; + readonly dbPath: string; } | { readonly type: "prepared"; @@ -78,7 +81,9 @@ export function decodeServiceUpdate(value: unknown): ServiceUpdateRecord | undef return undefined; } if (status === "pending") { - return { id, fromVersion, targetVersion, status }; + return typeof value.dbPath === "string" && value.dbPath.trim() !== "" + ? { id, fromVersion, targetVersion, dbPath: value.dbPath, status } + : undefined; } if ( (status === "committed" || status === "rolled-back" || status === "failed") && @@ -165,6 +170,16 @@ export function parseServiceState(value: string): ServiceState | undefined { } } +/** Detects an in-flight update across launcher protocol versions before replacing its state. */ +export function serviceStateHasPendingUpdate(value: string): boolean { + try { + const parsed: unknown = JSON.parse(value); + return isRecord(parsed) && isRecord(parsed.update) && parsed.update.status === "pending"; + } catch { + return false; + } +} + export function decodeServiceLauncherContext(value: string): ServiceLauncherContext | undefined { let parsed: unknown; try { @@ -202,8 +217,12 @@ export function decodeServiceLauncherChildMessage( value: unknown, ): ServiceLauncherChildMessage | undefined { if (!isRecord(value)) return undefined; - if (value.type === "request-update" && typeof value.targetVersion === "string") { - return { type: value.type, targetVersion: value.targetVersion }; + if ( + value.type === "request-update" && + typeof value.targetVersion === "string" && + typeof value.dbPath === "string" + ) { + return { type: value.type, targetVersion: value.targetVersion, dbPath: value.dbPath }; } return value.type === "prepared" && typeof value.updateId === "string" ? { type: value.type, updateId: value.updateId } diff --git a/apps/server/src/persistence/Layers/Sqlite.ts b/apps/server/src/persistence/Layers/Sqlite.ts index 3cfdb5c5c76192faadb045ab8103b687a6db6025..d1e002501263b7b797be47041385d9c3a2b59813 100644 --- a/apps/server/src/persistence/Layers/Sqlite.ts +++ b/apps/server/src/persistence/Layers/Sqlite.ts @@ -7,7 +7,6 @@ import type { SqlError } from "effect/unstable/sql/SqlError"; import { runMigrations } from "../Migrations.ts"; import { ServerConfig } from "../../config.ts"; -import * as ServiceLauncherClient from "../../cloud/serviceLauncherClient.ts"; type RuntimeSqliteLayerConfig = { readonly filename: string; @@ -31,28 +30,24 @@ const makeRuntimeSqliteLayer = Effect.fn("makeRuntimeSqliteLayer")(function* ( return clientModule.layer(config); }, Layer.unwrap); -const setup = (trial: boolean) => - Layer.effectDiscard( - Effect.gen(function* () { - const sql = yield* SqlClient.SqlClient; - yield* sql`PRAGMA foreign_keys = ON;`; - if (!trial) { - yield* sql`PRAGMA journal_mode = WAL;`; - yield* runMigrations(); - } - }), - ); +const setup = Layer.effectDiscard( + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql`PRAGMA foreign_keys = ON;`; + yield* sql`PRAGMA journal_mode = WAL;`; + yield* runMigrations(); + }), +); export const makeSqlitePersistenceLive = Effect.fn("makeSqlitePersistenceLive")(function* ( dbPath: string, - options?: { readonly trial?: boolean }, ) { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; yield* fs.makeDirectory(path.dirname(dbPath), { recursive: true }); return Layer.provideMerge( - setup(options?.trial === true), + setup, makeRuntimeSqliteLayer({ filename: dbPath, spanAttributes: { @@ -64,14 +59,13 @@ export const makeSqlitePersistenceLive = Effect.fn("makeSqlitePersistenceLive")( }, Layer.unwrap); export const SqlitePersistenceMemory = Layer.provideMerge( - setup(false), + setup, makeRuntimeSqliteLayer({ filename: ":memory:" }), ); export const layerConfig = Layer.unwrap( Effect.gen(function* () { const { dbPath } = yield* ServerConfig; - const launcher = yield* ServiceLauncherClient.resolveServiceLauncherMode(); - return makeSqlitePersistenceLive(dbPath, { trial: launcher.trial }); + return makeSqlitePersistenceLive(dbPath); }), ); diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index 21f3618d512438f6a2574101ad41df64cdc02296..4562c6a6de8ec75a03d38c532540ca76a79d84db 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -9,6 +9,7 @@ import { compareExactServiceVersions, decodeServiceState, isExactServiceVersion, + SERVICE_LAUNCHER_PROTOCOL, } from "./cloud/serviceProtocol.ts"; it("accepts only exact semantic versions", () => { @@ -33,12 +34,13 @@ it("orders exact semantic versions without treating build metadata as precedence it("rejects contradictory service state", () => { assert.isUndefined( decodeServiceState({ - protocol: 1, + protocol: SERVICE_LAUNCHER_PROTOCOL, activeVersion: "0.0.31", update: { id: "update-1", fromVersion: "0.0.30", targetVersion: "0.0.32", + dbPath: "/tmp/state.sqlite", status: "pending", }, }), @@ -46,12 +48,26 @@ it("rejects contradictory service state", () => { assert.isUndefined( decodeServiceState({ - protocol: 1, + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + update: { + id: "update-3", + fromVersion: "1.0.0", + targetVersion: "1.1.0", + status: "pending", + }, + }), + ); + + assert.isUndefined( + decodeServiceState({ + protocol: SERVICE_LAUNCHER_PROTOCOL, activeVersion: "1.0.0", update: { id: "update-2", fromVersion: "1.0.0", targetVersion: "0.9.0", + dbPath: "/tmp/state.sqlite", status: "pending", }, }), @@ -66,7 +82,7 @@ it.layer(NodeServices.layer)("service state persistence", (it) => { const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-test-" }); const statePath = path.join(root, "runtime", "service-state.json"); const state = { - protocol: 1, + protocol: SERVICE_LAUNCHER_PROTOCOL, activeVersion: "0.0.31", } as const; @@ -88,7 +104,7 @@ it.layer(NodeServices.layer)("service state persistence", (it) => { yield* fs.writeFileString(path.join(versionDir, ".install-complete"), "1.0.0\n"); yield* Effect.promise(() => writeServiceState(statePath, { - protocol: 1, + protocol: SERVICE_LAUNCHER_PROTOCOL, activeVersion: "1.0.0", }), ); @@ -106,6 +122,11 @@ it.layer(NodeServices.layer)("service state persistence", (it) => { const path = yield* Path.Path; const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-flow-" }); const statePath = path.join(root, "runtime", "service-state.json"); + const databasePath = path.join(root, "userdata", "state.sqlite"); + yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true }); + yield* fs.writeFileString(databasePath, "before trial"); + // @effect-diagnostics-next-line preferSchemaOverJson:off - embeds a path in fake child source. + const encodedDatabasePath = JSON.stringify(databasePath); const childSource = ` const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT); if (context.update?.status === "pending") { @@ -114,7 +135,7 @@ if (context.update?.status === "pending") { if (message.type === "committed") process.exit(0); }); } else if (context.update === undefined) { - process.send({ type: "request-update", targetVersion: "1.1.0" }); + process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: ${encodedDatabasePath} }); setInterval(() => {}, 1_000); } else { process.exit(0); @@ -129,7 +150,7 @@ if (context.update?.status === "pending") { } yield* Effect.promise(() => writeServiceState(statePath, { - protocol: 1, + protocol: SERVICE_LAUNCHER_PROTOCOL, activeVersion: "1.0.0", }), ); @@ -154,12 +175,17 @@ if (context.update?.status === "pending") { const path = yield* Path.Path; const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-rollback-" }); const statePath = path.join(root, "runtime", "service-state.json"); + const databasePath = path.join(root, "userdata", "state.sqlite"); + yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true }); + yield* fs.writeFileString(databasePath, "before trial"); + // @effect-diagnostics-next-line preferSchemaOverJson:off - embeds a path in fake child source. + const encodedDatabasePath = JSON.stringify(databasePath); const childSource = ` const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT); if (context.update?.status === "pending") { process.send({ type: "prepared", updateId: "wrong-update" }); } else if (context.update === undefined) { - process.send({ type: "request-update", targetVersion: "1.1.0" }); + process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: ${encodedDatabasePath} }); setInterval(() => {}, 1_000); } else { process.exit(0); @@ -174,7 +200,7 @@ if (context.update?.status === "pending") { } yield* Effect.promise(() => writeServiceState(statePath, { - protocol: 1, + protocol: SERVICE_LAUNCHER_PROTOCOL, activeVersion: "1.0.0", }), ); @@ -196,4 +222,65 @@ if (context.update?.status === "pending") { ); }), ); + + it.effect("restores the database when a migrating trial exits", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-db-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + const databasePath = path.join(root, "userdata", "state.sqlite"); + const original = "database before migration"; + yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true }); + yield* fs.writeFileString(databasePath, original); + // @effect-diagnostics-next-line preferSchemaOverJson:off - embeds a path in fake child source. + const encodedDatabasePath = JSON.stringify(databasePath); + const childSource = ` +import { writeFileSync } from "node:fs"; +const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT); +if (context.update?.status === "pending") { + writeFileSync(context.update.dbPath, "database after migration"); + writeFileSync(context.update.dbPath + "-wal", "trial wal"); + writeFileSync(context.update.dbPath + "-shm", "trial shm"); + process.exit(1); +} else if (context.update === undefined) { + process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: ${encodedDatabasePath} }); + setInterval(() => {}, 1_000); +} else { + process.exit(0); +} +`; + for (const version of ["1.0.0", "1.1.0"]) { + const versionDir = path.join(root, "runtime", "versions", version); + const entryPath = path.join(versionDir, "node_modules", "t3", "dist", "bin.mjs"); + yield* fs.makeDirectory(path.dirname(entryPath), { recursive: true }); + yield* fs.writeFileString(entryPath, childSource); + yield* fs.writeFileString(path.join(versionDir, ".install-complete"), `${version}\n`); + } + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + }), + ); + + const launcher = new Launcher(root, yield* Effect.promise(() => readServiceState(statePath))); + yield* Effect.promise(() => + launcher.run().then( + () => Promise.reject(new Error("launcher unexpectedly completed")), + () => Promise.resolve(), + ), + ); + + const state = yield* Effect.promise(() => readServiceState(statePath)); + assert.equal(state.activeVersion, "1.0.0"); + assert.equal(state.update?.status, "rolled-back"); + assert.equal(yield* fs.readFileString(databasePath), original); + assert.isFalse(yield* fs.exists(`${databasePath}-wal`)); + assert.isFalse(yield* fs.exists(`${databasePath}-shm`)); + const updateId = state.update?.id; + assert.isDefined(updateId); + assert.isFalse(yield* fs.exists(path.join(root, "runtime", "db-backup", updateId))); + }), + ); }); diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index d7ca33578023240852ea263cf84fb0f8137abd3f..211c0138bc4b6e176aab4899690b53f9c30d2388 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -48,6 +48,121 @@ const runtimePaths = (baseDir: string, version: string) => { }; }; +/** SQLite persists across the main file plus its WAL and shared-memory sidecars. */ +const DB_FILE_SUFFIXES = ["", "-wal", "-shm"] as const; +const RESTORE_MARKER = ".restore-pending"; + +const databaseBackupDir = (baseDir: string, updateId: string) => + NodePath.join(baseDir, "runtime", "db-backup", updateId); + +const databaseBackupFile = (backupDir: string, suffix: (typeof DB_FILE_SUFFIXES)[number]) => + NodePath.join(backupDir, suffix === "" ? "database" : `database${suffix}`); + +async function pathExists(target: string): Promise { + try { + await NodeFSP.access(target); + return true; + } catch (cause) { + if (cause instanceof Error && "code" in cause && cause.code === "ENOENT") return false; + throw cause; + } +} + +async function syncFile(filePath: string): Promise { + const handle = await NodeFSP.open(filePath, "r"); + try { + await handle.sync(); + } finally { + await handle.close(); + } +} + +async function syncDirectory(directory: string): Promise { + const handle = await NodeFSP.open(directory, "r"); + try { + await handle.sync(); + } finally { + await handle.close(); + } +} + +/** + * Snapshots the database once per update before the first trial. A completed + * backup is never overwritten because a restarted launcher may be looking at + * database writes from an earlier attempt by the same trial. + */ +async function backupDatabaseOnce(baseDir: string, pending: PendingServiceUpdate): Promise { + const backupDir = databaseBackupDir(baseDir, pending.id); + if (await pathExists(backupDir)) return; + + const stagingDir = `${backupDir}.staging`; + await NodeFSP.rm(stagingDir, { recursive: true, force: true }); + await NodeFSP.mkdir(stagingDir, { recursive: true, mode: 0o700 }); + try { + for (const suffix of DB_FILE_SUFFIXES) { + const source = `${pending.dbPath}${suffix}`; + if (suffix !== "" && !(await pathExists(source))) continue; + const destination = databaseBackupFile(stagingDir, suffix); + await NodeFSP.copyFile(source, destination); + await syncFile(destination); + } + await NodeFSP.rename(stagingDir, backupDir); + await syncDirectory(NodePath.dirname(backupDir)); + } catch (cause) { + await NodeFSP.rm(stagingDir, { recursive: true, force: true }).catch(() => undefined); + throw cause; + } +} + +const restoreMarkerPath = (baseDir: string, updateId: string) => + NodePath.join(databaseBackupDir(baseDir, updateId), RESTORE_MARKER); + +const databaseRestorePending = (baseDir: string, pending: PendingServiceUpdate) => + pathExists(restoreMarkerPath(baseDir, pending.id)); + +/** Mark rollback before changing live files so launcher recovery cannot boot a partial restore. */ +async function markDatabaseRestorePending(backupDir: string): Promise { + const markerPath = NodePath.join(backupDir, RESTORE_MARKER); + if (!(await pathExists(markerPath))) { + const handle = await NodeFSP.open(markerPath, "wx", 0o600); + try { + await handle.sync(); + } finally { + await handle.close(); + } + await syncDirectory(backupDir); + } +} + +/** Restore is retryable after any process crash while the backup directory remains. */ +async function restoreDatabaseBackup( + baseDir: string, + pending: PendingServiceUpdate, +): Promise { + const backupDir = databaseBackupDir(baseDir, pending.id); + if (!(await pathExists(backupDir))) return; + + await markDatabaseRestorePending(backupDir); + for (const suffix of DB_FILE_SUFFIXES) { + const target = `${pending.dbPath}${suffix}`; + const source = databaseBackupFile(backupDir, suffix); + if (await pathExists(source)) { + await NodeFSP.copyFile(source, target); + await syncFile(target); + } else { + await NodeFSP.rm(target, { force: true }); + } + } + await syncDirectory(NodePath.dirname(pending.dbPath)); +} + +async function discardDatabaseBackup(baseDir: string, updateId: string): Promise { + const backupDir = databaseBackupDir(baseDir, updateId); + if (!(await pathExists(backupDir))) return; + await NodeFSP.rm(backupDir, { recursive: true, force: true }); + await syncDirectory(NodePath.dirname(backupDir)); +} + export async function readServiceState(filePath: string): Promise { const contents = await NodeFSP.readFile(filePath, "utf8"); const state = parseServiceState(contents); @@ -217,9 +332,16 @@ export class Launcher { async #recover(): Promise { const update = this.#state.update; if (update?.status !== "pending") { + if (update !== undefined) { + await discardDatabaseBackup(this.#baseDir, update.id).catch(() => undefined); + } await this.#startChild(this.#state.activeVersion, "active", update); return; } + if (await databaseRestorePending(this.#baseDir, update)) { + await this.#returnToPrevious(update, "failed", "rollback-interrupted"); + return; + } if (!(await runtimeExists(this.#baseDir, update.targetVersion))) { await this.#returnToPrevious(update, "failed", "target-runtime-missing"); return; @@ -228,6 +350,13 @@ export class Launcher { } async #startTrial(pending: PendingServiceUpdate): Promise { + // The previous child is dead here, so all three SQLite files are quiescent. + try { + await backupDatabaseOnce(this.#baseDir, pending); + } catch { + await this.#returnToPrevious(pending, "failed", "db-backup-failed"); + return; + } try { await this.#startChild(pending.targetVersion, "trial", pending); } catch { @@ -322,6 +451,10 @@ export class Launcher { await reject("Remote updates must select a newer server version."); return; } + if (!NodePath.isAbsolute(message.dbPath)) { + await reject("The requested database path is not absolute."); + return; + } if (!(await runtimeExists(this.#baseDir, message.targetVersion))) { await reject("The requested target runtime is missing or incomplete."); return; @@ -331,6 +464,7 @@ export class Launcher { id: NodeCrypto.randomUUID(), fromVersion: child.version, targetVersion: message.targetVersion, + dbPath: message.dbPath, status: "pending", }; const next: ServiceState = { ...this.#state, update: pending }; @@ -375,6 +509,7 @@ export class Launcher { await writeServiceState(this.#statePath, next); this.#state = next; child.role = "active"; + await discardDatabaseBackup(this.#baseDir, committed.id).catch(() => undefined); await sendMessage(child.process, { type: "committed", updateId: committed.id }); } @@ -423,6 +558,11 @@ export class Launcher { reason: string, child?: ManagedChild, ): Promise { + if (child !== undefined) { + this.#child = null; + await terminateChild(child.process); + } + await restoreDatabaseBackup(this.#baseDir, pending); const outcome = terminalUpdate({ pending, status, reason }); const next: ServiceState = { ...this.#state, @@ -431,10 +571,7 @@ export class Launcher { }; await writeServiceState(this.#statePath, next); this.#state = next; - if (child !== undefined) { - this.#child = null; - await terminateChild(child.process); - } + await discardDatabaseBackup(this.#baseDir, pending.id).catch(() => undefined); await this.#startChild(next.activeVersion, "active", outcome); } } diff --git a/docs/internals/server-updates.md b/docs/internals/server-updates.md index f0a0034c3ee842a4980b4e603211ad6f9a66ee87..c2ba77f6f66ee203b51077de7fbd1886ae7fff59 100644 --- a/docs/internals/server-updates.md +++ b/docs/internals/server-updates.md @@ -29,39 +29,49 @@ Every write uses same-directory replacement plus file and directory fsync. ## Remote Update 1. The active server installs `t3@` into a unique staging directory. -2. The target runs `__service-preflight` against the active database in read-only mode. +2. The target runs `__service-preflight` and verifies that the stable launcher supports its update + protocol. 3. The staging directory is renamed to its immutable version path only after preflight succeeds. 4. The active child sends `request-update`. The launcher validates the child and target, writes pending state, generates the update ID, then replies `update-accepted`. 5. After a short response-flush grace period, the launcher stops the active child. -6. The launcher starts the target as a trial and gives it the pending update over IPC. -7. The trial acquires dependencies, binds HTTP, starts every long-running root fiber, and verifies - that each root is parked at the activation gate. It does not run migrations. -8. The trial sends `prepared`. The launcher durably commits B before replying `committed`. -9. The child opens the existing activation gate, accepts commands, and publishes lifecycle ready - with the terminal update outcome. +6. With SQLite quiescent, the launcher snapshots the database, WAL, and shared-memory files. +7. The launcher starts the target as a trial and gives it the pending update over IPC. +8. The trial runs migrations, acquires dependencies, binds HTTP, starts every long-running root + fiber, and verifies that each root is parked at the activation gate. +9. The trial sends `prepared`. The launcher durably commits B, deletes the snapshot, then replies + `committed`. +10. The child opens the existing activation gate, accepts commands, and publishes lifecycle ready + with the terminal update outcome. Post-commit startup does not call service `start`, `initialize`, `connect`, `load`, or `acquire` operations. It only opens prepared gates and publishes prepared lifecycle state. The launcher serializes child exits, IPC messages, and timers. A trial must report prepared within -120 seconds. If the trial exits or times out before prepared, the launcher records rollback and -starts A. After commit, B is active and normal systemd restart policy applies. +120 seconds. If the trial exits or times out before prepared, the launcher stops it, restores the +snapshot, records rollback, and starts A. A durable restore marker makes an interrupted restore +resume before either version can boot. After commit, B is active and normal systemd restart policy +applies. -## Migration Boundary +## Database Rollback -Remote update preflight requires exact equality between the database's applied `(id, name)` rows -and the target release's migration manifest. A target with a missing, additional, renamed, or -unknown migration is blocked. Remote updates never migrate or downgrade a database. +The launcher snapshots `state.sqlite`, `state.sqlite-wal`, and `state.sqlite-shm` after the old +server stops and before the trial starts. This makes trial migrations and writes reversible without +requiring down migrations. The snapshot is retained across launcher restarts and is removed only +after commit or after both restore and the terminal rollback state are durable. -This deliberately means any release containing a migration requires a local service update: +The protocol version is part of the safety boundary. A target that requires database snapshots is +blocked when the installed launcher is too old. Upgrade the launcher once with: ```sh npx t3@ service update ``` The local command stops the unit, selects the new launcher and exact runtime, then restarts the -service. Its normal server startup may run migrations. +service. Later releases, including releases with migrations, can use the remote trial path. + +Snapshots briefly require enough free disk for another copy of the SQLite files. Attachments and +other files under the state directory are outside this rollback boundary. ## Client Correlation diff --git a/docs/user/background-service.md b/docs/user/background-service.md index 0ab5b2013c730ed47650af0a1295c569778fb359..299e3b641f17e540171ada0bbde90925b81bc8e1 100644 --- a/docs/user/background-service.md +++ b/docs/user/background-service.md @@ -30,10 +30,13 @@ npx t3@latest service uninstall ``` Updating restarts T3 Code briefly. Let active agent work and terminal commands finish first. +If a remote update is already in progress, wait for it to finish before retrying a local update. The systemd unit runs a small stable launcher. Exact T3 Code versions are installed separately, so -a failed remote candidate can return to the previous version without rewriting the unit. Releases -that change the database must be installed with the local `service update` command above. +a failed remote candidate can return to the previous version without rewriting the unit. The +launcher snapshots the database before a remote candidate starts, so database updates roll back +with the server version. An older launcher may require one local `service update` before this is +available. ## Using It with T3 Connect diff --git a/docs/user/updating.md b/docs/user/updating.md index a0cc0e5d1e0c2c8aa21f334e32577d9c805ffd68..ea2ff644cf72186e67d946b066c04b4f72629188 100644 --- a/docs/user/updating.md +++ b/docs/user/updating.md @@ -31,9 +31,9 @@ The update does not remove saved threads, settings, or project files. The available action depends on how that server was started. T3 Code does not update connected servers silently in the background. -If the requested version includes a database update, remote installation stops before restart and -asks you to run the exact `npx t3@ service update` command on the server machine. This is -an intentional rollback-safety boundary. +An older background-service launcher may ask you to run the exact +`npx t3@ service update` command on the server machine. That one local update installs the +rollback support needed for later remote updates, including versions that change the database. After selecting **Update server**, the warning becomes a three-step progress rail: **Download**, **Install**, and **Resume**. The same progress appears in the conversation and in