| author |  Theo Browne <me@t3.gg> 2026-08-01 05:48:03-07:00 |
| committer |  GitHub <noreply@github.com> 2026-08-01 05:48:03-07:00 |
| log | d3037064e61a9f059eafbd4f9869679779bd2a7c |
| tree | dbabde7e2086de111b674d179e749d3a54fac5cb |
| parent | 0ad91b6e7fc1fcb6d5f4bc736d84c337e912bc62 |
| signature | Signed by PGP key B5690EEEBB952194 |
fix(server): make remote updates rollback-safe (#5181)
40 files changed, 2831 insertions(+), 2245 deletions(-)
apps
docs
internals
operations
packages
client-runtime
src
contracts
apps/server/package.json+1-1
| ... | ... | @@ -16,7 +16,7 @@ |
| 16 | 16 | "type": "module", |
| 17 | 17 | "scripts": { |
| 18 | 18 | "dev": "node --watch src/bin.ts", |
| 19 | "build:bundle": "vp pack", | |
| 19 | "build:bundle": "vp pack && vp pack src/service-launcher.ts --out-dir dist --no-clean", | |
| 20 | 20 | "start": "node dist/bin.mjs", |
| 21 | 21 | "typecheck": "tsgo --noEmit", |
| 22 | 22 | "test": "vp test run" |
apps/server/scripts/cli.ts+5-1
| ... | ... | @@ -223,7 +223,11 @@ const publishCmd = Command.make( |
| 223 | 223 | const packageJsonPath = path.join(serverDir, "package.json"); |
| 224 | 224 | |
| 225 | 225 | // Assert build assets exist |
| 226 | for (const relPath of ["dist/bin.mjs", "dist/client/index.html"]) { | |
| 226 | for (const relPath of [ | |
| 227 | "dist/bin.mjs", | |
| 228 | "dist/service-launcher.mjs", | |
| 229 | "dist/client/index.html", | |
| 230 | ]) { | |
| 227 | 231 | const abs = path.join(serverDir, relPath); |
| 228 | 232 | if (!(yield* fs.exists(abs))) { |
| 229 | 233 | return yield* new ServerCliBuildAssetMissingError({ assetPath: abs }); |
apps/server/src/bin.ts+2
| ... | ... | @@ -15,6 +15,7 @@ import { sharedServerCommandFlags } from "./cli/config.ts"; |
| 15 | 15 | import { projectCommand } from "./cli/project.ts"; |
| 16 | 16 | import { runServerCommand, serveCommand, startCommand } from "./cli/server.ts"; |
| 17 | 17 | import { serviceCommand } from "./cli/service.ts"; |
| 18 | import { servicePreflightCommand } from "./cli/servicePreflight.ts"; | |
| 18 | 19 | |
| 19 | 20 | const CliRuntimeLayer = Layer.mergeAll(NodeServices.layer, NetService.layer); |
| 20 | 21 | |
| ... | ... | @@ -53,6 +54,7 @@ export const makeCli = ({ cloudEnabled = hasCloudPublicConfig } = {}) => |
| 53 | 54 | authCommand, |
| 54 | 55 | projectCommand, |
| 55 | 56 | serviceCommand, |
| 57 | servicePreflightCommand, | |
| 56 | 58 | cloudEnabled ? connectCommand : connectUnavailableCommand, |
| 57 | 59 | ]), |
| 58 | 60 | ); |
apps/server/src/cli/servicePreflight.ts created+17
| ... | ... | @@ -0,0 +1,17 @@ |
| 1 | import * as Console from "effect/Console"; | |
| 2 | import * as Effect from "effect/Effect"; | |
| 3 | import { Command, Flag } from "effect/unstable/cli"; | |
| 4 | ||
| 5 | import { runServicePreflight } from "../cloud/servicePreflight.ts"; | |
| 6 | ||
| 7 | export const servicePreflightCommand = Command.make("__service-preflight", { | |
| 8 | databasePath: Flag.string("database-path"), | |
| 9 | launcherProtocol: Flag.integer("launcher-protocol"), | |
| 10 | }).pipe( | |
| 11 | Command.withHidden, | |
| 12 | Command.withHandler(({ databasePath, launcherProtocol }) => | |
| 13 | Console.log(JSON.stringify(runServicePreflight({ databasePath, launcherProtocol }))).pipe( | |
| 14 | Effect.asVoid, | |
| 15 | ), | |
| 16 | ), | |
| 17 | ); |
apps/server/src/cloud/bootService.test.ts+92-514
| ... | ... | @@ -1,558 +1,136 @@ |
| 1 | 1 | import * as NodeServices from "@effect/platform-node/NodeServices"; |
| 2 | import { assert, it } from "@effect/vitest"; | |
| 2 | import { expect, it } from "@effect/vitest"; | |
| 3 | import { | |
| 4 | HostProcessArguments, | |
| 5 | HostProcessExecutablePath, | |
| 6 | HostProcessPlatform, | |
| 7 | } from "@t3tools/shared/hostProcess"; | |
| 3 | 8 | import * as ConfigProvider from "effect/ConfigProvider"; |
| 4 | 9 | import * as Effect from "effect/Effect"; |
| 5 | 10 | import * as FileSystem from "effect/FileSystem"; |
| 6 | 11 | import * as Layer from "effect/Layer"; |
| 7 | 12 | import * as Path from "effect/Path"; |
| 8 | import * as Schema from "effect/Schema"; | |
| 9 | 13 | import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner"; |
| 10 | 14 | |
| 11 | import { | |
| 12 | HostProcessArguments, | |
| 13 | HostProcessExecutablePath, | |
| 14 | HostProcessPlatform, | |
| 15 | } from "@t3tools/shared/hostProcess"; | |
| 16 | ||
| 17 | import { reconcileService } from "../cli/service.ts"; | |
| 18 | 15 | import * as ProcessRunner from "../processRunner.ts"; |
| 19 | 16 | import * as BootService from "./bootService.ts"; |
| 17 | import { pinnedRuntimePaths } from "./pinnedRuntime.ts"; | |
| 18 | import { parseServiceState } from "./serviceProtocol.ts"; | |
| 20 | 19 | |
| 21 | const isUnsupportedError = Schema.is(BootService.BootServiceUnsupportedError); | |
| 22 | const isCommandError = Schema.is(BootService.BootServiceCommandError); | |
| 23 | ||
| 24 | interface RecordedCommand { | |
| 25 | readonly command: string; | |
| 26 | readonly args: ReadonlyArray<string>; | |
| 27 | } | |
| 28 | ||
| 29 | const makeRecordingRunnerLayer = ( | |
| 30 | commands: Array<RecordedCommand>, | |
| 31 | options?: { | |
| 32 | readonly failCommand?: string; | |
| 33 | readonly failWhen?: (command: string, args: ReadonlyArray<string>) => boolean; | |
| 34 | }, | |
| 35 | ) => | |
| 36 | Layer.succeed( | |
| 37 | ProcessRunner.ProcessRunner, | |
| 38 | ProcessRunner.ProcessRunner.of({ | |
| 39 | run: (input) => | |
| 40 | Effect.sync(() => { | |
| 41 | assert.isUndefined(input.env); | |
| 42 | commands.push({ command: input.command, args: input.args }); | |
| 43 | const failed = | |
| 44 | input.command === options?.failCommand || | |
| 45 | options?.failWhen?.(input.command, input.args) === true; | |
| 46 | return { | |
| 47 | stdout: "", | |
| 48 | stderr: failed ? `${input.command} exploded` : "", | |
| 49 | code: ChildProcessSpawner.ExitCode(failed ? 1 : 0), | |
| 50 | timedOut: false, | |
| 51 | stdoutTruncated: false, | |
| 52 | stderrTruncated: false, | |
| 53 | }; | |
| 54 | }), | |
| 55 | }), | |
| 56 | ); | |
| 57 | ||
| 58 | const makeHost = (entry: string): BootService.BootServiceHost => ({ | |
| 59 | execPath: "/usr/local/bin/node", | |
| 60 | cliEntryPath: entry, | |
| 61 | }); | |
| 62 | ||
| 63 | const provideHostRefs = (home: string, platform: NodeJS.Platform = "linux") => | |
| 64 | Effect.provide( | |
| 65 | Layer.mergeAll( | |
| 66 | Layer.succeed(HostProcessPlatform, platform), | |
| 67 | ConfigProvider.layer(ConfigProvider.fromEnv({ env: { HOME: home } })), | |
| 68 | ), | |
| 69 | ); | |
| 70 | ||
| 71 | const makeTestContext = Effect.fn("test.makeTestContext")(function* () { | |
| 72 | const fs = yield* FileSystem.FileSystem; | |
| 73 | const path = yield* Path.Path; | |
| 74 | const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-boot-service-test-" }); | |
| 75 | // A real file for the stable-entry cases so status can confirm the entry | |
| 76 | // point exists. | |
| 77 | const stableEntry = path.join(root, "bin.mjs"); | |
| 78 | yield* fs.writeFileString(stableEntry, "#!/usr/bin/env node\n"); | |
| 79 | return { | |
| 80 | fs, | |
| 81 | path, | |
| 82 | dirs: { | |
| 83 | home: root, | |
| 84 | baseDir: path.join(root, ".t3"), | |
| 85 | logsDir: path.join(root, ".t3", "userdata", "logs"), | |
| 86 | stableEntry, | |
| 87 | }, | |
| 88 | }; | |
| 89 | }); | |
| 90 | ||
| 91 | it("renders a systemd unit with absolute paths and append-mode logging", () => { | |
| 20 | it("keeps systemd pinned to the stable launcher rather than a versioned server", () => { | |
| 92 | 21 | const unit = BootService.renderBootServiceUnit({ |
| 93 | nodePath: "/usr/local/bin/node", | |
| 94 | t3EntryPath: "/home/theo/.t3/runtime/versions/0.0.27/node_modules/t3/dist/bin.mjs", | |
| 22 | nodePath: "/usr/bin/node", | |
| 23 | launcherPath: "/home/theo/.t3/runtime/service-launcher.mjs", | |
| 95 | 24 | baseDir: "/home/theo/.t3", |
| 96 | 25 | logPath: "/home/theo/.t3/userdata/logs/boot-service.log", |
| 97 | 26 | unitPath: "/home/theo/.config/systemd/user/t3code.service", |
| 98 | 27 | }); |
| 99 | 28 | |
| 100 | assert.equal( | |
| 101 | unit, | |
| 102 | [ | |
| 103 | "[Unit]", | |
| 104 | "Description=T3 Code server", | |
| 105 | "StartLimitIntervalSec=300", | |
| 106 | "StartLimitBurst=5", | |
| 107 | "", | |
| 108 | "[Service]", | |
| 109 | "Type=simple", | |
| 110 | "WorkingDirectory=%h", | |
| 111 | "Environment=T3CODE_HOME=/home/theo/.t3", | |
| 112 | "Environment=T3_BOOT_SERVICE_UNIT=t3code.service", | |
| 113 | "ExecStart=/usr/local/bin/node /home/theo/.t3/runtime/versions/0.0.27/node_modules/t3/dist/bin.mjs serve", | |
| 114 | "Restart=always", | |
| 115 | "RestartSec=5", | |
| 116 | "StandardOutput=append:/home/theo/.t3/userdata/logs/boot-service.log", | |
| 117 | "StandardError=append:/home/theo/.t3/userdata/logs/boot-service.log", | |
| 118 | "", | |
| 119 | "[Install]", | |
| 120 | "WantedBy=default.target", | |
| 121 | "", | |
| 122 | ].join("\n"), | |
| 123 | ); | |
| 29 | expect(unit).toContain("ExecStart=/usr/bin/node /home/theo/.t3/runtime/service-launcher.mjs"); | |
| 30 | expect(unit).toContain("KillMode=control-group"); | |
| 31 | expect(unit).not.toContain("versions/1.2.3"); | |
| 124 | 32 | }); |
| 125 | 33 | |
| 126 | it("quotes systemd values containing spaces and escapes percent specifiers", () => { | |
| 127 | assert.equal(BootService.quoteSystemdValue("/plain/path"), "/plain/path"); | |
| 128 | assert.equal(BootService.quoteSystemdValue("/home/me/T3 Data"), '"/home/me/T3 Data"'); | |
| 129 | assert.equal(BootService.quoteSystemdValue("/opt/100%cpu"), "/opt/100%%cpu"); | |
| 130 | ||
| 131 | const unit = BootService.renderBootServiceUnit({ | |
| 132 | nodePath: "/home/me/my tools/node", | |
| 133 | t3EntryPath: "/home/me/T3 Data/bin.mjs", | |
| 134 | baseDir: "/home/me/T3 Data", | |
| 135 | logPath: "/home/me/100%logs/boot.log", | |
| 136 | unitPath: "/home/me/.config/systemd/user/t3code.service", | |
| 34 | const makeHarness = Effect.fn("test.make_boot_service_harness")(function* ( | |
| 35 | platform: NodeJS.Platform = "linux", | |
| 36 | ) { | |
| 37 | const fs = yield* FileSystem.FileSystem; | |
| 38 | const path = yield* Path.Path; | |
| 39 | const home = yield* fs.makeTempDirectoryScoped({ prefix: "t3-boot-service-test-" }); | |
| 40 | const baseDir = path.join(home, ".t3"); | |
| 41 | const sourceLauncher = path.join(home, "service-launcher.mjs"); | |
| 42 | const statePath = path.join(baseDir, "runtime", "service-state.json"); | |
| 43 | yield* fs.writeFileString(sourceLauncher, "export {};\n"); | |
| 44 | const runtime = pinnedRuntimePaths(path, baseDir, "1.2.3"); | |
| 45 | yield* fs.makeDirectory(path.dirname(runtime.entryPath), { recursive: true }); | |
| 46 | yield* fs.writeFileString(runtime.entryPath, "export {};\n"); | |
| 47 | yield* fs.writeFileString(runtime.sentinelPath, "1.2.3\n"); | |
| 48 | ||
| 49 | const commands: string[] = []; | |
| 50 | const control: { failCommand: string | undefined } = { failCommand: undefined }; | |
| 51 | const runner = ProcessRunner.ProcessRunner.of({ | |
| 52 | run: (input) => | |
| 53 | Effect.sync(() => { | |
| 54 | const command = `${input.command} ${input.args.join(" ")}`; | |
| 55 | commands.push(command); | |
| 56 | return { | |
| 57 | stdout: input.args[1] === "--version" ? "t3 v1.2.3\n" : "", | |
| 58 | stderr: "", | |
| 59 | code: ChildProcessSpawner.ExitCode(command === control.failCommand ? 1 : 0), | |
| 60 | timedOut: false, | |
| 61 | stdoutTruncated: false, | |
| 62 | stderrTruncated: false, | |
| 63 | }; | |
| 64 | }), | |
| 137 | 65 | }); |
| 138 | assert.include(unit, 'ExecStart="/home/me/my tools/node" "/home/me/T3 Data/bin.mjs" serve'); | |
| 139 | assert.include(unit, 'Environment=T3CODE_HOME="/home/me/T3 Data"'); | |
| 140 | // append: paths take the rest of the line literally (spaces are fine, | |
| 141 | // quoting is not), but % still goes through specifier expansion. | |
| 142 | assert.include(unit, "StandardOutput=append:/home/me/100%%logs/boot.log"); | |
| 143 | assert.include(unit, "StandardError=append:/home/me/100%%logs/boot.log"); | |
| 144 | }); | |
| 145 | ||
| 146 | it("flags package-manager cache entry points as ephemeral", () => { | |
| 147 | assert.isTrue( | |
| 148 | BootService.isEphemeralCacheEntry("/home/theo/.npm/_npx/abc123/node_modules/t3/dist/bin.mjs"), | |
| 149 | ); | |
| 150 | assert.isTrue( | |
| 151 | BootService.isEphemeralCacheEntry("C:\\Users\\theo\\AppData\\npm-cache\\_npx\\abc\\bin.mjs"), | |
| 152 | ); | |
| 153 | assert.isTrue( | |
| 154 | BootService.isEphemeralCacheEntry( | |
| 155 | "/home/theo/.cache/pnpm/dlx/abc/node_modules/t3/dist/bin.mjs", | |
| 156 | ), | |
| 157 | ); | |
| 158 | assert.isTrue( | |
| 159 | BootService.isEphemeralCacheEntry("/home/theo/.bun/install/cache/t3@0.0.27/dist/bin.mjs"), | |
| 160 | ); | |
| 161 | assert.isFalse(BootService.isEphemeralCacheEntry("/usr/local/lib/node_modules/t3/dist/bin.mjs")); | |
| 162 | assert.isFalse( | |
| 163 | BootService.isEphemeralCacheEntry( | |
| 164 | "/home/theo/dev/pnpm/dlx-tools/t3/node_modules/t3/dist/bin.mjs", | |
| 165 | ), | |
| 166 | ); | |
| 167 | assert.isFalse( | |
| 168 | BootService.isEphemeralCacheEntry( | |
| 169 | "/home/theo/.t3/runtime/versions/0.0.27/node_modules/t3/dist/bin.mjs", | |
| 66 | const service = yield* BootService.make({ | |
| 67 | baseDir, | |
| 68 | logsDir: path.join(baseDir, "userdata", "logs"), | |
| 69 | cliVersion: "1.2.3", | |
| 70 | host: { | |
| 71 | execPath: "/usr/bin/node", | |
| 72 | cliEntryPath: path.join(home, "bin.mjs"), | |
| 73 | launcherSourcePath: sourceLauncher, | |
| 74 | }, | |
| 75 | }).pipe( | |
| 76 | Effect.provideService(ProcessRunner.ProcessRunner, runner), | |
| 77 | Effect.provide( | |
| 78 | Layer.mergeAll( | |
| 79 | Layer.succeed(HostProcessPlatform, platform), | |
| 80 | Layer.succeed(HostProcessExecutablePath, "/usr/bin/node"), | |
| 81 | Layer.succeed(HostProcessArguments, ["/usr/bin/node", path.join(home, "bin.mjs")]), | |
| 82 | ConfigProvider.layer(ConfigProvider.fromEnv({ env: { HOME: home } })), | |
| 83 | ), | |
| 170 | 84 | ), |
| 171 | 85 | ); |
| 86 | return { service, fs, statePath, commands, control }; | |
| 172 | 87 | }); |
| 173 | 88 | |
| 174 | it.layer(NodeServices.layer)("BootService", (it) => { | |
| 175 | it.effect("reconciles the standalone service once and is then idempotent", () => | |
| 176 | Effect.gen(function* () { | |
| 177 | const { dirs } = yield* makeTestContext(); | |
| 178 | const commands: Array<RecordedCommand> = []; | |
| 179 | const service = yield* BootService.make({ | |
| 180 | baseDir: dirs.baseDir, | |
| 181 | logsDir: dirs.logsDir, | |
| 182 | cliVersion: "0.0.27", | |
| 183 | host: makeHost(dirs.stableEntry), | |
| 184 | }).pipe(Effect.provide(makeRecordingRunnerLayer(commands)), provideHostRefs(dirs.home)); | |
| 185 | ||
| 186 | const first = yield* reconcileService().pipe( | |
| 187 | Effect.provideService(BootService.BootService, service), | |
| 188 | ); | |
| 189 | assert.isTrue(first.changed); | |
| 190 | if (!first.changed) return; | |
| 191 | assert.isFalse(first.previouslyInstalled); | |
| 192 | ||
| 193 | const commandCount = commands.length; | |
| 194 | const second = yield* reconcileService().pipe( | |
| 195 | Effect.provideService(BootService.BootService, service), | |
| 196 | ); | |
| 197 | assert.isFalse(second.changed); | |
| 198 | assert.lengthOf(commands, commandCount); | |
| 199 | }), | |
| 200 | ); | |
| 201 | ||
| 202 | it.effect("installs the unit, enables the service, and enables linger", () => | |
| 203 | Effect.gen(function* () { | |
| 204 | const { dirs, fs, path } = yield* makeTestContext(); | |
| 205 | const commands: Array<RecordedCommand> = []; | |
| 206 | const service = yield* BootService.make({ | |
| 207 | baseDir: dirs.baseDir, | |
| 208 | logsDir: dirs.logsDir, | |
| 209 | cliVersion: "0.0.27", | |
| 210 | host: makeHost(dirs.stableEntry), | |
| 211 | }).pipe(Effect.provide(makeRecordingRunnerLayer(commands)), provideHostRefs(dirs.home)); | |
| 212 | ||
| 213 | const plan = yield* service.install; | |
| 214 | ||
| 215 | // A stable entry point is reused directly — no npm install. | |
| 216 | assert.equal(plan.t3EntryPath, dirs.stableEntry); | |
| 217 | assert.deepEqual( | |
| 218 | commands.map((entry) => [entry.command, ...entry.args].join(" ")), | |
| 219 | [ | |
| 220 | "systemctl --user daemon-reload", | |
| 221 | "systemctl --user enable t3code.service", | |
| 222 | // restart (not enable --now) so repairing a stale unit replaces a | |
| 223 | // running process instead of leaving the old one until reboot. | |
| 224 | "systemctl --user restart t3code.service", | |
| 225 | "loginctl enable-linger", | |
| 226 | ], | |
| 227 | ); | |
| 228 | ||
| 229 | const unitPath = path.join(dirs.home, ".config", "systemd", "user", "t3code.service"); | |
| 230 | const unit = yield* fs.readFileString(unitPath); | |
| 231 | assert.include(unit, `ExecStart=/usr/local/bin/node ${dirs.stableEntry} serve`); | |
| 232 | assert.include(unit, `Environment=T3CODE_HOME=${dirs.baseDir}`); | |
| 233 | ||
| 234 | const status = yield* service.status; | |
| 235 | assert.isTrue(status.supported); | |
| 236 | assert.isTrue(status.installed); | |
| 237 | assert.isTrue(status.current); | |
| 238 | ||
| 239 | const removed = yield* service.uninstall; | |
| 240 | assert.isTrue(removed); | |
| 241 | assert.isFalse(yield* fs.exists(unitPath)); | |
| 242 | const statusAfter = yield* service.status; | |
| 243 | assert.isFalse(statusAfter.installed); | |
| 244 | const removedAgain = yield* service.uninstall; | |
| 245 | assert.isFalse(removedAgain); | |
| 246 | }), | |
| 247 | ); | |
| 248 | ||
| 249 | it.effect("pins a runtime via npm install when running from the npx cache", () => | |
| 89 | it.layer(NodeServices.layer)("boot service install", (it) => { | |
| 90 | it.effect("installs, reports current state, and uninstalls", () => | |
| 250 | 91 | Effect.gen(function* () { |
| 251 | const { dirs, fs, path } = yield* makeTestContext(); | |
| 252 | const commands: Array<RecordedCommand> = []; | |
| 253 | const service = yield* BootService.make({ | |
| 254 | baseDir: dirs.baseDir, | |
| 255 | logsDir: dirs.logsDir, | |
| 256 | cliVersion: "0.0.27", | |
| 257 | host: makeHost("/home/theo/.npm/_npx/abc/node_modules/t3/dist/bin.mjs"), | |
| 258 | }).pipe(Effect.provide(makeRecordingRunnerLayer(commands)), provideHostRefs(dirs.home)); | |
| 259 | ||
| 92 | const { service, fs, statePath, commands } = yield* makeHarness(); | |
| 260 | 93 | const plan = yield* service.install; |
| 261 | 94 | |
| 262 | const runtimeDir = path.join(dirs.baseDir, "runtime", "versions", "0.0.27"); | |
| 263 | assert.equal( | |
| 264 | plan.t3EntryPath, | |
| 265 | path.join(runtimeDir, "node_modules", "t3", "dist", "bin.mjs"), | |
| 266 | ); | |
| 267 | assert.deepEqual(commands[0], { | |
| 268 | command: "npm", | |
| 269 | args: ["install", "--prefix", runtimeDir, "--no-fund", "--no-audit", "t3@0.0.27"], | |
| 95 | expect(parseServiceState(yield* fs.readFileString(statePath))).toEqual({ | |
| 96 | protocol: 1, | |
| 97 | activeVersion: "1.2.3", | |
| 270 | 98 | }); |
| 271 | // Success is recorded via a sentinel so interrupted installs re-run. | |
| 272 | assert.isTrue(yield* fs.exists(path.join(runtimeDir, ".install-complete"))); | |
| 273 | }), | |
| 274 | ); | |
| 275 | ||
| 276 | it.effect("reinstalls a pinned runtime when its entry point is missing", () => | |
| 277 | Effect.gen(function* () { | |
| 278 | const { dirs, fs, path } = yield* makeTestContext(); | |
| 279 | const commands: Array<RecordedCommand> = []; | |
| 280 | const service = yield* BootService.make({ | |
| 281 | baseDir: dirs.baseDir, | |
| 282 | logsDir: dirs.logsDir, | |
| 283 | cliVersion: "0.0.27", | |
| 284 | host: makeHost("/home/theo/.npm/_npx/abc/node_modules/t3/dist/bin.mjs"), | |
| 285 | }).pipe(Effect.provide(makeRecordingRunnerLayer(commands)), provideHostRefs(dirs.home)); | |
| 286 | ||
| 287 | const plan = yield* service.install; | |
| 288 | yield* fs.makeDirectory(path.dirname(plan.t3EntryPath), { recursive: true }); | |
| 289 | yield* fs.writeFileString(plan.t3EntryPath, "#!/usr/bin/env node\n"); | |
| 290 | yield* fs.remove(plan.t3EntryPath); | |
| 291 | commands.length = 0; | |
| 292 | ||
| 293 | yield* service.install; | |
| 294 | ||
| 295 | assert.isTrue(commands.some(({ command }) => command === "npm")); | |
| 296 | }), | |
| 297 | ); | |
| 298 | ||
| 299 | it.effect("reads executable metadata from host process references", () => | |
| 300 | Effect.gen(function* () { | |
| 301 | const { dirs } = yield* makeTestContext(); | |
| 302 | const commands: Array<RecordedCommand> = []; | |
| 303 | const service = yield* BootService.make({ | |
| 304 | baseDir: dirs.baseDir, | |
| 305 | logsDir: dirs.logsDir, | |
| 306 | cliVersion: "0.0.27", | |
| 307 | }).pipe( | |
| 308 | Effect.provide(makeRecordingRunnerLayer(commands)), | |
| 309 | provideHostRefs(dirs.home), | |
| 310 | Effect.provideService(HostProcessExecutablePath, "/opt/node/bin/node"), | |
| 311 | Effect.provideService(HostProcessArguments, ["/opt/node/bin/node", dirs.stableEntry]), | |
| 312 | ); | |
| 313 | ||
| 314 | const plan = yield* service.install; | |
| 315 | assert.equal(plan.nodePath, "/opt/node/bin/node"); | |
| 316 | assert.equal(plan.t3EntryPath, dirs.stableEntry); | |
| 317 | }), | |
| 318 | ); | |
| 319 | ||
| 320 | it.effect("cleans up and fails when the pinned runtime install fails", () => | |
| 321 | Effect.gen(function* () { | |
| 322 | const { dirs, fs, path } = yield* makeTestContext(); | |
| 323 | const commands: Array<RecordedCommand> = []; | |
| 324 | const service = yield* BootService.make({ | |
| 325 | baseDir: dirs.baseDir, | |
| 326 | logsDir: dirs.logsDir, | |
| 327 | cliVersion: "0.0.27", | |
| 328 | host: makeHost("/home/theo/.npm/_npx/abc/node_modules/t3/dist/bin.mjs"), | |
| 329 | }).pipe( | |
| 330 | Effect.provide(makeRecordingRunnerLayer(commands, { failCommand: "npm" })), | |
| 331 | provideHostRefs(dirs.home), | |
| 332 | ); | |
| 333 | ||
| 334 | const error = yield* service.install.pipe(Effect.flip); | |
| 335 | assert.isTrue(isCommandError(error)); | |
| 336 | const runtimeDir = path.join(dirs.baseDir, "runtime", "versions", "0.0.27"); | |
| 337 | // The half-installed tree must not be reused by the next attempt. | |
| 338 | assert.isFalse(yield* fs.exists(runtimeDir)); | |
| 339 | assert.isFalse(yield* fs.exists(path.join(runtimeDir, ".install-complete"))); | |
| 340 | }), | |
| 341 | ); | |
| 342 | ||
| 343 | it.effect("reports an installed-but-stale unit so the lifecycle can offer a repair", () => | |
| 344 | Effect.gen(function* () { | |
| 345 | const { dirs, fs, path } = yield* makeTestContext(); | |
| 346 | const commands: Array<RecordedCommand> = []; | |
| 347 | const service = yield* BootService.make({ | |
| 348 | baseDir: dirs.baseDir, | |
| 349 | logsDir: dirs.logsDir, | |
| 350 | cliVersion: "0.0.27", | |
| 351 | host: makeHost(dirs.stableEntry), | |
| 352 | }).pipe(Effect.provide(makeRecordingRunnerLayer(commands)), provideHostRefs(dirs.home)); | |
| 353 | ||
| 354 | const unitDir = path.join(dirs.home, ".config", "systemd", "user"); | |
| 355 | yield* fs.makeDirectory(unitDir, { recursive: true }); | |
| 99 | expect(yield* fs.readFileString(plan.launcherPath)).toBe("export {};\n"); | |
| 100 | expect((yield* service.status).current).toBe(true); | |
| 356 | 101 | yield* fs.writeFileString( |
| 357 | path.join(unitDir, "t3code.service"), | |
| 358 | "[Service]\nExecStart=/old/node /old/t3 serve\n", | |
| 102 | statePath, | |
| 103 | '{"protocol":1,"activeVersion":"1.2.3","update":{"id":"u","fromVersion":"1.2.3","targetVersion":"1.2.4","status":"pending"}}', | |
| 359 | 104 | ); |
| 360 | ||
| 361 | const status = yield* service.status; | |
| 362 | assert.isTrue(status.supported); | |
| 363 | assert.isTrue(status.installed); | |
| 364 | assert.isFalse(status.current); | |
| 105 | expect((yield* service.status).current).toBe(false); | |
| 106 | expect(yield* service.uninstall).toBe(true); | |
| 107 | expect((yield* service.status).installed).toBe(false); | |
| 108 | expect(commands.some((command) => command.startsWith("npm "))).toBe(false); | |
| 365 | 109 | }), |
| 366 | 110 | ); |
| 367 | 111 | |
| 368 | it.effect("reports a current unit as stale when its entry point is gone", () => | |
| 112 | it.effect("restarts an installed service when repair fails", () => | |
| 369 | 113 | Effect.gen(function* () { |
| 370 | const { dirs, fs } = yield* makeTestContext(); | |
| 371 | const commands: Array<RecordedCommand> = []; | |
| 372 | const service = yield* BootService.make({ | |
| 373 | baseDir: dirs.baseDir, | |
| 374 | logsDir: dirs.logsDir, | |
| 375 | cliVersion: "0.0.27", | |
| 376 | host: makeHost(dirs.stableEntry), | |
| 377 | }).pipe(Effect.provide(makeRecordingRunnerLayer(commands)), provideHostRefs(dirs.home)); | |
| 378 | ||
| 114 | const { service, commands, control } = yield* makeHarness(); | |
| 379 | 115 | yield* service.install; |
| 380 | assert.isTrue((yield* service.status).current); | |
| 381 | ||
| 382 | // The pinned runtime (or global bin) was deleted to reclaim space; the | |
| 383 | // unit still matches byte-for-byte but would crashloop at boot. | |
| 384 | yield* fs.remove(dirs.stableEntry); | |
| 385 | const status = yield* service.status; | |
| 386 | assert.isTrue(status.installed); | |
| 387 | assert.isFalse(status.current); | |
| 388 | }), | |
| 389 | ); | |
| 390 | ||
| 391 | it.effect("fails on non-Linux platforms without touching the filesystem", () => | |
| 392 | Effect.gen(function* () { | |
| 393 | const { dirs, fs, path } = yield* makeTestContext(); | |
| 394 | const commands: Array<RecordedCommand> = []; | |
| 395 | const service = yield* BootService.make({ | |
| 396 | baseDir: dirs.baseDir, | |
| 397 | logsDir: dirs.logsDir, | |
| 398 | cliVersion: "0.0.27", | |
| 399 | host: makeHost("/usr/local/lib/node_modules/t3/dist/bin.mjs"), | |
| 400 | }).pipe( | |
| 401 | Effect.provide(makeRecordingRunnerLayer(commands)), | |
| 402 | provideHostRefs(dirs.home, "darwin"), | |
| 403 | ); | |
| 404 | ||
| 405 | const error = yield* service.install.pipe(Effect.flip); | |
| 406 | assert.isTrue(isUnsupportedError(error)); | |
| 407 | assert.lengthOf(commands, 0); | |
| 408 | assert.isFalse( | |
| 409 | yield* fs.exists(path.join(dirs.home, ".config", "systemd", "user", "t3code.service")), | |
| 410 | ); | |
| 411 | ||
| 412 | const status = yield* service.status; | |
| 413 | assert.isFalse(status.supported); | |
| 414 | assert.isFalse(status.installed); | |
| 415 | }), | |
| 416 | ); | |
| 417 | ||
| 418 | it.effect("removes the unit file when an activation step fails", () => | |
| 419 | Effect.gen(function* () { | |
| 420 | const { dirs, fs, path } = yield* makeTestContext(); | |
| 421 | const commands: Array<RecordedCommand> = []; | |
| 422 | const service = yield* BootService.make({ | |
| 423 | baseDir: dirs.baseDir, | |
| 424 | logsDir: dirs.logsDir, | |
| 425 | cliVersion: "0.0.27", | |
| 426 | host: makeHost("/usr/local/lib/node_modules/t3/dist/bin.mjs"), | |
| 427 | }).pipe( | |
| 428 | Effect.provide(makeRecordingRunnerLayer(commands, { failCommand: "loginctl" })), | |
| 429 | provideHostRefs(dirs.home), | |
| 430 | ); | |
| 116 | commands.length = 0; | |
| 117 | control.failCommand = "systemctl --user daemon-reload"; | |
| 431 | 118 | |
| 432 | 119 | const error = yield* service.install.pipe(Effect.flip); |
| 433 | assert.isTrue(isCommandError(error)); | |
| 434 | // A leftover unit would make status report "installed" even though | |
| 435 | // linger never happened. | |
| 436 | assert.isFalse( | |
| 437 | yield* fs.exists(path.join(dirs.home, ".config", "systemd", "user", "t3code.service")), | |
| 438 | ); | |
| 439 | const status = yield* service.status; | |
| 440 | assert.isFalse(status.installed); | |
| 441 | assert.isTrue( | |
| 442 | commands.some( | |
| 443 | ({ command, args }) => | |
| 444 | command === "systemctl" && args.join(" ") === "--user disable --now t3code.service", | |
| 445 | ), | |
| 446 | ); | |
| 120 | expect(error._tag).toBe("BootServiceCommandError"); | |
| 121 | expect(commands.filter((command) => command.startsWith("systemctl "))).toEqual([ | |
| 122 | "systemctl --user stop t3code.service", | |
| 123 | "systemctl --user daemon-reload", | |
| 124 | "systemctl --user restart t3code.service", | |
| 125 | ]); | |
| 447 | 126 | }), |
| 448 | 127 | ); |
| 449 | 128 | |
| 450 | it.effect("restores the previous unit when a repair cannot activate", () => | |
| 129 | it.effect("fails closed off Linux", () => | |
| 451 | 130 | Effect.gen(function* () { |
| 452 | const { dirs, fs, path } = yield* makeTestContext(); | |
| 453 | const initialCommands: Array<RecordedCommand> = []; | |
| 454 | const initialService = yield* BootService.make({ | |
| 455 | baseDir: dirs.baseDir, | |
| 456 | logsDir: dirs.logsDir, | |
| 457 | cliVersion: "0.0.27", | |
| 458 | host: makeHost(dirs.stableEntry), | |
| 459 | }).pipe( | |
| 460 | Effect.provide(makeRecordingRunnerLayer(initialCommands)), | |
| 461 | provideHostRefs(dirs.home), | |
| 462 | ); | |
| 463 | yield* initialService.install; | |
| 464 | ||
| 465 | const unitPath = path.join(dirs.home, ".config", "systemd", "user", "t3code.service"); | |
| 466 | const previousUnit = yield* fs.readFileString(unitPath); | |
| 467 | const replacementEntry = path.join(dirs.home, "replacement-bin.mjs"); | |
| 468 | yield* fs.writeFileString(replacementEntry, "#!/usr/bin/env node\n"); | |
| 469 | const repairCommands: Array<RecordedCommand> = []; | |
| 470 | const repairService = yield* BootService.make({ | |
| 471 | baseDir: dirs.baseDir, | |
| 472 | logsDir: dirs.logsDir, | |
| 473 | cliVersion: "0.0.28", | |
| 474 | host: makeHost(replacementEntry), | |
| 475 | }).pipe( | |
| 476 | Effect.provide(makeRecordingRunnerLayer(repairCommands, { failCommand: "loginctl" })), | |
| 477 | provideHostRefs(dirs.home), | |
| 478 | ); | |
| 479 | ||
| 480 | const error = yield* repairService.install.pipe(Effect.flip); | |
| 481 | ||
| 482 | assert.isTrue(isCommandError(error)); | |
| 483 | assert.equal(yield* fs.readFileString(unitPath), previousUnit); | |
| 484 | assert.isTrue( | |
| 485 | repairCommands.some( | |
| 486 | ({ command, args }) => | |
| 487 | command === "systemctl" && args.join(" ") === "--user restart t3code.service", | |
| 488 | ), | |
| 489 | ); | |
| 490 | }), | |
| 491 | ); | |
| 492 | ||
| 493 | it.effect("keeps the unit when stopping it during uninstall fails", () => | |
| 494 | Effect.gen(function* () { | |
| 495 | const { dirs, fs, path } = yield* makeTestContext(); | |
| 496 | const installCommands: Array<RecordedCommand> = []; | |
| 497 | const installedService = yield* BootService.make({ | |
| 498 | baseDir: dirs.baseDir, | |
| 499 | logsDir: dirs.logsDir, | |
| 500 | cliVersion: "0.0.27", | |
| 501 | host: makeHost(dirs.stableEntry), | |
| 502 | }).pipe( | |
| 503 | Effect.provide(makeRecordingRunnerLayer(installCommands)), | |
| 504 | provideHostRefs(dirs.home), | |
| 505 | ); | |
| 506 | yield* installedService.install; | |
| 507 | ||
| 508 | const uninstallCommands: Array<RecordedCommand> = []; | |
| 509 | const failingService = yield* BootService.make({ | |
| 510 | baseDir: dirs.baseDir, | |
| 511 | logsDir: dirs.logsDir, | |
| 512 | cliVersion: "0.0.27", | |
| 513 | host: makeHost(dirs.stableEntry), | |
| 514 | }).pipe( | |
| 515 | Effect.provide( | |
| 516 | makeRecordingRunnerLayer(uninstallCommands, { | |
| 517 | failWhen: (command, args) => | |
| 518 | command === "systemctl" && args.includes("disable") && args.includes("--now"), | |
| 519 | }), | |
| 520 | ), | |
| 521 | provideHostRefs(dirs.home), | |
| 522 | ); | |
| 523 | ||
| 524 | const error = yield* failingService.uninstall.pipe(Effect.flip); | |
| 525 | ||
| 526 | assert.isTrue(isCommandError(error)); | |
| 527 | assert.isTrue( | |
| 528 | yield* fs.exists(path.join(dirs.home, ".config", "systemd", "user", "t3code.service")), | |
| 529 | ); | |
| 530 | }), | |
| 531 | ); | |
| 532 | ||
| 533 | it.effect("appends failed steps to the boot-service log", () => | |
| 534 | Effect.gen(function* () { | |
| 535 | const { dirs, fs, path } = yield* makeTestContext(); | |
| 536 | const commands: Array<RecordedCommand> = []; | |
| 537 | const service = yield* BootService.make({ | |
| 538 | baseDir: dirs.baseDir, | |
| 539 | logsDir: dirs.logsDir, | |
| 540 | cliVersion: "0.0.27", | |
| 541 | host: makeHost("/usr/local/lib/node_modules/t3/dist/bin.mjs"), | |
| 542 | }).pipe( | |
| 543 | Effect.provide(makeRecordingRunnerLayer(commands, { failCommand: "systemctl" })), | |
| 544 | provideHostRefs(dirs.home), | |
| 545 | ); | |
| 546 | ||
| 547 | const error = yield* service.install.pipe(Effect.flip); | |
| 548 | assert.isTrue(isCommandError(error)); | |
| 549 | if (!isCommandError(error)) return; | |
| 550 | assert.equal(error.exitCode, 1); | |
| 551 | assert.equal(error.stderrLength, "systemctl exploded".length); | |
| 552 | ||
| 553 | const logPath = path.join(dirs.logsDir, "boot-service.log"); | |
| 554 | assert.isTrue(yield* fs.exists(logPath)); | |
| 555 | assert.include(yield* fs.readFileString(logPath), "exit code 1"); | |
| 131 | const { service } = yield* makeHarness("darwin"); | |
| 132 | expect((yield* service.status).supported).toBe(false); | |
| 133 | expect((yield* service.install.pipe(Effect.flip))._tag).toBe("BootServiceUnsupportedError"); | |
| 556 | 134 | }), |
| 557 | 135 | ); |
| 558 | 136 | }); |
apps/server/src/cloud/bootService.ts+163-188
| ... | ... | @@ -1,5 +1,10 @@ |
| 1 | import * as Context from "effect/Context"; | |
| 1 | import { | |
| 2 | HostProcessArguments, | |
| 3 | HostProcessExecutablePath, | |
| 4 | HostProcessPlatform, | |
| 5 | } from "@t3tools/shared/hostProcess"; | |
| 2 | 6 | import * as Config from "effect/Config"; |
| 7 | import * as Context from "effect/Context"; | |
| 3 | 8 | import * as DateTime from "effect/DateTime"; |
| 4 | 9 | import * as Duration from "effect/Duration"; |
| 5 | 10 | import * as Effect from "effect/Effect"; |
| ... | ... | @@ -9,58 +14,29 @@ import * as Option from "effect/Option"; |
| 9 | 14 | import * as Path from "effect/Path"; |
| 10 | 15 | import * as Schema from "effect/Schema"; |
| 11 | 16 | |
| 12 | import { | |
| 13 | HostProcessArguments, | |
| 14 | HostProcessExecutablePath, | |
| 15 | HostProcessPlatform, | |
| 16 | } from "@t3tools/shared/hostProcess"; | |
| 17 | ||
| 18 | 17 | import * as ProcessRunner from "../processRunner.ts"; |
| 19 | import { ensurePinnedRuntimeInstalled, pinnedRuntimePaths } from "./pinnedRuntime.ts"; | |
| 20 | ||
| 21 | /** | |
| 22 | * Installs T3 Code as a per-user boot service. Linux-only for now: systemd | |
| 23 | * user unit + loginctl enable-linger. The service runs a stable or pinned | |
| 24 | * runtime — never an ephemeral `npx t3` cache whose eviction could break | |
| 25 | * startup. | |
| 26 | */ | |
| 18 | import { | |
| 19 | ensurePinnedRuntimeInstalled, | |
| 20 | pinnedRuntimePaths, | |
| 21 | PinnedRuntimeInstallError, | |
| 22 | } from "./pinnedRuntime.ts"; | |
| 23 | import { | |
| 24 | SERVICE_LAUNCHER_FILE, | |
| 25 | SERVICE_LAUNCHER_PROTOCOL, | |
| 26 | SERVICE_STATE_FILE, | |
| 27 | parseServiceState, | |
| 28 | type ServiceState, | |
| 29 | } from "./serviceProtocol.ts"; | |
| 27 | 30 | |
| 28 | 31 | const BOOT_SERVICE_NAME = "t3code"; |
| 29 | ||
| 30 | 32 | export const BOOT_SERVICE_UNIT_FILE = `${BOOT_SERVICE_NAME}.service`; |
| 31 | 33 | export const BOOT_SERVICE_UNIT_ENV = "T3_BOOT_SERVICE_UNIT"; |
| 32 | 34 | |
| 33 | const EPHEMERAL_CACHE_SEGMENTS = [ | |
| 34 | "/_npx/", // npx | |
| 35 | "\\_npx\\", | |
| 36 | "/pnpm/dlx/", // pnpm dlx (~/.cache/pnpm/dlx and $PNPM_HOME/.pnpm/dlx) | |
| 37 | "/.pnpm/dlx/", | |
| 38 | "/.bun/install/cache/", // bunx | |
| 39 | ]; | |
| 40 | ||
| 41 | /** | |
| 42 | * `npx t3` (and pnpm dlx / bunx) run out of ephemeral package-manager | |
| 43 | * caches that can be evicted at any time — a boot service must never point | |
| 44 | * there. Global installs, repo checkouts, and the pinned runtime below are | |
| 45 | * all stable. | |
| 46 | */ | |
| 47 | export function isEphemeralCacheEntry(entryPath: string): boolean { | |
| 48 | return EPHEMERAL_CACHE_SEGMENTS.some((segment) => entryPath.includes(segment)); | |
| 49 | } | |
| 50 | ||
| 51 | /** | |
| 52 | * systemd expands `%` specifiers in most directive values, including the | |
| 53 | * `append:` file paths, which take the rest of the line literally and must | |
| 54 | * NOT be quoted. | |
| 55 | */ | |
| 35 | /** systemd expands `%` specifiers, including in unquoted append-log paths. */ | |
| 56 | 36 | export function escapeSystemdSpecifiers(value: string): string { |
| 57 | 37 | return value.replaceAll("%", "%%"); |
| 58 | 38 | } |
| 59 | 39 | |
| 60 | /** | |
| 61 | * systemd word-splits ExecStart and Environment values and expands `%` | |
| 62 | * specifiers, so paths with spaces or percents must be quoted and escaped. | |
| 63 | */ | |
| 64 | 40 | export function quoteSystemdValue(value: string): string { |
| 65 | 41 | const escaped = escapeSystemdSpecifiers(value); |
| 66 | 42 | return /[\s"'\\]/.test(escaped) |
| ... | ... | @@ -69,31 +45,19 @@ export function quoteSystemdValue(value: string): string { |
| 69 | 45 | } |
| 70 | 46 | |
| 71 | 47 | export interface BootServicePlan { |
| 72 | /** Absolute path of the node binary running this CLI. */ | |
| 73 | 48 | readonly nodePath: string; |
| 74 | /** Absolute path of the pinned t3 entry point the unit will run. */ | |
| 75 | readonly t3EntryPath: string; | |
| 49 | readonly launcherPath: string; | |
| 76 | 50 | readonly baseDir: string; |
| 77 | 51 | readonly logPath: string; |
| 78 | 52 | readonly unitPath: string; |
| 79 | 53 | } |
| 80 | 54 | |
| 81 | /** | |
| 82 | * Pure so it is testable byte-for-byte. systemd user units run with a | |
| 83 | * minimal environment: every path must be absolute, and the service must | |
| 84 | * not rely on PATH, nvm shims, or shell profiles. Failures land in | |
| 85 | * `logPath` because `systemctl --user` failures are otherwise invisible. | |
| 86 | */ | |
| 55 | /** Pure renderer: service units cannot rely on the user's shell or PATH. */ | |
| 87 | 56 | export function renderBootServiceUnit(plan: BootServicePlan): string { |
| 88 | // No After=network-online.target: it does not exist in the systemd *user* | |
| 89 | // manager, so ordering on it is silently ignored. The server retries its | |
| 90 | // relay connection, and Restart=always covers early-boot failures. | |
| 57 | // The user manager has no reliable network-online target; server networking retries itself. | |
| 91 | 58 | return [ |
| 92 | 59 | "[Unit]", |
| 93 | 60 | "Description=T3 Code server", |
| 94 | // Give up after 5 crashes in 5 minutes so a persistently broken install | |
| 95 | // (deleted runtime, broken workspace) stops instead of restarting every | |
| 96 | // 5s forever and growing the unrotated append log without bound. | |
| 97 | 61 | "StartLimitIntervalSec=300", |
| 98 | 62 | "StartLimitBurst=5", |
| 99 | 63 | "", |
| ... | ... | @@ -102,7 +66,8 @@ export function renderBootServiceUnit(plan: BootServicePlan): string { |
| 102 | 66 | "WorkingDirectory=%h", |
| 103 | 67 | `Environment=T3CODE_HOME=${quoteSystemdValue(plan.baseDir)}`, |
| 104 | 68 | `Environment=${BOOT_SERVICE_UNIT_ENV}=${BOOT_SERVICE_UNIT_FILE}`, |
| 105 | `ExecStart=${quoteSystemdValue(plan.nodePath)} ${quoteSystemdValue(plan.t3EntryPath)} serve`, | |
| 69 | `ExecStart=${quoteSystemdValue(plan.nodePath)} ${quoteSystemdValue(plan.launcherPath)}`, | |
| 70 | "KillMode=control-group", | |
| 106 | 71 | "Restart=always", |
| 107 | 72 | "RestartSec=5", |
| 108 | 73 | `StandardOutput=append:${escapeSystemdSpecifiers(plan.logPath)}`, |
| ... | ... | @@ -157,7 +122,6 @@ export type BootServiceError = |
| 157 | 122 | export interface BootServiceStatus { |
| 158 | 123 | readonly supported: boolean; |
| 159 | 124 | readonly installed: boolean; |
| 160 | /** False when the installed unit no longer matches what install would write. */ | |
| 161 | 125 | readonly current: boolean; |
| 162 | 126 | readonly unitPath: string; |
| 163 | 127 | readonly logPath: string; |
| ... | ... | @@ -166,12 +130,7 @@ export interface BootServiceStatus { |
| 166 | 130 | export class BootService extends Context.Service< |
| 167 | 131 | BootService, |
| 168 | 132 | { |
| 169 | /** Installs the pinned runtime + unit, enables linger, starts the service. */ | |
| 170 | 133 | readonly install: Effect.Effect<BootServicePlan, BootServiceError>; |
| 171 | /** | |
| 172 | * Stops and removes the unit; leaves the pinned runtime for reuse. | |
| 173 | * Returns whether a unit was actually removed. | |
| 174 | */ | |
| 175 | 134 | readonly uninstall: Effect.Effect<boolean, BootServiceError>; |
| 176 | 135 | readonly status: Effect.Effect<BootServiceStatus, BootServiceError>; |
| 177 | 136 | } |
| ... | ... | @@ -180,6 +139,7 @@ export class BootService extends Context.Service< |
| 180 | 139 | export interface BootServiceHost { |
| 181 | 140 | readonly execPath: string; |
| 182 | 141 | readonly cliEntryPath: string; |
| 142 | readonly launcherSourcePath?: string; | |
| 183 | 143 | } |
| 184 | 144 | |
| 185 | 145 | export const make = Effect.fn("cloud.boot_service.make")(function* (input: { |
| ... | ... | @@ -188,24 +148,45 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { |
| 188 | 148 | readonly cliVersion: string; |
| 189 | 149 | readonly host?: BootServiceHost; |
| 190 | 150 | }) { |
| 191 | const hostExecPath = yield* HostProcessExecutablePath; | |
| 192 | 151 | const hostArguments = yield* HostProcessArguments; |
| 193 | const host = input.host ?? { | |
| 194 | execPath: hostExecPath, | |
| 195 | // When running the packed CLI this is dist/bin.mjs; when stable (global | |
| 196 | // install, repo checkout) the boot service runs this same artifact. | |
| 197 | cliEntryPath: hostArguments[1] ?? "", | |
| 198 | }; | |
| 152 | const hostExecPath = yield* HostProcessExecutablePath; | |
| 199 | 153 | const platform = yield* HostProcessPlatform; |
| 200 | 154 | const homeDir = yield* Config.string("HOME").pipe(Config.withDefault("")); |
| 201 | 155 | const fs = yield* FileSystem.FileSystem; |
| 202 | 156 | const path = yield* Path.Path; |
| 203 | 157 | const runner = yield* ProcessRunner.ProcessRunner; |
| 158 | const host = input.host ?? { | |
| 159 | execPath: hostExecPath, | |
| 160 | cliEntryPath: hostArguments[1] ?? "", | |
| 161 | }; | |
| 204 | 162 | |
| 205 | 163 | const unitDir = path.join(homeDir, ".config", "systemd", "user"); |
| 206 | 164 | const unitPath = path.join(unitDir, BOOT_SERVICE_UNIT_FILE); |
| 207 | 165 | const logPath = path.join(input.logsDir, "boot-service.log"); |
| 166 | const launcherPath = path.join(input.baseDir, "runtime", SERVICE_LAUNCHER_FILE); | |
| 167 | const statePath = path.join(input.baseDir, "runtime", SERVICE_STATE_FILE); | |
| 168 | const launcherSourcePath = | |
| 169 | host.launcherSourcePath ?? path.join(path.dirname(host.cliEntryPath), SERVICE_LAUNCHER_FILE); | |
| 208 | 170 | const runtimePaths = pinnedRuntimePaths(path, input.baseDir, input.cliVersion); |
| 171 | const writeDurably = (filePath: string, contents: string) => | |
| 172 | Effect.scoped( | |
| 173 | Effect.gen(function* () { | |
| 174 | const directory = path.dirname(filePath); | |
| 175 | yield* fs.makeDirectory(directory, { recursive: true }); | |
| 176 | const tempPath = yield* fs.makeTempFileScoped({ directory, prefix: ".service-write-" }); | |
| 177 | yield* fs.writeFileString(tempPath, contents, { mode: 0o600 }); | |
| 178 | yield* (yield* fs.open(tempPath, { flag: "r" })).sync; | |
| 179 | yield* fs.rename(tempPath, filePath); | |
| 180 | yield* (yield* fs.open(directory, { flag: "r" })).sync; | |
| 181 | }), | |
| 182 | ).pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); | |
| 183 | const plan: BootServicePlan = { | |
| 184 | nodePath: host.execPath, | |
| 185 | launcherPath, | |
| 186 | baseDir: input.baseDir, | |
| 187 | logPath, | |
| 188 | unitPath, | |
| 189 | }; | |
| 209 | 190 | |
| 210 | 191 | const requireSystemdLinux = Effect.gen(function* () { |
| 211 | 192 | if (platform !== "linux" || homeDir === "") { |
| ... | ... | @@ -244,150 +225,130 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { |
| 244 | 225 | ); |
| 245 | 226 | }); |
| 246 | 227 | |
| 247 | /** | |
| 248 | * Ensures plannedEntryPath exists before the unit points at it. A stable | |
| 249 | * install (global bin, repo checkout) is used as-is; an ephemeral cache | |
| 250 | * entry is replaced by `npm install --prefix`-ing the exact running | |
| 251 | * version into <baseDir>/runtime/versions/<v>. A real install (not a copy | |
| 252 | * of bin.mjs) because t3 ships native deps like node-pty. | |
| 253 | */ | |
| 254 | const ensurePinnedRuntime = Effect.gen(function* () { | |
| 255 | if (!isEphemeralCacheEntry(host.cliEntryPath)) { | |
| 256 | return; | |
| 257 | } | |
| 228 | const install: BootService["Service"]["install"] = Effect.gen(function* () { | |
| 229 | yield* requireSystemdLinux; | |
| 230 | yield* fs | |
| 231 | .makeDirectory(input.logsDir, { recursive: true }) | |
| 232 | .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); | |
| 233 | ||
| 234 | // Prepare every immutable artifact before stopping the installed unit. | |
| 258 | 235 | yield* ensurePinnedRuntimeInstalled({ |
| 259 | 236 | baseDir: input.baseDir, |
| 260 | 237 | version: input.cliVersion, |
| 261 | 238 | fs, |
| 262 | 239 | path, |
| 263 | 240 | runner, |
| 241 | validate: (runtime) => | |
| 242 | runner | |
| 243 | .run({ | |
| 244 | command: host.execPath, | |
| 245 | args: [runtime.entryPath, "--version"], | |
| 246 | timeout: Duration.seconds(30), | |
| 247 | }) | |
| 248 | .pipe( | |
| 249 | Effect.mapError( | |
| 250 | (cause) => | |
| 251 | new PinnedRuntimeInstallError({ | |
| 252 | step: "verifying the pinned t3 runtime", | |
| 253 | cause, | |
| 254 | }), | |
| 255 | ), | |
| 256 | Effect.flatMap((result) => { | |
| 257 | const reportedVersion = /\bv(\S+)\s*$/.exec(result.stdout)?.[1]; | |
| 258 | return result.code === 0 && reportedVersion === input.cliVersion | |
| 259 | ? Effect.void | |
| 260 | : Effect.fail( | |
| 261 | new PinnedRuntimeInstallError({ | |
| 262 | step: "verifying the pinned t3 runtime", | |
| 263 | exitCode: Number(result.code), | |
| 264 | stdoutLength: result.stdout.length, | |
| 265 | stderrLength: result.stderr.length, | |
| 266 | }), | |
| 267 | ); | |
| 268 | }), | |
| 269 | ), | |
| 264 | 270 | }).pipe( |
| 265 | 271 | Effect.mapError((error) => |
| 266 | error.step.startsWith("installing") | |
| 272 | error._tag === "PinnedRuntimeInstallError" | |
| 267 | 273 | ? new BootServiceCommandError({ |
| 268 | 274 | step: error.step, |
| 269 | 275 | exitCode: error.exitCode, |
| 270 | 276 | stdoutLength: error.stdoutLength, |
| 271 | 277 | stderrLength: error.stderrLength, |
| 272 | cause: error.cause, | |
| 278 | cause: error, | |
| 273 | 279 | }) |
| 274 | 280 | : new BootServiceInstallError({ cause: error }), |
| 275 | 281 | ), |
| 276 | Effect.tapError((error) => | |
| 277 | DateTime.now.pipe( | |
| 278 | Effect.flatMap((now) => | |
| 279 | fs.writeFileString(logPath, `${DateTime.formatIso(now)} ${error.message}\n`, { | |
| 280 | flag: "a", | |
| 281 | }), | |
| 282 | ), | |
| 283 | Effect.ignore, | |
| 284 | ), | |
| 285 | ), | |
| 286 | 282 | ); |
| 287 | }); | |
| 288 | ||
| 289 | // Where the unit will point: derivable without touching the network, so | |
| 290 | // status can compare units purely; install materializes it first. | |
| 291 | const plannedEntryPath = isEphemeralCacheEntry(host.cliEntryPath) | |
| 292 | ? runtimePaths.entryPath | |
| 293 | : host.cliEntryPath; | |
| 294 | const plan: BootServicePlan = { | |
| 295 | nodePath: host.execPath, | |
| 296 | t3EntryPath: plannedEntryPath, | |
| 297 | baseDir: input.baseDir, | |
| 298 | logPath, | |
| 299 | unitPath, | |
| 300 | }; | |
| 301 | ||
| 302 | const install: BootService["Service"]["install"] = Effect.gen(function* () { | |
| 303 | yield* requireSystemdLinux; | |
| 304 | yield* fs | |
| 305 | .makeDirectory(input.logsDir, { recursive: true }) | |
| 283 | const launcherSource = yield* fs | |
| 284 | .readFileString(launcherSourcePath) | |
| 306 | 285 | .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); |
| 307 | 286 | |
| 308 | yield* ensurePinnedRuntime; | |
| 309 | ||
| 310 | const previousUnit = yield* fs.exists(unitPath).pipe( | |
| 311 | Effect.flatMap((exists) => | |
| 312 | exists | |
| 313 | ? fs.readFileString(unitPath).pipe(Effect.map(Option.some)) | |
| 314 | : Effect.succeed(Option.none<string>()), | |
| 315 | ), | |
| 316 | Effect.mapError((cause) => new BootServiceInstallError({ cause })), | |
| 317 | ); | |
| 318 | ||
| 319 | yield* fs.makeDirectory(unitDir, { recursive: true }).pipe( | |
| 320 | Effect.andThen(fs.writeFileString(unitPath, renderBootServiceUnit(plan))), | |
| 321 | Effect.mapError((cause) => new BootServiceInstallError({ cause })), | |
| 322 | ); | |
| 287 | const installed = yield* fs | |
| 288 | .exists(unitPath) | |
| 289 | .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); | |
| 290 | if (installed) { | |
| 291 | yield* runStep("stopping the installed service", "systemctl", [ | |
| 292 | "--user", | |
| 293 | "stop", | |
| 294 | BOOT_SERVICE_UNIT_FILE, | |
| 295 | ]); | |
| 296 | } | |
| 323 | 297 | |
| 324 | // If any activation step fails, remove the unit again: a leftover file | |
| 325 | // would make service status report it as installed even though it was | |
| 326 | // never enabled or lingered. | |
| 327 | 298 | yield* Effect.gen(function* () { |
| 299 | yield* fs | |
| 300 | .makeDirectory(unitDir, { recursive: true }) | |
| 301 | .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); | |
| 302 | yield* writeDurably(launcherPath, launcherSource); | |
| 303 | yield* writeDurably( | |
| 304 | statePath, | |
| 305 | // @effect-diagnostics-next-line preferSchemaOverJson:off - fixed launcher-owned document. | |
| 306 | `${JSON.stringify( | |
| 307 | { | |
| 308 | protocol: SERVICE_LAUNCHER_PROTOCOL, | |
| 309 | activeVersion: input.cliVersion, | |
| 310 | } satisfies ServiceState, | |
| 311 | null, | |
| 312 | 2, | |
| 313 | )}\n`, | |
| 314 | ); | |
| 315 | yield* writeDurably(unitPath, renderBootServiceUnit(plan)); | |
| 316 | ||
| 328 | 317 | yield* runStep("reloading systemd user units", "systemctl", ["--user", "daemon-reload"]); |
| 329 | 318 | yield* runStep("enabling the service", "systemctl", [ |
| 330 | 319 | "--user", |
| 331 | 320 | "enable", |
| 332 | 321 | BOOT_SERVICE_UNIT_FILE, |
| 333 | 322 | ]); |
| 334 | // restart rather than enable --now: --now does not replace an already | |
| 335 | // running process, so repairing a stale unit would leave the old | |
| 336 | // server running until reboot. restart also starts a stopped service. | |
| 323 | yield* runStep("enabling lingering for this user", "loginctl", ["enable-linger"]); | |
| 324 | // Start last. No administrative state write occurs after this succeeds. | |
| 337 | 325 | yield* runStep("starting the service", "systemctl", [ |
| 338 | 326 | "--user", |
| 339 | 327 | "restart", |
| 340 | 328 | BOOT_SERVICE_UNIT_FILE, |
| 341 | 329 | ]); |
| 342 | // Linger keeps the user manager (and this service) running without an | |
| 343 | // open session — the whole point on a box reached over SSH. No | |
| 344 | // username argument: loginctl defaults to the calling user, which is | |
| 345 | // always right, while $USER can be stale (su without -l) or unset. | |
| 346 | yield* runStep("enabling lingering for this user", "loginctl", ["enable-linger"]); | |
| 347 | }).pipe(Effect.tapError(() => rollbackFailedInstall(previousUnit))); | |
| 348 | ||
| 330 | }).pipe( | |
| 331 | Effect.tapError(() => | |
| 332 | installed | |
| 333 | ? runStep("restarting the service after a failed update", "systemctl", [ | |
| 334 | "--user", | |
| 335 | "restart", | |
| 336 | BOOT_SERVICE_UNIT_FILE, | |
| 337 | ]).pipe(Effect.ignore) | |
| 338 | : Effect.void, | |
| 339 | ), | |
| 340 | ); | |
| 349 | 341 | return plan; |
| 350 | 342 | }).pipe(Effect.withSpan("cloud.boot_service.install")); |
| 351 | 343 | |
| 352 | // If activation fails partway (e.g. enable succeeds but restart/linger | |
| 353 | // fails), leave nothing behind: disable removes the enable symlink, remove | |
| 354 | // deletes the file, daemon-reload clears the stale definition — otherwise a | |
| 355 | // dangling wants/ symlink logs "Failed to load unit" at every boot and the | |
| 356 | // next lifecycle command misreports the state. | |
| 357 | const rollbackFailedInstall = Effect.fn("cloud.boot_service.rollback_failed_install")(function* ( | |
| 358 | previousUnit: Option.Option<string>, | |
| 359 | ) { | |
| 360 | if (Option.isSome(previousUnit)) { | |
| 361 | yield* fs.writeFileString(unitPath, previousUnit.value).pipe(Effect.ignore); | |
| 362 | } else { | |
| 363 | yield* runStep("cleaning up the service", "systemctl", [ | |
| 364 | "--user", | |
| 365 | "disable", | |
| 366 | "--now", | |
| 367 | BOOT_SERVICE_UNIT_FILE, | |
| 368 | ]).pipe(Effect.ignore); | |
| 369 | yield* fs.remove(unitPath).pipe(Effect.ignore); | |
| 370 | } | |
| 371 | yield* runStep("reloading systemd user units", "systemctl", ["--user", "daemon-reload"]).pipe( | |
| 372 | Effect.ignore, | |
| 373 | ); | |
| 374 | if (Option.isSome(previousUnit)) { | |
| 375 | yield* runStep("restoring the previous service", "systemctl", [ | |
| 376 | "--user", | |
| 377 | "restart", | |
| 378 | BOOT_SERVICE_UNIT_FILE, | |
| 379 | ]).pipe(Effect.ignore); | |
| 380 | } | |
| 381 | }); | |
| 382 | ||
| 383 | 344 | const uninstall: BootService["Service"]["uninstall"] = Effect.gen(function* () { |
| 384 | 345 | yield* requireSystemdLinux; |
| 385 | const exists = yield* fs | |
| 386 | .exists(unitPath) | |
| 387 | .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); | |
| 388 | if (!exists) { | |
| 346 | if ( | |
| 347 | !(yield* fs | |
| 348 | .exists(unitPath) | |
| 349 | .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause })))) | |
| 350 | ) | |
| 389 | 351 | return false; |
| 390 | } | |
| 391 | 352 | yield* runStep("stopping the service", "systemctl", [ |
| 392 | 353 | "--user", |
| 393 | 354 | "disable", |
| ... | ... | @@ -405,18 +366,32 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { |
| 405 | 366 | if (platform !== "linux" || homeDir === "") { |
| 406 | 367 | return { supported: false, installed: false, current: false, unitPath, logPath }; |
| 407 | 368 | } |
| 408 | const unitExists = yield* fs.exists(unitPath); | |
| 409 | if (!unitExists) { | |
| 369 | if (!(yield* fs.exists(unitPath))) { | |
| 410 | 370 | return { supported: true, installed: false, current: false, unitPath, logPath }; |
| 411 | 371 | } |
| 412 | const unit = yield* fs.readFileString(unitPath); | |
| 413 | // A unit is current only if it matches what install would write now (an | |
| 414 | // older CLI wrote a different runtime/node path) AND the entry point it | |
| 415 | // references still exists (a pinned runtime under ~/.t3 can be deleted to | |
| 416 | // reclaim space). Either mismatch makes connect offer a repair. | |
| 417 | const entryExists = yield* fs.exists(plannedEntryPath); | |
| 418 | const current = unit === renderBootServiceUnit(plan) && entryExists; | |
| 419 | return { supported: true, installed: true, current, unitPath, logPath }; | |
| 372 | const [unit, launcherExists, runtimeEntryExists, runtimeSentinel, stateText] = | |
| 373 | yield* Effect.all([ | |
| 374 | fs.readFileString(unitPath), | |
| 375 | fs.exists(launcherPath), | |
| 376 | fs.exists(runtimePaths.entryPath), | |
| 377 | fs.readFileString(runtimePaths.sentinelPath).pipe(Effect.option), | |
| 378 | fs.readFileString(statePath).pipe(Effect.option), | |
| 379 | ]); | |
| 380 | const state = Option.isSome(stateText) ? parseServiceState(stateText.value) : undefined; | |
| 381 | return { | |
| 382 | supported: true, | |
| 383 | installed: true, | |
| 384 | current: | |
| 385 | unit === renderBootServiceUnit(plan) && | |
| 386 | launcherExists && | |
| 387 | runtimeEntryExists && | |
| 388 | Option.isSome(runtimeSentinel) && | |
| 389 | runtimeSentinel.value.trim() === input.cliVersion && | |
| 390 | state?.activeVersion === input.cliVersion && | |
| 391 | state?.update?.status !== "pending", | |
| 392 | unitPath, | |
| 393 | logPath, | |
| 394 | }; | |
| 420 | 395 | }).pipe( |
| 421 | 396 | Effect.mapError((cause) => new BootServiceInstallError({ cause })), |
| 422 | 397 | Effect.withSpan("cloud.boot_service.status"), |
apps/server/src/cloud/pinnedRuntime.test.ts+133-88
| ... | ... | @@ -2,8 +2,8 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; |
| 2 | 2 | import { assert, it } from "@effect/vitest"; |
| 3 | 3 | import * as Deferred from "effect/Deferred"; |
| 4 | 4 | import * as Effect from "effect/Effect"; |
| 5 | import * as Fiber from "effect/Fiber"; | |
| 6 | 5 | import * as FileSystem from "effect/FileSystem"; |
| 6 | import * as Fiber from "effect/Fiber"; | |
| 7 | 7 | import * as Path from "effect/Path"; |
| 8 | 8 | import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner"; |
| 9 | 9 | |
| ... | ... | @@ -11,119 +11,164 @@ import * as ProcessRunner from "../processRunner.ts"; |
| 11 | 11 | import { |
| 12 | 12 | ensurePinnedRuntimeInstalled, |
| 13 | 13 | pinnedRuntimePaths, |
| 14 | removePinnedRuntimeInstallation, | |
| 14 | PinnedRuntimeInstallError, | |
| 15 | 15 | } from "./pinnedRuntime.ts"; |
| 16 | 16 | |
| 17 | const successfulRunner = (fs: FileSystem.FileSystem, path: Path.Path) => | |
| 18 | ProcessRunner.ProcessRunner.of({ | |
| 19 | run: (input) => | |
| 20 | Effect.gen(function* () { | |
| 21 | const prefixIndex = input.args.indexOf("--prefix"); | |
| 22 | const stagingDir = input.args[prefixIndex + 1]; | |
| 23 | if (stagingDir === undefined) return yield* Effect.die("missing npm --prefix"); | |
| 24 | const entry = path.join(stagingDir, "node_modules", "t3", "dist", "bin.mjs"); | |
| 25 | yield* fs.makeDirectory(path.dirname(entry), { recursive: true }).pipe(Effect.orDie); | |
| 26 | yield* fs.writeFileString(entry, "export {};\n").pipe(Effect.orDie); | |
| 27 | return { | |
| 28 | stdout: "", | |
| 29 | stderr: "", | |
| 30 | code: ChildProcessSpawner.ExitCode(0), | |
| 31 | timedOut: false, | |
| 32 | stdoutTruncated: false, | |
| 33 | stderrTruncated: false, | |
| 34 | }; | |
| 35 | }), | |
| 36 | }); | |
| 37 | ||
| 17 | 38 | it.layer(NodeServices.layer)("ensurePinnedRuntimeInstalled", (it) => { |
| 18 | it.effect("serializes concurrent installs of the same runtime", () => | |
| 39 | it.effect("validates a staging tree before atomically publishing it", () => | |
| 19 | 40 | Effect.gen(function* () { |
| 20 | 41 | const fs = yield* FileSystem.FileSystem; |
| 21 | 42 | const path = yield* Path.Path; |
| 22 | 43 | const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-pinned-runtime-test-" }); |
| 23 | const installStarted = yield* Deferred.make<void>(); | |
| 24 | const allowInstallToFinish = yield* Deferred.make<void>(); | |
| 25 | const paths = pinnedRuntimePaths(path, baseDir, "0.0.29"); | |
| 26 | let npmRuns = 0; | |
| 44 | const finalPaths = pinnedRuntimePaths(path, baseDir, "1.2.3"); | |
| 45 | let validatedDirectory = ""; | |
| 27 | 46 | |
| 28 | const runner = ProcessRunner.ProcessRunner.of({ | |
| 29 | run: (_input) => | |
| 47 | const installed = yield* ensurePinnedRuntimeInstalled({ | |
| 48 | baseDir, | |
| 49 | version: "1.2.3", | |
| 50 | fs, | |
| 51 | path, | |
| 52 | runner: successfulRunner(fs, path), | |
| 53 | validate: (staging) => | |
| 30 | 54 | Effect.gen(function* () { |
| 31 | npmRuns += 1; | |
| 32 | yield* Deferred.succeed(installStarted, undefined); | |
| 33 | yield* Deferred.await(allowInstallToFinish); | |
| 34 | yield* fs | |
| 35 | .makeDirectory(path.dirname(paths.entryPath), { recursive: true }) | |
| 36 | .pipe(Effect.orDie); | |
| 37 | yield* fs.writeFileString(paths.entryPath, "export {};\n").pipe(Effect.orDie); | |
| 38 | return { | |
| 39 | stdout: "", | |
| 40 | stderr: "", | |
| 41 | code: ChildProcessSpawner.ExitCode(0), | |
| 42 | timedOut: false, | |
| 43 | stdoutTruncated: false, | |
| 44 | stderrTruncated: false, | |
| 45 | }; | |
| 46 | }), | |
| 55 | validatedDirectory = staging.versionDir; | |
| 56 | assert.isFalse(yield* fs.exists(finalPaths.versionDir)); | |
| 57 | assert.isTrue(yield* fs.exists(staging.entryPath)); | |
| 58 | }).pipe(Effect.orDie), | |
| 47 | 59 | }); |
| 48 | const install = ensurePinnedRuntimeInstalled({ | |
| 60 | ||
| 61 | assert.notEqual(validatedDirectory, finalPaths.versionDir); | |
| 62 | assert.deepEqual(installed, finalPaths); | |
| 63 | assert.isTrue(yield* fs.exists(finalPaths.entryPath)); | |
| 64 | assert.equal(yield* fs.readFileString(finalPaths.sentinelPath), "1.2.3\n"); | |
| 65 | }), | |
| 66 | ); | |
| 67 | ||
| 68 | it.effect("removes staging and leaves no final runtime when validation fails", () => | |
| 69 | Effect.gen(function* () { | |
| 70 | const fs = yield* FileSystem.FileSystem; | |
| 71 | const path = yield* Path.Path; | |
| 72 | const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-pinned-runtime-test-" }); | |
| 73 | const finalPaths = pinnedRuntimePaths(path, baseDir, "1.2.3"); | |
| 74 | ||
| 75 | yield* ensurePinnedRuntimeInstalled({ | |
| 49 | 76 | baseDir, |
| 50 | version: "0.0.29", | |
| 77 | version: "1.2.3", | |
| 51 | 78 | fs, |
| 52 | 79 | path, |
| 53 | runner, | |
| 54 | }); | |
| 80 | runner: successfulRunner(fs, path), | |
| 81 | validate: () => | |
| 82 | Effect.fail(new PinnedRuntimeInstallError({ step: "validating the staged runtime" })), | |
| 83 | }).pipe(Effect.flip); | |
| 55 | 84 | |
| 56 | const first = yield* Effect.forkChild(install, { startImmediately: true }); | |
| 57 | yield* Deferred.await(installStarted); | |
| 58 | const second = yield* Effect.forkChild(install, { startImmediately: true }); | |
| 59 | yield* Effect.yieldNow; | |
| 60 | assert.equal(npmRuns, 1); | |
| 85 | assert.isFalse(yield* fs.exists(finalPaths.versionDir)); | |
| 86 | assert.deepEqual( | |
| 87 | (yield* fs.readDirectory(path.dirname(finalPaths.versionDir))).filter((entry) => | |
| 88 | entry.startsWith(".staging-"), | |
| 89 | ), | |
| 90 | [], | |
| 91 | ); | |
| 92 | }), | |
| 93 | ); | |
| 61 | 94 | |
| 62 | yield* Deferred.succeed(allowInstallToFinish, undefined); | |
| 63 | yield* Fiber.join(first); | |
| 64 | yield* Fiber.join(second); | |
| 95 | it.effect("replaces an incomplete pinned runtime", () => | |
| 96 | Effect.gen(function* () { | |
| 97 | const fs = yield* FileSystem.FileSystem; | |
| 98 | const path = yield* Path.Path; | |
| 99 | const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-pinned-runtime-repair-" }); | |
| 100 | const finalPaths = pinnedRuntimePaths(path, baseDir, "1.2.3"); | |
| 101 | yield* fs.makeDirectory(finalPaths.versionDir, { recursive: true }); | |
| 102 | yield* fs.writeFileString(path.join(finalPaths.versionDir, "partial"), "incomplete\n"); | |
| 65 | 103 | |
| 66 | assert.equal(npmRuns, 1); | |
| 67 | assert.isTrue(yield* fs.exists(paths.sentinelPath)); | |
| 68 | assert.isTrue(yield* fs.exists(paths.entryPath)); | |
| 104 | yield* ensurePinnedRuntimeInstalled({ | |
| 105 | baseDir, | |
| 106 | version: "1.2.3", | |
| 107 | fs, | |
| 108 | path, | |
| 109 | runner: successfulRunner(fs, path), | |
| 110 | validate: () => Effect.void, | |
| 111 | }); | |
| 112 | ||
| 113 | assert.isFalse(yield* fs.exists(path.join(finalPaths.versionDir, "partial"))); | |
| 114 | assert.isTrue(yield* fs.exists(finalPaths.entryPath)); | |
| 69 | 115 | }), |
| 70 | 116 | ); |
| 71 | 117 | |
| 72 | it.effect("waits for an active install before removing its runtime", () => | |
| 118 | it.effect("preserves a completed runtime when validation fails", () => | |
| 73 | 119 | Effect.gen(function* () { |
| 74 | 120 | const fs = yield* FileSystem.FileSystem; |
| 75 | 121 | const path = yield* Path.Path; |
| 76 | const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-pinned-runtime-test-" }); | |
| 77 | const installStarted = yield* Deferred.make<void>(); | |
| 78 | const allowInstallToFinish = yield* Deferred.make<void>(); | |
| 79 | const paths = pinnedRuntimePaths(path, baseDir, "0.0.30"); | |
| 80 | const runner = ProcessRunner.ProcessRunner.of({ | |
| 81 | run: (_input) => | |
| 122 | const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-pinned-runtime-repair-" }); | |
| 123 | const finalPaths = pinnedRuntimePaths(path, baseDir, "1.2.3"); | |
| 124 | yield* fs.makeDirectory(path.dirname(finalPaths.entryPath), { recursive: true }); | |
| 125 | yield* fs.writeFileString(finalPaths.entryPath, "broken\n"); | |
| 126 | yield* fs.writeFileString(finalPaths.sentinelPath, "1.2.3\n"); | |
| 127 | ||
| 128 | let validations = 0; | |
| 129 | yield* ensurePinnedRuntimeInstalled({ | |
| 130 | baseDir, | |
| 131 | version: "1.2.3", | |
| 132 | fs, | |
| 133 | path, | |
| 134 | runner: successfulRunner(fs, path), | |
| 135 | validate: (paths) => | |
| 82 | 136 | Effect.gen(function* () { |
| 83 | yield* Deferred.succeed(installStarted, undefined); | |
| 84 | yield* Deferred.await(allowInstallToFinish); | |
| 85 | yield* fs | |
| 86 | .makeDirectory(path.dirname(paths.entryPath), { recursive: true }) | |
| 87 | .pipe(Effect.orDie); | |
| 88 | yield* fs.writeFileString(paths.entryPath, "export {};\n").pipe(Effect.orDie); | |
| 89 | return { | |
| 90 | stdout: "", | |
| 91 | stderr: "", | |
| 92 | code: ChildProcessSpawner.ExitCode(0), | |
| 93 | timedOut: false, | |
| 94 | stdoutTruncated: false, | |
| 95 | stderrTruncated: false, | |
| 96 | }; | |
| 137 | validations += 1; | |
| 138 | const source = yield* fs.readFileString(paths.entryPath).pipe(Effect.orDie); | |
| 139 | if (source === "broken\n") { | |
| 140 | return yield* new PinnedRuntimeInstallError({ step: "validating the runtime" }); | |
| 141 | } | |
| 97 | 142 | }), |
| 98 | }); | |
| 143 | }).pipe(Effect.flip); | |
| 99 | 144 | |
| 100 | const installFiber = yield* Effect.forkChild( | |
| 101 | ensurePinnedRuntimeInstalled({ | |
| 102 | baseDir, | |
| 103 | version: "0.0.30", | |
| 104 | fs, | |
| 105 | path, | |
| 106 | runner, | |
| 107 | }), | |
| 108 | { startImmediately: true }, | |
| 109 | ); | |
| 110 | yield* Deferred.await(installStarted); | |
| 111 | const removeFiber = yield* Effect.forkChild( | |
| 112 | removePinnedRuntimeInstallation({ | |
| 113 | baseDir, | |
| 114 | version: "0.0.30", | |
| 115 | fs, | |
| 116 | path, | |
| 117 | }), | |
| 118 | { startImmediately: true }, | |
| 119 | ); | |
| 120 | yield* Effect.yieldNow; | |
| 121 | assert.isTrue(yield* fs.exists(paths.versionDir)); | |
| 145 | assert.equal(validations, 1); | |
| 146 | assert.equal(yield* fs.readFileString(finalPaths.entryPath), "broken\n"); | |
| 147 | }), | |
| 148 | ); | |
| 149 | ||
| 150 | it.effect("removes staging when installation is interrupted", () => | |
| 151 | Effect.gen(function* () { | |
| 152 | const fs = yield* FileSystem.FileSystem; | |
| 153 | const path = yield* Path.Path; | |
| 154 | const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-pinned-runtime-interrupt-" }); | |
| 155 | const started = yield* Deferred.make<void>(); | |
| 156 | const runner = ProcessRunner.ProcessRunner.of({ | |
| 157 | run: () => Deferred.succeed(started, undefined).pipe(Effect.andThen(Effect.never)), | |
| 158 | }); | |
| 159 | const install = yield* ensurePinnedRuntimeInstalled({ | |
| 160 | baseDir, | |
| 161 | version: "1.2.3", | |
| 162 | fs, | |
| 163 | path, | |
| 164 | runner, | |
| 165 | validate: () => Effect.void, | |
| 166 | }).pipe(Effect.forkScoped); | |
| 122 | 167 | |
| 123 | yield* Deferred.succeed(allowInstallToFinish, undefined); | |
| 124 | yield* Fiber.join(installFiber); | |
| 125 | yield* Fiber.join(removeFiber); | |
| 126 | assert.isFalse(yield* fs.exists(paths.versionDir)); | |
| 168 | yield* Deferred.await(started); | |
| 169 | yield* Fiber.interrupt(install); | |
| 170 | const versionsDir = path.join(baseDir, "runtime", "versions"); | |
| 171 | assert.deepEqual(yield* fs.readDirectory(versionsDir), []); | |
| 127 | 172 | }), |
| 128 | 173 | ); |
| 129 | 174 | }); |
apps/server/src/cloud/pinnedRuntime.ts+150-104
| ... | ... | @@ -3,6 +3,7 @@ import * as Effect from "effect/Effect"; |
| 3 | 3 | import * as FileSystem from "effect/FileSystem"; |
| 4 | 4 | import * as Path from "effect/Path"; |
| 5 | 5 | import * as Schema from "effect/Schema"; |
| 6 | import * as Option from "effect/Option"; | |
| 6 | 7 | import * as Semaphore from "effect/Semaphore"; |
| 7 | 8 | |
| 8 | 9 | import * as ProcessRunner from "../processRunner.ts"; |
| ... | ... | @@ -11,15 +12,14 @@ import * as ProcessRunner from "../processRunner.ts"; |
| 11 | 12 | * A pinned runtime is an exact `t3@<version>` npm-installed into |
| 12 | 13 | * <baseDir>/runtime/versions/<version>. The boot service points its systemd |
| 13 | 14 | * unit here, and server self-update installs the target version here before |
| 14 | * switching over — never `npx t3`, whose cache is ephemeral and whose | |
| 15 | * switching over, never `npx t3`, whose cache is ephemeral and whose | |
| 15 | 16 | * registry fetch at boot would make startup depend on the network. |
| 16 | 17 | */ |
| 17 | 18 | |
| 18 | 19 | const PINNED_RUNTIME_DIR = "runtime"; |
| 19 | 20 | const PINNED_RUNTIME_INSTALL_TIMEOUT = Duration.minutes(10); |
| 20 | // Boot-service setup and remote self-update share this module but can be | |
| 21 | // constructed in separate layers. Serialize the complete check/install/ | |
| 22 | // sentinel transaction across all callers in this process. | |
| 21 | // Boot-service setup and remote update can construct separate layers. Serialize | |
| 22 | // the complete install transaction across every caller in this process. | |
| 23 | 23 | const pinnedRuntimeInstallLock = Semaphore.makeUnsafe(1); |
| 24 | 24 | |
| 25 | 25 | export interface PinnedRuntimePaths { |
| ... | ... | @@ -58,119 +58,165 @@ export class PinnedRuntimeInstallError extends Schema.TaggedErrorClass<PinnedRun |
| 58 | 58 | } |
| 59 | 59 | } |
| 60 | 60 | |
| 61 | export class PinnedRuntimePreflightBlockedError extends Schema.TaggedErrorClass<PinnedRuntimePreflightBlockedError>()( | |
| 62 | "PinnedRuntimePreflightBlockedError", | |
| 63 | { | |
| 64 | version: Schema.String, | |
| 65 | reason: Schema.String, | |
| 66 | }, | |
| 67 | ) { | |
| 68 | override get message(): string { | |
| 69 | return this.reason; | |
| 70 | } | |
| 71 | } | |
| 72 | ||
| 61 | 73 | /** |
| 62 | 74 | * Installs `t3@<version>` into the pinned runtime directory unless a complete |
| 63 | 75 | * install is already there, and returns its paths. The sentinel is written |
| 64 | * only after npm exits 0; checking the entry file alone is not enough — npm | |
| 76 | * only after npm exits 0; checking the entry file alone is not enough. npm | |
| 65 | 77 | * extracts files before running native builds (node-pty), so a killed |
| 66 | 78 | * install leaves a plausible-looking but broken tree behind. |
| 67 | 79 | */ |
| 68 | export const ensurePinnedRuntimeInstalled = Effect.fn("cloud.pinned_runtime.ensure_installed")( | |
| 69 | function* (input: { | |
| 70 | readonly baseDir: string; | |
| 71 | readonly version: string; | |
| 72 | readonly fs: FileSystem.FileSystem; | |
| 73 | readonly path: Path.Path; | |
| 74 | readonly runner: ProcessRunner.ProcessRunner["Service"]; | |
| 75 | }) { | |
| 76 | const { fs, runner } = input; | |
| 77 | const paths = pinnedRuntimePaths(input.path, input.baseDir, input.version); | |
| 80 | interface PinnedRuntimeInstallInput { | |
| 81 | readonly baseDir: string; | |
| 82 | readonly version: string; | |
| 83 | readonly fs: FileSystem.FileSystem; | |
| 84 | readonly path: Path.Path; | |
| 85 | readonly runner: ProcessRunner.ProcessRunner["Service"]; | |
| 86 | readonly validate: ( | |
| 87 | paths: PinnedRuntimePaths, | |
| 88 | ) => Effect.Effect<void, PinnedRuntimeInstallError | PinnedRuntimePreflightBlockedError>; | |
| 89 | } | |
| 90 | ||
| 91 | const installPinnedRuntime = Effect.fn("cloud.pinned_runtime.ensure_installed")(function* ( | |
| 92 | input: PinnedRuntimeInstallInput, | |
| 93 | ) { | |
| 94 | const { fs, runner } = input; | |
| 95 | const paths = pinnedRuntimePaths(input.path, input.baseDir, input.version); | |
| 96 | const [versionDirExists, entryExists, sentinel] = yield* Effect.all([ | |
| 97 | fs.exists(paths.versionDir), | |
| 98 | fs.exists(paths.entryPath), | |
| 99 | fs.readFileString(paths.sentinelPath).pipe(Effect.option), | |
| 100 | ]).pipe( | |
| 101 | Effect.mapError( | |
| 102 | (cause) => new PinnedRuntimeInstallError({ step: "checking the pinned runtime", cause }), | |
| 103 | ), | |
| 104 | ); | |
| 105 | const alreadyPinned = | |
| 106 | entryExists && Option.isSome(sentinel) && sentinel.value.trim() === input.version; | |
| 107 | if (alreadyPinned) { | |
| 108 | yield* input.validate(paths); | |
| 109 | return paths; | |
| 110 | } | |
| 111 | if (versionDirExists) { | |
| 112 | yield* fs.remove(paths.versionDir, { recursive: true, force: true }).pipe( | |
| 113 | Effect.mapError( | |
| 114 | (cause) => | |
| 115 | new PinnedRuntimeInstallError({ | |
| 116 | step: "removing an incomplete pinned runtime", | |
| 117 | cause, | |
| 118 | }), | |
| 119 | ), | |
| 120 | ); | |
| 121 | } | |
| 122 | ||
| 123 | const versionsDir = input.path.dirname(paths.versionDir); | |
| 124 | yield* fs.makeDirectory(versionsDir, { recursive: true }).pipe( | |
| 125 | Effect.mapError( | |
| 126 | (cause) => | |
| 127 | new PinnedRuntimeInstallError({ | |
| 128 | step: "preparing the pinned runtime directory", | |
| 129 | cause, | |
| 130 | }), | |
| 131 | ), | |
| 132 | ); | |
| 133 | const stagingDir = yield* fs | |
| 134 | .makeTempDirectory({ | |
| 135 | directory: versionsDir, | |
| 136 | prefix: ".staging-", | |
| 137 | }) | |
| 138 | .pipe( | |
| 139 | Effect.mapError( | |
| 140 | (cause) => | |
| 141 | new PinnedRuntimeInstallError({ | |
| 142 | step: "preparing the pinned runtime directory", | |
| 143 | cause, | |
| 144 | }), | |
| 145 | ), | |
| 146 | ); | |
| 147 | const stagingPaths: PinnedRuntimePaths = { | |
| 148 | versionDir: stagingDir, | |
| 149 | entryPath: input.path.join(stagingDir, "node_modules", "t3", "dist", "bin.mjs"), | |
| 150 | sentinelPath: input.path.join(stagingDir, ".install-complete"), | |
| 151 | }; | |
| 78 | 152 | |
| 79 | return yield* pinnedRuntimeInstallLock.withPermit( | |
| 80 | Effect.gen(function* () { | |
| 81 | const alreadyPinned = yield* Effect.all([ | |
| 82 | fs.exists(paths.sentinelPath), | |
| 153 | return yield* Effect.gen(function* () { | |
| 154 | const installStep = "installing the pinned t3 runtime (this can take a few minutes)"; | |
| 155 | yield* runner | |
| 156 | .run({ | |
| 157 | command: "npm", | |
| 158 | args: ["install", "--prefix", stagingDir, "--no-fund", "--no-audit", `t3@${input.version}`], | |
| 159 | // Native dependencies may compile from source on slower machines. | |
| 160 | timeout: PINNED_RUNTIME_INSTALL_TIMEOUT, | |
| 161 | }) | |
| 162 | .pipe( | |
| 163 | Effect.mapError((cause) => new PinnedRuntimeInstallError({ step: installStep, cause })), | |
| 164 | Effect.filterOrFail( | |
| 165 | (result) => result.code === 0, | |
| 166 | (result) => | |
| 167 | new PinnedRuntimeInstallError({ | |
| 168 | step: installStep, | |
| 169 | exitCode: Number(result.code), | |
| 170 | stdoutLength: result.stdout.length, | |
| 171 | stderrLength: result.stderr.length, | |
| 172 | }), | |
| 173 | ), | |
| 174 | ); | |
| 175 | ||
| 176 | yield* input.validate(stagingPaths); | |
| 177 | yield* fs | |
| 178 | .writeFileString(stagingPaths.sentinelPath, `${input.version}\n`) | |
| 179 | .pipe( | |
| 180 | Effect.mapError( | |
| 181 | (cause) => | |
| 182 | new PinnedRuntimeInstallError({ step: "recording the completed install", cause }), | |
| 183 | ), | |
| 184 | ); | |
| 185 | const published = yield* fs.rename(stagingDir, paths.versionDir).pipe( | |
| 186 | Effect.as(true), | |
| 187 | Effect.catch((cause) => | |
| 188 | Effect.all([ | |
| 83 | 189 | fs.exists(paths.entryPath), |
| 190 | fs.readFileString(paths.sentinelPath).pipe(Effect.option), | |
| 84 | 191 | ]).pipe( |
| 85 | Effect.map(([sentinelExists, entryExists]) => sentinelExists && entryExists), | |
| 86 | Effect.mapError( | |
| 87 | (cause) => | |
| 88 | new PinnedRuntimeInstallError({ step: "checking the pinned runtime", cause }), | |
| 89 | ), | |
| 90 | ); | |
| 91 | if (alreadyPinned) { | |
| 92 | return paths; | |
| 93 | } | |
| 94 | ||
| 95 | yield* fs.remove(paths.versionDir, { recursive: true, force: true }).pipe( | |
| 96 | Effect.andThen(fs.makeDirectory(paths.versionDir, { recursive: true })), | |
| 97 | 192 | Effect.mapError( |
| 98 | (cause) => | |
| 193 | (checkCause) => | |
| 99 | 194 | new PinnedRuntimeInstallError({ |
| 100 | step: "preparing the pinned runtime directory", | |
| 101 | cause, | |
| 195 | step: "checking a concurrently published pinned runtime", | |
| 196 | cause: checkCause, | |
| 102 | 197 | }), |
| 103 | 198 | ), |
| 104 | ); | |
| 105 | ||
| 106 | const installStep = "installing the pinned t3 runtime (this can take a few minutes)"; | |
| 107 | yield* runner | |
| 108 | .run({ | |
| 109 | command: "npm", | |
| 110 | args: [ | |
| 111 | "install", | |
| 112 | "--prefix", | |
| 113 | paths.versionDir, | |
| 114 | "--no-fund", | |
| 115 | "--no-audit", | |
| 116 | `t3@${input.version}`, | |
| 117 | ], | |
| 118 | // Native deps (node-pty) can compile from source on slow boxes; the | |
| 119 | // ProcessRunner default of 60s would kill a healthy install. | |
| 120 | timeout: PINNED_RUNTIME_INSTALL_TIMEOUT, | |
| 121 | }) | |
| 122 | .pipe( | |
| 123 | Effect.mapError((cause) => new PinnedRuntimeInstallError({ step: installStep, cause })), | |
| 124 | Effect.filterOrFail( | |
| 125 | (result) => result.code === 0, | |
| 126 | (result) => | |
| 127 | new PinnedRuntimeInstallError({ | |
| 128 | step: installStep, | |
| 129 | exitCode: Number(result.code), | |
| 130 | stdoutLength: result.stdout.length, | |
| 131 | stderrLength: result.stderr.length, | |
| 132 | }), | |
| 133 | ), | |
| 134 | Effect.tapError(() => | |
| 135 | fs.remove(paths.versionDir, { recursive: true, force: true }).pipe(Effect.ignore), | |
| 136 | ), | |
| 137 | ); | |
| 138 | ||
| 139 | yield* fs | |
| 140 | .writeFileString(paths.sentinelPath, `${input.version}\n`) | |
| 141 | .pipe( | |
| 142 | Effect.mapError( | |
| 143 | (cause) => | |
| 144 | new PinnedRuntimeInstallError({ step: "recording the completed install", cause }), | |
| 145 | ), | |
| 146 | ); | |
| 147 | ||
| 148 | return paths; | |
| 149 | }), | |
| 150 | ); | |
| 151 | }, | |
| 152 | ); | |
| 153 | ||
| 154 | /** Removes one pinned runtime while holding the same process-wide lock used | |
| 155 | * by install/check/sentinel work, so cleanup cannot race another caller that | |
| 156 | * is materializing or reusing the runtime tree. */ | |
| 157 | export const removePinnedRuntimeInstallation = Effect.fn("cloud.pinned_runtime.remove")( | |
| 158 | function* (input: { | |
| 159 | readonly baseDir: string; | |
| 160 | readonly version: string; | |
| 161 | readonly fs: FileSystem.FileSystem; | |
| 162 | readonly path: Path.Path; | |
| 163 | }) { | |
| 164 | const paths = pinnedRuntimePaths(input.path, input.baseDir, input.version); | |
| 165 | yield* pinnedRuntimeInstallLock.withPermit( | |
| 166 | input.fs | |
| 167 | .remove(paths.versionDir, { recursive: true, force: true }) | |
| 168 | .pipe( | |
| 169 | Effect.mapError( | |
| 170 | (cause) => | |
| 171 | new PinnedRuntimeInstallError({ step: "removing the pinned runtime", cause }), | |
| 199 | Effect.flatMap(([publishedEntryExists, publishedSentinel]) => | |
| 200 | publishedEntryExists && | |
| 201 | Option.isSome(publishedSentinel) && | |
| 202 | publishedSentinel.value.trim() === input.version | |
| 203 | ? Effect.succeed(false) | |
| 204 | : Effect.fail( | |
| 205 | new PinnedRuntimeInstallError({ | |
| 206 | step: "publishing the pinned runtime", | |
| 207 | cause, | |
| 208 | }), | |
| 209 | ), | |
| 172 | 210 | ), |
| 173 | 211 | ), |
| 212 | ), | |
| 174 | 213 | ); |
| 175 | }, | |
| 176 | ); | |
| 214 | if (!published) yield* input.validate(paths); | |
| 215 | return paths; | |
| 216 | }).pipe( | |
| 217 | Effect.ensuring(fs.remove(stagingDir, { recursive: true, force: true }).pipe(Effect.ignore)), | |
| 218 | ); | |
| 219 | }); | |
| 220 | ||
| 221 | export const ensurePinnedRuntimeInstalled = (input: PinnedRuntimeInstallInput) => | |
| 222 | pinnedRuntimeInstallLock.withPermit(installPinnedRuntime(input)); |
apps/server/src/cloud/selfUpdate.test.ts+103-591
| ... | ... | @@ -1,632 +1,144 @@ |
| 1 | 1 | import * as NodeServices from "@effect/platform-node/NodeServices"; |
| 2 | import { assert, it } from "@effect/vitest"; | |
| 3 | import * as Duration from "effect/Duration"; | |
| 2 | import { expect, it } from "@effect/vitest"; | |
| 3 | import { HostProcessExecutablePath } from "@t3tools/shared/hostProcess"; | |
| 4 | import * as Deferred from "effect/Deferred"; | |
| 4 | 5 | import * as Effect from "effect/Effect"; |
| 5 | 6 | import * as FileSystem from "effect/FileSystem"; |
| 6 | import * as Layer from "effect/Layer"; | |
| 7 | import * as Fiber from "effect/Fiber"; | |
| 7 | 8 | import * as Path from "effect/Path"; |
| 8 | import * as TestClock from "effect/testing/TestClock"; | |
| 9 | 9 | import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner"; |
| 10 | 10 | |
| 11 | import { | |
| 12 | HostProcessArguments, | |
| 13 | HostProcessEnvironment, | |
| 14 | HostProcessExecutablePath, | |
| 15 | HostProcessPlatform, | |
| 16 | } from "@t3tools/shared/hostProcess"; | |
| 17 | ||
| 18 | 11 | import * as ServerConfig from "../config.ts"; |
| 19 | 12 | import * as ProcessRunner from "../processRunner.ts"; |
| 20 | import { | |
| 21 | BOOT_SERVICE_UNIT_ENV, | |
| 22 | BOOT_SERVICE_UNIT_FILE, | |
| 23 | renderBootServiceUnit, | |
| 24 | } from "./bootService.ts"; | |
| 25 | import * as SelfUpdate from "./selfUpdate.ts"; | |
| 26 | ||
| 27 | const NODE_PATH = "/usr/local/bin/node"; | |
| 13 | import * as ServiceLauncherClient from "./serviceLauncherClient.ts"; | |
| 14 | import * as ServerSelfUpdate from "./selfUpdate.ts"; | |
| 15 | ||
| 16 | interface HarnessOptions { | |
| 17 | readonly mode?: "web" | "desktop"; | |
| 18 | readonly managed?: boolean; | |
| 19 | readonly preflight?: "ready" | "blocked"; | |
| 20 | readonly requestUpdate?: ServiceLauncherClient.ServiceLauncherClient["Service"]["requestUpdate"]; | |
| 21 | } | |
| 28 | 22 | |
| 29 | const eventuallyFileString = Effect.fn("test.eventuallyFileString")(function* ( | |
| 30 | filePath: string, | |
| 31 | expected: string, | |
| 23 | const makeHarness = Effect.fn("test.make_self_update_harness")(function* ( | |
| 24 | options: HarnessOptions = {}, | |
| 32 | 25 | ) { |
| 33 | 26 | const fs = yield* FileSystem.FileSystem; |
| 34 | for (let iteration = 0; iteration < 1_000; iteration += 1) { | |
| 35 | const contents = yield* fs.readFileString(filePath); | |
| 36 | if (contents === expected) { | |
| 37 | return; | |
| 38 | } | |
| 39 | // The rollback performs real filesystem I/O on a detached fiber, which | |
| 40 | // advancing TestClock does not await. | |
| 41 | yield* Effect.yieldNow; | |
| 42 | } | |
| 43 | return yield* Effect.die(new Error(`Expected file contents were not observed at ${filePath}.`)); | |
| 44 | }); | |
| 45 | ||
| 46 | const eventuallyTrue = Effect.fn("test.eventuallyTrue")(function* (predicate: () => boolean) { | |
| 47 | for (let iteration = 0; iteration < 1_000; iteration += 1) { | |
| 48 | if (predicate()) { | |
| 49 | return; | |
| 50 | } | |
| 51 | yield* Effect.yieldNow; | |
| 52 | } | |
| 53 | return yield* Effect.die(new Error("Expected condition was not observed.")); | |
| 54 | }); | |
| 55 | ||
| 56 | interface RecordedCommand { | |
| 57 | readonly command: string; | |
| 58 | readonly args: ReadonlyArray<string>; | |
| 59 | } | |
| 60 | ||
| 61 | const makeRecordingRunnerLayer = ( | |
| 62 | commands: Array<RecordedCommand>, | |
| 63 | options?: { | |
| 64 | readonly failWhen?: ((command: string, args: ReadonlyArray<string>) => boolean) | undefined; | |
| 65 | readonly stdoutFor?: | |
| 66 | | ((command: string, args: ReadonlyArray<string>) => string | undefined) | |
| 67 | | undefined; | |
| 68 | }, | |
| 69 | ) => | |
| 70 | Layer.succeed( | |
| 71 | ProcessRunner.ProcessRunner, | |
| 72 | ProcessRunner.ProcessRunner.of({ | |
| 73 | run: (input) => | |
| 74 | Effect.sync(() => { | |
| 75 | commands.push({ command: input.command, args: input.args }); | |
| 76 | const failed = options?.failWhen?.(input.command, input.args) === true; | |
| 77 | const versionFromPath = | |
| 78 | input.command === NODE_PATH && input.args[1] === "--version" | |
| 79 | ? /[/\\]runtime[/\\]versions[/\\]([^/\\]+)/.exec(input.args[0] ?? "")?.[1] | |
| 80 | : undefined; | |
| 27 | const path = yield* Path.Path; | |
| 28 | const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-self-update-test-" }); | |
| 29 | const order: string[] = []; | |
| 30 | const runner = ProcessRunner.ProcessRunner.of({ | |
| 31 | run: (input) => | |
| 32 | Effect.gen(function* () { | |
| 33 | if (input.command === "npm") { | |
| 34 | order.push("install"); | |
| 35 | const prefix = input.args[input.args.indexOf("--prefix") + 1]; | |
| 36 | if (prefix === undefined) return yield* Effect.die("missing npm prefix"); | |
| 37 | const entry = path.join(prefix, "node_modules", "t3", "dist", "bin.mjs"); | |
| 38 | yield* fs.makeDirectory(path.dirname(entry), { recursive: true }).pipe(Effect.orDie); | |
| 39 | yield* fs.writeFileString(entry, "export {};\n").pipe(Effect.orDie); | |
| 81 | 40 | return { |
| 82 | stdout: | |
| 83 | options?.stdoutFor?.(input.command, input.args) ?? | |
| 84 | (versionFromPath === undefined ? "" : `t3 v${versionFromPath}\n`), | |
| 85 | stderr: failed ? `${input.command} exploded` : "", | |
| 86 | code: ChildProcessSpawner.ExitCode(failed ? 1 : 0), | |
| 41 | stdout: "", | |
| 42 | stderr: "", | |
| 43 | code: ChildProcessSpawner.ExitCode(0), | |
| 87 | 44 | timedOut: false, |
| 88 | 45 | stdoutTruncated: false, |
| 89 | 46 | stderrTruncated: false, |
| 90 | 47 | }; |
| 91 | }), | |
| 92 | }), | |
| 93 | ); | |
| 94 | ||
| 95 | const provideHostRefs = (input: { | |
| 96 | readonly platform: NodeJS.Platform; | |
| 97 | readonly env: NodeJS.ProcessEnv; | |
| 98 | readonly entryPath: string; | |
| 99 | }) => | |
| 100 | Effect.provide( | |
| 101 | Layer.mergeAll( | |
| 102 | Layer.succeed(HostProcessPlatform, input.platform), | |
| 103 | Layer.succeed(HostProcessEnvironment, input.env), | |
| 104 | Layer.succeed(HostProcessExecutablePath, NODE_PATH), | |
| 105 | Layer.succeed(HostProcessArguments, [NODE_PATH, input.entryPath, "serve"]), | |
| 106 | ), | |
| 107 | ); | |
| 108 | ||
| 109 | it("recognizes published npm artifacts as swappable entry points", () => { | |
| 110 | assert.isTrue(SelfUpdate.isPublishedCliEntry("/usr/local/lib/node_modules/t3/dist/bin.mjs")); | |
| 111 | assert.isTrue( | |
| 112 | SelfUpdate.isPublishedCliEntry("/home/theo/.npm/_npx/abc123/node_modules/t3/dist/bin.mjs"), | |
| 113 | ); | |
| 114 | assert.isTrue( | |
| 115 | SelfUpdate.isPublishedCliEntry( | |
| 116 | "C:\\Users\\theo\\AppData\\Roaming\\npm\\node_modules\\t3\\dist\\bin.mjs", | |
| 117 | ), | |
| 118 | ); | |
| 119 | // Dev checkouts and the desktop bundle run apps/server/dist directly. | |
| 120 | assert.isFalse(SelfUpdate.isPublishedCliEntry("/home/theo/dev/t3/apps/server/dist/bin.mjs")); | |
| 121 | assert.isFalse(SelfUpdate.isPublishedCliEntry("")); | |
| 122 | }); | |
| 123 | ||
| 124 | it.layer(NodeServices.layer)("resolveServerSelfUpdateCapability", (it) => { | |
| 125 | const makeHome = Effect.fn("test.makeHome")(function* () { | |
| 126 | const fs = yield* FileSystem.FileSystem; | |
| 127 | const path = yield* Path.Path; | |
| 128 | const home = yield* fs.makeTempDirectoryScoped({ prefix: "t3-self-update-test-" }); | |
| 129 | return { fs, path, home }; | |
| 130 | }); | |
| 131 | ||
| 132 | const writeUnitReferencing = Effect.fn("test.writeUnitReferencing")(function* ( | |
| 133 | home: string, | |
| 134 | entryPath: string, | |
| 135 | ) { | |
| 136 | const fs = yield* FileSystem.FileSystem; | |
| 137 | const path = yield* Path.Path; | |
| 138 | const unitDir = path.join(home, ".config", "systemd", "user"); | |
| 139 | yield* fs.makeDirectory(unitDir, { recursive: true }); | |
| 140 | yield* fs.writeFileString( | |
| 141 | path.join(unitDir, "t3code.service"), | |
| 142 | renderBootServiceUnit({ | |
| 143 | nodePath: NODE_PATH, | |
| 144 | t3EntryPath: entryPath, | |
| 145 | baseDir: path.join(home, ".t3"), | |
| 146 | logPath: path.join(home, ".t3", "userdata", "logs", "boot-service.log"), | |
| 147 | unitPath: path.join(unitDir, "t3code.service"), | |
| 48 | } | |
| 49 | order.push("preflight"); | |
| 50 | const result = | |
| 51 | options.preflight === "blocked" | |
| 52 | ? { status: "blocked", version: "1.1.0", reason: "local update required" } | |
| 53 | : { status: "ready", version: "1.1.0", launcherProtocol: 1 }; | |
| 54 | return { | |
| 55 | // @effect-diagnostics-next-line preferSchemaOverJson:off - fake child-process stdout. | |
| 56 | stdout: JSON.stringify(result), | |
| 57 | stderr: "", | |
| 58 | code: ChildProcessSpawner.ExitCode(0), | |
| 59 | timedOut: false, | |
| 60 | stdoutTruncated: false, | |
| 61 | stderrTruncated: false, | |
| 62 | }; | |
| 148 | 63 | }), |
| 149 | ); | |
| 150 | 64 | }); |
| 151 | ||
| 152 | it.effect("reports boot-service for the systemd-spawned unit process", () => | |
| 153 | Effect.gen(function* () { | |
| 154 | const { home, path } = yield* makeHome(); | |
| 155 | const entryPath = path.join(home, ".t3/runtime/versions/0.0.28/node_modules/t3/dist/bin.mjs"); | |
| 156 | yield* writeUnitReferencing(home, entryPath); | |
| 157 | const method = yield* SelfUpdate.resolveServerSelfUpdateCapability({ | |
| 158 | desktopManaged: false, | |
| 159 | }).pipe( | |
| 160 | provideHostRefs({ | |
| 161 | platform: "linux", | |
| 162 | env: { | |
| 163 | HOME: home, | |
| 164 | INVOCATION_ID: "abc123", | |
| 165 | [BOOT_SERVICE_UNIT_ENV]: BOOT_SERVICE_UNIT_FILE, | |
| 166 | }, | |
| 167 | entryPath, | |
| 168 | }), | |
| 169 | ); | |
| 170 | assert.equal(method, "boot-service"); | |
| 171 | }), | |
| 172 | ); | |
| 173 | ||
| 174 | it.effect("does not claim a systemd process owned by another unit", () => | |
| 175 | Effect.gen(function* () { | |
| 176 | const { home, path } = yield* makeHome(); | |
| 177 | const entryPath = path.join(home, ".t3/runtime/versions/0.0.28/node_modules/t3/dist/bin.mjs"); | |
| 178 | yield* writeUnitReferencing(home, entryPath); | |
| 179 | const method = yield* SelfUpdate.resolveServerSelfUpdateCapability({ | |
| 180 | desktopManaged: false, | |
| 181 | }).pipe( | |
| 182 | provideHostRefs({ | |
| 183 | platform: "linux", | |
| 184 | env: { HOME: home, INVOCATION_ID: "abc123" }, | |
| 185 | entryPath, | |
| 186 | }), | |
| 187 | ); | |
| 188 | assert.isNull(method); | |
| 189 | }), | |
| 190 | ); | |
| 191 | ||
| 192 | it.effect("reports respawn for a manual run of the pinned artifact", () => | |
| 193 | Effect.gen(function* () { | |
| 194 | const { home, path } = yield* makeHome(); | |
| 195 | const entryPath = path.join(home, ".t3/runtime/versions/0.0.28/node_modules/t3/dist/bin.mjs"); | |
| 196 | yield* writeUnitReferencing(home, entryPath); | |
| 197 | // Same unit on disk, but no INVOCATION_ID: restarting the unit would | |
| 198 | // not replace this process, so it must respawn itself instead. | |
| 199 | const method = yield* SelfUpdate.resolveServerSelfUpdateCapability({ | |
| 200 | desktopManaged: false, | |
| 201 | }).pipe(provideHostRefs({ platform: "linux", env: { HOME: home }, entryPath })); | |
| 202 | assert.equal(method, "respawn"); | |
| 203 | }), | |
| 204 | ); | |
| 205 | ||
| 206 | it.effect("reports respawn for a foreground npx artifact on darwin", () => | |
| 207 | Effect.gen(function* () { | |
| 208 | const { home } = yield* makeHome(); | |
| 209 | const method = yield* SelfUpdate.resolveServerSelfUpdateCapability({ | |
| 210 | desktopManaged: false, | |
| 211 | }).pipe( | |
| 212 | provideHostRefs({ | |
| 213 | platform: "darwin", | |
| 214 | env: { HOME: home }, | |
| 215 | entryPath: `${home}/.npm/_npx/abc123/node_modules/t3/dist/bin.mjs`, | |
| 216 | }), | |
| 217 | ); | |
| 218 | assert.equal(method, "respawn"); | |
| 219 | }), | |
| 220 | ); | |
| 221 | ||
| 222 | it.effect("reports desktop-managed for desktop-supervised backends", () => | |
| 223 | Effect.gen(function* () { | |
| 224 | const { home, path } = yield* makeHome(); | |
| 225 | // Desktop ownership wins over every process-shape heuristic: even a | |
| 226 | // systemd-looking pinned artifact belongs to the app that spawned it. | |
| 227 | const entryPath = path.join(home, ".t3/runtime/versions/0.0.28/node_modules/t3/dist/bin.mjs"); | |
| 228 | yield* writeUnitReferencing(home, entryPath); | |
| 229 | const method = yield* SelfUpdate.resolveServerSelfUpdateCapability({ | |
| 230 | desktopManaged: true, | |
| 231 | }).pipe( | |
| 232 | provideHostRefs({ | |
| 233 | platform: "linux", | |
| 234 | env: { | |
| 235 | HOME: home, | |
| 236 | INVOCATION_ID: "abc123", | |
| 237 | [BOOT_SERVICE_UNIT_ENV]: BOOT_SERVICE_UNIT_FILE, | |
| 238 | }, | |
| 239 | entryPath, | |
| 240 | }), | |
| 241 | ); | |
| 242 | assert.equal(method, "desktop-managed"); | |
| 243 | }), | |
| 244 | ); | |
| 245 | ||
| 246 | it.effect("reports no method for dev checkouts and Windows", () => | |
| 247 | Effect.gen(function* () { | |
| 248 | const { home } = yield* makeHome(); | |
| 249 | const devMethod = yield* SelfUpdate.resolveServerSelfUpdateCapability({ | |
| 250 | desktopManaged: false, | |
| 251 | }).pipe( | |
| 252 | provideHostRefs({ | |
| 253 | platform: "darwin", | |
| 254 | env: { HOME: home }, | |
| 255 | entryPath: `${home}/dev/t3/apps/server/dist/bin.mjs`, | |
| 256 | }), | |
| 257 | ); | |
| 258 | assert.isNull(devMethod); | |
| 259 | const windowsMethod = yield* SelfUpdate.resolveServerSelfUpdateCapability({ | |
| 260 | desktopManaged: false, | |
| 261 | }).pipe( | |
| 262 | provideHostRefs({ | |
| 263 | platform: "win32", | |
| 264 | env: { HOME: home }, | |
| 265 | entryPath: "C:\\Users\\theo\\AppData\\Roaming\\npm\\node_modules\\t3\\dist\\bin.mjs", | |
| 266 | }), | |
| 267 | ); | |
| 268 | assert.isNull(windowsMethod); | |
| 269 | }), | |
| 270 | ); | |
| 271 | }); | |
| 272 | ||
| 273 | it.layer(NodeServices.layer)("ServerSelfUpdate.update", (it) => { | |
| 274 | interface RecordedSpawn { | |
| 275 | readonly command: string; | |
| 276 | readonly args: ReadonlyArray<string>; | |
| 277 | } | |
| 278 | ||
| 279 | const makeContext = Effect.fn("test.makeContext")(function* (options?: { | |
| 280 | readonly platform?: NodeJS.Platform; | |
| 281 | readonly bootService?: boolean; | |
| 282 | readonly desktopManaged?: boolean; | |
| 283 | readonly entryPath?: string; | |
| 284 | readonly failWhen?: (command: string, args: ReadonlyArray<string>) => boolean; | |
| 285 | readonly stdoutFor?: (command: string, args: ReadonlyArray<string>) => string | undefined; | |
| 286 | readonly failSpawn?: boolean; | |
| 287 | }) { | |
| 288 | const fs = yield* FileSystem.FileSystem; | |
| 289 | const path = yield* Path.Path; | |
| 290 | const home = yield* fs.makeTempDirectoryScoped({ prefix: "t3-self-update-test-" }); | |
| 291 | const baseDir = path.join(home, ".t3"); | |
| 292 | const entryPath = | |
| 293 | options?.entryPath ?? | |
| 294 | path.join(home, ".t3/runtime/versions/0.0.28/node_modules/t3/dist/bin.mjs"); | |
| 295 | const env: NodeJS.ProcessEnv = | |
| 296 | options?.bootService === true | |
| 297 | ? { | |
| 298 | HOME: home, | |
| 299 | INVOCATION_ID: "abc123", | |
| 300 | [BOOT_SERVICE_UNIT_ENV]: BOOT_SERVICE_UNIT_FILE, | |
| 301 | } | |
| 302 | : { HOME: home }; | |
| 303 | if (options?.bootService === true) { | |
| 304 | const unitDir = path.join(home, ".config", "systemd", "user"); | |
| 305 | yield* fs.makeDirectory(unitDir, { recursive: true }); | |
| 306 | yield* fs.writeFileString( | |
| 307 | path.join(unitDir, "t3code.service"), | |
| 308 | renderBootServiceUnit({ | |
| 309 | nodePath: NODE_PATH, | |
| 310 | t3EntryPath: entryPath, | |
| 311 | baseDir, | |
| 312 | logPath: path.join(baseDir, "userdata", "logs", "boot-service.log"), | |
| 313 | unitPath: path.join(unitDir, "t3code.service"), | |
| 314 | }), | |
| 315 | ); | |
| 316 | } | |
| 317 | ||
| 318 | const commands: Array<RecordedCommand> = []; | |
| 319 | const spawns: Array<RecordedSpawn> = []; | |
| 320 | let exited = 0; | |
| 321 | // layerTest always reports mode "web"; desktop-managed contexts overlay | |
| 322 | // the mode the desktop app's bootstrap envelope would set. | |
| 323 | const configLayer = | |
| 324 | options?.desktopManaged === true | |
| 325 | ? Layer.effect( | |
| 326 | ServerConfig.ServerConfig, | |
| 327 | Effect.gen(function* () { | |
| 328 | const config = yield* ServerConfig.ServerConfig; | |
| 329 | return { ...config, mode: "desktop" as const }; | |
| 330 | }), | |
| 331 | ).pipe(Layer.provide(ServerConfig.layerTest(home, baseDir))) | |
| 332 | : ServerConfig.layerTest(home, baseDir); | |
| 333 | const service = yield* SelfUpdate.make({ | |
| 334 | host: { | |
| 335 | spawnDetached: (command, args) => | |
| 336 | Effect.sync(() => spawns.push({ command, args })).pipe( | |
| 337 | Effect.andThen( | |
| 338 | options?.failSpawn === true | |
| 339 | ? Effect.fail( | |
| 340 | new ProcessRunner.ProcessSpawnError({ | |
| 341 | command, | |
| 342 | argumentCount: args.length, | |
| 343 | cause: new Error("detached spawn failed"), | |
| 344 | }), | |
| 345 | ) | |
| 346 | : Effect.void, | |
| 347 | ), | |
| 348 | ), | |
| 349 | exitProcess: () => { | |
| 350 | exited += 1; | |
| 351 | }, | |
| 352 | }, | |
| 353 | }).pipe( | |
| 354 | Effect.provide( | |
| 355 | Layer.mergeAll( | |
| 356 | makeRecordingRunnerLayer(commands, { | |
| 357 | failWhen: options?.failWhen, | |
| 358 | stdoutFor: options?.stdoutFor, | |
| 359 | }), | |
| 360 | configLayer, | |
| 361 | ), | |
| 362 | ), | |
| 363 | provideHostRefs({ platform: options?.platform ?? "linux", env, entryPath }), | |
| 364 | ); | |
| 365 | return { | |
| 366 | fs, | |
| 367 | path, | |
| 368 | home, | |
| 369 | baseDir, | |
| 370 | entryPath, | |
| 371 | commands, | |
| 372 | spawns, | |
| 373 | exitCount: () => exited, | |
| 374 | service, | |
| 375 | }; | |
| 65 | const launcher = ServiceLauncherClient.ServiceLauncherClient.of({ | |
| 66 | managed: options.managed ?? true, | |
| 67 | trial: false, | |
| 68 | requestUpdate: | |
| 69 | options.requestUpdate ?? | |
| 70 | (() => | |
| 71 | Effect.sync(() => { | |
| 72 | order.push("accept"); | |
| 73 | return "launcher-id"; | |
| 74 | })), | |
| 75 | prepareTrial: Effect.sync((): undefined => undefined), | |
| 376 | 76 | }); |
| 377 | ||
| 378 | it.effect("rejects dist-tags and other non-exact versions", () => | |
| 379 | Effect.gen(function* () { | |
| 380 | const context = yield* makeContext(); | |
| 381 | const error = yield* context.service.update({ targetVersion: "latest" }).pipe(Effect.flip); | |
| 382 | assert.include(error.reason, "not an exact t3 version"); | |
| 383 | assert.lengthOf(context.commands, 0); | |
| 384 | }), | |
| 385 | ); | |
| 386 | ||
| 387 | it.effect("refuses to update a desktop-managed backend and points at the app", () => | |
| 388 | Effect.gen(function* () { | |
| 389 | const context = yield* makeContext({ desktopManaged: true, bootService: true }); | |
| 390 | const error = yield* context.service.update({ targetVersion: "0.0.29" }).pipe(Effect.flip); | |
| 391 | assert.include(error.reason, "desktop app"); | |
| 392 | assert.lengthOf(context.commands, 0); | |
| 393 | assert.lengthOf(context.spawns, 0); | |
| 394 | }), | |
| 395 | ); | |
| 396 | ||
| 397 | it.effect("fails without touching anything when no update method applies", () => | |
| 398 | Effect.gen(function* () { | |
| 399 | const context = yield* makeContext({ | |
| 400 | entryPath: "/home/theo/dev/t3/apps/server/dist/bin.mjs", | |
| 401 | }); | |
| 402 | const error = yield* context.service.update({ targetVersion: "0.0.29" }).pipe(Effect.flip); | |
| 403 | assert.include(error.reason, "cannot update itself"); | |
| 404 | assert.lengthOf(context.commands, 0); | |
| 405 | }), | |
| 406 | ); | |
| 407 | ||
| 408 | it.effect("surfaces a failed npm install and never schedules a restart", () => | |
| 409 | Effect.gen(function* () { | |
| 410 | const context = yield* makeContext({ failWhen: (command) => command === "npm" }); | |
| 411 | const error = yield* context.service.update({ targetVersion: "0.0.29" }).pipe(Effect.flip); | |
| 412 | assert.equal(error.reason, "Could not install the requested t3 version."); | |
| 413 | yield* TestClock.adjust(Duration.seconds(10)); | |
| 414 | assert.lengthOf(context.spawns, 0); | |
| 415 | assert.equal(context.exitCount(), 0); | |
| 416 | }).pipe(Effect.provide(TestClock.layer())), | |
| 77 | const config = yield* ServerConfig.ServerConfig.pipe( | |
| 78 | Effect.provide(ServerConfig.layerTest(process.cwd(), baseDir)), | |
| 417 | 79 | ); |
| 418 | ||
| 419 | it.effect("reinstalls the same version after a failed preflight", () => | |
| 420 | Effect.gen(function* () { | |
| 421 | let preflightAttempts = 0; | |
| 422 | const context = yield* makeContext({ | |
| 423 | failWhen: (command) => { | |
| 424 | if (command !== NODE_PATH) return false; | |
| 425 | preflightAttempts += 1; | |
| 426 | return preflightAttempts === 1; | |
| 427 | }, | |
| 428 | }); | |
| 429 | const versionDir = context.path.join(context.baseDir, "runtime", "versions", "0.0.29"); | |
| 430 | const entryPath = context.path.join(versionDir, "node_modules", "t3", "dist", "bin.mjs"); | |
| 431 | yield* context.fs.makeDirectory(context.path.dirname(entryPath), { recursive: true }); | |
| 432 | yield* context.fs.writeFileString(entryPath, "export {};\n"); | |
| 433 | yield* context.fs.writeFileString( | |
| 434 | context.path.join(versionDir, ".install-complete"), | |
| 435 | "0.0.29\n", | |
| 436 | ); | |
| 437 | ||
| 438 | const firstError = yield* context.service | |
| 439 | .update({ targetVersion: "0.0.29" }) | |
| 440 | .pipe(Effect.flip); | |
| 441 | assert.include(firstError.reason, "failed its version check"); | |
| 442 | assert.isFalse(yield* context.fs.exists(versionDir)); | |
| 443 | ||
| 444 | const result = yield* context.service.update({ targetVersion: "0.0.29" }); | |
| 445 | assert.deepEqual(result, { targetVersion: "0.0.29", method: "respawn" }); | |
| 446 | assert.deepEqual( | |
| 447 | context.commands.map((entry) => entry.command), | |
| 448 | [NODE_PATH, "npm", NODE_PATH], | |
| 449 | ); | |
| 450 | }).pipe(Effect.provide(TestClock.layer())), | |
| 80 | const selfUpdate = yield* ServerSelfUpdate.make().pipe( | |
| 81 | Effect.provideService(ProcessRunner.ProcessRunner, runner), | |
| 82 | Effect.provideService(ServiceLauncherClient.ServiceLauncherClient, launcher), | |
| 83 | Effect.provideService(HostProcessExecutablePath, "/usr/bin/node"), | |
| 84 | Effect.provide(ServerConfig.layer({ ...config, mode: options.mode ?? "web" })), | |
| 451 | 85 | ); |
| 86 | return { selfUpdate, order }; | |
| 87 | }); | |
| 452 | 88 | |
| 453 | it.effect("rejects and removes an installed runtime that reports the wrong version", () => | |
| 89 | it.layer(NodeServices.layer)("server self update", (it) => { | |
| 90 | it.effect("stages and preflights before asking the launcher for an update ID", () => | |
| 454 | 91 | Effect.gen(function* () { |
| 455 | const context = yield* makeContext({ | |
| 456 | stdoutFor: (command, args) => | |
| 457 | command === NODE_PATH && args[1] === "--version" ? "t3 v0.0.28\n" : undefined, | |
| 92 | const { selfUpdate, order } = yield* makeHarness(); | |
| 93 | expect(yield* selfUpdate.update({ targetVersion: "1.1.0" })).toEqual({ | |
| 94 | targetVersion: "1.1.0", | |
| 95 | method: "boot-service", | |
| 96 | updateId: "launcher-id", | |
| 458 | 97 | }); |
| 459 | const versionDir = context.path.join(context.baseDir, "runtime", "versions", "0.0.29"); | |
| 460 | ||
| 461 | const error = yield* context.service.update({ targetVersion: "0.0.29" }).pipe(Effect.flip); | |
| 462 | ||
| 463 | assert.include(error.reason, "did not report the requested"); | |
| 464 | assert.isFalse(yield* context.fs.exists(versionDir)); | |
| 465 | assert.lengthOf(context.spawns, 0); | |
| 98 | expect(order).toEqual(["install", "preflight", "accept"]); | |
| 466 | 99 | }), |
| 467 | 100 | ); |
| 468 | 101 | |
| 469 | it.effect("reports a detached replacement spawn failure and leaves updates retryable", () => | |
| 102 | it.effect("rejects invalid versions and desktop-managed servers before staging", () => | |
| 470 | 103 | Effect.gen(function* () { |
| 471 | const context = yield* makeContext({ failSpawn: true }); | |
| 472 | ||
| 473 | const first = yield* context.service.update({ targetVersion: "0.0.29" }).pipe(Effect.flip); | |
| 474 | assert.include(first.reason, "Could not start the replacement"); | |
| 475 | ||
| 476 | const second = yield* context.service.update({ targetVersion: "0.0.29" }).pipe(Effect.flip); | |
| 477 | assert.include(second.reason, "Could not start the replacement"); | |
| 478 | assert.notInclude(second.reason, "already in progress"); | |
| 479 | assert.lengthOf(context.spawns, 2); | |
| 480 | assert.equal(context.exitCount(), 0); | |
| 104 | const web = yield* makeHarness(); | |
| 105 | expect( | |
| 106 | (yield* web.selfUpdate.update({ targetVersion: "latest" }).pipe(Effect.flip)).reason, | |
| 107 | ).toBe("'latest' is not an exact t3 version."); | |
| 108 | const desktop = yield* makeHarness({ mode: "desktop" }); | |
| 109 | expect( | |
| 110 | (yield* desktop.selfUpdate.update({ targetVersion: "1.1.0" }).pipe(Effect.flip)).reason, | |
| 111 | ).toContain("desktop app"); | |
| 112 | expect([...web.order, ...desktop.order]).toEqual([]); | |
| 481 | 113 | }), |
| 482 | 114 | ); |
| 483 | 115 | |
| 484 | it.effect("installs, preflights, and respawns a foreground server", () => | |
| 116 | it.effect("preserves the preflight refusal reason", () => | |
| 485 | 117 | Effect.gen(function* () { |
| 486 | const context = yield* makeContext(); | |
| 487 | const progress: Array<string> = []; | |
| 488 | const result = yield* context.service.update({ targetVersion: "0.0.29" }, (stage) => | |
| 489 | Effect.sync(() => progress.push(stage)), | |
| 118 | const { selfUpdate } = yield* makeHarness({ preflight: "blocked" }); | |
| 119 | expect((yield* selfUpdate.update({ targetVersion: "1.1.0" }).pipe(Effect.flip)).reason).toBe( | |
| 120 | "local update required", | |
| 490 | 121 | ); |
| 491 | assert.deepEqual(result, { targetVersion: "0.0.29", method: "respawn" }); | |
| 492 | assert.deepEqual(progress, ["downloading", "installing"]); | |
| 493 | assert.lengthOf(context.spawns, 1); | |
| 494 | ||
| 495 | const concurrentError = yield* context.service | |
| 496 | .update({ targetVersion: "0.0.30" }) | |
| 497 | .pipe(Effect.flip); | |
| 498 | assert.include(concurrentError.reason, "already in progress"); | |
| 499 | ||
| 500 | const pinnedEntry = context.path.join( | |
| 501 | context.baseDir, | |
| 502 | "runtime/versions/0.0.29/node_modules/t3/dist/bin.mjs", | |
| 503 | ); | |
| 504 | assert.deepEqual( | |
| 505 | context.commands.map((entry) => [entry.command, ...entry.args].join(" ")), | |
| 506 | [ | |
| 507 | `npm install --prefix ${context.path.join(context.baseDir, "runtime/versions/0.0.29")} --no-fund --no-audit t3@0.0.29`, | |
| 508 | `${NODE_PATH} ${pinnedEntry} --version`, | |
| 509 | ], | |
| 510 | ); | |
| 511 | ||
| 512 | // The restart is deferred so the RPC acknowledgement flushes first. | |
| 513 | yield* TestClock.adjust(Duration.seconds(10)); | |
| 514 | assert.lengthOf(context.spawns, 1); | |
| 515 | const spawn = context.spawns[0]; | |
| 516 | assert.equal(spawn?.command, "/bin/sh"); | |
| 517 | assert.include(spawn?.args ?? [], pinnedEntry); | |
| 518 | // The replacement replays the original CLI arguments. | |
| 519 | assert.include(spawn?.args ?? [], "serve"); | |
| 520 | assert.equal(context.exitCount(), 1); | |
| 521 | }).pipe(Effect.provide(TestClock.layer())), | |
| 522 | ); | |
| 523 | ||
| 524 | it.effect("rewrites the systemd unit and restarts the boot service", () => | |
| 525 | Effect.gen(function* () { | |
| 526 | const context = yield* makeContext({ bootService: true }); | |
| 527 | const result = yield* context.service.update({ targetVersion: "0.0.29" }); | |
| 528 | assert.deepEqual(result, { targetVersion: "0.0.29", method: "boot-service" }); | |
| 529 | ||
| 530 | const pinnedEntry = context.path.join( | |
| 531 | context.baseDir, | |
| 532 | "runtime/versions/0.0.29/node_modules/t3/dist/bin.mjs", | |
| 533 | ); | |
| 534 | const unit = yield* context.fs.readFileString( | |
| 535 | context.path.join(context.home, ".config", "systemd", "user", "t3code.service"), | |
| 536 | ); | |
| 537 | assert.include(unit, `ExecStart=${NODE_PATH} ${pinnedEntry} serve`); | |
| 538 | assert.deepEqual( | |
| 539 | context.commands.map((entry) => entry.command), | |
| 540 | ["npm", NODE_PATH, "systemctl"], | |
| 541 | ); | |
| 542 | assert.deepEqual(context.commands[2]?.args, ["--user", "daemon-reload"]); | |
| 543 | ||
| 544 | // Restart waits until after the update acknowledgement can flush. | |
| 545 | yield* TestClock.adjust(Duration.seconds(10)); | |
| 546 | assert.deepEqual(context.commands[3], { | |
| 547 | command: "systemctl", | |
| 548 | args: ["--user", "restart", "--no-block", "t3code.service"], | |
| 549 | }); | |
| 550 | assert.lengthOf(context.spawns, 0); | |
| 551 | // systemd replaces the process; the server must not exit itself. | |
| 552 | assert.equal(context.exitCount(), 0); | |
| 553 | ||
| 554 | // The queued restart returns while this process is still shutting | |
| 555 | // down; the lock must stay held so a second update cannot rewrite the | |
| 556 | // unit mid-teardown. | |
| 557 | const concurrentError = yield* context.service | |
| 558 | .update({ targetVersion: "0.0.30" }) | |
| 559 | .pipe(Effect.flip); | |
| 560 | assert.include(concurrentError.reason, "already in progress"); | |
| 561 | }).pipe(Effect.provide(TestClock.layer())), | |
| 122 | }), | |
| 562 | 123 | ); |
| 563 | 124 | |
| 564 | it.effect("restores the previous unit and permits a retry when systemd restart fails", () => | |
| 125 | it.effect("allows only one update at a time", () => | |
| 565 | 126 | Effect.gen(function* () { |
| 566 | let failRestart = true; | |
| 567 | const context = yield* makeContext({ | |
| 568 | bootService: true, | |
| 569 | failWhen: (command, args) => { | |
| 570 | if (command !== "systemctl" || args[1] !== "restart" || !failRestart) { | |
| 571 | return false; | |
| 572 | } | |
| 573 | failRestart = false; | |
| 574 | return true; | |
| 575 | }, | |
| 127 | const requested = yield* Deferred.make<void>(); | |
| 128 | const accepted = yield* Deferred.make<string>(); | |
| 129 | const { selfUpdate } = yield* makeHarness({ | |
| 130 | requestUpdate: () => | |
| 131 | Deferred.succeed(requested, undefined).pipe(Effect.andThen(Deferred.await(accepted))), | |
| 576 | 132 | }); |
| 577 | const unitPath = context.path.join( | |
| 578 | context.home, | |
| 579 | ".config", | |
| 580 | "systemd", | |
| 581 | "user", | |
| 582 | BOOT_SERVICE_UNIT_FILE, | |
| 583 | ); | |
| 584 | const previousUnit = yield* context.fs.readFileString(unitPath); | |
| 585 | ||
| 586 | const first = yield* context.service.update({ targetVersion: "0.0.29" }); | |
| 587 | assert.deepEqual(first, { targetVersion: "0.0.29", method: "boot-service" }); | |
| 588 | yield* TestClock.adjust(Duration.seconds(10)); | |
| 589 | yield* eventuallyFileString(unitPath, previousUnit); | |
| 590 | yield* eventuallyTrue(() => context.commands.at(-1)?.args[1] === "daemon-reload"); | |
| 591 | assert.deepEqual( | |
| 592 | context.commands.slice(-2).map((entry) => entry.args), | |
| 593 | [ | |
| 594 | ["--user", "restart", "--no-block", BOOT_SERVICE_UNIT_FILE], | |
| 595 | ["--user", "daemon-reload"], | |
| 596 | ], | |
| 597 | ); | |
| 598 | ||
| 599 | const retry = yield* context.service.update({ targetVersion: "0.0.30" }); | |
| 600 | assert.deepEqual(retry, { targetVersion: "0.0.30", method: "boot-service" }); | |
| 601 | }).pipe(Effect.provide(TestClock.layer())), | |
| 602 | ); | |
| 603 | ||
| 604 | it.effect("restores the previous systemd unit when daemon-reload fails", () => | |
| 605 | Effect.gen(function* () { | |
| 606 | const context = yield* makeContext({ | |
| 607 | bootService: true, | |
| 608 | failWhen: (command) => command === "systemctl", | |
| 133 | const first = yield* Effect.forkChild(selfUpdate.update({ targetVersion: "1.1.0" }), { | |
| 134 | startImmediately: true, | |
| 609 | 135 | }); |
| 610 | const unitPath = context.path.join( | |
| 611 | context.home, | |
| 612 | ".config", | |
| 613 | "systemd", | |
| 614 | "user", | |
| 615 | BOOT_SERVICE_UNIT_FILE, | |
| 616 | ); | |
| 617 | const previousUnit = yield* context.fs.readFileString(unitPath); | |
| 618 | ||
| 619 | const error = yield* context.service.update({ targetVersion: "0.0.29" }).pipe(Effect.flip); | |
| 620 | assert.include(error.reason, "Reloading systemd units failed"); | |
| 621 | assert.equal(yield* context.fs.readFileString(unitPath), previousUnit); | |
| 622 | assert.deepEqual( | |
| 623 | context.commands.map((entry) => entry.command), | |
| 624 | ["npm", NODE_PATH, "systemctl", "systemctl"], | |
| 136 | yield* Deferred.await(requested); | |
| 137 | expect((yield* selfUpdate.update({ targetVersion: "1.1.1" }).pipe(Effect.flip)).reason).toBe( | |
| 138 | "A server update is already in progress.", | |
| 625 | 139 | ); |
| 626 | ||
| 627 | yield* TestClock.adjust(Duration.seconds(10)); | |
| 628 | assert.lengthOf(context.spawns, 0); | |
| 629 | assert.equal(context.exitCount(), 0); | |
| 630 | }).pipe(Effect.provide(TestClock.layer())), | |
| 140 | yield* Deferred.succeed(accepted, "launcher-id"); | |
| 141 | expect((yield* Fiber.join(first)).updateId).toBe("launcher-id"); | |
| 142 | }), | |
| 631 | 143 | ); |
| 632 | 144 | }); |
apps/server/src/cloud/selfUpdate.ts+113-364
| ... | ... | @@ -1,7 +1,3 @@ |
| 1 | // @effect-diagnostics nodeBuiltinImport:off | |
| 2 | // node:child_process directly: the foreground-server replacement must be a | |
| 3 | // detached fire-and-forget child that outlives this process, while Effect's | |
| 4 | // ChildProcessSpawner ties every child to a scope that kills it. | |
| 5 | 1 | import { |
| 6 | 2 | ServerSelfUpdateError, |
| 7 | 3 | type ServerSelfUpdateCapability, |
| ... | ... | @@ -9,13 +5,7 @@ import { |
| 9 | 5 | type ServerSelfUpdateProgressStage, |
| 10 | 6 | type ServerSelfUpdateResult, |
| 11 | 7 | } from "@t3tools/contracts"; |
| 12 | import { | |
| 13 | HostProcessArguments, | |
| 14 | HostProcessEnvironment, | |
| 15 | HostProcessExecutablePath, | |
| 16 | HostProcessPlatform, | |
| 17 | } from "@t3tools/shared/hostProcess"; | |
| 18 | import * as NodeChildProcess from "node:child_process"; | |
| 8 | import { HostProcessExecutablePath } from "@t3tools/shared/hostProcess"; | |
| 19 | 9 | import * as Context from "effect/Context"; |
| 20 | 10 | import * as Duration from "effect/Duration"; |
| 21 | 11 | import * as Effect from "effect/Effect"; |
| ... | ... | @@ -25,208 +15,52 @@ import * as Path from "effect/Path"; |
| 25 | 15 | import * as Ref from "effect/Ref"; |
| 26 | 16 | |
| 27 | 17 | import * as ServerConfig from "../config.ts"; |
| 28 | import { writeFileStringAtomically } from "../atomicWrite.ts"; | |
| 29 | 18 | import * as ProcessRunner from "../processRunner.ts"; |
| 30 | 19 | import { |
| 31 | BOOT_SERVICE_UNIT_ENV, | |
| 32 | BOOT_SERVICE_UNIT_FILE, | |
| 33 | quoteSystemdValue, | |
| 34 | renderBootServiceUnit, | |
| 35 | } from "./bootService.ts"; | |
| 36 | import { ensurePinnedRuntimeInstalled, removePinnedRuntimeInstallation } from "./pinnedRuntime.ts"; | |
| 37 | ||
| 38 | /** | |
| 39 | * Lets a connected client replace this server with another published `t3` | |
| 40 | * version over RPC — the only update path that works when the user is not at | |
| 41 | * the machine (phone against a home server, relay-managed box). The target | |
| 42 | * version is npm-installed into the pinned runtime and verified before | |
| 43 | * anything restarts, so a failed install leaves the running server untouched. | |
| 44 | */ | |
| 20 | ensurePinnedRuntimeInstalled, | |
| 21 | PinnedRuntimeInstallError, | |
| 22 | PinnedRuntimePreflightBlockedError, | |
| 23 | } from "./pinnedRuntime.ts"; | |
| 24 | import { decodeServicePreflightResult } from "./servicePreflight.ts"; | |
| 25 | import * as ServiceLauncherClient from "./serviceLauncherClient.ts"; | |
| 26 | import { isExactServiceVersion, SERVICE_LAUNCHER_PROTOCOL } from "./serviceProtocol.ts"; | |
| 45 | 27 | |
| 46 | 28 | const PREFLIGHT_TIMEOUT = Duration.seconds(30); |
| 47 | /** Grace between acknowledging the RPC and killing the process, so the | |
| 48 | response (and its relay hop) flushes before the socket drops. */ | |
| 49 | const RESTART_DELAY = Duration.seconds(2); | |
| 50 | ||
| 51 | /** Exact npm versions only — never dist-tags — so the acknowledgement names | |
| 52 | the version that was actually installed. Also keeps the value safe to | |
| 53 | pass to npm and embed in filesystem paths. */ | |
| 54 | const EXACT_VERSION_PATTERN = /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$/; | |
| 55 | ||
| 56 | export interface ServerSelfUpdateHost { | |
| 57 | readonly execPath: string; | |
| 58 | readonly cliEntryPath: string; | |
| 59 | /** Original CLI arguments after the entry path, replayed on respawn. */ | |
| 60 | readonly cliArgs: ReadonlyArray<string>; | |
| 61 | /** Resolves once the foreground replacement process has actually spawned. */ | |
| 62 | readonly spawnDetached: ( | |
| 63 | command: string, | |
| 64 | args: ReadonlyArray<string>, | |
| 65 | ) => Effect.Effect<void, ProcessRunner.ProcessSpawnError>; | |
| 66 | readonly exitProcess: () => void; | |
| 67 | } | |
| 68 | ||
| 69 | function normalizeEntryPath(entryPath: string): string { | |
| 70 | return entryPath.replaceAll("\\", "/"); | |
| 71 | } | |
| 72 | 29 | |
| 73 | /** | |
| 74 | * Only a published npm artifact can be swapped for another version: dev | |
| 75 | * checkouts (apps/server/dist) and the desktop app's bundled backend have no | |
| 76 | * npm identity, and the desktop manages its own updates. | |
| 77 | */ | |
| 78 | export function isPublishedCliEntry(entryPath: string): boolean { | |
| 79 | return normalizeEntryPath(entryPath).includes("/node_modules/t3/dist/"); | |
| 80 | } | |
| 81 | ||
| 82 | /** | |
| 83 | * The update path this process can offer, or null when only a manual | |
| 84 | * relaunch works. "desktop-managed" — the T3 Code desktop app spawned this | |
| 85 | * backend and owns its version; only updating the app updates it. | |
| 86 | * "boot-service" — this is the systemd-supervised process from | |
| 87 | * bootService.ts: rewrite the unit and let systemd swap it. "respawn" — a | |
| 88 | * foreground POSIX process running a published artifact: replace it with a | |
| 89 | * detached child. Windows foreground runs are unsupported for now (no | |
| 90 | * equivalent of the detach-and-exec handoff below). | |
| 91 | */ | |
| 92 | export const resolveServerSelfUpdateCapability = Effect.fn( | |
| 93 | "cloud.server_self_update.resolve_capability", | |
| 94 | )(function* (input: { | |
| 95 | /** True when the desktop app supervises this backend (mode "desktop"). */ | |
| 30 | export function resolveServerSelfUpdateCapability(input: { | |
| 96 | 31 | readonly desktopManaged: boolean; |
| 97 | }) { | |
| 98 | if (input.desktopManaged) { | |
| 99 | return "desktop-managed" as const; | |
| 100 | } | |
| 101 | ||
| 102 | const platform = yield* HostProcessPlatform; | |
| 103 | const env = yield* HostProcessEnvironment; | |
| 104 | const hostArguments = yield* HostProcessArguments; | |
| 105 | const fs = yield* FileSystem.FileSystem; | |
| 106 | const path = yield* Path.Path; | |
| 107 | ||
| 108 | const entryPath = hostArguments[1] ?? ""; | |
| 109 | if (entryPath === "") { | |
| 110 | return null; | |
| 111 | } | |
| 112 | ||
| 113 | const homeDir = env.HOME ?? ""; | |
| 114 | if (platform === "linux" && homeDir !== "") { | |
| 115 | const unitPath = path.join(homeDir, ".config", "systemd", "user", BOOT_SERVICE_UNIT_FILE); | |
| 116 | const unitReferencesEntry = yield* fs.readFileString(unitPath).pipe( | |
| 117 | Effect.map((unit) => unit.includes(quoteSystemdValue(entryPath))), | |
| 118 | Effect.orElseSucceed(() => false), | |
| 119 | ); | |
| 120 | // INVOCATION_ID only proves that some systemd unit launched us. The | |
| 121 | // explicit marker written into t3code.service identifies this unit as the | |
| 122 | // supervisor that will replace the current process when restarted. | |
| 123 | if ( | |
| 124 | unitReferencesEntry && | |
| 125 | (env.INVOCATION_ID ?? "") !== "" && | |
| 126 | env[BOOT_SERVICE_UNIT_ENV] === BOOT_SERVICE_UNIT_FILE | |
| 127 | ) { | |
| 128 | return "boot-service" as const; | |
| 129 | } | |
| 130 | ||
| 131 | // A process owned by another (or a legacy unmarked) systemd unit must not | |
| 132 | // use the foreground respawn path: Restart=always could otherwise launch | |
| 133 | // the old unit beside the detached replacement. | |
| 134 | if ((env.INVOCATION_ID ?? "") !== "") { | |
| 135 | return null; | |
| 136 | } | |
| 137 | } | |
| 138 | ||
| 139 | if ((platform === "linux" || platform === "darwin") && isPublishedCliEntry(entryPath)) { | |
| 140 | return "respawn" as const; | |
| 141 | } | |
| 142 | ||
| 143 | return null; | |
| 144 | }); | |
| 32 | readonly launcherManaged: boolean; | |
| 33 | }): ServerSelfUpdateCapability | null { | |
| 34 | if (input.desktopManaged) return "desktop-managed" as const; | |
| 35 | return input.launcherManaged ? ("boot-service" as const) : null; | |
| 36 | } | |
| 145 | 37 | |
| 146 | 38 | export class ServerSelfUpdate extends Context.Service< |
| 147 | 39 | ServerSelfUpdate, |
| 148 | 40 | { |
| 149 | 41 | readonly update: ( |
| 150 | 42 | input: ServerSelfUpdateInput, |
| 151 | reportProgress?: (stage: ServerSelfUpdateProgressStage) => Effect.Effect<void, never, never>, | |
| 43 | reportProgress?: (stage: ServerSelfUpdateProgressStage) => Effect.Effect<void>, | |
| 152 | 44 | ) => Effect.Effect<ServerSelfUpdateResult, ServerSelfUpdateError>; |
| 153 | 45 | } |
| 154 | 46 | >()("t3/cloud/selfUpdate/ServerSelfUpdate") {} |
| 155 | 47 | |
| 156 | export const make = Effect.fn("cloud.server_self_update.make")(function* (options?: { | |
| 157 | readonly host?: Partial<ServerSelfUpdateHost>; | |
| 158 | }) { | |
| 48 | export const make = Effect.fn("cloud.server_self_update.make")(function* () { | |
| 159 | 49 | const serverConfig = yield* ServerConfig.ServerConfig; |
| 50 | const launcher = yield* ServiceLauncherClient.ServiceLauncherClient; | |
| 51 | const runner = yield* ProcessRunner.ProcessRunner; | |
| 160 | 52 | const fs = yield* FileSystem.FileSystem; |
| 161 | 53 | const path = yield* Path.Path; |
| 162 | const runner = yield* ProcessRunner.ProcessRunner; | |
| 163 | const env = yield* HostProcessEnvironment; | |
| 164 | const hostExecPath = yield* HostProcessExecutablePath; | |
| 165 | const hostArguments = yield* HostProcessArguments; | |
| 166 | const capability: ServerSelfUpdateCapability | null = yield* resolveServerSelfUpdateCapability({ | |
| 167 | desktopManaged: serverConfig.mode === "desktop", | |
| 168 | }); | |
| 169 | ||
| 170 | const host: ServerSelfUpdateHost = { | |
| 171 | execPath: options?.host?.execPath ?? hostExecPath, | |
| 172 | cliEntryPath: options?.host?.cliEntryPath ?? hostArguments[1] ?? "", | |
| 173 | cliArgs: options?.host?.cliArgs ?? hostArguments.slice(2), | |
| 174 | spawnDetached: | |
| 175 | options?.host?.spawnDetached ?? | |
| 176 | ((command, args) => | |
| 177 | Effect.callback<void, ProcessRunner.ProcessSpawnError>((resume) => { | |
| 178 | const spawnError = (cause: unknown) => | |
| 179 | new ProcessRunner.ProcessSpawnError({ | |
| 180 | command, | |
| 181 | argumentCount: args.length, | |
| 182 | cause, | |
| 183 | }); | |
| 184 | let child: NodeChildProcess.ChildProcess; | |
| 185 | try { | |
| 186 | child = NodeChildProcess.spawn(command, [...args], { | |
| 187 | detached: true, | |
| 188 | stdio: "ignore", | |
| 189 | }); | |
| 190 | } catch (cause) { | |
| 191 | resume(Effect.fail(spawnError(cause))); | |
| 192 | return; | |
| 193 | } | |
| 194 | ||
| 195 | const onSpawnError = (cause: Error) => resume(Effect.fail(spawnError(cause))); | |
| 196 | child.once("error", onSpawnError); | |
| 197 | child.once("spawn", () => { | |
| 198 | child.removeListener("error", onSpawnError); | |
| 199 | // Keep asynchronous child errors from becoming uncaught after the | |
| 200 | // successful spawn handoff has already been acknowledged. | |
| 201 | child.on("error", () => undefined); | |
| 202 | child.unref(); | |
| 203 | resume(Effect.void); | |
| 204 | }); | |
| 205 | })), | |
| 206 | exitProcess: options?.host?.exitProcess ?? (() => process.exit(0)), | |
| 207 | }; | |
| 208 | ||
| 54 | const execPath = yield* HostProcessExecutablePath; | |
| 209 | 55 | const inFlight = yield* Ref.make(false); |
| 210 | 56 | |
| 57 | const capability: ServerSelfUpdateCapability | null = | |
| 58 | serverConfig.mode === "desktop" ? "desktop-managed" : launcher.managed ? "boot-service" : null; | |
| 211 | 59 | const failWith = (reason: string, cause?: unknown) => |
| 212 | 60 | cause === undefined |
| 213 | 61 | ? new ServerSelfUpdateError({ reason }) |
| 214 | 62 | : new ServerSelfUpdateError({ reason, cause }); |
| 215 | 63 | |
| 216 | /** Deferred so the RPC acknowledgement flushes before the process dies. | |
| 217 | Detached from the request scope: the triggering connection is exactly | |
| 218 | what the restart tears down. */ | |
| 219 | const scheduleRestart = (restart: Effect.Effect<void>) => | |
| 220 | Effect.sleep(RESTART_DELAY).pipe( | |
| 221 | Effect.andThen(restart), | |
| 222 | Effect.forkDetach({ startImmediately: true }), | |
| 223 | ); | |
| 224 | const writeUnitAtomically = (filePath: string, contents: string) => | |
| 225 | writeFileStringAtomically({ filePath, contents }).pipe( | |
| 226 | Effect.provideService(FileSystem.FileSystem, fs), | |
| 227 | Effect.provideService(Path.Path, path), | |
| 228 | ); | |
| 229 | ||
| 230 | 64 | const update: ServerSelfUpdate["Service"]["update"] = Effect.fn( |
| 231 | 65 | "cloud.server_self_update.update", |
| 232 | 66 | )(function* (input, reportProgress = () => Effect.void) { |
| ... | ... | @@ -237,208 +71,123 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* (option |
| 237 | 71 | } |
| 238 | 72 | if (capability === null) { |
| 239 | 73 | return yield* failWith( |
| 240 | "This server cannot update itself; relaunch it manually with the new version.", | |
| 74 | "Remote updates require the T3 Code background service. Run `t3 service install` on the server machine.", | |
| 241 | 75 | ); |
| 242 | 76 | } |
| 243 | const activeMethod = capability; | |
| 77 | ||
| 244 | 78 | const targetVersion = input.targetVersion.trim(); |
| 245 | if (!EXACT_VERSION_PATTERN.test(targetVersion)) { | |
| 79 | if (!isExactServiceVersion(targetVersion)) { | |
| 246 | 80 | return yield* failWith(`'${targetVersion}' is not an exact t3 version.`); |
| 247 | 81 | } |
| 248 | ||
| 249 | const alreadyRunning = yield* Ref.getAndSet(inFlight, true); | |
| 250 | if (alreadyRunning) { | |
| 82 | if (yield* Ref.getAndSet(inFlight, true)) { | |
| 251 | 83 | return yield* failWith("A server update is already in progress."); |
| 252 | 84 | } |
| 253 | 85 | |
| 254 | 86 | return yield* Effect.gen(function* () { |
| 255 | 87 | yield* reportProgress("downloading"); |
| 256 | const runtimePaths = yield* ensurePinnedRuntimeInstalled({ | |
| 88 | const paths = yield* ensurePinnedRuntimeInstalled({ | |
| 257 | 89 | baseDir: serverConfig.baseDir, |
| 258 | 90 | version: targetVersion, |
| 259 | 91 | fs, |
| 260 | 92 | path, |
| 261 | 93 | runner, |
| 94 | validate: (runtime) => | |
| 95 | runner | |
| 96 | .run({ | |
| 97 | command: execPath, | |
| 98 | args: [ | |
| 99 | runtime.entryPath, | |
| 100 | "__service-preflight", | |
| 101 | "--database-path", | |
| 102 | serverConfig.dbPath, | |
| 103 | "--launcher-protocol", | |
| 104 | String(SERVICE_LAUNCHER_PROTOCOL), | |
| 105 | ], | |
| 106 | timeout: PREFLIGHT_TIMEOUT, | |
| 107 | }) | |
| 108 | .pipe( | |
| 109 | Effect.mapError( | |
| 110 | (cause) => | |
| 111 | new PinnedRuntimeInstallError({ | |
| 112 | step: "running the staged service preflight", | |
| 113 | cause, | |
| 114 | }), | |
| 115 | ), | |
| 116 | Effect.flatMap( | |
| 117 | ( | |
| 118 | result, | |
| 119 | ): Effect.Effect< | |
| 120 | void, | |
| 121 | PinnedRuntimeInstallError | PinnedRuntimePreflightBlockedError | |
| 122 | > => { | |
| 123 | if (result.code !== 0) { | |
| 124 | return Effect.fail( | |
| 125 | new PinnedRuntimeInstallError({ | |
| 126 | step: "running the staged service preflight", | |
| 127 | exitCode: Number(result.code), | |
| 128 | stdoutLength: result.stdout.length, | |
| 129 | stderrLength: result.stderr.length, | |
| 130 | }), | |
| 131 | ); | |
| 132 | } | |
| 133 | let parsed: unknown; | |
| 134 | try { | |
| 135 | parsed = JSON.parse(result.stdout.trim()); | |
| 136 | } catch (cause) { | |
| 137 | return Effect.fail( | |
| 138 | new PinnedRuntimeInstallError({ | |
| 139 | step: "decoding the staged service preflight", | |
| 140 | cause, | |
| 141 | }), | |
| 142 | ); | |
| 143 | } | |
| 144 | const preflight = decodeServicePreflightResult(parsed); | |
| 145 | if (preflight === undefined || preflight.version !== targetVersion) { | |
| 146 | return Effect.fail( | |
| 147 | new PinnedRuntimeInstallError({ | |
| 148 | step: "verifying the staged service preflight", | |
| 149 | }), | |
| 150 | ); | |
| 151 | } | |
| 152 | return preflight.status === "ready" | |
| 153 | ? Effect.void | |
| 154 | : Effect.fail( | |
| 155 | new PinnedRuntimePreflightBlockedError({ | |
| 156 | version: targetVersion, | |
| 157 | reason: preflight.reason, | |
| 158 | }), | |
| 159 | ); | |
| 160 | }, | |
| 161 | ), | |
| 162 | ), | |
| 262 | 163 | }).pipe( |
| 263 | Effect.mapError((error) => failWith("Could not install the requested t3 version.", error)), | |
| 164 | Effect.mapError((error) => | |
| 165 | error._tag === "PinnedRuntimePreflightBlockedError" | |
| 166 | ? failWith(error.reason, error) | |
| 167 | : failWith(`Could not prepare t3@${targetVersion}.`, error), | |
| 168 | ), | |
| 264 | 169 | ); |
| 265 | 170 | |
| 266 | 171 | yield* reportProgress("installing"); |
| 267 | // A broken artifact (failed native build, incompatible node) must be | |
| 268 | // caught while the current server is still alive to report it. | |
| 269 | const preflight = yield* runner | |
| 270 | .run({ | |
| 271 | command: host.execPath, | |
| 272 | args: [runtimePaths.entryPath, "--version"], | |
| 273 | timeout: PREFLIGHT_TIMEOUT, | |
| 274 | }) | |
| 172 | const updateId = yield* launcher | |
| 173 | .requestUpdate({ targetVersion }) | |
| 275 | 174 | .pipe( |
| 276 | Effect.mapError((cause) => | |
| 277 | failWith(`Could not verify the installed t3@${targetVersion}.`, cause), | |
| 278 | ), | |
| 279 | ); | |
| 280 | // Effect CLI's unstable formatVersion currently emits `${name} v${version}`. | |
| 281 | // Extract the version token so surrounding presentation changes do not break updates. | |
| 282 | const reportedVersion = /\bv(\S+)\s*$/.exec(preflight.stdout)?.[1]; | |
| 283 | if (preflight.code !== 0 || reportedVersion !== targetVersion) { | |
| 284 | // A completed npm install can still be unusable under this Node or on | |
| 285 | // this machine. Remove its sentinel and tree so a retry of the same | |
| 286 | // version performs a clean install instead of reusing a known-bad one. | |
| 287 | yield* removePinnedRuntimeInstallation({ | |
| 288 | baseDir: serverConfig.baseDir, | |
| 289 | version: targetVersion, | |
| 290 | fs, | |
| 291 | path, | |
| 292 | }).pipe( | |
| 293 | 175 | Effect.mapError((error) => |
| 294 | failWith(`Could not remove the failed t3@${targetVersion} installation.`, error), | |
| 295 | ), | |
| 296 | ); | |
| 297 | return yield* failWith( | |
| 298 | preflight.code !== 0 | |
| 299 | ? `The installed t3@${targetVersion} failed its version check (exit code ${String(preflight.code)}).` | |
| 300 | : `The installed runtime did not report the requested t3@${targetVersion} version.`, | |
| 301 | ); | |
| 302 | } | |
| 303 | ||
| 304 | if (activeMethod === "boot-service") { | |
| 305 | const homeDir = env.HOME ?? ""; | |
| 306 | const unitPath = path.join(homeDir, ".config", "systemd", "user", BOOT_SERVICE_UNIT_FILE); | |
| 307 | const previousUnit = yield* fs | |
| 308 | .readFileString(unitPath) | |
| 309 | .pipe( | |
| 310 | Effect.mapError((cause) => failWith("Could not read the current systemd unit.", cause)), | |
| 311 | ); | |
| 312 | // Same shape bootService.install writes, so host lifecycle commands | |
| 313 | // still recognize the unit as current. | |
| 314 | const unit = renderBootServiceUnit({ | |
| 315 | nodePath: host.execPath, | |
| 316 | t3EntryPath: runtimePaths.entryPath, | |
| 317 | baseDir: serverConfig.baseDir, | |
| 318 | logPath: path.join(serverConfig.logsDir, "boot-service.log"), | |
| 319 | unitPath, | |
| 320 | }); | |
| 321 | yield* writeUnitAtomically(unitPath, unit).pipe( | |
| 322 | Effect.mapError((cause) => failWith("Could not update the systemd unit.", cause)), | |
| 323 | ); | |
| 324 | ||
| 325 | const reloadSystemd = Effect.fn("cloud.server_self_update.reload_systemd")(function* () { | |
| 326 | const reload = yield* runner | |
| 327 | .run({ command: "systemctl", args: ["--user", "daemon-reload"] }) | |
| 328 | .pipe(Effect.mapError((cause) => failWith("Could not reload systemd units.", cause))); | |
| 329 | if (reload.code !== 0) { | |
| 330 | return yield* failWith( | |
| 331 | `Reloading systemd units failed (exit code ${String(reload.code)}).`, | |
| 332 | ); | |
| 333 | } | |
| 334 | }); | |
| 335 | ||
| 336 | yield* reloadSystemd().pipe( | |
| 337 | Effect.catch((reloadError) => | |
| 338 | writeUnitAtomically(unitPath, previousUnit).pipe( | |
| 339 | Effect.mapError((rollbackCause) => | |
| 340 | failWith("Could not restore the previous systemd unit.", { | |
| 341 | reloadError, | |
| 342 | rollbackCause, | |
| 343 | }), | |
| 344 | ), | |
| 345 | // Systemd should still have the old unit in memory after the | |
| 346 | // failed reload, but retry after restoring in case it applied a | |
| 347 | // partial update before returning an error. | |
| 348 | Effect.andThen(reloadSystemd().pipe(Effect.ignore)), | |
| 349 | Effect.andThen(Effect.fail(reloadError)), | |
| 350 | ), | |
| 351 | ), | |
| 352 | ); | |
| 353 | yield* Effect.logInfo("Server self-update installed; restarting boot service.", { | |
| 354 | targetVersion, | |
| 355 | }); | |
| 356 | // Restart after the acknowledgement has had time to cross any relay | |
| 357 | // hop. --no-block queues the restart job and exits before systemd | |
| 358 | // stops this unit: a blocking restart's SIGTERM reaches the systemctl | |
| 359 | // child (it shares this service's cgroup), which read as a restart | |
| 360 | // failure and rolled the new unit back while the old server finished | |
| 361 | // shutting down. With the handoff race gone, a non-zero exit or spawn | |
| 362 | // error means systemd genuinely rejected the job while this process is | |
| 363 | // still alive, so restoring the previous unit below stays correct. | |
| 364 | yield* scheduleRestart( | |
| 365 | Effect.gen(function* () { | |
| 366 | const restart = yield* runner | |
| 367 | .run({ | |
| 368 | command: "systemctl", | |
| 369 | args: ["--user", "restart", "--no-block", BOOT_SERVICE_UNIT_FILE], | |
| 370 | }) | |
| 371 | .pipe( | |
| 372 | Effect.mapError((cause) => | |
| 373 | failWith("Could not restart the systemd boot service.", cause), | |
| 374 | ), | |
| 375 | ); | |
| 376 | if (restart.code !== 0) { | |
| 377 | return yield* failWith( | |
| 378 | `Restarting the systemd boot service failed (exit code ${String(restart.code)}).`, | |
| 379 | ); | |
| 380 | } | |
| 381 | }).pipe( | |
| 382 | Effect.catch((restartError) => | |
| 383 | writeUnitAtomically(unitPath, previousUnit).pipe( | |
| 384 | Effect.andThen(reloadSystemd()), | |
| 385 | Effect.mapError((rollbackError) => | |
| 386 | failWith("Could not restore the previous systemd unit.", { | |
| 387 | restartError, | |
| 388 | rollbackError, | |
| 389 | }), | |
| 390 | ), | |
| 391 | Effect.andThen(Effect.fail(restartError)), | |
| 392 | ), | |
| 393 | ), | |
| 394 | Effect.catch((error) => | |
| 395 | Effect.logError("Server self-update could not restart the boot service.").pipe( | |
| 396 | Effect.annotateLogs({ targetVersion, error: error.reason }), | |
| 397 | // Permit a retry only after the failed handoff was rolled | |
| 398 | // back. A queued restart returns while this process is still | |
| 399 | // shutting down; releasing the lock then would let a second | |
| 400 | // update rewrite the unit mid-teardown. | |
| 401 | Effect.andThen(Ref.set(inFlight, false)), | |
| 402 | ), | |
| 403 | ), | |
| 404 | ), | |
| 405 | ); | |
| 406 | } else { | |
| 407 | // Spawn the shim before acknowledging the RPC so ENOENT/EACCES and | |
| 408 | // other launch failures leave this server alive and return a useful | |
| 409 | // error. The shim itself waits until after the acknowledgement and | |
| 410 | // deferred exit before binding the replacement server. | |
| 411 | yield* host | |
| 412 | .spawnDetached("/bin/sh", [ | |
| 413 | "-c", | |
| 414 | 'sleep 3; exec "$@"', | |
| 415 | "t3-self-update", | |
| 416 | host.execPath, | |
| 417 | runtimePaths.entryPath, | |
| 418 | ...host.cliArgs, | |
| 419 | ]) | |
| 420 | .pipe( | |
| 421 | Effect.mapError((cause) => | |
| 422 | failWith("Could not start the replacement t3 process.", cause), | |
| 423 | ), | |
| 424 | ); | |
| 425 | yield* Effect.logInfo("Server self-update installed; respawning.", { targetVersion }); | |
| 426 | yield* scheduleRestart( | |
| 427 | Effect.try({ | |
| 428 | try: () => host.exitProcess(), | |
| 429 | catch: (cause) => failWith("Could not exit the replaced t3 process.", cause), | |
| 430 | }).pipe( | |
| 431 | Effect.catch((error) => | |
| 432 | Effect.logError("Server self-update could not exit the replaced process.").pipe( | |
| 433 | Effect.annotateLogs({ targetVersion, error: error.reason }), | |
| 434 | Effect.ensuring(Ref.set(inFlight, false)), | |
| 435 | ), | |
| 176 | failWith( | |
| 177 | error._tag === "ServiceLauncherRejectedError" | |
| 178 | ? error.reason | |
| 179 | : "Could not ask the service launcher to activate the prepared update.", | |
| 180 | error, | |
| 436 | 181 | ), |
| 437 | 182 | ), |
| 438 | 183 | ); |
| 439 | } | |
| 440 | 184 | |
| 441 | return { targetVersion, method: activeMethod }; | |
| 185 | yield* Effect.logInfo("Server update prepared; handing off to the service launcher.", { | |
| 186 | updateId, | |
| 187 | targetVersion, | |
| 188 | runtimePath: paths.entryPath, | |
| 189 | }); | |
| 190 | return { targetVersion, method: "boot-service" as const, updateId }; | |
| 442 | 191 | }).pipe(Effect.onError(() => Ref.set(inFlight, false))); |
| 443 | 192 | }); |
| 444 | 193 |
apps/server/src/cloud/serviceLauncherClient.test.ts created+130
| ... | ... | @@ -0,0 +1,130 @@ |
| 1 | import { expect, it } from "@effect/vitest"; | |
| 2 | import { HostProcessEnvironment } from "@t3tools/shared/hostProcess"; | |
| 3 | import * as Effect from "effect/Effect"; | |
| 4 | import * as Fiber from "effect/Fiber"; | |
| 5 | ||
| 6 | import { | |
| 7 | SERVICE_LAUNCHER_CONTEXT_ENV, | |
| 8 | type ServiceLauncherChildMessage, | |
| 9 | type ServiceLauncherParentMessage, | |
| 10 | } from "./serviceProtocol.ts"; | |
| 11 | import * as ServiceLauncherClient from "./serviceLauncherClient.ts"; | |
| 12 | ||
| 13 | class FakeLauncherProcess { | |
| 14 | readonly connected = true; | |
| 15 | readonly env: Record<string, string | undefined>; | |
| 16 | readonly sent: ServiceLauncherChildMessage[] = []; | |
| 17 | readonly #listeners = new Map<string, Set<(...args: ReadonlyArray<unknown>) => void>>(); | |
| 18 | ||
| 19 | constructor(context: unknown) { | |
| 20 | this.env = { [SERVICE_LAUNCHER_CONTEXT_ENV]: JSON.stringify(context) }; | |
| 21 | } | |
| 22 | ||
| 23 | send = (message: ServiceLauncherChildMessage, callback?: (error: Error | null) => void) => { | |
| 24 | this.sent.push(message); | |
| 25 | callback?.(null); | |
| 26 | return true; | |
| 27 | }; | |
| 28 | ||
| 29 | on = (event: "message" | "disconnect", listener: (...args: ReadonlyArray<unknown>) => void) => { | |
| 30 | const listeners = this.#listeners.get(event) ?? new Set(); | |
| 31 | listeners.add(listener); | |
| 32 | this.#listeners.set(event, listeners); | |
| 33 | }; | |
| 34 | ||
| 35 | off = (event: "message" | "disconnect", listener: (...args: ReadonlyArray<unknown>) => void) => { | |
| 36 | this.#listeners.get(event)?.delete(listener); | |
| 37 | }; | |
| 38 | ||
| 39 | emit(message: ServiceLauncherParentMessage) { | |
| 40 | for (const listener of this.#listeners.get("message") ?? []) listener(message); | |
| 41 | } | |
| 42 | } | |
| 43 | ||
| 44 | const makeClient = (host: FakeLauncherProcess, currentVersion: string) => | |
| 45 | ServiceLauncherClient.make({ currentVersion }).pipe( | |
| 46 | Effect.provideService(ServiceLauncherClient.ServiceLauncherHostProcess, host), | |
| 47 | Effect.provideService(HostProcessEnvironment, host.env), | |
| 48 | ); | |
| 49 | ||
| 50 | it.effect("waits for the launcher to durably commit the trial update ID", () => | |
| 51 | Effect.gen(function* () { | |
| 52 | const pending = { | |
| 53 | id: "update-1", | |
| 54 | fromVersion: "1.0.0", | |
| 55 | targetVersion: "1.1.0", | |
| 56 | status: "pending" as const, | |
| 57 | }; | |
| 58 | const host = new FakeLauncherProcess({ | |
| 59 | protocol: 1, | |
| 60 | childVersion: "1.1.0", | |
| 61 | update: pending, | |
| 62 | }); | |
| 63 | const client = yield* makeClient(host, "1.1.0"); | |
| 64 | const prepared = yield* Effect.forkChild(client.prepareTrial, { startImmediately: true }); | |
| 65 | yield* Effect.yieldNow; | |
| 66 | expect(host.sent).toEqual([{ type: "prepared", updateId: "update-1" }]); | |
| 67 | ||
| 68 | const committed = { | |
| 69 | id: pending.id, | |
| 70 | fromVersion: pending.fromVersion, | |
| 71 | targetVersion: pending.targetVersion, | |
| 72 | status: "committed" as const, | |
| 73 | }; | |
| 74 | host.emit({ type: "committed", updateId: committed.id }); | |
| 75 | expect(yield* Fiber.join(prepared)).toEqual(committed); | |
| 76 | }), | |
| 77 | ); | |
| 78 | ||
| 79 | it.effect("returns the launcher-generated ID only after update acceptance", () => | |
| 80 | Effect.gen(function* () { | |
| 81 | const host = new FakeLauncherProcess({ | |
| 82 | protocol: 1, | |
| 83 | childVersion: "1.0.0", | |
| 84 | }); | |
| 85 | const client = yield* makeClient(host, "1.0.0"); | |
| 86 | const requested = yield* Effect.forkChild(client.requestUpdate({ targetVersion: "1.1.0" }), { | |
| 87 | startImmediately: true, | |
| 88 | }); | |
| 89 | yield* Effect.yieldNow; | |
| 90 | host.emit({ | |
| 91 | type: "update-accepted", | |
| 92 | updateId: "launcher-id", | |
| 93 | }); | |
| 94 | expect(yield* Fiber.join(requested)).toBe("launcher-id"); | |
| 95 | }), | |
| 96 | ); | |
| 97 | ||
| 98 | it.effect("preserves a launcher rejection as a distinct error", () => | |
| 99 | Effect.gen(function* () { | |
| 100 | const host = new FakeLauncherProcess({ protocol: 1, childVersion: "1.0.0" }); | |
| 101 | const client = yield* makeClient(host, "1.0.0"); | |
| 102 | const requested = yield* Effect.forkChild(client.requestUpdate({ targetVersion: "1.1.0" }), { | |
| 103 | startImmediately: true, | |
| 104 | }); | |
| 105 | yield* Effect.yieldNow; | |
| 106 | host.emit({ type: "update-rejected", reason: "requires local update" }); | |
| 107 | expect(yield* Fiber.join(requested).pipe(Effect.flip)).toMatchObject({ | |
| 108 | _tag: "ServiceLauncherRejectedError", | |
| 109 | targetVersion: "1.1.0", | |
| 110 | reason: "requires local update", | |
| 111 | }); | |
| 112 | }), | |
| 113 | ); | |
| 114 | ||
| 115 | it.effect("rejects contradictory trial context instead of leaving activation closed", () => | |
| 116 | Effect.gen(function* () { | |
| 117 | const host = new FakeLauncherProcess({ | |
| 118 | protocol: 1, | |
| 119 | childVersion: "1.1.0", | |
| 120 | update: { | |
| 121 | id: "update-1", | |
| 122 | fromVersion: "1.0.0", | |
| 123 | targetVersion: "1.2.0", | |
| 124 | status: "pending", | |
| 125 | }, | |
| 126 | }); | |
| 127 | const error = yield* makeClient(host, "1.1.0").pipe(Effect.flip); | |
| 128 | expect(error.message).toBe("The service launcher supplied invalid startup context."); | |
| 129 | }), | |
| 130 | ); |
apps/server/src/cloud/serviceLauncherClient.ts created+249
| ... | ... | @@ -0,0 +1,249 @@ |
| 1 | import type { ServerSelfUpdateOutcome } from "@t3tools/contracts"; | |
| 2 | import { HostProcessEnvironment } from "@t3tools/shared/hostProcess"; | |
| 3 | import * as Context from "effect/Context"; | |
| 4 | import * as Effect from "effect/Effect"; | |
| 5 | import * as Layer from "effect/Layer"; | |
| 6 | import * as Schema from "effect/Schema"; | |
| 7 | ||
| 8 | import packageJson from "../../package.json" with { type: "json" }; | |
| 9 | import { | |
| 10 | decodeServiceLauncherContext, | |
| 11 | decodeServiceLauncherParentMessage, | |
| 12 | SERVICE_LAUNCHER_CONTEXT_ENV, | |
| 13 | type ServiceLauncherChildMessage, | |
| 14 | type ServiceLauncherParentMessage, | |
| 15 | } from "./serviceProtocol.ts"; | |
| 16 | ||
| 17 | export class ServiceLauncherClientError extends Schema.TaggedErrorClass<ServiceLauncherClientError>()( | |
| 18 | "ServiceLauncherClientError", | |
| 19 | { | |
| 20 | operation: Schema.Literals([ | |
| 21 | "decode-context", | |
| 22 | "version-mismatch", | |
| 23 | "ipc-unavailable", | |
| 24 | "unmanaged", | |
| 25 | "send", | |
| 26 | "disconnect", | |
| 27 | "timeout", | |
| 28 | ]), | |
| 29 | cause: Schema.optional(Schema.Defect()), | |
| 30 | }, | |
| 31 | ) { | |
| 32 | override get message(): string { | |
| 33 | switch (this.operation) { | |
| 34 | case "decode-context": | |
| 35 | return "The service launcher supplied invalid startup context."; | |
| 36 | case "version-mismatch": | |
| 37 | return "The service launcher started a different t3 version."; | |
| 38 | case "ipc-unavailable": | |
| 39 | return "The service launcher IPC channel is unavailable."; | |
| 40 | case "unmanaged": | |
| 41 | return "This server is not managed by the launcher."; | |
| 42 | case "send": | |
| 43 | return "Could not send a request to the service launcher."; | |
| 44 | case "disconnect": | |
| 45 | return "The service launcher disconnected before acknowledging the request."; | |
| 46 | case "timeout": | |
| 47 | return "The service launcher did not respond within 30 seconds."; | |
| 48 | } | |
| 49 | } | |
| 50 | } | |
| 51 | ||
| 52 | export class ServiceLauncherRejectedError extends Schema.TaggedErrorClass<ServiceLauncherRejectedError>()( | |
| 53 | "ServiceLauncherRejectedError", | |
| 54 | { | |
| 55 | targetVersion: Schema.String, | |
| 56 | reason: Schema.String, | |
| 57 | }, | |
| 58 | ) { | |
| 59 | override get message(): string { | |
| 60 | return this.reason; | |
| 61 | } | |
| 62 | } | |
| 63 | ||
| 64 | interface ServiceLauncherProcess { | |
| 65 | readonly connected: boolean; | |
| 66 | readonly send: ( | |
| 67 | message: ServiceLauncherChildMessage, | |
| 68 | callback?: (error: Error | null) => void, | |
| 69 | ) => boolean; | |
| 70 | readonly on: ( | |
| 71 | event: "message" | "disconnect", | |
| 72 | listener: (...args: ReadonlyArray<unknown>) => void, | |
| 73 | ) => void; | |
| 74 | readonly off: ( | |
| 75 | event: "message" | "disconnect", | |
| 76 | listener: (...args: ReadonlyArray<unknown>) => void, | |
| 77 | ) => void; | |
| 78 | } | |
| 79 | ||
| 80 | export const ServiceLauncherHostProcess = Context.Reference<ServiceLauncherProcess>( | |
| 81 | "t3/cloud/serviceLauncherHostProcess", | |
| 82 | { | |
| 83 | defaultValue: () => ({ | |
| 84 | connected: process.connected && process.send !== undefined, | |
| 85 | send: (message, callback) => { | |
| 86 | if (process.send === undefined) return false; | |
| 87 | return callback === undefined ? process.send(message) : process.send(message, callback); | |
| 88 | }, | |
| 89 | on: (event, listener) => { | |
| 90 | process.on(event, listener); | |
| 91 | }, | |
| 92 | off: (event, listener) => { | |
| 93 | process.off(event, listener); | |
| 94 | }, | |
| 95 | }), | |
| 96 | }, | |
| 97 | ); | |
| 98 | ||
| 99 | export class ServiceLauncherClient extends Context.Service< | |
| 100 | ServiceLauncherClient, | |
| 101 | { | |
| 102 | readonly managed: boolean; | |
| 103 | readonly trial: boolean; | |
| 104 | readonly requestUpdate: (input: { | |
| 105 | readonly targetVersion: string; | |
| 106 | }) => Effect.Effect<string, ServiceLauncherClientError | ServiceLauncherRejectedError>; | |
| 107 | readonly prepareTrial: Effect.Effect< | |
| 108 | ServerSelfUpdateOutcome | undefined, | |
| 109 | ServiceLauncherClientError | |
| 110 | >; | |
| 111 | } | |
| 112 | >()("t3/cloud/serviceLauncherClient") {} | |
| 113 | ||
| 114 | const resolveStartup = Effect.fn("cloud.service_launcher_client.resolve_startup")( | |
| 115 | function* (options?: { readonly currentVersion?: string }) { | |
| 116 | const host = yield* ServiceLauncherHostProcess; | |
| 117 | const environment = yield* HostProcessEnvironment; | |
| 118 | const currentVersion = options?.currentVersion ?? packageJson.version; | |
| 119 | const rawContext = environment[SERVICE_LAUNCHER_CONTEXT_ENV]; | |
| 120 | const context = rawContext === undefined ? undefined : decodeServiceLauncherContext(rawContext); | |
| 121 | ||
| 122 | if (rawContext !== undefined && context === undefined) { | |
| 123 | return yield* new ServiceLauncherClientError({ operation: "decode-context" }); | |
| 124 | } | |
| 125 | if (context !== undefined && context.childVersion !== currentVersion) { | |
| 126 | return yield* new ServiceLauncherClientError({ operation: "version-mismatch" }); | |
| 127 | } | |
| 128 | ||
| 129 | const managed = context !== undefined && host.connected; | |
| 130 | if (context !== undefined && !managed) { | |
| 131 | return yield* new ServiceLauncherClientError({ operation: "ipc-unavailable" }); | |
| 132 | } | |
| 133 | ||
| 134 | return { host, context, managed }; | |
| 135 | }, | |
| 136 | ); | |
| 137 | ||
| 138 | export const resolveServiceLauncherMode = Effect.fn("cloud.service_launcher_client.resolve_mode")( | |
| 139 | function* () { | |
| 140 | const { context, managed } = yield* resolveStartup(); | |
| 141 | return { managed, trial: context?.update?.status === "pending" }; | |
| 142 | }, | |
| 143 | ); | |
| 144 | ||
| 145 | export const make = Effect.fn("cloud.service_launcher_client.make")(function* (options?: { | |
| 146 | readonly currentVersion?: string; | |
| 147 | }) { | |
| 148 | const { host, context, managed } = yield* resolveStartup(options); | |
| 149 | ||
| 150 | const exchange = ( | |
| 151 | message: ServiceLauncherChildMessage, | |
| 152 | accept: (reply: ServiceLauncherParentMessage) => boolean, | |
| 153 | ) => | |
| 154 | Effect.callback<ServiceLauncherParentMessage, ServiceLauncherClientError>((resume) => { | |
| 155 | if (!managed) { | |
| 156 | resume(Effect.fail(new ServiceLauncherClientError({ operation: "unmanaged" }))); | |
| 157 | return; | |
| 158 | } | |
| 159 | ||
| 160 | let settled = false; | |
| 161 | const cleanup = () => { | |
| 162 | host.off("message", onMessage); | |
| 163 | host.off("disconnect", onDisconnect); | |
| 164 | }; | |
| 165 | const settle = ( | |
| 166 | effect: Effect.Effect<ServiceLauncherParentMessage, ServiceLauncherClientError>, | |
| 167 | ) => { | |
| 168 | if (settled) return; | |
| 169 | settled = true; | |
| 170 | cleanup(); | |
| 171 | resume(effect); | |
| 172 | }; | |
| 173 | const onMessage = (...args: ReadonlyArray<unknown>) => { | |
| 174 | const reply = decodeServiceLauncherParentMessage(args[0]); | |
| 175 | if (reply !== undefined && accept(reply)) settle(Effect.succeed(reply)); | |
| 176 | }; | |
| 177 | const onDisconnect = () => | |
| 178 | settle(Effect.fail(new ServiceLauncherClientError({ operation: "disconnect" }))); | |
| 179 | ||
| 180 | host.on("message", onMessage); | |
| 181 | host.on("disconnect", onDisconnect); | |
| 182 | try { | |
| 183 | host.send(message, (error) => { | |
| 184 | if (error !== null) { | |
| 185 | settle( | |
| 186 | Effect.fail(new ServiceLauncherClientError({ operation: "send", cause: error })), | |
| 187 | ); | |
| 188 | } | |
| 189 | }); | |
| 190 | } catch (cause) { | |
| 191 | settle(Effect.fail(new ServiceLauncherClientError({ operation: "send", cause }))); | |
| 192 | } | |
| 193 | ||
| 194 | return Effect.sync(cleanup); | |
| 195 | }).pipe( | |
| 196 | Effect.timeoutOrElse({ | |
| 197 | duration: "30 seconds", | |
| 198 | orElse: () => Effect.fail(new ServiceLauncherClientError({ operation: "timeout" })), | |
| 199 | }), | |
| 200 | ); | |
| 201 | ||
| 202 | const requestUpdate = (input: { readonly targetVersion: string }) => | |
| 203 | exchange( | |
| 204 | { type: "request-update", ...input }, | |
| 205 | (reply) => reply.type === "update-accepted" || reply.type === "update-rejected", | |
| 206 | ).pipe( | |
| 207 | Effect.flatMap((reply) => | |
| 208 | reply.type === "update-accepted" | |
| 209 | ? Effect.succeed(reply.updateId) | |
| 210 | : reply.type === "update-rejected" | |
| 211 | ? Effect.fail( | |
| 212 | new ServiceLauncherRejectedError({ | |
| 213 | targetVersion: input.targetVersion, | |
| 214 | reason: reply.reason, | |
| 215 | }), | |
| 216 | ) | |
| 217 | : Effect.die("service launcher returned an impossible update response"), | |
| 218 | ), | |
| 219 | ); | |
| 220 | ||
| 221 | const pending = context?.update?.status === "pending" ? context.update : undefined; | |
| 222 | const outcome = | |
| 223 | context?.update === undefined || context.update.status === "pending" | |
| 224 | ? undefined | |
| 225 | : context.update; | |
| 226 | const prepareTrial = | |
| 227 | pending !== undefined | |
| 228 | ? exchange( | |
| 229 | { type: "prepared", updateId: pending.id }, | |
| 230 | (reply) => reply.type === "committed" && reply.updateId === pending.id, | |
| 231 | ).pipe( | |
| 232 | Effect.flatMap((reply) => { | |
| 233 | if (reply.type !== "committed") { | |
| 234 | return Effect.die("service launcher returned an impossible prepared response"); | |
| 235 | } | |
| 236 | return Effect.succeed({ ...pending, status: "committed" as const }); | |
| 237 | }), | |
| 238 | ) | |
| 239 | : Effect.succeed(outcome); | |
| 240 | ||
| 241 | return ServiceLauncherClient.of({ | |
| 242 | managed, | |
| 243 | trial: pending !== undefined, | |
| 244 | requestUpdate, | |
| 245 | prepareTrial, | |
| 246 | }); | |
| 247 | }); | |
| 248 | ||
| 249 | export const layer = Layer.effect(ServiceLauncherClient, make()); |
apps/server/src/cloud/servicePreflight.test.ts created+47
| ... | ... | @@ -0,0 +1,47 @@ |
| 1 | // @effect-diagnostics nodeBuiltinImport:off | |
| 2 | import * as NodeServices from "@effect/platform-node/NodeServices"; | |
| 3 | import { expect, it } from "@effect/vitest"; | |
| 4 | import * as Effect from "effect/Effect"; | |
| 5 | import * as FileSystem from "effect/FileSystem"; | |
| 6 | import * as Path from "effect/Path"; | |
| 7 | import * as NodeSqlite from "node:sqlite"; | |
| 8 | ||
| 9 | import { migrationManifest } from "../persistence/Migrations.ts"; | |
| 10 | import { runServicePreflight } from "./servicePreflight.ts"; | |
| 11 | ||
| 12 | it.layer(NodeServices.layer)("service update preflight", (it) => { | |
| 13 | it.effect("requires exact migration-manifest equality without mutating the database", () => | |
| 14 | Effect.gen(function* () { | |
| 15 | const fs = yield* FileSystem.FileSystem; | |
| 16 | const path = yield* Path.Path; | |
| 17 | const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-preflight-test-" }); | |
| 18 | const databasePath = path.join(root, "state.sqlite"); | |
| 19 | const database = new NodeSqlite.DatabaseSync(databasePath); | |
| 20 | database.exec("CREATE TABLE effect_sql_migrations (migration_id INTEGER, name TEXT)"); | |
| 21 | const insert = database.prepare( | |
| 22 | "INSERT INTO effect_sql_migrations (migration_id, name) VALUES (?, ?)", | |
| 23 | ); | |
| 24 | for (const [id, name] of migrationManifest) insert.run(id, name); | |
| 25 | database.close(); | |
| 26 | ||
| 27 | expect(runServicePreflight({ databasePath, launcherProtocol: 1, version: "1.2.3" })).toEqual({ | |
| 28 | status: "ready", | |
| 29 | version: "1.2.3", | |
| 30 | launcherProtocol: 1, | |
| 31 | }); | |
| 32 | ||
| 33 | const changed = new NodeSqlite.DatabaseSync(databasePath); | |
| 34 | changed.exec("DELETE FROM effect_sql_migrations WHERE migration_id = 35"); | |
| 35 | changed.close(); | |
| 36 | const blocked = runServicePreflight({ | |
| 37 | databasePath, | |
| 38 | launcherProtocol: 1, | |
| 39 | version: "1.2.3", | |
| 40 | }); | |
| 41 | expect(blocked.status).toBe("blocked"); | |
| 42 | if (blocked.status === "blocked") { | |
| 43 | expect(blocked.reason).toContain("npx t3@1.2.3 service update"); | |
| 44 | } | |
| 45 | }), | |
| 46 | ); | |
| 47 | }); |
apps/server/src/cloud/servicePreflight.ts created+96
| ... | ... | @@ -0,0 +1,96 @@ |
| 1 | import * as NodeSqlite from "node:sqlite"; | |
| 2 | ||
| 3 | import packageJson from "../../package.json" with { type: "json" }; | |
| 4 | import { migrationManifest } from "../persistence/Migrations.ts"; | |
| 5 | import { SERVICE_LAUNCHER_PROTOCOL } from "./serviceProtocol.ts"; | |
| 6 | ||
| 7 | export type ServicePreflightResult = | |
| 8 | | { | |
| 9 | readonly status: "ready"; | |
| 10 | readonly version: string; | |
| 11 | readonly launcherProtocol: typeof SERVICE_LAUNCHER_PROTOCOL; | |
| 12 | } | |
| 13 | | { | |
| 14 | readonly status: "blocked"; | |
| 15 | readonly version: string; | |
| 16 | readonly reason: string; | |
| 17 | }; | |
| 18 | ||
| 19 | const localUpdateReason = (version: string) => | |
| 20 | `This version includes a database update and cannot be installed remotely. Run \`npx t3@${version} service update\` on the server machine.`; | |
| 21 | ||
| 22 | const isMigrationRow = ( | |
| 23 | value: unknown, | |
| 24 | ): value is { readonly migration_id: number; readonly name: string } => | |
| 25 | typeof value === "object" && | |
| 26 | value !== null && | |
| 27 | "migration_id" in value && | |
| 28 | typeof value.migration_id === "number" && | |
| 29 | "name" in value && | |
| 30 | typeof value.name === "string"; | |
| 31 | ||
| 32 | export function runServicePreflight(input: { | |
| 33 | readonly databasePath: string; | |
| 34 | readonly launcherProtocol: number; | |
| 35 | readonly version?: string; | |
| 36 | }): ServicePreflightResult { | |
| 37 | const version = input.version ?? packageJson.version; | |
| 38 | if (input.launcherProtocol !== SERVICE_LAUNCHER_PROTOCOL) { | |
| 39 | return { | |
| 40 | status: "blocked", | |
| 41 | version, | |
| 42 | reason: | |
| 43 | "This release requires a newer T3 Code service launcher. Update it on the server machine.", | |
| 44 | }; | |
| 45 | } | |
| 46 | ||
| 47 | try { | |
| 48 | const database = new NodeSqlite.DatabaseSync(input.databasePath, { readOnly: true }); | |
| 49 | try { | |
| 50 | const rows: ReadonlyArray<unknown> = database | |
| 51 | .prepare("SELECT migration_id, name FROM effect_sql_migrations ORDER BY migration_id") | |
| 52 | .all(); | |
| 53 | const exact = | |
| 54 | rows.length === migrationManifest.length && | |
| 55 | rows.every((row, index) => { | |
| 56 | const expected = migrationManifest[index]; | |
| 57 | return ( | |
| 58 | isMigrationRow(row) && row.migration_id === expected?.[0] && row.name === expected?.[1] | |
| 59 | ); | |
| 60 | }); | |
| 61 | if (!exact) return { status: "blocked", version, reason: localUpdateReason(version) }; | |
| 62 | } finally { | |
| 63 | database.close(); | |
| 64 | } | |
| 65 | } catch { | |
| 66 | return { status: "blocked", version, reason: localUpdateReason(version) }; | |
| 67 | } | |
| 68 | ||
| 69 | return { status: "ready", version, launcherProtocol: SERVICE_LAUNCHER_PROTOCOL }; | |
| 70 | } | |
| 71 | ||
| 72 | export function decodeServicePreflightResult(value: unknown): ServicePreflightResult | undefined { | |
| 73 | if (typeof value !== "object" || value === null) { | |
| 74 | return undefined; | |
| 75 | } | |
| 76 | const record = value as Record<string, unknown>; | |
| 77 | if ( | |
| 78 | record.status === "ready" && | |
| 79 | record.launcherProtocol === SERVICE_LAUNCHER_PROTOCOL && | |
| 80 | typeof record.version === "string" | |
| 81 | ) { | |
| 82 | return { | |
| 83 | status: "ready", | |
| 84 | version: record.version, | |
| 85 | launcherProtocol: SERVICE_LAUNCHER_PROTOCOL, | |
| 86 | }; | |
| 87 | } | |
| 88 | if ( | |
| 89 | record.status === "blocked" && | |
| 90 | typeof record.version === "string" && | |
| 91 | typeof record.reason === "string" | |
| 92 | ) { | |
| 93 | return { status: "blocked", version: record.version, reason: record.reason }; | |
| 94 | } | |
| 95 | return undefined; | |
| 96 | } |
apps/server/src/cloud/serviceProtocol.ts created+226
| ... | ... | @@ -0,0 +1,226 @@ |
| 1 | import type { ServerSelfUpdateOutcome } from "@t3tools/contracts"; | |
| 2 | ||
| 3 | export const SERVICE_LAUNCHER_PROTOCOL = 1 as const; | |
| 4 | export const SERVICE_LAUNCHER_CONTEXT_ENV = "T3_SERVICE_LAUNCHER_CONTEXT"; | |
| 5 | export const SERVICE_LAUNCHER_FILE = "service-launcher.mjs"; | |
| 6 | export const SERVICE_STATE_FILE = "service-state.json"; | |
| 7 | ||
| 8 | export interface PendingServiceUpdate { | |
| 9 | readonly id: string; | |
| 10 | readonly fromVersion: string; | |
| 11 | readonly targetVersion: string; | |
| 12 | readonly status: "pending"; | |
| 13 | } | |
| 14 | ||
| 15 | export type ServiceUpdateRecord = PendingServiceUpdate | ServerSelfUpdateOutcome; | |
| 16 | ||
| 17 | export interface ServiceState { | |
| 18 | readonly protocol: typeof SERVICE_LAUNCHER_PROTOCOL; | |
| 19 | readonly activeVersion: string; | |
| 20 | readonly update?: ServiceUpdateRecord; | |
| 21 | } | |
| 22 | ||
| 23 | /** Context is copied from launcher-owned state when a child is spawned. */ | |
| 24 | export interface ServiceLauncherContext { | |
| 25 | readonly protocol: typeof SERVICE_LAUNCHER_PROTOCOL; | |
| 26 | readonly childVersion: string; | |
| 27 | readonly update?: ServiceUpdateRecord; | |
| 28 | } | |
| 29 | ||
| 30 | export type ServiceLauncherChildMessage = | |
| 31 | | { | |
| 32 | readonly type: "request-update"; | |
| 33 | readonly targetVersion: string; | |
| 34 | } | |
| 35 | | { | |
| 36 | readonly type: "prepared"; | |
| 37 | readonly updateId: string; | |
| 38 | }; | |
| 39 | ||
| 40 | export type ServiceLauncherParentMessage = | |
| 41 | | { | |
| 42 | readonly type: "update-accepted"; | |
| 43 | readonly updateId: string; | |
| 44 | } | |
| 45 | | { | |
| 46 | readonly type: "update-rejected"; | |
| 47 | readonly reason: string; | |
| 48 | } | |
| 49 | | { | |
| 50 | readonly type: "committed"; | |
| 51 | readonly updateId: string; | |
| 52 | }; | |
| 53 | ||
| 54 | const SEMVER_NUMBER = "(?:0|[1-9]\\d*)"; | |
| 55 | const SEMVER_PRERELEASE = `(?:${SEMVER_NUMBER}|[0-9]*[A-Za-z-][0-9A-Za-z-]*)`; | |
| 56 | const EXACT_SERVICE_VERSION = new RegExp( | |
| 57 | `^${SEMVER_NUMBER}\\.${SEMVER_NUMBER}\\.${SEMVER_NUMBER}(?:-${SEMVER_PRERELEASE}(?:\\.${SEMVER_PRERELEASE})*)?(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?$`, | |
| 58 | ); | |
| 59 | ||
| 60 | /** Accepts exact SemVer only: never dist-tags or ranges passed to npm or filesystem paths. */ | |
| 61 | export const isExactServiceVersion = (version: string): boolean => | |
| 62 | EXACT_SERVICE_VERSION.test(version); | |
| 63 | ||
| 64 | const isRecord = (value: unknown): value is Record<string, unknown> => | |
| 65 | typeof value === "object" && value !== null && !Array.isArray(value); | |
| 66 | ||
| 67 | export function decodeServiceUpdate(value: unknown): ServiceUpdateRecord | undefined { | |
| 68 | if (!isRecord(value)) return undefined; | |
| 69 | const { id, fromVersion, targetVersion, status } = value; | |
| 70 | if ( | |
| 71 | typeof id !== "string" || | |
| 72 | id.trim() === "" || | |
| 73 | typeof fromVersion !== "string" || | |
| 74 | !isExactServiceVersion(fromVersion) || | |
| 75 | typeof targetVersion !== "string" || | |
| 76 | !isExactServiceVersion(targetVersion) | |
| 77 | ) { | |
| 78 | return undefined; | |
| 79 | } | |
| 80 | if (status === "pending") { | |
| 81 | return { id, fromVersion, targetVersion, status }; | |
| 82 | } | |
| 83 | if ( | |
| 84 | (status === "committed" || status === "rolled-back" || status === "failed") && | |
| 85 | (value.reason === undefined || (typeof value.reason === "string" && value.reason.trim() !== "")) | |
| 86 | ) { | |
| 87 | return { | |
| 88 | id, | |
| 89 | fromVersion, | |
| 90 | targetVersion, | |
| 91 | status, | |
| 92 | ...(typeof value.reason === "string" ? { reason: value.reason } : {}), | |
| 93 | }; | |
| 94 | } | |
| 95 | return undefined; | |
| 96 | } | |
| 97 | ||
| 98 | /** SemVer precedence for exact versions. Build metadata is ignored. */ | |
| 99 | export function compareExactServiceVersions(left: string, right: string): number { | |
| 100 | const parse = (version: string) => { | |
| 101 | const withoutBuild = version.split("+", 1)[0] ?? version; | |
| 102 | const separator = withoutBuild.indexOf("-"); | |
| 103 | const core = separator === -1 ? withoutBuild : withoutBuild.slice(0, separator); | |
| 104 | const prerelease = separator === -1 ? undefined : withoutBuild.slice(separator + 1); | |
| 105 | const [major = "0", minor = "0", patch = "0"] = core.split("."); | |
| 106 | return { | |
| 107 | core: [BigInt(major), BigInt(minor), BigInt(patch)] as const, | |
| 108 | prerelease: prerelease?.split(".") ?? [], | |
| 109 | }; | |
| 110 | }; | |
| 111 | const a = parse(left); | |
| 112 | const b = parse(right); | |
| 113 | for (let index = 0; index < 3; index += 1) { | |
| 114 | const x = a.core[index] ?? 0n; | |
| 115 | const y = b.core[index] ?? 0n; | |
| 116 | if (x !== y) return x < y ? -1 : 1; | |
| 117 | } | |
| 118 | if (a.prerelease.length === 0 || b.prerelease.length === 0) { | |
| 119 | return a.prerelease.length === b.prerelease.length ? 0 : a.prerelease.length === 0 ? 1 : -1; | |
| 120 | } | |
| 121 | const count = Math.max(a.prerelease.length, b.prerelease.length); | |
| 122 | for (let index = 0; index < count; index += 1) { | |
| 123 | const x = a.prerelease[index]; | |
| 124 | const y = b.prerelease[index]; | |
| 125 | if (x === undefined || y === undefined) return x === undefined ? -1 : 1; | |
| 126 | if (x === y) continue; | |
| 127 | const xNumeric = /^\d+$/.test(x); | |
| 128 | const yNumeric = /^\d+$/.test(y); | |
| 129 | if (xNumeric && yNumeric) return BigInt(x) < BigInt(y) ? -1 : 1; | |
| 130 | if (xNumeric !== yNumeric) return xNumeric ? -1 : 1; | |
| 131 | return x < y ? -1 : 1; | |
| 132 | } | |
| 133 | return 0; | |
| 134 | } | |
| 135 | ||
| 136 | export function decodeServiceState(value: unknown): ServiceState | undefined { | |
| 137 | if (!isRecord(value)) return undefined; | |
| 138 | const update = value.update === undefined ? undefined : decodeServiceUpdate(value.update); | |
| 139 | if ( | |
| 140 | value.protocol !== SERVICE_LAUNCHER_PROTOCOL || | |
| 141 | typeof value.activeVersion !== "string" || | |
| 142 | !isExactServiceVersion(value.activeVersion) || | |
| 143 | (value.update !== undefined && update === undefined) || | |
| 144 | (update !== undefined && | |
| 145 | compareExactServiceVersions(update.targetVersion, update.fromVersion) <= 0) || | |
| 146 | (update?.status === "pending" && update.fromVersion !== value.activeVersion) || | |
| 147 | (update?.status === "committed" && update.targetVersion !== value.activeVersion) || | |
| 148 | ((update?.status === "rolled-back" || update?.status === "failed") && | |
| 149 | update.fromVersion !== value.activeVersion) | |
| 150 | ) { | |
| 151 | return undefined; | |
| 152 | } | |
| 153 | return { | |
| 154 | protocol: SERVICE_LAUNCHER_PROTOCOL, | |
| 155 | activeVersion: value.activeVersion, | |
| 156 | ...(update === undefined ? {} : { update }), | |
| 157 | }; | |
| 158 | } | |
| 159 | ||
| 160 | export function parseServiceState(value: string): ServiceState | undefined { | |
| 161 | try { | |
| 162 | return decodeServiceState(JSON.parse(value) as unknown); | |
| 163 | } catch { | |
| 164 | return undefined; | |
| 165 | } | |
| 166 | } | |
| 167 | ||
| 168 | export function decodeServiceLauncherContext(value: string): ServiceLauncherContext | undefined { | |
| 169 | let parsed: unknown; | |
| 170 | try { | |
| 171 | parsed = JSON.parse(value) as unknown; | |
| 172 | } catch { | |
| 173 | return undefined; | |
| 174 | } | |
| 175 | if ( | |
| 176 | !isRecord(parsed) || | |
| 177 | parsed.protocol !== SERVICE_LAUNCHER_PROTOCOL || | |
| 178 | typeof parsed.childVersion !== "string" || | |
| 179 | !isExactServiceVersion(parsed.childVersion) | |
| 180 | ) { | |
| 181 | return undefined; | |
| 182 | } | |
| 183 | const update = parsed.update === undefined ? undefined : decodeServiceUpdate(parsed.update); | |
| 184 | if (parsed.update !== undefined && update === undefined) return undefined; | |
| 185 | const selectedVersion = | |
| 186 | update?.status === "pending" || update?.status === "committed" | |
| 187 | ? update.targetVersion | |
| 188 | : update === undefined | |
| 189 | ? parsed.childVersion | |
| 190 | : update.fromVersion; | |
| 191 | if (parsed.childVersion !== selectedVersion) { | |
| 192 | return undefined; | |
| 193 | } | |
| 194 | return { | |
| 195 | protocol: SERVICE_LAUNCHER_PROTOCOL, | |
| 196 | childVersion: parsed.childVersion, | |
| 197 | ...(update === undefined ? {} : { update }), | |
| 198 | }; | |
| 199 | } | |
| 200 | ||
| 201 | export function decodeServiceLauncherChildMessage( | |
| 202 | value: unknown, | |
| 203 | ): ServiceLauncherChildMessage | undefined { | |
| 204 | if (!isRecord(value)) return undefined; | |
| 205 | if (value.type === "request-update" && typeof value.targetVersion === "string") { | |
| 206 | return { type: value.type, targetVersion: value.targetVersion }; | |
| 207 | } | |
| 208 | return value.type === "prepared" && typeof value.updateId === "string" | |
| 209 | ? { type: value.type, updateId: value.updateId } | |
| 210 | : undefined; | |
| 211 | } | |
| 212 | ||
| 213 | export function decodeServiceLauncherParentMessage( | |
| 214 | value: unknown, | |
| 215 | ): ServiceLauncherParentMessage | undefined { | |
| 216 | if (!isRecord(value)) return undefined; | |
| 217 | if (value.type === "update-rejected" && typeof value.reason === "string") { | |
| 218 | return { type: value.type, reason: value.reason }; | |
| 219 | } | |
| 220 | if (value.type === "update-accepted" && typeof value.updateId === "string") { | |
| 221 | return { type: value.type, updateId: value.updateId }; | |
| 222 | } | |
| 223 | return value.type === "committed" && typeof value.updateId === "string" | |
| 224 | ? { type: value.type, updateId: value.updateId } | |
| 225 | : undefined; | |
| 226 | } |
apps/server/src/environment/ServerEnvironment.ts+5-4
| ... | ... | @@ -10,6 +10,7 @@ import * as Schema from "effect/Schema"; |
| 10 | 10 | |
| 11 | 11 | import packageJson from "../../package.json" with { type: "json" }; |
| 12 | 12 | import { resolveServerSelfUpdateCapability } from "../cloud/selfUpdate.ts"; |
| 13 | import { resolveServiceLauncherMode } from "../cloud/serviceLauncherClient.ts"; | |
| 13 | 14 | import * as ServerConfig from "../config.ts"; |
| 14 | 15 | import * as ProcessRunner from "../processRunner.ts"; |
| 15 | 16 | import { resolveServerEnvironmentLabel } from "./ServerEnvironmentLabel.ts"; |
| ... | ... | @@ -125,8 +126,10 @@ export const make = Effect.gen(function* () { |
| 125 | 126 | const environmentId = EnvironmentId.make(environmentIdRaw); |
| 126 | 127 | const cwdBaseName = path.basename(serverConfig.cwd).trim(); |
| 127 | 128 | const label = yield* resolveServerEnvironmentLabel({ cwdBaseName }); |
| 128 | const serverSelfUpdate = yield* resolveServerSelfUpdateCapability({ | |
| 129 | const launcher = yield* resolveServiceLauncherMode(); | |
| 130 | const serverSelfUpdate = resolveServerSelfUpdateCapability({ | |
| 129 | 131 | desktopManaged: serverConfig.mode === "desktop", |
| 132 | launcherManaged: launcher.managed, | |
| 130 | 133 | }); |
| 131 | 134 | |
| 132 | 135 | const descriptor: ExecutionEnvironmentDescriptor = { |
| ... | ... | @@ -144,9 +147,7 @@ export const make = Effect.gen(function* () { |
| 144 | 147 | threadSnooze: true, |
| 145 | 148 | threadTitleRegeneration: true, |
| 146 | 149 | ...(serverSelfUpdate === null ? {} : { serverSelfUpdate }), |
| 147 | ...(serverSelfUpdate === "boot-service" || serverSelfUpdate === "respawn" | |
| 148 | ? { serverSelfUpdateProgress: true } | |
| 149 | : {}), | |
| 150 | ...(serverSelfUpdate === "boot-service" ? { serverSelfUpdateProgress: true } : {}), | |
| 150 | 151 | }, |
| 151 | 152 | }; |
| 152 | 153 |
apps/server/src/orchestration/Layers/CheckpointReactor.ts+3-2
| ... | ... | @@ -29,6 +29,7 @@ import { |
| 29 | 29 | import * as CheckpointStore from "../../checkpointing/CheckpointStore.ts"; |
| 30 | 30 | import { ProviderService } from "../../provider/Services/ProviderService.ts"; |
| 31 | 31 | import { CheckpointReactor, type CheckpointReactorShape } from "../Services/CheckpointReactor.ts"; |
| 32 | import { forkParked } from "../../serverActivation.ts"; | |
| 32 | 33 | import { OrchestrationEngineService } from "../Services/OrchestrationEngine.ts"; |
| 33 | 34 | import { ProjectionSnapshotQuery } from "../Services/ProjectionSnapshotQuery.ts"; |
| 34 | 35 | import { RuntimeReceiptBus } from "../Services/RuntimeReceiptBus.ts"; |
| ... | ... | @@ -912,7 +913,7 @@ const make = Effect.gen(function* () { |
| 912 | 913 | const worker = yield* makeDrainableWorker(processInputSafely); |
| 913 | 914 | |
| 914 | 915 | const start: CheckpointReactorShape["start"] = Effect.fn("start")(function* () { |
| 915 | yield* Effect.forkScoped( | |
| 916 | yield* forkParked( | |
| 916 | 917 | Stream.runForEach(orchestrationEngine.streamDomainEvents, (event) => { |
| 917 | 918 | if ( |
| 918 | 919 | event.type !== "thread.turn-start-requested" && |
| ... | ... | @@ -926,7 +927,7 @@ const make = Effect.gen(function* () { |
| 926 | 927 | }), |
| 927 | 928 | ); |
| 928 | 929 | |
| 929 | yield* Effect.forkScoped( | |
| 930 | yield* forkParked( | |
| 930 | 931 | Stream.runForEach(providerService.streamEvents, (event) => { |
| 931 | 932 | if (event.type !== "turn.started" && event.type !== "turn.completed") { |
| 932 | 933 | return Effect.void; |
apps/server/src/orchestration/Layers/ProviderCommandReactor.ts+39-15
| ... | ... | @@ -38,6 +38,7 @@ import { |
| 38 | 38 | ProviderCommandReactor, |
| 39 | 39 | type ProviderCommandReactorShape, |
| 40 | 40 | } from "../Services/ProviderCommandReactor.ts"; |
| 41 | import { forkParked, ServerActivation } from "../../serverActivation.ts"; | |
| 41 | 42 | import { |
| 42 | 43 | resolveSourceControlWriterModelSelection, |
| 43 | 44 | ServerSettingsService, |
| ... | ... | @@ -899,19 +900,25 @@ const make = Effect.gen(function* () { |
| 899 | 900 | ...(input.title !== undefined ? { title: input.title } : {}), |
| 900 | 901 | }); |
| 901 | 902 | }); |
| 902 | const clearInterruptedThreadTitleRegenerations = Effect.fn( | |
| 903 | "clearInterruptedThreadTitleRegenerations", | |
| 903 | const findInterruptedThreadTitleRegenerations = Effect.fn( | |
| 904 | "findInterruptedThreadTitleRegenerations", | |
| 904 | 905 | )(function* () { |
| 905 | 906 | const readModel = yield* projectionSnapshotQuery.getCommandReadModel(); |
| 907 | return readModel.threads.flatMap((thread) => { | |
| 908 | const requestId = thread.titleRegeneration?.requestId; | |
| 909 | return requestId === undefined ? [] : [{ threadId: thread.id, requestId }]; | |
| 910 | }); | |
| 911 | }); | |
| 912 | const clearInterruptedThreadTitleRegenerations = Effect.fn( | |
| 913 | "clearInterruptedThreadTitleRegenerations", | |
| 914 | )(function* ( | |
| 915 | interrupted: ReadonlyArray<{ readonly threadId: ThreadId; readonly requestId: CommandId }>, | |
| 916 | ) { | |
| 906 | 917 | yield* Effect.forEach( |
| 907 | readModel.threads, | |
| 908 | (thread) => { | |
| 909 | const requestId = thread.titleRegeneration?.requestId; | |
| 910 | if (requestId === undefined) { | |
| 911 | return Effect.void; | |
| 912 | } | |
| 918 | interrupted, | |
| 919 | ({ threadId, requestId }) => { | |
| 913 | 920 | return dispatchThreadTitleRegenerationCompletion({ |
| 914 | threadId: thread.id, | |
| 921 | threadId, | |
| 915 | 922 | requestId, |
| 916 | 923 | }).pipe( |
| 917 | 924 | Effect.catchCause((cause) => { |
| ... | ... | @@ -921,7 +928,7 @@ const make = Effect.gen(function* () { |
| 921 | 928 | return Effect.logWarning( |
| 922 | 929 | "provider command reactor failed to clear interrupted title regeneration", |
| 923 | 930 | { |
| 924 | threadId: thread.id, | |
| 931 | threadId, | |
| 925 | 932 | cause: Cause.pretty(cause), |
| 926 | 933 | }, |
| 927 | 934 | ); |
| ... | ... | @@ -1304,7 +1311,7 @@ const make = Effect.gen(function* () { |
| 1304 | 1311 | processDomainEvent(event).pipe( |
| 1305 | 1312 | Effect.catchCause((cause) => { |
| 1306 | 1313 | if (Cause.hasInterruptsOnly(cause)) { |
| 1307 | return Effect.failCause(cause); | |
| 1314 | return Effect.interrupt; | |
| 1308 | 1315 | } |
| 1309 | 1316 | return Effect.logWarning("provider command reactor failed to process event", { |
| 1310 | 1317 | eventType: event.type, |
| ... | ... | @@ -1316,6 +1323,17 @@ const make = Effect.gen(function* () { |
| 1316 | 1323 | const worker = yield* makeDrainableWorker(processDomainEventSafely); |
| 1317 | 1324 | |
| 1318 | 1325 | const start: ProviderCommandReactorShape["start"] = Effect.fn("start")(function* () { |
| 1326 | const interruptedTitleRegenerations = yield* findInterruptedThreadTitleRegenerations().pipe( | |
| 1327 | Effect.catchCause((cause) => { | |
| 1328 | if (Cause.hasInterruptsOnly(cause)) { | |
| 1329 | return Effect.interrupt; | |
| 1330 | } | |
| 1331 | return Effect.logWarning( | |
| 1332 | "provider command reactor failed to find interrupted title regenerations", | |
| 1333 | { cause: Cause.pretty(cause) }, | |
| 1334 | ).pipe(Effect.as([])); | |
| 1335 | }), | |
| 1336 | ); | |
| 1319 | 1337 | const processEvent = Effect.fn("processEvent")(function* (event: OrchestrationEvent) { |
| 1320 | 1338 | if ( |
| 1321 | 1339 | (event.type === "thread.meta-updated" && event.payload.regenerateTitle === true) || |
| ... | ... | @@ -1330,14 +1348,14 @@ const make = Effect.gen(function* () { |
| 1330 | 1348 | } |
| 1331 | 1349 | }); |
| 1332 | 1350 | |
| 1333 | yield* Effect.forkScoped( | |
| 1334 | Stream.runForEach(orchestrationEngine.streamDomainEvents, processEvent), | |
| 1335 | ); | |
| 1351 | yield* forkParked(Stream.runForEach(orchestrationEngine.streamDomainEvents, processEvent)); | |
| 1336 | 1352 | |
| 1337 | 1353 | // The domain event stream is hot, so work pending before this reactor |
| 1338 | 1354 | // starts cannot be resumed. Correlated completions only clear the request |
| 1339 | 1355 | // captured here, leaving any newer request untouched. |
| 1340 | yield* clearInterruptedThreadTitleRegenerations().pipe( | |
| 1356 | const clearInterrupted = clearInterruptedThreadTitleRegenerations( | |
| 1357 | interruptedTitleRegenerations, | |
| 1358 | ).pipe( | |
| 1341 | 1359 | Effect.catchCause((cause) => { |
| 1342 | 1360 | if (Cause.hasInterruptsOnly(cause)) { |
| 1343 | 1361 | return Effect.interrupt; |
| ... | ... | @@ -1350,6 +1368,12 @@ const make = Effect.gen(function* () { |
| 1350 | 1368 | ); |
| 1351 | 1369 | }), |
| 1352 | 1370 | ); |
| 1371 | const activation = yield* ServerActivation; | |
| 1372 | if (activation === undefined) { | |
| 1373 | yield* clearInterrupted; | |
| 1374 | } else { | |
| 1375 | yield* forkParked(clearInterrupted); | |
| 1376 | } | |
| 1353 | 1377 | }); |
| 1354 | 1378 | |
| 1355 | 1379 | return { |
apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts+3-2
| ... | ... | @@ -37,6 +37,7 @@ import { |
| 37 | 37 | ProviderRuntimeIngestionService, |
| 38 | 38 | type ProviderRuntimeIngestionShape, |
| 39 | 39 | } from "../Services/ProviderRuntimeIngestion.ts"; |
| 40 | import { forkParked } from "../../serverActivation.ts"; | |
| 40 | 41 | import { ServerSettingsService } from "../../serverSettings.ts"; |
| 41 | 42 | |
| 42 | 43 | const providerTurnKey = (threadId: ThreadId, turnId: TurnId) => `${threadId}:${turnId}`; |
| ... | ... | @@ -1804,12 +1805,12 @@ const make = Effect.gen(function* () { |
| 1804 | 1805 | |
| 1805 | 1806 | const start: ProviderRuntimeIngestionShape["start"] = () => |
| 1806 | 1807 | Effect.gen(function* () { |
| 1807 | yield* Effect.forkScoped( | |
| 1808 | yield* forkParked( | |
| 1808 | 1809 | Stream.runForEach(providerService.streamEvents, (event) => |
| 1809 | 1810 | worker.enqueue({ source: "runtime", event }), |
| 1810 | 1811 | ), |
| 1811 | 1812 | ); |
| 1812 | yield* Effect.forkScoped( | |
| 1813 | yield* forkParked( | |
| 1813 | 1814 | Stream.runForEach(orchestrationEngine.streamDomainEvents, (event) => { |
| 1814 | 1815 | if (event.type !== "thread.turn-start-requested") { |
| 1815 | 1816 | return Effect.void; |
apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts+2-1
| ... | ... | @@ -12,6 +12,7 @@ import { |
| 12 | 12 | ThreadDeletionReactor, |
| 13 | 13 | type ThreadDeletionReactorShape, |
| 14 | 14 | } from "../Services/ThreadDeletionReactor.ts"; |
| 15 | import { forkParked } from "../../serverActivation.ts"; | |
| 15 | 16 | |
| 16 | 17 | type ThreadDeletedEvent = Extract<OrchestrationEvent, { type: "thread.deleted" }>; |
| 17 | 18 | |
| ... | ... | @@ -80,7 +81,7 @@ const make = Effect.gen(function* () { |
| 80 | 81 | const worker = yield* makeDrainableWorker(processThreadDeletedSafely); |
| 81 | 82 | |
| 82 | 83 | const start: ThreadDeletionReactorShape["start"] = Effect.fn("start")(function* () { |
| 83 | yield* Effect.forkScoped( | |
| 84 | yield* forkParked( | |
| 84 | 85 | Stream.runForEach(orchestrationEngine.streamDomainEvents, (event) => { |
| 85 | 86 | if (event.type !== "thread.deleted") { |
| 86 | 87 | return Effect.void; |
apps/server/src/persistence/Layers/Sqlite.ts+20-11
| ... | ... | @@ -7,6 +7,7 @@ import type { SqlError } from "effect/unstable/sql/SqlError"; |
| 7 | 7 | |
| 8 | 8 | import { runMigrations } from "../Migrations.ts"; |
| 9 | 9 | import { ServerConfig } from "../../config.ts"; |
| 10 | import * as ServiceLauncherClient from "../../cloud/serviceLauncherClient.ts"; | |
| 10 | 11 | |
| 11 | 12 | type RuntimeSqliteLayerConfig = { |
| 12 | 13 | readonly filename: string; |
| ... | ... | @@ -30,24 +31,28 @@ const makeRuntimeSqliteLayer = Effect.fn("makeRuntimeSqliteLayer")(function* ( |
| 30 | 31 | return clientModule.layer(config); |
| 31 | 32 | }, Layer.unwrap); |
| 32 | 33 | |
| 33 | const setup = Layer.effectDiscard( | |
| 34 | Effect.gen(function* () { | |
| 35 | const sql = yield* SqlClient.SqlClient; | |
| 36 | yield* sql`PRAGMA journal_mode = WAL;`; | |
| 37 | yield* sql`PRAGMA foreign_keys = ON;`; | |
| 38 | yield* runMigrations(); | |
| 39 | }), | |
| 40 | ); | |
| 34 | const setup = (trial: boolean) => | |
| 35 | Layer.effectDiscard( | |
| 36 | Effect.gen(function* () { | |
| 37 | const sql = yield* SqlClient.SqlClient; | |
| 38 | yield* sql`PRAGMA foreign_keys = ON;`; | |
| 39 | if (!trial) { | |
| 40 | yield* sql`PRAGMA journal_mode = WAL;`; | |
| 41 | yield* runMigrations(); | |
| 42 | } | |
| 43 | }), | |
| 44 | ); | |
| 41 | 45 | |
| 42 | 46 | export const makeSqlitePersistenceLive = Effect.fn("makeSqlitePersistenceLive")(function* ( |
| 43 | 47 | dbPath: string, |
| 48 | options?: { readonly trial?: boolean }, | |
| 44 | 49 | ) { |
| 45 | 50 | const fs = yield* FileSystem.FileSystem; |
| 46 | 51 | const path = yield* Path.Path; |
| 47 | 52 | yield* fs.makeDirectory(path.dirname(dbPath), { recursive: true }); |
| 48 | 53 | |
| 49 | 54 | return Layer.provideMerge( |
| 50 | setup, | |
| 55 | setup(options?.trial === true), | |
| 51 | 56 | makeRuntimeSqliteLayer({ |
| 52 | 57 | filename: dbPath, |
| 53 | 58 | spanAttributes: { |
| ... | ... | @@ -59,10 +64,14 @@ export const makeSqlitePersistenceLive = Effect.fn("makeSqlitePersistenceLive")( |
| 59 | 64 | }, Layer.unwrap); |
| 60 | 65 | |
| 61 | 66 | export const SqlitePersistenceMemory = Layer.provideMerge( |
| 62 | setup, | |
| 67 | setup(false), | |
| 63 | 68 | makeRuntimeSqliteLayer({ filename: ":memory:" }), |
| 64 | 69 | ); |
| 65 | 70 | |
| 66 | 71 | export const layerConfig = Layer.unwrap( |
| 67 | Effect.map(Effect.service(ServerConfig), ({ dbPath }) => makeSqlitePersistenceLive(dbPath)), | |
| 72 | Effect.gen(function* () { | |
| 73 | const { dbPath } = yield* ServerConfig; | |
| 74 | const launcher = yield* ServiceLauncherClient.resolveServiceLauncherMode(); | |
| 75 | return makeSqlitePersistenceLive(dbPath, { trial: launcher.trial }); | |
| 76 | }), | |
| 68 | 77 | ); |
apps/server/src/persistence/Migrations.ts+3-1
| ... | ... | @@ -9,8 +9,8 @@ |
| 9 | 9 | */ |
| 10 | 10 | |
| 11 | 11 | import * as Migrator from "effect/unstable/sql/Migrator"; |
| 12 | import * as Layer from "effect/Layer"; | |
| 13 | 12 | import * as Effect from "effect/Effect"; |
| 13 | import * as Layer from "effect/Layer"; | |
| 14 | 14 | |
| 15 | 15 | // Import all migrations statically |
| 16 | 16 | import Migration0001 from "./Migrations/001_OrchestrationEvents.ts"; |
| ... | ... | @@ -97,6 +97,8 @@ export const migrationEntries = [ |
| 97 | 97 | [35, "ProjectionThreadTitleRegeneration", Migration0035], |
| 98 | 98 | ] as const; |
| 99 | 99 | |
| 100 | export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const); | |
| 101 | ||
| 100 | 102 | export const makeMigrationLoader = (throughId?: number) => |
| 101 | 103 | Migrator.fromRecord( |
| 102 | 104 | Object.fromEntries( |
apps/server/src/provider/Layers/ProviderSessionReaper.ts+2-1
| ... | ... | @@ -11,6 +11,7 @@ import { |
| 11 | 11 | ProviderSessionReaper, |
| 12 | 12 | type ProviderSessionReaperShape, |
| 13 | 13 | } from "../Services/ProviderSessionReaper.ts"; |
| 14 | import { forkParked } from "../../serverActivation.ts"; | |
| 14 | 15 | import { ProviderService } from "../Services/ProviderService.ts"; |
| 15 | 16 | |
| 16 | 17 | const DEFAULT_INACTIVITY_THRESHOLD_MS = 30 * 60 * 1000; |
| ... | ... | @@ -105,7 +106,7 @@ const makeProviderSessionReaper = (options?: ProviderSessionReaperLiveOptions) = |
| 105 | 106 | |
| 106 | 107 | const start: ProviderSessionReaperShape["start"] = () => |
| 107 | 108 | Effect.gen(function* () { |
| 108 | yield* Effect.forkScoped( | |
| 109 | yield* forkParked( | |
| 109 | 110 | sweep.pipe( |
| 110 | 111 | Effect.catch((error: unknown) => |
| 111 | 112 | Effect.logWarning("provider.session.reaper.sweep-failed", { |
apps/server/src/relay/AgentAwarenessRelay.ts+3-2
| ... | ... | @@ -44,6 +44,7 @@ import { getOrCreateEnvironmentKeyPairFromSecretStore } from "../cloud/environme |
| 44 | 44 | import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; |
| 45 | 45 | import * as OrchestrationEngine from "../orchestration/Services/OrchestrationEngine.ts"; |
| 46 | 46 | import * as ProjectionSnapshotQuery from "../orchestration/Services/ProjectionSnapshotQuery.ts"; |
| 47 | import { forkParked } from "../serverActivation.ts"; | |
| 47 | 48 | |
| 48 | 49 | export class AgentAwarenessRelay extends Context.Service< |
| 49 | 50 | AgentAwarenessRelay, |
| ... | ... | @@ -599,12 +600,12 @@ export const make = Effect.gen(function* () { |
| 599 | 600 | }); |
| 600 | 601 | break; |
| 601 | 602 | } |
| 602 | yield* Effect.forkScoped( | |
| 603 | yield* forkParked( | |
| 603 | 604 | Effect.sleep("1 second").pipe( |
| 604 | 605 | Effect.andThen(publishActiveThreadsOnceWhenConfigured(startupState !== "enabled")), |
| 605 | 606 | ), |
| 606 | 607 | ); |
| 607 | yield* Effect.forkScoped( | |
| 608 | yield* forkParked( | |
| 608 | 609 | Stream.runForEach(orchestrationEngine.streamDomainEvents, (event) => { |
| 609 | 610 | const threadId = eventThreadId(event); |
| 610 | 611 | if (threadId === null) { |
apps/server/src/server.test.ts+41-4
| ... | ... | @@ -89,6 +89,7 @@ import * as ProviderRegistry from "./provider/Services/ProviderRegistry.ts"; |
| 89 | 89 | import { makeManualOnlyProviderMaintenanceCapabilities } from "./provider/providerMaintenance.ts"; |
| 90 | 90 | import * as ServerLifecycleEvents from "./serverLifecycleEvents.ts"; |
| 91 | 91 | import * as ServerRuntimeStartup from "./serverRuntimeStartup.ts"; |
| 92 | import * as ServiceLauncherClient from "./cloud/serviceLauncherClient.ts"; | |
| 92 | 93 | import * as ServerSettings from "./serverSettings.ts"; |
| 93 | 94 | import * as TerminalManager from "./terminal/Manager.ts"; |
| 94 | 95 | import * as PreviewManager from "./preview/Manager.ts"; |
| ... | ... | @@ -550,10 +551,13 @@ const buildAppUnderTest = (options?: { |
| 550 | 551 | ), |
| 551 | 552 | ); |
| 552 | 553 | |
| 553 | const servedRoutesLayer = HttpRouter.serve(makeRoutesLayer, { | |
| 554 | disableListenLog: true, | |
| 555 | disableLogger: true, | |
| 556 | }).pipe( | |
| 554 | const servedRoutesLayer = HttpRouter.serve( | |
| 555 | makeRoutesLayer.pipe(Layer.provide(ServiceLauncherClient.layer)), | |
| 556 | { | |
| 557 | disableListenLog: true, | |
| 558 | disableLogger: true, | |
| 559 | }, | |
| 560 | ).pipe( | |
| 557 | 561 | Layer.provide( |
| 558 | 562 | Layer.mock(Keybindings.Keybindings)({ |
| 559 | 563 | loadConfigState: Effect.succeed({ |
| ... | ... | @@ -1318,6 +1322,39 @@ const getWsServerUrl = ( |
| 1318 | 1322 | }); |
| 1319 | 1323 | |
| 1320 | 1324 | it.layer(NodeServices.layer)("server router seam", (it) => { |
| 1325 | it.effect("parks HTTP ingress until command readiness", () => | |
| 1326 | Effect.gen(function* () { | |
| 1327 | const fileSystem = yield* FileSystem.FileSystem; | |
| 1328 | const path = yield* Path.Path; | |
| 1329 | const staticDir = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-router-gate-" }); | |
| 1330 | yield* fileSystem.writeFileString(path.join(staticDir, "index.html"), "ready"); | |
| 1331 | const entered = yield* Deferred.make<void>(); | |
| 1332 | const ready = yield* Deferred.make<void>(); | |
| 1333 | const completed = yield* Deferred.make<void>(); | |
| 1334 | ||
| 1335 | yield* buildAppUnderTest({ | |
| 1336 | config: { staticDir }, | |
| 1337 | layers: { | |
| 1338 | serverRuntimeStartup: { | |
| 1339 | awaitCommandReady: Deferred.succeed(entered, undefined).pipe( | |
| 1340 | Effect.andThen(Deferred.await(ready)), | |
| 1341 | ), | |
| 1342 | }, | |
| 1343 | }, | |
| 1344 | }); | |
| 1345 | const request = yield* HttpClient.get("/").pipe( | |
| 1346 | Effect.tap(() => Deferred.succeed(completed, undefined)), | |
| 1347 | Effect.forkChild, | |
| 1348 | ); | |
| 1349 | yield* Deferred.await(entered); | |
| 1350 | assert.isFalse(yield* Deferred.isDone(completed)); | |
| 1351 | ||
| 1352 | yield* Deferred.succeed(ready, undefined); | |
| 1353 | assert.equal((yield* Fiber.join(request)).status, 200); | |
| 1354 | assert.isTrue(yield* Deferred.isDone(completed)); | |
| 1355 | }).pipe(Effect.provide(NodeHttpServer.layerTest)), | |
| 1356 | ); | |
| 1357 | ||
| 1321 | 1358 | it.effect("serves static index content for GET / when staticDir is configured", () => |
| 1322 | 1359 | Effect.gen(function* () { |
| 1323 | 1360 | const fileSystem = yield* FileSystem.FileSystem; |
apps/server/src/server.ts+102-60
| ... | ... | @@ -1,5 +1,6 @@ |
| 1 | 1 | import { EnvironmentHttpApi } from "@t3tools/contracts"; |
| 2 | 2 | import * as Duration from "effect/Duration"; |
| 3 | import * as Deferred from "effect/Deferred"; | |
| 3 | 4 | import * as Effect from "effect/Effect"; |
| 4 | 5 | import * as Layer from "effect/Layer"; |
| 5 | 6 | import * as Schedule from "effect/Schedule"; |
| ... | ... | @@ -91,6 +92,7 @@ import * as CloudManagedEndpointRuntime from "./cloud/ManagedEndpointRuntime.ts" |
| 91 | 92 | import * as CloudCliTokenManager from "./cloud/CliTokenManager.ts"; |
| 92 | 93 | import * as CloudCliState from "./cloud/CliState.ts"; |
| 93 | 94 | import * as ServerSelfUpdate from "./cloud/selfUpdate.ts"; |
| 95 | import * as ServiceLauncherClient from "./cloud/serviceLauncherClient.ts"; | |
| 94 | 96 | import * as ProcessDiagnostics from "./diagnostics/ProcessDiagnostics.ts"; |
| 95 | 97 | import * as ProcessResourceMonitor from "./diagnostics/ProcessResourceMonitor.ts"; |
| 96 | 98 | import * as TraceDiagnostics from "./diagnostics/TraceDiagnostics.ts"; |
| ... | ... | @@ -109,6 +111,7 @@ import { orchestrationHttpApiLayer } from "./orchestration/http.ts"; |
| 109 | 111 | import * as NetService from "@t3tools/shared/Net"; |
| 110 | 112 | import * as RelayClient from "@t3tools/shared/relayClient"; |
| 111 | 113 | import { disableTailscaleServe, ensureTailscaleServe } from "@t3tools/tailscale"; |
| 114 | import { forkParked, ServerActivation } from "./serverActivation.ts"; | |
| 112 | 115 | |
| 113 | 116 | // Effect's default preemptive shutdown waits 20s before finalizing request scopes. |
| 114 | 117 | // T3's primary transport is long-lived WebSocket RPC, whose Effect scope finalizer |
| ... | ... | @@ -396,8 +399,12 @@ const RuntimeDependenciesLive = RuntimeCoreDependenciesLive.pipe( |
| 396 | 399 | Layer.provide(NetService.layer), |
| 397 | 400 | ); |
| 398 | 401 | |
| 399 | const RuntimeServicesLive = ServerRuntimeStartup.layer.pipe( | |
| 400 | Layer.provideMerge(RuntimeDependenciesLive), | |
| 402 | const commandReadinessLayer = HttpRouter.middleware( | |
| 403 | (httpEffect) => | |
| 404 | Effect.flatMap(ServerRuntimeStartup.ServerRuntimeStartup, (startup) => | |
| 405 | startup.awaitCommandReady.pipe(Effect.orDie, Effect.andThen(httpEffect)), | |
| 406 | ), | |
| 407 | { global: true }, | |
| 401 | 408 | ); |
| 402 | 409 | |
| 403 | 410 | export const makeRoutesLayer = Layer.mergeAll( |
| ... | ... | @@ -418,6 +425,7 @@ export const makeRoutesLayer = Layer.mergeAll( |
| 418 | 425 | ).pipe( |
| 419 | 426 | Layer.provide(PreviewAutomationBroker.layer), |
| 420 | 427 | Layer.provide(ServerSelfUpdate.layer), |
| 428 | Layer.provide(commandReadinessLayer), | |
| 421 | 429 | Layer.provide(browserApiCorsLayer), |
| 422 | 430 | Layer.provide(httpCompressionLayer), |
| 423 | 431 | ); |
| ... | ... | @@ -425,6 +433,14 @@ export const makeRoutesLayer = Layer.mergeAll( |
| 425 | 433 | export const makeServerLayer = Layer.unwrap( |
| 426 | 434 | Effect.gen(function* () { |
| 427 | 435 | const config = yield* ServerConfig.ServerConfig; |
| 436 | const activation = yield* Deferred.make<void>(); | |
| 437 | const awaitActivation = Deferred.await(activation); | |
| 438 | const activationLayer = Layer.succeed(ServerActivation, awaitActivation); | |
| 439 | const runtimeStateParked = yield* Deferred.make<void>(); | |
| 440 | const tailscaleParked = yield* Deferred.make<void>(); | |
| 441 | const cloudLinkParked = yield* Deferred.make<void>(); | |
| 442 | const routesReady = yield* Deferred.make<void>(); | |
| 443 | const launcherLayer = ServiceLauncherClient.layer; | |
| 428 | 444 | |
| 429 | 445 | yield* fixPath(); |
| 430 | 446 | |
| ... | ... | @@ -438,6 +454,8 @@ export const makeServerLayer = Layer.unwrap( |
| 438 | 454 | const runtimeStateLayer = Layer.effectDiscard( |
| 439 | 455 | Effect.acquireRelease( |
| 440 | 456 | Effect.gen(function* () { |
| 457 | yield* Deferred.succeed(runtimeStateParked, undefined).pipe(Effect.orDie); | |
| 458 | yield* awaitActivation; | |
| 441 | 459 | const server = yield* HttpServer.HttpServer; |
| 442 | 460 | const address = server.address; |
| 443 | 461 | if (typeof address === "string" || !("port" in address)) { |
| ... | ... | @@ -451,15 +469,26 @@ export const makeServerLayer = Layer.unwrap( |
| 451 | 469 | yield* persistServerRuntimeState({ |
| 452 | 470 | path: config.serverRuntimeStatePath, |
| 453 | 471 | state, |
| 454 | }); | |
| 472 | }).pipe( | |
| 473 | Effect.catchCause((cause) => | |
| 474 | Effect.logWarning("Failed to persist server runtime state", { cause }), | |
| 475 | ), | |
| 476 | ); | |
| 455 | 477 | }), |
| 456 | () => clearPersistedServerRuntimeState(config.serverRuntimeStatePath), | |
| 478 | () => | |
| 479 | clearPersistedServerRuntimeState(config.serverRuntimeStatePath).pipe( | |
| 480 | Effect.catchCause((cause) => | |
| 481 | Effect.logWarning("Failed to clear server runtime state", { cause }), | |
| 482 | ), | |
| 483 | ), | |
| 457 | 484 | ), |
| 458 | 485 | ); |
| 459 | 486 | const tailscaleServeLayer = config.tailscaleServeEnabled |
| 460 | 487 | ? Layer.effectDiscard( |
| 461 | 488 | Effect.acquireRelease( |
| 462 | 489 | Effect.gen(function* () { |
| 490 | yield* Deferred.succeed(tailscaleParked, undefined).pipe(Effect.orDie); | |
| 491 | yield* awaitActivation; | |
| 463 | 492 | const server = yield* HttpServer.HttpServer; |
| 464 | 493 | const address = server.address; |
| 465 | 494 | if (typeof address === "string" || !("port" in address)) { |
| ... | ... | @@ -509,68 +538,80 @@ export const makeServerLayer = Layer.unwrap( |
| 509 | 538 | : Layer.empty; |
| 510 | 539 | const cloudDesiredLinkReconcileLayer = Layer.effectDiscard( |
| 511 | 540 | Effect.gen(function* () { |
| 512 | if (!hasCloudPublicConfig) return; | |
| 513 | // Idle Cloudflare tunnels are billed, so a stopping server releases its | |
| 514 | // tunnel; the persisted desired link brings one back — same hostname, | |
| 515 | // fresh tunnel — when the environment starts again. Registered even | |
| 516 | // when no link is desired yet: a client can link a running server, and | |
| 517 | // that tunnel needs the same disposal on shutdown. | |
| 518 | yield* Effect.addFinalizer(() => | |
| 519 | releaseManagedTunnelOnShutdown().pipe( | |
| 520 | Effect.timeout("10 seconds"), | |
| 521 | Effect.tap((released) => | |
| 522 | released ? Effect.logInfo("Released the managed tunnel on shutdown") : Effect.void, | |
| 523 | ), | |
| 524 | Effect.catchCause((cause) => | |
| 525 | Effect.logWarning( | |
| 526 | "Failed to release the managed tunnel on shutdown; the next link reuses it", | |
| 527 | { cause }, | |
| 528 | ), | |
| 529 | ), | |
| 530 | Effect.asVoid, | |
| 531 | ), | |
| 532 | ); | |
| 533 | if (!(yield* CloudCliState.readCliDesiredCloudLink)) return; | |
| 534 | const server = yield* HttpServer.HttpServer; | |
| 535 | const address = server.address; | |
| 536 | if (typeof address === "string" || !("port" in address)) return; | |
| 537 | yield* Effect.forkScoped( | |
| 538 | Effect.sleep("250 millis").pipe( | |
| 539 | Effect.andThen(reconcileDesiredCloudLink(`http://127.0.0.1:${address.port}`)), | |
| 540 | // On reboot this races NIC/DNS bring-up, so back off exponentially | |
| 541 | // (capped at 30s) instead of burning all retries in a second. | |
| 542 | // Bounded overall so a permanently broken setup still surfaces the | |
| 543 | // warning below. Bad-request/unauthorized/conflict are | |
| 544 | // deterministic failures (malformed origin, not linked yet, linked | |
| 545 | // to a different cloud account) that no amount of retrying | |
| 546 | // converges. | |
| 547 | Effect.retry({ | |
| 548 | while: (error) => | |
| 549 | error._tag !== "EnvironmentHttpBadRequestError" && | |
| 550 | error._tag !== "EnvironmentHttpUnauthorizedError" && | |
| 551 | error._tag !== "EnvironmentHttpConflictError", | |
| 552 | schedule: Schedule.exponential("1 second").pipe( | |
| 553 | Schedule.modifyDelay(({ duration }) => | |
| 554 | Effect.succeed(Duration.min(duration, Duration.seconds(30))), | |
| 541 | if (!hasCloudPublicConfig) { | |
| 542 | yield* Deferred.succeed(cloudLinkParked, undefined).pipe(Effect.orDie); | |
| 543 | return; | |
| 544 | } | |
| 545 | yield* forkParked( | |
| 546 | Effect.gen(function* () { | |
| 547 | // Only an activated runtime owns the tunnel cleanup finalizer. | |
| 548 | yield* Effect.addFinalizer(() => | |
| 549 | releaseManagedTunnelOnShutdown().pipe( | |
| 550 | Effect.timeout("10 seconds"), | |
| 551 | Effect.tap((released) => | |
| 552 | released | |
| 553 | ? Effect.logInfo("Released the managed tunnel on shutdown") | |
| 554 | : Effect.void, | |
| 555 | 555 | ), |
| 556 | Schedule.upTo({ duration: "10 minutes" }), | |
| 556 | Effect.catchCause((cause) => | |
| 557 | Effect.logWarning( | |
| 558 | "Failed to release the managed tunnel on shutdown; the next link reuses it", | |
| 559 | { cause }, | |
| 560 | ), | |
| 561 | ), | |
| 562 | Effect.asVoid, | |
| 557 | 563 | ), |
| 558 | }), | |
| 559 | Effect.tap(() => Effect.logInfo("T3 Connect desired link reconciled on startup")), | |
| 560 | Effect.catch((cause) => | |
| 561 | Effect.logWarning("Failed to reconcile T3 Connect desired link on startup", { | |
| 562 | cause, | |
| 564 | ); | |
| 565 | if (!(yield* CloudCliState.readCliDesiredCloudLink)) return; | |
| 566 | const server = yield* HttpServer.HttpServer; | |
| 567 | const address = server.address; | |
| 568 | if (typeof address === "string" || !("port" in address)) return; | |
| 569 | yield* Effect.sleep("250 millis").pipe( | |
| 570 | Effect.andThen(reconcileDesiredCloudLink(`http://127.0.0.1:${address.port}`)), | |
| 571 | Effect.retry({ | |
| 572 | while: (error) => | |
| 573 | error._tag !== "EnvironmentHttpBadRequestError" && | |
| 574 | error._tag !== "EnvironmentHttpUnauthorizedError" && | |
| 575 | error._tag !== "EnvironmentHttpConflictError", | |
| 576 | schedule: Schedule.exponential("1 second").pipe( | |
| 577 | Schedule.modifyDelay(({ duration }) => | |
| 578 | Effect.succeed(Duration.min(duration, Duration.seconds(30))), | |
| 579 | ), | |
| 580 | Schedule.upTo({ duration: "10 minutes" }), | |
| 581 | ), | |
| 563 | 582 | }), |
| 564 | ), | |
| 565 | ), | |
| 583 | Effect.tap(() => Effect.logInfo("T3 Connect desired link reconciled on startup")), | |
| 584 | Effect.catch((cause) => | |
| 585 | Effect.logWarning("Failed to reconcile T3 Connect desired link on startup", { | |
| 586 | cause, | |
| 587 | }), | |
| 588 | ), | |
| 589 | ); | |
| 590 | }), | |
| 566 | 591 | ); |
| 592 | yield* Deferred.succeed(cloudLinkParked, undefined).pipe(Effect.orDie); | |
| 567 | 593 | }), |
| 568 | 594 | ); |
| 569 | 595 | |
| 596 | const runtimeServicesLive = ServerRuntimeStartup.layerWithOptions({ | |
| 597 | activate: Deferred.succeed(activation, undefined).pipe(Effect.asVoid), | |
| 598 | abort: (error) => Deferred.die(activation, error).pipe(Effect.asVoid), | |
| 599 | awaitAuxiliaryParked: Effect.all( | |
| 600 | [ | |
| 601 | Deferred.await(runtimeStateParked), | |
| 602 | Deferred.await(cloudLinkParked), | |
| 603 | Deferred.await(routesReady), | |
| 604 | ...(config.tailscaleServeEnabled ? [Deferred.await(tailscaleParked)] : []), | |
| 605 | ], | |
| 606 | { concurrency: "unbounded" }, | |
| 607 | ).pipe(Effect.asVoid), | |
| 608 | }).pipe(Layer.provideMerge(RuntimeDependenciesLive), Layer.provide(launcherLayer)); | |
| 609 | ||
| 610 | const routesLayer = HttpRouter.serve(makeRoutesLayer.pipe(Layer.provide(launcherLayer)), { | |
| 611 | disableLogger: !config.logWebSocketEvents, | |
| 612 | }).pipe(Layer.tap(() => Deferred.succeed(routesReady, undefined).pipe(Effect.orDie))); | |
| 570 | 613 | const serverApplicationLayer = Layer.mergeAll( |
| 571 | HttpRouter.serve(makeRoutesLayer, { | |
| 572 | disableLogger: !config.logWebSocketEvents, | |
| 573 | }), | |
| 614 | routesLayer, | |
| 574 | 615 | httpListeningLayer, |
| 575 | 616 | runtimeStateLayer, |
| 576 | 617 | tailscaleServeLayer, |
| ... | ... | @@ -578,7 +619,8 @@ export const makeServerLayer = Layer.unwrap( |
| 578 | 619 | ); |
| 579 | 620 | |
| 580 | 621 | return serverApplicationLayer.pipe( |
| 581 | Layer.provideMerge(RuntimeServicesLive), | |
| 622 | Layer.provideMerge(runtimeServicesLive), | |
| 623 | Layer.provide(activationLayer), | |
| 582 | 624 | Layer.provideMerge(serverRelayBrokerTracingLayer), |
| 583 | 625 | Layer.provideMerge(HttpResponseCompressionLive), |
| 584 | 626 | Layer.provideMerge(HttpServerLive), |
| ... | ... | @@ -590,5 +632,5 @@ export const makeServerLayer = Layer.unwrap( |
| 590 | 632 | }), |
| 591 | 633 | ); |
| 592 | 634 | |
| 593 | // Important: Only `ServerConfig` should be provided by the CLI layer!!! Don't let other requirements leak into the launch layer. | |
| 635 | // The CLI supplies configuration. | |
| 594 | 636 | export const runServer = Layer.launch(makeServerLayer); |
apps/server/src/serverActivation.test.ts created+23
| ... | ... | @@ -0,0 +1,23 @@ |
| 1 | import { expect, it } from "@effect/vitest"; | |
| 2 | import * as Deferred from "effect/Deferred"; | |
| 3 | import * as Effect from "effect/Effect"; | |
| 4 | ||
| 5 | import { forkParked, ServerActivation } from "./serverActivation.ts"; | |
| 6 | ||
| 7 | it.effect("proves a root is parked before returning and releases it with one gate", () => | |
| 8 | Effect.scoped( | |
| 9 | Effect.gen(function* () { | |
| 10 | const activation = yield* Deferred.make<void>(); | |
| 11 | const ran = yield* Deferred.make<void>(); | |
| 12 | ||
| 13 | yield* forkParked(Deferred.succeed(ran, undefined)).pipe( | |
| 14 | Effect.provideService(ServerActivation, Deferred.await(activation)), | |
| 15 | ); | |
| 16 | expect(yield* Deferred.isDone(ran)).toBe(false); | |
| 17 | ||
| 18 | yield* Deferred.succeed(activation, undefined); | |
| 19 | yield* Deferred.await(ran); | |
| 20 | expect(yield* Deferred.isDone(ran)).toBe(true); | |
| 21 | }), | |
| 22 | ), | |
| 23 | ); |
apps/server/src/serverActivation.ts created+26
| ... | ... | @@ -0,0 +1,26 @@ |
| 1 | import * as Context from "effect/Context"; | |
| 2 | import * as Deferred from "effect/Deferred"; | |
| 3 | import * as Effect from "effect/Effect"; | |
| 4 | import type * as Scope from "effect/Scope"; | |
| 5 | ||
| 6 | export class ServerActivation extends Context.Reference<Effect.Effect<void> | undefined>( | |
| 7 | "t3/serverActivation", | |
| 8 | { defaultValue: () => undefined }, | |
| 9 | ) {} | |
| 10 | ||
| 11 | /** Forks a long-running root before commit and proves it is parked at the activation boundary. */ | |
| 12 | export const forkParked = <A, E, R>( | |
| 13 | effect: Effect.Effect<A, E, R>, | |
| 14 | ): Effect.Effect<void, never, Scope.Scope | R> => | |
| 15 | Effect.gen(function* () { | |
| 16 | const activation = yield* ServerActivation; | |
| 17 | if (activation === undefined) { | |
| 18 | yield* Effect.forkScoped(effect); | |
| 19 | return; | |
| 20 | } | |
| 21 | const parked = yield* Deferred.make<void>(); | |
| 22 | yield* Effect.forkScoped( | |
| 23 | Deferred.succeed(parked, undefined).pipe(Effect.andThen(activation), Effect.andThen(effect)), | |
| 24 | ); | |
| 25 | yield* Deferred.await(parked); | |
| 26 | }); |
apps/server/src/serverRuntimeStartup.ts+182-160
| ... | ... | @@ -34,6 +34,8 @@ import * as AnalyticsService from "./telemetry/AnalyticsService.ts"; |
| 34 | 34 | import * as ServerEnvironment from "./environment/ServerEnvironment.ts"; |
| 35 | 35 | import * as EnvironmentAuth from "./auth/EnvironmentAuth.ts"; |
| 36 | 36 | import * as ProviderSessionReaper from "./provider/Services/ProviderSessionReaper.ts"; |
| 37 | import { forkParked } from "./serverActivation.ts"; | |
| 38 | import * as ServiceLauncherClient from "./cloud/serviceLauncherClient.ts"; | |
| 37 | 39 | import { |
| 38 | 40 | formatHeadlessServeOutput, |
| 39 | 41 | formatHostForUrl, |
| ... | ... | @@ -288,151 +290,161 @@ const runStartupPhase = <A, E, R>(phase: string, effect: Effect.Effect<A, E, R>) |
| 288 | 290 | Effect.withSpan(`server.startup.${phase}`), |
| 289 | 291 | ); |
| 290 | 292 | |
| 291 | export const make = Effect.gen(function* () { | |
| 292 | const serverConfig = yield* ServerConfig.ServerConfig; | |
| 293 | const keybindings = yield* Keybindings.Keybindings; | |
| 294 | const orchestrationReactor = yield* OrchestrationReactor.OrchestrationReactor; | |
| 295 | const providerSessionReaper = yield* ProviderSessionReaper.ProviderSessionReaper; | |
| 296 | const lifecycleEvents = yield* ServerLifecycleEvents.ServerLifecycleEvents; | |
| 297 | const serverSettings = yield* ServerSettings.ServerSettingsService; | |
| 298 | const serverEnvironment = yield* ServerEnvironment.ServerEnvironment; | |
| 299 | const crypto = yield* Crypto.Crypto; | |
| 293 | interface StartupOptions { | |
| 294 | readonly activate?: Effect.Effect<void>; | |
| 295 | readonly awaitAuxiliaryParked?: Effect.Effect<void>; | |
| 296 | readonly abort?: (error: ServerRuntimeStartupError) => Effect.Effect<void>; | |
| 297 | } | |
| 300 | 298 | |
| 301 | const commandGate = yield* makeCommandGate; | |
| 302 | const httpListening = yield* Deferred.make<void>(); | |
| 303 | const reactorScope = yield* Scope.make("sequential"); | |
| 304 | ||
| 305 | yield* Effect.addFinalizer(() => Scope.close(reactorScope, Exit.void)); | |
| 306 | ||
| 307 | const startup = Effect.gen(function* () { | |
| 308 | yield* Effect.logDebug("startup phase: starting keybindings runtime"); | |
| 309 | yield* runStartupPhase( | |
| 310 | "keybindings.start", | |
| 311 | keybindings.start.pipe( | |
| 312 | Effect.catch((error) => | |
| 313 | Effect.logWarning("failed to start keybindings runtime", { | |
| 314 | path: error.configPath, | |
| 315 | detail: error.detail, | |
| 316 | cause: error.cause, | |
| 317 | }), | |
| 299 | export const make = (options?: StartupOptions) => | |
| 300 | Effect.gen(function* () { | |
| 301 | const serverConfig = yield* ServerConfig.ServerConfig; | |
| 302 | const keybindings = yield* Keybindings.Keybindings; | |
| 303 | const orchestrationReactor = yield* OrchestrationReactor.OrchestrationReactor; | |
| 304 | const providerSessionReaper = yield* ProviderSessionReaper.ProviderSessionReaper; | |
| 305 | const lifecycleEvents = yield* ServerLifecycleEvents.ServerLifecycleEvents; | |
| 306 | const serverSettings = yield* ServerSettings.ServerSettingsService; | |
| 307 | const serverEnvironment = yield* ServerEnvironment.ServerEnvironment; | |
| 308 | const crypto = yield* Crypto.Crypto; | |
| 309 | const launcher = yield* ServiceLauncherClient.ServiceLauncherClient; | |
| 310 | ||
| 311 | const commandGate = yield* makeCommandGate; | |
| 312 | const httpListening = yield* Deferred.make<void>(); | |
| 313 | const reactorScope = yield* Scope.make("sequential"); | |
| 314 | ||
| 315 | yield* Effect.addFinalizer(() => Scope.close(reactorScope, Exit.void)); | |
| 316 | ||
| 317 | const startup = Effect.gen(function* () { | |
| 318 | yield* Effect.logDebug("startup phase: starting keybindings runtime"); | |
| 319 | yield* runStartupPhase( | |
| 320 | "keybindings.start", | |
| 321 | keybindings.start.pipe( | |
| 322 | Effect.catch((error) => | |
| 323 | Effect.logWarning("failed to start keybindings runtime", { | |
| 324 | path: error.configPath, | |
| 325 | detail: error.detail, | |
| 326 | cause: error.cause, | |
| 327 | }), | |
| 328 | ), | |
| 318 | 329 | ), |
| 319 | Effect.forkScoped, | |
| 320 | ), | |
| 321 | ); | |
| 330 | ); | |
| 322 | 331 | |
| 323 | yield* Effect.logDebug("startup phase: starting server settings runtime"); | |
| 324 | yield* runStartupPhase( | |
| 325 | "settings.start", | |
| 326 | serverSettings.start.pipe( | |
| 327 | Effect.catch((error) => | |
| 328 | Effect.logWarning("failed to start server settings runtime", { | |
| 329 | path: error.settingsPath, | |
| 330 | operation: error.operation, | |
| 331 | providerInstanceId: error.providerInstanceId, | |
| 332 | environmentVariable: error.environmentVariable, | |
| 333 | cause: error.cause, | |
| 334 | }), | |
| 332 | yield* Effect.logDebug("startup phase: starting server settings runtime"); | |
| 333 | yield* runStartupPhase( | |
| 334 | "settings.start", | |
| 335 | serverSettings.start.pipe( | |
| 336 | Effect.catch((error) => | |
| 337 | Effect.logWarning("failed to start server settings runtime", { | |
| 338 | path: error.settingsPath, | |
| 339 | operation: error.operation, | |
| 340 | providerInstanceId: error.providerInstanceId, | |
| 341 | environmentVariable: error.environmentVariable, | |
| 342 | cause: error.cause, | |
| 343 | }), | |
| 344 | ), | |
| 335 | 345 | ), |
| 336 | Effect.forkScoped, | |
| 337 | ), | |
| 338 | ); | |
| 346 | ); | |
| 339 | 347 | |
| 340 | yield* Effect.logDebug("startup phase: starting orchestration reactors"); | |
| 341 | yield* runStartupPhase( | |
| 342 | "reactors.start", | |
| 343 | Effect.gen(function* () { | |
| 344 | yield* orchestrationReactor.start().pipe(Scope.provide(reactorScope)); | |
| 345 | yield* providerSessionReaper.start().pipe(Scope.provide(reactorScope)); | |
| 346 | }), | |
| 347 | ); | |
| 348 | yield* Effect.logDebug("startup phase: parking orchestration roots at activation"); | |
| 349 | yield* runStartupPhase( | |
| 350 | "reactors.start", | |
| 351 | Effect.gen(function* () { | |
| 352 | yield* orchestrationReactor.start().pipe(Scope.provide(reactorScope)); | |
| 353 | yield* providerSessionReaper.start().pipe(Scope.provide(reactorScope)); | |
| 354 | }), | |
| 355 | ); | |
| 348 | 356 | |
| 349 | const welcomeBase = yield* resolveWelcomeBase; | |
| 350 | const environment = yield* serverEnvironment.getDescriptor; | |
| 351 | yield* Effect.logDebug("startup phase: preparing welcome payload"); | |
| 352 | yield* Effect.logDebug("startup phase: publishing welcome event", { | |
| 353 | environmentId: environment.environmentId, | |
| 354 | cwd: welcomeBase.cwd, | |
| 355 | projectName: welcomeBase.projectName, | |
| 356 | }); | |
| 357 | yield* runStartupPhase( | |
| 358 | "welcome.publish", | |
| 359 | lifecycleEvents.publish({ | |
| 360 | version: 1, | |
| 361 | type: "welcome", | |
| 362 | payload: { | |
| 363 | environment, | |
| 364 | ...welcomeBase, | |
| 365 | }, | |
| 366 | }), | |
| 367 | ); | |
| 357 | const welcomeBase = yield* resolveWelcomeBase; | |
| 358 | const environment = yield* serverEnvironment.getDescriptor; | |
| 359 | yield* Effect.logDebug("startup phase: preparing welcome payload"); | |
| 360 | ||
| 361 | if (serverConfig.autoBootstrapProjectFromCwd) { | |
| 362 | yield* forkParked( | |
| 363 | runStartupPhase( | |
| 364 | "welcome.autobootstrap", | |
| 365 | Effect.gen(function* () { | |
| 366 | const bootstrapTargets = yield* resolveAutoBootstrapWelcomeTargets.pipe( | |
| 367 | Effect.provideService(Crypto.Crypto, crypto), | |
| 368 | ); | |
| 369 | if (!bootstrapTargets.bootstrapProjectId && !bootstrapTargets.bootstrapThreadId) { | |
| 370 | return; | |
| 371 | } | |
| 372 | ||
| 373 | yield* Effect.logDebug("startup phase: publishing bootstrapped welcome event", { | |
| 374 | environmentId: environment.environmentId, | |
| 375 | cwd: welcomeBase.cwd, | |
| 376 | projectName: welcomeBase.projectName, | |
| 377 | bootstrapProjectId: bootstrapTargets.bootstrapProjectId, | |
| 378 | bootstrapThreadId: bootstrapTargets.bootstrapThreadId, | |
| 379 | }); | |
| 380 | yield* lifecycleEvents.publish({ | |
| 381 | version: 1, | |
| 382 | type: "welcome", | |
| 383 | payload: { | |
| 384 | environment, | |
| 385 | ...welcomeBase, | |
| 386 | ...bootstrapTargets, | |
| 387 | }, | |
| 388 | }); | |
| 389 | }).pipe( | |
| 390 | Effect.catch((cause) => | |
| 391 | Effect.logWarning("startup auto-bootstrap welcome failed", { | |
| 392 | cause, | |
| 393 | }), | |
| 394 | ), | |
| 395 | ), | |
| 396 | ), | |
| 397 | ); | |
| 398 | } | |
| 368 | 399 | |
| 369 | if (serverConfig.autoBootstrapProjectFromCwd) { | |
| 370 | yield* Effect.forkScoped( | |
| 371 | runStartupPhase( | |
| 372 | "welcome.autobootstrap", | |
| 373 | Effect.gen(function* () { | |
| 374 | const bootstrapTargets = yield* resolveAutoBootstrapWelcomeTargets.pipe( | |
| 375 | Effect.provideService(Crypto.Crypto, crypto), | |
| 400 | yield* forkParked( | |
| 401 | Effect.gen(function* () { | |
| 402 | yield* Effect.logDebug("startup phase: recording startup heartbeat"); | |
| 403 | yield* recordStartupHeartbeat.pipe( | |
| 404 | Effect.annotateSpans({ "startup.phase": "heartbeat.record" }), | |
| 405 | Effect.withSpan("server.startup.heartbeat.record"), | |
| 406 | Effect.ignoreCause({ log: true }), | |
| 407 | ); | |
| 408 | if (serverConfig.startupPresentation === "headless") { | |
| 409 | const accessInfo = yield* issueHeadlessServeAccessInfo(); | |
| 410 | yield* runStartupPhase( | |
| 411 | "headless.output", | |
| 412 | Console.log(formatHeadlessServeOutput(accessInfo)), | |
| 376 | 413 | ); |
| 377 | if (!bootstrapTargets.bootstrapProjectId && !bootstrapTargets.bootstrapThreadId) { | |
| 378 | return; | |
| 414 | } else { | |
| 415 | const startupBrowserTarget = yield* resolveStartupBrowserTarget; | |
| 416 | if (serverConfig.mode !== "desktop") { | |
| 417 | yield* Effect.logInfo( | |
| 418 | "Authentication required. Open T3 Code using the pairing URL.", | |
| 419 | ).pipe(Effect.annotateLogs({ pairingUrl: startupBrowserTarget })); | |
| 379 | 420 | } |
| 421 | yield* runStartupPhase("browser.open", maybeOpenBrowser(startupBrowserTarget)); | |
| 422 | } | |
| 423 | }), | |
| 424 | ); | |
| 380 | 425 | |
| 381 | yield* Effect.logDebug("startup phase: publishing bootstrapped welcome event", { | |
| 382 | environmentId: environment.environmentId, | |
| 383 | cwd: welcomeBase.cwd, | |
| 384 | projectName: welcomeBase.projectName, | |
| 385 | bootstrapProjectId: bootstrapTargets.bootstrapProjectId, | |
| 386 | bootstrapThreadId: bootstrapTargets.bootstrapThreadId, | |
| 387 | }); | |
| 388 | yield* lifecycleEvents.publish({ | |
| 389 | version: 1, | |
| 390 | type: "welcome", | |
| 391 | payload: { | |
| 392 | environment, | |
| 393 | ...welcomeBase, | |
| 394 | ...bootstrapTargets, | |
| 395 | }, | |
| 396 | }); | |
| 397 | }).pipe( | |
| 398 | Effect.catch((cause) => | |
| 399 | Effect.logWarning("startup auto-bootstrap welcome failed", { | |
| 400 | cause, | |
| 401 | }), | |
| 402 | ), | |
| 403 | ), | |
| 404 | ), | |
| 426 | yield* Effect.logDebug("startup phase: waiting for http listener"); | |
| 427 | yield* runStartupPhase("http.wait", Deferred.await(httpListening)); | |
| 428 | yield* runStartupPhase( | |
| 429 | "auxiliary-roots.parked", | |
| 430 | options?.awaitAuxiliaryParked ?? Effect.void, | |
| 405 | 431 | ); |
| 406 | } | |
| 407 | }).pipe( | |
| 408 | Effect.annotateSpans({ | |
| 409 | "server.mode": serverConfig.mode, | |
| 410 | "server.port": serverConfig.port, | |
| 411 | "server.host": serverConfig.host ?? "default", | |
| 412 | }), | |
| 413 | Effect.withSpan("server.startup", { kind: "server", root: true }), | |
| 414 | ); | |
| 415 | 432 | |
| 416 | yield* Effect.forkScoped( | |
| 417 | Effect.gen(function* () { | |
| 418 | const startupExit = yield* Effect.exit(startup); | |
| 419 | if (Exit.isFailure(startupExit)) { | |
| 420 | const error = new ServerRuntimeStartupError({ | |
| 421 | mode: serverConfig.mode, | |
| 422 | host: serverConfig.host ?? null, | |
| 423 | port: serverConfig.port, | |
| 424 | cause: startupExit.cause, | |
| 425 | }); | |
| 426 | yield* Effect.logError("server runtime startup failed", { cause: startupExit.cause }); | |
| 427 | yield* commandGate.failCommandReady(error); | |
| 428 | return; | |
| 429 | } | |
| 433 | // This is the prepared boundary. Every dependency has been acquired and | |
| 434 | // every runtime root has confirmed that it is parked before this request. | |
| 435 | const updateOutcome = yield* launcher.prepareTrial; | |
| 436 | yield* runStartupPhase( | |
| 437 | "welcome.publish", | |
| 438 | lifecycleEvents.publish({ | |
| 439 | version: 1, | |
| 440 | type: "welcome", | |
| 441 | payload: { environment, ...welcomeBase }, | |
| 442 | }), | |
| 443 | ); | |
| 444 | yield* options?.activate ?? Effect.void; | |
| 430 | 445 | |
| 431 | 446 | yield* Effect.logDebug("Accepting commands"); |
| 432 | 447 | yield* commandGate.signalCommandReady; |
| 433 | yield* Effect.logDebug("startup phase: waiting for http listener"); | |
| 434 | yield* runStartupPhase("http.wait", Deferred.await(httpListening)); | |
| 435 | yield* Effect.logDebug("startup phase: publishing ready event"); | |
| 436 | 448 | yield* runStartupPhase( |
| 437 | 449 | "ready.publish", |
| 438 | 450 | lifecycleEvents.publish({ |
| ... | ... | @@ -440,39 +452,49 @@ export const make = Effect.gen(function* () { |
| 440 | 452 | type: "ready", |
| 441 | 453 | payload: { |
| 442 | 454 | at: DateTime.formatIso(yield* DateTime.now), |
| 443 | environment: yield* serverEnvironment.getDescriptor, | |
| 455 | environment, | |
| 456 | ...(updateOutcome === undefined ? {} : { updateOutcome }), | |
| 444 | 457 | }, |
| 445 | 458 | }), |
| 446 | 459 | ); |
| 447 | ||
| 448 | yield* Effect.logDebug("startup phase: recording startup heartbeat"); | |
| 449 | yield* launchStartupHeartbeat; | |
| 450 | if (serverConfig.startupPresentation === "headless") { | |
| 451 | yield* Effect.logDebug("startup phase: headless access info"); | |
| 452 | const accessInfo = yield* issueHeadlessServeAccessInfo(); | |
| 453 | yield* runStartupPhase( | |
| 454 | "headless.output", | |
| 455 | Console.log(formatHeadlessServeOutput(accessInfo)), | |
| 456 | ); | |
| 457 | } else { | |
| 458 | yield* Effect.logDebug("startup phase: browser open check"); | |
| 459 | const startupBrowserTarget = yield* resolveStartupBrowserTarget; | |
| 460 | if (serverConfig.mode !== "desktop") { | |
| 461 | yield* Effect.logInfo( | |
| 462 | "Authentication required. Open T3 Code using the pairing URL.", | |
| 463 | ).pipe(Effect.annotateLogs({ pairingUrl: startupBrowserTarget })); | |
| 464 | } | |
| 465 | yield* runStartupPhase("browser.open", maybeOpenBrowser(startupBrowserTarget)); | |
| 466 | } | |
| 467 | 460 | yield* Effect.logDebug("startup phase: complete"); |
| 468 | }), | |
| 469 | ); | |
| 461 | }).pipe( | |
| 462 | Effect.annotateSpans({ | |
| 463 | "server.mode": serverConfig.mode, | |
| 464 | "server.port": serverConfig.port, | |
| 465 | "server.host": serverConfig.host ?? "default", | |
| 466 | }), | |
| 467 | Effect.withSpan("server.startup", { kind: "server", root: true }), | |
| 468 | ); | |
| 470 | 469 | |
| 471 | return { | |
| 472 | awaitCommandReady: commandGate.awaitCommandReady, | |
| 473 | markHttpListening: Deferred.succeed(httpListening, undefined), | |
| 474 | enqueueCommand: commandGate.enqueueCommand, | |
| 475 | } satisfies ServerRuntimeStartup["Service"]; | |
| 476 | }); | |
| 470 | yield* Effect.forkScoped( | |
| 471 | Effect.exit(startup).pipe( | |
| 472 | Effect.flatMap((startupExit) => { | |
| 473 | if (Exit.isSuccess(startupExit)) return Effect.void; | |
| 474 | const error = new ServerRuntimeStartupError({ | |
| 475 | mode: serverConfig.mode, | |
| 476 | host: serverConfig.host ?? null, | |
| 477 | port: serverConfig.port, | |
| 478 | cause: startupExit.cause, | |
| 479 | }); | |
| 480 | return Effect.logError("server runtime startup failed", { | |
| 481 | cause: startupExit.cause, | |
| 482 | }).pipe( | |
| 483 | Effect.andThen(commandGate.failCommandReady(error)), | |
| 484 | Effect.andThen(options?.abort?.(error) ?? Effect.void), | |
| 485 | ); | |
| 486 | }), | |
| 487 | ), | |
| 488 | ); | |
| 489 | ||
| 490 | return { | |
| 491 | awaitCommandReady: commandGate.awaitCommandReady, | |
| 492 | markHttpListening: Deferred.succeed(httpListening, undefined), | |
| 493 | enqueueCommand: commandGate.enqueueCommand, | |
| 494 | } satisfies ServerRuntimeStartup["Service"]; | |
| 495 | }); | |
| 496 | ||
| 497 | export const layerWithOptions = (options?: StartupOptions) => | |
| 498 | Layer.effect(ServerRuntimeStartup, make(options)); | |
| 477 | 499 | |
| 478 | export const layer = Layer.effect(ServerRuntimeStartup, make); | |
| 500 | export const layer = layerWithOptions(); |
apps/server/src/service-launcher.ts created+1
| ... | ... | @@ -0,0 +1 @@ |
| 1 | import "./serviceLauncher.ts"; |
apps/server/src/serviceLauncher.test.ts created+199
| ... | ... | @@ -0,0 +1,199 @@ |
| 1 | import * as NodeServices from "@effect/platform-node/NodeServices"; | |
| 2 | import { assert, it } from "@effect/vitest"; | |
| 3 | import * as Effect from "effect/Effect"; | |
| 4 | import * as FileSystem from "effect/FileSystem"; | |
| 5 | import * as Path from "effect/Path"; | |
| 6 | ||
| 7 | import { Launcher, readServiceState, writeServiceState } from "./serviceLauncher.ts"; | |
| 8 | import { | |
| 9 | compareExactServiceVersions, | |
| 10 | decodeServiceState, | |
| 11 | isExactServiceVersion, | |
| 12 | } from "./cloud/serviceProtocol.ts"; | |
| 13 | ||
| 14 | it("accepts only exact semantic versions", () => { | |
| 15 | for (const version of ["0.0.0", "1.2.3", "1.2.3-alpha.1", "1.2.3-0", "1.2.3+001"]) { | |
| 16 | assert.isTrue(isExactServiceVersion(version), version); | |
| 17 | } | |
| 18 | for (const version of ["latest", "01.2.3", "1.2.3-01", "1.2.3-alpha..1", "1.2.3+."]) { | |
| 19 | assert.isFalse(isExactServiceVersion(version), version); | |
| 20 | } | |
| 21 | }); | |
| 22 | ||
| 23 | it("orders exact semantic versions without treating build metadata as precedence", () => { | |
| 24 | assert.equal(compareExactServiceVersions("1.2.3", "1.2.3"), 0); | |
| 25 | assert.equal(compareExactServiceVersions("1.2.4", "1.2.3"), 1); | |
| 26 | assert.equal(compareExactServiceVersions("2.0.0-alpha.1", "2.0.0-alpha.2"), -1); | |
| 27 | assert.equal(compareExactServiceVersions("2.0.0-alpha.2", "2.0.0-alpha.beta"), -1); | |
| 28 | assert.equal(compareExactServiceVersions("2.0.0-alpha-beta", "2.0.0-alpha-alpha"), 1); | |
| 29 | assert.equal(compareExactServiceVersions("2.0.0", "2.0.0-rc.1"), 1); | |
| 30 | assert.equal(compareExactServiceVersions("2.0.0+one", "2.0.0+two"), 0); | |
| 31 | }); | |
| 32 | ||
| 33 | it("rejects contradictory service state", () => { | |
| 34 | assert.isUndefined( | |
| 35 | decodeServiceState({ | |
| 36 | protocol: 1, | |
| 37 | activeVersion: "0.0.31", | |
| 38 | update: { | |
| 39 | id: "update-1", | |
| 40 | fromVersion: "0.0.30", | |
| 41 | targetVersion: "0.0.32", | |
| 42 | status: "pending", | |
| 43 | }, | |
| 44 | }), | |
| 45 | ); | |
| 46 | ||
| 47 | assert.isUndefined( | |
| 48 | decodeServiceState({ | |
| 49 | protocol: 1, | |
| 50 | activeVersion: "1.0.0", | |
| 51 | update: { | |
| 52 | id: "update-2", | |
| 53 | fromVersion: "1.0.0", | |
| 54 | targetVersion: "0.9.0", | |
| 55 | status: "pending", | |
| 56 | }, | |
| 57 | }), | |
| 58 | ); | |
| 59 | }); | |
| 60 | ||
| 61 | it.layer(NodeServices.layer)("service state persistence", (it) => { | |
| 62 | it.effect("durably replaces and strictly reads one state document", () => | |
| 63 | Effect.gen(function* () { | |
| 64 | const fs = yield* FileSystem.FileSystem; | |
| 65 | const path = yield* Path.Path; | |
| 66 | const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-test-" }); | |
| 67 | const statePath = path.join(root, "runtime", "service-state.json"); | |
| 68 | const state = { | |
| 69 | protocol: 1, | |
| 70 | activeVersion: "0.0.31", | |
| 71 | } as const; | |
| 72 | ||
| 73 | yield* Effect.promise(() => writeServiceState(statePath, state)); | |
| 74 | assert.deepEqual(yield* Effect.promise(() => readServiceState(statePath)), state); | |
| 75 | }), | |
| 76 | ); | |
| 77 | ||
| 78 | it.effect("serializes shutdown with launcher recovery", () => | |
| 79 | Effect.gen(function* () { | |
| 80 | const fs = yield* FileSystem.FileSystem; | |
| 81 | const path = yield* Path.Path; | |
| 82 | const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-stop-" }); | |
| 83 | const statePath = path.join(root, "runtime", "service-state.json"); | |
| 84 | const versionDir = path.join(root, "runtime", "versions", "1.0.0"); | |
| 85 | const entryPath = path.join(versionDir, "node_modules", "t3", "dist", "bin.mjs"); | |
| 86 | yield* fs.makeDirectory(path.dirname(entryPath), { recursive: true }); | |
| 87 | yield* fs.writeFileString(entryPath, "setInterval(() => {}, 1_000);\n"); | |
| 88 | yield* fs.writeFileString(path.join(versionDir, ".install-complete"), "1.0.0\n"); | |
| 89 | yield* Effect.promise(() => | |
| 90 | writeServiceState(statePath, { | |
| 91 | protocol: 1, | |
| 92 | activeVersion: "1.0.0", | |
| 93 | }), | |
| 94 | ); | |
| 95 | ||
| 96 | const launcher = new Launcher(root, yield* Effect.promise(() => readServiceState(statePath))); | |
| 97 | const running = launcher.run(); | |
| 98 | yield* Effect.promise(() => launcher.stop("SIGTERM")); | |
| 99 | yield* Effect.promise(() => running); | |
| 100 | }), | |
| 101 | ); | |
| 102 | ||
| 103 | it.effect("commits only after the trial reports prepared", () => | |
| 104 | Effect.gen(function* () { | |
| 105 | const fs = yield* FileSystem.FileSystem; | |
| 106 | const path = yield* Path.Path; | |
| 107 | const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-flow-" }); | |
| 108 | const statePath = path.join(root, "runtime", "service-state.json"); | |
| 109 | const childSource = ` | |
| 110 | const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT); | |
| 111 | if (context.update?.status === "pending") { | |
| 112 | process.send({ type: "prepared", updateId: context.update.id }); | |
| 113 | process.on("message", (message) => { | |
| 114 | if (message.type === "committed") process.exit(0); | |
| 115 | }); | |
| 116 | } else if (context.update === undefined) { | |
| 117 | process.send({ type: "request-update", targetVersion: "1.1.0" }); | |
| 118 | setInterval(() => {}, 1_000); | |
| 119 | } else { | |
| 120 | process.exit(0); | |
| 121 | } | |
| 122 | `; | |
| 123 | for (const version of ["1.0.0", "1.1.0"]) { | |
| 124 | const versionDir = path.join(root, "runtime", "versions", version); | |
| 125 | const entryPath = path.join(versionDir, "node_modules", "t3", "dist", "bin.mjs"); | |
| 126 | yield* fs.makeDirectory(path.dirname(entryPath), { recursive: true }); | |
| 127 | yield* fs.writeFileString(entryPath, childSource); | |
| 128 | yield* fs.writeFileString(path.join(versionDir, ".install-complete"), `${version}\n`); | |
| 129 | } | |
| 130 | yield* Effect.promise(() => | |
| 131 | writeServiceState(statePath, { | |
| 132 | protocol: 1, | |
| 133 | activeVersion: "1.0.0", | |
| 134 | }), | |
| 135 | ); | |
| 136 | ||
| 137 | const launcher = new Launcher(root, yield* Effect.promise(() => readServiceState(statePath))); | |
| 138 | yield* Effect.promise(() => | |
| 139 | launcher.run().then( | |
| 140 | () => Promise.reject(new Error("launcher unexpectedly completed")), | |
| 141 | () => Promise.resolve(), | |
| 142 | ), | |
| 143 | ); | |
| 144 | ||
| 145 | const state = yield* Effect.promise(() => readServiceState(statePath)); | |
| 146 | assert.equal(state.activeVersion, "1.1.0"); | |
| 147 | assert.equal(state.update?.status, "committed"); | |
| 148 | }), | |
| 149 | ); | |
| 150 | ||
| 151 | it.effect("rolls back a trial that reports the wrong update ID", () => | |
| 152 | Effect.gen(function* () { | |
| 153 | const fs = yield* FileSystem.FileSystem; | |
| 154 | const path = yield* Path.Path; | |
| 155 | const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-rollback-" }); | |
| 156 | const statePath = path.join(root, "runtime", "service-state.json"); | |
| 157 | const childSource = ` | |
| 158 | const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT); | |
| 159 | if (context.update?.status === "pending") { | |
| 160 | process.send({ type: "prepared", updateId: "wrong-update" }); | |
| 161 | } else if (context.update === undefined) { | |
| 162 | process.send({ type: "request-update", targetVersion: "1.1.0" }); | |
| 163 | setInterval(() => {}, 1_000); | |
| 164 | } else { | |
| 165 | process.exit(0); | |
| 166 | } | |
| 167 | `; | |
| 168 | for (const version of ["1.0.0", "1.1.0"]) { | |
| 169 | const versionDir = path.join(root, "runtime", "versions", version); | |
| 170 | const entryPath = path.join(versionDir, "node_modules", "t3", "dist", "bin.mjs"); | |
| 171 | yield* fs.makeDirectory(path.dirname(entryPath), { recursive: true }); | |
| 172 | yield* fs.writeFileString(entryPath, childSource); | |
| 173 | yield* fs.writeFileString(path.join(versionDir, ".install-complete"), `${version}\n`); | |
| 174 | } | |
| 175 | yield* Effect.promise(() => | |
| 176 | writeServiceState(statePath, { | |
| 177 | protocol: 1, | |
| 178 | activeVersion: "1.0.0", | |
| 179 | }), | |
| 180 | ); | |
| 181 | ||
| 182 | const launcher = new Launcher(root, yield* Effect.promise(() => readServiceState(statePath))); | |
| 183 | yield* Effect.promise(() => | |
| 184 | launcher.run().then( | |
| 185 | () => Promise.reject(new Error("launcher unexpectedly completed")), | |
| 186 | () => Promise.resolve(), | |
| 187 | ), | |
| 188 | ); | |
| 189 | ||
| 190 | const state = yield* Effect.promise(() => readServiceState(statePath)); | |
| 191 | assert.equal(state.activeVersion, "1.0.0"); | |
| 192 | assert.equal(state.update?.status, "rolled-back"); | |
| 193 | assert.equal( | |
| 194 | state.update?.status === "rolled-back" ? state.update.reason : undefined, | |
| 195 | "invalid-prepared", | |
| 196 | ); | |
| 197 | }), | |
| 198 | ); | |
| 199 | }); |
apps/server/src/serviceLauncher.ts created+458
| ... | ... | @@ -0,0 +1,458 @@ |
| 1 | // @effect-diagnostics nodeBuiltinImport:off | |
| 2 | // @effect-diagnostics globalDate:off | |
| 3 | // @effect-diagnostics globalTimers:off | |
| 4 | // This file is shipped as a standalone bundle and copied to a stable path by | |
| 5 | // `t3 service update`. Keep runtime imports limited to Node built-ins. | |
| 6 | import * as NodeChildProcess from "node:child_process"; | |
| 7 | import * as NodeCrypto from "node:crypto"; | |
| 8 | import * as NodeFSP from "node:fs/promises"; | |
| 9 | import * as NodePath from "node:path"; | |
| 10 | ||
| 11 | import type { | |
| 12 | PendingServiceUpdate, | |
| 13 | ServiceLauncherChildMessage, | |
| 14 | ServiceLauncherContext, | |
| 15 | ServiceLauncherParentMessage, | |
| 16 | ServiceState, | |
| 17 | ServiceUpdateRecord, | |
| 18 | } from "./cloud/serviceProtocol.ts"; | |
| 19 | import { | |
| 20 | compareExactServiceVersions, | |
| 21 | decodeServiceLauncherChildMessage, | |
| 22 | isExactServiceVersion, | |
| 23 | parseServiceState, | |
| 24 | SERVICE_LAUNCHER_CONTEXT_ENV, | |
| 25 | SERVICE_LAUNCHER_PROTOCOL, | |
| 26 | SERVICE_STATE_FILE, | |
| 27 | } from "./cloud/serviceProtocol.ts"; | |
| 28 | ||
| 29 | const HANDOFF_DELAY_MS = 2_000; | |
| 30 | const PREPARED_TIMEOUT_MS = 120_000; | |
| 31 | const TERMINATE_GRACE_MS = 5_000; | |
| 32 | ||
| 33 | type TerminalStatus = "committed" | "rolled-back" | "failed"; | |
| 34 | type ChildRole = "active" | "trial"; | |
| 35 | ||
| 36 | interface ManagedChild { | |
| 37 | readonly version: string; | |
| 38 | role: ChildRole; | |
| 39 | readonly process: NodeChildProcess.ChildProcess; | |
| 40 | } | |
| 41 | ||
| 42 | const runtimePaths = (baseDir: string, version: string) => { | |
| 43 | const versionDir = NodePath.join(baseDir, "runtime", "versions", version); | |
| 44 | return { | |
| 45 | versionDir, | |
| 46 | entryPath: NodePath.join(versionDir, "node_modules", "t3", "dist", "bin.mjs"), | |
| 47 | sentinelPath: NodePath.join(versionDir, ".install-complete"), | |
| 48 | }; | |
| 49 | }; | |
| 50 | ||
| 51 | export async function readServiceState(filePath: string): Promise<ServiceState> { | |
| 52 | const contents = await NodeFSP.readFile(filePath, "utf8"); | |
| 53 | const state = parseServiceState(contents); | |
| 54 | if (state === undefined) throw new Error("Service state is invalid or unsupported."); | |
| 55 | return state; | |
| 56 | } | |
| 57 | ||
| 58 | /** Durable same-directory replacement used for every runtime state transition. */ | |
| 59 | export async function writeServiceState(filePath: string, state: ServiceState): Promise<void> { | |
| 60 | const directory = NodePath.dirname(filePath); | |
| 61 | await NodeFSP.mkdir(directory, { recursive: true, mode: 0o700 }); | |
| 62 | const tempPath = NodePath.join( | |
| 63 | directory, | |
| 64 | `.${NodePath.basename(filePath)}.${process.pid}.${NodeCrypto.randomUUID()}`, | |
| 65 | ); | |
| 66 | let handle: NodeFSP.FileHandle | undefined; | |
| 67 | try { | |
| 68 | handle = await NodeFSP.open(tempPath, "wx", 0o600); | |
| 69 | await handle.writeFile(`${JSON.stringify(state, null, 2)}\n`, "utf8"); | |
| 70 | await handle.sync(); | |
| 71 | await handle.close(); | |
| 72 | handle = undefined; | |
| 73 | await NodeFSP.rename(tempPath, filePath); | |
| 74 | const directoryHandle = await NodeFSP.open(directory, "r"); | |
| 75 | try { | |
| 76 | await directoryHandle.sync(); | |
| 77 | } finally { | |
| 78 | await directoryHandle.close(); | |
| 79 | } | |
| 80 | } finally { | |
| 81 | await handle?.close().catch(() => undefined); | |
| 82 | await NodeFSP.rm(tempPath, { force: true }).catch(() => undefined); | |
| 83 | } | |
| 84 | } | |
| 85 | ||
| 86 | async function runtimeExists(baseDir: string, version: string): Promise<boolean> { | |
| 87 | const paths = runtimePaths(baseDir, version); | |
| 88 | try { | |
| 89 | const [entry, sentinel] = await Promise.all([ | |
| 90 | NodeFSP.stat(paths.entryPath), | |
| 91 | NodeFSP.readFile(paths.sentinelPath, "utf8"), | |
| 92 | ]); | |
| 93 | return entry.isFile() && sentinel.trim() === version; | |
| 94 | } catch { | |
| 95 | return false; | |
| 96 | } | |
| 97 | } | |
| 98 | ||
| 99 | function terminalUpdate<S extends TerminalStatus>(input: { | |
| 100 | readonly pending: PendingServiceUpdate; | |
| 101 | readonly status: S; | |
| 102 | readonly reason?: string; | |
| 103 | }): Exclude<ServiceUpdateRecord, PendingServiceUpdate> & { readonly status: S } { | |
| 104 | return { | |
| 105 | id: input.pending.id, | |
| 106 | fromVersion: input.pending.fromVersion, | |
| 107 | targetVersion: input.pending.targetVersion, | |
| 108 | status: input.status, | |
| 109 | ...(input.reason === undefined ? {} : { reason: input.reason }), | |
| 110 | }; | |
| 111 | } | |
| 112 | ||
| 113 | function sendMessage( | |
| 114 | child: NodeChildProcess.ChildProcess, | |
| 115 | message: ServiceLauncherParentMessage, | |
| 116 | ): Promise<void> { | |
| 117 | return new Promise((resolve, reject) => { | |
| 118 | if (!child.connected || child.send === undefined) { | |
| 119 | reject(new Error("service child IPC is disconnected.")); | |
| 120 | return; | |
| 121 | } | |
| 122 | child.send(message, (error) => (error === null ? resolve() : reject(error))); | |
| 123 | }); | |
| 124 | } | |
| 125 | ||
| 126 | function waitForExit(child: NodeChildProcess.ChildProcess): Promise<void> { | |
| 127 | if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve(); | |
| 128 | return new Promise((resolve) => child.once("exit", () => resolve())); | |
| 129 | } | |
| 130 | ||
| 131 | async function terminateChild( | |
| 132 | child: NodeChildProcess.ChildProcess, | |
| 133 | signal: NodeJS.Signals = "SIGTERM", | |
| 134 | ): Promise<void> { | |
| 135 | if (child.exitCode !== null || child.signalCode !== null) return; | |
| 136 | child.kill(signal); | |
| 137 | const force = setTimeout(() => child.kill("SIGKILL"), TERMINATE_GRACE_MS); | |
| 138 | try { | |
| 139 | await waitForExit(child); | |
| 140 | } finally { | |
| 141 | clearTimeout(force); | |
| 142 | } | |
| 143 | } | |
| 144 | ||
| 145 | export class Launcher { | |
| 146 | readonly #baseDir: string; | |
| 147 | readonly #statePath: string; | |
| 148 | #state: ServiceState; | |
| 149 | #child: ManagedChild | null = null; | |
| 150 | #timer: NodeJS.Timeout | undefined; | |
| 151 | #transitions: Promise<void> = Promise.resolve(); | |
| 152 | #stopping = false; | |
| 153 | #done = false; | |
| 154 | readonly #completion = Promise.withResolvers<void>(); | |
| 155 | ||
| 156 | constructor(baseDir: string, state: ServiceState) { | |
| 157 | this.#baseDir = baseDir; | |
| 158 | this.#statePath = NodePath.join(baseDir, "runtime", SERVICE_STATE_FILE); | |
| 159 | this.#state = state; | |
| 160 | } | |
| 161 | ||
| 162 | async run(): Promise<void> { | |
| 163 | const onSigterm = () => void this.stop("SIGTERM"); | |
| 164 | const onSigint = () => void this.stop("SIGINT"); | |
| 165 | process.once("SIGTERM", onSigterm); | |
| 166 | process.once("SIGINT", onSigint); | |
| 167 | try { | |
| 168 | this.#enqueue(() => this.#recover()); | |
| 169 | await this.#completion.promise; | |
| 170 | } finally { | |
| 171 | process.off("SIGTERM", onSigterm); | |
| 172 | process.off("SIGINT", onSigint); | |
| 173 | } | |
| 174 | } | |
| 175 | ||
| 176 | #enqueue(transition: () => Promise<void>): void { | |
| 177 | this.#transitions = this.#transitions | |
| 178 | .then(transition, transition) | |
| 179 | .catch((cause: unknown) => | |
| 180 | this.#fatal(cause instanceof Error ? cause : new Error(String(cause))), | |
| 181 | ); | |
| 182 | } | |
| 183 | ||
| 184 | async #fatal(error: Error): Promise<void> { | |
| 185 | if (this.#done) return; | |
| 186 | this.#done = true; | |
| 187 | this.#stopping = true; | |
| 188 | this.#clearTimer(); | |
| 189 | const child = this.#child?.process; | |
| 190 | this.#child = null; | |
| 191 | if (child !== undefined) await terminateChild(child); | |
| 192 | this.#completion.reject(error); | |
| 193 | } | |
| 194 | ||
| 195 | async stop(signal: NodeJS.Signals): Promise<void> { | |
| 196 | if (this.#stopping) { | |
| 197 | await this.#completion.promise.catch(() => undefined); | |
| 198 | return; | |
| 199 | } | |
| 200 | this.#stopping = true; | |
| 201 | this.#clearTimer(); | |
| 202 | this.#enqueue(async () => { | |
| 203 | const child = this.#child?.process; | |
| 204 | this.#child = null; | |
| 205 | if (child !== undefined) await terminateChild(child, signal); | |
| 206 | this.#done = true; | |
| 207 | this.#completion.resolve(); | |
| 208 | }); | |
| 209 | await this.#completion.promise.catch(() => undefined); | |
| 210 | } | |
| 211 | ||
| 212 | #clearTimer(): void { | |
| 213 | clearTimeout(this.#timer); | |
| 214 | this.#timer = undefined; | |
| 215 | } | |
| 216 | ||
| 217 | async #recover(): Promise<void> { | |
| 218 | const update = this.#state.update; | |
| 219 | if (update?.status !== "pending") { | |
| 220 | await this.#startChild(this.#state.activeVersion, "active", update); | |
| 221 | return; | |
| 222 | } | |
| 223 | if (!(await runtimeExists(this.#baseDir, update.targetVersion))) { | |
| 224 | await this.#returnToPrevious(update, "failed", "target-runtime-missing"); | |
| 225 | return; | |
| 226 | } | |
| 227 | await this.#startTrial(update); | |
| 228 | } | |
| 229 | ||
| 230 | async #startTrial(pending: PendingServiceUpdate): Promise<void> { | |
| 231 | try { | |
| 232 | await this.#startChild(pending.targetVersion, "trial", pending); | |
| 233 | } catch { | |
| 234 | await this.#returnToPrevious(pending, "failed", "candidate-start-failed"); | |
| 235 | } | |
| 236 | } | |
| 237 | ||
| 238 | async #startChild(version: string, role: ChildRole, update?: ServiceUpdateRecord): Promise<void> { | |
| 239 | if (this.#stopping) return; | |
| 240 | if (!(await runtimeExists(this.#baseDir, version))) { | |
| 241 | throw new Error(`Selected t3@${version} runtime is missing or incomplete.`); | |
| 242 | } | |
| 243 | if (this.#stopping) return; | |
| 244 | const paths = runtimePaths(this.#baseDir, version); | |
| 245 | const context: ServiceLauncherContext = { | |
| 246 | protocol: SERVICE_LAUNCHER_PROTOCOL, | |
| 247 | childVersion: version, | |
| 248 | ...(update === undefined ? {} : { update }), | |
| 249 | }; | |
| 250 | const child = NodeChildProcess.spawn(process.execPath, [paths.entryPath, "serve"], { | |
| 251 | env: { ...process.env, [SERVICE_LAUNCHER_CONTEXT_ENV]: JSON.stringify(context) }, | |
| 252 | stdio: ["inherit", "inherit", "inherit", "ipc"], | |
| 253 | }); | |
| 254 | await new Promise<void>((resolve, reject) => { | |
| 255 | const onError = (error: Error) => reject(error); | |
| 256 | child.once("error", onError); | |
| 257 | child.once("spawn", () => { | |
| 258 | child.removeListener("error", onError); | |
| 259 | child.on("error", (error) => this.#enqueue(() => Promise.reject(error))); | |
| 260 | resolve(); | |
| 261 | }); | |
| 262 | }); | |
| 263 | if (this.#stopping) { | |
| 264 | await terminateChild(child); | |
| 265 | return; | |
| 266 | } | |
| 267 | ||
| 268 | const managed: ManagedChild = { | |
| 269 | version, | |
| 270 | role, | |
| 271 | process: child, | |
| 272 | }; | |
| 273 | this.#child = managed; | |
| 274 | child.on("message", (value) => { | |
| 275 | const message = decodeServiceLauncherChildMessage(value); | |
| 276 | if (message !== undefined) this.#enqueue(() => this.#handleMessage(managed, message)); | |
| 277 | }); | |
| 278 | child.once("exit", (code, signal) => | |
| 279 | this.#enqueue(() => this.#handleExit(managed, code, signal)), | |
| 280 | ); | |
| 281 | ||
| 282 | if (role === "trial") { | |
| 283 | this.#timer = setTimeout( | |
| 284 | () => this.#enqueue(() => this.#handlePreparedTimeout(managed)), | |
| 285 | PREPARED_TIMEOUT_MS, | |
| 286 | ); | |
| 287 | } | |
| 288 | } | |
| 289 | ||
| 290 | async #handleMessage(child: ManagedChild, message: ServiceLauncherChildMessage): Promise<void> { | |
| 291 | if (this.#child !== child || this.#stopping) return; | |
| 292 | if (message.type === "request-update") { | |
| 293 | await this.#handleUpdateRequest(child, message); | |
| 294 | return; | |
| 295 | } | |
| 296 | await this.#handlePrepared(child, message.updateId); | |
| 297 | } | |
| 298 | ||
| 299 | async #handleUpdateRequest( | |
| 300 | child: ManagedChild, | |
| 301 | message: Extract<ServiceLauncherChildMessage, { readonly type: "request-update" }>, | |
| 302 | ): Promise<void> { | |
| 303 | const reject = (reason: string) => | |
| 304 | sendMessage(child.process, { type: "update-rejected", reason }); | |
| 305 | if (child.role !== "active") { | |
| 306 | await reject("Only the active server can request an update."); | |
| 307 | return; | |
| 308 | } | |
| 309 | if (child.version !== this.#state.activeVersion) { | |
| 310 | await reject("The requesting server is not the selected active version."); | |
| 311 | return; | |
| 312 | } | |
| 313 | if (this.#state.update?.status === "pending") { | |
| 314 | await reject("Another server update is already pending."); | |
| 315 | return; | |
| 316 | } | |
| 317 | if (!isExactServiceVersion(message.targetVersion)) { | |
| 318 | await reject("The requested target is not an exact version."); | |
| 319 | return; | |
| 320 | } | |
| 321 | if (compareExactServiceVersions(message.targetVersion, child.version) <= 0) { | |
| 322 | await reject("Remote updates must select a newer server version."); | |
| 323 | return; | |
| 324 | } | |
| 325 | if (!(await runtimeExists(this.#baseDir, message.targetVersion))) { | |
| 326 | await reject("The requested target runtime is missing or incomplete."); | |
| 327 | return; | |
| 328 | } | |
| 329 | ||
| 330 | const pending: PendingServiceUpdate = { | |
| 331 | id: NodeCrypto.randomUUID(), | |
| 332 | fromVersion: child.version, | |
| 333 | targetVersion: message.targetVersion, | |
| 334 | status: "pending", | |
| 335 | }; | |
| 336 | const next: ServiceState = { ...this.#state, update: pending }; | |
| 337 | await writeServiceState(this.#statePath, next); | |
| 338 | this.#state = next; | |
| 339 | await sendMessage(child.process, { type: "update-accepted", updateId: pending.id }); | |
| 340 | this.#timer = setTimeout(() => this.#enqueue(() => this.#beginTrial(child)), HANDOFF_DELAY_MS); | |
| 341 | } | |
| 342 | ||
| 343 | async #beginTrial(child: ManagedChild): Promise<void> { | |
| 344 | const pending = this.#state.update; | |
| 345 | if (this.#child !== child || child.role !== "active" || pending?.status !== "pending") { | |
| 346 | return; | |
| 347 | } | |
| 348 | this.#timer = undefined; | |
| 349 | this.#child = null; | |
| 350 | await terminateChild(child.process); | |
| 351 | await this.#startTrial(pending); | |
| 352 | } | |
| 353 | ||
| 354 | async #handlePrepared(child: ManagedChild, updateId: string): Promise<void> { | |
| 355 | const pending = this.#state.update; | |
| 356 | if ( | |
| 357 | child.role !== "trial" || | |
| 358 | pending?.status !== "pending" || | |
| 359 | pending.id !== updateId || | |
| 360 | pending.targetVersion !== child.version | |
| 361 | ) { | |
| 362 | if (child.role === "trial" && pending?.status === "pending") { | |
| 363 | await this.#returnToPrevious(pending, "rolled-back", "invalid-prepared", child); | |
| 364 | return; | |
| 365 | } | |
| 366 | throw new Error("Trial child reported prepared for an unexpected update."); | |
| 367 | } | |
| 368 | this.#clearTimer(); | |
| 369 | const committed = terminalUpdate({ pending, status: "committed" }); | |
| 370 | const next: ServiceState = { | |
| 371 | ...this.#state, | |
| 372 | activeVersion: pending.targetVersion, | |
| 373 | update: committed, | |
| 374 | }; | |
| 375 | await writeServiceState(this.#statePath, next); | |
| 376 | this.#state = next; | |
| 377 | child.role = "active"; | |
| 378 | await sendMessage(child.process, { type: "committed", updateId: committed.id }); | |
| 379 | } | |
| 380 | ||
| 381 | async #handlePreparedTimeout(child: ManagedChild): Promise<void> { | |
| 382 | const pending = this.#state.update; | |
| 383 | if (this.#child !== child || child.role !== "trial" || pending?.status !== "pending") { | |
| 384 | return; | |
| 385 | } | |
| 386 | this.#timer = undefined; | |
| 387 | await this.#returnToPrevious(pending, "rolled-back", "prepared-timeout", child); | |
| 388 | } | |
| 389 | ||
| 390 | async #handleExit( | |
| 391 | child: ManagedChild, | |
| 392 | code: number | null, | |
| 393 | signal: NodeJS.Signals | null, | |
| 394 | ): Promise<void> { | |
| 395 | if (this.#child !== child || this.#stopping) return; | |
| 396 | this.#child = null; | |
| 397 | if (child.role === "trial") { | |
| 398 | this.#clearTimer(); | |
| 399 | const pending = this.#state.update; | |
| 400 | if (pending?.status !== "pending") { | |
| 401 | throw new Error("Trial child exited without matching pending state."); | |
| 402 | } | |
| 403 | await this.#returnToPrevious( | |
| 404 | pending, | |
| 405 | "rolled-back", | |
| 406 | `candidate-exited:${String(code ?? signal ?? "unknown")}`, | |
| 407 | ); | |
| 408 | return; | |
| 409 | } | |
| 410 | ||
| 411 | this.#clearTimer(); | |
| 412 | const pending = this.#state.update; | |
| 413 | if (pending?.status === "pending") { | |
| 414 | await this.#startTrial(pending); | |
| 415 | return; | |
| 416 | } | |
| 417 | throw new Error(`Active child exited unexpectedly (${String(code ?? signal ?? "unknown")}).`); | |
| 418 | } | |
| 419 | ||
| 420 | async #returnToPrevious( | |
| 421 | pending: PendingServiceUpdate, | |
| 422 | status: "rolled-back" | "failed", | |
| 423 | reason: string, | |
| 424 | child?: ManagedChild, | |
| 425 | ): Promise<void> { | |
| 426 | const outcome = terminalUpdate({ pending, status, reason }); | |
| 427 | const next: ServiceState = { | |
| 428 | ...this.#state, | |
| 429 | activeVersion: pending.fromVersion, | |
| 430 | update: outcome, | |
| 431 | }; | |
| 432 | await writeServiceState(this.#statePath, next); | |
| 433 | this.#state = next; | |
| 434 | if (child !== undefined) { | |
| 435 | this.#child = null; | |
| 436 | await terminateChild(child.process); | |
| 437 | } | |
| 438 | await this.#startChild(next.activeVersion, "active", outcome); | |
| 439 | } | |
| 440 | } | |
| 441 | ||
| 442 | async function main(): Promise<void> { | |
| 443 | const baseDir = process.env.T3CODE_HOME?.trim(); | |
| 444 | if (baseDir === undefined || baseDir === "") { | |
| 445 | throw new Error("T3CODE_HOME is required by the T3 Code service launcher."); | |
| 446 | } | |
| 447 | const statePath = NodePath.join(baseDir, "runtime", SERVICE_STATE_FILE); | |
| 448 | const state = await readServiceState(statePath); | |
| 449 | await new Launcher(baseDir, state).run(); | |
| 450 | } | |
| 451 | ||
| 452 | if (import.meta.main) { | |
| 453 | main().catch((cause: unknown) => { | |
| 454 | const error = cause instanceof Error ? cause : new Error(String(cause)); | |
| 455 | process.stderr.write(`[service-launcher] ${error.message}\n`); | |
| 456 | process.exitCode = 1; | |
| 457 | }); | |
| 458 | } |
docs/internals/server-updates.md+68-120
| ... | ... | @@ -2,139 +2,87 @@ |
| 2 | 2 | |
| 3 | 3 | > For maintainers. Using T3 Code? See [docs/user](../user/). |
| 4 | 4 | |
| 5 | T3 Code can update a connected server to the exact version of the client that detected version | |
| 6 | drift. This path exists primarily for remote environments, where the user may not have a terminal | |
| 7 | open on the server machine. | |
| 8 | ||
| 9 | The feature has three boundaries: | |
| 10 | ||
| 11 | - the server advertises whether and how it can be replaced; | |
| 12 | - the client chooses the matching user action; | |
| 13 | - the server installs and verifies the replacement before handing off the process. | |
| 14 | ||
| 15 | ## Detection and Presentation | |
| 16 | ||
| 17 | `ExecutionEnvironmentDescriptor` includes the server version and an optional | |
| 18 | `capabilities.serverSelfUpdate` value. Progress-capable servers also advertise | |
| 19 | `capabilities.serverSelfUpdateProgress`. The client compares the server version with `APP_VERSION` | |
| 20 | after loading server config. | |
| 21 | ||
| 22 | The optional capability is intentionally backward compatible. An older server does not know about | |
| 23 | the field, so a missing value means the client must offer a manual relaunch instead of sending an | |
| 24 | unknown RPC. | |
| 25 | ||
| 26 | The shared `ServerUpdateAction` is rendered in both user-facing version-drift surfaces: | |
| 27 | ||
| 28 | - the conversation banner in `ChatView`; | |
| 29 | - primary and saved environment rows in **Settings** → **Connections**. | |
| 30 | ||
| 31 | Both surfaces target the client's exact version. When the reconnected server reports that version, | |
| 32 | the mismatch and action disappear. | |
| 33 | ||
| 34 | The operation state lives in `packages/client-runtime`, keyed by environment. Both web surfaces read | |
| 35 | the same `downloading`, `installing`, or `resuming` state, so route changes do not own or cancel the | |
| 36 | operation. | |
| 37 | ||
| 38 | ## Capability Selection | |
| 39 | ||
| 40 | The server resolves its capability once at startup and publishes it in the environment descriptor. | |
| 41 | ||
| 42 | | Advertised value | Process shape | Client behavior | | |
| 43 | | ----------------- | --------------------------------------------------------------------------------------------- | --------------------------------------------------------------------- | | |
| 44 | | `boot-service` | Linux server running under the T3-managed systemd user service | Call the update RPC; the service unit is replaced and restarted. | | |
| 45 | | `respawn` | Published npm CLI running in the foreground on macOS or Linux | Call the update RPC; the process hands off to a detached replacement. | | |
| 46 | | `desktop-managed` | Backend supervised by the desktop app | Tell the user to update the desktop app on the server machine. | | |
| 47 | | absent | Older server, development checkout, Windows foreground process, or an unrecognized supervisor | Offer the exact manual relaunch command. | | |
| 48 | ||
| 49 | Desktop ownership takes precedence over process-shape detection. A desktop-managed backend must | |
| 50 | never spawn a second CLI server beside the app-owned process. Likewise, a process launched by an | |
| 51 | unrecognized systemd unit does not claim the foreground respawn path because its supervisor could | |
| 52 | bring the old version back. | |
| 53 | ||
| 54 | ## Update Flow | |
| 55 | ||
| 56 | ```mermaid | |
| 57 | flowchart TD | |
| 58 | A[Client detects different versions] --> B{Advertised update path} | |
| 59 | B -->|desktop-managed| C[Update desktop app on server machine] | |
| 60 | B -->|missing| D[Copy exact manual relaunch command] | |
| 61 | B -->|boot-service or respawn| E{Progress capability} | |
| 62 | E -->|present| F[server.updateServerWithProgress] | |
| 63 | E -->|missing| G[server.updateServer fallback] | |
| 64 | F --> H[Install exact t3 version in pinned runtime] | |
| 65 | G --> H | |
| 66 | H --> I[Run version preflight] | |
| 67 | I -->|bad code or version| J[Remove candidate runtime and keep current server] | |
| 68 | I -->|cannot run preflight| J2[Keep candidate and current server] | |
| 69 | I -->|passes| K{Handoff method} | |
| 70 | K -->|boot-service| L[Rewrite and restart T3 systemd unit] | |
| 71 | K -->|respawn| M[Start delayed replacement and exit current process] | |
| 72 | L --> N[Reconnect with fresh backoff] | |
| 73 | M --> N | |
| 74 | N --> O[Replacement publishes ready at target version] | |
| 75 | ``` | |
| 5 | Remote server updates use one stable systemd launcher. Foreground CLI processes do not self-update, | |
| 6 | and a running server never edits its systemd unit or durable service state. | |
| 7 | ||
| 8 | ## Ownership | |
| 9 | ||
| 10 | The service files under `<baseDir>/runtime` are: | |
| 11 | ||
| 12 | - `service-launcher.mjs`, the stable process selected by systemd; | |
| 13 | - `service-state.json`, the launcher's durable selection state; | |
| 14 | - `versions/<version>`, immutable exact-version npm installs. | |
| 15 | ||
| 16 | The launcher is the only runtime writer of `service-state.json`. `t3 service install` and | |
| 17 | `t3 service update` may replace the launcher and state while the unit is stopped. Server children | |
| 18 | only communicate with the launcher over their inherited IPC channel. | |
| 76 | 19 | |
| 77 | Both update RPCs require the environment's `orchestration:operate` authorization scope. Their | |
| 78 | payload accepts only an exact npm version, including an exact prerelease version; dist-tags such as | |
| 79 | `latest` and `nightly` are rejected. The unary `server.updateServer` method remains available so a | |
| 80 | new client can still repair skew with an older server. | |
| 20 | The state contains one active version and, at most, one update record: | |
| 81 | 21 | |
| 82 | The update service permits one update at a time. It installs `t3@<version>` under | |
| 83 | `<baseDir>/runtime/versions/<version>` and writes an install-complete sentinel only after npm exits | |
| 84 | successfully. Boot-service setup and self-update share the same process-wide installation lock, so | |
| 85 | they cannot mutate a pinned runtime concurrently. | |
| 22 | - `pending A → B` selects B as a retryable trial; | |
| 23 | - `committed A → B` selects B for ordinary restarts; | |
| 24 | - `rolled-back A → B` or `failed A → B` selects A; | |
| 25 | - invalid state fails closed so systemd cannot guess at a runtime. | |
| 86 | 26 | |
| 87 | Before any restart, the current Node executable runs the replacement with `--version`. A failed | |
| 88 | install, failed preflight, or wrong reported version leaves the current server running. | |
| 27 | Every write uses same-directory replacement plus file and directory fsync. | |
| 89 | 28 | |
| 90 | Candidate cleanup is narrower than "any failed preflight". The candidate runtime is removed only when | |
| 91 | the preflight process actually completes and reports a bad exit code or the wrong version: that is | |
| 92 | the case where a completed npm install produced an unusable tree, so retrying the same version must | |
| 93 | perform a clean install rather than reuse it. If the preflight cannot run at all, for example a spawn | |
| 94 | error or the `PREFLIGHT_TIMEOUT` elapsing, the update fails before reaching cleanup and the candidate | |
| 95 | directory is left in place. | |
| 29 | ## Remote Update | |
| 96 | 30 | |
| 97 | ## Host Service Lifecycle | |
| 31 | 1. The active server installs `t3@<target>` into a unique staging directory. | |
| 32 | 2. The target runs `__service-preflight` against the active database in read-only mode. | |
| 33 | 3. The staging directory is renamed to its immutable version path only after preflight succeeds. | |
| 34 | 4. The active child sends `request-update`. The launcher validates the child and target, writes | |
| 35 | pending state, generates the update ID, then replies `update-accepted`. | |
| 36 | 5. After a short response-flush grace period, the launcher stops the active child. | |
| 37 | 6. The launcher starts the target as a trial and gives it the pending update over IPC. | |
| 38 | 7. The trial acquires dependencies, binds HTTP, starts every long-running root fiber, and verifies | |
| 39 | that each root is parked at the activation gate. It does not run migrations. | |
| 40 | 8. The trial sends `prepared`. The launcher durably commits B before replying `committed`. | |
| 41 | 9. The child opens the existing activation gate, accepts commands, and publishes lifecycle ready | |
| 42 | with the terminal update outcome. | |
| 98 | 43 | |
| 99 | The systemd user service is a host lifecycle concern, not a T3 Connect resource. The standalone | |
| 100 | `t3 service install`, `uninstall`, `update`, and `status` commands own it. Install and update both | |
| 101 | reconcile the unit through `BootService`; running `npx t3@latest service update` therefore pins and | |
| 102 | activates the latest CLI release without requiring a connected client. | |
| 44 | Post-commit startup does not call service `start`, `initialize`, `connect`, `load`, or `acquire` | |
| 45 | operations. It only opens prepared gates and publishes prepared lifecycle state. | |
| 103 | 46 | |
| 104 | The `t3 connect` onboarding flow may offer service installation, but it calls the same reconciliation | |
| 105 | operation as `t3 service install`. Connect logout only disables cloud access and clears its | |
| 106 | authorization; it does not uninstall the host service. | |
| 47 | The launcher serializes child exits, IPC messages, and timers. A trial must report prepared within | |
| 48 | 120 seconds. If the trial exits or times out before prepared, the launcher records rollback and | |
| 49 | starts A. After commit, B is active and normal systemd restart policy applies. | |
| 107 | 50 | |
| 108 | ## Process Handoff | |
| 51 | ## Migration Boundary | |
| 52 | ||
| 53 | Remote update preflight requires exact equality between the database's applied `(id, name)` rows | |
| 54 | and the target release's migration manifest. A target with a missing, additional, renamed, or | |
| 55 | unknown migration is blocked. Remote updates never migrate or downgrade a database. | |
| 56 | ||
| 57 | This deliberately means any release containing a migration requires a local service update: | |
| 58 | ||
| 59 | ```sh | |
| 60 | npx t3@<version> service update | |
| 61 | ``` | |
| 109 | 62 | |
| 110 | For `boot-service`, the server atomically rewrites the T3-managed user unit to point at the verified | |
| 111 | runtime and reloads systemd. It acknowledges the handoff, then restarts the unit after the same | |
| 112 | short grace period used by foreground respawn. A rejected deferred restart restores the previous | |
| 113 | unit and is logged by the still-running process. | |
| 63 | The local command stops the unit, selects the new launcher and exact runtime, then restarts the | |
| 64 | service. Its normal server startup may run migrations. | |
| 114 | 65 | |
| 115 | For `respawn`, the server starts a detached, delayed replacement that replays the original CLI | |
| 116 | arguments. It then acknowledges the request and schedules the current process to exit. The delays | |
| 117 | give the acknowledgement time to cross direct or relayed connections before the socket closes. | |
| 66 | ## Client Correlation | |
| 118 | 67 | |
| 119 | Progress-capable servers emit `downloading` before installing the pinned runtime and `installing` | |
| 120 | before preflight and handoff. A terminal stream event acknowledges that restart is scheduled. The | |
| 121 | client then enters `resuming`, waits for the replacement lifecycle stream to publish `ready` with | |
| 122 | the target version, and only then completes the operation. It watches for the intentional | |
| 123 | disconnect's first backoff state and requests one fresh retry, which clears historical backoff debt | |
| 124 | without adding a separate reconnect loop. | |
| 68 | The update acknowledgement includes the launcher-generated update ID. After reconnecting, clients | |
| 69 | wait for a lifecycle ready event carrying that same ID. `committed` completes the operation only | |
| 70 | when the ready server is the target version. `rolled-back` and `failed` end it immediately with the | |
| 71 | recorded reason. Older servers without an ID retain version-only reconnect behavior. | |
| 125 | 72 | |
| 126 | ## Release Invariant | |
| 73 | ## Capability and Compatibility | |
| 127 | 74 | |
| 128 | The exact client version must exist as the `t3` npm package before a client carrying that version is | |
| 129 | published. The release workflow therefore makes the GitHub release depend on CLI publication, and | |
| 130 | the hosted web deployment depends on that release. See [Release Checklist](../operations/release.md#server-self-update-release-invariant). | |
| 75 | The existing additive RPC and lifecycle schemas remain compatible with older clients. New servers | |
| 76 | advertise remote self-update only when they have valid launcher context and a live IPC channel. | |
| 77 | Desktop-managed servers direct the user to update the desktop app. Other process shapes provide a | |
| 78 | manual command; the old detached foreground respawn path no longer exists. | |
| 131 | 79 | |
| 132 | 80 | ## Source Map |
| 133 | 81 | |
| 134 | - Capability contract: `packages/contracts/src/environment.ts` | |
| 135 | - Update RPC contract: `packages/contracts/src/server.ts` and `packages/contracts/src/rpc.ts` | |
| 136 | - Capability detection and handoff: `apps/server/src/cloud/selfUpdate.ts` | |
| 137 | - Host service commands: `apps/server/src/cli/service.ts` | |
| 138 | - Pinned runtime installation: `apps/server/src/cloud/pinnedRuntime.ts` | |
| 139 | - Client version comparison: `apps/web/src/versionSkew.ts` | |
| 140 | - Shared update action: `apps/web/src/components/ServerUpdateAction.tsx` | |
| 82 | - Launcher and state machine: `apps/server/src/serviceLauncher.ts` | |
| 83 | - IPC and durable state types: `apps/server/src/cloud/serviceProtocol.ts` | |
| 84 | - Child IPC adapter: `apps/server/src/cloud/serviceLauncherClient.ts` | |
| 85 | - Staging and preflight: `apps/server/src/cloud/pinnedRuntime.ts` and `servicePreflight.ts` | |
| 86 | - Service installation: `apps/server/src/cloud/bootService.ts` | |
| 87 | - Activation boundary: `apps/server/src/serverRuntimeStartup.ts` and `serverActivation.ts` | |
| 88 | - Client outcome correlation: `packages/client-runtime/src/state/server.ts` |
docs/operations/release.md+4-2
| ... | ... | @@ -187,8 +187,10 @@ the **Update server** action targeting a package version that does not exist yet |
| 187 | 187 | |
| 188 | 188 | For a release smoke test, confirm `npm view t3@<version> version` returns the expected version, then |
| 189 | 189 | connect the new client to a server on the previous version and verify that the update action |
| 190 | reconnects to the matching server. Test one automatic path and the manual or desktop-managed | |
| 191 | guidance when those environments are available. | |
| 190 | reconnects to the matching server. Use releases with identical migration manifests for the | |
| 191 | automatic path. When the manifest changed, verify that the remote action stops before restart and | |
| 192 | shows the exact local `npx t3@<version> service update` command. Also test the manual or | |
| 193 | desktop-managed guidance when those environments are available. | |
| 192 | 194 | |
| 193 | 195 | ## Desktop auto-update notes |
| 194 | 196 |
docs/user/background-service.md+4
| ... | ... | @@ -31,6 +31,10 @@ npx t3@latest service uninstall |
| 31 | 31 | |
| 32 | 32 | Updating restarts T3 Code briefly. Let active agent work and terminal commands finish first. |
| 33 | 33 | |
| 34 | The systemd unit runs a small stable launcher. Exact T3 Code versions are installed separately, so | |
| 35 | a failed remote candidate can return to the previous version without rewriting the unit. Releases | |
| 36 | that change the database must be installed with the local `service update` command above. | |
| 37 | ||
| 34 | 38 | ## Using It with T3 Connect |
| 35 | 39 | |
| 36 | 40 | T3 Connect may offer to install the service during setup so the host stays reachable after you log |
docs/user/updating.md+7-3
| ... | ... | @@ -24,13 +24,17 @@ The update does not remove saved threads, settings, or project files. |
| 24 | 24 | |
| 25 | 25 | | Action | What to do | |
| 26 | 26 | | -------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | |
| 27 | | **Update server** | Select the button and leave T3 Code open. It prepares the matching version, restarts the server, and reconnects automatically. This can take several minutes. | | |
| 27 | | **Update server** | Available for the T3 Code Linux background service. Select the button and leave T3 Code open while it prepares, tests, restarts, and reconnects. | | |
| 28 | 28 | | **Update the desktop app** | Open the T3 Code desktop app on the machine that runs the server and install the app update there. Reopen it if needed. | |
| 29 | 29 | | **Copy update command** | Copy the command, open a terminal on the server machine, stop the current T3 Code server, and relaunch it with the copied command and any startup options you normally use. | |
| 30 | 30 | |
| 31 | 31 | The available action depends on how that server was started. T3 Code does not update connected |
| 32 | 32 | servers silently in the background. |
| 33 | 33 | |
| 34 | If the requested version includes a database update, remote installation stops before restart and | |
| 35 | asks you to run the exact `npx t3@<version> service update` command on the server machine. This is | |
| 36 | an intentional rollback-safety boundary. | |
| 37 | ||
| 34 | 38 | After selecting **Update server**, the warning becomes a three-step progress rail: |
| 35 | 39 | **Download**, **Install**, and **Resume**. The same progress appears in the conversation and in |
| 36 | 40 | Connections, so navigating between them does not lose the update. A failed step remains visible |
| ... | ... | @@ -56,8 +60,8 @@ commands. |
| 56 | 60 | ## After the Update |
| 57 | 61 | |
| 58 | 62 | Keep the web or desktop app open while the server restarts. The update completes only after the |
| 59 | replacement server reports the requested version and is ready to accept commands. The warning and | |
| 60 | progress rail then disappear. | |
| 63 | service launcher reports that exact update committed and the replacement server is ready to accept | |
| 64 | commands. A rollback is reported immediately instead of waiting for a generic reconnect timeout. | |
| 61 | 65 | |
| 62 | 66 | If a step fails: |
| 63 | 67 |
packages/client-runtime/src/state/server.test.ts+36
| ... | ... | @@ -29,11 +29,13 @@ import { |
| 29 | 29 | applyServerConfigProjection, |
| 30 | 30 | makeEnvironmentServerConfigState, |
| 31 | 31 | isLegacyUpdateHandoffLoss, |
| 32 | matchesServerUpdateReadyEvent, | |
| 32 | 33 | projectServerWelcome, |
| 33 | 34 | resolveServerConfigValue, |
| 34 | 35 | resolveServerUpdateProgressResult, |
| 35 | 36 | serverUpdateStateForProgressEvent, |
| 36 | 37 | serverUpdateStateForServerVersion, |
| 38 | validateServerUpdateReadyEvent, | |
| 37 | 39 | } from "./server.ts"; |
| 38 | 40 | |
| 39 | 41 | const CONFIG = { |
| ... | ... | @@ -173,6 +175,40 @@ describe("server state projection", () => { |
| 173 | 175 | expect(serverUpdateStateForServerVersion(failed, "0.0.31")).toEqual({ status: "idle" }); |
| 174 | 176 | }); |
| 175 | 177 | |
| 178 | it.effect("correlates launcher outcomes and fails immediately after rollback", () => | |
| 179 | Effect.gen(function* () { | |
| 180 | const result = { | |
| 181 | targetVersion: "0.0.31", | |
| 182 | method: "boot-service" as const, | |
| 183 | updateId: "update-1", | |
| 184 | }; | |
| 185 | const ready = (status: "committed" | "rolled-back") => | |
| 186 | ({ | |
| 187 | version: 1 as const, | |
| 188 | sequence: 1, | |
| 189 | type: "ready" as const, | |
| 190 | payload: { | |
| 191 | at: "2026-08-01T00:00:00.000Z", | |
| 192 | environment: { serverVersion: status === "committed" ? "0.0.31" : "0.0.30" }, | |
| 193 | updateOutcome: { | |
| 194 | id: "update-1", | |
| 195 | fromVersion: "0.0.30", | |
| 196 | targetVersion: "0.0.31", | |
| 197 | status, | |
| 198 | ...(status === "rolled-back" ? { reason: "prepared-timeout" } : {}), | |
| 199 | }, | |
| 200 | }, | |
| 201 | }) as Parameters<typeof matchesServerUpdateReadyEvent>[1]; | |
| 202 | ||
| 203 | expect(matchesServerUpdateReadyEvent(result, ready("committed"))).toBe(true); | |
| 204 | yield* validateServerUpdateReadyEvent(result, ready("committed")); | |
| 205 | const rollback = yield* Effect.flip( | |
| 206 | validateServerUpdateReadyEvent(result, ready("rolled-back")), | |
| 207 | ); | |
| 208 | expect(rollback.message).toBe("prepared-timeout"); | |
| 209 | }), | |
| 210 | ); | |
| 211 | ||
| 176 | 212 | it("applies every config category to the projected snapshot", () => { |
| 177 | 213 | const snapshot = applyServerConfigProjection(Option.none(), { |
| 178 | 214 | version: 1, |
packages/client-runtime/src/state/server.ts+56-3
| ... | ... | @@ -3,6 +3,7 @@ import { |
| 3 | 3 | type ServerConfig, |
| 4 | 4 | type ServerConfigStreamEvent, |
| 5 | 5 | type ServerLifecycleWelcomePayload, |
| 6 | type ServerLifecycleStreamReadyEvent, | |
| 6 | 7 | type ServerSelfUpdateProgressEvent, |
| 7 | 8 | type ServerSelfUpdateResult, |
| 8 | 9 | WS_METHODS, |
| ... | ... | @@ -97,6 +98,57 @@ export class ServerUpdateProgressIncompleteError extends Schema.TaggedErrorClass |
| 97 | 98 | } |
| 98 | 99 | } |
| 99 | 100 | |
| 101 | export class ServerUpdateTerminalError extends Schema.TaggedErrorClass<ServerUpdateTerminalError>()( | |
| 102 | "ServerUpdateTerminalError", | |
| 103 | { | |
| 104 | targetVersion: Schema.String, | |
| 105 | status: Schema.Literals(["committed", "rolled-back", "failed"]), | |
| 106 | reason: Schema.optional(Schema.String), | |
| 107 | }, | |
| 108 | ) { | |
| 109 | override get message(): string { | |
| 110 | return this.reason ?? `The t3@${this.targetVersion} update ${this.status}.`; | |
| 111 | } | |
| 112 | } | |
| 113 | ||
| 114 | // Covers the 120-second trial deadline and a final restart of the previous | |
| 115 | // version when the trial rolls back. | |
| 116 | const SERVER_UPDATE_RESUME_TIMEOUT = Duration.minutes(4); | |
| 117 | ||
| 118 | export function matchesServerUpdateReadyEvent( | |
| 119 | result: ServerSelfUpdateResult, | |
| 120 | event: ServerLifecycleStreamReadyEvent, | |
| 121 | ): boolean { | |
| 122 | return result.updateId === undefined | |
| 123 | ? event.payload.environment.serverVersion === result.targetVersion | |
| 124 | : event.payload.updateOutcome?.id === result.updateId; | |
| 125 | } | |
| 126 | ||
| 127 | export function validateServerUpdateReadyEvent( | |
| 128 | result: ServerSelfUpdateResult, | |
| 129 | event: ServerLifecycleStreamReadyEvent, | |
| 130 | ): Effect.Effect<void, ServerUpdateTerminalError> { | |
| 131 | if (result.updateId === undefined) return Effect.void; | |
| 132 | const outcome = event.payload.updateOutcome; | |
| 133 | if ( | |
| 134 | outcome?.id === result.updateId && | |
| 135 | outcome.status === "committed" && | |
| 136 | outcome.targetVersion === result.targetVersion && | |
| 137 | event.payload.environment.serverVersion === result.targetVersion | |
| 138 | ) { | |
| 139 | return Effect.void; | |
| 140 | } | |
| 141 | return Effect.fail( | |
| 142 | new ServerUpdateTerminalError({ | |
| 143 | targetVersion: result.targetVersion, | |
| 144 | status: outcome?.status ?? "failed", | |
| 145 | reason: | |
| 146 | outcome?.reason ?? | |
| 147 | "The service launcher resumed without committing the requested server version.", | |
| 148 | }), | |
| 149 | ); | |
| 150 | } | |
| 151 | ||
| 100 | 152 | export function serverUpdateStateForProgressEvent( |
| 101 | 153 | fromVersion: string, |
| 102 | 154 | targetVersion: string, |
| ... | ... | @@ -554,11 +606,11 @@ export function createServerEnvironmentAtoms<R, E>( |
| 554 | 606 | .followStream(target.environmentId, subscribe(WS_METHODS.subscribeServerLifecycle, {})) |
| 555 | 607 | .pipe( |
| 556 | 608 | Stream.filter( |
| 557 | (event) => | |
| 558 | event.type === "ready" && event.payload.environment.serverVersion === targetVersion, | |
| 609 | (event): event is ServerLifecycleStreamReadyEvent => | |
| 610 | event.type === "ready" && matchesServerUpdateReadyEvent(result, event), | |
| 559 | 611 | ), |
| 560 | 612 | Stream.runHead, |
| 561 | Effect.timeoutOption(Duration.seconds(120)), | |
| 613 | Effect.timeoutOption(SERVER_UPDATE_RESUME_TIMEOUT), | |
| 562 | 614 | Effect.map(Option.flatten), |
| 563 | 615 | ); |
| 564 | 616 | if (Option.isNone(resumed)) { |
| ... | ... | @@ -567,6 +619,7 @@ export function createServerEnvironmentAtoms<R, E>( |
| 567 | 619 | targetVersion, |
| 568 | 620 | }); |
| 569 | 621 | } |
| 622 | yield* validateServerUpdateReadyEvent(result, resumed.value); | |
| 570 | 623 | |
| 571 | 624 | atomRegistry.set(stateAtom, IDLE_SERVER_UPDATE_STATE); |
| 572 | 625 | return result; |
packages/contracts/src/environment.ts+3-3
| ... | ... | @@ -20,9 +20,9 @@ export const ExecutionEnvironmentPlatform = Schema.Struct({ |
| 20 | 20 | }); |
| 21 | 21 | export type ExecutionEnvironmentPlatform = typeof ExecutionEnvironmentPlatform.Type; |
| 22 | 22 | |
| 23 | /** How a server can replace itself with another version when asked over RPC: | |
| 24 | "boot-service" rewrites the systemd user unit and restarts it; "respawn" | |
| 25 | installs the target version and respawns the foreground process. */ | |
| 23 | /** How a server can replace itself with another version when asked over RPC. | |
| 24 | New servers only advertise the stable launcher-backed "boot-service" path; | |
| 25 | "respawn" remains decodable for compatibility with older servers. */ | |
| 26 | 26 | export const ServerSelfUpdateMethod = Schema.Literals(["boot-service", "respawn"]); |
| 27 | 27 | export type ServerSelfUpdateMethod = typeof ServerSelfUpdateMethod.Type; |
| 28 | 28 |
packages/contracts/src/server.ts+14
| ... | ... | @@ -521,9 +521,21 @@ export const ServerConfigStreamEvent = Schema.Union([ |
| 521 | 521 | ]); |
| 522 | 522 | export type ServerConfigStreamEvent = typeof ServerConfigStreamEvent.Type; |
| 523 | 523 | |
| 524 | /** Terminal selection recorded by the service launcher for one update. */ | |
| 525 | export const ServerSelfUpdateOutcome = Schema.Struct({ | |
| 526 | id: TrimmedNonEmptyString, | |
| 527 | fromVersion: TrimmedNonEmptyString, | |
| 528 | targetVersion: TrimmedNonEmptyString, | |
| 529 | status: Schema.Literals(["committed", "rolled-back", "failed"]), | |
| 530 | reason: Schema.optionalKey(TrimmedNonEmptyString), | |
| 531 | }); | |
| 532 | export type ServerSelfUpdateOutcome = typeof ServerSelfUpdateOutcome.Type; | |
| 533 | ||
| 524 | 534 | export const ServerLifecycleReadyPayload = Schema.Struct({ |
| 525 | 535 | at: IsoDateTime, |
| 526 | 536 | environment: ExecutionEnvironmentDescriptor, |
| 537 | /** Present when this process resumed a launcher-managed update. */ | |
| 538 | updateOutcome: Schema.optionalKey(ServerSelfUpdateOutcome), | |
| 527 | 539 | }); |
| 528 | 540 | export type ServerLifecycleReadyPayload = typeof ServerLifecycleReadyPayload.Type; |
| 529 | 541 | |
| ... | ... | @@ -594,6 +606,8 @@ export type ServerSelfUpdateInput = typeof ServerSelfUpdateInput.Type; |
| 594 | 606 | export const ServerSelfUpdateResult = Schema.Struct({ |
| 595 | 607 | targetVersion: TrimmedNonEmptyString, |
| 596 | 608 | method: ServerSelfUpdateMethod, |
| 609 | /** Launcher-generated correlation ID. Absent when talking to older servers. */ | |
| 610 | updateId: Schema.optionalKey(TrimmedNonEmptyString), | |
| 597 | 611 | }); |
| 598 | 612 | export type ServerSelfUpdateResult = typeof ServerSelfUpdateResult.Type; |
| 599 | 613 |