| author |  Theo Browne <me@t3.gg> 2026-08-05 21:28:43-07:00 |
| committer |  Theo Browne <me@theo.li> 2026-08-06 08:09:51-04:00 |
| log | 808d6853555c8b57f414a06e0a79eb1b9e192ef5 |
| tree | 3ee03f5422092b66f9ef3c4b8dbc2a537be7ee3d |
| parent | 80720ad592dc68c29677444fd7b6abb97b82c03e |
fix(server): keep the managed tunnel across remote update restarts
Measured on a live update: the server was back in ~9s but stayed
unreachable for ~96s, because releasing the tunnel on shutdown forces
the replacement tunnel's hostname route through 1-2 minutes of edge
propagation. An update handoff always brings a server right back
(new version or rollback), so the tunnel is never orphaned; skip the
release when the launcher state file records a pending update. The
next boot respawns the connector from the stored config against the
same tunnel and is reachable as soon as it connects.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>2 files changed, 102 insertions(+), 0 deletions(-)
apps
server
src
cloud
apps/server/src/cloud/http.test.ts+71
| ... | @@ -1,7 +1,9 @@ | ... | @@ -1,7 +1,9 @@ |
| 1 | import * as NodeServices from "@effect/platform-node/NodeServices"; | 1 | import * as NodeServices from "@effect/platform-node/NodeServices"; |
| 2 | import { describe, expect, it } from "@effect/vitest"; | 2 | import { describe, expect, it } from "@effect/vitest"; |
| 3 | import * as Effect from "effect/Effect"; | 3 | import * as Effect from "effect/Effect"; |
| 4 | import * as FileSystem from "effect/FileSystem"; | ||
| 4 | import * as Option from "effect/Option"; | 5 | import * as Option from "effect/Option"; |
| 6 | import * as Path from "effect/Path"; | ||
| 5 | import * as PlatformError from "effect/PlatformError"; | 7 | import * as PlatformError from "effect/PlatformError"; |
| 6 | import * as Tracer from "effect/Tracer"; | 8 | import * as Tracer from "effect/Tracer"; |
| 7 | import { | 9 | import { |
| ... | @@ -15,6 +17,7 @@ import { EnvironmentId } from "@t3tools/contracts"; | ... | @@ -15,6 +17,7 @@ import { EnvironmentId } from "@t3tools/contracts"; |
| 15 | import { RelayClientTracer } from "@t3tools/shared/relayTracing"; | 17 | import { RelayClientTracer } from "@t3tools/shared/relayTracing"; |
| 16 | import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts"; | 18 | import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts"; |
| 17 | import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; | 19 | import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; |
| 20 | import * as ServerConfigModule from "../config.ts"; | ||
| 18 | import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; | 21 | import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; |
| 19 | import { CLOUD_CLI_DESIRED_LINK_SECRET } from "./CliState.ts"; | 22 | import { CLOUD_CLI_DESIRED_LINK_SECRET } from "./CliState.ts"; |
| 20 | import * as CliTokenManager from "./CliTokenManager.ts"; | 23 | import * as CliTokenManager from "./CliTokenManager.ts"; |
| ... | @@ -256,6 +259,21 @@ describe("releaseManagedTunnelOnShutdown", () => { | ... | @@ -256,6 +259,21 @@ describe("releaseManagedTunnelOnShutdown", () => { |
| 256 | readonly respond?: () => Response; | 259 | readonly respond?: () => Response; |
| 257 | } | 260 | } |
| 258 | 261 | ||
| 262 | // Writes the launcher's durable state file into this test's baseDir; the | ||
| 263 | // release reads it to detect an in-flight update handoff. | ||
| 264 | const writeServiceState = (update: Record<string, unknown>) => | ||
| 265 | Effect.gen(function* () { | ||
| 266 | const fs = yield* FileSystem.FileSystem; | ||
| 267 | const path = yield* Path.Path; | ||
| 268 | const config = yield* ServerConfigModule.ServerConfig; | ||
| 269 | const statePath = path.join(config.baseDir, "runtime", "service-state.json"); | ||
| 270 | yield* fs.makeDirectory(path.dirname(statePath), { recursive: true }); | ||
| 271 | yield* fs.writeFileString( | ||
| 272 | statePath, | ||
| 273 | JSON.stringify({ version: 1, activeVersion: "0.0.30", update }), | ||
| 274 | ); | ||
| 275 | }); | ||
| 276 | |||
| 259 | const provideReleaseHarness = | 277 | const provideReleaseHarness = |
| 260 | (harness: ReleaseHarness) => | 278 | (harness: ReleaseHarness) => |
| 261 | <A, E, R>(effect: Effect.Effect<A, E, R>) => | 279 | <A, E, R>(effect: Effect.Effect<A, E, R>) => |
| ... | @@ -306,6 +324,11 @@ describe("releaseManagedTunnelOnShutdown", () => { | ... | @@ -306,6 +324,11 @@ describe("releaseManagedTunnelOnShutdown", () => { |
| 306 | }), | 324 | }), |
| 307 | ), | 325 | ), |
| 308 | ), | 326 | ), |
| 327 | // The release consults the launcher state file under the configured | ||
| 328 | // baseDir, so every harness run gets a scoped temp baseDir. | ||
| 329 | Effect.provide(ServerConfigModule.layerTest("/", { prefix: "t3-http-release-test-" })), | ||
| 330 | Effect.provide(NodeServices.layer), | ||
| 331 | Effect.scoped, | ||
| 309 | ); | 332 | ); |
| 310 | 333 | ||
| 311 | // The persisted state of a CLI-managed link whose tunnel is releasable. | 334 | // The persisted state of a CLI-managed link whose tunnel is releasable. |
| ... | @@ -390,6 +413,54 @@ describe("releaseManagedTunnelOnShutdown", () => { | ... | @@ -390,6 +413,54 @@ describe("releaseManagedTunnelOnShutdown", () => { |
| 390 | }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); | 413 | }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); |
| 391 | }); | 414 | }); |
| 392 | 415 | ||
| 416 | it.effect("keeps the tunnel when shutdown hands off to a pending update", () => { | ||
| 417 | const { store, values } = makeMemorySecretStore(managedLinkSecrets); | ||
| 418 | const applyConfigCalls: Array<unknown> = []; | ||
| 419 | const requests: Array<HttpClientRequest.HttpClientRequest> = []; | ||
| 420 | |||
| 421 | return Effect.gen(function* () { | ||
| 422 | yield* writeServiceState({ | ||
| 423 | id: "update-1", | ||
| 424 | fromVersion: "0.0.30", | ||
| 425 | targetVersion: "0.0.31", | ||
| 426 | dbPath: "/tmp/state.sqlite", | ||
| 427 | status: "pending", | ||
| 428 | }); | ||
| 429 | |||
| 430 | const released = yield* releaseManagedTunnelOnShutdown(); | ||
| 431 | |||
| 432 | // The launcher restarts a server immediately, so the tunnel is not | ||
| 433 | // orphaned; keeping it avoids the hostname route re-propagation that | ||
| 434 | // dominates update downtime. The stored config must survive so the | ||
| 435 | // next boot respawns the connector against the same tunnel. | ||
| 436 | expect(released).toBe(false); | ||
| 437 | expect(applyConfigCalls).toEqual([]); | ||
| 438 | expect(requests).toEqual([]); | ||
| 439 | expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(true); | ||
| 440 | }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); | ||
| 441 | }); | ||
| 442 | |||
| 443 | it.effect("still releases when the recorded update already settled", () => { | ||
| 444 | const { store, values } = makeMemorySecretStore(managedLinkSecrets); | ||
| 445 | const applyConfigCalls: Array<unknown> = []; | ||
| 446 | const requests: Array<HttpClientRequest.HttpClientRequest> = []; | ||
| 447 | |||
| 448 | return Effect.gen(function* () { | ||
| 449 | yield* writeServiceState({ | ||
| 450 | id: "update-1", | ||
| 451 | fromVersion: "0.0.30", | ||
| 452 | targetVersion: "0.0.31", | ||
| 453 | status: "committed", | ||
| 454 | }); | ||
| 455 | |||
| 456 | const released = yield* releaseManagedTunnelOnShutdown(); | ||
| 457 | |||
| 458 | expect(released).toBe(true); | ||
| 459 | expect(requests).toHaveLength(1); | ||
| 460 | expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(false); | ||
| 461 | }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); | ||
| 462 | }); | ||
| 463 | |||
| 393 | it.effect("keeps a runtime config that a fast restart replaced mid-release", () => { | 464 | it.effect("keeps a runtime config that a fast restart replaced mid-release", () => { |
| 394 | const { store, values } = makeMemorySecretStore(managedLinkSecrets); | 465 | const { store, values } = makeMemorySecretStore(managedLinkSecrets); |
| 395 | const applyConfigCalls: Array<unknown> = []; | 466 | const applyConfigCalls: Array<unknown> = []; |
apps/server/src/cloud/http.ts+31
| ... | @@ -46,7 +46,9 @@ import * as DateTime from "effect/DateTime"; | ... | @@ -46,7 +46,9 @@ import * as DateTime from "effect/DateTime"; |
| 46 | import * as Crypto from "effect/Crypto"; | 46 | import * as Crypto from "effect/Crypto"; |
| 47 | import * as Duration from "effect/Duration"; | 47 | import * as Duration from "effect/Duration"; |
| 48 | import * as Effect from "effect/Effect"; | 48 | import * as Effect from "effect/Effect"; |
| 49 | import * as FileSystem from "effect/FileSystem"; | ||
| 49 | import * as Option from "effect/Option"; | 50 | import * as Option from "effect/Option"; |
| 51 | import * as Path from "effect/Path"; | ||
| 50 | import * as Schema from "effect/Schema"; | 52 | import * as Schema from "effect/Schema"; |
| 51 | import * as HttpEffect from "effect/unstable/http/HttpEffect"; | 53 | import * as HttpEffect from "effect/unstable/http/HttpEffect"; |
| 52 | import { HttpServerRequest, HttpServerResponse } from "effect/unstable/http"; | 54 | import { HttpServerRequest, HttpServerResponse } from "effect/unstable/http"; |
| ... | @@ -56,8 +58,10 @@ import * as HttpApiBuilder from "effect/unstable/httpapi/HttpApiBuilder"; | ... | @@ -56,8 +58,10 @@ import * as HttpApiBuilder from "effect/unstable/httpapi/HttpApiBuilder"; |
| 56 | import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts"; | 58 | import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts"; |
| 57 | import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; | 59 | import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; |
| 58 | import { requireEnvironmentScope } from "../auth/http.ts"; | 60 | import { requireEnvironmentScope } from "../auth/http.ts"; |
| 61 | import * as ServerConfig from "../config.ts"; | ||
| 59 | import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; | 62 | import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; |
| 60 | import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts"; | 63 | import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts"; |
| 64 | import { SERVICE_STATE_FILE, serviceStateHasPendingUpdate } from "./serviceProtocol.ts"; | ||
| 61 | import { | 65 | import { |
| 62 | CLOUD_ENDPOINT_RUNTIME_CONFIG, | 66 | CLOUD_ENDPOINT_RUNTIME_CONFIG, |
| 63 | CLOUD_LINKED_USER_ID, | 67 | CLOUD_LINKED_USER_ID, |
| ... | @@ -627,6 +631,20 @@ export const reconcileDesiredCloudLink = Effect.fn("environment.cloud.reconcileD | ... | @@ -627,6 +631,20 @@ export const reconcileDesiredCloudLink = Effect.fn("environment.cloud.reconcileD |
| 627 | }, | 631 | }, |
| 628 | ); | 632 | ); |
| 629 | 633 | ||
| 634 | // The launcher records an in-flight update in its durable state file. Read | ||
| 635 | // straight from disk: the launcher owns that file, and this runs while the | ||
| 636 | // server is tearing down. Unreadable or unparseable state means no pending | ||
| 637 | // update — the release then proceeds as before. | ||
| 638 | const pendingServiceUpdateExists = Effect.gen(function* () { | ||
| 639 | const config = yield* ServerConfig.ServerConfig; | ||
| 640 | const fs = yield* FileSystem.FileSystem; | ||
| 641 | const path = yield* Path.Path; | ||
| 642 | const stateText = yield* fs | ||
| 643 | .readFileString(path.join(config.baseDir, "runtime", SERVICE_STATE_FILE)) | ||
| 644 | .pipe(Effect.option); | ||
| 645 | return Option.isSome(stateText) && serviceStateHasPendingUpdate(stateText.value); | ||
| 646 | }); | ||
| 647 | |||
| 630 | // Cloudflare bills per provisioned tunnel, so an environment that goes offline | 648 | // Cloudflare bills per provisioned tunnel, so an environment that goes offline |
| 631 | // must not leave its tunnel behind. Releasing deletes only the tunnel — the | 649 | // must not leave its tunnel behind. Releasing deletes only the tunnel — the |
| 632 | // relay keeps the link and its hostname reservation, and the next startup's | 650 | // relay keeps the link and its hostname reservation, and the next startup's |
| ... | @@ -649,6 +667,19 @@ export const releaseManagedTunnelOnShutdown = Effect.fn( | ... | @@ -649,6 +667,19 @@ export const releaseManagedTunnelOnShutdown = Effect.fn( |
| 649 | if (!(yield* readCliDesiredCloudLink) || (yield* readCliDesiredLinkMode) !== "managed") { | 667 | if (!(yield* readCliDesiredCloudLink) || (yield* readCliDesiredLinkMode) !== "managed") { |
| 650 | return false; | 668 | return false; |
| 651 | } | 669 | } |
| 670 | // A shutdown that hands off to a pending remote update is not the | ||
| 671 | // environment going offline: the launcher immediately brings a server back | ||
| 672 | // (the new version, or the old one after a rollback). Deleting the tunnel | ||
| 673 | // here forces that server to provision a replacement UUID, and the public | ||
| 674 | // hostname's route to the new tunnel takes 1-2 minutes to propagate — the | ||
| 675 | // dominant cost of an update restart. Keep the tunnel instead: the next | ||
| 676 | // boot respawns the connector from the stored config and is reachable as | ||
| 677 | // soon as it connects, and the reconcile confirms the still-live tunnel | ||
| 678 | // without replacing it. | ||
| 679 | if (yield* pendingServiceUpdateExists) { | ||
| 680 | yield* Effect.logInfo("Keeping the managed tunnel across the update restart"); | ||
| 681 | return false; | ||
| 682 | } | ||
| 652 | const token = yield* dependencies.cliTokenManager.getExisting; | 683 | const token = yield* dependencies.cliTokenManager.getExisting; |
| 653 | if (Option.isNone(token)) { | 684 | if (Option.isNone(token)) { |
| 654 | return false; | 685 | return false; |