From 808d6853555c8b57f414a06e0a79eb1b9e192ef5 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Wed, 5 Aug 2026 21:28:43 -0700 Subject: [PATCH] fix(server): keep the managed tunnel across remote update restarts Measured on a live update: the server was back in ~9s but stayed unreachable for ~96s, because releasing the tunnel on shutdown forces the replacement tunnel's hostname route through 1-2 minutes of edge propagation. An update handoff always brings a server right back (new version or rollback), so the tunnel is never orphaned; skip the release when the launcher state file records a pending update. The next boot respawns the connector from the stored config against the same tunnel and is reachable as soon as it connects. Co-Authored-By: Claude Fable 5 --- apps/server/src/cloud/http.test.ts | 71 ++++++++++++++++++++++++++++++ apps/server/src/cloud/http.ts | 31 +++++++++++++ 2 files changed, 102 insertions(+) diff --git a/apps/server/src/cloud/http.test.ts b/apps/server/src/cloud/http.test.ts index c779c13aaf4dc13b5c85b2b7a238c78ddce0c961..897912bc62e01e508d5f6b21f93cd1b53cfe6ad2 100644 --- a/apps/server/src/cloud/http.test.ts +++ b/apps/server/src/cloud/http.test.ts @@ -1,7 +1,9 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { describe, expect, it } from "@effect/vitest"; import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; import * as Option from "effect/Option"; +import * as Path from "effect/Path"; import * as PlatformError from "effect/PlatformError"; import * as Tracer from "effect/Tracer"; import { @@ -15,6 +17,7 @@ import { EnvironmentId } from "@t3tools/contracts"; import { RelayClientTracer } from "@t3tools/shared/relayTracing"; import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts"; import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; +import * as ServerConfigModule from "../config.ts"; import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; import { CLOUD_CLI_DESIRED_LINK_SECRET } from "./CliState.ts"; import * as CliTokenManager from "./CliTokenManager.ts"; @@ -256,6 +259,21 @@ describe("releaseManagedTunnelOnShutdown", () => { readonly respond?: () => Response; } + // Writes the launcher's durable state file into this test's baseDir; the + // release reads it to detect an in-flight update handoff. + const writeServiceState = (update: Record) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const config = yield* ServerConfigModule.ServerConfig; + const statePath = path.join(config.baseDir, "runtime", "service-state.json"); + yield* fs.makeDirectory(path.dirname(statePath), { recursive: true }); + yield* fs.writeFileString( + statePath, + JSON.stringify({ version: 1, activeVersion: "0.0.30", update }), + ); + }); + const provideReleaseHarness = (harness: ReleaseHarness) => (effect: Effect.Effect) => @@ -306,6 +324,11 @@ describe("releaseManagedTunnelOnShutdown", () => { }), ), ), + // The release consults the launcher state file under the configured + // baseDir, so every harness run gets a scoped temp baseDir. + Effect.provide(ServerConfigModule.layerTest("/", { prefix: "t3-http-release-test-" })), + Effect.provide(NodeServices.layer), + Effect.scoped, ); // The persisted state of a CLI-managed link whose tunnel is releasable. @@ -390,6 +413,54 @@ describe("releaseManagedTunnelOnShutdown", () => { }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); }); + it.effect("keeps the tunnel when shutdown hands off to a pending update", () => { + const { store, values } = makeMemorySecretStore(managedLinkSecrets); + const applyConfigCalls: Array = []; + const requests: Array = []; + + return Effect.gen(function* () { + yield* writeServiceState({ + id: "update-1", + fromVersion: "0.0.30", + targetVersion: "0.0.31", + dbPath: "/tmp/state.sqlite", + status: "pending", + }); + + const released = yield* releaseManagedTunnelOnShutdown(); + + // The launcher restarts a server immediately, so the tunnel is not + // orphaned; keeping it avoids the hostname route re-propagation that + // dominates update downtime. The stored config must survive so the + // next boot respawns the connector against the same tunnel. + expect(released).toBe(false); + expect(applyConfigCalls).toEqual([]); + expect(requests).toEqual([]); + expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(true); + }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); + }); + + it.effect("still releases when the recorded update already settled", () => { + const { store, values } = makeMemorySecretStore(managedLinkSecrets); + const applyConfigCalls: Array = []; + const requests: Array = []; + + return Effect.gen(function* () { + yield* writeServiceState({ + id: "update-1", + fromVersion: "0.0.30", + targetVersion: "0.0.31", + status: "committed", + }); + + const released = yield* releaseManagedTunnelOnShutdown(); + + expect(released).toBe(true); + expect(requests).toHaveLength(1); + expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(false); + }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); + }); + it.effect("keeps a runtime config that a fast restart replaced mid-release", () => { const { store, values } = makeMemorySecretStore(managedLinkSecrets); const applyConfigCalls: Array = []; diff --git a/apps/server/src/cloud/http.ts b/apps/server/src/cloud/http.ts index eaf31a37f277ce7ef247d4a0bf8be3fe69619012..e51ec6f3c88bf6af75ba40e2850431e80e0ae3a6 100644 --- a/apps/server/src/cloud/http.ts +++ b/apps/server/src/cloud/http.ts @@ -46,7 +46,9 @@ import * as DateTime from "effect/DateTime"; import * as Crypto from "effect/Crypto"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; import * as Option from "effect/Option"; +import * as Path from "effect/Path"; import * as Schema from "effect/Schema"; import * as HttpEffect from "effect/unstable/http/HttpEffect"; import { HttpServerRequest, HttpServerResponse } from "effect/unstable/http"; @@ -56,8 +58,10 @@ import * as HttpApiBuilder from "effect/unstable/httpapi/HttpApiBuilder"; import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts"; import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; import { requireEnvironmentScope } from "../auth/http.ts"; +import * as ServerConfig from "../config.ts"; import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts"; +import { SERVICE_STATE_FILE, serviceStateHasPendingUpdate } from "./serviceProtocol.ts"; import { CLOUD_ENDPOINT_RUNTIME_CONFIG, CLOUD_LINKED_USER_ID, @@ -627,6 +631,20 @@ export const reconcileDesiredCloudLink = Effect.fn("environment.cloud.reconcileD }, ); +// The launcher records an in-flight update in its durable state file. Read +// straight from disk: the launcher owns that file, and this runs while the +// server is tearing down. Unreadable or unparseable state means no pending +// update — the release then proceeds as before. +const pendingServiceUpdateExists = Effect.gen(function* () { + const config = yield* ServerConfig.ServerConfig; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const stateText = yield* fs + .readFileString(path.join(config.baseDir, "runtime", SERVICE_STATE_FILE)) + .pipe(Effect.option); + return Option.isSome(stateText) && serviceStateHasPendingUpdate(stateText.value); +}); + // Cloudflare bills per provisioned tunnel, so an environment that goes offline // must not leave its tunnel behind. Releasing deletes only the tunnel — the // relay keeps the link and its hostname reservation, and the next startup's @@ -649,6 +667,19 @@ export const releaseManagedTunnelOnShutdown = Effect.fn( if (!(yield* readCliDesiredCloudLink) || (yield* readCliDesiredLinkMode) !== "managed") { return false; } + // A shutdown that hands off to a pending remote update is not the + // environment going offline: the launcher immediately brings a server back + // (the new version, or the old one after a rollback). Deleting the tunnel + // here forces that server to provision a replacement UUID, and the public + // hostname's route to the new tunnel takes 1-2 minutes to propagate — the + // dominant cost of an update restart. Keep the tunnel instead: the next + // boot respawns the connector from the stored config and is reachable as + // soon as it connects, and the reconcile confirms the still-live tunnel + // without replacing it. + if (yield* pendingServiceUpdateExists) { + yield* Effect.logInfo("Keeping the managed tunnel across the update restart"); + return false; + } const token = yield* dependencies.cliTokenManager.getExisting; if (Option.isNone(token)) { return false; -- 2.54.0