authorgravatar for me@t3.ggTheo Browne <me@t3.gg> 2026-08-05 21:28:43-07:00
committergravatar for me@theo.liTheo Browne <me@theo.li> 2026-08-06 08:09:51-04:00
log808d6853555c8b57f414a06e0a79eb1b9e192ef5
tree3ee03f5422092b66f9ef3c4b8dbc2a537be7ee3d
parent80720ad592dc68c29677444fd7b6abb97b82c03e

fix(server): keep the managed tunnel across remote update restarts

Measured on a live update: the server was back in ~9s but stayed unreachable for ~96s, because releasing the tunnel on shutdown forces the replacement tunnel's hostname route through 1-2 minutes of edge propagation. An update handoff always brings a server right back (new version or rollback), so the tunnel is never orphaned; skip the release when the launcher state file records a pending update. The next boot respawns the connector from the stored config against the same tunnel and is reachable as soon as it connects. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

2 files changed, 102 insertions(+), 0 deletions(-)

apps/server/src/cloud/http.test.ts+71
......@@ -1,7 +1,9 @@
11import * as NodeServices from "@effect/platform-node/NodeServices";
22import { describe, expect, it } from "@effect/vitest";
33import * as Effect from "effect/Effect";
4import * as FileSystem from "effect/FileSystem";
45import * as Option from "effect/Option";
6import * as Path from "effect/Path";
57import * as PlatformError from "effect/PlatformError";
68import * as Tracer from "effect/Tracer";
79import {
......@@ -15,6 +17,7 @@ import { EnvironmentId } from "@t3tools/contracts";
1517import { RelayClientTracer } from "@t3tools/shared/relayTracing";
1618import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts";
1719import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
20import * as ServerConfigModule from "../config.ts";
1821import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
1922import { CLOUD_CLI_DESIRED_LINK_SECRET } from "./CliState.ts";
2023import * as CliTokenManager from "./CliTokenManager.ts";
......@@ -256,6 +259,21 @@ describe("releaseManagedTunnelOnShutdown", () => {
256259 readonly respond?: () => Response;
257260 }
258261
262 // Writes the launcher's durable state file into this test's baseDir; the
263 // release reads it to detect an in-flight update handoff.
264 const writeServiceState = (update: Record<string, unknown>) =>
265 Effect.gen(function* () {
266 const fs = yield* FileSystem.FileSystem;
267 const path = yield* Path.Path;
268 const config = yield* ServerConfigModule.ServerConfig;
269 const statePath = path.join(config.baseDir, "runtime", "service-state.json");
270 yield* fs.makeDirectory(path.dirname(statePath), { recursive: true });
271 yield* fs.writeFileString(
272 statePath,
273 JSON.stringify({ version: 1, activeVersion: "0.0.30", update }),
274 );
275 });
276
259277 const provideReleaseHarness =
260278 (harness: ReleaseHarness) =>
261279 <A, E, R>(effect: Effect.Effect<A, E, R>) =>
......@@ -306,6 +324,11 @@ describe("releaseManagedTunnelOnShutdown", () => {
306324 }),
307325 ),
308326 ),
327 // The release consults the launcher state file under the configured
328 // baseDir, so every harness run gets a scoped temp baseDir.
329 Effect.provide(ServerConfigModule.layerTest("/", { prefix: "t3-http-release-test-" })),
330 Effect.provide(NodeServices.layer),
331 Effect.scoped,
309332 );
310333
311334 // The persisted state of a CLI-managed link whose tunnel is releasable.
......@@ -390,6 +413,54 @@ describe("releaseManagedTunnelOnShutdown", () => {
390413 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
391414 });
392415
416 it.effect("keeps the tunnel when shutdown hands off to a pending update", () => {
417 const { store, values } = makeMemorySecretStore(managedLinkSecrets);
418 const applyConfigCalls: Array<unknown> = [];
419 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
420
421 return Effect.gen(function* () {
422 yield* writeServiceState({
423 id: "update-1",
424 fromVersion: "0.0.30",
425 targetVersion: "0.0.31",
426 dbPath: "/tmp/state.sqlite",
427 status: "pending",
428 });
429
430 const released = yield* releaseManagedTunnelOnShutdown();
431
432 // The launcher restarts a server immediately, so the tunnel is not
433 // orphaned; keeping it avoids the hostname route re-propagation that
434 // dominates update downtime. The stored config must survive so the
435 // next boot respawns the connector against the same tunnel.
436 expect(released).toBe(false);
437 expect(applyConfigCalls).toEqual([]);
438 expect(requests).toEqual([]);
439 expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(true);
440 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
441 });
442
443 it.effect("still releases when the recorded update already settled", () => {
444 const { store, values } = makeMemorySecretStore(managedLinkSecrets);
445 const applyConfigCalls: Array<unknown> = [];
446 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
447
448 return Effect.gen(function* () {
449 yield* writeServiceState({
450 id: "update-1",
451 fromVersion: "0.0.30",
452 targetVersion: "0.0.31",
453 status: "committed",
454 });
455
456 const released = yield* releaseManagedTunnelOnShutdown();
457
458 expect(released).toBe(true);
459 expect(requests).toHaveLength(1);
460 expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(false);
461 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
462 });
463
393464 it.effect("keeps a runtime config that a fast restart replaced mid-release", () => {
394465 const { store, values } = makeMemorySecretStore(managedLinkSecrets);
395466 const applyConfigCalls: Array<unknown> = [];
apps/server/src/cloud/http.ts+31
......@@ -46,7 +46,9 @@ import * as DateTime from "effect/DateTime";
4646import * as Crypto from "effect/Crypto";
4747import * as Duration from "effect/Duration";
4848import * as Effect from "effect/Effect";
49import * as FileSystem from "effect/FileSystem";
4950import * as Option from "effect/Option";
51import * as Path from "effect/Path";
5052import * as Schema from "effect/Schema";
5153import * as HttpEffect from "effect/unstable/http/HttpEffect";
5254import { HttpServerRequest, HttpServerResponse } from "effect/unstable/http";
......@@ -56,8 +58,10 @@ import * as HttpApiBuilder from "effect/unstable/httpapi/HttpApiBuilder";
5658import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts";
5759import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
5860import { requireEnvironmentScope } from "../auth/http.ts";
61import * as ServerConfig from "../config.ts";
5962import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
6063import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts";
64import { SERVICE_STATE_FILE, serviceStateHasPendingUpdate } from "./serviceProtocol.ts";
6165import {
6266 CLOUD_ENDPOINT_RUNTIME_CONFIG,
6367 CLOUD_LINKED_USER_ID,
......@@ -627,6 +631,20 @@ export const reconcileDesiredCloudLink = Effect.fn("environment.cloud.reconcileD
627631 },
628632);
629633
634// The launcher records an in-flight update in its durable state file. Read
635// straight from disk: the launcher owns that file, and this runs while the
636// server is tearing down. Unreadable or unparseable state means no pending
637// update — the release then proceeds as before.
638const pendingServiceUpdateExists = Effect.gen(function* () {
639 const config = yield* ServerConfig.ServerConfig;
640 const fs = yield* FileSystem.FileSystem;
641 const path = yield* Path.Path;
642 const stateText = yield* fs
643 .readFileString(path.join(config.baseDir, "runtime", SERVICE_STATE_FILE))
644 .pipe(Effect.option);
645 return Option.isSome(stateText) && serviceStateHasPendingUpdate(stateText.value);
646});
647
630648// Cloudflare bills per provisioned tunnel, so an environment that goes offline
631649// must not leave its tunnel behind. Releasing deletes only the tunnel — the
632650// relay keeps the link and its hostname reservation, and the next startup's
......@@ -649,6 +667,19 @@ export const releaseManagedTunnelOnShutdown = Effect.fn(
649667 if (!(yield* readCliDesiredCloudLink) || (yield* readCliDesiredLinkMode) !== "managed") {
650668 return false;
651669 }
670 // A shutdown that hands off to a pending remote update is not the
671 // environment going offline: the launcher immediately brings a server back
672 // (the new version, or the old one after a rollback). Deleting the tunnel
673 // here forces that server to provision a replacement UUID, and the public
674 // hostname's route to the new tunnel takes 1-2 minutes to propagate — the
675 // dominant cost of an update restart. Keep the tunnel instead: the next
676 // boot respawns the connector from the stored config and is reachable as
677 // soon as it connects, and the reconcile confirms the still-live tunnel
678 // without replacing it.
679 if (yield* pendingServiceUpdateExists) {
680 yield* Effect.logInfo("Keeping the managed tunnel across the update restart");
681 return false;
682 }
652683 const token = yield* dependencies.cliTokenManager.getExisting;
653684 if (Option.isNone(token)) {
654685 return false;