authorgravatar for me@t3.ggTheo Browne <me@t3.gg> 2026-09-24 17:17:41-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-09-24 17:17:41-07:00
log8d7b5e998c72b4674326cbac7d23bafe9b222668
treefcbfc93d4f6343e71e840c3de3cc4903508e93a7
parentd4a33457cb0da797728f4846a9da8592d7d81d36
signature Signed by PGP key B5690EEEBB952194

fix(connect): remove tunnels after hosts go offline (#9386)

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: Julius Marminge <julius0216@outlook.com> Co-authored-by: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com>

36 files changed, 7563 insertions(+), 388 deletions(-)

.github/workflows/deploy-relay.yml+1
......@@ -28,6 +28,7 @@ jobs:
2828 RELAY_DOMAIN: ${{ vars.RELAY_DOMAIN }}
2929 RELAY_API_ZONE_NAME: ${{ vars.RELAY_API_ZONE_NAME }}
3030 RELAY_TUNNEL_ZONE_NAME: ${{ vars.RELAY_TUNNEL_ZONE_NAME }}
31 RELAY_TUNNEL_CLEANUP_MODE: ${{ vars.RELAY_TUNNEL_CLEANUP_MODE }}
3132 CLERK_PUBLISHABLE_KEY: ${{ vars.CLERK_PUBLISHABLE_KEY }}
3233 CLERK_JWT_AUDIENCE: ${{ vars.CLERK_JWT_AUDIENCE }}
3334 APNS_ENVIRONMENT: ${{ vars.APNS_ENVIRONMENT }}
apps/server/src/cloud/CliState.test.ts+2
......@@ -8,6 +8,7 @@ import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
88import { ServerConfig } from "../config.ts";
99import * as CliState from "./CliState.ts";
1010import {
11 CLOUD_ENDPOINT_CONFIRMED_ORIGIN,
1112 CLOUD_ENDPOINT_RUNTIME_CONFIG,
1213 CLOUD_LINKED_USER_ID,
1314 CLOUD_MINT_PUBLIC_KEY,
......@@ -24,6 +25,7 @@ const persistedCloudLinkSecrets = [
2425 RELAY_ENVIRONMENT_CREDENTIAL_SECRET,
2526 CLOUD_MINT_PUBLIC_KEY,
2627 CLOUD_ENDPOINT_RUNTIME_CONFIG,
28 CLOUD_ENDPOINT_CONFIRMED_ORIGIN,
2729 PUBLISH_AGENT_ACTIVITY_SECRET,
2830] as const;
2931
apps/server/src/cloud/CliState.ts+2
......@@ -3,6 +3,7 @@ import * as Option from "effect/Option";
33
44import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
55import {
6 CLOUD_ENDPOINT_CONFIRMED_ORIGIN,
67 CLOUD_ENDPOINT_RUNTIME_CONFIG,
78 CLOUD_LINKED_USER_ID,
89 CLOUD_MINT_PUBLIC_KEY,
......@@ -67,6 +68,7 @@ export const clearPersistedCloudLink = Effect.gen(function* () {
6768 secrets.remove(RELAY_ENVIRONMENT_CREDENTIAL_SECRET),
6869 secrets.remove(CLOUD_MINT_PUBLIC_KEY),
6970 secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG),
71 secrets.remove(CLOUD_ENDPOINT_CONFIRMED_ORIGIN),
7072 secrets.remove(PUBLISH_AGENT_ACTIVITY_SECRET),
7173 ],
7274 { concurrency: "unbounded" },
apps/server/src/cloud/ManagedEndpointRuntime.test.ts+214-3
......@@ -7,10 +7,12 @@ import * as Fiber from "effect/Fiber";
77import * as Layer from "effect/Layer";
88import * as Option from "effect/Option";
99import * as PlatformError from "effect/PlatformError";
10import * as Queue from "effect/Queue";
1011import * as Sink from "effect/Sink";
1112import * as Stream from "effect/Stream";
1213import * as TestClock from "effect/testing/TestClock";
1314import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process";
15import type { RelayManagedEndpointRuntimeConfig } from "@t3tools/contracts/relay";
1416import * as RelayClient from "@t3tools/shared/relayClient";
1517
1618import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
......@@ -62,6 +64,7 @@ function makeHandle(input: {
6264 readonly onKill: () => void;
6365 readonly isRunning?: () => boolean;
6466 readonly exitCode?: Effect.Effect<ChildProcessSpawner.ExitCode>;
67 readonly output?: Stream.Stream<Uint8Array>;
6568}) {
6669 return ChildProcessSpawner.makeHandle({
6770 pid: ChildProcessSpawner.ProcessId(input.pid),
......@@ -75,13 +78,70 @@ function makeHandle(input: {
7578 stdin: Sink.drain,
7679 stdout: Stream.empty,
7780 stderr: Stream.empty,
78 all: Stream.empty,
81 all: input.output ?? Stream.empty,
7982 getInputFd: () => Sink.drain,
8083 getOutputFd: () => Stream.empty,
8184 });
8285}
8386
8487describe("CloudManagedEndpointRuntime", () => {
88 it("retries connector startup failures but stops for unsupported runtimes", () => {
89 expect(
90 ManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus({
91 status: "failed",
92 failure: "not-installed",
93 reason: "The relay client is not installed.",
94 }),
95 ).toBe(true);
96 expect(
97 ManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus({
98 status: "failed",
99 failure: "spawn-failed",
100 reason: "spawn failed",
101 }),
102 ).toBe(true);
103 expect(
104 ManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus({
105 status: "failed",
106 failure: "unsupported-platform",
107 reason: "Relay client is unsupported on linux-arm.",
108 }),
109 ).toBe(false);
110 expect(
111 ManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus({ status: "unsupported" }),
112 ).toBe(false);
113 });
114
115 it.effect("serializes updates to persisted cloud link state", () =>
116 Effect.gen(function* () {
117 const firstEntered = yield* Deferred.make<void>();
118 const releaseFirst = yield* Deferred.make<void>();
119 const secondEntered = yield* Deferred.make<void>();
120 const runtime = yield* buildCloudManagedEndpointRuntime(
121 ChildProcessSpawner.make(() => Effect.die("unused")),
122 );
123
124 const first = yield* runtime
125 .withLinkStateLock(
126 Deferred.succeed(firstEntered, undefined).pipe(
127 Effect.andThen(Deferred.await(releaseFirst)),
128 ),
129 )
130 .pipe(Effect.forkChild);
131 yield* Deferred.await(firstEntered);
132
133 const second = yield* runtime
134 .withLinkStateLock(Deferred.succeed(secondEntered, undefined))
135 .pipe(Effect.forkChild);
136 expect(yield* Deferred.isDone(secondEntered)).toBe(false);
137
138 yield* Deferred.succeed(releaseFirst, undefined);
139 yield* Fiber.join(first);
140 yield* Fiber.join(second);
141 expect(yield* Deferred.isDone(secondEntered)).toBe(true);
142 }),
143 );
144
85145 it("classifies Cloudflare connection and warning output", () => {
86146 expect(
87147 ManagedEndpointRuntime.classifyRelayClientOutput(
......@@ -109,6 +169,125 @@ describe("CloudManagedEndpointRuntime", () => {
109169 ).toBe("warning");
110170 });
111171
172 it("recognizes tunnel authorization failures without matching ordinary transport errors", () => {
173 expect(
174 ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput(
175 '2026-09-15T06:30:43Z ERR Register tunnel error from server side error="Failed to get tunnel" connIndex=0 event=0 ip=198.41.200.23',
176 ),
177 ).toBe(true);
178 expect(
179 ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput(
180 '2026-06-17T02:00:00Z ERR Register tunnel error from server side error="Unauthorized: Record for tunnel not found" connIndex=0',
181 ),
182 ).toBe(true);
183 expect(
184 ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput(
185 '2026-06-17T02:00:00Z ERR Register tunnel error from server side error="Unauthorized: Invalid tunnel secret" connIndex=0',
186 ),
187 ).toBe(true);
188 expect(
189 ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput(
190 '2026-06-17T02:00:00Z ERR Register tunnel error from server side error="connection timed out" connIndex=0',
191 ),
192 ).toBe(false);
193 });
194
195 it.effect("keeps recovery requests sent before the server starts consuming them", () =>
196 Effect.gen(function* () {
197 const runtime = yield* buildCloudManagedEndpointRuntime(
198 ChildProcessSpawner.make(() => Effect.die("unused")),
199 );
200 const config = {
201 providerKind: "cloudflare_tunnel" as const,
202 connectorToken: "token",
203 tunnelId: "tunnel-1",
204 };
205
206 yield* runtime.requestRecovery(config);
207
208 expect(Option.getOrNull(yield* Stream.runHead(runtime.recoveryRequests))).toEqual(config);
209 }),
210 );
211
212 it.effect("recovers a rejected tunnel without waiting for the connector to exit", () =>
213 Effect.gen(function* () {
214 const output = yield* Queue.unbounded<Uint8Array>();
215 const firstBatchObserved = yield* Deferred.make<void>();
216 const secondBatchObserved = yield* Deferred.make<void>();
217 const recoveryRequested = yield* Deferred.make<RelayManagedEndpointRuntimeConfig>();
218 const recoveryRetried = yield* Deferred.make<RelayManagedEndpointRuntimeConfig>();
219 let recoveryRequestCount = 0;
220 const spawned: Array<number> = [];
221 const encoder = new TextEncoder();
222 const connectorOutput = Stream.fromQueue(output).pipe(
223 Stream.tap((chunk) => {
224 const line = new TextDecoder().decode(chunk);
225 if (line === "first checkpoint\n") {
226 return Deferred.succeed(firstBatchObserved, undefined).pipe(Effect.asVoid);
227 }
228 if (line === "second checkpoint\n") {
229 return Deferred.succeed(secondBatchObserved, undefined).pipe(Effect.asVoid);
230 }
231 return Effect.void;
232 }),
233 );
234 const spawner = ChildProcessSpawner.make(() =>
235 Effect.gen(function* () {
236 const pid = 600;
237 spawned.push(pid);
238 const handle = makeHandle({ pid, onKill: () => {}, output: connectorOutput });
239 yield* Effect.addFinalizer(() => handle.kill().pipe(Effect.ignore));
240 return handle;
241 }),
242 );
243 const runtime = yield* buildCloudManagedEndpointRuntime(spawner);
244 const config = {
245 providerKind: "cloudflare_tunnel" as const,
246 connectorToken: "token",
247 tunnelId: "deleted-tunnel",
248 };
249 const rejectedLine =
250 '2026-09-15T06:30:43Z ERR Register tunnel error from server side error="Failed to get tunnel" connIndex=0 event=0 ip=198.41.200.23\n';
251
252 yield* runtime.recoveryRequests.pipe(
253 Stream.runForEach((requested) => {
254 recoveryRequestCount += 1;
255 return Deferred.succeed(
256 recoveryRequestCount === 1 ? recoveryRequested : recoveryRetried,
257 requested,
258 ).pipe(Effect.asVoid);
259 }),
260 Effect.forkChild,
261 );
262 yield* runtime.applyConfig(config);
263
264 yield* Queue.offer(output, encoder.encode(rejectedLine.repeat(3)));
265 yield* Queue.offer(output, encoder.encode("first checkpoint\n"));
266 yield* Deferred.await(firstBatchObserved);
267 expect(yield* Deferred.isDone(recoveryRequested)).toBe(false);
268
269 yield* Queue.offer(
270 output,
271 encoder.encode(
272 "2026-06-17T02:00:00Z INF Registered tunnel connection connIndex=0\n" +
273 rejectedLine.repeat(3),
274 ),
275 );
276 yield* Queue.offer(output, encoder.encode("second checkpoint\n"));
277 yield* Deferred.await(secondBatchObserved);
278 expect(yield* Deferred.isDone(recoveryRequested)).toBe(false);
279
280 yield* Queue.offer(output, encoder.encode(rejectedLine));
281
282 expect(yield* Deferred.await(recoveryRequested)).toEqual(config);
283
284 yield* Queue.offer(output, encoder.encode(rejectedLine.repeat(4)));
285
286 expect(yield* Deferred.await(recoveryRetried)).toEqual(config);
287 expect(spawned).toEqual([600]);
288 }),
289 );
290
112291 it.effect("starts, deduplicates, rotates, and stops the Cloudflare connector", () =>
113292 Effect.gen(function* () {
114293 const spawned: Array<ChildProcess.StandardCommand> = [];
......@@ -156,8 +335,8 @@ describe("CloudManagedEndpointRuntime", () => {
156335
157336 expect(spawned.map((command) => command.command)).toEqual(["cloudflared", "cloudflared"]);
158337 expect(spawned.map((command) => command.args)).toEqual([
159 ["tunnel", "run"],
160 ["tunnel", "run"],
338 ["tunnel", "--no-autoupdate", "--loglevel", "info", "--output", "default", "run"],
339 ["tunnel", "--no-autoupdate", "--loglevel", "info", "--output", "default", "run"],
161340 ]);
162341 expect(spawned.map((command) => command.options.env?.TUNNEL_TOKEN)).toEqual([
163342 "token-1",
......@@ -378,6 +557,37 @@ describe("CloudManagedEndpointRuntime", () => {
378557 }).pipe(Effect.provide(TestClock.layer())),
379558 );
380559
560 it.effect("a recovery that returns the same config keeps the crash backoff", () =>
561 Effect.gen(function* () {
562 const { spawner, spawned, exits, spawnSignals } = yield* makeCrashLoopSpawner(900, 4);
563 const runtime = yield* buildCloudManagedEndpointRuntime(spawner);
564 const config = {
565 providerKind: "cloudflare_tunnel" as const,
566 connectorToken: "same-token",
567 tunnelId: "same-tunnel",
568 };
569 // The startup consumer re-applies whatever the relay hands back. When the
570 // relay confirms the current tunnel, that must not look like a config change.
571 yield* runtime.recoveryRequests.pipe(
572 Stream.runForEach((requested) => runtime.applyConfig(requested).pipe(Effect.asVoid)),
573 Effect.forkChild,
574 );
575
576 yield* runtime.applyConfig(config);
577 yield* Deferred.succeed(exits[0]!, ChildProcessSpawner.ExitCode(1));
578 yield* Deferred.await(spawnSignals[1]!);
579 expect(spawned).toEqual([900, 901]);
580
581 // Second rapid crash still waits out the base delay.
582 yield* Deferred.succeed(exits[1]!, ChildProcessSpawner.ExitCode(1));
583 yield* TestClock.adjust(Duration.millis(999));
584 expect(spawned).toEqual([900, 901]);
585 yield* TestClock.adjust(Duration.millis(1));
586 yield* Deferred.await(spawnSignals[2]!);
587 expect(spawned).toEqual([900, 901, 902]);
588 }).pipe(Effect.provide(TestClock.layer())),
589 );
590
381591 it.effect("an explicit config change clears the backoff and preempts a delayed restart", () =>
382592 Effect.gen(function* () {
383593 const { spawner, spawned, exits, spawnSignals } = yield* makeCrashLoopSpawner(800, 3);
......@@ -510,6 +720,7 @@ describe("CloudManagedEndpointRuntime", () => {
510720 expect(status).toEqual({
511721 status: "failed",
512722 providerKind: "cloudflare_tunnel",
723 failure: "not-installed",
513724 reason: "The relay client is not installed.",
514725 });
515726 expect(spawn).not.toHaveBeenCalled();
apps/server/src/cloud/ManagedEndpointRuntime.ts+89-41
......@@ -6,7 +6,7 @@ import * as Duration from "effect/Duration";
66import * as Effect from "effect/Effect";
77import * as Exit from "effect/Exit";
88import * as Layer from "effect/Layer";
9import * as Option from "effect/Option";
9import * as Queue from "effect/Queue";
1010import * as Ref from "effect/Ref";
1111import * as Result from "effect/Result";
1212import * as Semaphore from "effect/Semaphore";
......@@ -15,22 +15,6 @@ import * as Stream from "effect/Stream";
1515import * as ChildProcess from "effect/unstable/process/ChildProcess";
1616import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner";
1717
18import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
19import { CLOUD_ENDPOINT_RUNTIME_CONFIG, decodeRuntimeConfig } from "./config.ts";
20
21function bytesToString(bytes: Uint8Array): string {
22 return new TextDecoder().decode(bytes);
23}
24
25const readRuntimeConfig = Effect.gen(function* () {
26 const secrets = yield* ServerSecretStore.ServerSecretStore;
27 const bytes = yield* secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG);
28 if (Option.isNone(bytes)) {
29 return null;
30 }
31 return Option.getOrNull(decodeRuntimeConfig(bytesToString(bytes.value)));
32});
33
3418export type CloudManagedEndpointRuntimeStatus =
3519 | {
3620 readonly status: "disabled";
......@@ -38,6 +22,7 @@ export type CloudManagedEndpointRuntimeStatus =
3822 | {
3923 readonly status: "failed";
4024 readonly providerKind: RelayManagedEndpointRuntimeConfig["providerKind"];
25 readonly failure: "unsupported-platform" | "not-installed" | "spawn-failed";
4126 readonly reason: string;
4227 readonly tunnelId?: string;
4328 readonly tunnelName?: string;
......@@ -60,6 +45,9 @@ export class CloudManagedEndpointRuntime extends Context.Service<
6045 readonly applyConfig: (
6146 config: RelayManagedEndpointRuntimeConfig | null,
6247 ) => Effect.Effect<CloudManagedEndpointRuntimeStatus>;
48 readonly recoveryRequests: Stream.Stream<RelayManagedEndpointRuntimeConfig>;
49 readonly requestRecovery: (config: RelayManagedEndpointRuntimeConfig) => Effect.Effect<void>;
50 readonly withLinkStateLock: <A, E, R>(effect: Effect.Effect<A, E, R>) => Effect.Effect<A, E, R>;
6351 }
6452>()("t3/cloud/ManagedEndpointRuntime/CloudManagedEndpointRuntime") {}
6553
......@@ -79,6 +67,8 @@ interface ActiveConnector {
7967const RELAY_RESTART_STABLE_UPTIME_MS = 30_000;
8068const RELAY_RESTART_BACKOFF_BASE_MS = 1_000;
8169const RELAY_RESTART_BACKOFF_MAX_MS = 60_000;
70// Newly created tunnels can fail authorization briefly while Cloudflare propagates their token.
71const TUNNEL_AUTHORIZATION_FAILURES_BEFORE_RECOVERY = 4;
8272
8373export function classifyRelayClientOutput(line: string): "connected" | "warning" | "debug" {
8474 if (/\bRegistered tunnel connection\b/iu.test(line)) {
......@@ -90,6 +80,32 @@ export function classifyRelayClientOutput(line: string): "connected" | "warning"
9080 return /\b(?:ERR|WRN|FTL|PNC)\b/u.test(line) ? "warning" : "debug";
9181}
9282
83/**
84 * Cloudflare's edge rejects a connector whose tunnel was deleted or whose
85 * token no longer matches. Current edge output is
86 * `error="Failed to get tunnel"` with no prefix; older edges prefixed the
87 * same messages with `Unauthorized:`. Match both so recovery fires on either.
88 */
89export function isRejectedRelayClientTunnelOutput(line: string): boolean {
90 return (
91 /\bRegister tunnel error from server side\b/iu.test(line) &&
92 /error="(?:Unauthorized:\s*)?(?:Failed to get tunnel|Record for tunnel not found|Invalid tunnel secret)"/iu.test(
93 line,
94 )
95 );
96}
97
98/** Connector startup failures can clear after installation or a later spawn attempt. */
99export function isRetryableManagedEndpointRuntimeStatus(status: unknown): boolean {
100 if (typeof status !== "object" || status === null || !("status" in status)) {
101 return false;
102 }
103 if (status.status !== "failed" || !("failure" in status)) {
104 return false;
105 }
106 return status.failure === "not-installed" || status.failure === "spawn-failed";
107}
108
93109function runtimeConfigKey(config: RelayManagedEndpointRuntimeConfig): string {
94110 return JSON.stringify({
95111 providerKind: config.providerKind,
......@@ -117,8 +133,10 @@ export const make = Effect.gen(function* () {
117133 const relayClient = yield* RelayClient.RelayClient;
118134 const activeRef = yield* Ref.make<ActiveConnector | null>(null);
119135 const desiredConfigRef = yield* Ref.make<RelayManagedEndpointRuntimeConfig | null>(null);
136 const recoveryRequests = yield* Queue.sliding<RelayManagedEndpointRuntimeConfig>(1);
120137 const reconcileSemaphore = yield* Semaphore.make(1);
121138 const restartDelayRef = yield* Ref.make(0);
139 const linkStateSemaphore = yield* Semaphore.make(1);
122140 let reconcileConfig: CloudManagedEndpointRuntime["Service"]["applyConfig"];
123141
124142 const stopActive = Effect.gen(function* () {
......@@ -191,6 +209,7 @@ export const make = Effect.gen(function* () {
191209 tunnelId: connector.config.tunnelId,
192210 tunnelName: connector.config.tunnelName,
193211 });
212 yield* Queue.offer(recoveryRequests, connector.config);
194213 yield* reconcileConfig(desiredConfig);
195214 }),
196215 );
......@@ -198,8 +217,10 @@ export const make = Effect.gen(function* () {
198217 Effect.catchCause((cause) => Effect.logWarning("Relay client supervisor failed", { cause })),
199218 );
200219
201 const observeConnectorOutput = (connector: ActiveConnector) =>
202 connector.child.all.pipe(
220 const observeConnectorOutput = (connector: ActiveConnector) => {
221 let rejectedRegistrations = 0;
222
223 return connector.child.all.pipe(
203224 Stream.decodeText(),
204225 Stream.splitLines,
205226 Stream.map((line) => line.trim()),
......@@ -214,8 +235,22 @@ export const make = Effect.gen(function* () {
214235 };
215236 switch (classifyRelayClientOutput(line)) {
216237 case "connected":
238 rejectedRegistrations = 0;
217239 return Effect.logInfo("Relay client tunnel connection registered", attributes);
218240 case "warning":
241 if (isRejectedRelayClientTunnelOutput(line)) {
242 rejectedRegistrations += 1;
243 if (rejectedRegistrations >= TUNNEL_AUTHORIZATION_FAILURES_BEFORE_RECOVERY) {
244 rejectedRegistrations = 0;
245 return Effect.logWarning(
246 "Relay client tunnel was rejected; requesting recovery",
247 attributes,
248 ).pipe(
249 Effect.andThen(Queue.offer(recoveryRequests, connector.config)),
250 Effect.asVoid,
251 );
252 }
253 }
219254 return Effect.logWarning("Relay client reported a transport warning", attributes);
220255 case "debug":
221256 return Effect.logDebug("Relay client output", attributes);
......@@ -230,6 +265,7 @@ export const make = Effect.gen(function* () {
230265 }),
231266 ),
232267 );
268 };
233269
234270 reconcileConfig = Effect.fn("CloudManagedEndpointRuntime.reconcileConfig")(function* (config) {
235271 if (!config || config.providerKind !== "cloudflare_tunnel") {
......@@ -261,6 +297,7 @@ export const make = Effect.gen(function* () {
261297 return {
262298 status: "failed",
263299 providerKind: "cloudflare_tunnel",
300 failure: executable.status === "unsupported" ? "unsupported-platform" : "not-installed",
264301 reason:
265302 executable.status === "unsupported"
266303 ? `Relay client is unsupported on ${executable.platform}-${executable.arch}.`
......@@ -273,16 +310,20 @@ export const make = Effect.gen(function* () {
273310 const connectorScope = yield* Scope.make("sequential");
274311 const child = yield* spawner
275312 .spawn(
276 ChildProcess.make(executable.executablePath, ["tunnel", "run"], {
277 detached: false,
278 env: {
279 ...process.env,
280 TUNNEL_TOKEN: config.connectorToken,
313 ChildProcess.make(
314 executable.executablePath,
315 ["tunnel", "--no-autoupdate", "--loglevel", "info", "--output", "default", "run"],
316 {
317 detached: false,
318 env: {
319 ...process.env,
320 TUNNEL_TOKEN: config.connectorToken,
321 },
322 shell: false,
323 stderr: "pipe",
324 stdout: "pipe",
281325 },
282 shell: false,
283 stderr: "pipe",
284 stdout: "pipe",
285 }),
326 ),
286327 )
287328 .pipe(
288329 Effect.provideService(Scope.Scope, connectorScope),
......@@ -303,6 +344,7 @@ export const make = Effect.gen(function* () {
303344 Effect.as({
304345 status: "failed",
305346 providerKind: "cloudflare_tunnel",
347 failure: "spawn-failed",
306348 reason: String(cause),
307349 ...(config.tunnelId ? { tunnelId: config.tunnelId } : {}),
308350 ...(config.tunnelName ? { tunnelName: config.tunnelName } : {}),
......@@ -338,6 +380,7 @@ export const make = Effect.gen(function* () {
338380 return {
339381 status: "failed",
340382 providerKind: "cloudflare_tunnel",
383 failure: "spawn-failed",
341384 reason: "Relay client did not start.",
342385 ...(config.tunnelId ? { tunnelId: config.tunnelId } : {}),
343386 ...(config.tunnelName ? { tunnelName: config.tunnelName } : {}),
......@@ -347,26 +390,31 @@ export const make = Effect.gen(function* () {
347390 const applyConfig = Effect.fn("CloudManagedEndpointRuntime.applyConfig")(
348391 (config: RelayManagedEndpointRuntimeConfig | null) =>
349392 reconcileSemaphore.withPermits(1)(
350 // An explicit config change starts over with a fresh backoff.
351 Ref.set(restartDelayRef, 0).pipe(
352 Effect.andThen(Ref.set(desiredConfigRef, config)),
353 Effect.andThen(reconcileConfig(config)),
354 ),
393 Effect.gen(function* () {
394 // A real config change starts over with a fresh backoff. Recovery
395 // that hands back the same tunnel and token must keep the delay, or
396 // a crash-looping connector respawns on every recovery round trip.
397 const desired = yield* Ref.get(desiredConfigRef);
398 const unchanged =
399 desired !== null &&
400 config !== null &&
401 runtimeConfigKey(desired) === runtimeConfigKey(config);
402 if (!unchanged) {
403 yield* Ref.set(restartDelayRef, 0);
404 }
405 yield* Ref.set(desiredConfigRef, config);
406 return yield* reconcileConfig(config);
407 }),
355408 ),
356409 );
357410
358411 const runtime = CloudManagedEndpointRuntime.of({
359412 applyConfig,
413 recoveryRequests: Stream.fromQueue(recoveryRequests),
414 requestRecovery: (config) => Queue.offer(recoveryRequests, config).pipe(Effect.asVoid),
415 withLinkStateLock: linkStateSemaphore.withPermits(1),
360416 });
361417
362 const initialConfig = yield* readRuntimeConfig.pipe(
363 Effect.catch((cause) =>
364 Effect.logWarning("Failed to read managed endpoint runtime config", { cause }).pipe(
365 Effect.as(null),
366 ),
367 ),
368 );
369 yield* runtime.applyConfig(initialConfig);
370418 yield* Effect.addFinalizer(() => runtime.applyConfig(null));
371419 return runtime;
372420});
apps/server/src/cloud/config.ts+18-1
......@@ -1,4 +1,7 @@
1import { RelayManagedEndpointRuntimeConfig } from "@t3tools/contracts/relay";
1import {
2 RelayManagedEndpointOrigin,
3 RelayManagedEndpointRuntimeConfig,
4} from "@t3tools/contracts/relay";
25import * as Effect from "effect/Effect";
36import * as Option from "effect/Option";
47import * as Schema from "effect/Schema";
......@@ -7,6 +10,7 @@ import type * as ServerSecretStore from "../auth/ServerSecretStore.ts";
710
811export const CLOUD_MINT_PUBLIC_KEY = "cloud-mint-ed25519-public-key";
912export const CLOUD_ENDPOINT_RUNTIME_CONFIG = "cloud-endpoint-runtime-config";
13export const CLOUD_ENDPOINT_CONFIRMED_ORIGIN = "cloud-endpoint-confirmed-origin";
1014export const CLOUD_LINKED_USER_ID = "cloud-linked-user-id";
1115export const RELAY_URL_SECRET = "cloud-relay-url";
1216export const RELAY_ISSUER_SECRET = "cloud-relay-issuer";
......@@ -21,6 +25,19 @@ export const decodeRuntimeConfig = Schema.decodeUnknownOption(
2125 Schema.fromJsonString(RelayManagedEndpointRuntimeConfig),
2226);
2327
28export const ManagedEndpointConfirmedOrigin = Schema.Struct({
29 config: RelayManagedEndpointRuntimeConfig,
30 origin: RelayManagedEndpointOrigin,
31});
32
33export const encodeConfirmedOriginJson = Schema.encodeEffect(
34 Schema.fromJsonString(ManagedEndpointConfirmedOrigin),
35);
36
37export const decodeConfirmedOrigin = Schema.decodeUnknownOption(
38 Schema.fromJsonString(ManagedEndpointConfirmedOrigin),
39);
40
2441export function isAgentActivityPublishingEnabledValue(value: string | null): boolean {
2542 return value === "true";
2643}
apps/server/src/cloud/http.test.ts+605-11
......@@ -1,14 +1,18 @@
11import * as NodeServices from "@effect/platform-node/NodeServices";
22import { describe, expect, it } from "@effect/vitest";
3import * as Deferred from "effect/Deferred";
34import * as Duration from "effect/Duration";
45import * as Effect from "effect/Effect";
6import * as Fiber from "effect/Fiber";
57import * as FileSystem from "effect/FileSystem";
68import * as Layer from "effect/Layer";
79import * as Option from "effect/Option";
810import * as Path from "effect/Path";
911import * as PlatformError from "effect/PlatformError";
12import * as Schema from "effect/Schema";
1013import * as TestClock from "effect/testing/TestClock";
1114import * as Tracer from "effect/Tracer";
15import * as Stream from "effect/Stream";
1216import {
1317 HttpClient,
1418 HttpClientResponse,
......@@ -31,16 +35,37 @@ import {
3135import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
3236import { CLOUD_CLI_DESIRED_LINK_SECRET } from "./CliState.ts";
3337import * as CliTokenManager from "./CliTokenManager.ts";
34import type { RelayLinkProofRequest } from "@t3tools/contracts/relay";
35import { CLOUD_ENDPOINT_RUNTIME_CONFIG, RELAY_URL_SECRET } from "./config.ts";
38import {
39 RelayManagedEndpointRecoveryRegistrationRequest,
40 type RelayLinkProofRequest,
41} from "@t3tools/contracts/relay";
42import {
43 CLOUD_ENDPOINT_CONFIRMED_ORIGIN,
44 CLOUD_ENDPOINT_RUNTIME_CONFIG,
45 CLOUD_LINKED_USER_ID,
46 decodeConfirmedOrigin,
47 decodeRuntimeConfig,
48 RELAY_ENVIRONMENT_CREDENTIAL_SECRET,
49 RELAY_URL_SECRET,
50} from "./config.ts";
3651import {
3752 consumeCloudReplayGuards,
3853 isSupportedLinkProviderKind,
3954 linkProofScopes,
4055 pendingServiceUpdateExists,
56 parseManagedEndpointLocalOrigin,
4157 reconcileDesiredCloudLink,
58 reconcileDesiredCloudLinkIfStillDesired,
59 recoverManagedCloudTunnel,
60 registerManagedCloudTunnelRecovery,
4261 releaseManagedTunnelOnShutdown,
62 startManagedCloudTunnelIfOriginConfirmed,
4363} from "./http.ts";
64import {
65 managedTunnelStartupAction,
66 retryManagedTunnelRegistration,
67} from "./managedTunnelStartup.ts";
68import { shouldRetryCloudLink } from "./relayResponse.ts";
4469import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts";
4570import { traceAuthenticatedRelayRequest, traceRelayRequest } from "./traceRelayRequest.ts";
4671
......@@ -56,6 +81,9 @@ const storeFailure = (tag: "AlreadyExists" | "PermissionDenied") =>
5681 });
5782
5883const unusedSecretStoreOperation = () => Effect.die("unused secret-store operation");
84const decodeManagedTunnelRecoveryRegistration = Schema.decodeUnknownEffect(
85 Schema.fromJsonString(RelayManagedEndpointRecoveryRegistrationRequest),
86);
5987
6088function makeSecretStore(
6189 create: ServerSecretStore.ServerSecretStore["Service"]["create"],
......@@ -211,6 +239,9 @@ describe("reconcileDesiredCloudLink", () => {
211239 ManagedEndpointRuntime.CloudManagedEndpointRuntime,
212240 ManagedEndpointRuntime.CloudManagedEndpointRuntime.of({
213241 applyConfig: unusedSecretStoreOperation,
242 recoveryRequests: Stream.empty,
243 requestRecovery: () => Effect.void,
244 withLinkStateLock: (effect) => effect,
214245 } satisfies ManagedEndpointRuntime.CloudManagedEndpointRuntime["Service"]),
215246 ),
216247 Effect.provideService(
......@@ -236,6 +267,39 @@ describe("reconcileDesiredCloudLink", () => {
236267 );
237268});
238269
270describe("parseManagedEndpointLocalOrigin", () => {
271 it.each([
272 {
273 input: "http://127.0.0.1:80",
274 httpBaseUrl: "http://127.0.0.1",
275 wsBaseUrl: "ws://127.0.0.1",
276 port: 80,
277 },
278 {
279 input: "https://127.0.0.1:443",
280 httpBaseUrl: "https://127.0.0.1",
281 wsBaseUrl: "wss://127.0.0.1",
282 port: 443,
283 },
284 ])("accepts an explicit default port in $input", ({ input, httpBaseUrl, wsBaseUrl, port }) => {
285 expect(parseManagedEndpointLocalOrigin(input)).toEqual({
286 httpBaseUrl,
287 wsBaseUrl,
288 origin: { localHttpHost: "127.0.0.1", localHttpPort: port },
289 });
290 });
291
292 it.each([
293 "ftp://127.0.0.1:3773",
294 "http://user:password@127.0.0.1:3773",
295 "http://127.0.0.1:3773/api",
296 "http://127.0.0.1:3773?mode=test",
297 "http://127.0.0.1:3773#fragment",
298 ])("rejects non-origin URL %s", (input) => {
299 expect(() => parseManagedEndpointLocalOrigin(input)).toThrow("Invalid local origin");
300 });
301});
302
239303describe("releaseManagedTunnelOnShutdown", () => {
240304 const cliToken: CliTokenManager.PersistedToken = {
241305 accessToken: "cli-access-token",
......@@ -253,7 +317,10 @@ describe("releaseManagedTunnelOnShutdown", () => {
253317 Effect.sync(() => {
254318 values.set(name, value);
255319 }),
256 create: unusedSecretStoreOperation,
320 create: (name, value) =>
321 Effect.sync(() => {
322 values.set(name, value);
323 }),
257324 getOrCreateRandom: unusedSecretStoreOperation,
258325 remove: (name) =>
259326 Effect.sync(() => {
......@@ -267,7 +334,9 @@ describe("releaseManagedTunnelOnShutdown", () => {
267334 readonly store: ServerSecretStore.ServerSecretStore["Service"];
268335 readonly applyConfigCalls: Array<unknown>;
269336 readonly requests: Array<HttpClientRequest.HttpClientRequest>;
337 readonly onRequest?: (request: HttpClientRequest.HttpClientRequest) => Effect.Effect<void>;
270338 readonly respond?: () => Response;
339 readonly respondEffect?: Effect.Effect<Response>;
271340 }
272341
273342 // Writes the launcher's durable state file into this test's baseDir with
......@@ -319,10 +388,19 @@ describe("releaseManagedTunnelOnShutdown", () => {
319388 applyConfig: (config) =>
320389 Effect.sync(() => {
321390 harness.applyConfigCalls.push(config);
322 return {
323 status: "disabled",
324 } satisfies ManagedEndpointRuntime.CloudManagedEndpointRuntimeStatus;
391 return config === null
392 ? ({
393 status: "disabled",
394 } satisfies ManagedEndpointRuntime.CloudManagedEndpointRuntimeStatus)
395 : ({
396 status: "running",
397 providerKind: "cloudflare_tunnel",
398 pid: 123,
399 } satisfies ManagedEndpointRuntime.CloudManagedEndpointRuntimeStatus);
325400 }),
401 recoveryRequests: Stream.empty,
402 requestRecovery: () => Effect.void,
403 withLinkStateLock: (effect) => effect,
326404 }),
327405 ),
328406 Effect.provideService(
......@@ -344,11 +422,14 @@ describe("releaseManagedTunnelOnShutdown", () => {
344422 HttpClient.make((request) =>
345423 Effect.sync(() => {
346424 harness.requests.push(request);
347 return HttpClientResponse.fromWeb(
348 request,
349 (harness.respond ?? (() => Response.json({ ok: true })))(),
350 );
351 }),
425 }).pipe(
426 Effect.andThen(harness.onRequest?.(request) ?? Effect.void),
427 Effect.andThen(
428 harness.respondEffect ??
429 Effect.sync(() => (harness.respond ?? (() => Response.json({ ok: true })))()),
430 ),
431 Effect.map((response) => HttpClientResponse.fromWeb(request, response)),
432 ),
352433 ),
353434 ),
354435 // The release consults the launcher state file under the configured
......@@ -364,10 +445,27 @@ describe("releaseManagedTunnelOnShutdown", () => {
364445 // The persisted state of a CLI-managed link whose tunnel is releasable.
365446 const managedLinkSecrets = [
366447 [CLOUD_ENDPOINT_RUNTIME_CONFIG, "runtime-config"],
448 [CLOUD_ENDPOINT_CONFIRMED_ORIGIN, "confirmed-origin"],
367449 [RELAY_URL_SECRET, "https://relay.example.test"],
368450 [CLOUD_CLI_DESIRED_LINK_SECRET, "managed"],
369451 ] as const;
370452
453 it.effect("does not recreate a link that was unlinked while startup registration retried", () => {
454 const { store, values } = makeMemorySecretStore(managedLinkSecrets);
455 const applyConfigCalls: Array<unknown> = [];
456 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
457
458 return Effect.gen(function* () {
459 // Registration started while this marker existed. Unlink removes it
460 // before startup receives the relay's final not_linked response.
461 values.delete(CLOUD_CLI_DESIRED_LINK_SECRET);
462
463 expect(yield* reconcileDesiredCloudLinkIfStillDesired("http://127.0.0.1:3773")).toBeNull();
464 expect(requests).toEqual([]);
465 expect(applyConfigCalls).toEqual([]);
466 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
467 });
468
371469 it.effect("stops the connector, releases the relay tunnel, and drops the dead token", () => {
372470 const { store, values } = makeMemorySecretStore(managedLinkSecrets);
373471 const applyConfigCalls: Array<unknown> = [];
......@@ -386,6 +484,7 @@ describe("releaseManagedTunnelOnShutdown", () => {
386484 );
387485 expect(request.headers.authorization).toBe("Bearer cli-access-token");
388486 expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(false);
487 expect(values.has(CLOUD_ENDPOINT_CONFIRMED_ORIGIN)).toBe(false);
389488 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
390489 });
391490
......@@ -627,6 +726,501 @@ describe("releaseManagedTunnelOnShutdown", () => {
627726 }),
628727 );
629728 });
729
730 it.effect("registers an existing tunnel and starts the confirmed connector", () => {
731 const { store } = makeMemorySecretStore([
732 [
733 CLOUD_ENDPOINT_RUNTIME_CONFIG,
734 '{"providerKind":"cloudflare_tunnel","connectorToken":"existing-token","tunnelId":"existing-tunnel"}',
735 ],
736 [RELAY_URL_SECRET, "https://relay.example.test"],
737 [CLOUD_LINKED_USER_ID, "user-123"],
738 [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"],
739 ]);
740 const applyConfigCalls: Array<unknown> = [];
741 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
742
743 return Effect.gen(function* () {
744 expect(yield* registerManagedCloudTunnelRecovery("http://127.0.0.1:3773")).toMatchObject({
745 status: "ready",
746 });
747 expect(requests).toHaveLength(1);
748 expect(requests[0]?.method).toBe("POST");
749 expect(requests[0]?.url).toBe(
750 "https://relay.example.test/v1/environments/env_123/tunnel/recovery",
751 );
752 expect(requests[0]?.headers.authorization).toBe("Bearer environment-credential");
753 const body = requests[0]?.body;
754 expect(body?._tag).toBe("Uint8Array");
755 if (body?._tag === "Uint8Array") {
756 expect(
757 yield* decodeManagedTunnelRecoveryRegistration(new TextDecoder().decode(body.body)),
758 ).toMatchObject({
759 cloudUserId: "user-123",
760 tunnelId: "existing-tunnel",
761 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
762 });
763 }
764 expect(applyConfigCalls).toHaveLength(1);
765 }).pipe(
766 provideReleaseHarness({
767 store,
768 applyConfigCalls,
769 requests,
770 respond: () => Response.json({ status: "ready" }),
771 }),
772 );
773 });
774
775 it.effect("reconciles a changed port after a relay outage outlasts the startup fallback", () => {
776 const config = {
777 providerKind: "cloudflare_tunnel" as const,
778 connectorToken: "existing-token",
779 tunnelId: "existing-tunnel",
780 };
781 const { store } = makeMemorySecretStore([
782 [CLOUD_ENDPOINT_RUNTIME_CONFIG, JSON.stringify(config)],
783 [
784 CLOUD_ENDPOINT_CONFIRMED_ORIGIN,
785 JSON.stringify({
786 config,
787 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
788 }),
789 ],
790 [RELAY_URL_SECRET, "https://relay.example.test"],
791 [CLOUD_LINKED_USER_ID, "user-123"],
792 [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"],
793 ]);
794 const applyConfigCalls: Array<unknown> = [];
795 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
796 let relayAvailable = false;
797 const localOrigin = "http://127.0.0.1:4884";
798
799 return Effect.gen(function* () {
800 const fallbackStarted = yield* Deferred.make<void>();
801 const firstFailure = yield* Deferred.make<void>();
802 expect(yield* startManagedCloudTunnelIfOriginConfirmed(localOrigin)).toBe(false);
803 const registration = yield* Effect.forkChild(
804 retryManagedTunnelRegistration(
805 registerManagedCloudTunnelRecovery(localOrigin).pipe(
806 Effect.tapError(() => Deferred.succeed(firstFailure, undefined)),
807 ),
808 shouldRetryCloudLink,
809 startManagedCloudTunnelIfOriginConfirmed(localOrigin, {
810 requireConfirmedOrigin: false,
811 }).pipe(
812 Effect.orDie,
813 Effect.tap((started) => {
814 expect(started).toBe(true);
815 return Deferred.succeed(fallbackStarted, undefined);
816 }),
817 Effect.asVoid,
818 ),
819 ),
820 { startImmediately: true },
821 );
822 yield* Deferred.await(firstFailure);
823 yield* TestClock.adjust("15 minutes");
824 yield* Effect.raceFirst(
825 Deferred.await(fallbackStarted),
826 Fiber.join(registration).pipe(
827 Effect.andThen(Effect.die("Registration ended before starting the fallback")),
828 ),
829 );
830 expect(applyConfigCalls).toEqual([config]);
831 const attemptsBeforeRecovery = requests.length;
832
833 relayAvailable = true;
834 yield* TestClock.adjust("1 minute");
835 expect(yield* Fiber.join(registration)).toMatchObject({ status: "ready" });
836 expect(requests.length).toBeGreaterThan(attemptsBeforeRecovery);
837 const marker = yield* store.get(CLOUD_ENDPOINT_CONFIRMED_ORIGIN);
838 expect(Option.isSome(marker)).toBe(true);
839 if (Option.isSome(marker)) {
840 expect(
841 Option.getOrThrow(decodeConfirmedOrigin(new TextDecoder().decode(marker.value))),
842 ).toEqual({
843 config,
844 origin: { localHttpHost: "127.0.0.1", localHttpPort: 4884 },
845 });
846 }
847 }).pipe(
848 provideReleaseHarness({
849 store,
850 applyConfigCalls,
851 requests,
852 respond: () =>
853 relayAvailable
854 ? Response.json({ status: "ready" })
855 : Response.json({ message: "relay unavailable" }, { status: 503 }),
856 }),
857 );
858 });
859
860 it.effect(
861 "starts a connector with a marker for the current origin without contacting relay",
862 () => {
863 const configJson =
864 '{"providerKind":"cloudflare_tunnel","connectorToken":"existing-token","tunnelId":"existing-tunnel"}';
865 const config = {
866 providerKind: "cloudflare_tunnel" as const,
867 connectorToken: "existing-token",
868 tunnelId: "existing-tunnel",
869 };
870 const { store } = makeMemorySecretStore([
871 [CLOUD_ENDPOINT_RUNTIME_CONFIG, configJson],
872 [
873 CLOUD_ENDPOINT_CONFIRMED_ORIGIN,
874 `{"config":${configJson},"origin":{"localHttpHost":"127.0.0.1","localHttpPort":3773}}`,
875 ],
876 ]);
877 const applyConfigCalls: Array<unknown> = [];
878 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
879
880 return Effect.gen(function* () {
881 expect(yield* startManagedCloudTunnelIfOriginConfirmed("http://127.0.0.1:3773")).toBe(true);
882 expect(applyConfigCalls).toEqual([config]);
883 expect(requests).toEqual([]);
884 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
885 },
886 );
887
888 it.effect.each([
889 { name: "missing", marker: undefined, origin: "http://127.0.0.1:3773" },
890 {
891 name: "stale",
892 marker:
893 '{"config":{"providerKind":"cloudflare_tunnel","connectorToken":"existing-token","tunnelId":"existing-tunnel"},"origin":{"localHttpHost":"127.0.0.1","localHttpPort":3773}}',
894 origin: "http://127.0.0.1:4884",
895 },
896 ])("does not start a connector with a $name origin marker", ({ marker, origin }) => {
897 const entries: Array<readonly [string, string]> = [
898 [
899 CLOUD_ENDPOINT_RUNTIME_CONFIG,
900 '{"providerKind":"cloudflare_tunnel","connectorToken":"existing-token","tunnelId":"existing-tunnel"}',
901 ],
902 ];
903 if (marker !== undefined) entries.push([CLOUD_ENDPOINT_CONFIRMED_ORIGIN, marker]);
904 const { store } = makeMemorySecretStore(entries);
905 const applyConfigCalls: Array<unknown> = [];
906 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
907
908 return Effect.gen(function* () {
909 expect(yield* startManagedCloudTunnelIfOriginConfirmed(origin)).toBe(false);
910 expect(applyConfigCalls).toEqual([]);
911 expect(requests).toEqual([]);
912 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
913 });
914
915 it.effect(
916 "starts the stored connector without a marker when confirmation is not required",
917 () => {
918 const config = {
919 providerKind: "cloudflare_tunnel" as const,
920 connectorToken: "existing-token",
921 tunnelId: "existing-tunnel",
922 };
923 const { store } = makeMemorySecretStore([
924 [CLOUD_ENDPOINT_RUNTIME_CONFIG, JSON.stringify(config)],
925 ]);
926 const applyConfigCalls: Array<unknown> = [];
927 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
928
929 return Effect.gen(function* () {
930 expect(
931 yield* startManagedCloudTunnelIfOriginConfirmed("http://127.0.0.1:3773", {
932 requireConfirmedOrigin: false,
933 }),
934 ).toBe(true);
935 expect(applyConfigCalls).toEqual([config]);
936 expect(requests).toEqual([]);
937 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
938 },
939 );
940
941 it.effect.each(["replaced", "removed"] as const)(
942 "does not activate a tunnel when its runtime config is %s during registration",
943 (mutation) => {
944 const originalConfig =
945 '{"providerKind":"cloudflare_tunnel","connectorToken":"existing-token","tunnelId":"existing-tunnel"}';
946 const { store, values } = makeMemorySecretStore([
947 [CLOUD_ENDPOINT_RUNTIME_CONFIG, originalConfig],
948 [RELAY_URL_SECRET, "https://relay.example.test"],
949 [CLOUD_LINKED_USER_ID, "user-123"],
950 [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"],
951 ]);
952 const applyConfigCalls: Array<unknown> = [];
953 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
954
955 return Effect.gen(function* () {
956 expect(yield* registerManagedCloudTunnelRecovery("http://127.0.0.1:3773")).toEqual({
957 status: "superseded",
958 });
959 expect(applyConfigCalls).toEqual([]);
960 expect(values.has(CLOUD_ENDPOINT_CONFIRMED_ORIGIN)).toBe(false);
961 }).pipe(
962 provideReleaseHarness({
963 store,
964 applyConfigCalls,
965 requests,
966 respond: () => {
967 if (mutation === "replaced") {
968 values.set(
969 CLOUD_ENDPOINT_RUNTIME_CONFIG,
970 new TextEncoder().encode(
971 '{"providerKind":"cloudflare_tunnel","connectorToken":"fresh-token","tunnelId":"fresh-tunnel"}',
972 ),
973 );
974 } else {
975 values.delete(CLOUD_ENDPOINT_RUNTIME_CONFIG);
976 }
977 return Response.json({ status: "ready" });
978 },
979 }),
980 );
981 },
982 );
983
984 it.effect("requests startup recovery for a legacy config without a recorded tunnel ID", () => {
985 const { store } = makeMemorySecretStore([
986 [
987 CLOUD_ENDPOINT_RUNTIME_CONFIG,
988 '{"providerKind":"cloudflare_tunnel","connectorToken":"token"}',
989 ],
990 [RELAY_URL_SECRET, "https://relay.example.test"],
991 [CLOUD_LINKED_USER_ID, "user-123"],
992 [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"],
993 ]);
994 const applyConfigCalls: Array<unknown> = [];
995 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
996
997 return Effect.gen(function* () {
998 const registration = yield* registerManagedCloudTunnelRecovery("http://127.0.0.1:3773");
999 expect(registration).toEqual({
1000 status: "recovery_required",
1001 config: { providerKind: "cloudflare_tunnel", connectorToken: "token" },
1002 });
1003 expect(
1004 managedTunnelStartupAction({
1005 wantsCliLink: false,
1006 registration,
1007 }),
1008 ).toEqual({
1009 action: "request_recovery",
1010 config: { providerKind: "cloudflare_tunnel", connectorToken: "token" },
1011 });
1012 expect(requests).toEqual([]);
1013 expect(applyConfigCalls).toEqual([]);
1014 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
1015 });
1016
1017 it.effect("recovers a web-linked tunnel with its environment credential", () => {
1018 const oldConfig =
1019 '{"providerKind":"cloudflare_tunnel","connectorToken":"old-token","tunnelId":"old-tunnel"}';
1020 const nextConfig = {
1021 providerKind: "cloudflare_tunnel",
1022 connectorToken: "new-token",
1023 tunnelId: "new-tunnel",
1024 } as const;
1025 const { store, values } = makeMemorySecretStore([
1026 [CLOUD_ENDPOINT_RUNTIME_CONFIG, oldConfig],
1027 [RELAY_URL_SECRET, "https://relay.example.test"],
1028 [CLOUD_LINKED_USER_ID, "user-123"],
1029 [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"],
1030 ]);
1031 const applyConfigCalls: Array<unknown> = [];
1032 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
1033
1034 return Effect.gen(function* () {
1035 expect(yield* recoverManagedCloudTunnel("http://127.0.0.1:3773")).toBe(true);
1036 expect(requests).toHaveLength(1);
1037 expect(requests[0]?.method).toBe("POST");
1038 expect(requests[0]?.url).toBe("https://relay.example.test/v1/environments/env_123/tunnel");
1039 expect(requests[0]?.headers.authorization).toBe("Bearer environment-credential");
1040 expect(applyConfigCalls).toEqual([nextConfig]);
1041 expect(
1042 Option.getOrNull(
1043 decodeRuntimeConfig(new TextDecoder().decode(values.get(CLOUD_ENDPOINT_RUNTIME_CONFIG))),
1044 ),
1045 ).toEqual(nextConfig);
1046 }).pipe(
1047 provideReleaseHarness({
1048 store,
1049 applyConfigCalls,
1050 requests,
1051 respond: () =>
1052 Response.json({
1053 endpoint: {
1054 httpBaseUrl: "https://environment.example.test/",
1055 wsBaseUrl: "wss://environment.example.test/ws",
1056 providerKind: "cloudflare_tunnel",
1057 },
1058 endpointRuntime: nextConfig,
1059 }),
1060 }),
1061 );
1062 });
1063
1064 it.effect("allows managed tunnel provisioning to take longer than ten seconds", () =>
1065 Effect.gen(function* () {
1066 const oldConfig =
1067 '{"providerKind":"cloudflare_tunnel","connectorToken":"old-token","tunnelId":"old-tunnel"}';
1068 const nextConfig = {
1069 providerKind: "cloudflare_tunnel" as const,
1070 connectorToken: "new-token",
1071 tunnelId: "new-tunnel",
1072 };
1073 const { store } = makeMemorySecretStore([
1074 [CLOUD_ENDPOINT_RUNTIME_CONFIG, oldConfig],
1075 [RELAY_URL_SECRET, "https://relay.example.test"],
1076 [CLOUD_LINKED_USER_ID, "user-123"],
1077 [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"],
1078 ]);
1079 const applyConfigCalls: Array<unknown> = [];
1080 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
1081 const requestStarted = yield* Deferred.make<void>();
1082 const response = yield* Deferred.make<Response>();
1083 const recovery = yield* recoverManagedCloudTunnel("http://127.0.0.1:3773").pipe(
1084 provideReleaseHarness({
1085 store,
1086 applyConfigCalls,
1087 requests,
1088 onRequest: () => Deferred.succeed(requestStarted, undefined),
1089 respondEffect: Deferred.await(response),
1090 }),
1091 Effect.forkChild({ startImmediately: true }),
1092 );
1093
1094 yield* Deferred.await(requestStarted);
1095 expect(requests).toHaveLength(1);
1096 yield* TestClock.adjust("11 seconds");
1097 yield* Effect.yieldNow;
1098 yield* Deferred.succeed(
1099 response,
1100 Response.json({
1101 endpoint: {
1102 httpBaseUrl: "https://environment.example.test/",
1103 wsBaseUrl: "wss://environment.example.test/ws",
1104 providerKind: "cloudflare_tunnel",
1105 },
1106 endpointRuntime: nextConfig,
1107 }),
1108 );
1109
1110 expect(yield* Fiber.join(recovery)).toBe(true);
1111 expect(requests).toHaveLength(1);
1112 expect(applyConfigCalls).toEqual([nextConfig]);
1113 }),
1114 );
1115
1116 it.effect("does not recover an environment without a managed tunnel credential", () => {
1117 const { store } = makeMemorySecretStore([
1118 [CLOUD_ENDPOINT_RUNTIME_CONFIG, "old-config"],
1119 [RELAY_URL_SECRET, "https://relay.example.test"],
1120 ]);
1121 const applyConfigCalls: Array<unknown> = [];
1122 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
1123
1124 return Effect.gen(function* () {
1125 expect(yield* recoverManagedCloudTunnel("http://127.0.0.1:3773")).toBe(false);
1126 expect(applyConfigCalls).toEqual([]);
1127 expect(requests).toEqual([]);
1128 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
1129 });
1130
1131 it.effect("ignores recovery requests for a tunnel that has already been replaced", () => {
1132 const { store } = makeMemorySecretStore([
1133 [
1134 CLOUD_ENDPOINT_RUNTIME_CONFIG,
1135 '{"providerKind":"cloudflare_tunnel","connectorToken":"current-token","tunnelId":"current-tunnel"}',
1136 ],
1137 [RELAY_URL_SECRET, "https://relay.example.test"],
1138 [CLOUD_LINKED_USER_ID, "user-123"],
1139 [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"],
1140 ]);
1141 const applyConfigCalls: Array<unknown> = [];
1142 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
1143
1144 return Effect.gen(function* () {
1145 expect(
1146 yield* recoverManagedCloudTunnel("http://127.0.0.1:3773", {
1147 providerKind: "cloudflare_tunnel",
1148 connectorToken: "old-token",
1149 tunnelId: "old-tunnel",
1150 }),
1151 ).toBe(false);
1152 expect(requests).toEqual([]);
1153 expect(applyConfigCalls).toEqual([]);
1154 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
1155 });
1156
1157 it.effect.each([
1158 { status: 401, errorTag: "EnvironmentHttpUnauthorizedError" },
1159 { status: 403, errorTag: "EnvironmentHttpForbiddenError" },
1160 { status: 409, errorTag: "EnvironmentHttpBadRequestError" },
1161 ])("preserves a permanent $status relay recovery failure", ({ status, errorTag }) => {
1162 const { store } = makeMemorySecretStore([
1163 [CLOUD_ENDPOINT_RUNTIME_CONFIG, "old-config"],
1164 [RELAY_URL_SECRET, "https://relay.example.test"],
1165 [CLOUD_LINKED_USER_ID, "user-123"],
1166 [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"],
1167 ]);
1168 const applyConfigCalls: Array<unknown> = [];
1169 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
1170
1171 return Effect.gen(function* () {
1172 const error = yield* Effect.flip(recoverManagedCloudTunnel("http://127.0.0.1:3773"));
1173
1174 expect(error._tag).toBe(errorTag);
1175 expect(requests).toHaveLength(1);
1176 expect(applyConfigCalls).toEqual([]);
1177 }).pipe(
1178 provideReleaseHarness({
1179 store,
1180 applyConfigCalls,
1181 requests,
1182 respond: () => Response.json({}, { status }),
1183 }),
1184 );
1185 });
1186
1187 it.effect("keeps a tunnel configuration replaced during recovery", () => {
1188 const { store, values } = makeMemorySecretStore([
1189 [CLOUD_ENDPOINT_RUNTIME_CONFIG, "old-config"],
1190 [RELAY_URL_SECRET, "https://relay.example.test"],
1191 [CLOUD_LINKED_USER_ID, "user-123"],
1192 [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"],
1193 ]);
1194 const applyConfigCalls: Array<unknown> = [];
1195 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
1196 const freshConfig = new TextEncoder().encode("fresh-config");
1197
1198 return Effect.gen(function* () {
1199 expect(yield* recoverManagedCloudTunnel("http://127.0.0.1:3773")).toBe(false);
1200 expect(values.get(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(freshConfig);
1201 expect(applyConfigCalls).toEqual([]);
1202 }).pipe(
1203 provideReleaseHarness({
1204 store,
1205 applyConfigCalls,
1206 requests,
1207 respond: () => {
1208 values.set(CLOUD_ENDPOINT_RUNTIME_CONFIG, freshConfig);
1209 return Response.json({
1210 endpoint: {
1211 httpBaseUrl: "https://environment.example.test/",
1212 wsBaseUrl: "wss://environment.example.test/ws",
1213 providerKind: "cloudflare_tunnel",
1214 },
1215 endpointRuntime: {
1216 providerKind: "cloudflare_tunnel",
1217 connectorToken: "replacement-token",
1218 },
1219 });
1220 },
1221 }),
1222 );
1223 });
6301224});
6311225
6321226describe("link proof provider kinds", () => {
apps/server/src/cloud/http.ts+583-84
......@@ -29,6 +29,10 @@ import {
2929 RelayEnvironmentLinkProofPayload,
3030 RelayLinkProofRequest,
3131 RelayManagedEndpointOrigin,
32 RelayManagedEndpointRecoveryProofPayload,
33 RelayManagedEndpointRecoveryRegistrationResponse,
34 RelayManagedEndpointRecoveryResponse,
35 type RelayManagedEndpointRuntimeConfig,
3236 RelayOkResponse,
3337} from "@t3tools/contracts/relay";
3438import { withRelayClientTracing } from "@t3tools/shared/relayTracing";
......@@ -37,6 +41,7 @@ import {
3741 RELAY_HEALTH_REQUEST_TYP,
3842 RELAY_HEALTH_RESPONSE_TYP,
3943 RELAY_LINK_PROOF_TYP,
44 RELAY_MANAGED_TUNNEL_RECOVERY_TYP,
4045 RELAY_MINT_REQUEST_TYP,
4146 RELAY_MINT_RESPONSE_TYP,
4247 signRelayJwt,
......@@ -52,10 +57,12 @@ import * as FileSystem from "effect/FileSystem";
5257import * as Option from "effect/Option";
5358import * as Path from "effect/Path";
5459import * as Schema from "effect/Schema";
60import * as Schedule from "effect/Schedule";
5561import * as HttpEffect from "effect/unstable/http/HttpEffect";
5662import { HttpServerRequest, HttpServerResponse } from "effect/unstable/http";
5763import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http";
5864import * as HttpApiBuilder from "effect/unstable/httpapi/HttpApiBuilder";
65import * as HttpServer from "effect/unstable/http/HttpServer";
5966
6067import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts";
6168import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
......@@ -70,9 +77,13 @@ import {
7077} from "./serviceProtocol.ts";
7178import {
7279 CLOUD_ENDPOINT_RUNTIME_CONFIG,
80 CLOUD_ENDPOINT_CONFIRMED_ORIGIN,
81 decodeConfirmedOrigin,
7382 CLOUD_LINKED_USER_ID,
7483 CLOUD_MINT_PUBLIC_KEY,
84 decodeRuntimeConfig,
7585 encodeEndpointRuntimeConfigJson,
86 encodeConfirmedOriginJson,
7687 PUBLISH_AGENT_ACTIVITY_SECRET,
7788 RELAY_ENVIRONMENT_CREDENTIAL_SECRET,
7889 RELAY_ISSUER_SECRET,
......@@ -87,7 +98,7 @@ import {
8798import * as CliTokenManager from "./CliTokenManager.ts";
8899import { getOrCreateEnvironmentKeyPairFromSecretStore } from "./environmentKeys.ts";
89100import { traceRelayRequest } from "./traceRelayRequest.ts";
90import { filterRelayResponse, relayRequestError } from "./relayResponse.ts";
101import { filterRelayResponse, relayRequestError, shouldRetryCloudLink } from "./relayResponse.ts";
91102
92103const CLOUD_MINT_NONCE_PREFIX = "cloud-mint-nonce-";
93104const CLOUD_MINT_JTI_PREFIX = "cloud-mint-jti-";
......@@ -97,6 +108,7 @@ const CLOUD_PROOF_MAX_LIFETIME_SECONDS = 5 * 60;
97108const CLOUD_PROOF_CLOCK_SKEW_SECONDS = 60;
98109// The desktop app stops its backends within seconds of writing the marker.
99110const DESKTOP_UPDATE_RESTART_MARKER_TTL = Duration.minutes(1);
111const MANAGED_ENDPOINT_PROVISION_REQUEST_TIMEOUT = Duration.minutes(2);
100112const LOOPBACK_HOSTNAMES = new Set(["127.0.0.1", "::1", "localhost"]);
101113const CLOUD_CREDENTIAL_RESPONSE_HEADERS = {
102114 "cache-control": "no-store",
......@@ -301,6 +313,33 @@ function endpointRequestPort(url: URL): number {
301313 return Number(url.port || (url.protocol === "https:" ? 443 : 80));
302314}
303315
316export function parseManagedEndpointLocalOrigin(localOrigin: string) {
317 const url = new URL(localOrigin);
318 if (
319 localOrigin !== localOrigin.trim() ||
320 (url.protocol !== "http:" && url.protocol !== "https:") ||
321 url.username !== "" ||
322 url.password !== "" ||
323 url.pathname !== "/" ||
324 url.search !== "" ||
325 url.hash !== "" ||
326 localOrigin.includes("?") ||
327 localOrigin.includes("#")
328 ) {
329 throw new Error("Invalid local origin");
330 }
331 const wsUrl = new URL(url.origin);
332 wsUrl.protocol = url.protocol === "https:" ? "wss:" : "ws:";
333 return {
334 httpBaseUrl: url.origin,
335 wsBaseUrl: wsUrl.origin,
336 origin: {
337 localHttpHost: url.hostname,
338 localHttpPort: endpointRequestPort(url),
339 } satisfies RelayManagedEndpointOrigin,
340 };
341}
342
304343function isAllowedEndpointOrigin(input: {
305344 readonly origin: RelayManagedEndpointOrigin;
306345 readonly requestUrl: string;
......@@ -455,55 +494,249 @@ const cloudLinkProofHandler = Effect.fn("environment.cloud.linkProof")(
455494 ),
456495);
457496
458const applyCloudRelayConfig = Effect.fn("environment.cloud.applyRelayConfig")(function* (
497function managedEndpointRuntimeConfigsMatch(
498 left: RelayManagedEndpointRuntimeConfig,
499 right: RelayManagedEndpointRuntimeConfig,
500): boolean {
501 return (
502 left.providerKind === right.providerKind &&
503 left.connectorToken === right.connectorToken &&
504 left.tunnelId === right.tunnelId &&
505 left.tunnelName === right.tunnelName
506 );
507}
508
509const activateManagedTunnel = Effect.fn("environment.cloud.activateManagedTunnel")(function* (
459510 dependencies: CloudHttpDependencies,
460 payload: RelayEnvironmentConfigRequest,
511 input: {
512 readonly config: RelayManagedEndpointRuntimeConfig;
513 readonly configJson: string;
514 readonly origin: RelayManagedEndpointOrigin;
515 },
461516) {
462 yield* validateRelayConfigPayload(payload);
463 yield* validateLinkedCloudUser({
464 secrets: dependencies.secrets,
465 cloudUserId: payload.cloudUserId,
517 return yield* dependencies.endpointRuntime.withLinkStateLock(
518 Effect.gen(function* () {
519 const currentConfig = yield* dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG);
520 if (Option.isNone(currentConfig) || bytesToString(currentConfig.value) !== input.configJson) {
521 return null;
522 }
523 const status = yield* dependencies.endpointRuntime.applyConfig(input.config);
524 if (status.status !== "running") {
525 return yield* new EnvironmentCloudEndpointUnavailableError({
526 message: "Managed endpoint runtime could not be started.",
527 endpointRuntimeStatus: status,
528 });
529 }
530 const marker = yield* encodeConfirmedOriginJson({
531 config: input.config,
532 origin: input.origin,
533 });
534 yield* dependencies.secrets.set(CLOUD_ENDPOINT_CONFIRMED_ORIGIN, stringToBytes(marker));
535 return status;
536 }),
537 );
538});
539
540const activateManagedTunnelWithRetry = (
541 dependencies: CloudHttpDependencies,
542 input: {
543 readonly config: RelayManagedEndpointRuntimeConfig;
544 readonly configJson: string;
545 readonly origin: RelayManagedEndpointOrigin;
546 },
547 retryRuntimeFailures: boolean,
548) => {
549 const activate = activateManagedTunnel(dependencies, input);
550 return retryRuntimeFailures
551 ? activate.pipe(
552 Effect.retry({
553 while: (error) =>
554 error._tag === "EnvironmentCloudEndpointUnavailableError" &&
555 ManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus(
556 error.endpointRuntimeStatus,
557 ),
558 schedule: Schedule.exponential("1 second").pipe(
559 Schedule.modifyDelay(({ duration }) =>
560 Effect.succeed(Duration.min(duration, Duration.seconds(30))),
561 ),
562 Schedule.jittered,
563 ),
564 }),
565 )
566 : activate;
567};
568
569export const startManagedCloudTunnelIfOriginConfirmed = Effect.fn(
570 "environment.cloud.startManagedCloudTunnelIfOriginConfirmed",
571)(function* (localOrigin: string, options?: { readonly requireConfirmedOrigin?: boolean }) {
572 const dependencies = yield* cloudHttpDependencies;
573 const requireConfirmedOrigin = options?.requireConfirmedOrigin ?? true;
574 const parsedOrigin = yield* Effect.try({
575 try: () => parseManagedEndpointLocalOrigin(localOrigin),
576 catch: () =>
577 new EnvironmentHttpBadRequestError({
578 message: "Could not resolve local environment origin.",
579 }),
466580 });
467 yield* validateCloudMintPublicKey(payload.cloudMintPublicKey);
468 const endpointRuntimeStatus = yield* dependencies.endpointRuntime.applyConfig(
469 payload.endpointRuntime,
581 return yield* dependencies.endpointRuntime.withLinkStateLock(
582 Effect.gen(function* () {
583 const [runtimeBytes, markerBytes] = yield* Effect.all([
584 dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG),
585 dependencies.secrets.get(CLOUD_ENDPOINT_CONFIRMED_ORIGIN),
586 ]);
587 if (Option.isNone(runtimeBytes)) return false;
588 const config = Option.getOrNull(decodeRuntimeConfig(bytesToString(runtimeBytes.value)));
589 if (config === null || config.providerKind !== "cloudflare_tunnel") return false;
590 // With the marker required, only a config the relay already confirmed on
591 // this port may start. Without it, startup is falling back after the
592 // relay stayed unreachable: an unconfirmed origin may send traffic to a
593 // stale port, but that beats no remote access at all.
594 if (requireConfirmedOrigin) {
595 if (Option.isNone(markerBytes)) return false;
596 const marker = Option.getOrNull(decodeConfirmedOrigin(bytesToString(markerBytes.value)));
597 if (
598 marker === null ||
599 !managedEndpointRuntimeConfigsMatch(marker.config, config) ||
600 marker.origin.localHttpHost !== parsedOrigin.origin.localHttpHost ||
601 marker.origin.localHttpPort !== parsedOrigin.origin.localHttpPort
602 ) {
603 return false;
604 }
605 }
606 const status = yield* dependencies.endpointRuntime.applyConfig(config);
607 if (status.status !== "running") {
608 return yield* new EnvironmentCloudEndpointUnavailableError({
609 message: "Managed endpoint runtime could not be started.",
610 endpointRuntimeStatus: status,
611 });
612 }
613 return true;
614 }),
470615 );
471 const ok =
472 endpointRuntimeStatus.status === "disabled" || endpointRuntimeStatus.status === "running";
473 if (!ok) {
474 return yield* new EnvironmentCloudEndpointUnavailableError({
475 message: "Managed endpoint runtime could not be started.",
476 endpointRuntimeStatus,
616});
617
618const applyCloudRelayConfig = Effect.fn("environment.cloud.applyRelayConfig")(function* (
619 dependencies: CloudHttpDependencies,
620 payload: RelayEnvironmentConfigRequest,
621 options?: {
622 readonly lockHeld?: boolean;
623 readonly confirmedOrigin?: RelayManagedEndpointOrigin;
624 },
625) {
626 const apply = Effect.gen(function* () {
627 yield* validateRelayConfigPayload(payload);
628 yield* validateLinkedCloudUser({
629 secrets: dependencies.secrets,
630 cloudUserId: payload.cloudUserId,
477631 });
478 }
632 yield* validateCloudMintPublicKey(payload.cloudMintPublicKey);
633 // Reject unsupported runtimes before touching the connector so a bad
634 // payload cannot stop a healthy tunnel on its way to a 503.
635 if (
636 payload.endpointRuntime !== null &&
637 payload.endpointRuntime.providerKind !== "cloudflare_tunnel"
638 ) {
639 return yield* new EnvironmentCloudEndpointUnavailableError({
640 message: "Managed endpoint runtime could not be started.",
641 endpointRuntimeStatus: {
642 status: "unsupported",
643 providerKind: payload.endpointRuntime.providerKind,
644 },
645 });
646 }
647 yield* dependencies.endpointRuntime.applyConfig(null);
648 yield* dependencies.secrets.remove(CLOUD_ENDPOINT_CONFIRMED_ORIGIN);
479649
480 yield* dependencies.secrets.set(RELAY_URL_SECRET, stringToBytes(payload.relayUrl));
481 yield* dependencies.secrets.set(
482 RELAY_ISSUER_SECRET,
483 stringToBytes(payload.relayIssuer ?? payload.relayUrl),
484 );
485 yield* dependencies.secrets.set(CLOUD_LINKED_USER_ID, stringToBytes(payload.cloudUserId));
486 yield* dependencies.secrets.set(
487 RELAY_ENVIRONMENT_CREDENTIAL_SECRET,
488 stringToBytes(payload.environmentCredential),
489 );
490 yield* dependencies.secrets.set(CLOUD_MINT_PUBLIC_KEY, stringToBytes(payload.cloudMintPublicKey));
491 if (payload.endpointRuntime) {
492 const endpointRuntimeJson = yield* encodeEndpointRuntimeConfigJson(payload.endpointRuntime);
650 yield* dependencies.secrets.set(RELAY_URL_SECRET, stringToBytes(payload.relayUrl));
493651 yield* dependencies.secrets.set(
494 CLOUD_ENDPOINT_RUNTIME_CONFIG,
495 stringToBytes(endpointRuntimeJson),
652 RELAY_ISSUER_SECRET,
653 stringToBytes(payload.relayIssuer ?? payload.relayUrl),
496654 );
497 } else {
498 yield* dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG);
499 }
500 return { ok, endpointRuntimeStatus } satisfies EnvironmentCloudRelayConfigResult;
655 yield* dependencies.secrets.set(CLOUD_LINKED_USER_ID, stringToBytes(payload.cloudUserId));
656 yield* dependencies.secrets.set(
657 RELAY_ENVIRONMENT_CREDENTIAL_SECRET,
658 stringToBytes(payload.environmentCredential),
659 );
660 yield* dependencies.secrets.set(
661 CLOUD_MINT_PUBLIC_KEY,
662 stringToBytes(payload.cloudMintPublicKey),
663 );
664 if (payload.endpointRuntime) {
665 const endpointRuntimeJson = yield* encodeEndpointRuntimeConfigJson(payload.endpointRuntime);
666 yield* dependencies.secrets.set(
667 CLOUD_ENDPOINT_RUNTIME_CONFIG,
668 stringToBytes(endpointRuntimeJson),
669 );
670 } else {
671 yield* dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG);
672 }
673 if (payload.endpointRuntime === null || options?.confirmedOrigin === undefined) {
674 return {
675 ok: true,
676 endpointRuntimeStatus: { status: "disabled" },
677 } satisfies EnvironmentCloudRelayConfigResult;
678 }
679 const endpointRuntimeStatus = yield* dependencies.endpointRuntime.applyConfig(
680 payload.endpointRuntime,
681 );
682 if (endpointRuntimeStatus.status !== "running") {
683 return yield* new EnvironmentCloudEndpointUnavailableError({
684 message: "Managed endpoint runtime could not be started.",
685 endpointRuntimeStatus,
686 });
687 }
688 const marker = yield* encodeConfirmedOriginJson({
689 config: payload.endpointRuntime,
690 origin: options.confirmedOrigin,
691 });
692 yield* dependencies.secrets.set(CLOUD_ENDPOINT_CONFIRMED_ORIGIN, stringToBytes(marker));
693 return { ok: true, endpointRuntimeStatus } satisfies EnvironmentCloudRelayConfigResult;
694 });
695 return yield* options?.lockHeld ? apply : dependencies.endpointRuntime.withLinkStateLock(apply);
501696});
502697
503698const cloudRelayConfigHandler = Effect.fn("environment.cloud.relayConfig")(
504699 function* (dependencies: CloudHttpDependencies, payload: RelayEnvironmentConfigRequest) {
505700 yield* requireEnvironmentScope(AuthRelayWriteScope);
506 return yield* applyCloudRelayConfig(dependencies, payload);
701 const result = yield* applyCloudRelayConfig(dependencies, payload);
702 if (payload.endpointRuntime?.providerKind === "cloudflare_tunnel") {
703 const server = yield* HttpServer.HttpServer;
704 const address = server.address;
705 if (typeof address === "string" || !("port" in address)) {
706 return yield* new EnvironmentHttpInternalServerError({
707 message: "Could not resolve the local server origin.",
708 });
709 }
710 const registration = yield* registerManagedCloudTunnelRecovery(
711 `http://127.0.0.1:${address.port}`,
712 ).pipe(
713 Effect.retry({
714 times: 2,
715 while: (error) =>
716 shouldRetryCloudLink(error) &&
717 error._tag !== "EnvironmentCloudEndpointUnavailableError",
718 }),
719 );
720 if (registration.status === "superseded") {
721 return yield* new EnvironmentHttpConflictError({
722 message: "The managed tunnel configuration changed during registration.",
723 });
724 }
725 if (registration.status === "recovery_required") {
726 yield* dependencies.endpointRuntime.requestRecovery(registration.config);
727 }
728 if (registration.status !== "ready") {
729 return yield* new EnvironmentCloudEndpointUnavailableError({
730 message: "Managed endpoint origin could not be confirmed.",
731 endpointRuntimeStatus: { status: "disabled" },
732 });
733 }
734 return {
735 ok: true,
736 endpointRuntimeStatus: registration.endpointRuntimeStatus,
737 } satisfies EnvironmentCloudRelayConfigResult;
738 }
739 return result;
507740 },
508741 Effect.catchIf(EnvironmentAuth.isServerAuthInternalError, (error) =>
509742 failEnvironmentCloudInternalError(error.message)(error),
......@@ -512,10 +745,14 @@ const cloudRelayConfigHandler = Effect.fn("environment.cloud.relayConfig")(
512745 ServerSecretStore.isSecretStoreError,
513746 failEnvironmentCloudInternalError("Could not persist environment relay configuration."),
514747 ),
515 Effect.catchTag(
516 "SchemaError",
517 failEnvironmentCloudInternalError("Could not persist environment relay configuration."),
518 ),
748 Effect.catchTags({
749 SchemaError: failEnvironmentCloudInternalError(
750 "Could not persist environment relay configuration.",
751 ),
752 PlatformError: failEnvironmentCloudInternalError(
753 "Could not register the managed endpoint origin.",
754 ),
755 }),
519756);
520757
521758const relayClientRequest = <A>(
......@@ -525,6 +762,7 @@ const relayClientRequest = <A>(
525762 readonly token: string;
526763 readonly payload: unknown;
527764 readonly schema: Schema.Decoder<A>;
765 readonly timeout?: Duration.Input;
528766 },
529767) =>
530768 HttpClientRequest.post(input.url).pipe(
......@@ -533,25 +771,20 @@ const relayClientRequest = <A>(
533771 Effect.flatMap(dependencies.httpClient.execute),
534772 Effect.flatMap(filterRelayResponse),
535773 Effect.flatMap(HttpClientResponse.schemaBodyJson(input.schema)),
774 Effect.timeout(input.timeout ?? "10 seconds"),
536775 Effect.mapError(relayRequestError),
537776 withRelayClientTracing,
538777 );
539778
540779const reconcileDesiredCloudLinkWith = Effect.fn("environment.cloud.reconcileDesiredLinkWith")(
541780 function* (dependencies: CloudHttpDependencies, localOrigin: string) {
542 const localUrl = yield* Effect.try({
543 try: () => new URL(localOrigin),
781 const parsedOrigin = yield* Effect.try({
782 try: () => parseManagedEndpointLocalOrigin(localOrigin),
544783 catch: () =>
545784 new EnvironmentHttpBadRequestError({
546785 message: "Could not resolve local environment origin.",
547786 }),
548787 });
549 if (localUrl.origin !== localOrigin) {
550 return yield* new EnvironmentHttpBadRequestError({
551 message: "Could not resolve local environment origin.",
552 });
553 }
554 const localWsOrigin = localOrigin.replace(/^http/u, "ws");
555788 const token = yield* dependencies.cliTokenManager.getExisting.pipe(
556789 Effect.flatMap(
557790 Option.match({
......@@ -584,16 +817,13 @@ const reconcileDesiredCloudLinkWith = Effect.fn("environment.cloud.reconcileDesi
584817 challenge: challenge.challenge,
585818 relayIssuer: relayUrl,
586819 endpoint: {
587 httpBaseUrl: localOrigin,
588 wsBaseUrl: localWsOrigin,
820 httpBaseUrl: parsedOrigin.httpBaseUrl,
821 wsBaseUrl: parsedOrigin.wsBaseUrl,
589822 providerKind: managedTunnelsEnabled ? "cloudflare_tunnel" : "manual",
590823 },
591 origin: {
592 localHttpHost: localUrl.hostname,
593 localHttpPort: endpointRequestPort(localUrl),
594 },
824 origin: parsedOrigin.origin,
595825 },
596 localOrigin,
826 parsedOrigin.httpBaseUrl,
597827 );
598828 const link = yield* relayClientRequest(dependencies, {
599829 url: `${relayUrl}/v1/client/environment-links`,
......@@ -605,16 +835,27 @@ const reconcileDesiredCloudLinkWith = Effect.fn("environment.cloud.reconcileDesi
605835 managedTunnelsEnabled,
606836 },
607837 schema: RelayEnvironmentLinkResponse,
838 timeout: MANAGED_ENDPOINT_PROVISION_REQUEST_TIMEOUT,
608839 });
609840 yield* setCliDesiredCloudLink(true, mode);
610 return yield* applyCloudRelayConfig(dependencies, {
611 relayUrl,
612 relayIssuer: link.relayIssuer,
613 cloudUserId: link.cloudUserId,
614 environmentCredential: link.environmentCredential,
615 cloudMintPublicKey: link.cloudMintPublicKey,
616 endpointRuntime: link.endpointRuntime,
617 });
841 yield* applyCloudRelayConfig(
842 dependencies,
843 {
844 relayUrl,
845 relayIssuer: link.relayIssuer,
846 cloudUserId: link.cloudUserId,
847 environmentCredential: link.environmentCredential,
848 cloudMintPublicKey: link.cloudMintPublicKey,
849 endpointRuntime: link.endpointRuntime,
850 },
851 {
852 lockHeld: true,
853 confirmedOrigin: parsedOrigin.origin,
854 },
855 );
856 // Callers decide on managed tunnel recovery from the mode this link
857 // actually used, not from a value read before the relay round trip.
858 return mode;
618859 },
619860 Effect.catchIf(
620861 ServerSecretStore.isSecretStoreError,
......@@ -631,7 +872,260 @@ const reconcileDesiredCloudLinkWith = Effect.fn("environment.cloud.reconcileDesi
631872
632873export const reconcileDesiredCloudLink = Effect.fn("environment.cloud.reconcileDesiredLink")(
633874 function* (localOrigin: string) {
634 return yield* reconcileDesiredCloudLinkWith(yield* cloudHttpDependencies, localOrigin);
875 const dependencies = yield* cloudHttpDependencies;
876 return yield* dependencies.endpointRuntime.withLinkStateLock(
877 reconcileDesiredCloudLinkWith(dependencies, localOrigin),
878 );
879 },
880);
881
882export const reconcileDesiredCloudLinkIfStillDesired = Effect.fn(
883 "environment.cloud.reconcileDesiredLinkIfStillDesired",
884)(function* (localOrigin: string) {
885 const dependencies = yield* cloudHttpDependencies;
886 return yield* dependencies.endpointRuntime.withLinkStateLock(
887 Effect.gen(function* () {
888 if (!(yield* readCliDesiredCloudLink)) {
889 return null;
890 }
891 return yield* reconcileDesiredCloudLinkWith(dependencies, localOrigin);
892 }),
893 );
894});
895
896type ManagedTunnelRecoveryProofInput = {
897 readonly environmentId: RelayManagedEndpointRecoveryProofPayload["environmentId"];
898 readonly cloudUserId: string;
899 readonly relayUrl: string;
900} & (
901 | {
902 readonly action: "register";
903 readonly tunnelId: string;
904 readonly origin: RelayManagedEndpointOrigin;
905 }
906 | { readonly action: "recover"; readonly origin: RelayManagedEndpointOrigin }
907);
908
909const makeManagedTunnelRecoveryProof = Effect.fn(
910 "environment.cloud.makeManagedTunnelRecoveryProof",
911)(function* (dependencies: CloudHttpDependencies, input: ManagedTunnelRecoveryProofInput) {
912 const keyPair = yield* getOrCreateEnvironmentKeyPairFromSecretStore(dependencies.secrets);
913 const configuredIssuer = yield* dependencies.secrets.get(RELAY_ISSUER_SECRET);
914 const now = yield* DateTime.now;
915 const issuedAt = Math.floor(now.epochMilliseconds / 1_000);
916 const claims = {
917 iss: `t3-env:${input.environmentId}`,
918 aud: normalizeRelayIssuer(
919 Option.isSome(configuredIssuer) ? bytesToString(configuredIssuer.value) : input.relayUrl,
920 ),
921 sub: input.environmentId,
922 jti: yield* Crypto.Crypto.pipe(Effect.flatMap((crypto) => crypto.randomUUIDv4)),
923 iat: issuedAt,
924 exp: issuedAt + 60,
925 environmentId: input.environmentId,
926 cloudUserId: input.cloudUserId,
927 };
928 const payload =
929 input.action === "register"
930 ? {
931 ...claims,
932 action: "register" as const,
933 tunnelId: input.tunnelId,
934 origin: input.origin,
935 }
936 : { ...claims, action: "recover" as const, origin: input.origin };
937
938 return yield* signRelayJwt({
939 privateKey: keyPair.privateKey,
940 typ: RELAY_MANAGED_TUNNEL_RECOVERY_TYP,
941 payload,
942 }).pipe(
943 Effect.mapError(
944 () =>
945 new EnvironmentHttpInternalServerError({
946 message: "Could not sign the managed tunnel recovery request.",
947 }),
948 ),
949 );
950});
951
952export const registerManagedCloudTunnelRecovery = Effect.fn(
953 "environment.cloud.registerManagedCloudTunnelRecovery",
954)(function* (localOrigin: string, options?: { readonly retryRuntimeFailures?: boolean }) {
955 const dependencies = yield* cloudHttpDependencies;
956 const [runtimeConfig, relayUrl, cloudUserId, environmentCredential] = yield* Effect.all([
957 dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG),
958 dependencies.secrets.get(RELAY_URL_SECRET),
959 dependencies.secrets.get(CLOUD_LINKED_USER_ID),
960 dependencies.secrets.get(RELAY_ENVIRONMENT_CREDENTIAL_SECRET),
961 ]);
962 if (
963 Option.isNone(runtimeConfig) ||
964 Option.isNone(relayUrl) ||
965 Option.isNone(cloudUserId) ||
966 Option.isNone(environmentCredential)
967 ) {
968 return { status: "not_linked" as const };
969 }
970
971 const config = Option.getOrNull(decodeRuntimeConfig(bytesToString(runtimeConfig.value)));
972 if (config?.providerKind !== "cloudflare_tunnel") {
973 return { status: "not_linked" as const };
974 }
975
976 const parsedOrigin = yield* Effect.try({
977 try: () => parseManagedEndpointLocalOrigin(localOrigin),
978 catch: () =>
979 new EnvironmentHttpBadRequestError({
980 message: "Could not resolve local environment origin.",
981 }),
982 });
983 if (config.tunnelId === undefined) {
984 return { status: "recovery_required" as const, config };
985 }
986 const origin = parsedOrigin.origin;
987 const environmentId = yield* dependencies.environment.getEnvironmentId;
988 const relayUrlValue = bytesToString(relayUrl.value);
989 const cloudUserIdValue = bytesToString(cloudUserId.value);
990 const proof = yield* makeManagedTunnelRecoveryProof(dependencies, {
991 action: "register",
992 environmentId,
993 cloudUserId: cloudUserIdValue,
994 relayUrl: relayUrlValue,
995 tunnelId: config.tunnelId,
996 origin,
997 });
998 const registered = yield* relayClientRequest(dependencies, {
999 url: `${relayUrlValue}/v1/environments/${encodeURIComponent(environmentId)}/tunnel/recovery`,
1000 token: bytesToString(environmentCredential.value),
1001 payload: {
1002 cloudUserId: cloudUserIdValue,
1003 tunnelId: config.tunnelId,
1004 origin,
1005 proof,
1006 },
1007 schema: RelayManagedEndpointRecoveryRegistrationResponse,
1008 });
1009 if (registered.status === "recovery_required") {
1010 return { status: registered.status, config };
1011 }
1012 const endpointRuntimeStatus = yield* activateManagedTunnelWithRetry(
1013 dependencies,
1014 {
1015 config,
1016 configJson: bytesToString(runtimeConfig.value),
1017 origin,
1018 },
1019 options?.retryRuntimeFailures === true,
1020 );
1021 return endpointRuntimeStatus === null
1022 ? { status: "superseded" as const }
1023 : { status: "ready" as const, endpointRuntimeStatus };
1024});
1025
1026export const recoverManagedCloudTunnel = Effect.fn("environment.cloud.recoverManagedCloudTunnel")(
1027 function* (
1028 localOrigin: string,
1029 expectedConfig?: RelayManagedEndpointRuntimeConfig,
1030 options?: { readonly retryRuntimeFailures?: boolean },
1031 ) {
1032 const dependencies = yield* cloudHttpDependencies;
1033 const [runtimeConfig, relayUrl, cloudUserId, environmentCredential] = yield* Effect.all([
1034 dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG),
1035 dependencies.secrets.get(RELAY_URL_SECRET),
1036 dependencies.secrets.get(CLOUD_LINKED_USER_ID),
1037 dependencies.secrets.get(RELAY_ENVIRONMENT_CREDENTIAL_SECRET),
1038 ]);
1039 if (
1040 Option.isNone(runtimeConfig) ||
1041 Option.isNone(relayUrl) ||
1042 Option.isNone(cloudUserId) ||
1043 Option.isNone(environmentCredential)
1044 ) {
1045 return false;
1046 }
1047 if (expectedConfig !== undefined) {
1048 const current = Option.getOrNull(decodeRuntimeConfig(bytesToString(runtimeConfig.value)));
1049 if (
1050 current === null ||
1051 current.providerKind !== expectedConfig.providerKind ||
1052 current.connectorToken !== expectedConfig.connectorToken ||
1053 current.tunnelId !== expectedConfig.tunnelId ||
1054 current.tunnelName !== expectedConfig.tunnelName
1055 ) {
1056 return false;
1057 }
1058 }
1059
1060 const parsedOrigin = yield* Effect.try({
1061 try: () => parseManagedEndpointLocalOrigin(localOrigin),
1062 catch: () =>
1063 new EnvironmentHttpBadRequestError({
1064 message: "Could not resolve local environment origin.",
1065 }),
1066 });
1067
1068 const environmentId = yield* dependencies.environment.getEnvironmentId;
1069 const relayUrlValue = bytesToString(relayUrl.value);
1070 const cloudUserIdValue = bytesToString(cloudUserId.value);
1071 const origin = parsedOrigin.origin;
1072 const proof = yield* makeManagedTunnelRecoveryProof(dependencies, {
1073 action: "recover",
1074 environmentId,
1075 cloudUserId: cloudUserIdValue,
1076 relayUrl: relayUrlValue,
1077 origin,
1078 });
1079 const recovered = yield* relayClientRequest(dependencies, {
1080 url: `${relayUrlValue}/v1/environments/${encodeURIComponent(environmentId)}/tunnel`,
1081 token: bytesToString(environmentCredential.value),
1082 payload: {
1083 cloudUserId: cloudUserIdValue,
1084 origin,
1085 proof,
1086 },
1087 schema: RelayManagedEndpointRecoveryResponse,
1088 timeout: MANAGED_ENDPOINT_PROVISION_REQUEST_TIMEOUT,
1089 });
1090 if (recovered.endpointRuntime.providerKind !== "cloudflare_tunnel") {
1091 return yield* new EnvironmentHttpInternalServerError({
1092 message: "T3 Connect returned an unsupported managed tunnel configuration.",
1093 });
1094 }
1095
1096 const encoded = yield* encodeEndpointRuntimeConfigJson(recovered.endpointRuntime).pipe(
1097 Effect.mapError(
1098 () =>
1099 new EnvironmentHttpInternalServerError({
1100 message: "Could not persist the recovered managed tunnel configuration.",
1101 }),
1102 ),
1103 );
1104 const stored = yield* dependencies.endpointRuntime.withLinkStateLock(
1105 Effect.gen(function* () {
1106 const currentConfig = yield* dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG);
1107 if (
1108 Option.isNone(currentConfig) ||
1109 bytesToString(currentConfig.value) !== bytesToString(runtimeConfig.value)
1110 ) {
1111 return false;
1112 }
1113 yield* dependencies.secrets.set(CLOUD_ENDPOINT_RUNTIME_CONFIG, stringToBytes(encoded));
1114 yield* dependencies.secrets.remove(CLOUD_ENDPOINT_CONFIRMED_ORIGIN);
1115 return true;
1116 }),
1117 );
1118 if (!stored) return false;
1119 const status = yield* activateManagedTunnelWithRetry(
1120 dependencies,
1121 {
1122 config: recovered.endpointRuntime,
1123 configJson: encoded,
1124 origin,
1125 },
1126 options?.retryRuntimeFailures === true,
1127 );
1128 return status !== null;
6351129 },
6361130);
6371131
......@@ -704,11 +1198,10 @@ export const releaseManagedTunnelOnShutdown = Effect.fn(
7041198 if (Option.isNone(runtimeConfig)) {
7051199 return false;
7061200 }
707 // Only CLI-desired managed links release on shutdown, because the startup
708 // reconcile that provisions the replacement tunnel only runs for them. A
709 // link installed by a web/mobile client comes back after a restart by
710 // reapplying the stored connector token — it has no boot-time re-provision
711 // path — so its tunnel must survive the restart. (Unlink still deletes it.)
1201 // Only CLI-desired managed links release eagerly because this request uses
1202 // CLI authorization. Web/mobile links register startup recovery with their
1203 // environment credential, and the relay reaper removes them after they are
1204 // down for the configured grace period. Unlink still deletes either kind.
7121205 if (!(yield* readCliDesiredCloudLink) || (yield* readCliDesiredLinkMode) !== "managed") {
7131206 return false;
7141207 }
......@@ -765,6 +1258,7 @@ export const releaseManagedTunnelOnShutdown = Effect.fn(
7651258 bytesToString(storedConfig.value) === bytesToString(runtimeConfig.value)
7661259 ) {
7671260 yield* dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG);
1261 yield* dependencies.secrets.remove(CLOUD_ENDPOINT_CONFIRMED_ORIGIN);
7681262 }
7691263 return true;
7701264});
......@@ -811,21 +1305,26 @@ const cloudLinkStateHandler = Effect.fn("environment.cloud.linkState")(
8111305const cloudUnlinkHandler = Effect.fn("environment.cloud.unlink")(
8121306 function* (dependencies: CloudHttpDependencies) {
8131307 yield* requireEnvironmentScope(AuthRelayWriteScope);
814 const endpointRuntimeStatus = yield* dependencies.endpointRuntime.applyConfig(null);
815 yield* Effect.all(
816 [
817 dependencies.secrets.remove(CLOUD_LINKED_USER_ID),
818 dependencies.secrets.remove(RELAY_URL_SECRET),
819 dependencies.secrets.remove(RELAY_ISSUER_SECRET),
820 dependencies.secrets.remove(RELAY_ENVIRONMENT_CREDENTIAL_SECRET),
821 dependencies.secrets.remove(CLOUD_MINT_PUBLIC_KEY),
822 dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG),
823 dependencies.secrets.remove(PUBLISH_AGENT_ACTIVITY_SECRET),
824 ],
825 { concurrency: 7 },
1308 return yield* dependencies.endpointRuntime.withLinkStateLock(
1309 Effect.gen(function* () {
1310 const endpointRuntimeStatus = yield* dependencies.endpointRuntime.applyConfig(null);
1311 yield* Effect.all(
1312 [
1313 dependencies.secrets.remove(CLOUD_LINKED_USER_ID),
1314 dependencies.secrets.remove(RELAY_URL_SECRET),
1315 dependencies.secrets.remove(RELAY_ISSUER_SECRET),
1316 dependencies.secrets.remove(RELAY_ENVIRONMENT_CREDENTIAL_SECRET),
1317 dependencies.secrets.remove(CLOUD_MINT_PUBLIC_KEY),
1318 dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG),
1319 dependencies.secrets.remove(CLOUD_ENDPOINT_CONFIRMED_ORIGIN),
1320 dependencies.secrets.remove(PUBLISH_AGENT_ACTIVITY_SECRET),
1321 ],
1322 { concurrency: 8 },
1323 );
1324 yield* setCliDesiredCloudLink(false);
1325 return { ok: true, endpointRuntimeStatus } satisfies EnvironmentCloudRelayConfigResult;
1326 }),
8261327 );
827 yield* setCliDesiredCloudLink(false);
828 return { ok: true, endpointRuntimeStatus } satisfies EnvironmentCloudRelayConfigResult;
8291328 },
8301329 Effect.catchIf(
8311330 ServerSecretStore.isSecretStoreError,
apps/server/src/cloud/managedTunnelStartup.test.ts created+132
......@@ -0,0 +1,132 @@
1import { describe, expect, it } from "@effect/vitest";
2import * as Deferred from "effect/Deferred";
3import * as Effect from "effect/Effect";
4import * as Fiber from "effect/Fiber";
5import * as TestClock from "effect/testing/TestClock";
6
7import {
8 managedTunnelStartupAction,
9 retryManagedTunnelRegistration,
10} from "./managedTunnelStartup.ts";
11
12describe("managedTunnelStartupAction", () => {
13 const config = {
14 providerKind: "cloudflare_tunnel" as const,
15 connectorToken: "connector-token",
16 tunnelId: "tunnel-1",
17 };
18
19 it("requests tunnel recovery only when the relay proves it is needed", () => {
20 expect(
21 managedTunnelStartupAction({
22 wantsCliLink: true,
23 registration: { status: "recovery_required", config },
24 }),
25 ).toEqual({ action: "request_recovery", config });
26 });
27
28 it("creates a desired CLI link only when no local managed link exists", () => {
29 expect(
30 managedTunnelStartupAction({
31 wantsCliLink: true,
32 registration: { status: "not_linked" },
33 }),
34 ).toEqual({ action: "reconcile_link" });
35 });
36
37 it.each(["ready", "unavailable"] as const)(
38 "does not provision after a %s registration result",
39 (status) => {
40 expect(
41 managedTunnelStartupAction({
42 wantsCliLink: true,
43 registration: { status },
44 }),
45 ).toEqual({ action: "none" });
46 },
47 );
48});
49
50describe("retryManagedTunnelRegistration", () => {
51 it.effect("does not fall back or retry when registration is permanently rejected", () =>
52 Effect.gen(function* () {
53 let attempts = 0;
54 let fallbacks = 0;
55 const error = yield* Effect.flip(
56 retryManagedTunnelRegistration(
57 Effect.suspend(() => {
58 attempts += 1;
59 return Effect.fail("not authorized");
60 }),
61 () => false,
62 Effect.sync(() => {
63 fallbacks += 1;
64 }),
65 ),
66 );
67 expect(error).toBe("not authorized");
68 expect(attempts).toBe(1);
69 expect(fallbacks).toBe(0);
70 }),
71 );
72
73 it.effect("stops retrying after the retry window so startup can fall back", () =>
74 Effect.gen(function* () {
75 let attempts = 0;
76 const registration = Effect.suspend(() => {
77 attempts += 1;
78 return Effect.fail("relay unavailable" as const);
79 });
80 const fiber = yield* Effect.forkChild(
81 Effect.flip(retryManagedTunnelRegistration(registration, () => true)),
82 { startImmediately: true },
83 );
84 yield* TestClock.adjust("15 minutes");
85 expect(yield* Fiber.join(fiber)).toBe("relay unavailable");
86 // Capped at 30 seconds between attempts, ten minutes allows a bounded run.
87 expect(attempts).toBeGreaterThan(5);
88 expect(attempts).toBeLessThan(60);
89 }),
90 );
91
92 it.effect("waits for successful registration before it activates the connector", () =>
93 Effect.gen(function* () {
94 const firstAttempt = yield* Deferred.make<void>();
95 let attempts = 0;
96 let activations = 0;
97 let reconciliations = 0;
98 const registration = Effect.suspend(() => {
99 attempts += 1;
100 if (attempts === 1) {
101 return Deferred.succeed(firstAttempt, undefined).pipe(
102 Effect.andThen(Effect.fail("relay unavailable" as const)),
103 );
104 }
105 return Effect.succeed({ status: "ready" as const });
106 });
107 const startup = retryManagedTunnelRegistration(registration, () => true).pipe(
108 Effect.tap((result) =>
109 Effect.sync(() => {
110 const action = managedTunnelStartupAction({ wantsCliLink: true, registration: result });
111 if (action.action === "reconcile_link") {
112 reconciliations += 1;
113 }
114 activations += 1;
115 }),
116 ),
117 );
118
119 const fiber = yield* Effect.forkChild(startup, { startImmediately: true });
120 yield* Deferred.await(firstAttempt);
121 expect(attempts).toBe(1);
122 expect(activations).toBe(0);
123
124 yield* TestClock.adjust("2 seconds");
125 yield* Fiber.join(fiber);
126
127 expect(attempts).toBe(2);
128 expect(activations).toBe(1);
129 expect(reconciliations).toBe(0);
130 }),
131 );
132});
apps/server/src/cloud/managedTunnelStartup.ts created+77
......@@ -0,0 +1,77 @@
1import type { RelayManagedEndpointRuntimeConfig } from "@t3tools/contracts/relay";
2import * as Duration from "effect/Duration";
3import * as Effect from "effect/Effect";
4import * as Schedule from "effect/Schedule";
5
6export type ManagedTunnelRegistrationResult =
7 | { readonly status: "not_linked" | "ready" | "unavailable" | "superseded" }
8 | {
9 readonly status: "recovery_required";
10 readonly config: RelayManagedEndpointRuntimeConfig;
11 };
12
13export type ManagedTunnelStartupAction =
14 | { readonly action: "none" }
15 | { readonly action: "reconcile_link" }
16 | {
17 readonly action: "request_recovery";
18 readonly config: RelayManagedEndpointRuntimeConfig;
19 };
20
21export function managedTunnelStartupAction(input: {
22 readonly wantsCliLink: boolean;
23 readonly registration: ManagedTunnelRegistrationResult;
24}): ManagedTunnelStartupAction {
25 if (input.registration.status === "recovery_required") {
26 return {
27 action: "request_recovery",
28 config: input.registration.config,
29 };
30 }
31 if (input.wantsCliLink && input.registration.status === "not_linked") {
32 return { action: "reconcile_link" };
33 }
34 return { action: "none" };
35}
36
37// After this window the host can start its stored connector config while
38// registration keeps retrying to reconcile the origin when the relay returns.
39const MANAGED_TUNNEL_REGISTRATION_RETRY_WINDOW = Duration.minutes(10);
40
41export const retryManagedTunnelRegistration = <A, E, R>(
42 registration: Effect.Effect<A, E, R>,
43 isRetryable: (error: E) => boolean,
44 onRetryWindowExhausted?: Effect.Effect<void, never, R>,
45) => {
46 const schedule = Schedule.exponential("1 second").pipe(
47 Schedule.modifyDelay(({ duration }) =>
48 Effect.succeed(Duration.min(duration, Duration.seconds(30))),
49 ),
50 Schedule.jittered,
51 );
52 return registration.pipe(
53 Effect.retry({
54 while: isRetryable,
55 schedule: schedule.pipe(
56 Schedule.upTo({ duration: MANAGED_TUNNEL_REGISTRATION_RETRY_WINDOW }),
57 ),
58 }),
59 Effect.catch((error) =>
60 onRetryWindowExhausted !== undefined && isRetryable(error)
61 ? onRetryWindowExhausted.pipe(
62 Effect.andThen(registration.pipe(Effect.retry({ while: isRetryable, schedule }))),
63 )
64 : Effect.fail(error),
65 ),
66 );
67};
68
69// A host asks the relay for a replacement tunnel at most this often. Every
70// managed host shares one relay, so a host stuck in a bad loop must not turn
71// into a fleet-wide request storm.
72export const MANAGED_TUNNEL_RECOVERY_COOLDOWN = Duration.minutes(2);
73
74// Existing hosts register on their first boot after an upgrade, and desktop
75// auto-update delivers that boot to many hosts at once. Spread the first
76// registration so the relay and Cloudflare see a ramp instead of a spike.
77export const MANAGED_TUNNEL_FIRST_REGISTRATION_JITTER = Duration.seconds(30);
apps/server/src/server.test.ts+244-33
......@@ -562,6 +562,7 @@ const buildAppUnderTest = (options?: {
562562 >;
563563 relayClient?: Partial<RelayClient.RelayClient["Service"]>;
564564 cloudCliTokenManager?: Partial<CloudCliTokenManager.CloudCliTokenManager["Service"]>;
565 httpClient?: HttpClient.HttpClient;
565566 nativeTelemetryClient?: Partial<NativeTelemetryClient.NativeTelemetryClient["Service"]>;
566567 desktopTelemetryReceiver?: Partial<
567568 DesktopTelemetryReceiver.DesktopTelemetryReceiver["Service"]
......@@ -1176,6 +1177,9 @@ const buildAppUnderTest = (options?: {
11761177 CloudManagedEndpointRuntime.CloudManagedEndpointRuntime,
11771178 CloudManagedEndpointRuntime.CloudManagedEndpointRuntime.of({
11781179 applyConfig: () => Effect.succeed({ status: "disabled" }),
1180 recoveryRequests: Stream.empty,
1181 requestRecovery: () => Effect.void,
1182 withLinkStateLock: (effect) => effect,
11791183 ...options?.layers?.cloudManagedEndpointRuntime,
11801184 }),
11811185 ),
......@@ -1224,7 +1228,11 @@ const buildAppUnderTest = (options?: {
12241228 Layer.provideMerge(makeAuthTestLayer()),
12251229 Layer.provideMerge(ServerSecretStore.layer),
12261230 Layer.provide(workspaceAndProjectServicesLayer),
1227 Layer.provideMerge(FetchHttpClient.layer),
1231 Layer.provideMerge(
1232 options?.layers?.httpClient === undefined
1233 ? FetchHttpClient.layer
1234 : Layer.succeed(HttpClient.HttpClient, options.layers.httpClient),
1235 ),
12281236 Layer.provide(GitHubCli.layer.pipe(Layer.provideMerge(VcsProcess.layer))),
12291237 Layer.provide(layerConfig),
12301238 );
......@@ -3223,6 +3231,68 @@ it.layer(NodeServices.layer)("server router seam", (it) => {
32233231 }).pipe(Effect.provide(NodeHttpServer.layerTest)),
32243232 );
32253233
3234 it.effect("rejects a non-Cloudflare managed endpoint runtime without persisting the link", () =>
3235 Effect.gen(function* () {
3236 const appliedRuntimeConfigs: Array<unknown> = [];
3237 yield* buildAppUnderTest({
3238 layers: {
3239 cloudManagedEndpointRuntime: {
3240 applyConfig: (config) =>
3241 Effect.sync(() => {
3242 appliedRuntimeConfigs.push(config);
3243 return config === null
3244 ? ({ status: "disabled" } as const)
3245 : ({ status: "unsupported", providerKind: config.providerKind } as const);
3246 }),
3247 },
3248 },
3249 });
3250
3251 const cloudKeyPair = NodeCrypto.generateKeyPairSync("ed25519", {
3252 privateKeyEncoding: { format: "pem", type: "pkcs8" },
3253 publicKeyEncoding: { format: "pem", type: "spki" },
3254 });
3255 const ownerCookie = yield* getAuthenticatedSessionCookieHeader();
3256 const relayConfigUrl = yield* getHttpServerUrl("/api/connect/relay-config");
3257 const relayConfigResponse = yield* fetchEffect(relayConfigUrl, {
3258 method: "POST",
3259 headers: {
3260 cookie: ownerCookie,
3261 "content-type": "application/json",
3262 },
3263 body: jsonRequestBody({
3264 relayUrl: "https://relay.example.test",
3265 cloudUserId: "user_123",
3266 environmentCredential: "t3env_test_credential",
3267 cloudMintPublicKey: cloudKeyPair.publicKey,
3268 endpointRuntime: {
3269 providerKind: "manual",
3270 connectorToken: "manual-token",
3271 },
3272 }),
3273 });
3274 const relayConfigBody = yield* responseJsonEffect<{
3275 readonly _tag?: string;
3276 readonly endpointRuntimeStatus?: { readonly status?: string };
3277 }>(relayConfigResponse);
3278 const linkStateUrl = yield* getHttpServerUrl("/api/connect/link-state");
3279 const linkStateResponse = yield* fetchEffect(linkStateUrl, {
3280 headers: { cookie: ownerCookie },
3281 });
3282 const linkStateBody = yield* responseJsonEffect<{ readonly linked?: boolean }>(
3283 linkStateResponse,
3284 );
3285
3286 assert.equal(relayConfigResponse.status, 503);
3287 assert.equal(relayConfigBody._tag, "EnvironmentCloudEndpointUnavailableError");
3288 assert.equal(relayConfigBody.endpointRuntimeStatus?.status, "unsupported");
3289 // The connector is never touched for a rejected runtime.
3290 assert.deepEqual(appliedRuntimeConfigs, []);
3291 assert.equal(linkStateResponse.status, 200);
3292 assert.equal(linkStateBody.linked, false);
3293 }).pipe(Effect.provide(NodeHttpServer.layerTest)),
3294 );
3295
32263296 it.effect("reports local cloud link state from persisted relay config", () =>
32273297 Effect.gen(function* () {
32283298 yield* buildAppUnderTest();
......@@ -3301,6 +3371,7 @@ it.layer(NodeServices.layer)("server router seam", (it) => {
33013371 it.effect("unlinks local cloud state and disables the managed endpoint runtime", () =>
33023372 Effect.gen(function* () {
33033373 const appliedRuntimeConfigs: Array<unknown> = [];
3374 const requestedRecoveryConfigs: Array<unknown> = [];
33043375 yield* buildAppUnderTest({
33053376 layers: {
33063377 cloudManagedEndpointRuntime: {
......@@ -3317,7 +3388,14 @@ it.layer(NodeServices.layer)("server router seam", (it) => {
33173388 ...(config.tunnelName ? { tunnelName: config.tunnelName } : {}),
33183389 });
33193390 },
3391 requestRecovery: (config) =>
3392 Effect.sync(() => {
3393 requestedRecoveryConfigs.push(config);
3394 }),
33203395 },
3396 httpClient: HttpClient.make((request) =>
3397 Effect.succeed(HttpClientResponse.fromWeb(request, Response.json({ status: "ready" }))),
3398 ),
33213399 },
33223400 });
33233401
......@@ -3383,6 +3461,7 @@ it.layer(NodeServices.layer)("server router seam", (it) => {
33833461 assert.equal(linkStateBody.relayUrl, null);
33843462 assert.equal(linkStateBody.relayIssuer, null);
33853463 assert.deepEqual(appliedRuntimeConfigs, [
3464 null,
33863465 {
33873466 providerKind: "cloudflare_tunnel",
33883467 connectorToken: "connector-token",
......@@ -3391,6 +3470,7 @@ it.layer(NodeServices.layer)("server router seam", (it) => {
33913470 },
33923471 null,
33933472 ]);
3473 assert.deepEqual(requestedRecoveryConfigs, []);
33943474 }).pipe(Effect.provide(NodeHttpServer.layerTest)),
33953475 );
33963476
......@@ -3701,19 +3781,169 @@ it.layer(NodeServices.layer)("server router seam", (it) => {
37013781 }).pipe(Effect.provide(NodeHttpServer.layerTest)),
37023782 );
37033783
3784 it.effect("keeps a managed connector stopped when relay registration fails", () =>
3785 Effect.gen(function* () {
3786 const appliedRuntimeConfigs: Array<unknown> = [];
3787 const relayRequests: Array<HttpClientRequest.HttpClientRequest> = [];
3788 yield* buildAppUnderTest({
3789 layers: {
3790 cloudManagedEndpointRuntime: {
3791 applyConfig: (config) =>
3792 Effect.sync(() => {
3793 appliedRuntimeConfigs.push(config);
3794 return config === null
3795 ? ({ status: "disabled" } as const)
3796 : ({ status: "running", providerKind: "cloudflare_tunnel", pid: 123 } as const);
3797 }),
3798 },
3799 httpClient: HttpClient.make((request) =>
3800 Effect.sync(() => {
3801 relayRequests.push(request);
3802 return HttpClientResponse.fromWeb(
3803 request,
3804 Response.json({ message: "relay unavailable" }, { status: 503 }),
3805 );
3806 }),
3807 ),
3808 },
3809 });
3810
3811 const cloudKeyPair = NodeCrypto.generateKeyPairSync("ed25519", {
3812 privateKeyEncoding: { format: "pem", type: "pkcs8" },
3813 publicKeyEncoding: { format: "pem", type: "spki" },
3814 });
3815 const ownerCookie = yield* getAuthenticatedSessionCookieHeader();
3816 const relayConfigUrl = yield* getHttpServerUrl("/api/connect/relay-config");
3817 const relayConfigResponse = yield* fetchEffect(relayConfigUrl, {
3818 method: "POST",
3819 headers: {
3820 cookie: ownerCookie,
3821 "content-type": "application/json",
3822 },
3823 body: jsonRequestBody({
3824 relayUrl: "https://relay.example.test",
3825 cloudUserId: "user_123",
3826 environmentCredential: "t3env_test_credential",
3827 cloudMintPublicKey: cloudKeyPair.publicKey,
3828 endpointRuntime: {
3829 providerKind: "cloudflare_tunnel",
3830 connectorToken: "connector-token",
3831 tunnelId: "tunnel-1",
3832 },
3833 }),
3834 });
3835 const relayConfigBody = yield* responseJsonEffect<{ readonly _tag?: string }>(
3836 relayConfigResponse,
3837 );
3838
3839 assert.equal(relayConfigResponse.status, 500);
3840 assert.equal(relayConfigBody._tag, "EnvironmentHttpInternalServerError");
3841 assert.equal(relayRequests.length, 3);
3842 assert.deepEqual(appliedRuntimeConfigs, [null]);
3843 }).pipe(Effect.provide(NodeHttpServer.layerTest)),
3844 );
3845
3846 it.effect(
3847 "queues recovery without starting a connector when relay registration requires it",
3848 () =>
3849 Effect.gen(function* () {
3850 const appliedRuntimeConfigs: Array<unknown> = [];
3851 const requestedRecoveryConfigs: Array<unknown> = [];
3852 const relayRequests: Array<HttpClientRequest.HttpClientRequest> = [];
3853 yield* buildAppUnderTest({
3854 layers: {
3855 cloudManagedEndpointRuntime: {
3856 applyConfig: (config) =>
3857 Effect.sync(() => {
3858 appliedRuntimeConfigs.push(config);
3859 return config === null
3860 ? ({ status: "disabled" } as const)
3861 : ({ status: "running", providerKind: "cloudflare_tunnel", pid: 123 } as const);
3862 }),
3863 requestRecovery: (config) =>
3864 Effect.sync(() => {
3865 requestedRecoveryConfigs.push(config);
3866 }),
3867 },
3868 httpClient: HttpClient.make((request) =>
3869 Effect.sync(() => {
3870 relayRequests.push(request);
3871 return HttpClientResponse.fromWeb(
3872 request,
3873 Response.json({ status: "recovery_required" }),
3874 );
3875 }),
3876 ),
3877 },
3878 });
3879
3880 const cloudKeyPair = NodeCrypto.generateKeyPairSync("ed25519", {
3881 privateKeyEncoding: { format: "pem", type: "pkcs8" },
3882 publicKeyEncoding: { format: "pem", type: "spki" },
3883 });
3884 const ownerCookie = yield* getAuthenticatedSessionCookieHeader();
3885 const relayConfigUrl = yield* getHttpServerUrl("/api/connect/relay-config");
3886 const relayConfigResponse = yield* fetchEffect(relayConfigUrl, {
3887 method: "POST",
3888 headers: {
3889 cookie: ownerCookie,
3890 "content-type": "application/json",
3891 },
3892 body: jsonRequestBody({
3893 relayUrl: "https://relay.example.test",
3894 cloudUserId: "user_123",
3895 environmentCredential: "t3env_test_credential",
3896 cloudMintPublicKey: cloudKeyPair.publicKey,
3897 endpointRuntime: {
3898 providerKind: "cloudflare_tunnel",
3899 connectorToken: "connector-token",
3900 tunnelId: "tunnel-1",
3901 },
3902 }),
3903 });
3904 const relayConfigBody = yield* responseJsonEffect<{
3905 readonly _tag?: string;
3906 readonly endpointRuntimeStatus?: { readonly status?: string };
3907 }>(relayConfigResponse);
3908
3909 assert.equal(relayConfigResponse.status, 503);
3910 assert.equal(relayConfigBody._tag, "EnvironmentCloudEndpointUnavailableError");
3911 assert.equal(relayConfigBody.endpointRuntimeStatus?.status, "disabled");
3912 assert.equal(relayRequests.length, 1);
3913 assert.deepEqual(appliedRuntimeConfigs, [null]);
3914 assert.deepEqual(requestedRecoveryConfigs, [
3915 {
3916 providerKind: "cloudflare_tunnel",
3917 connectorToken: "connector-token",
3918 tunnelId: "tunnel-1",
3919 },
3920 ]);
3921 }).pipe(Effect.provide(NodeHttpServer.layerTest)),
3922 );
3923
37043924 it.effect("fails relay config when the managed endpoint connector cannot start", () =>
37053925 Effect.gen(function* () {
3926 const appliedRuntimeConfigs: Array<unknown> = [];
37063927 yield* buildAppUnderTest({
37073928 layers: {
37083929 cloudManagedEndpointRuntime: {
3709 applyConfig: () =>
3710 Effect.succeed({
3711 status: "failed",
3712 providerKind: "cloudflare_tunnel",
3713 reason: "cloudflared missing",
3714 tunnelId: "tunnel-1",
3930 applyConfig: (config) =>
3931 Effect.sync(() => {
3932 appliedRuntimeConfigs.push(config);
3933 return config === null
3934 ? ({ status: "disabled" } as const)
3935 : ({
3936 status: "failed",
3937 providerKind: "cloudflare_tunnel",
3938 failure: "not-installed",
3939 reason: "cloudflared missing",
3940 tunnelId: "tunnel-1",
3941 } as const);
37153942 }),
37163943 },
3944 httpClient: HttpClient.make((request) =>
3945 Effect.succeed(HttpClientResponse.fromWeb(request, Response.json({ status: "ready" }))),
3946 ),
37173947 },
37183948 });
37193949
......@@ -3752,33 +3982,14 @@ it.layer(NodeServices.layer)("server router seam", (it) => {
37523982 assert.equal(relayConfigBody.message, "Managed endpoint runtime could not be started.");
37533983 assert.equal(relayConfigBody.endpointRuntimeStatus?.status, "failed");
37543984 assert.equal(relayConfigBody.endpointRuntimeStatus?.reason, "cloudflared missing");
3755
3756 const now = yield* DateTime.now;
3757 const healthRequest = makeCloudEnvironmentHealthRequest({
3758 privateKey: cloudKeyPair.privateKey,
3759 environmentId: testEnvironmentDescriptor.environmentId,
3760 nonce: "cloud-health-after-failed-runtime",
3761 issuedAt: DateTime.formatIso(now),
3762 expiresAt: DateTime.formatIso(DateTime.add(now, { minutes: 5 })),
3763 });
3764 const healthUrl = yield* getHttpServerUrl("/api/t3-connect/health");
3765 const healthResponse = yield* fetchEffect(healthUrl, {
3766 method: "POST",
3767 headers: {
3768 "content-type": "application/json",
3985 assert.deepEqual(appliedRuntimeConfigs, [
3986 null,
3987 {
3988 providerKind: "cloudflare_tunnel",
3989 connectorToken: "connector-token",
3990 tunnelId: "tunnel-1",
37693991 },
3770 body: jsonRequestBody(healthRequest),
3771 });
3772 const healthBody = yield* responseJsonEffect<{
3773 _tag?: string;
3774 message?: string;
3775 }>(healthResponse);
3776 assert.equal(healthResponse.status, 500);
3777 assert.equal(healthBody._tag, "EnvironmentHttpInternalServerError");
3778 assert.equal(
3779 healthBody.message,
3780 "Cloud mint public key is not installed for this environment.",
3781 );
3992 ]);
37823993 }).pipe(Effect.provide(NodeHttpServer.layerTest)),
37833994 );
37843995
apps/server/src/server.ts+179-19
......@@ -8,12 +8,16 @@ import {
88 ProviderDriverKind,
99 type RepositoryIdentity,
1010} from "@t3tools/contracts";
11import type { RelayManagedEndpointRuntimeConfig } from "@t3tools/contracts/relay";
1112import * as Cause from "effect/Cause";
13import * as Clock from "effect/Clock";
1214import * as Duration from "effect/Duration";
1315import * as Deferred from "effect/Deferred";
1416import * as Effect from "effect/Effect";
1517import * as Layer from "effect/Layer";
18import * as Random from "effect/Random";
1619import * as Schedule from "effect/Schedule";
20import * as Semaphore from "effect/Semaphore";
1721import * as Stream from "effect/Stream";
1822import { FetchHttpClient, HttpRouter, HttpServer } from "effect/unstable/http";
1923import * as HttpApiBuilder from "effect/unstable/httpapi/HttpApiBuilder";
......@@ -123,12 +127,21 @@ import * as EnvironmentAuth from "./auth/EnvironmentAuth.ts";
123127import {
124128 connectHttpApiLayer,
125129 pendingServiceUpdateExists,
126 reconcileDesiredCloudLink,
130 reconcileDesiredCloudLinkIfStillDesired,
131 recoverManagedCloudTunnel,
132 registerManagedCloudTunnelRecovery,
133 startManagedCloudTunnelIfOriginConfirmed,
127134 releaseManagedTunnelOnShutdown,
128135} from "./cloud/http.ts";
129136import { serverRelayBrokerTracingLayer } from "./cloud/relayTracing.ts";
130137import { shouldRetryCloudLink } from "./cloud/relayResponse.ts";
131138import * as CloudManagedEndpointRuntime from "./cloud/ManagedEndpointRuntime.ts";
139import {
140 MANAGED_TUNNEL_FIRST_REGISTRATION_JITTER,
141 MANAGED_TUNNEL_RECOVERY_COOLDOWN,
142 managedTunnelStartupAction,
143 retryManagedTunnelRegistration,
144} from "./cloud/managedTunnelStartup.ts";
132145import * as CloudCliTokenManager from "./cloud/CliTokenManager.ts";
133146import * as CloudCliState from "./cloud/CliState.ts";
134147import * as ServerSelfUpdate from "./cloud/selfUpdate.ts";
......@@ -708,10 +721,6 @@ const makeServerLayer = Layer.unwrap(
708721 : Layer.empty;
709722 const cloudDesiredLinkReconcileLayer = Layer.effectDiscard(
710723 Effect.gen(function* () {
711 if (!hasCloudPublicConfig) {
712 yield* Deferred.succeed(cloudLinkParked, undefined).pipe(Effect.orDie);
713 return;
714 }
715724 const releaseManagedTunnel = releaseManagedTunnelOnShutdown().pipe(
716725 Effect.timeout("10 seconds"),
717726 Effect.tap((released) =>
......@@ -740,33 +749,184 @@ const makeServerLayer = Layer.unwrap(
740749 if (!cleanupBeforeActivation) {
741750 yield* Effect.addFinalizer(() => releaseManagedTunnel);
742751 }
743 if (!(yield* CloudCliState.readCliDesiredCloudLink)) return;
744752 const server = yield* HttpServer.HttpServer;
745753 const address = server.address;
746754 if (typeof address === "string" || !("port" in address)) return;
755 const localOrigin = `http://127.0.0.1:${address.port}`;
756 const endpointRuntime = yield* CloudManagedEndpointRuntime.CloudManagedEndpointRuntime;
757 const recoveryLock = yield* Semaphore.make(1);
758 let lastRecoveryAtMillis = 0;
759 const recoverManagedTunnel = (config: RelayManagedEndpointRuntimeConfig) =>
760 recoveryLock.withPermits(1)(
761 Effect.gen(function* () {
762 const elapsed = (yield* Clock.currentTimeMillis) - lastRecoveryAtMillis;
763 const wait = Duration.toMillis(MANAGED_TUNNEL_RECOVERY_COOLDOWN) - elapsed;
764 if (wait > 0) yield* Effect.sleep(Duration.millis(wait));
765 lastRecoveryAtMillis = yield* Clock.currentTimeMillis;
766 }).pipe(
767 Effect.andThen(
768 recoverManagedCloudTunnel(localOrigin, config, {
769 retryRuntimeFailures: true,
770 }),
771 ),
772 Effect.retry({
773 while: (error) =>
774 shouldRetryCloudLink(error) &&
775 error._tag !== "EnvironmentCloudEndpointUnavailableError",
776 schedule: Schedule.exponential("1 second").pipe(
777 Schedule.modifyDelay(({ duration }) =>
778 Effect.succeed(Duration.min(duration, Duration.seconds(30))),
779 ),
780 Schedule.jittered,
781 ),
782 }),
783 Effect.tap((recovered) =>
784 recovered ? Effect.logInfo("T3 Connect managed tunnel recovered") : Effect.void,
785 ),
786 Effect.catchCause((cause) =>
787 Cause.hasInterrupts(cause)
788 ? Effect.interrupt
789 : Effect.logWarning("Failed to recover the T3 Connect managed tunnel", {
790 cause,
791 }),
792 ),
793 ),
794 );
795 yield* endpointRuntime.recoveryRequests.pipe(
796 Stream.runForEach(recoverManagedTunnel),
797 Effect.forkScoped,
798 );
747799 // No settling delay before the first attempt: routes are already
748800 // serving by the time activation opens this gate (the startup
749801 // sequence awaits routesReady), and the retry schedule below
750802 // covers anything this sleep used to hedge against. Every
751803 // millisecond here is dead time on the path to remote
752804 // reachability after a restart.
753 yield* reconcileDesiredCloudLink(`http://127.0.0.1:${address.port}`).pipe(
754 Effect.retry({
755 while: shouldRetryCloudLink,
756 schedule: Schedule.exponential("1 second").pipe(
757 Schedule.modifyDelay(({ duration }) =>
758 Effect.succeed(Duration.min(duration, Duration.seconds(30))),
805 const wantsCliLink = hasCloudPublicConfig
806 ? yield* CloudCliState.readCliDesiredCloudLink.pipe(
807 Effect.catch((cause) =>
808 Effect.logWarning("Failed to read the desired T3 Connect link", { cause }).pipe(
809 Effect.as(false),
810 ),
759811 ),
760 Schedule.upTo({ duration: "10 minutes" }),
761 ),
762 }),
763 Effect.tap(() => Effect.logInfo("T3 Connect desired link reconciled on startup")),
812 )
813 : false;
814 // A failed read must not end this fiber before it registers
815 // recovery and starts consuming recovery requests. "managed" is
816 // what a missing value means, so it is the safe fallback.
817 const desiredCliLinkMode = wantsCliLink
818 ? yield* CloudCliState.readCliDesiredLinkMode.pipe(
819 Effect.catch((cause) =>
820 Effect.logWarning("Failed to read the desired T3 Connect link mode", {
821 cause,
822 }).pipe(Effect.as("managed" as const)),
823 ),
824 )
825 : null;
826 // A publish-only link must not expose the host, even if a managed
827 // config from an earlier link is still stored.
828 const startedConfirmed =
829 desiredCliLinkMode === "publish_only"
830 ? false
831 : yield* startManagedCloudTunnelIfOriginConfirmed(localOrigin).pipe(
832 Effect.catch((cause) =>
833 Effect.logWarning("Failed to start the confirmed T3 Connect tunnel", {
834 cause,
835 }).pipe(Effect.as(false)),
836 ),
837 );
838 const startStoredManagedTunnel = startManagedCloudTunnelIfOriginConfirmed(localOrigin, {
839 requireConfirmedOrigin: false,
840 }).pipe(
841 Effect.tap((started) =>
842 started
843 ? Effect.logWarning(
844 "T3 Connect started the stored tunnel without relay confirmation",
845 )
846 : Effect.void,
847 ),
764848 Effect.catch((cause) =>
765 Effect.logWarning("Failed to reconcile T3 Connect desired link on startup", {
766 message: cause.message,
767 }),
849 Effect.logWarning("Failed to start the stored T3 Connect tunnel", { cause }),
768850 ),
851 Effect.asVoid,
769852 );
853 const registerManagedTunnel = retryManagedTunnelRegistration(
854 registerManagedCloudTunnelRecovery(localOrigin, {
855 retryRuntimeFailures: true,
856 }),
857 (error) =>
858 shouldRetryCloudLink(error) &&
859 error._tag !== "EnvironmentCloudEndpointUnavailableError",
860 startedConfirmed ? Effect.void : startStoredManagedTunnel,
861 ).pipe(
862 Effect.tap((result) =>
863 result.status === "ready"
864 ? Effect.logInfo("T3 Connect managed tunnel recovery registered")
865 : Effect.void,
866 ),
867 Effect.catchCause((cause) =>
868 Cause.hasInterrupts(cause)
869 ? Effect.interrupt
870 : Effect.logWarning("Failed to register T3 Connect managed tunnel recovery", {
871 cause,
872 }).pipe(Effect.as({ status: "unavailable" as const })),
873 ),
874 );
875 // A host without a confirmed marker is on its first boot after the
876 // upgrade. Spread those registrations so an auto-update wave does
877 // not hit the relay all at once.
878 if (!startedConfirmed && desiredCliLinkMode !== "publish_only") {
879 const jitter = yield* Random.nextIntBetween(
880 0,
881 Duration.toMillis(MANAGED_TUNNEL_FIRST_REGISTRATION_JITTER),
882 );
883 yield* Effect.sleep(Duration.millis(jitter));
884 }
885 const registration =
886 desiredCliLinkMode === "publish_only"
887 ? { status: "not_linked" as const }
888 : yield* registerManagedTunnel;
889 // A terminal registration failure also allows the stored config
890 // to start. Transient outages use the fallback above and keep
891 // registration retrying in this scoped startup fiber.
892 if (registration.status === "unavailable" && !startedConfirmed) {
893 yield* startStoredManagedTunnel;
894 }
895 const startupAction = managedTunnelStartupAction({ wantsCliLink, registration });
896 if (startupAction.action === "request_recovery") {
897 yield* endpointRuntime.requestRecovery(startupAction.config);
898 }
899 if (startupAction.action === "reconcile_link") {
900 const reconciledMode = yield* reconcileDesiredCloudLinkIfStillDesired(
901 localOrigin,
902 ).pipe(
903 Effect.retry({
904 while: shouldRetryCloudLink,
905 schedule: Schedule.exponential("1 second").pipe(
906 Schedule.modifyDelay(({ duration }) =>
907 Effect.succeed(Duration.min(duration, Duration.seconds(30))),
908 ),
909 Schedule.upTo({ duration: "10 minutes" }),
910 ),
911 }),
912 Effect.tap((mode) =>
913 mode === null
914 ? Effect.void
915 : Effect.logInfo("T3 Connect desired link reconciled on startup"),
916 ),
917 Effect.catch((cause) =>
918 Effect.logWarning("Failed to reconcile T3 Connect desired link on startup", {
919 cause,
920 }).pipe(Effect.as(null)),
921 ),
922 );
923 if (reconciledMode === "managed") {
924 const afterReconcile = yield* registerManagedTunnel;
925 if (afterReconcile.status === "recovery_required") {
926 yield* endpointRuntime.requestRecovery(afterReconcile.config);
927 }
928 }
929 }
770930 }),
771931 );
772932 yield* Deferred.succeed(cloudLinkParked, undefined).pipe(Effect.orDie);
docs/internals/t3-connect.md+41
......@@ -65,6 +65,47 @@ teardown, because a database failure must leave the active link usable. Failed
6565teardown retains enough state to retry. See the
6666[managed endpoint lifecycle](../../infra/relay/src/environments/ManagedEndpointProvider.ts).
6767
68## Idle tunnels are reclaimed and recovered
69
70Cloudflare bills a tunnel whether or not a connector is attached, so a laptop
71that sleeps with a linked environment leaves a paid tunnel behind. The relay's
72five-minute maintenance job can reclaim those tunnels. `RELAY_TUNNEL_CLEANUP_MODE`
73selects `off`, `dry-run`, or `enabled`, with `off` as the default. The mode is
74read at deploy time, so changing it means a relay deploy, not a variable flip.
75A candidate is a same-stage tunnel that Cloudflare reports down for at least
76five minutes, or one that never connected and is at least an hour old. The
77longer grace for never-connected tunnels covers a pairing still in progress.
78
79Cleanup deletes only tunnels whose host has registered recovery. Allocations
80without recovery registration belong to hosts that cannot replace a deleted
81tunnel and are left alone. Allocations with no recorded tunnel ID, or a
82different tunnel ID, are skipped because a provision may own them. A tunnel with
83no allocation row at all is counted as `skippedOrphan` and never deleted: there
84is no row to lock, so a relink that adopts it by name could race the delete.
85Clear those by hand. Each sweep is bounded: at most ten list requests, 100 deletions, a
86two-minute deadline, and an early stop on a Cloudflare rate limit. Each sweep
87starts one budget further along the candidate list, so a block of deletes that
88keep failing cannot starve the tunnels listed after them. See the
89[reaper](../../infra/relay/src/environments/ManagedEndpointReaper.ts).
90
91A host registers recovery at startup by sending its tunnel ID and loopback
92origin with a short-lived signature from the environment key. Registration
93touches Cloudflare only when the local host or port changed, and once per
94existing allocation on the first registration after the upgrade because the
95stored origin is empty. First registrations are jittered so an auto-update wave
96does not hit the relay at once. The host stores a confirmed-origin marker with
97the connector config, and a later boot starts the connector before registration
98only when that marker matches the current config and port. If registration
99cannot reach the relay for ten minutes, the host starts its stored config anyway
100and keeps registering in the background until it can reconcile the origin.
101If the connector exits, or `cloudflared` reports repeated tunnel
102rejections, the host asks the relay for a replacement, at most once every two
103minutes. The relay
104provisions under the same allocation, so the hostname and DNS record survive
105and clients keep their bindings. Every mutation on an allocation bumps its
106`generation`, and deletion locks the row at the generation it claimed, so a
107host that reconnects mid-sweep wins.
108
68109## OAuth traps
69110
70111Interactive clients and the headless CLI use the same Clerk application but
docs/operations/release.md+51
......@@ -132,6 +132,8 @@ Required `production` environment variables:
132132Optional `production` environment variables:
133133
134134- `RELAY_DOMAIN` when overriding the derived `relay.<RELAY_API_ZONE_NAME>` domain
135- `RELAY_TUNNEL_CLEANUP_MODE` with `off`, `dry-run`, or `enabled`. Missing and blank values use
136 `off`.
135137
136138Required `production` environment secrets:
137139
......@@ -151,6 +153,55 @@ Developers deploy personal stages locally rather than through pull-request autom
151153vp run --filter t3code-relay deploy -- --stage "$USER" --env-file .env.local
152154```
153155
156### Managed tunnel cleanup rollout
157
158Keep `RELAY_TUNNEL_CLEANUP_MODE=off` for the first production deploy. That deploy applies the
159nullable allocation migration and adds the recovery endpoints. Web and mobile clients need no
160coordinated release. CLI and desktop server builds must reach users before cleanup is enabled,
161because those builds register recovery and replace a deleted tunnel after wake.
162
1631. Deploy the relay and migration with cleanup `off`.
1642. Release the server build and confirm current hosts register recovery. Older hosts stay marked
165 legacy and are never candidates.
1663. Set `dry-run`, deploy, and read the sweep counters (`scanned`, `wouldDelete`, `skippedLegacy`,
167 `skippedOrphan`, `failed`, `truncated`) across several sweeps.
1684. Run the disposable-host canary below.
1695. Set `enabled` only after the canary recovers without a server restart.
170
171The job runs every five minutes with a five-minute grace period for tunnels that lost their
172connector, so a candidate is usually removed five to ten minutes after it goes down. Tunnels that
173never connected wait an hour. One sweep attempts at most 100 deletions, so a backlog takes longer.
174`RELAY_TUNNEL_CLEANUP_MODE` is read at deploy time. Changing it, including turning cleanup off during
175an incident, needs a relay deploy.
176
177To roll back, set cleanup to `off` and deploy the relay before downgrading any host. Keep the
178recovery endpoints deployed while current server builds are in use. The nullable columns can stay.
179
180### Disposable-host canary
181
182This test has not been run against a real Cloudflare account. Run it against a disposable relay
183stage, test Cloudflare account, disposable host, and disposable T3 home. Keep production cleanup at
184`off` or `dry-run` until it passes. Do not stop a daily-use T3 server.
185
1861. Deploy the disposable stage with cleanup `dry-run`. Link a first disposable environment through
187 web or mobile settings and confirm its tunnel is healthy and recovery is registered.
1882. Stop that host and restart the same T3 home on a different local port. Confirm the public
189 hostname reaches the new port and sends nothing to the old one.
1903. Link a second disposable environment with a server build that predates recovery registration.
191 Capture its managed `cloudflared` child PID, confirm it belongs to that host, and pause only that
192 child with `kill -STOP <legacy-pid>`. Wait until Cloudflare reports it down for over five minutes.
1934. Capture the first environment's `cloudflared` child PID from its server logs, confirm ownership,
194 and pause it with `kill -STOP <first-pid>`. Wait until Cloudflare reports it down for over five
195 minutes.
1965. Confirm dry-run counts the first tunnel in `wouldDelete` and the second in `skippedLegacy`.
1976. Set cleanup `enabled` on the disposable stage and deploy. Confirm in the test Cloudflare account
198 that the first tunnel is deleted and the legacy tunnel still exists.
1997. Resume the first child with `kill -CONT <first-pid>`. Confirm the running server detects the
200 repeated rejection, requests recovery, and becomes reachable at the same hostname without a
201 restart.
2028. Resume the legacy child with `kill -CONT <legacy-pid>` and confirm its tunnel reconnects.
2039. Repeat with a physical sleep and wake cycle on a disposable laptop before broad rollout.
204
154205## Marketing site deployment
155206
156207After a nightly release is published, the release workflow deploys the same commit
docs/user/remote-access.md+6
......@@ -158,6 +158,12 @@ page, or **Settings → T3 Connect** on mobile, and choose **Deregister**. This
158158revokes its cloud access and frees its host space even when the environment is
159159offline or has been wiped.
160160
161When idle tunnel cleanup is enabled, T3 Connect removes a linked environment's
162tunnel after it stays offline for several minutes. The environment stays linked
163and keeps the same address. When the host starts again or wakes, T3 Connect
164creates a replacement tunnel on its own. You do not need to pair again. Cleanup
165usually runs five to ten minutes after the tunnel goes down.
166
161167On a command-line host, `t3 connect unlink` disables exposure while retaining
162168your login; `t3 connect logout` also clears that login. Background-service
163169[removal](./background-service.md#manage-the-service) is separate.
infra/relay/.env.example+4
......@@ -5,6 +5,10 @@
55RELAY_API_ZONE_NAME=example.com
66RELAY_TUNNEL_ZONE_NAME=tunnels.example.com
77
8# Optional: inactive tunnel cleanup. Start with dry-run, verify the cleanup
9# logs, then set enabled. Unset and off both disable cleanup.
10# RELAY_TUNNEL_CLEANUP_MODE=off
11
812# Optional: Relay domain override
913# Set this only when the derived relay hostname should not be used.
1014# RELAY_DOMAIN=relay.example.com
infra/relay/migrations/postgres/20260919015455_managed_endpoint_recovery/migration.sql created+4
......@@ -0,0 +1,4 @@
1ALTER TABLE "relay_managed_endpoint_allocations" ADD COLUMN "recovery_enabled_at" varchar(64);--> statement-breakpoint
2ALTER TABLE "relay_managed_endpoint_allocations" ADD COLUMN "recovery_environment_public_key" text;--> statement-breakpoint
3ALTER TABLE "relay_managed_endpoint_allocations" ADD COLUMN "origin" jsonb;--> statement-breakpoint
4ALTER TABLE "relay_managed_endpoint_allocations" ADD COLUMN "generation" integer DEFAULT 0 NOT NULL;
infra/relay/migrations/postgres/20260919015455_managed_endpoint_recovery/snapshot.json created+1568
......@@ -0,0 +1,1568 @@
1{
2 "dialect": "postgres",
3 "id": "3809c51e-3821-4a08-818d-e5d28dd3e9b4",
4 "prevIds": ["4b6d0d21-8d78-4499-9dde-b42bcf633d05"],
5 "version": "8",
6 "ddl": [
7 {
8 "isRlsEnabled": false,
9 "name": "relay_agent_activity_rows",
10 "entityType": "tables",
11 "schema": "public"
12 },
13 {
14 "isRlsEnabled": false,
15 "name": "relay_delivery_attempts",
16 "entityType": "tables",
17 "schema": "public"
18 },
19 {
20 "isRlsEnabled": false,
21 "name": "relay_dpop_proofs",
22 "entityType": "tables",
23 "schema": "public"
24 },
25 {
26 "isRlsEnabled": false,
27 "name": "relay_environment_credentials",
28 "entityType": "tables",
29 "schema": "public"
30 },
31 {
32 "isRlsEnabled": false,
33 "name": "relay_environment_links",
34 "entityType": "tables",
35 "schema": "public"
36 },
37 {
38 "isRlsEnabled": false,
39 "name": "relay_live_activities",
40 "entityType": "tables",
41 "schema": "public"
42 },
43 {
44 "isRlsEnabled": false,
45 "name": "relay_managed_endpoint_allocations",
46 "entityType": "tables",
47 "schema": "public"
48 },
49 {
50 "isRlsEnabled": false,
51 "name": "relay_managed_tunnel_limits",
52 "entityType": "tables",
53 "schema": "public"
54 },
55 {
56 "isRlsEnabled": false,
57 "name": "relay_mobile_devices",
58 "entityType": "tables",
59 "schema": "public"
60 },
61 {
62 "type": "varchar(191)",
63 "typeSchema": null,
64 "notNull": true,
65 "dimensions": 0,
66 "default": null,
67 "generated": null,
68 "identity": null,
69 "name": "environment_id",
70 "entityType": "columns",
71 "schema": "public",
72 "table": "relay_agent_activity_rows"
73 },
74 {
75 "type": "text",
76 "typeSchema": null,
77 "notNull": true,
78 "dimensions": 0,
79 "default": null,
80 "generated": null,
81 "identity": null,
82 "name": "environment_public_key",
83 "entityType": "columns",
84 "schema": "public",
85 "table": "relay_agent_activity_rows"
86 },
87 {
88 "type": "varchar(512)",
89 "typeSchema": null,
90 "notNull": true,
91 "dimensions": 0,
92 "default": null,
93 "generated": null,
94 "identity": null,
95 "name": "thread_id",
96 "entityType": "columns",
97 "schema": "public",
98 "table": "relay_agent_activity_rows"
99 },
100 {
101 "type": "jsonb",
102 "typeSchema": null,
103 "notNull": true,
104 "dimensions": 0,
105 "default": null,
106 "generated": null,
107 "identity": null,
108 "name": "state_json",
109 "entityType": "columns",
110 "schema": "public",
111 "table": "relay_agent_activity_rows"
112 },
113 {
114 "type": "varchar(64)",
115 "typeSchema": null,
116 "notNull": true,
117 "dimensions": 0,
118 "default": null,
119 "generated": null,
120 "identity": null,
121 "name": "updated_at",
122 "entityType": "columns",
123 "schema": "public",
124 "table": "relay_agent_activity_rows"
125 },
126 {
127 "type": "varchar(64)",
128 "typeSchema": null,
129 "notNull": true,
130 "dimensions": 0,
131 "default": null,
132 "generated": null,
133 "identity": null,
134 "name": "created_at",
135 "entityType": "columns",
136 "schema": "public",
137 "table": "relay_agent_activity_rows"
138 },
139 {
140 "type": "varchar(36)",
141 "typeSchema": null,
142 "notNull": true,
143 "dimensions": 0,
144 "default": null,
145 "generated": null,
146 "identity": null,
147 "name": "id",
148 "entityType": "columns",
149 "schema": "public",
150 "table": "relay_delivery_attempts"
151 },
152 {
153 "type": "varchar(64)",
154 "typeSchema": null,
155 "notNull": true,
156 "dimensions": 0,
157 "default": null,
158 "generated": null,
159 "identity": null,
160 "name": "created_at",
161 "entityType": "columns",
162 "schema": "public",
163 "table": "relay_delivery_attempts"
164 },
165 {
166 "type": "varchar(255)",
167 "typeSchema": null,
168 "notNull": false,
169 "dimensions": 0,
170 "default": null,
171 "generated": null,
172 "identity": null,
173 "name": "user_id",
174 "entityType": "columns",
175 "schema": "public",
176 "table": "relay_delivery_attempts"
177 },
178 {
179 "type": "varchar(191)",
180 "typeSchema": null,
181 "notNull": false,
182 "dimensions": 0,
183 "default": null,
184 "generated": null,
185 "identity": null,
186 "name": "environment_id",
187 "entityType": "columns",
188 "schema": "public",
189 "table": "relay_delivery_attempts"
190 },
191 {
192 "type": "varchar(512)",
193 "typeSchema": null,
194 "notNull": false,
195 "dimensions": 0,
196 "default": null,
197 "generated": null,
198 "identity": null,
199 "name": "thread_id",
200 "entityType": "columns",
201 "schema": "public",
202 "table": "relay_delivery_attempts"
203 },
204 {
205 "type": "varchar(255)",
206 "typeSchema": null,
207 "notNull": false,
208 "dimensions": 0,
209 "default": null,
210 "generated": null,
211 "identity": null,
212 "name": "device_id",
213 "entityType": "columns",
214 "schema": "public",
215 "table": "relay_delivery_attempts"
216 },
217 {
218 "type": "varchar(64)",
219 "typeSchema": null,
220 "notNull": true,
221 "dimensions": 0,
222 "default": null,
223 "generated": null,
224 "identity": null,
225 "name": "kind",
226 "entityType": "columns",
227 "schema": "public",
228 "table": "relay_delivery_attempts"
229 },
230 {
231 "type": "varchar(64)",
232 "typeSchema": null,
233 "notNull": false,
234 "dimensions": 0,
235 "default": null,
236 "generated": null,
237 "identity": null,
238 "name": "source_job_id",
239 "entityType": "columns",
240 "schema": "public",
241 "table": "relay_delivery_attempts"
242 },
243 {
244 "type": "varchar(16)",
245 "typeSchema": null,
246 "notNull": false,
247 "dimensions": 0,
248 "default": null,
249 "generated": null,
250 "identity": null,
251 "name": "token_suffix",
252 "entityType": "columns",
253 "schema": "public",
254 "table": "relay_delivery_attempts"
255 },
256 {
257 "type": "integer",
258 "typeSchema": null,
259 "notNull": false,
260 "dimensions": 0,
261 "default": null,
262 "generated": null,
263 "identity": null,
264 "name": "apns_status",
265 "entityType": "columns",
266 "schema": "public",
267 "table": "relay_delivery_attempts"
268 },
269 {
270 "type": "text",
271 "typeSchema": null,
272 "notNull": false,
273 "dimensions": 0,
274 "default": null,
275 "generated": null,
276 "identity": null,
277 "name": "apns_reason",
278 "entityType": "columns",
279 "schema": "public",
280 "table": "relay_delivery_attempts"
281 },
282 {
283 "type": "varchar(128)",
284 "typeSchema": null,
285 "notNull": false,
286 "dimensions": 0,
287 "default": null,
288 "generated": null,
289 "identity": null,
290 "name": "apns_id",
291 "entityType": "columns",
292 "schema": "public",
293 "table": "relay_delivery_attempts"
294 },
295 {
296 "type": "text",
297 "typeSchema": null,
298 "notNull": false,
299 "dimensions": 0,
300 "default": null,
301 "generated": null,
302 "identity": null,
303 "name": "transport_error",
304 "entityType": "columns",
305 "schema": "public",
306 "table": "relay_delivery_attempts"
307 },
308 {
309 "type": "varchar(128)",
310 "typeSchema": null,
311 "notNull": true,
312 "dimensions": 0,
313 "default": null,
314 "generated": null,
315 "identity": null,
316 "name": "thumbprint",
317 "entityType": "columns",
318 "schema": "public",
319 "table": "relay_dpop_proofs"
320 },
321 {
322 "type": "varchar(255)",
323 "typeSchema": null,
324 "notNull": true,
325 "dimensions": 0,
326 "default": null,
327 "generated": null,
328 "identity": null,
329 "name": "jti",
330 "entityType": "columns",
331 "schema": "public",
332 "table": "relay_dpop_proofs"
333 },
334 {
335 "type": "integer",
336 "typeSchema": null,
337 "notNull": true,
338 "dimensions": 0,
339 "default": null,
340 "generated": null,
341 "identity": null,
342 "name": "iat",
343 "entityType": "columns",
344 "schema": "public",
345 "table": "relay_dpop_proofs"
346 },
347 {
348 "type": "varchar(64)",
349 "typeSchema": null,
350 "notNull": true,
351 "dimensions": 0,
352 "default": null,
353 "generated": null,
354 "identity": null,
355 "name": "expires_at",
356 "entityType": "columns",
357 "schema": "public",
358 "table": "relay_dpop_proofs"
359 },
360 {
361 "type": "varchar(64)",
362 "typeSchema": null,
363 "notNull": true,
364 "dimensions": 0,
365 "default": null,
366 "generated": null,
367 "identity": null,
368 "name": "created_at",
369 "entityType": "columns",
370 "schema": "public",
371 "table": "relay_dpop_proofs"
372 },
373 {
374 "type": "varchar(64)",
375 "typeSchema": null,
376 "notNull": true,
377 "dimensions": 0,
378 "default": null,
379 "generated": null,
380 "identity": null,
381 "name": "credential_id",
382 "entityType": "columns",
383 "schema": "public",
384 "table": "relay_environment_credentials"
385 },
386 {
387 "type": "varchar(191)",
388 "typeSchema": null,
389 "notNull": true,
390 "dimensions": 0,
391 "default": null,
392 "generated": null,
393 "identity": null,
394 "name": "environment_id",
395 "entityType": "columns",
396 "schema": "public",
397 "table": "relay_environment_credentials"
398 },
399 {
400 "type": "text",
401 "typeSchema": null,
402 "notNull": true,
403 "dimensions": 0,
404 "default": null,
405 "generated": null,
406 "identity": null,
407 "name": "environment_public_key",
408 "entityType": "columns",
409 "schema": "public",
410 "table": "relay_environment_credentials"
411 },
412 {
413 "type": "varchar(191)",
414 "typeSchema": null,
415 "notNull": true,
416 "dimensions": 0,
417 "default": null,
418 "generated": null,
419 "identity": null,
420 "name": "credential_hash",
421 "entityType": "columns",
422 "schema": "public",
423 "table": "relay_environment_credentials"
424 },
425 {
426 "type": "varchar(64)",
427 "typeSchema": null,
428 "notNull": false,
429 "dimensions": 0,
430 "default": null,
431 "generated": null,
432 "identity": null,
433 "name": "revoked_at",
434 "entityType": "columns",
435 "schema": "public",
436 "table": "relay_environment_credentials"
437 },
438 {
439 "type": "varchar(64)",
440 "typeSchema": null,
441 "notNull": true,
442 "dimensions": 0,
443 "default": null,
444 "generated": null,
445 "identity": null,
446 "name": "created_at",
447 "entityType": "columns",
448 "schema": "public",
449 "table": "relay_environment_credentials"
450 },
451 {
452 "type": "varchar(64)",
453 "typeSchema": null,
454 "notNull": true,
455 "dimensions": 0,
456 "default": null,
457 "generated": null,
458 "identity": null,
459 "name": "updated_at",
460 "entityType": "columns",
461 "schema": "public",
462 "table": "relay_environment_credentials"
463 },
464 {
465 "type": "varchar(191)",
466 "typeSchema": null,
467 "notNull": true,
468 "dimensions": 0,
469 "default": null,
470 "generated": null,
471 "identity": null,
472 "name": "user_id",
473 "entityType": "columns",
474 "schema": "public",
475 "table": "relay_environment_links"
476 },
477 {
478 "type": "varchar(191)",
479 "typeSchema": null,
480 "notNull": true,
481 "dimensions": 0,
482 "default": null,
483 "generated": null,
484 "identity": null,
485 "name": "environment_id",
486 "entityType": "columns",
487 "schema": "public",
488 "table": "relay_environment_links"
489 },
490 {
491 "type": "text",
492 "typeSchema": null,
493 "notNull": true,
494 "dimensions": 0,
495 "default": "'T3 Environment'",
496 "generated": null,
497 "identity": null,
498 "name": "environment_label",
499 "entityType": "columns",
500 "schema": "public",
501 "table": "relay_environment_links"
502 },
503 {
504 "type": "text",
505 "typeSchema": null,
506 "notNull": true,
507 "dimensions": 0,
508 "default": null,
509 "generated": null,
510 "identity": null,
511 "name": "environment_public_key",
512 "entityType": "columns",
513 "schema": "public",
514 "table": "relay_environment_links"
515 },
516 {
517 "type": "text",
518 "typeSchema": null,
519 "notNull": true,
520 "dimensions": 0,
521 "default": null,
522 "generated": null,
523 "identity": null,
524 "name": "endpoint_http_base_url",
525 "entityType": "columns",
526 "schema": "public",
527 "table": "relay_environment_links"
528 },
529 {
530 "type": "text",
531 "typeSchema": null,
532 "notNull": true,
533 "dimensions": 0,
534 "default": null,
535 "generated": null,
536 "identity": null,
537 "name": "endpoint_ws_base_url",
538 "entityType": "columns",
539 "schema": "public",
540 "table": "relay_environment_links"
541 },
542 {
543 "type": "varchar(32)",
544 "typeSchema": null,
545 "notNull": true,
546 "dimensions": 0,
547 "default": null,
548 "generated": null,
549 "identity": null,
550 "name": "endpoint_provider_kind",
551 "entityType": "columns",
552 "schema": "public",
553 "table": "relay_environment_links"
554 },
555 {
556 "type": "boolean",
557 "typeSchema": null,
558 "notNull": true,
559 "dimensions": 0,
560 "default": "true",
561 "generated": null,
562 "identity": null,
563 "name": "notifications_enabled",
564 "entityType": "columns",
565 "schema": "public",
566 "table": "relay_environment_links"
567 },
568 {
569 "type": "boolean",
570 "typeSchema": null,
571 "notNull": true,
572 "dimensions": 0,
573 "default": "true",
574 "generated": null,
575 "identity": null,
576 "name": "live_activities_enabled",
577 "entityType": "columns",
578 "schema": "public",
579 "table": "relay_environment_links"
580 },
581 {
582 "type": "boolean",
583 "typeSchema": null,
584 "notNull": true,
585 "dimensions": 0,
586 "default": "false",
587 "generated": null,
588 "identity": null,
589 "name": "managed_tunnels_enabled",
590 "entityType": "columns",
591 "schema": "public",
592 "table": "relay_environment_links"
593 },
594 {
595 "type": "varchar(191)",
596 "typeSchema": null,
597 "notNull": false,
598 "dimensions": 0,
599 "default": null,
600 "generated": null,
601 "identity": null,
602 "name": "created_by_device_id",
603 "entityType": "columns",
604 "schema": "public",
605 "table": "relay_environment_links"
606 },
607 {
608 "type": "varchar(64)",
609 "typeSchema": null,
610 "notNull": false,
611 "dimensions": 0,
612 "default": null,
613 "generated": null,
614 "identity": null,
615 "name": "revoked_at",
616 "entityType": "columns",
617 "schema": "public",
618 "table": "relay_environment_links"
619 },
620 {
621 "type": "varchar(64)",
622 "typeSchema": null,
623 "notNull": true,
624 "dimensions": 0,
625 "default": null,
626 "generated": null,
627 "identity": null,
628 "name": "created_at",
629 "entityType": "columns",
630 "schema": "public",
631 "table": "relay_environment_links"
632 },
633 {
634 "type": "varchar(64)",
635 "typeSchema": null,
636 "notNull": true,
637 "dimensions": 0,
638 "default": null,
639 "generated": null,
640 "identity": null,
641 "name": "updated_at",
642 "entityType": "columns",
643 "schema": "public",
644 "table": "relay_environment_links"
645 },
646 {
647 "type": "varchar(255)",
648 "typeSchema": null,
649 "notNull": true,
650 "dimensions": 0,
651 "default": null,
652 "generated": null,
653 "identity": null,
654 "name": "user_id",
655 "entityType": "columns",
656 "schema": "public",
657 "table": "relay_live_activities"
658 },
659 {
660 "type": "varchar(255)",
661 "typeSchema": null,
662 "notNull": true,
663 "dimensions": 0,
664 "default": null,
665 "generated": null,
666 "identity": null,
667 "name": "device_id",
668 "entityType": "columns",
669 "schema": "public",
670 "table": "relay_live_activities"
671 },
672 {
673 "type": "text",
674 "typeSchema": null,
675 "notNull": false,
676 "dimensions": 0,
677 "default": null,
678 "generated": null,
679 "identity": null,
680 "name": "activity_push_token",
681 "entityType": "columns",
682 "schema": "public",
683 "table": "relay_live_activities"
684 },
685 {
686 "type": "varchar(64)",
687 "typeSchema": null,
688 "notNull": false,
689 "dimensions": 0,
690 "default": null,
691 "generated": null,
692 "identity": null,
693 "name": "remote_start_queued_at",
694 "entityType": "columns",
695 "schema": "public",
696 "table": "relay_live_activities"
697 },
698 {
699 "type": "varchar(64)",
700 "typeSchema": null,
701 "notNull": false,
702 "dimensions": 0,
703 "default": null,
704 "generated": null,
705 "identity": null,
706 "name": "remote_started_at",
707 "entityType": "columns",
708 "schema": "public",
709 "table": "relay_live_activities"
710 },
711 {
712 "type": "varchar(64)",
713 "typeSchema": null,
714 "notNull": false,
715 "dimensions": 0,
716 "default": null,
717 "generated": null,
718 "identity": null,
719 "name": "ended_at",
720 "entityType": "columns",
721 "schema": "public",
722 "table": "relay_live_activities"
723 },
724 {
725 "type": "jsonb",
726 "typeSchema": null,
727 "notNull": false,
728 "dimensions": 0,
729 "default": null,
730 "generated": null,
731 "identity": null,
732 "name": "last_aggregate_json",
733 "entityType": "columns",
734 "schema": "public",
735 "table": "relay_live_activities"
736 },
737 {
738 "type": "varchar(64)",
739 "typeSchema": null,
740 "notNull": false,
741 "dimensions": 0,
742 "default": null,
743 "generated": null,
744 "identity": null,
745 "name": "last_live_activity_delivery_at",
746 "entityType": "columns",
747 "schema": "public",
748 "table": "relay_live_activities"
749 },
750 {
751 "type": "varchar(64)",
752 "typeSchema": null,
753 "notNull": true,
754 "dimensions": 0,
755 "default": null,
756 "generated": null,
757 "identity": null,
758 "name": "created_at",
759 "entityType": "columns",
760 "schema": "public",
761 "table": "relay_live_activities"
762 },
763 {
764 "type": "varchar(64)",
765 "typeSchema": null,
766 "notNull": true,
767 "dimensions": 0,
768 "default": null,
769 "generated": null,
770 "identity": null,
771 "name": "updated_at",
772 "entityType": "columns",
773 "schema": "public",
774 "table": "relay_live_activities"
775 },
776 {
777 "type": "varchar(191)",
778 "typeSchema": null,
779 "notNull": true,
780 "dimensions": 0,
781 "default": null,
782 "generated": null,
783 "identity": null,
784 "name": "user_id",
785 "entityType": "columns",
786 "schema": "public",
787 "table": "relay_managed_endpoint_allocations"
788 },
789 {
790 "type": "varchar(191)",
791 "typeSchema": null,
792 "notNull": true,
793 "dimensions": 0,
794 "default": null,
795 "generated": null,
796 "identity": null,
797 "name": "environment_id",
798 "entityType": "columns",
799 "schema": "public",
800 "table": "relay_managed_endpoint_allocations"
801 },
802 {
803 "type": "text",
804 "typeSchema": null,
805 "notNull": true,
806 "dimensions": 0,
807 "default": null,
808 "generated": null,
809 "identity": null,
810 "name": "hostname",
811 "entityType": "columns",
812 "schema": "public",
813 "table": "relay_managed_endpoint_allocations"
814 },
815 {
816 "type": "varchar(191)",
817 "typeSchema": null,
818 "notNull": false,
819 "dimensions": 0,
820 "default": null,
821 "generated": null,
822 "identity": null,
823 "name": "tunnel_id",
824 "entityType": "columns",
825 "schema": "public",
826 "table": "relay_managed_endpoint_allocations"
827 },
828 {
829 "type": "text",
830 "typeSchema": null,
831 "notNull": true,
832 "dimensions": 0,
833 "default": null,
834 "generated": null,
835 "identity": null,
836 "name": "tunnel_name",
837 "entityType": "columns",
838 "schema": "public",
839 "table": "relay_managed_endpoint_allocations"
840 },
841 {
842 "type": "varchar(191)",
843 "typeSchema": null,
844 "notNull": false,
845 "dimensions": 0,
846 "default": null,
847 "generated": null,
848 "identity": null,
849 "name": "dns_record_id",
850 "entityType": "columns",
851 "schema": "public",
852 "table": "relay_managed_endpoint_allocations"
853 },
854 {
855 "type": "varchar(64)",
856 "typeSchema": null,
857 "notNull": false,
858 "dimensions": 0,
859 "default": null,
860 "generated": null,
861 "identity": null,
862 "name": "ready_at",
863 "entityType": "columns",
864 "schema": "public",
865 "table": "relay_managed_endpoint_allocations"
866 },
867 {
868 "type": "varchar(64)",
869 "typeSchema": null,
870 "notNull": false,
871 "dimensions": 0,
872 "default": null,
873 "generated": null,
874 "identity": null,
875 "name": "recovery_enabled_at",
876 "entityType": "columns",
877 "schema": "public",
878 "table": "relay_managed_endpoint_allocations"
879 },
880 {
881 "type": "text",
882 "typeSchema": null,
883 "notNull": false,
884 "dimensions": 0,
885 "default": null,
886 "generated": null,
887 "identity": null,
888 "name": "recovery_environment_public_key",
889 "entityType": "columns",
890 "schema": "public",
891 "table": "relay_managed_endpoint_allocations"
892 },
893 {
894 "type": "jsonb",
895 "typeSchema": null,
896 "notNull": false,
897 "dimensions": 0,
898 "default": null,
899 "generated": null,
900 "identity": null,
901 "name": "origin",
902 "entityType": "columns",
903 "schema": "public",
904 "table": "relay_managed_endpoint_allocations"
905 },
906 {
907 "type": "integer",
908 "typeSchema": null,
909 "notNull": true,
910 "dimensions": 0,
911 "default": "0",
912 "generated": null,
913 "identity": null,
914 "name": "generation",
915 "entityType": "columns",
916 "schema": "public",
917 "table": "relay_managed_endpoint_allocations"
918 },
919 {
920 "type": "varchar(64)",
921 "typeSchema": null,
922 "notNull": true,
923 "dimensions": 0,
924 "default": null,
925 "generated": null,
926 "identity": null,
927 "name": "created_at",
928 "entityType": "columns",
929 "schema": "public",
930 "table": "relay_managed_endpoint_allocations"
931 },
932 {
933 "type": "varchar(64)",
934 "typeSchema": null,
935 "notNull": true,
936 "dimensions": 0,
937 "default": null,
938 "generated": null,
939 "identity": null,
940 "name": "updated_at",
941 "entityType": "columns",
942 "schema": "public",
943 "table": "relay_managed_endpoint_allocations"
944 },
945 {
946 "type": "varchar(191)",
947 "typeSchema": null,
948 "notNull": true,
949 "dimensions": 0,
950 "default": null,
951 "generated": null,
952 "identity": null,
953 "name": "user_id",
954 "entityType": "columns",
955 "schema": "public",
956 "table": "relay_managed_tunnel_limits"
957 },
958 {
959 "type": "integer",
960 "typeSchema": null,
961 "notNull": true,
962 "dimensions": 0,
963 "default": null,
964 "generated": null,
965 "identity": null,
966 "name": "max_tunnels",
967 "entityType": "columns",
968 "schema": "public",
969 "table": "relay_managed_tunnel_limits"
970 },
971 {
972 "type": "varchar(64)",
973 "typeSchema": null,
974 "notNull": true,
975 "dimensions": 0,
976 "default": null,
977 "generated": null,
978 "identity": null,
979 "name": "created_at",
980 "entityType": "columns",
981 "schema": "public",
982 "table": "relay_managed_tunnel_limits"
983 },
984 {
985 "type": "varchar(64)",
986 "typeSchema": null,
987 "notNull": true,
988 "dimensions": 0,
989 "default": null,
990 "generated": null,
991 "identity": null,
992 "name": "updated_at",
993 "entityType": "columns",
994 "schema": "public",
995 "table": "relay_managed_tunnel_limits"
996 },
997 {
998 "type": "varchar(255)",
999 "typeSchema": null,
1000 "notNull": true,
1001 "dimensions": 0,
1002 "default": null,
1003 "generated": null,
1004 "identity": null,
1005 "name": "user_id",
1006 "entityType": "columns",
1007 "schema": "public",
1008 "table": "relay_mobile_devices"
1009 },
1010 {
1011 "type": "varchar(255)",
1012 "typeSchema": null,
1013 "notNull": true,
1014 "dimensions": 0,
1015 "default": null,
1016 "generated": null,
1017 "identity": null,
1018 "name": "device_id",
1019 "entityType": "columns",
1020 "schema": "public",
1021 "table": "relay_mobile_devices"
1022 },
1023 {
1024 "type": "text",
1025 "typeSchema": null,
1026 "notNull": true,
1027 "dimensions": 0,
1028 "default": "'iOS device'",
1029 "generated": null,
1030 "identity": null,
1031 "name": "label",
1032 "entityType": "columns",
1033 "schema": "public",
1034 "table": "relay_mobile_devices"
1035 },
1036 {
1037 "type": "varchar(16)",
1038 "typeSchema": null,
1039 "notNull": true,
1040 "dimensions": 0,
1041 "default": null,
1042 "generated": null,
1043 "identity": null,
1044 "name": "platform",
1045 "entityType": "columns",
1046 "schema": "public",
1047 "table": "relay_mobile_devices"
1048 },
1049 {
1050 "type": "integer",
1051 "typeSchema": null,
1052 "notNull": false,
1053 "dimensions": 0,
1054 "default": null,
1055 "generated": null,
1056 "identity": null,
1057 "name": "ios_major_version",
1058 "entityType": "columns",
1059 "schema": "public",
1060 "table": "relay_mobile_devices"
1061 },
1062 {
1063 "type": "integer",
1064 "typeSchema": null,
1065 "notNull": false,
1066 "dimensions": 0,
1067 "default": null,
1068 "generated": null,
1069 "identity": null,
1070 "name": "android_api_level",
1071 "entityType": "columns",
1072 "schema": "public",
1073 "table": "relay_mobile_devices"
1074 },
1075 {
1076 "type": "varchar(64)",
1077 "typeSchema": null,
1078 "notNull": false,
1079 "dimensions": 0,
1080 "default": null,
1081 "generated": null,
1082 "identity": null,
1083 "name": "app_version",
1084 "entityType": "columns",
1085 "schema": "public",
1086 "table": "relay_mobile_devices"
1087 },
1088 {
1089 "type": "varchar(255)",
1090 "typeSchema": null,
1091 "notNull": false,
1092 "dimensions": 0,
1093 "default": null,
1094 "generated": null,
1095 "identity": null,
1096 "name": "bundle_id",
1097 "entityType": "columns",
1098 "schema": "public",
1099 "table": "relay_mobile_devices"
1100 },
1101 {
1102 "type": "varchar(16)",
1103 "typeSchema": null,
1104 "notNull": false,
1105 "dimensions": 0,
1106 "default": null,
1107 "generated": null,
1108 "identity": null,
1109 "name": "aps_environment",
1110 "entityType": "columns",
1111 "schema": "public",
1112 "table": "relay_mobile_devices"
1113 },
1114 {
1115 "type": "text",
1116 "typeSchema": null,
1117 "notNull": false,
1118 "dimensions": 0,
1119 "default": null,
1120 "generated": null,
1121 "identity": null,
1122 "name": "push_token",
1123 "entityType": "columns",
1124 "schema": "public",
1125 "table": "relay_mobile_devices"
1126 },
1127 {
1128 "type": "text",
1129 "typeSchema": null,
1130 "notNull": false,
1131 "dimensions": 0,
1132 "default": null,
1133 "generated": null,
1134 "identity": null,
1135 "name": "push_to_start_token",
1136 "entityType": "columns",
1137 "schema": "public",
1138 "table": "relay_mobile_devices"
1139 },
1140 {
1141 "type": "jsonb",
1142 "typeSchema": null,
1143 "notNull": true,
1144 "dimensions": 0,
1145 "default": null,
1146 "generated": null,
1147 "identity": null,
1148 "name": "preferences_json",
1149 "entityType": "columns",
1150 "schema": "public",
1151 "table": "relay_mobile_devices"
1152 },
1153 {
1154 "type": "varchar(64)",
1155 "typeSchema": null,
1156 "notNull": true,
1157 "dimensions": 0,
1158 "default": null,
1159 "generated": null,
1160 "identity": null,
1161 "name": "created_at",
1162 "entityType": "columns",
1163 "schema": "public",
1164 "table": "relay_mobile_devices"
1165 },
1166 {
1167 "type": "varchar(64)",
1168 "typeSchema": null,
1169 "notNull": true,
1170 "dimensions": 0,
1171 "default": null,
1172 "generated": null,
1173 "identity": null,
1174 "name": "updated_at",
1175 "entityType": "columns",
1176 "schema": "public",
1177 "table": "relay_mobile_devices"
1178 },
1179 {
1180 "nameExplicit": true,
1181 "columns": [
1182 {
1183 "value": "updated_at",
1184 "isExpression": false,
1185 "asc": true,
1186 "nullsFirst": false,
1187 "opclass": null
1188 }
1189 ],
1190 "isUnique": false,
1191 "where": null,
1192 "with": "",
1193 "method": "btree",
1194 "concurrently": false,
1195 "name": "idx_relay_agent_activity_rows_updated",
1196 "entityType": "indexes",
1197 "schema": "public",
1198 "table": "relay_agent_activity_rows"
1199 },
1200 {
1201 "nameExplicit": true,
1202 "columns": [
1203 {
1204 "value": "environment_id",
1205 "isExpression": false,
1206 "asc": true,
1207 "nullsFirst": false,
1208 "opclass": null
1209 },
1210 {
1211 "value": "thread_id",
1212 "isExpression": false,
1213 "asc": true,
1214 "nullsFirst": false,
1215 "opclass": null
1216 },
1217 {
1218 "value": "created_at",
1219 "isExpression": false,
1220 "asc": true,
1221 "nullsFirst": false,
1222 "opclass": null
1223 }
1224 ],
1225 "isUnique": false,
1226 "where": null,
1227 "with": "",
1228 "method": "btree",
1229 "concurrently": false,
1230 "name": "idx_relay_delivery_attempts_environment",
1231 "entityType": "indexes",
1232 "schema": "public",
1233 "table": "relay_delivery_attempts"
1234 },
1235 {
1236 "nameExplicit": true,
1237 "columns": [
1238 {
1239 "value": "source_job_id",
1240 "isExpression": false,
1241 "asc": true,
1242 "nullsFirst": false,
1243 "opclass": null
1244 }
1245 ],
1246 "isUnique": true,
1247 "where": null,
1248 "with": "",
1249 "method": "btree",
1250 "concurrently": false,
1251 "name": "idx_relay_delivery_attempts_source_job",
1252 "entityType": "indexes",
1253 "schema": "public",
1254 "table": "relay_delivery_attempts"
1255 },
1256 {
1257 "nameExplicit": true,
1258 "columns": [
1259 {
1260 "value": "expires_at",
1261 "isExpression": false,
1262 "asc": true,
1263 "nullsFirst": false,
1264 "opclass": null
1265 }
1266 ],
1267 "isUnique": false,
1268 "where": null,
1269 "with": "",
1270 "method": "btree",
1271 "concurrently": false,
1272 "name": "idx_relay_dpop_proofs_expires_at",
1273 "entityType": "indexes",
1274 "schema": "public",
1275 "table": "relay_dpop_proofs"
1276 },
1277 {
1278 "nameExplicit": true,
1279 "columns": [
1280 {
1281 "value": "credential_hash",
1282 "isExpression": false,
1283 "asc": true,
1284 "nullsFirst": false,
1285 "opclass": null
1286 }
1287 ],
1288 "isUnique": true,
1289 "where": null,
1290 "with": "",
1291 "method": "btree",
1292 "concurrently": false,
1293 "name": "idx_relay_environment_credentials_hash",
1294 "entityType": "indexes",
1295 "schema": "public",
1296 "table": "relay_environment_credentials"
1297 },
1298 {
1299 "nameExplicit": true,
1300 "columns": [
1301 {
1302 "value": "environment_id",
1303 "isExpression": false,
1304 "asc": true,
1305 "nullsFirst": false,
1306 "opclass": null
1307 },
1308 {
1309 "value": "revoked_at",
1310 "isExpression": false,
1311 "asc": true,
1312 "nullsFirst": false,
1313 "opclass": null
1314 }
1315 ],
1316 "isUnique": false,
1317 "where": null,
1318 "with": "",
1319 "method": "btree",
1320 "concurrently": false,
1321 "name": "idx_relay_environment_credentials_environment",
1322 "entityType": "indexes",
1323 "schema": "public",
1324 "table": "relay_environment_credentials"
1325 },
1326 {
1327 "nameExplicit": true,
1328 "columns": [
1329 {
1330 "value": "environment_id",
1331 "isExpression": false,
1332 "asc": true,
1333 "nullsFirst": false,
1334 "opclass": null
1335 },
1336 {
1337 "value": "environment_public_key",
1338 "isExpression": false,
1339 "asc": true,
1340 "nullsFirst": false,
1341 "opclass": null
1342 },
1343 {
1344 "value": "revoked_at",
1345 "isExpression": false,
1346 "asc": true,
1347 "nullsFirst": false,
1348 "opclass": null
1349 }
1350 ],
1351 "isUnique": false,
1352 "where": null,
1353 "with": "",
1354 "method": "btree",
1355 "concurrently": false,
1356 "name": "idx_relay_environment_credentials_environment_key",
1357 "entityType": "indexes",
1358 "schema": "public",
1359 "table": "relay_environment_credentials"
1360 },
1361 {
1362 "nameExplicit": true,
1363 "columns": [
1364 {
1365 "value": "environment_id",
1366 "isExpression": false,
1367 "asc": true,
1368 "nullsFirst": false,
1369 "opclass": null
1370 },
1371 {
1372 "value": "revoked_at",
1373 "isExpression": false,
1374 "asc": true,
1375 "nullsFirst": false,
1376 "opclass": null
1377 }
1378 ],
1379 "isUnique": false,
1380 "where": null,
1381 "with": "",
1382 "method": "btree",
1383 "concurrently": false,
1384 "name": "idx_relay_environment_links_environment",
1385 "entityType": "indexes",
1386 "schema": "public",
1387 "table": "relay_environment_links"
1388 },
1389 {
1390 "nameExplicit": true,
1391 "columns": [
1392 {
1393 "value": "activity_push_token",
1394 "isExpression": false,
1395 "asc": true,
1396 "nullsFirst": false,
1397 "opclass": null
1398 }
1399 ],
1400 "isUnique": true,
1401 "where": null,
1402 "with": "",
1403 "method": "btree",
1404 "concurrently": false,
1405 "name": "idx_relay_live_activities_activity_push_token",
1406 "entityType": "indexes",
1407 "schema": "public",
1408 "table": "relay_live_activities"
1409 },
1410 {
1411 "nameExplicit": true,
1412 "columns": [
1413 {
1414 "value": "hostname",
1415 "isExpression": false,
1416 "asc": true,
1417 "nullsFirst": false,
1418 "opclass": null
1419 }
1420 ],
1421 "isUnique": true,
1422 "where": null,
1423 "with": "",
1424 "method": "btree",
1425 "concurrently": false,
1426 "name": "idx_relay_managed_endpoint_allocations_hostname",
1427 "entityType": "indexes",
1428 "schema": "public",
1429 "table": "relay_managed_endpoint_allocations"
1430 },
1431 {
1432 "nameExplicit": true,
1433 "columns": [
1434 {
1435 "value": "tunnel_name",
1436 "isExpression": false,
1437 "asc": true,
1438 "nullsFirst": false,
1439 "opclass": null
1440 }
1441 ],
1442 "isUnique": true,
1443 "where": null,
1444 "with": "",
1445 "method": "btree",
1446 "concurrently": false,
1447 "name": "idx_relay_managed_endpoint_allocations_tunnel_name",
1448 "entityType": "indexes",
1449 "schema": "public",
1450 "table": "relay_managed_endpoint_allocations"
1451 },
1452 {
1453 "nameExplicit": true,
1454 "columns": [
1455 {
1456 "value": "push_token",
1457 "isExpression": false,
1458 "asc": true,
1459 "nullsFirst": false,
1460 "opclass": null
1461 }
1462 ],
1463 "isUnique": true,
1464 "where": null,
1465 "with": "",
1466 "method": "btree",
1467 "concurrently": false,
1468 "name": "idx_relay_mobile_devices_push_token",
1469 "entityType": "indexes",
1470 "schema": "public",
1471 "table": "relay_mobile_devices"
1472 },
1473 {
1474 "nameExplicit": true,
1475 "columns": [
1476 {
1477 "value": "push_to_start_token",
1478 "isExpression": false,
1479 "asc": true,
1480 "nullsFirst": false,
1481 "opclass": null
1482 }
1483 ],
1484 "isUnique": true,
1485 "where": null,
1486 "with": "",
1487 "method": "btree",
1488 "concurrently": false,
1489 "name": "idx_relay_mobile_devices_push_to_start_token",
1490 "entityType": "indexes",
1491 "schema": "public",
1492 "table": "relay_mobile_devices"
1493 },
1494 {
1495 "columns": ["environment_id", "environment_public_key", "thread_id"],
1496 "nameExplicit": false,
1497 "name": "relay_agent_activity_rows_pkey",
1498 "entityType": "pks",
1499 "schema": "public",
1500 "table": "relay_agent_activity_rows"
1501 },
1502 {
1503 "columns": ["thumbprint", "jti"],
1504 "nameExplicit": false,
1505 "name": "relay_dpop_proofs_pkey",
1506 "entityType": "pks",
1507 "schema": "public",
1508 "table": "relay_dpop_proofs"
1509 },
1510 {
1511 "columns": ["user_id", "environment_id"],
1512 "nameExplicit": false,
1513 "name": "relay_environment_links_pkey",
1514 "entityType": "pks",
1515 "schema": "public",
1516 "table": "relay_environment_links"
1517 },
1518 {
1519 "columns": ["user_id", "device_id"],
1520 "nameExplicit": false,
1521 "name": "relay_live_activities_pkey",
1522 "entityType": "pks",
1523 "schema": "public",
1524 "table": "relay_live_activities"
1525 },
1526 {
1527 "columns": ["user_id", "environment_id"],
1528 "nameExplicit": false,
1529 "name": "relay_managed_endpoint_allocations_pkey",
1530 "entityType": "pks",
1531 "schema": "public",
1532 "table": "relay_managed_endpoint_allocations"
1533 },
1534 {
1535 "columns": ["user_id", "device_id"],
1536 "nameExplicit": false,
1537 "name": "relay_mobile_devices_pkey",
1538 "entityType": "pks",
1539 "schema": "public",
1540 "table": "relay_mobile_devices"
1541 },
1542 {
1543 "columns": ["id"],
1544 "nameExplicit": false,
1545 "name": "relay_delivery_attempts_pkey",
1546 "schema": "public",
1547 "table": "relay_delivery_attempts",
1548 "entityType": "pks"
1549 },
1550 {
1551 "columns": ["credential_id"],
1552 "nameExplicit": false,
1553 "name": "relay_environment_credentials_pkey",
1554 "schema": "public",
1555 "table": "relay_environment_credentials",
1556 "entityType": "pks"
1557 },
1558 {
1559 "columns": ["user_id"],
1560 "nameExplicit": false,
1561 "name": "relay_managed_tunnel_limits_pkey",
1562 "schema": "public",
1563 "table": "relay_managed_tunnel_limits",
1564 "entityType": "pks"
1565 }
1566 ],
1567 "renames": []
1568}
infra/relay/src/Config.test.ts created+35
......@@ -0,0 +1,35 @@
1import { expect, it } from "@effect/vitest";
2import * as ConfigProvider from "effect/ConfigProvider";
3import * as Effect from "effect/Effect";
4
5import { managedEndpointCleanupModeConfig } from "./Config.ts";
6
7it.effect.each([
8 { name: "missing", env: {}, expected: "off" },
9 { name: "empty", env: { RELAY_TUNNEL_CLEANUP_MODE: "" }, expected: "off" },
10 { name: "whitespace", env: { RELAY_TUNNEL_CLEANUP_MODE: " \t" }, expected: "off" },
11 { name: "off", env: { RELAY_TUNNEL_CLEANUP_MODE: "off" }, expected: "off" },
12 {
13 name: "dry-run",
14 env: { RELAY_TUNNEL_CLEANUP_MODE: "dry-run" },
15 expected: "dry-run",
16 },
17 { name: "enabled", env: { RELAY_TUNNEL_CLEANUP_MODE: "enabled" }, expected: "enabled" },
18] as const)("loads $name cleanup mode as $expected", ({ env, expected }) =>
19 Effect.gen(function* () {
20 const provider = ConfigProvider.fromEnv({ env });
21 expect(yield* managedEndpointCleanupModeConfig.parse(provider)).toBe(expected);
22 }),
23);
24
25it.effect("rejects an invalid cleanup mode", () =>
26 Effect.gen(function* () {
27 const provider = ConfigProvider.fromEnv({
28 env: { RELAY_TUNNEL_CLEANUP_MODE: "delete-everything" },
29 });
30 const error = yield* Effect.flip(managedEndpointCleanupModeConfig.parse(provider));
31
32 expect(error._tag).toBe("ConfigError");
33 expect(error.message).toContain('Expected "off" | "dry-run" | "enabled"');
34 }),
35);
infra/relay/src/Config.ts+17
......@@ -1,4 +1,6 @@
1import * as Config from "effect/Config";
12import * as Context from "effect/Context";
3import * as Effect from "effect/Effect";
24import * as Layer from "effect/Layer";
35import * as Redacted from "effect/Redacted";
46import * as Schema from "effect/Schema";
......@@ -6,6 +8,20 @@ import * as Schema from "effect/Schema";
68export const ApnsEnvironment = Schema.Literals(["sandbox", "production"]);
79export type ApnsEnvironment = typeof ApnsEnvironment.Type;
810
11export const ManagedEndpointCleanupMode = Schema.Literals(["off", "dry-run", "enabled"]);
12export type ManagedEndpointCleanupMode = typeof ManagedEndpointCleanupMode.Type;
13const decodeManagedEndpointCleanupMode = Schema.decodeUnknownEffect(ManagedEndpointCleanupMode);
14
15export const managedEndpointCleanupModeConfig = Config.String("RELAY_TUNNEL_CLEANUP_MODE").pipe(
16 Config.withDefault("off"),
17 Config.map((value) => value.trim() || "off"),
18 Config.mapEffect((value) =>
19 decodeManagedEndpointCleanupMode(value).pipe(
20 Effect.mapError((error) => new Config.ConfigError(error)),
21 ),
22 ),
23);
24
925export interface ApnsCredentials {
1026 readonly teamId: string;
1127 readonly keyId: string;
......@@ -28,6 +44,7 @@ export class RelayConfiguration extends Context.Service<
2844 readonly cloudMintPublicKey: string;
2945 readonly managedEndpointBaseDomain: string | undefined;
3046 readonly managedEndpointNamespace: string | undefined;
47 readonly managedEndpointCleanupMode?: ManagedEndpointCleanupMode;
3148 }
3249>()("t3code-relay/Config/RelayConfiguration") {}
3350
infra/relay/src/deploymentConfig.test.ts+4
......@@ -7,6 +7,7 @@ import {
77 managedEndpointHostname,
88 isManagedEndpointHostname,
99 managedEndpointTunnelName,
10 managedEndpointTunnelNamePrefix,
1011 relayOwnsManagedEndpointZone,
1112 RelayPublicDomainLabelTooLongError,
1213 relayPublicDomainForStage,
......@@ -84,6 +85,9 @@ describe("managed endpoint names", () => {
8485 expect(managedEndpointTunnelName("dev_julius", hash)).toBe(
8586 "t3coderelay-managedendpoint-dev-julius-abcdef0123456789",
8687 );
88 expect(managedEndpointTunnelNamePrefix("dev_julius")).toBe(
89 "t3coderelay-managedendpoint-dev-julius-",
90 );
8791 });
8892
8993 it("keeps the DNS label within the provider limit for long stage names", () => {
infra/relay/src/deploymentConfig.ts+5-1
......@@ -117,6 +117,10 @@ export function managedEndpointForHostname(hostname: string): RelayManagedEndpoi
117117 };
118118}
119119
120export function managedEndpointTunnelNamePrefix(stage: string): string {
121 return `${MANAGED_ENDPOINT_TUNNEL_PREFIX}-${relayStageSlug(stage)}-`;
122}
123
120124export function managedEndpointTunnelName(stage: string, hash: string): string {
121 return `${MANAGED_ENDPOINT_TUNNEL_PREFIX}-${relayStageSlug(stage)}-${stableSuffix(hash)}`;
125 return `${managedEndpointTunnelNamePrefix(stage)}${stableSuffix(hash)}`;
122126}
infra/relay/src/environments/EnvironmentConnector.test.ts+7
......@@ -188,7 +188,9 @@ function makeAllocations(
188188 tunnelName: "tunnel-name",
189189 dnsRecordId: "dns-record-id",
190190 readyAt: "2026-05-25T00:00:00.000Z",
191 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
191192 updatedAt: "2026-05-25T00:00:00.000Z",
193 generation: 1,
192194 },
193195): ManagedEndpointAllocations.ManagedEndpointAllocations["Service"] {
194196 return {
......@@ -197,7 +199,10 @@ function makeAllocations(
197199 recordTunnel: () => Effect.die("unused"),
198200 recordDns: () => Effect.die("unused"),
199201 markReady: () => Effect.die("unused"),
202 enableRecovery: () => Effect.die("unused"),
203 listByTunnelNames: () => Effect.die("unused"),
200204 claimRelease: () => Effect.die("unused"),
205 withClaimedTunnel: () => Effect.die("unused"),
201206 claimDeprovision: () => Effect.die("unused"),
202207 remove: () => Effect.die("unused"),
203208 removeClaimed: () => Effect.die("unused"),
......@@ -469,7 +474,9 @@ describe("EnvironmentConnector", () => {
469474 tunnelName: "tunnel-name",
470475 dnsRecordId: "dns-record-id",
471476 readyAt: null,
477 origin: null,
472478 updatedAt: "2026-05-25T00:00:00.000Z",
479 generation: 1,
473480 }),
474481 }),
475482 ),
infra/relay/src/environments/EnvironmentLinker.test.ts+3-1
......@@ -134,8 +134,9 @@ function testLayer(input?: {
134134 revokeForEnvironmentPublicKey: () => Effect.succeed(false),
135135 }),
136136 Layer.succeed(ManagedEndpointProvider.ManagedEndpointProvider, {
137 reconcileOrigin: () => Effect.succeed("ready"),
137138 prepareDeprovision: () => Effect.succeed(null),
138 deprovision: input?.deprovision ?? (() => Effect.void),
139 deprovision: input?.deprovision ?? (() => Effect.succeed(true)),
139140 release: () => Effect.succeed(true),
140141 provision: () =>
141142 Effect.succeed({
......@@ -241,6 +242,7 @@ describe("EnvironmentLinker", () => {
241242 deprovision: (input) =>
242243 Effect.sync(() => {
243244 deprovisionedEnvironmentId = input.environmentId;
245 return true;
244246 }),
245247 }),
246248 ),
infra/relay/src/environments/ManagedEndpointAllocations.test.ts+289-7
......@@ -1,6 +1,8 @@
11import { describe, expect, it } from "@effect/vitest";
22import * as Effect from "effect/Effect";
33import * as Layer from "effect/Layer";
4import * as Option from "effect/Option";
5import { PgDialect } from "drizzle-orm/pg-core";
46
57import * as RelayDb from "../db.ts";
68import { relayManagedEndpointAllocations } from "../persistence/schema.ts";
......@@ -10,17 +12,253 @@ const layerWithDb = (db: RelayDb.RelayDb["Service"]) =>
1012 ManagedEndpointAllocations.layer.pipe(Layer.provide(Layer.succeed(RelayDb.RelayDb, db)));
1113
1214describe("ManagedEndpointAllocations", () => {
15 it.effect("clears endpoint readiness and recovery only when the recorded tunnel changes", () => {
16 let updated:
17 | {
18 readonly tunnelId: string;
19 readonly readyAt: unknown;
20 readonly recoveryEnabledAt: unknown;
21 readonly recoveryEnvironmentPublicKey: unknown;
22 }
23 | undefined;
24 const fakeDb = {
25 update: (table: unknown) => {
26 expect(table).toBe(relayManagedEndpointAllocations);
27 return {
28 set: (values: {
29 readonly tunnelId: string;
30 readonly readyAt: unknown;
31 readonly recoveryEnabledAt: unknown;
32 readonly recoveryEnvironmentPublicKey: unknown;
33 }) => {
34 updated = values;
35 return {
36 where: () => ({
37 returning: () => Effect.succeed([{ generation: 8 }]),
38 }),
39 };
40 },
41 };
42 },
43 } as unknown as RelayDb.RelayDb["Service"];
44
45 return Effect.gen(function* () {
46 const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations;
47 expect(
48 yield* allocations.recordTunnel({
49 userId: "user-1",
50 environmentId: "environment-1",
51 tunnelId: "replacement-tunnel",
52 generation: 7,
53 }),
54 ).toBe(8);
55 expect(updated?.tunnelId).toBe("replacement-tunnel");
56 const query = new PgDialect().sqlToQuery(updated?.readyAt as never);
57 expect(query.sql).toBe(
58 'case when "relay_managed_endpoint_allocations"."tunnel_id" = $1 then "relay_managed_endpoint_allocations"."ready_at" else null end',
59 );
60 expect(query.params).toEqual(["replacement-tunnel"]);
61 for (const [column, value] of [
62 ["recovery_enabled_at", updated?.recoveryEnabledAt],
63 ["recovery_environment_public_key", updated?.recoveryEnvironmentPublicKey],
64 ] as const) {
65 const recoveryQuery = new PgDialect().sqlToQuery(value as never);
66 expect(recoveryQuery.sql).toBe(
67 `case when "relay_managed_endpoint_allocations"."tunnel_id" = $1 then "relay_managed_endpoint_allocations"."${column}" else null end`,
68 );
69 expect(recoveryQuery.params).toEqual(["replacement-tunnel"]);
70 }
71 }).pipe(Effect.provide(layerWithDb(fakeDb)));
72 });
73
74 it.effect("records recovery support and advances the allocation generation", () => {
75 let updated:
76 | {
77 readonly recoveryEnabledAt: string;
78 readonly recoveryEnvironmentPublicKey: string;
79 readonly updatedAt: string;
80 }
81 | undefined;
82 let condition: unknown;
83 const fakeDb = {
84 update: (table: unknown) => {
85 expect(table).toBe(relayManagedEndpointAllocations);
86 return {
87 set: (values: {
88 readonly recoveryEnabledAt: string;
89 readonly recoveryEnvironmentPublicKey: string;
90 readonly updatedAt: string;
91 }) => {
92 updated = values;
93 return {
94 where: (where: unknown) => {
95 condition = where;
96 return {
97 returning: () => Effect.succeed([{ environmentId: "environment-1" }]),
98 };
99 },
100 };
101 },
102 };
103 },
104 } as unknown as RelayDb.RelayDb["Service"];
105
106 return Effect.gen(function* () {
107 const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations;
108 expect(
109 yield* allocations.enableRecovery({
110 userId: "user-1",
111 environmentId: "environment-1",
112 tunnelId: "tunnel-1",
113 environmentPublicKey: "public-key",
114 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
115 }),
116 ).toBe(true);
117
118 expect(updated?.recoveryEnabledAt).toBe(updated?.updatedAt);
119 expect(updated?.recoveryEnabledAt).toBeDefined();
120 expect(updated?.recoveryEnvironmentPublicKey).toBe("public-key");
121 const query = new PgDialect().sqlToQuery(condition as never);
122 expect(query.sql).toContain('"relay_managed_endpoint_allocations"."tunnel_id"');
123 expect(query.sql).toContain('"relay_environment_links"."environment_public_key"');
124 expect(query.sql).toContain('"relay_environment_links"."endpoint_provider_kind"');
125 expect(query.sql).toContain('"relay_environment_links"."revoked_at" is null');
126 expect(query.sql).toContain("for update");
127 expect(query.params).toContain("tunnel-1");
128 expect(query.params).toContain("public-key");
129 expect(query.params).toContain("cloudflare_tunnel");
130 }).pipe(Effect.provide(layerWithDb(fakeDb)));
131 });
132
133 it.effect("rejects recovery when the tunnel or active link no longer matches", () => {
134 const fakeDb = {
135 update: () => ({
136 set: () => ({
137 where: () => ({
138 returning: () => Effect.succeed([]),
139 }),
140 }),
141 }),
142 } as unknown as RelayDb.RelayDb["Service"];
143
144 return Effect.gen(function* () {
145 const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations;
146 expect(
147 yield* allocations.enableRecovery({
148 userId: "user-1",
149 environmentId: "environment-1",
150 tunnelId: "missing-tunnel",
151 environmentPublicKey: "public-key",
152 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
153 }),
154 ).toBe(false);
155 }).pipe(Effect.provide(layerWithDb(fakeDb)));
156 });
157
158 it.effect("returns recovery support with tunnel allocation lookups", () => {
159 const base = {
160 userId: "user-1",
161 hostname: "environment.example.test",
162 tunnelName: "managed-tunnel",
163 dnsRecordId: "dns-1",
164 readyAt: "2026-08-25T12:00:00.000Z",
165 updatedAt: "2026-08-25T12:00:00.000Z",
166 generation: 1,
167 };
168 const fakeDb = {
169 select: () => ({
170 from: (table: unknown) => {
171 expect(table).toBe(relayManagedEndpointAllocations);
172 return {
173 leftJoin: () => ({
174 where: () =>
175 Effect.succeed([
176 {
177 ...base,
178 environmentId: "environment-1",
179 tunnelId: "tunnel-1",
180 recoveryEnabledAt: "2026-08-25T12:00:00.000Z",
181 recoveryEnvironmentPublicKey: "current-key",
182 linkedEnvironmentPublicKey: "current-key",
183 },
184 {
185 ...base,
186 environmentId: "environment-2",
187 tunnelId: "tunnel-2",
188 recoveryEnabledAt: null,
189 recoveryEnvironmentPublicKey: null,
190 linkedEnvironmentPublicKey: "current-key",
191 },
192 {
193 ...base,
194 environmentId: "environment-3",
195 tunnelId: "tunnel-3",
196 recoveryEnabledAt: "2026-08-25T12:00:00.000Z",
197 recoveryEnvironmentPublicKey: "old-key",
198 linkedEnvironmentPublicKey: "new-key",
199 },
200 ]),
201 }),
202 };
203 },
204 }),
205 } as unknown as RelayDb.RelayDb["Service"];
206
207 return Effect.gen(function* () {
208 const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations;
209 const result = yield* allocations.listByTunnelNames([
210 "first-tunnel",
211 "second-tunnel",
212 "third-tunnel",
213 ]);
214
215 expect(result.map((entry) => [entry.tunnelId, entry.recoveryEnabled])).toEqual([
216 ["tunnel-1", true],
217 ["tunnel-2", false],
218 ["tunnel-3", false],
219 ]);
220 }).pipe(Effect.provide(layerWithDb(fakeDb)));
221 });
222
223 it.effect("skips the database for an empty tunnel lookup", () =>
224 Effect.gen(function* () {
225 const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations;
226 expect(yield* allocations.listByTunnelNames([])).toEqual([]);
227 }).pipe(Effect.provide(layerWithDb({} as RelayDb.RelayDb["Service"]))),
228 );
229
230 it.effect("splits large tunnel lookups into bounded database queries", () => {
231 const batchSizes: number[] = [];
232 const fakeDb = {
233 select: () => ({
234 from: () => ({
235 leftJoin: () => ({
236 where: (condition: unknown) => {
237 batchSizes.push(new PgDialect().sqlToQuery(condition as never).params.length);
238 return Effect.succeed([]);
239 },
240 }),
241 }),
242 }),
243 } as unknown as RelayDb.RelayDb["Service"];
244
245 return Effect.gen(function* () {
246 const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations;
247 const names = Array.from({ length: 1_001 }, (_, index) => `tunnel-${index}`);
248 expect(yield* allocations.listByTunnelNames(names)).toEqual([]);
249 expect(batchSizes).toEqual([500, 500, 1]);
250 }).pipe(Effect.provide(layerWithDb(fakeDb)));
251 });
252
13253 it.effect("returns a claim generation only when deprovision wins the allocation CAS", () => {
14 let claimedAt: string | undefined;
15254 const fakeDb = {
16255 update: (table: unknown) => {
17256 expect(table).toBe(relayManagedEndpointAllocations);
18257 return {
19 set: (values: { readonly updatedAt: string }) => {
20 claimedAt = values.updatedAt;
258 set: (_values: { readonly updatedAt: string }) => {
21259 return {
22260 where: () => ({
23 returning: () => Effect.succeed([{ userId: "user-1" }]),
261 returning: () => Effect.succeed([{ generation: 8 }]),
24262 }),
25263 };
26264 },
......@@ -33,14 +271,58 @@ describe("ManagedEndpointAllocations", () => {
33271 const generation = yield* allocations.claimDeprovision({
34272 userId: "user-1",
35273 environmentId: "environment-1",
36 updatedAt: "captured-generation",
274 generation: 7,
37275 });
38276
39 expect(generation).toBe(claimedAt);
277 expect(generation).toBe(8);
40278 expect(generation).not.toBeNull();
41279 }).pipe(Effect.provide(layerWithDb(fakeDb)));
42280 });
43281
282 it.effect("holds the claimed allocation row while deleting its tunnel", () => {
283 const operations: string[] = [];
284 const fakeDb = {
285 $client: {
286 withTransaction: <A, E, R>(effect: Effect.Effect<A, E, R>) =>
287 Effect.sync(() => {
288 operations.push("transaction");
289 }).pipe(Effect.andThen(effect)),
290 },
291 select: () => ({
292 from: () => ({
293 where: () => ({
294 limit: () => ({
295 for: (strength: string) =>
296 Effect.sync(() => {
297 operations.push(`lock:${strength}`);
298 return [{ generation: 7 }];
299 }),
300 }),
301 }),
302 }),
303 }),
304 } as unknown as RelayDb.RelayDb["Service"];
305
306 return Effect.gen(function* () {
307 const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations;
308 const result = yield* allocations.withClaimedTunnel(
309 {
310 userId: "user-1",
311 environmentId: "environment-1",
312 tunnelId: "tunnel-1",
313 generation: 7,
314 },
315 Effect.sync(() => {
316 operations.push("delete");
317 return true;
318 }),
319 );
320
321 expect(Option.getOrNull(result)).toBe(true);
322 expect(operations).toEqual(["transaction", "lock:update", "delete"]);
323 }).pipe(Effect.provide(layerWithDb(fakeDb)));
324 });
325
44326 it.effect("does not remove an allocation superseded after a deprovision claim", () => {
45327 const fakeDb = {
46328 delete: (table: unknown) => {
......@@ -59,7 +341,7 @@ describe("ManagedEndpointAllocations", () => {
59341 yield* allocations.removeClaimed({
60342 userId: "user-1",
61343 environmentId: "environment-1",
62 updatedAt: "outdated-claim-generation",
344 generation: 7,
63345 }),
64346 ).toBe(false);
65347 }).pipe(Effect.provide(layerWithDb(fakeDb)));
infra/relay/src/environments/ManagedEndpointAllocations.ts+270-34
......@@ -1,14 +1,17 @@
1import type { RelayManagedEndpoint } from "@t3tools/contracts/relay";
2import { and, eq } from "drizzle-orm";
1import type { RelayManagedEndpoint, RelayManagedEndpointOrigin } from "@t3tools/contracts/relay";
2import { and, eq, exists, inArray, isNull, sql } from "drizzle-orm";
3import { QueryBuilder } from "drizzle-orm/pg-core";
34import * as Context from "effect/Context";
45import * as DateTime from "effect/DateTime";
56import * as Effect from "effect/Effect";
67import * as Layer from "effect/Layer";
8import * as Option from "effect/Option";
79import * as Schema from "effect/Schema";
10import * as SqlError from "effect/unstable/sql/SqlError";
811
912import * as RelayDb from "../db.ts";
1013import { isManagedEndpointHostname, managedEndpointForHostname } from "../deploymentConfig.ts";
11import { relayManagedEndpointAllocations } from "../persistence/schema.ts";
14import { relayEnvironmentLinks, relayManagedEndpointAllocations } from "../persistence/schema.ts";
1215
1316export interface ManagedEndpointAllocation {
1417 readonly userId: string;
......@@ -18,13 +21,17 @@ export interface ManagedEndpointAllocation {
1821 readonly tunnelName: string;
1922 readonly dnsRecordId: string | null;
2023 readonly readyAt: string | null;
21 /**
22 * Doubles as the allocation's generation marker: every mutation rewrites it,
23 * so `claimRelease` can detect a provision that raced a release.
24 */
24 readonly origin: RelayManagedEndpointOrigin | null;
2525 readonly updatedAt: string;
26 readonly generation: number;
2627}
2728
29export interface ManagedEndpointTunnelAllocation extends ManagedEndpointAllocation {
30 readonly recoveryEnabled: boolean;
31}
32
33export const MANAGED_ENDPOINT_ALLOCATION_LOOKUP_BATCH_SIZE = 500;
34
2835export function resolveReadyManagedEndpoint(input: {
2936 readonly allocation: ManagedEndpointAllocation;
3037 readonly baseDomain: string | undefined;
......@@ -50,6 +57,9 @@ export class ManagedEndpointAllocationPersistenceError extends Schema.TaggedErro
5057 "record-tunnel",
5158 "record-dns",
5259 "mark-ready",
60 "enable-recovery",
61 "list-tunnels",
62 "lock-tunnel",
5363 "claim-release",
5464 "claim-deprovision",
5565 "remove",
......@@ -82,23 +92,38 @@ interface ReserveManagedEndpointAllocationInput extends ManagedEndpointAllocatio
8292
8393interface RecordManagedEndpointTunnelInput extends ManagedEndpointAllocationKey {
8494 readonly tunnelId: string;
95 readonly generation: number;
8596}
8697
8798interface RecordManagedEndpointDnsInput extends ManagedEndpointAllocationKey {
8899 readonly dnsRecordId: string;
100 readonly tunnelId: string;
101 readonly generation: number;
102}
103
104interface MarkManagedEndpointReadyInput extends ManagedEndpointAllocationKey {
105 readonly tunnelId: string;
106 readonly generation: number;
107 readonly origin: RelayManagedEndpointOrigin;
89108}
90109
91110interface ClaimManagedEndpointReleaseInput extends ManagedEndpointAllocationKey {
92111 readonly tunnelId: string;
93 readonly updatedAt: string;
112 readonly generation: number;
113}
114
115interface EnableManagedEndpointRecoveryInput extends ManagedEndpointAllocationKey {
116 readonly tunnelId: string;
117 readonly environmentPublicKey: string;
118 readonly origin: RelayManagedEndpointOrigin;
94119}
95120
96121interface ClaimManagedEndpointDeprovisionInput extends ManagedEndpointAllocationKey {
97 readonly updatedAt: string;
122 readonly generation: number;
98123}
99124
100125interface RemoveClaimedManagedEndpointAllocationInput extends ManagedEndpointAllocationKey {
101 readonly updatedAt: string;
126 readonly generation: number;
102127}
103128
104129export class ManagedEndpointAllocations extends Context.Service<
......@@ -112,23 +137,36 @@ export class ManagedEndpointAllocations extends Context.Service<
112137 ) => Effect.Effect<ManagedEndpointAllocation, ManagedEndpointAllocationPersistenceError>;
113138 readonly recordTunnel: (
114139 input: RecordManagedEndpointTunnelInput,
115 ) => Effect.Effect<void, ManagedEndpointAllocationPersistenceError>;
140 ) => Effect.Effect<number | null, ManagedEndpointAllocationPersistenceError>;
116141 readonly recordDns: (
117142 input: RecordManagedEndpointDnsInput,
118 ) => Effect.Effect<void, ManagedEndpointAllocationPersistenceError>;
143 ) => Effect.Effect<number | null, ManagedEndpointAllocationPersistenceError>;
119144 readonly markReady: (
120 input: ManagedEndpointAllocationKey,
121 ) => Effect.Effect<void, ManagedEndpointAllocationPersistenceError>;
145 input: MarkManagedEndpointReadyInput,
146 ) => Effect.Effect<boolean, ManagedEndpointAllocationPersistenceError>;
147 readonly enableRecovery: (
148 input: EnableManagedEndpointRecoveryInput,
149 ) => Effect.Effect<boolean, ManagedEndpointAllocationPersistenceError>;
150 readonly listByTunnelNames: (
151 tunnelNames: ReadonlyArray<string>,
152 ) => Effect.Effect<
153 ReadonlyArray<ManagedEndpointTunnelAllocation>,
154 ManagedEndpointAllocationPersistenceError
155 >;
122156 /**
123157 * Atomically claims the right to delete the allocation's tunnel: succeeds
124158 * only while the recorded tunnel and generation still match what the
125 * caller loaded. A concurrent provision rewrites `updatedAt` when it
159 * caller loaded. A concurrent provision increments `generation` when it
126160 * records its tunnel, which makes a stale claim fail and keeps the freshly
127161 * issued tunnel alive.
128162 */
129163 readonly claimRelease: (
130164 input: ClaimManagedEndpointReleaseInput,
131 ) => Effect.Effect<boolean, ManagedEndpointAllocationPersistenceError>;
165 ) => Effect.Effect<number | null, ManagedEndpointAllocationPersistenceError>;
166 readonly withClaimedTunnel: <A, E, R>(
167 input: ClaimManagedEndpointReleaseInput,
168 effect: Effect.Effect<A, E, R>,
169 ) => Effect.Effect<Option.Option<A>, E | ManagedEndpointAllocationPersistenceError, R>;
132170 /**
133171 * Claims the complete allocation for teardown only if its generation still
134172 * matches the snapshot captured by the unlink operation.
......@@ -138,7 +176,7 @@ export class ManagedEndpointAllocations extends Context.Service<
138176 */
139177 readonly claimDeprovision: (
140178 input: ClaimManagedEndpointDeprovisionInput,
141 ) => Effect.Effect<string | null, ManagedEndpointAllocationPersistenceError>;
179 ) => Effect.Effect<number | null, ManagedEndpointAllocationPersistenceError>;
142180 readonly remove: (
143181 input: ManagedEndpointAllocationKey,
144182 ) => Effect.Effect<void, ManagedEndpointAllocationPersistenceError>;
......@@ -156,7 +194,9 @@ const allocationSelection = {
156194 tunnelName: relayManagedEndpointAllocations.tunnelName,
157195 dnsRecordId: relayManagedEndpointAllocations.dnsRecordId,
158196 readyAt: relayManagedEndpointAllocations.readyAt,
197 origin: relayManagedEndpointAllocations.origin,
159198 updatedAt: relayManagedEndpointAllocations.updatedAt,
199 generation: relayManagedEndpointAllocations.generation,
160200};
161201
162202const whereAllocation = (input: ManagedEndpointAllocationKey) =>
......@@ -248,14 +288,28 @@ export const make = Effect.gen(function* () {
248288 recordTunnel: Effect.fn("relay.managed_endpoint_allocations.record_tunnel")(function* (
249289 input: RecordManagedEndpointTunnelInput,
250290 ) {
251 yield* db
291 return yield* db
252292 .update(relayManagedEndpointAllocations)
253293 .set({
254294 tunnelId: input.tunnelId,
295 readyAt: sql`case when ${relayManagedEndpointAllocations.tunnelId} = ${input.tunnelId} then ${relayManagedEndpointAllocations.readyAt} else null end`,
296 origin: sql`case when ${relayManagedEndpointAllocations.tunnelId} = ${input.tunnelId} then ${relayManagedEndpointAllocations.origin} else null end`,
297 // Recovery registration is per tunnel: a replacement must register
298 // again before the reaper may treat it as recoverable.
299 recoveryEnabledAt: sql`case when ${relayManagedEndpointAllocations.tunnelId} = ${input.tunnelId} then ${relayManagedEndpointAllocations.recoveryEnabledAt} else null end`,
300 recoveryEnvironmentPublicKey: sql`case when ${relayManagedEndpointAllocations.tunnelId} = ${input.tunnelId} then ${relayManagedEndpointAllocations.recoveryEnvironmentPublicKey} else null end`,
255301 updatedAt: DateTime.formatIso(yield* DateTime.now),
302 generation: sql`${relayManagedEndpointAllocations.generation} + 1`,
256303 })
257 .where(whereAllocation(input))
304 .where(
305 and(
306 whereAllocation(input),
307 eq(relayManagedEndpointAllocations.generation, input.generation),
308 ),
309 )
310 .returning({ generation: relayManagedEndpointAllocations.generation })
258311 .pipe(
312 Effect.map((rows) => rows[0]?.generation ?? null),
259313 Effect.mapError(
260314 (cause) =>
261315 new ManagedEndpointAllocationPersistenceError({
......@@ -270,14 +324,23 @@ export const make = Effect.gen(function* () {
270324 recordDns: Effect.fn("relay.managed_endpoint_allocations.record_dns")(function* (
271325 input: RecordManagedEndpointDnsInput,
272326 ) {
273 yield* db
327 return yield* db
274328 .update(relayManagedEndpointAllocations)
275329 .set({
276330 dnsRecordId: input.dnsRecordId,
277331 updatedAt: DateTime.formatIso(yield* DateTime.now),
332 generation: sql`${relayManagedEndpointAllocations.generation} + 1`,
278333 })
279 .where(whereAllocation(input))
334 .where(
335 and(
336 whereAllocation(input),
337 eq(relayManagedEndpointAllocations.tunnelId, input.tunnelId),
338 eq(relayManagedEndpointAllocations.generation, input.generation),
339 ),
340 )
341 .returning({ generation: relayManagedEndpointAllocations.generation })
280342 .pipe(
343 Effect.map((rows) => rows[0]?.generation ?? null),
281344 Effect.mapError(
282345 (cause) =>
283346 new ManagedEndpointAllocationPersistenceError({
......@@ -290,17 +353,27 @@ export const make = Effect.gen(function* () {
290353 );
291354 }),
292355 markReady: Effect.fn("relay.managed_endpoint_allocations.mark_ready")(function* (
293 input: ManagedEndpointAllocationKey,
356 input: MarkManagedEndpointReadyInput,
294357 ) {
295358 const now = DateTime.formatIso(yield* DateTime.now);
296 yield* db
359 return yield* db
297360 .update(relayManagedEndpointAllocations)
298361 .set({
299362 readyAt: now,
363 origin: input.origin,
300364 updatedAt: now,
365 generation: sql`${relayManagedEndpointAllocations.generation} + 1`,
301366 })
302 .where(whereAllocation(input))
367 .where(
368 and(
369 whereAllocation(input),
370 eq(relayManagedEndpointAllocations.tunnelId, input.tunnelId),
371 eq(relayManagedEndpointAllocations.generation, input.generation),
372 ),
373 )
374 .returning({ environmentId: relayManagedEndpointAllocations.environmentId })
303375 .pipe(
376 Effect.map((rows) => rows.length > 0),
304377 Effect.mapError(
305378 (cause) =>
306379 new ManagedEndpointAllocationPersistenceError({
......@@ -312,6 +385,124 @@ export const make = Effect.gen(function* () {
312385 ),
313386 );
314387 }),
388 enableRecovery: Effect.fn("relay.managed_endpoint_allocations.enable_recovery")(function* (
389 input: EnableManagedEndpointRecoveryInput,
390 ) {
391 const now = DateTime.formatIso(yield* DateTime.now);
392 return yield* db
393 .update(relayManagedEndpointAllocations)
394 .set({
395 recoveryEnabledAt: now,
396 recoveryEnvironmentPublicKey: input.environmentPublicKey,
397 updatedAt: now,
398 generation: sql`${relayManagedEndpointAllocations.generation} + 1`,
399 })
400 .where(
401 and(
402 whereAllocation(input),
403 eq(relayManagedEndpointAllocations.tunnelId, input.tunnelId),
404 eq(relayManagedEndpointAllocations.origin, input.origin),
405 exists(
406 new QueryBuilder()
407 .select({ userId: relayEnvironmentLinks.userId })
408 .from(relayEnvironmentLinks)
409 .where(
410 and(
411 eq(relayEnvironmentLinks.userId, input.userId),
412 eq(relayEnvironmentLinks.environmentId, input.environmentId),
413 eq(relayEnvironmentLinks.environmentPublicKey, input.environmentPublicKey),
414 eq(relayEnvironmentLinks.endpointProviderKind, "cloudflare_tunnel"),
415 isNull(relayEnvironmentLinks.revokedAt),
416 ),
417 )
418 .for("update"),
419 ),
420 ),
421 )
422 .returning({ environmentId: relayManagedEndpointAllocations.environmentId })
423 .pipe(
424 Effect.map((rows) => rows.length > 0),
425 Effect.mapError(
426 (cause) =>
427 new ManagedEndpointAllocationPersistenceError({
428 operation: "enable-recovery",
429 stage: "database-request",
430 ...input,
431 cause,
432 }),
433 ),
434 );
435 }),
436 listByTunnelNames: Effect.fn("relay.managed_endpoint_allocations.list_by_tunnel_names")(
437 function* (tunnelNames: ReadonlyArray<string>) {
438 if (tunnelNames.length === 0) {
439 return [];
440 }
441 const batches = Array.from(
442 { length: Math.ceil(tunnelNames.length / MANAGED_ENDPOINT_ALLOCATION_LOOKUP_BATCH_SIZE) },
443 (_, index) =>
444 tunnelNames.slice(
445 index * MANAGED_ENDPOINT_ALLOCATION_LOOKUP_BATCH_SIZE,
446 (index + 1) * MANAGED_ENDPOINT_ALLOCATION_LOOKUP_BATCH_SIZE,
447 ),
448 );
449 const results = yield* Effect.forEach(
450 batches,
451 (batch) =>
452 db
453 .select({
454 ...allocationSelection,
455 recoveryEnabledAt: relayManagedEndpointAllocations.recoveryEnabledAt,
456 recoveryEnvironmentPublicKey:
457 relayManagedEndpointAllocations.recoveryEnvironmentPublicKey,
458 linkedEnvironmentPublicKey: relayEnvironmentLinks.environmentPublicKey,
459 })
460 .from(relayManagedEndpointAllocations)
461 .leftJoin(
462 relayEnvironmentLinks,
463 and(
464 eq(relayEnvironmentLinks.userId, relayManagedEndpointAllocations.userId),
465 eq(
466 relayEnvironmentLinks.environmentId,
467 relayManagedEndpointAllocations.environmentId,
468 ),
469 isNull(relayEnvironmentLinks.revokedAt),
470 ),
471 )
472 .where(inArray(relayManagedEndpointAllocations.tunnelName, batch))
473 .pipe(
474 Effect.map((rows) =>
475 rows.map(
476 ({
477 recoveryEnabledAt,
478 recoveryEnvironmentPublicKey,
479 linkedEnvironmentPublicKey,
480 ...allocation
481 }) => ({
482 ...allocation,
483 recoveryEnabled:
484 recoveryEnabledAt !== null &&
485 recoveryEnvironmentPublicKey !== null &&
486 recoveryEnvironmentPublicKey === linkedEnvironmentPublicKey,
487 }),
488 ),
489 ),
490 Effect.mapError(
491 (cause) =>
492 new ManagedEndpointAllocationPersistenceError({
493 operation: "list-tunnels",
494 stage: "database-request",
495 userId: "*",
496 environmentId: "*",
497 cause,
498 }),
499 ),
500 ),
501 { concurrency: 1 },
502 );
503 return results.flat();
504 },
505 ),
315506 claimRelease: Effect.fn("relay.managed_endpoint_allocations.claim_release")(function* (
316507 input: ClaimManagedEndpointReleaseInput,
317508 ) {
......@@ -319,17 +510,18 @@ export const make = Effect.gen(function* () {
319510 .update(relayManagedEndpointAllocations)
320511 .set({
321512 updatedAt: DateTime.formatIso(yield* DateTime.now),
513 generation: sql`${relayManagedEndpointAllocations.generation} + 1`,
322514 })
323515 .where(
324516 and(
325517 whereAllocation(input),
326518 eq(relayManagedEndpointAllocations.tunnelId, input.tunnelId),
327 eq(relayManagedEndpointAllocations.updatedAt, input.updatedAt),
519 eq(relayManagedEndpointAllocations.generation, input.generation),
328520 ),
329521 )
330 .returning({ userId: relayManagedEndpointAllocations.userId })
522 .returning({ generation: relayManagedEndpointAllocations.generation })
331523 .pipe(
332 Effect.map((rows) => rows.length > 0),
524 Effect.map((rows) => rows[0]?.generation ?? null),
333525 Effect.mapError(
334526 (cause) =>
335527 new ManagedEndpointAllocationPersistenceError({
......@@ -344,22 +536,66 @@ export const make = Effect.gen(function* () {
344536 );
345537 return claimed;
346538 }),
539 withClaimedTunnel: Effect.fn("relay.managed_endpoint_allocations.with_claimed_tunnel")(
540 function* <A, E, R>(
541 input: ClaimManagedEndpointReleaseInput,
542 effect: Effect.Effect<A, E, R>,
543 ): Effect.fn.Return<Option.Option<A>, E | ManagedEndpointAllocationPersistenceError, R> {
544 const lockError = (cause: unknown) =>
545 new ManagedEndpointAllocationPersistenceError({
546 operation: "lock-tunnel",
547 stage: "database-request",
548 userId: input.userId,
549 environmentId: input.environmentId,
550 tunnelId: input.tunnelId,
551 cause,
552 });
553 return yield* db.$client
554 .withTransaction(
555 db
556 .select({ generation: relayManagedEndpointAllocations.generation })
557 .from(relayManagedEndpointAllocations)
558 .where(
559 and(
560 whereAllocation(input),
561 eq(relayManagedEndpointAllocations.tunnelId, input.tunnelId),
562 eq(relayManagedEndpointAllocations.generation, input.generation),
563 ),
564 )
565 .limit(1)
566 .for("update")
567 .pipe(
568 Effect.mapError(lockError),
569 Effect.flatMap((rows) =>
570 rows.length === 0
571 ? Effect.succeed(Option.none())
572 : effect.pipe(Effect.map(Option.some)),
573 ),
574 ),
575 )
576 .pipe(
577 Effect.mapError((cause) => (SqlError.isSqlError(cause) ? lockError(cause) : cause)),
578 );
579 },
580 ),
347581 claimDeprovision: Effect.fn("relay.managed_endpoint_allocations.claim_deprovision")(function* (
348582 input: ClaimManagedEndpointDeprovisionInput,
349583 ) {
350 const claimedAt = DateTime.formatIso(yield* DateTime.now);
351584 const claimed = yield* db
352585 .update(relayManagedEndpointAllocations)
353 .set({ updatedAt: claimedAt })
586 .set({
587 updatedAt: DateTime.formatIso(yield* DateTime.now),
588 generation: sql`${relayManagedEndpointAllocations.generation} + 1`,
589 })
354590 .where(
355591 and(
356592 whereAllocation(input),
357 eq(relayManagedEndpointAllocations.updatedAt, input.updatedAt),
593 eq(relayManagedEndpointAllocations.generation, input.generation),
358594 ),
359595 )
360 .returning({ userId: relayManagedEndpointAllocations.userId })
596 .returning({ generation: relayManagedEndpointAllocations.generation })
361597 .pipe(
362 Effect.map((rows) => rows.length > 0),
598 Effect.map((rows) => rows[0]?.generation ?? null),
363599 Effect.mapError(
364600 (cause) =>
365601 new ManagedEndpointAllocationPersistenceError({
......@@ -371,7 +607,7 @@ export const make = Effect.gen(function* () {
371607 }),
372608 ),
373609 );
374 return claimed ? claimedAt : null;
610 return claimed;
375611 }),
376612 remove: Effect.fn("relay.managed_endpoint_allocations.remove")(function* (
377613 input: ManagedEndpointAllocationKey,
......@@ -399,7 +635,7 @@ export const make = Effect.gen(function* () {
399635 .where(
400636 and(
401637 whereAllocation(input),
402 eq(relayManagedEndpointAllocations.updatedAt, input.updatedAt),
638 eq(relayManagedEndpointAllocations.generation, input.generation),
403639 ),
404640 )
405641 .returning({ userId: relayManagedEndpointAllocations.userId })
infra/relay/src/environments/ManagedEndpointProvider.test.ts+546-10
......@@ -6,6 +6,7 @@ import * as Alchemy from "alchemy";
66import * as Cloudflare from "alchemy/Cloudflare";
77import * as Effect from "effect/Effect";
88import * as Layer from "effect/Layer";
9import * as Option from "effect/Option";
910import * as Redacted from "effect/Redacted";
1011
1112import * as RelayConfiguration from "../Config.ts";
......@@ -33,7 +34,7 @@ const config = RelayConfiguration.RelayConfiguration.of({
3334});
3435
3536interface TunnelCall {
36 readonly operation: "list" | "create" | "putConfiguration" | "getToken" | "delete";
37 readonly operation: "get" | "list" | "create" | "putConfiguration" | "getToken" | "delete";
3738 readonly input: unknown;
3839}
3940
......@@ -62,6 +63,16 @@ function allocationKey(input: { readonly userId: string; readonly environmentId:
6263
6364function makeTunnelClient(calls: TunnelCall[] = []) {
6465 return ManagedEndpointProvider.ManagedEndpointTunnelClient.of({
66 get: (tunnelId) =>
67 Effect.sync(() => {
68 calls.push({ operation: "get", input: tunnelId });
69 return {
70 id: tunnelId,
71 name: "managed-tunnel",
72 status: "down",
73 connsInactiveAt: "2026-06-01T00:00:00.000Z",
74 };
75 }),
6576 list: (request) =>
6677 Effect.sync(() => {
6778 calls.push({ operation: "list", input: request });
......@@ -91,6 +102,23 @@ function makeTunnelClient(calls: TunnelCall[] = []) {
91102function makePersistentTunnelClient(calls: TunnelCall[] = []) {
92103 let tunnel: { readonly id: string; readonly name: string } | null = null;
93104 return ManagedEndpointProvider.ManagedEndpointTunnelClient.of({
105 get: (tunnelId) =>
106 Effect.suspend(() => {
107 calls.push({ operation: "get", input: tunnelId });
108 return tunnel === null
109 ? Effect.fail(
110 new ManagedEndpointProvider.ManagedEndpointTunnelClientError({
111 operation: "get",
112 tunnelId,
113 cause: { _tag: "NotFound" },
114 }),
115 )
116 : Effect.succeed({
117 ...tunnel,
118 status: "down",
119 connsInactiveAt: "2026-06-01T00:00:00.000Z",
120 });
121 }),
94122 list: (request) =>
95123 Effect.sync(() => {
96124 calls.push({ operation: "list", input: request });
......@@ -159,6 +187,7 @@ function makeDnsClient(
159187
160188function makeAllocations(calls: AllocationCall[] = []) {
161189 const allocations = new Map<string, ManagedEndpointAllocations.ManagedEndpointAllocation>();
190 const recoveryEnabled = new Set<string>();
162191 let generation = 0;
163192 const mutate = (
164193 key: string,
......@@ -168,7 +197,11 @@ function makeAllocations(calls: AllocationCall[] = []) {
168197 ) => {
169198 const allocation = allocations.get(key);
170199 if (allocation !== undefined) {
171 allocations.set(key, { ...change(allocation), updatedAt: `generation-${++generation}` });
200 allocations.set(key, {
201 ...change(allocation),
202 generation: allocation.generation + 1,
203 updatedAt: `generation-${++generation}`,
204 });
172205 }
173206 };
174207 return ManagedEndpointAllocations.ManagedEndpointAllocations.of({
......@@ -185,7 +218,9 @@ function makeAllocations(calls: AllocationCall[] = []) {
185218 tunnelId: null,
186219 dnsRecordId: null,
187220 readyAt: null,
221 origin: null,
188222 updatedAt: `generation-${++generation}`,
223 generation: 0,
189224 };
190225 allocations.set(allocationKey(input), allocation);
191226 return allocation;
......@@ -193,27 +228,62 @@ function makeAllocations(calls: AllocationCall[] = []) {
193228 recordTunnel: (input) =>
194229 Effect.sync(() => {
195230 calls.push({ operation: "recordTunnel", input });
231 const current = allocations.get(allocationKey(input));
232 if (current?.generation !== input.generation) {
233 return null;
234 }
196235 mutate(allocationKey(input), (allocation) => ({
197236 ...allocation,
198237 tunnelId: input.tunnelId,
238 readyAt: allocation.tunnelId === input.tunnelId ? allocation.readyAt : null,
199239 }));
240 return allocations.get(allocationKey(input))?.generation ?? null;
200241 }),
201242 recordDns: (input) =>
202243 Effect.sync(() => {
203244 calls.push({ operation: "recordDns", input });
245 const current = allocations.get(allocationKey(input));
246 if (current?.generation !== input.generation || current.tunnelId !== input.tunnelId) {
247 return null;
248 }
204249 mutate(allocationKey(input), (allocation) => ({
205250 ...allocation,
206251 dnsRecordId: input.dnsRecordId,
207252 }));
253 return allocations.get(allocationKey(input))?.generation ?? null;
208254 }),
209255 markReady: (input) =>
210256 Effect.sync(() => {
211257 calls.push({ operation: "markReady", input });
258 const current = allocations.get(allocationKey(input));
259 if (current?.generation !== input.generation || current.tunnelId !== input.tunnelId) {
260 return false;
261 }
212262 mutate(allocationKey(input), (allocation) => ({
213263 ...allocation,
214264 readyAt: "2026-06-02T00:00:00.000Z",
265 origin: input.origin,
215266 }));
267 return true;
268 }),
269 enableRecovery: (input) =>
270 Effect.sync(() => {
271 const allocation = allocations.get(allocationKey(input));
272 if (allocation?.tunnelId !== input.tunnelId) {
273 return false;
274 }
275 recoveryEnabled.add(allocationKey(input));
276 return true;
216277 }),
278 listByTunnelNames: (tunnelNames) =>
279 Effect.sync(() =>
280 [...allocations.values()]
281 .filter((allocation) => tunnelNames.includes(allocation.tunnelName))
282 .map((allocation) => ({
283 ...allocation,
284 recoveryEnabled: recoveryEnabled.has(allocationKey(allocation)),
285 })),
286 ),
217287 claimRelease: (input) =>
218288 Effect.sync(() => {
219289 calls.push({ operation: "claimRelease", input });
......@@ -221,22 +291,29 @@ function makeAllocations(calls: AllocationCall[] = []) {
221291 if (
222292 allocation === undefined ||
223293 allocation.tunnelId !== input.tunnelId ||
224 allocation.updatedAt !== input.updatedAt
294 allocation.generation !== input.generation
225295 ) {
226 return false;
296 return null;
227297 }
228298 mutate(allocationKey(input), (current) => current);
229 return true;
299 return allocations.get(allocationKey(input))?.generation ?? null;
300 }),
301 withClaimedTunnel: (input, effect) =>
302 Effect.suspend(() => {
303 const current = allocations.get(allocationKey(input));
304 return current?.tunnelId === input.tunnelId && current.generation === input.generation
305 ? effect.pipe(Effect.map(Option.some))
306 : Effect.succeed(Option.none());
230307 }),
231308 claimDeprovision: (input) =>
232309 Effect.sync(() => {
233310 calls.push({ operation: "claimDeprovision", input });
234311 const allocation = allocations.get(allocationKey(input));
235 if (allocation === undefined || allocation.updatedAt !== input.updatedAt) {
312 if (allocation === undefined || allocation.generation !== input.generation) {
236313 return null;
237314 }
238315 mutate(allocationKey(input), (current) => current);
239 return allocations.get(allocationKey(input))?.updatedAt ?? null;
316 return allocations.get(allocationKey(input))?.generation ?? null;
240317 }),
241318 remove: (input) =>
242319 Effect.sync(() => {
......@@ -247,7 +324,7 @@ function makeAllocations(calls: AllocationCall[] = []) {
247324 Effect.sync(() => {
248325 calls.push({ operation: "removeClaimed", input });
249326 const allocation = allocations.get(allocationKey(input));
250 if (allocation === undefined || allocation.updatedAt !== input.updatedAt) {
327 if (allocation === undefined || allocation.generation !== input.generation) {
251328 return false;
252329 }
253330 allocations.delete(allocationKey(input));
......@@ -306,6 +383,7 @@ function expectedManagedTunnelName(environmentId: string, userId = "user_ABC"):
306383describe("ManagedEndpointProvider", () => {
307384 it.effect("does not require the deployment RuntimeContext when building the Worker layer", () => {
308385 const tunnelClient = {
386 get: () => Effect.succeed({ id: "tunnel-id", name: "managed-tunnel" }),
309387 list: () => Effect.succeed({ result: [] }),
310388 create: (request: { readonly name: string }) =>
311389 Effect.succeed({ id: "tunnel-id", name: request.name }),
......@@ -731,7 +809,8 @@ describe("ManagedEndpointProvider", () => {
731809 }).pipe(Effect.andThen(Effect.fail(failure))),
732810 deleteRecord: () => Effect.void,
733811 });
734 const layer = providerLayer(makePersistentTunnelClient(), dnsClient, makeAllocations());
812 const allocations = makeAllocations();
813 const layer = providerLayer(makePersistentTunnelClient(), dnsClient, allocations);
735814
736815 return Effect.gen(function* () {
737816 const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
......@@ -754,6 +833,10 @@ describe("ManagedEndpointProvider", () => {
754833 "createRecord",
755834 "updateRecord",
756835 ]);
836 expect(yield* allocations.get(request)).toMatchObject({
837 tunnelId: "tunnel-id",
838 readyAt: "2026-06-02T00:00:00.000Z",
839 });
757840 }).pipe(Effect.provide(layer));
758841 });
759842
......@@ -914,7 +997,7 @@ describe("ManagedEndpointProvider", () => {
914997 // longer matches what the release loaded, so the claim fails.
915998 const outdated = ManagedEndpointAllocations.ManagedEndpointAllocations.of({
916999 ...allocations,
917 claimRelease: () => Effect.succeed(false),
1000 claimRelease: () => Effect.succeed(null),
9181001 });
9191002 const layer = providerLayer(makePersistentTunnelClient(tunnelCalls), makeDnsClient(), outdated);
9201003
......@@ -939,6 +1022,459 @@ describe("ManagedEndpointProvider", () => {
9391022 }).pipe(Effect.provide(layer));
9401023 });
9411024
1025 it.effect("does not release a tunnel when the requested tunnel id is outdated", () => {
1026 const tunnelCalls: TunnelCall[] = [];
1027 const layer = providerLayer(
1028 makePersistentTunnelClient(tunnelCalls),
1029 makeDnsClient(),
1030 makeAllocations(),
1031 );
1032
1033 return Effect.gen(function* () {
1034 const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
1035 const key = { userId: "user_ABC", environmentId: "env_ABC" } as const;
1036 yield* provider.provision({
1037 ...key,
1038 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
1039 });
1040
1041 expect(yield* provider.release({ ...key, expectedTunnelId: "old-tunnel-id" })).toBe(false);
1042 expect(tunnelCalls.map((call) => call.operation)).not.toContain("delete");
1043 }).pipe(Effect.provide(layer));
1044 });
1045
1046 it.effect("keeps a tunnel when a provision replaces an ordinary release generation", () => {
1047 const tunnelCalls: TunnelCall[] = [];
1048 const allocations = makeAllocations();
1049 let replaceAfterClaim = false;
1050 const replaced = ManagedEndpointAllocations.ManagedEndpointAllocations.of({
1051 ...allocations,
1052 claimRelease: (input) =>
1053 allocations.claimRelease(input).pipe(
1054 Effect.tap((claimedGeneration) => {
1055 if (claimedGeneration === null || replaceAfterClaim) return Effect.void;
1056 replaceAfterClaim = true;
1057 return allocations
1058 .recordTunnel({
1059 userId: input.userId,
1060 environmentId: input.environmentId,
1061 tunnelId: "replacement-tunnel",
1062 generation: claimedGeneration,
1063 })
1064 .pipe(Effect.asVoid);
1065 }),
1066 ),
1067 });
1068 const layer = providerLayer(makePersistentTunnelClient(tunnelCalls), makeDnsClient(), replaced);
1069
1070 return Effect.gen(function* () {
1071 const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
1072 const key = { userId: "user_ABC", environmentId: "env_ABC" } as const;
1073 yield* provider.provision({
1074 ...key,
1075 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
1076 });
1077
1078 expect(yield* provider.release(key)).toBe(false);
1079 expect(tunnelCalls.map((call) => call.operation)).not.toContain("delete");
1080 }).pipe(Effect.provide(layer));
1081 });
1082
1083 it.effect("does no Cloudflare work when the registered origin is unchanged", () => {
1084 const tunnelCalls: TunnelCall[] = [];
1085 const layer = providerLayer(makePersistentTunnelClient(tunnelCalls));
1086
1087 return Effect.gen(function* () {
1088 const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
1089 const key = { userId: "user_ABC", environmentId: "env_ABC" } as const;
1090 const origin = { localHttpHost: "127.0.0.1", localHttpPort: 3773 } as const;
1091 const provisioned = yield* provider.provision({ ...key, origin });
1092 tunnelCalls.length = 0;
1093
1094 expect(
1095 yield* provider.reconcileOrigin({
1096 ...key,
1097 tunnelId: provisioned.runtime.tunnelId!,
1098 origin,
1099 endpoint: provisioned.endpoint,
1100 }),
1101 ).toBe("ready");
1102 expect(tunnelCalls).toEqual([]);
1103 }).pipe(Effect.provide(layer));
1104 });
1105
1106 it.effect("updates Cloudflare ingress once when the registered port changes", () => {
1107 const tunnelCalls: TunnelCall[] = [];
1108 const layer = providerLayer(makePersistentTunnelClient(tunnelCalls));
1109
1110 return Effect.gen(function* () {
1111 const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
1112 const key = { userId: "user_ABC", environmentId: "env_ABC" } as const;
1113 const provisioned = yield* provider.provision({
1114 ...key,
1115 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
1116 });
1117 tunnelCalls.length = 0;
1118
1119 expect(
1120 yield* provider.reconcileOrigin({
1121 ...key,
1122 tunnelId: provisioned.runtime.tunnelId!,
1123 origin: { localHttpHost: "127.0.0.1", localHttpPort: 4884 },
1124 endpoint: provisioned.endpoint,
1125 }),
1126 ).toBe("ready");
1127 expect(tunnelCalls).toEqual([
1128 {
1129 operation: "putConfiguration",
1130 input: {
1131 tunnelId: "tunnel-id",
1132 tunnelConfig: {
1133 ingress: [
1134 {
1135 hostname: expectedManagedHostname("env_ABC"),
1136 service: "http://127.0.0.1:4884",
1137 },
1138 { service: "http_status:404" },
1139 ],
1140 },
1141 },
1142 },
1143 ]);
1144 }).pipe(Effect.provide(layer));
1145 });
1146
1147 it.effect("requires recovery when Cloudflare reports the registered tunnel is missing", () => {
1148 const tunnelCalls: TunnelCall[] = [];
1149 const baseTunnelClient = makePersistentTunnelClient(tunnelCalls);
1150 let tunnelMissing = false;
1151 const tunnelClient = ManagedEndpointProvider.ManagedEndpointTunnelClient.of({
1152 ...baseTunnelClient,
1153 putConfiguration: (tunnelId, tunnelConfig) =>
1154 tunnelMissing
1155 ? Effect.fail(
1156 new ManagedEndpointProvider.ManagedEndpointTunnelClientError({
1157 operation: "put-configuration",
1158 tunnelId,
1159 cause: { _tag: "TunnelNotFound" },
1160 }),
1161 )
1162 : baseTunnelClient.putConfiguration(tunnelId, tunnelConfig),
1163 });
1164 const layer = providerLayer(tunnelClient);
1165
1166 return Effect.gen(function* () {
1167 const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
1168 const key = { userId: "user_ABC", environmentId: "env_ABC" } as const;
1169 const provisioned = yield* provider.provision({
1170 ...key,
1171 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
1172 });
1173 tunnelMissing = true;
1174
1175 expect(
1176 yield* provider.reconcileOrigin({
1177 ...key,
1178 tunnelId: provisioned.runtime.tunnelId!,
1179 origin: { localHttpHost: "127.0.0.1", localHttpPort: 4884 },
1180 endpoint: provisioned.endpoint,
1181 }),
1182 ).toBe("recovery_required");
1183 }).pipe(Effect.provide(layer));
1184 });
1185
1186 it.effect("fails origin sync when the allocation generation changes", () => {
1187 const allocations = makeAllocations();
1188 let loseClaim = false;
1189 const changed = ManagedEndpointAllocations.ManagedEndpointAllocations.of({
1190 ...allocations,
1191 withClaimedTunnel: (input, effect) =>
1192 loseClaim ? Effect.succeed(Option.none()) : allocations.withClaimedTunnel(input, effect),
1193 });
1194 const layer = providerLayer(makePersistentTunnelClient(), makeDnsClient(), changed);
1195
1196 return Effect.gen(function* () {
1197 const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
1198 const key = { userId: "user_ABC", environmentId: "env_ABC" } as const;
1199 const provisioned = yield* provider.provision({
1200 ...key,
1201 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
1202 });
1203 loseClaim = true;
1204
1205 const error = yield* Effect.flip(
1206 provider.reconcileOrigin({
1207 ...key,
1208 tunnelId: provisioned.runtime.tunnelId!,
1209 origin: { localHttpHost: "127.0.0.1", localHttpPort: 4884 },
1210 endpoint: provisioned.endpoint,
1211 }),
1212 );
1213 expect(error).toMatchObject({
1214 _tag: "ManagedEndpointProvisioningFailed",
1215 stage: "sync-origin",
1216 });
1217 }).pipe(Effect.provide(layer));
1218 });
1219
1220 it.effect("rejects an active endpoint that does not match the allocation hostname", () => {
1221 const layer = providerLayer(makePersistentTunnelClient());
1222
1223 return Effect.gen(function* () {
1224 const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
1225 const key = { userId: "user_ABC", environmentId: "env_ABC" } as const;
1226 const provisioned = yield* provider.provision({
1227 ...key,
1228 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
1229 });
1230
1231 const error = yield* Effect.flip(
1232 provider.reconcileOrigin({
1233 ...key,
1234 tunnelId: provisioned.runtime.tunnelId!,
1235 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
1236 endpoint: {
1237 ...provisioned.endpoint,
1238 httpBaseUrl: "https://different-host.t3code.test/",
1239 },
1240 }),
1241 );
1242 expect(error).toMatchObject({
1243 _tag: "ManagedEndpointProvisioningFailed",
1244 stage: "verify-endpoint",
1245 });
1246 }).pipe(Effect.provide(layer));
1247 });
1248
1249 it.effect(
1250 "keeps a newly created tunnel available for retry after losing its allocation claim",
1251 () => {
1252 const tunnelCalls: TunnelCall[] = [];
1253 const allocations = makeAllocations();
1254 let changeGeneration = true;
1255 const changed = ManagedEndpointAllocations.ManagedEndpointAllocations.of({
1256 ...allocations,
1257 recordTunnel: (input) =>
1258 Effect.gen(function* () {
1259 if (changeGeneration) {
1260 changeGeneration = false;
1261 yield* allocations.claimDeprovision(input);
1262 }
1263 return yield* allocations.recordTunnel(input);
1264 }),
1265 });
1266 const layer = providerLayer(
1267 makePersistentTunnelClient(tunnelCalls),
1268 makeDnsClient(),
1269 changed,
1270 );
1271
1272 return Effect.gen(function* () {
1273 const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
1274 const input = {
1275 userId: "user_ABC",
1276 environmentId: "env_ABC",
1277 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
1278 };
1279 const error = yield* Effect.flip(provider.provision(input));
1280
1281 expect(error).toMatchObject({
1282 _tag: "ManagedEndpointProvisioningFailed",
1283 stage: "record-tunnel",
1284 });
1285 expect(tunnelCalls.map((call) => call.operation)).toEqual(["list", "create"]);
1286 expect((yield* provider.provision(input)).runtime.tunnelId).toBe("tunnel-id");
1287 expect(tunnelCalls.filter((call) => call.operation === "create")).toHaveLength(1);
1288 expect(tunnelCalls.filter((call) => call.operation === "delete")).toEqual([]);
1289 }).pipe(Effect.provide(layer));
1290 },
1291 );
1292
1293 it.effect("does not overwrite DNS when tunnel ownership changes during provisioning", () => {
1294 const allocations = makeAllocations();
1295 const changed = ManagedEndpointAllocations.ManagedEndpointAllocations.of({
1296 ...allocations,
1297 recordDns: () => Effect.succeed(null),
1298 });
1299 const layer = providerLayer(makePersistentTunnelClient(), makeDnsClient(), changed);
1300
1301 return Effect.gen(function* () {
1302 const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
1303 const error = yield* Effect.flip(
1304 provider.provision({
1305 userId: "user_ABC",
1306 environmentId: "env_ABC",
1307 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
1308 }),
1309 );
1310
1311 expect(error).toMatchObject({
1312 _tag: "ManagedEndpointProvisioningFailed",
1313 stage: "record-dns",
1314 });
1315 }).pipe(Effect.provide(layer));
1316 });
1317
1318 it.effect("does not change tunnel ingress after another provision takes ownership", () => {
1319 const tunnelCalls: TunnelCall[] = [];
1320 const allocations = makeAllocations();
1321 const changed = ManagedEndpointAllocations.ManagedEndpointAllocations.of({
1322 ...allocations,
1323 withClaimedTunnel: () => Effect.succeed(Option.none()),
1324 });
1325 const layer = providerLayer(makePersistentTunnelClient(tunnelCalls), makeDnsClient(), changed);
1326
1327 return Effect.gen(function* () {
1328 const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
1329 const error = yield* Effect.flip(
1330 provider.provision({
1331 userId: "user_ABC",
1332 environmentId: "env_ABC",
1333 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
1334 }),
1335 );
1336
1337 expect(error).toMatchObject({
1338 _tag: "ManagedEndpointProvisioningFailed",
1339 stage: "configure-tunnel",
1340 });
1341 expect(tunnelCalls.map((call) => call.operation)).not.toContain("putConfiguration");
1342 }).pipe(Effect.provide(layer));
1343 });
1344
1345 it.effect("does not change DNS after another provision takes ownership", () => {
1346 const dnsCalls: DnsCall[] = [];
1347 const allocations = makeAllocations();
1348 let lockCount = 0;
1349 const changed = ManagedEndpointAllocations.ManagedEndpointAllocations.of({
1350 ...allocations,
1351 withClaimedTunnel: (input, effect) =>
1352 ++lockCount === 1
1353 ? allocations.withClaimedTunnel(input, effect)
1354 : Effect.succeed(Option.none()),
1355 });
1356 const layer = providerLayer(makePersistentTunnelClient(), makeDnsClient(dnsCalls), changed);
1357
1358 return Effect.gen(function* () {
1359 const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
1360 const error = yield* Effect.flip(
1361 provider.provision({
1362 userId: "user_ABC",
1363 environmentId: "env_ABC",
1364 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
1365 }),
1366 );
1367
1368 expect(error).toMatchObject({
1369 _tag: "ManagedEndpointProvisioningFailed",
1370 stage: "record-dns",
1371 });
1372 expect(dnsCalls).toEqual([]);
1373 }).pipe(Effect.provide(layer));
1374 });
1375
1376 it.effect("does not mark a superseded tunnel allocation as ready", () => {
1377 const allocations = makeAllocations();
1378 const changed = ManagedEndpointAllocations.ManagedEndpointAllocations.of({
1379 ...allocations,
1380 markReady: () => Effect.succeed(false),
1381 });
1382 const layer = providerLayer(makePersistentTunnelClient(), makeDnsClient(), changed);
1383
1384 return Effect.gen(function* () {
1385 const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
1386 const error = yield* Effect.flip(
1387 provider.provision({
1388 userId: "user_ABC",
1389 environmentId: "env_ABC",
1390 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
1391 }),
1392 );
1393
1394 expect(error).toMatchObject({
1395 _tag: "ManagedEndpointProvisioningFailed",
1396 stage: "mark-allocation-ready",
1397 });
1398 }).pipe(Effect.provide(layer));
1399 });
1400
1401 it.effect("keeps a tunnel that reconnects before scheduled deletion", () => {
1402 const tunnelCalls: TunnelCall[] = [];
1403 const tunnelClient = ManagedEndpointProvider.ManagedEndpointTunnelClient.of({
1404 ...makePersistentTunnelClient(tunnelCalls),
1405 get: (tunnelId) =>
1406 Effect.succeed({
1407 id: tunnelId,
1408 name: expectedManagedTunnelName("env_ABC"),
1409 status: "healthy",
1410 connsInactiveAt: null,
1411 }),
1412 });
1413 const layer = providerLayer(tunnelClient, makeDnsClient(), makeAllocations());
1414
1415 return Effect.gen(function* () {
1416 const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
1417 const key = { userId: "user_ABC", environmentId: "env_ABC" } as const;
1418 yield* provider.provision({
1419 ...key,
1420 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
1421 });
1422
1423 expect(
1424 yield* provider.release({
1425 ...key,
1426 expectedTunnelId: "tunnel-id",
1427 expectedStatus: "down",
1428 expectedInactiveBefore: "2026-06-01T00:05:00.000Z",
1429 }),
1430 ).toBe(false);
1431 expect(tunnelCalls.map((call) => call.operation)).not.toContain("delete");
1432 }).pipe(Effect.provide(layer));
1433 });
1434
1435 it.effect("keeps a tunnel when a new provision replaces the release generation", () => {
1436 const tunnelCalls: TunnelCall[] = [];
1437 const allocations = makeAllocations();
1438 const replaced = ManagedEndpointAllocations.ManagedEndpointAllocations.of({
1439 ...allocations,
1440 claimRelease: (input) =>
1441 allocations.claimRelease(input).pipe(
1442 Effect.tap((claimedGeneration) =>
1443 claimedGeneration === null
1444 ? Effect.void
1445 : allocations
1446 .recordTunnel({
1447 userId: input.userId,
1448 environmentId: input.environmentId,
1449 tunnelId: "replacement-tunnel",
1450 generation: claimedGeneration,
1451 })
1452 .pipe(Effect.asVoid),
1453 ),
1454 ),
1455 });
1456 const layer = providerLayer(makePersistentTunnelClient(tunnelCalls), makeDnsClient(), replaced);
1457
1458 return Effect.gen(function* () {
1459 const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
1460 const key = { userId: "user_ABC", environmentId: "env_ABC" } as const;
1461 yield* provider.provision({
1462 ...key,
1463 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
1464 });
1465
1466 expect(
1467 yield* provider.release({
1468 ...key,
1469 expectedTunnelId: "tunnel-id",
1470 expectedStatus: "down",
1471 expectedInactiveBefore: "2026-06-01T00:05:00.000Z",
1472 }),
1473 ).toBe(false);
1474 expect(tunnelCalls.map((call) => call.operation)).not.toContain("delete");
1475 }).pipe(Effect.provide(layer));
1476 });
1477
9421478 it.effect("treats an already deleted tunnel as successfully released", () => {
9431479 const notFound = { _tag: "NotFound" } as const;
9441480 const tunnelClient = ManagedEndpointProvider.ManagedEndpointTunnelClient.of({
infra/relay/src/environments/ManagedEndpointProvider.ts+545-119
......@@ -3,6 +3,7 @@ import * as Cloudflare from "alchemy/Cloudflare";
33import * as Arr from "effect/Array";
44import * as Context from "effect/Context";
55import * as Crypto from "effect/Crypto";
6import * as DateTime from "effect/DateTime";
67import * as Effect from "effect/Effect";
78import * as Encoding from "effect/Encoding";
89import * as Layer from "effect/Layer";
......@@ -52,6 +53,9 @@ const ManagedEndpointProvisioningStage = Schema.Literals([
5253 "record-dns",
5354 "get-tunnel-token",
5455 "mark-allocation-ready",
56 "load-allocation",
57 "verify-endpoint",
58 "sync-origin",
5559]);
5660
5761export class ManagedEndpointProvisioningFailed extends Schema.TaggedError<ManagedEndpointProvisioningFailed>()(
......@@ -76,6 +80,7 @@ export class ManagedEndpointProvisioningFailed extends Schema.TaggedError<Manage
7680
7781const ManagedEndpointDeprovisioningStage = Schema.Literals([
7882 "load-allocation",
83 "load-tunnel",
7984 "claim-release",
8085 "claim-deprovision",
8186 "delete-dns-record",
......@@ -124,6 +129,8 @@ export interface ManagedEndpointProvisioningResult {
124129 readonly runtime: RelayManagedEndpointRuntimeConfig;
125130}
126131
132export type ManagedEndpointOriginSyncResult = "ready" | "recovery_required";
133
127134export type ManagedEndpointDeprovisionTarget = ManagedEndpointAllocations.ManagedEndpointAllocation;
128135
129136export class ManagedEndpointProvider extends Context.Service<
......@@ -134,6 +141,13 @@ export class ManagedEndpointProvider extends Context.Service<
134141 readonly environmentId: string;
135142 readonly origin: RelayManagedEndpointOrigin;
136143 }) => Effect.Effect<ManagedEndpointProvisioningResult, ManagedEndpointProviderError>;
144 readonly reconcileOrigin: (input: {
145 readonly userId: string;
146 readonly environmentId: string;
147 readonly tunnelId: string;
148 readonly origin: RelayManagedEndpointOrigin;
149 readonly endpoint: RelayManagedEndpoint;
150 }) => Effect.Effect<ManagedEndpointOriginSyncResult, ManagedEndpointProviderError>;
137151 /**
138152 * Captures the allocation generation owned by an unlink before its link
139153 * revocation commits. Passing this target to `deprovision` prevents a
......@@ -150,7 +164,7 @@ export class ManagedEndpointProvider extends Context.Service<
150164 readonly userId: string;
151165 readonly environmentId: string;
152166 readonly target?: ManagedEndpointDeprovisionTarget | null;
153 }) => Effect.Effect<void, ManagedEndpointDeprovisioningFailed>;
167 }) => Effect.Effect<boolean, ManagedEndpointDeprovisioningFailed>;
154168 /**
155169 * Deletes the provisioned Cloudflare tunnel while keeping the allocation
156170 * (hostname + tunnel name reservation) and DNS record. Cloudflare bills per
......@@ -166,16 +180,34 @@ export class ManagedEndpointProvider extends Context.Service<
166180 readonly release: (input: {
167181 readonly userId: string;
168182 readonly environmentId: string;
183 readonly expectedTunnelId?: string;
184 readonly expectedInactiveBefore?: string;
185 readonly expectedStatus?: "inactive" | "down";
169186 }) => Effect.Effect<boolean, ManagedEndpointDeprovisioningFailed>;
170187 }
171188>()("t3code-relay/environments/ManagedEndpointProvider") {}
172189
173interface ManagedEndpointTunnel {
190export interface ManagedEndpointTunnel {
174191 readonly id?: string | null;
175192 readonly name?: string | null;
193 readonly status?: string | null;
194 readonly createdAt?: string | null;
195 readonly connsInactiveAt?: string | null;
196}
197
198export interface ManagedEndpointTunnelListRequest {
199 readonly isDeleted: false;
200 readonly name?: string;
201 readonly includePrefix?: string;
202 readonly status?: "inactive" | "down";
203 readonly existedAt?: string;
204 readonly wasInactiveAt?: string;
205 readonly page?: number;
206 readonly perPage?: number;
176207}
177208
178209const ManagedEndpointTunnelClientOperation = Schema.Literals([
210 "get",
179211 "list",
180212 "create",
181213 "put-configuration",
......@@ -201,11 +233,18 @@ export class ManagedEndpointTunnelClientError extends Schema.TaggedError<Managed
201233export class ManagedEndpointTunnelClient extends Context.Service<
202234 ManagedEndpointTunnelClient,
203235 {
204 readonly list: (request: {
205 readonly name: string;
206 readonly isDeleted: false;
207 }) => Effect.Effect<
208 { readonly result: ReadonlyArray<ManagedEndpointTunnel> },
236 readonly get: (
237 tunnelId: string,
238 ) => Effect.Effect<ManagedEndpointTunnel, ManagedEndpointTunnelClientError>;
239 readonly list: (request: ManagedEndpointTunnelListRequest) => Effect.Effect<
240 {
241 readonly result: ReadonlyArray<ManagedEndpointTunnel>;
242 readonly resultInfo?: {
243 readonly page?: number | null;
244 readonly perPage?: number | null;
245 readonly totalCount?: number | null;
246 } | null;
247 },
209248 ManagedEndpointTunnelClientError
210249 >;
211250 readonly create: (request: {
......@@ -333,17 +372,17 @@ function isLoopbackOrigin(origin: RelayManagedEndpointOrigin): boolean {
333372 );
334373}
335374
336function isNotFoundCause(cause: unknown): boolean {
375export function isManagedEndpointNotFound(cause: unknown): boolean {
337376 if (typeof cause !== "object" || cause === null) {
338377 return false;
339378 }
340 if ("_tag" in cause && cause._tag === "NotFound") {
379 if ("_tag" in cause && (cause._tag === "NotFound" || cause._tag === "TunnelNotFound")) {
341380 return true;
342381 }
343382 if ("status" in cause && cause.status === 404) {
344383 return true;
345384 }
346 return "cause" in cause && isNotFoundCause(cause.cause);
385 return "cause" in cause && isManagedEndpointNotFound(cause.cause);
347386}
348387
349388type ManagedEndpointClientError = ManagedEndpointTunnelClientError | ManagedEndpointDnsClientError;
......@@ -355,9 +394,9 @@ const ignoreNotFound = <A>(
355394 Effect.asVoid,
356395 Effect.catchTags({
357396 ManagedEndpointTunnelClientError: (error) =>
358 isNotFoundCause(error.cause) ? Effect.void : Effect.fail(error),
397 isManagedEndpointNotFound(error.cause) ? Effect.void : Effect.fail(error),
359398 ManagedEndpointDnsClientError: (error) =>
360 isNotFoundCause(error.cause) ? Effect.void : Effect.fail(error),
399 isManagedEndpointNotFound(error.cause) ? Effect.void : Effect.fail(error),
361400 }),
362401 );
363402
......@@ -399,7 +438,7 @@ export const make = Effect.gen(function* () {
399438 Effect.as(true),
400439 Effect.catchTags({
401440 ManagedEndpointDnsClientError: (error) =>
402 isNotFoundCause(error.cause) ? Effect.succeed(false) : Effect.fail(error),
441 isManagedEndpointNotFound(error.cause) ? Effect.succeed(false) : Effect.fail(error),
403442 }),
404443 );
405444 if (checkpointedRecordUpdated) {
......@@ -455,8 +494,129 @@ export const make = Effect.gen(function* () {
455494 },
456495 );
457496
497 const reconcileOrigin = Effect.fn("relay.managed_endpoint_provider.reconcile_origin")(
498 function* (input: {
499 readonly userId: string;
500 readonly environmentId: string;
501 readonly tunnelId: string;
502 readonly origin: RelayManagedEndpointOrigin;
503 readonly endpoint: RelayManagedEndpoint;
504 }) {
505 if (!isLoopbackOrigin(input.origin)) {
506 return yield* new ManagedEndpointOriginNotAllowed({
507 userId: input.userId,
508 environmentId: input.environmentId,
509 host: input.origin.localHttpHost,
510 port: input.origin.localHttpPort,
511 });
512 }
513 const allocation = yield* allocations.get(input).pipe(
514 Effect.mapError(
515 (cause) =>
516 new ManagedEndpointProvisioningFailed({
517 ...input,
518 stage: "load-allocation",
519 cause,
520 }),
521 ),
522 );
523 if (
524 allocation === null ||
525 allocation.tunnelId !== input.tunnelId ||
526 allocation.dnsRecordId === null ||
527 allocation.readyAt === null
528 ) {
529 return "recovery_required";
530 }
531 const cf = yield* requireCloudflareSettings(config, input);
532 const recordedEndpoint = ManagedEndpointAllocations.resolveReadyManagedEndpoint({
533 allocation,
534 baseDomain: cf.baseDomain,
535 });
536 if (
537 recordedEndpoint === null ||
538 recordedEndpoint.httpBaseUrl !== input.endpoint.httpBaseUrl ||
539 recordedEndpoint.wsBaseUrl !== input.endpoint.wsBaseUrl ||
540 recordedEndpoint.providerKind !== input.endpoint.providerKind
541 ) {
542 return yield* new ManagedEndpointProvisioningFailed({
543 ...input,
544 stage: "verify-endpoint",
545 hostname: allocation.hostname,
546 });
547 }
548 if (
549 allocation.origin?.localHttpHost === input.origin.localHttpHost &&
550 allocation.origin.localHttpPort === input.origin.localHttpPort
551 ) {
552 return "ready";
553 }
554
555 const updated = yield* allocations
556 .withClaimedTunnel(
557 {
558 userId: input.userId,
559 environmentId: input.environmentId,
560 tunnelId: input.tunnelId,
561 generation: allocation.generation,
562 },
563 tunnels
564 .putConfiguration(input.tunnelId, {
565 ingress: [
566 {
567 hostname: allocation.hostname,
568 service: formatOriginService(input.origin),
569 },
570 { service: "http_status:404" },
571 ],
572 })
573 .pipe(
574 Effect.as("configured" as const),
575 Effect.catchTags({
576 ManagedEndpointTunnelClientError: (error) =>
577 isManagedEndpointNotFound(error.cause)
578 ? Effect.succeed("missing" as const)
579 : Effect.fail(error),
580 }),
581 Effect.flatMap((result) =>
582 result === "missing"
583 ? Effect.succeed(result)
584 : allocations
585 .markReady({
586 ...input,
587 generation: allocation.generation,
588 })
589 .pipe(
590 Effect.map((updated) =>
591 updated ? ("configured" as const) : ("stale" as const),
592 ),
593 ),
594 ),
595 ),
596 )
597 .pipe(
598 Effect.mapError(
599 (cause) =>
600 new ManagedEndpointProvisioningFailed({
601 ...input,
602 stage: "sync-origin",
603 cause,
604 }),
605 ),
606 );
607 if (Option.isNone(updated) || updated.value === "stale") {
608 return yield* new ManagedEndpointProvisioningFailed({
609 ...input,
610 stage: "sync-origin",
611 });
612 }
613 return updated.value === "configured" ? "ready" : "recovery_required";
614 },
615 );
616
458617 return ManagedEndpointProvider.of({
459618 prepareDeprovision,
619 reconcileOrigin,
460620 deprovision: Effect.fn("relay.managed_endpoint_provider.deprovision")(function* (input) {
461621 yield* Effect.annotateCurrentSpan({
462622 "relay.user_id": input.userId,
......@@ -465,13 +625,13 @@ export const make = Effect.gen(function* () {
465625 const allocation =
466626 input.target === undefined ? yield* prepareDeprovision(input) : input.target;
467627 if (allocation === null) {
468 return;
628 return true;
469629 }
470 const claimedAt = yield* allocations
630 const claimedGeneration = yield* allocations
471631 .claimDeprovision({
472632 userId: input.userId,
473633 environmentId: input.environmentId,
474 updatedAt: allocation.updatedAt,
634 generation: allocation.generation,
475635 })
476636 .pipe(
477637 Effect.mapError(
......@@ -485,55 +645,86 @@ export const make = Effect.gen(function* () {
485645 }),
486646 ),
487647 );
488 if (claimedAt === null) {
489 return;
490 }
491 const dnsRecordId = allocation.dnsRecordId;
492 if (dnsRecordId !== null) {
493 yield* ignoreNotFound(dns.deleteRecord(dnsRecordId)).pipe(
494 Effect.mapError(
495 (cause) =>
496 new ManagedEndpointDeprovisioningFailed({
497 ...input,
498 stage: "delete-dns-record",
499 dnsRecordId,
500 cause,
501 }),
502 ),
503 );
648 if (claimedGeneration === null) {
649 return false;
504650 }
505651 const tunnelId = allocation.tunnelId;
506 if (tunnelId !== null) {
507 yield* ignoreNotFound(tunnels.delete(tunnelId)).pipe(
508 Effect.mapError(
509 (cause) =>
510 new ManagedEndpointDeprovisioningFailed({
511 ...input,
512 stage: "delete-tunnel",
513 tunnelId,
514 cause,
515 }),
516 ),
517 );
652 const deprovision = Effect.gen(function* () {
653 const dnsRecordId = allocation.dnsRecordId;
654 if (dnsRecordId !== null) {
655 yield* ignoreNotFound(dns.deleteRecord(dnsRecordId)).pipe(
656 Effect.mapError(
657 (cause) =>
658 new ManagedEndpointDeprovisioningFailed({
659 ...input,
660 stage: "delete-dns-record",
661 dnsRecordId,
662 cause,
663 }),
664 ),
665 );
666 }
667 if (tunnelId !== null) {
668 yield* ignoreNotFound(tunnels.delete(tunnelId)).pipe(
669 Effect.mapError(
670 (cause) =>
671 new ManagedEndpointDeprovisioningFailed({
672 ...input,
673 stage: "delete-tunnel",
674 tunnelId,
675 cause,
676 }),
677 ),
678 );
679 }
680 return yield* allocations
681 .removeClaimed({
682 userId: input.userId,
683 environmentId: input.environmentId,
684 generation: claimedGeneration,
685 })
686 .pipe(
687 Effect.mapError(
688 (cause) =>
689 new ManagedEndpointDeprovisioningFailed({
690 ...input,
691 stage: "remove-allocation",
692 ...(allocation.tunnelId === null ? {} : { tunnelId: allocation.tunnelId }),
693 ...(allocation.dnsRecordId === null
694 ? {}
695 : { dnsRecordId: allocation.dnsRecordId }),
696 cause,
697 }),
698 ),
699 );
700 });
701 if (tunnelId === null) {
702 return yield* deprovision;
518703 }
519 yield* allocations
520 .removeClaimed({
521 userId: input.userId,
522 environmentId: input.environmentId,
523 updatedAt: claimedAt,
524 })
704 const removed = yield* allocations
705 .withClaimedTunnel(
706 {
707 userId: input.userId,
708 environmentId: input.environmentId,
709 tunnelId,
710 generation: claimedGeneration,
711 },
712 deprovision,
713 )
525714 .pipe(
526 Effect.mapError(
527 (cause) =>
528 new ManagedEndpointDeprovisioningFailed({
529 ...input,
530 stage: "remove-allocation",
531 ...(allocation.tunnelId === null ? {} : { tunnelId: allocation.tunnelId }),
532 ...(allocation.dnsRecordId === null ? {} : { dnsRecordId: allocation.dnsRecordId }),
533 cause,
534 }),
535 ),
715 Effect.catchTags({
716 ManagedEndpointAllocationPersistenceError: (cause) =>
717 Effect.fail(
718 new ManagedEndpointDeprovisioningFailed({
719 ...input,
720 stage: "claim-deprovision",
721 tunnelId,
722 cause,
723 }),
724 ),
725 }),
536726 );
727 return Option.getOrElse(removed, () => false);
537728 }),
538729 release: Effect.fn("relay.managed_endpoint_provider.release")(function* (input) {
539730 yield* Effect.annotateCurrentSpan({
......@@ -554,19 +745,22 @@ export const make = Effect.gen(function* () {
554745 if (allocation === null || tunnelId === null) {
555746 return true;
556747 }
748 if (input.expectedTunnelId !== undefined && input.expectedTunnelId !== tunnelId) {
749 return false;
750 }
557751 // Claim the release against the allocation's current generation before
558752 // touching Cloudflare. A provision racing this release (fast environment
559 // restart) rewrites updatedAt when it records its tunnel, so a stale
753 // restart) increments the generation when it records its tunnel, so a stale
560754 // claim means the recorded tunnel may already back a fresh connector and
561755 // must be left alive. A provision that starts after the claim instead
562756 // fails loudly on the deleted tunnel and the client-side retry
563757 // provisions a replacement.
564 const claimed = yield* allocations
758 const claimedGeneration = yield* allocations
565759 .claimRelease({
566760 userId: input.userId,
567761 environmentId: input.environmentId,
568762 tunnelId,
569 updatedAt: allocation.updatedAt,
763 generation: allocation.generation,
570764 })
571765 .pipe(
572766 Effect.mapError(
......@@ -579,10 +773,10 @@ export const make = Effect.gen(function* () {
579773 }),
580774 ),
581775 );
582 if (!claimed) {
776 if (claimedGeneration === null) {
583777 return false;
584778 }
585 yield* ignoreNotFound(tunnels.delete(tunnelId)).pipe(
779 const deleteTunnel = ignoreNotFound(tunnels.delete(tunnelId)).pipe(
586780 Effect.mapError(
587781 (cause) =>
588782 new ManagedEndpointDeprovisioningFailed({
......@@ -593,13 +787,103 @@ export const make = Effect.gen(function* () {
593787 }),
594788 ),
595789 );
790 if (input.expectedInactiveBefore !== undefined && input.expectedStatus !== undefined) {
791 const expectedStatus = input.expectedStatus;
792 const inactiveBefore = input.expectedInactiveBefore;
793 const currentTunnel = yield* tunnels.get(tunnelId).pipe(
794 Effect.map(Option.some),
795 Effect.catchTags({
796 ManagedEndpointTunnelClientError: (cause) =>
797 isManagedEndpointNotFound(cause.cause)
798 ? Effect.succeed(Option.none())
799 : Effect.fail(
800 new ManagedEndpointDeprovisioningFailed({
801 ...input,
802 stage: "load-tunnel",
803 tunnelId,
804 cause,
805 }),
806 ),
807 }),
808 );
809 if (Option.isNone(currentTunnel)) {
810 return true;
811 }
812 const inactiveAt =
813 expectedStatus === "down"
814 ? currentTunnel.value.connsInactiveAt
815 : currentTunnel.value.createdAt;
816 if (
817 currentTunnel.value.id !== tunnelId ||
818 currentTunnel.value.status !== expectedStatus ||
819 typeof inactiveAt !== "string"
820 ) {
821 return false;
822 }
823 const inactiveTime = DateTime.make(inactiveAt);
824 const cutoff = DateTime.make(inactiveBefore);
825 if (
826 Option.isNone(inactiveTime) ||
827 Option.isNone(cutoff) ||
828 inactiveTime.value.epochMilliseconds > cutoff.value.epochMilliseconds
829 ) {
830 return false;
831 }
832 }
833 const released = yield* allocations
834 .withClaimedTunnel(
835 {
836 userId: input.userId,
837 environmentId: input.environmentId,
838 tunnelId,
839 generation: claimedGeneration,
840 },
841 Effect.gen(function* () {
842 const finalGeneration = yield* allocations
843 .claimRelease({
844 userId: input.userId,
845 environmentId: input.environmentId,
846 tunnelId,
847 generation: claimedGeneration,
848 })
849 .pipe(
850 Effect.mapError(
851 (cause) =>
852 new ManagedEndpointDeprovisioningFailed({
853 ...input,
854 stage: "claim-release",
855 tunnelId,
856 cause,
857 }),
858 ),
859 );
860 if (finalGeneration === null) {
861 return false;
862 }
863 yield* deleteTunnel;
864 return true;
865 }),
866 )
867 .pipe(
868 Effect.catchTags({
869 ManagedEndpointAllocationPersistenceError: (cause) =>
870 Effect.fail(
871 new ManagedEndpointDeprovisioningFailed({
872 ...input,
873 stage: "claim-release",
874 tunnelId,
875 cause,
876 }),
877 ),
878 }),
879 );
596880 // The recorded tunnelId is now stale, but the allocation row is left
597881 // untouched deliberately: connect/status authorization requires a fully
598882 // recorded allocation, and an offline environment must keep reporting
599883 // "offline" (health probe fails) rather than "not authorized". The next
600884 // provision lists tunnels by name, finds none, creates a replacement and
601885 // re-records the fresh id.
602 return true;
886 return Option.getOrElse(released, () => false);
603887 }),
604888 provision: Effect.fn("relay.managed_endpoint_provider.provision")(function* (input) {
605889 yield* Effect.annotateCurrentSpan({
......@@ -717,11 +1001,12 @@ export const make = Effect.gen(function* () {
7171001 });
7181002 }
7191003 const tunnel = { id: tunnelResponse.id, name: tunnelResponse.name };
720 yield* allocations
1004 const tunnelGeneration = yield* allocations
7211005 .recordTunnel({
7221006 userId: input.userId,
7231007 environmentId: input.environmentId,
7241008 tunnelId: tunnel.id,
1009 generation: allocation.generation,
7251010 })
7261011 .pipe(
7271012 Effect.mapError(
......@@ -737,31 +1022,78 @@ export const make = Effect.gen(function* () {
7371022 }),
7381023 ),
7391024 );
1025 if (tunnelGeneration === null) {
1026 // A newer provision can adopt this tunnel by name at any point after
1027 // our claim fails. Leave it available for that provision or a retry.
1028 return yield* new ManagedEndpointProvisioningFailed({
1029 userId: input.userId,
1030 environmentId: input.environmentId,
1031 stage: "record-tunnel",
1032 hostname,
1033 tunnelName,
1034 tunnelId: tunnel.id,
1035 });
1036 }
7401037
741 yield* tunnels
742 .putConfiguration(tunnel.id, {
743 ingress: [
744 {
745 hostname,
746 service: formatOriginService(input.origin),
747 },
748 { service: "http_status:404" },
749 ],
750 })
1038 const configured = yield* allocations
1039 .withClaimedTunnel(
1040 {
1041 userId: input.userId,
1042 environmentId: input.environmentId,
1043 tunnelId: tunnel.id,
1044 generation: tunnelGeneration,
1045 },
1046 tunnels
1047 .putConfiguration(tunnel.id, {
1048 ingress: [
1049 {
1050 hostname,
1051 service: formatOriginService(input.origin),
1052 },
1053 { service: "http_status:404" },
1054 ],
1055 })
1056 .pipe(
1057 Effect.mapError(
1058 (cause) =>
1059 new ManagedEndpointProvisioningFailed({
1060 userId: input.userId,
1061 environmentId: input.environmentId,
1062 stage: "configure-tunnel",
1063 hostname,
1064 tunnelName,
1065 tunnelId: tunnel.id,
1066 cause,
1067 }),
1068 ),
1069 ),
1070 )
7511071 .pipe(
752 Effect.mapError(
753 (cause) =>
754 new ManagedEndpointProvisioningFailed({
755 userId: input.userId,
756 environmentId: input.environmentId,
757 stage: "configure-tunnel",
758 hostname,
759 tunnelName,
760 tunnelId: tunnel.id,
761 cause,
762 }),
763 ),
1072 Effect.catchTags({
1073 ManagedEndpointAllocationPersistenceError: (cause) =>
1074 Effect.fail(
1075 new ManagedEndpointProvisioningFailed({
1076 userId: input.userId,
1077 environmentId: input.environmentId,
1078 stage: "configure-tunnel",
1079 hostname,
1080 tunnelName,
1081 tunnelId: tunnel.id,
1082 cause,
1083 }),
1084 ),
1085 }),
7641086 );
1087 if (Option.isNone(configured)) {
1088 return yield* new ManagedEndpointProvisioningFailed({
1089 userId: input.userId,
1090 environmentId: input.environmentId,
1091 stage: "configure-tunnel",
1092 hostname,
1093 tunnelName,
1094 tunnelId: tunnel.id,
1095 });
1096 }
7651097
7661098 const dnsRecord = {
7671099 type: "CNAME",
......@@ -771,31 +1103,61 @@ export const make = Effect.gen(function* () {
7711103 proxied: true,
7721104 } as const;
7731105
774 const dnsRecordId = yield* ensureDnsRecord(hostname, allocation.dnsRecordId, dnsRecord).pipe(
775 Effect.mapError(
776 (cause) =>
777 new ManagedEndpointProvisioningFailed({
778 userId: input.userId,
779 environmentId: input.environmentId,
780 stage: "ensure-dns-record",
1106 const recordedDns = yield* allocations
1107 .withClaimedTunnel(
1108 {
1109 userId: input.userId,
1110 environmentId: input.environmentId,
1111 tunnelId: tunnel.id,
1112 generation: tunnelGeneration,
1113 },
1114 Effect.gen(function* () {
1115 const dnsRecordId = yield* ensureDnsRecord(
7811116 hostname,
782 tunnelName,
783 tunnelId: tunnel.id,
784 ...(allocation.dnsRecordId === null ? {} : { dnsRecordId: allocation.dnsRecordId }),
785 cause,
786 }),
787 ),
788 );
789 yield* allocations
790 .recordDns({
791 userId: input.userId,
792 environmentId: input.environmentId,
793 dnsRecordId,
794 })
795 .pipe(
796 Effect.mapError(
797 (cause) =>
798 new ManagedEndpointProvisioningFailed({
1117 allocation.dnsRecordId,
1118 dnsRecord,
1119 ).pipe(
1120 Effect.mapError(
1121 (cause) =>
1122 new ManagedEndpointProvisioningFailed({
1123 userId: input.userId,
1124 environmentId: input.environmentId,
1125 stage: "ensure-dns-record",
1126 hostname,
1127 tunnelName,
1128 tunnelId: tunnel.id,
1129 ...(allocation.dnsRecordId === null
1130 ? {}
1131 : { dnsRecordId: allocation.dnsRecordId }),
1132 cause,
1133 }),
1134 ),
1135 );
1136 const dnsGeneration = yield* allocations
1137 .recordDns({
1138 userId: input.userId,
1139 environmentId: input.environmentId,
1140 dnsRecordId,
1141 tunnelId: tunnel.id,
1142 generation: tunnelGeneration,
1143 })
1144 .pipe(
1145 Effect.mapError(
1146 (cause) =>
1147 new ManagedEndpointProvisioningFailed({
1148 userId: input.userId,
1149 environmentId: input.environmentId,
1150 stage: "record-dns",
1151 hostname,
1152 tunnelName,
1153 tunnelId: tunnel.id,
1154 dnsRecordId,
1155 cause,
1156 }),
1157 ),
1158 );
1159 if (dnsGeneration === null) {
1160 return yield* new ManagedEndpointProvisioningFailed({
7991161 userId: input.userId,
8001162 environmentId: input.environmentId,
8011163 stage: "record-dns",
......@@ -803,10 +1165,38 @@ export const make = Effect.gen(function* () {
8031165 tunnelName,
8041166 tunnelId: tunnel.id,
8051167 dnsRecordId,
806 cause,
807 }),
808 ),
1168 });
1169 }
1170 return { dnsRecordId, dnsGeneration };
1171 }),
1172 )
1173 .pipe(
1174 Effect.catchTags({
1175 ManagedEndpointAllocationPersistenceError: (cause) =>
1176 Effect.fail(
1177 new ManagedEndpointProvisioningFailed({
1178 userId: input.userId,
1179 environmentId: input.environmentId,
1180 stage: "record-dns",
1181 hostname,
1182 tunnelName,
1183 tunnelId: tunnel.id,
1184 cause,
1185 }),
1186 ),
1187 }),
8091188 );
1189 if (Option.isNone(recordedDns)) {
1190 return yield* new ManagedEndpointProvisioningFailed({
1191 userId: input.userId,
1192 environmentId: input.environmentId,
1193 stage: "record-dns",
1194 hostname,
1195 tunnelName,
1196 tunnelId: tunnel.id,
1197 });
1198 }
1199 const { dnsRecordId, dnsGeneration } = recordedDns.value;
8101200
8111201 const connectorToken = yield* tunnels.getToken(tunnel.id).pipe(
8121202 Effect.mapError(
......@@ -823,10 +1213,13 @@ export const make = Effect.gen(function* () {
8231213 }),
8241214 ),
8251215 );
826 yield* allocations
1216 const ready = yield* allocations
8271217 .markReady({
8281218 userId: input.userId,
8291219 environmentId: input.environmentId,
1220 tunnelId: tunnel.id,
1221 generation: dnsGeneration,
1222 origin: input.origin,
8301223 })
8311224 .pipe(
8321225 Effect.mapError(
......@@ -843,6 +1236,17 @@ export const make = Effect.gen(function* () {
8431236 }),
8441237 ),
8451238 );
1239 if (!ready) {
1240 return yield* new ManagedEndpointProvisioningFailed({
1241 userId: input.userId,
1242 environmentId: input.environmentId,
1243 stage: "mark-allocation-ready",
1244 hostname,
1245 tunnelName,
1246 tunnelId: tunnel.id,
1247 dnsRecordId,
1248 });
1249 }
8461250
8471251 return {
8481252 endpoint: managedEndpointForHostname(hostname),
......@@ -865,16 +1269,30 @@ export const layerCloudflareBindings = (
8651269 alchemyRuntimeContext: Alchemy.BaseRuntimeContext,
8661270) =>
8671271 layer.pipe(
868 Layer.provide(
1272 Layer.provideMerge(
8691273 Layer.mergeAll(
8701274 layerTunnelClient({
1275 get: (tunnelId) =>
1276 tunnelClient.get(tunnelId).pipe(
1277 Effect.timeout("8 seconds"),
1278 Effect.mapError(
1279 (cause) =>
1280 new ManagedEndpointTunnelClientError({
1281 operation: "get",
1282 tunnelId,
1283 cause,
1284 }),
1285 ),
1286 Effect.provideService(Alchemy.RuntimeContext, alchemyRuntimeContext),
1287 ),
8711288 list: (request) =>
8721289 tunnelClient.list(request).pipe(
1290 Effect.timeout("8 seconds"),
8731291 Effect.mapError(
8741292 (cause) =>
8751293 new ManagedEndpointTunnelClientError({
8761294 operation: "list",
877 tunnelName: request.name,
1295 ...(request.name === undefined ? {} : { tunnelName: request.name }),
8781296 cause,
8791297 }),
8801298 ),
......@@ -882,6 +1300,7 @@ export const layerCloudflareBindings = (
8821300 ),
8831301 create: (request) =>
8841302 tunnelClient.create(request).pipe(
1303 Effect.timeout("8 seconds"),
8851304 Effect.mapError(
8861305 (cause) =>
8871306 new ManagedEndpointTunnelClientError({
......@@ -894,6 +1313,7 @@ export const layerCloudflareBindings = (
8941313 ),
8951314 putConfiguration: (tunnelId, config) =>
8961315 tunnelClient.putConfiguration(tunnelId, config).pipe(
1316 Effect.timeout("8 seconds"),
8971317 Effect.mapError(
8981318 (cause) =>
8991319 new ManagedEndpointTunnelClientError({
......@@ -906,6 +1326,7 @@ export const layerCloudflareBindings = (
9061326 ),
9071327 getToken: (tunnelId) =>
9081328 tunnelClient.getToken(tunnelId).pipe(
1329 Effect.timeout("8 seconds"),
9091330 Effect.mapError(
9101331 (cause) =>
9111332 new ManagedEndpointTunnelClientError({
......@@ -918,6 +1339,7 @@ export const layerCloudflareBindings = (
9181339 ),
9191340 delete: (tunnelId) =>
9201341 tunnelClient.delete(tunnelId).pipe(
1342 Effect.timeout("8 seconds"),
9211343 Effect.mapError(
9221344 (cause) =>
9231345 new ManagedEndpointTunnelClientError({
......@@ -932,6 +1354,7 @@ export const layerCloudflareBindings = (
9321354 layerDnsClient({
9331355 listRecords: (hostname) =>
9341356 dnsClient.listDnsRecords({ search: hostname }).pipe(
1357 Effect.timeout("8 seconds"),
9351358 Effect.map((response) =>
9361359 response.result.filter(
9371360 (record): record is typeof record & { readonly id: string } =>
......@@ -951,6 +1374,7 @@ export const layerCloudflareBindings = (
9511374 ),
9521375 createRecord: (request) =>
9531376 dnsClient.createDnsRecord(request).pipe(
1377 Effect.timeout("8 seconds"),
9541378 Effect.map((response) => ({ id: response.id })),
9551379 Effect.mapError(
9561380 (cause) =>
......@@ -964,6 +1388,7 @@ export const layerCloudflareBindings = (
9641388 ),
9651389 updateRecord: (dnsRecordId, request) =>
9661390 dnsClient.updateDnsRecord(dnsRecordId, request).pipe(
1391 Effect.timeout("8 seconds"),
9671392 Effect.mapError(
9681393 (cause) =>
9691394 new ManagedEndpointDnsClientError({
......@@ -977,6 +1402,7 @@ export const layerCloudflareBindings = (
9771402 ),
9781403 deleteRecord: (dnsRecordId) =>
9791404 dnsClient.deleteDnsRecord(dnsRecordId).pipe(
1405 Effect.timeout("8 seconds"),
9801406 Effect.mapError(
9811407 (cause) =>
9821408 new ManagedEndpointDnsClientError({
infra/relay/src/environments/ManagedEndpointReaper.test.ts created+761
......@@ -0,0 +1,761 @@
1import { describe, expect, it } from "@effect/vitest";
2import * as DateTime from "effect/DateTime";
3import * as Effect from "effect/Effect";
4import * as Layer from "effect/Layer";
5import * as Redacted from "effect/Redacted";
6import * as TestClock from "effect/testing/TestClock";
7
8import * as RelayConfiguration from "../Config.ts";
9import * as ManagedEndpointAllocations from "./ManagedEndpointAllocations.ts";
10import * as ManagedEndpointProvider from "./ManagedEndpointProvider.ts";
11import * as ManagedEndpointReaper from "./ManagedEndpointReaper.ts";
12
13const NOW = "2026-08-25T12:00:00.000Z";
14const NOW_MILLIS = DateTime.makeUnsafe(NOW).epochMilliseconds;
15const PREFIX = "t3coderelay-managedendpoint-prod-";
16
17function tunnel(input: {
18 readonly id: string;
19 readonly suffix: string;
20 readonly status: "down" | "inactive" | "healthy" | "degraded";
21 readonly timestamp?: string | null;
22 readonly prefix?: string;
23}): ManagedEndpointProvider.ManagedEndpointTunnel {
24 return {
25 id: input.id,
26 name: `${input.prefix ?? PREFIX}${input.suffix}`,
27 status: input.status,
28 ...(input.timestamp === undefined
29 ? {}
30 : input.status === "inactive"
31 ? { createdAt: input.timestamp }
32 : { connsInactiveAt: input.timestamp }),
33 };
34}
35
36function recoverableOwners(
37 tunnels: ReadonlyArray<ManagedEndpointProvider.ManagedEndpointTunnel>,
38): ReadonlyArray<ManagedEndpointAllocations.ManagedEndpointTunnelAllocation> {
39 return tunnels.map((entry) => allocation({ tunnelId: entry.id!, recoveryEnabled: true }));
40}
41
42function allocation(input: {
43 readonly tunnelId: string | null;
44 readonly recoveryEnabled: boolean;
45}): ManagedEndpointAllocations.ManagedEndpointTunnelAllocation {
46 return {
47 userId: "user-1",
48 environmentId: `environment-${input.tunnelId ?? "pending"}`,
49 hostname: `${input.tunnelId ?? "pending"}.example.test`,
50 tunnelId: input.tunnelId,
51 tunnelName: `${PREFIX}aaaaaaaaaaaaaaaa`,
52 dnsRecordId: "dns-1",
53 readyAt: "2026-08-25T11:00:00.000Z",
54 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
55 updatedAt: "2026-08-25T11:00:00.000Z",
56 generation: 1,
57 recoveryEnabled: input.recoveryEnabled,
58 };
59}
60
61function harness(input?: {
62 readonly tunnels?: ReadonlyArray<ManagedEndpointProvider.ManagedEndpointTunnel>;
63 readonly allocations?: ReadonlyArray<ManagedEndpointAllocations.ManagedEndpointTunnelAllocation>;
64 readonly namespace?: string;
65 readonly failTunnelId?: string;
66 readonly rateLimitedTunnelId?: string;
67 readonly failAllDeletes?: boolean;
68 readonly failDeleteWhen?: (tunnelId: string) => boolean;
69 readonly missingOnDeleteTunnelId?: string;
70 readonly missingOnGetTunnelId?: string;
71 readonly reserveOnGetTunnelId?: string;
72 readonly refreshedTunnels?: ReadonlyMap<string, ManagedEndpointProvider.ManagedEndpointTunnel>;
73 readonly skipTunnelId?: string;
74 readonly cleanupMode?: RelayConfiguration.ManagedEndpointCleanupMode;
75}) {
76 const listRequests: ManagedEndpointProvider.ManagedEndpointTunnelListRequest[] = [];
77 const deleted: string[] = [];
78 const releases: Array<
79 Parameters<ManagedEndpointProvider.ManagedEndpointProvider["Service"]["release"]>[0]
80 > = [];
81 const remaining = [...(input?.tunnels ?? [])];
82 const recorded = (input?.allocations ?? []).map((entry) => {
83 const matching = remaining.find((candidate) => candidate.id === entry.tunnelId);
84 return typeof matching?.name === "string" ? { ...entry, tunnelName: matching.name } : entry;
85 });
86 const tunnelClient = ManagedEndpointProvider.ManagedEndpointTunnelClient.of({
87 get: (tunnelId) =>
88 Effect.suspend(() => {
89 if (tunnelId === input?.missingOnGetTunnelId) {
90 return Effect.fail(
91 new ManagedEndpointProvider.ManagedEndpointTunnelClientError({
92 operation: "get",
93 tunnelId,
94 cause: { _tag: "NotFound" },
95 }),
96 );
97 }
98 const found =
99 input?.refreshedTunnels?.get(tunnelId) ??
100 remaining.find((candidate) => candidate.id === tunnelId);
101 if (found === undefined) {
102 return Effect.fail(
103 new ManagedEndpointProvider.ManagedEndpointTunnelClientError({
104 operation: "get",
105 tunnelId,
106 cause: { _tag: "NotFound" },
107 }),
108 );
109 }
110 if (tunnelId === input?.reserveOnGetTunnelId && typeof found.name === "string") {
111 recorded.push({
112 ...allocation({ tunnelId, recoveryEnabled: false }),
113 tunnelName: found.name,
114 });
115 }
116 return Effect.succeed(found);
117 }),
118 list: (request) =>
119 Effect.sync(() => {
120 listRequests.push(request);
121 const matching = remaining.filter((entry) => entry.status === request.status);
122 const start = ((request.page ?? 1) - 1) * (request.perPage ?? 100);
123 return {
124 result: matching.slice(start, start + (request.perPage ?? 100)),
125 resultInfo: {
126 page: request.page ?? 1,
127 perPage: request.perPage ?? 100,
128 totalCount: matching.length,
129 },
130 };
131 }),
132 create: () => Effect.die("unused"),
133 putConfiguration: () => Effect.die("unused"),
134 getToken: () => Effect.die("unused"),
135 delete: (tunnelId) =>
136 input?.failAllDeletes === true ||
137 input?.failDeleteWhen?.(tunnelId) === true ||
138 tunnelId === input?.failTunnelId ||
139 tunnelId === input?.rateLimitedTunnelId ||
140 tunnelId === input?.missingOnDeleteTunnelId
141 ? Effect.fail(
142 new ManagedEndpointProvider.ManagedEndpointTunnelClientError({
143 operation: "delete",
144 tunnelId,
145 cause:
146 tunnelId === input?.missingOnDeleteTunnelId
147 ? { _tag: "NotFound" }
148 : tunnelId === input?.rateLimitedTunnelId
149 ? {
150 cause: {
151 _tag: "TooManyRequests",
152 message: "Cloudflare rate limit exceeded",
153 retryAfter: 60,
154 },
155 }
156 : "Cloudflare refused the deletion",
157 }),
158 )
159 : Effect.sync(() => {
160 deleted.push(tunnelId);
161 const index = remaining.findIndex((candidate) => candidate.id === tunnelId);
162 if (index !== -1) {
163 remaining.splice(index, 1);
164 }
165 }),
166 });
167 const allocationService = ManagedEndpointAllocations.ManagedEndpointAllocations.of({
168 get: () => Effect.die("unused"),
169 reserve: () => Effect.die("unused"),
170 recordTunnel: () => Effect.die("unused"),
171 recordDns: () => Effect.die("unused"),
172 markReady: () => Effect.die("unused"),
173 enableRecovery: () => Effect.die("unused"),
174 listByTunnelNames: (tunnelNames) =>
175 Effect.succeed(recorded.filter((entry) => tunnelNames.includes(entry.tunnelName))),
176 claimRelease: () => Effect.die("unused"),
177 withClaimedTunnel: () => Effect.die("unused"),
178 claimDeprovision: () => Effect.die("unused"),
179 remove: () => Effect.die("unused"),
180 removeClaimed: () => Effect.die("unused"),
181 });
182 const provider = ManagedEndpointProvider.ManagedEndpointProvider.of({
183 provision: () => Effect.die("unused"),
184 reconcileOrigin: () => Effect.die("unused"),
185 prepareDeprovision: () => Effect.die("unused"),
186 deprovision: () => Effect.die("unused"),
187 release: (request) =>
188 Effect.gen(function* () {
189 releases.push(request);
190 if (request.expectedTunnelId === input?.skipTunnelId) {
191 return false;
192 }
193 if (request.expectedTunnelId !== undefined) {
194 // Surface Cloudflare failures the same way the real release does.
195 yield* tunnelClient.delete(request.expectedTunnelId).pipe(
196 Effect.mapError(
197 (cause) =>
198 new ManagedEndpointProvider.ManagedEndpointDeprovisioningFailed({
199 stage: "delete-tunnel",
200 userId: request.userId,
201 environmentId: request.environmentId,
202 tunnelId: request.expectedTunnelId!,
203 cause,
204 }),
205 ),
206 );
207 }
208 return true;
209 }),
210 });
211 const config = RelayConfiguration.RelayConfiguration.of({
212 relayIssuer: "https://relay.example.test",
213 apns: {
214 environment: "sandbox",
215 teamId: "team-id",
216 keyId: "key-id",
217 privateKey: Redacted.make("private-key"),
218 bundleId: "com.t3tools.t3code.dev",
219 },
220 apnsDeliveryJobSigningSecret: Redacted.make("job-secret"),
221 clerkSecretKey: Redacted.make("clerk-secret"),
222 clerkPublishableKey: "pk_test_test",
223 clerkJwtAudience: "t3-code-relay",
224 cloudMintPrivateKey: Redacted.make("cloud-private-key"),
225 cloudMintPublicKey: "cloud-public-key",
226 managedEndpointBaseDomain: "example.test",
227 managedEndpointNamespace: input?.namespace ?? "prod",
228 managedEndpointCleanupMode: input?.cleanupMode ?? "enabled",
229 });
230
231 return {
232 listRequests,
233 deleted,
234 releases,
235 layer: ManagedEndpointReaper.layer.pipe(
236 Layer.provide(
237 Layer.mergeAll(
238 RelayConfiguration.layer(config),
239 ManagedEndpointProvider.layerTunnelClient(tunnelClient),
240 Layer.succeed(ManagedEndpointProvider.ManagedEndpointProvider, provider),
241 Layer.succeed(ManagedEndpointAllocations.ManagedEndpointAllocations, allocationService),
242 ),
243 ),
244 ),
245 };
246}
247
248describe("ManagedEndpointReaper", () => {
249 it.effect("removes expired down and inactive tunnels from recoverable environments", () => {
250 const state = harness({
251 tunnels: [
252 tunnel({
253 id: "down-1",
254 suffix: "aaaaaaaaaaaaaaaa",
255 status: "down",
256 timestamp: "2026-08-25T11:55:00.000Z",
257 }),
258 tunnel({
259 id: "inactive-1",
260 suffix: "bbbbbbbbbbbbbbbb",
261 status: "inactive",
262 timestamp: "2026-08-25T10:59:00.000Z",
263 }),
264 ],
265 allocations: [
266 allocation({ tunnelId: "down-1", recoveryEnabled: true }),
267 allocation({ tunnelId: "inactive-1", recoveryEnabled: true }),
268 ],
269 });
270
271 return Effect.gen(function* () {
272 yield* TestClock.setTime(NOW_MILLIS);
273 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
274 expect(yield* reaper.sweep).toMatchObject({
275 scanned: 2,
276 deleted: 2,
277 skippedLegacy: 0,
278 failed: 0,
279 });
280 expect(state.deleted).toEqual(["down-1", "inactive-1"]);
281 expect(state.releases.map((request) => request.expectedTunnelId)).toEqual([
282 "down-1",
283 "inactive-1",
284 ]);
285 expect(state.listRequests).toEqual([
286 {
287 isDeleted: false,
288 includePrefix: PREFIX,
289 status: "down",
290 existedAt: "2026-08-25T11:55:00.000Z",
291 wasInactiveAt: "2026-08-25T11:55:00.000Z",
292 page: 1,
293 perPage: 100,
294 },
295 {
296 isDeleted: false,
297 includePrefix: PREFIX,
298 status: "inactive",
299 existedAt: "2026-08-25T11:00:00.000Z",
300 page: 1,
301 perPage: 100,
302 },
303 ]);
304 }).pipe(Effect.provide(state.layer));
305 });
306
307 it.effect("keeps a tunnel that never connected until it is an hour old", () => {
308 const state = harness({
309 tunnels: [
310 tunnel({
311 id: "pairing",
312 suffix: "cccccccccccccccc",
313 status: "inactive",
314 timestamp: "2026-08-25T11:30:00.000Z",
315 }),
316 ],
317 allocations: [allocation({ tunnelId: "pairing", recoveryEnabled: true })],
318 });
319
320 return Effect.gen(function* () {
321 yield* TestClock.setTime(NOW_MILLIS);
322 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
323 expect((yield* reaper.sweep).deleted).toBe(0);
324 expect(state.deleted).toEqual([]);
325 }).pipe(Effect.provide(state.layer));
326 });
327
328 it.effect("keeps recent tunnels, other stages, and tunnels without valid timestamps", () => {
329 const state = harness({
330 tunnels: [
331 tunnel({
332 id: "recent",
333 suffix: "aaaaaaaaaaaaaaaa",
334 status: "down",
335 timestamp: "2026-08-25T11:55:01.000Z",
336 }),
337 tunnel({
338 id: "other-stage",
339 prefix: `${PREFIX}julius-`,
340 suffix: "bbbbbbbbbbbbbbbb",
341 status: "down",
342 timestamp: "2026-08-25T11:00:00.000Z",
343 }),
344 tunnel({
345 id: "missing-time",
346 suffix: "cccccccccccccccc",
347 status: "inactive",
348 timestamp: null,
349 }),
350 ],
351 });
352
353 return Effect.gen(function* () {
354 yield* TestClock.setTime(NOW_MILLIS);
355 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
356 expect(yield* reaper.sweep).toMatchObject({
357 scanned: 0,
358 deleted: 0,
359 skippedLegacy: 0,
360 failed: 0,
361 });
362 expect(state.deleted).toEqual([]);
363 }).pipe(Effect.provide(state.layer));
364 });
365
366 it.effect("keeps tunnels owned by environments that cannot recover yet", () => {
367 const state = harness({
368 tunnels: [
369 tunnel({
370 id: "legacy",
371 suffix: "aaaaaaaaaaaaaaaa",
372 status: "down",
373 timestamp: "2026-08-25T11:00:00.000Z",
374 }),
375 ],
376 allocations: [allocation({ tunnelId: "legacy", recoveryEnabled: false })],
377 });
378
379 return Effect.gen(function* () {
380 yield* TestClock.setTime(NOW_MILLIS);
381 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
382 expect(yield* reaper.sweep).toMatchObject({
383 scanned: 1,
384 deleted: 0,
385 skippedLegacy: 1,
386 failed: 0,
387 });
388 expect(state.deleted).toEqual([]);
389 }).pipe(Effect.provide(state.layer));
390 });
391
392 it.effect("counts an expired tunnel with no allocation instead of deleting it", () => {
393 const state = harness({
394 tunnels: [
395 tunnel({
396 id: "orphan",
397 suffix: "cccccccccccccccc",
398 status: "down",
399 timestamp: "2026-08-25T11:00:00.000Z",
400 }),
401 ],
402 });
403
404 return Effect.gen(function* () {
405 yield* TestClock.setTime(NOW_MILLIS);
406 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
407 expect(yield* reaper.sweep).toMatchObject({
408 scanned: 1,
409 wouldDelete: 0,
410 deleted: 0,
411 skippedOrphan: 1,
412 });
413 expect(state.deleted).toEqual([]);
414 }).pipe(Effect.provide(state.layer));
415 });
416 it.effect("keeps an expired tunnel while its allocation is incomplete", () => {
417 const state = harness({
418 tunnels: [
419 tunnel({
420 id: "unrecorded",
421 suffix: "aaaaaaaaaaaaaaaa",
422 status: "inactive",
423 timestamp: "2026-08-25T11:00:00.000Z",
424 }),
425 ],
426 allocations: [allocation({ tunnelId: null, recoveryEnabled: false })],
427 });
428
429 return Effect.gen(function* () {
430 yield* TestClock.setTime(NOW_MILLIS);
431 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
432 expect((yield* reaper.sweep).deleted).toBe(0);
433 expect(state.deleted).toEqual([]);
434 }).pipe(Effect.provide(state.layer));
435 });
436
437 it.effect("does not count a tunnel that was replaced before its release", () => {
438 const state = harness({
439 tunnels: [
440 tunnel({
441 id: "replaced",
442 suffix: "aaaaaaaaaaaaaaaa",
443 status: "down",
444 timestamp: "2026-08-25T11:00:00.000Z",
445 }),
446 ],
447 allocations: [allocation({ tunnelId: "replaced", recoveryEnabled: true })],
448 skipTunnelId: "replaced",
449 });
450
451 return Effect.gen(function* () {
452 yield* TestClock.setTime(NOW_MILLIS);
453 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
454 expect((yield* reaper.sweep).deleted).toBe(0);
455 expect(state.deleted).toEqual([]);
456 }).pipe(Effect.provide(state.layer));
457 });
458
459 it.effect("stops the sweep after a structured Cloudflare rate limit error", () => {
460 const state = harness({
461 tunnels: [
462 tunnel({
463 id: "limited",
464 suffix: "aaaaaaaaaaaaaaaa",
465 status: "down",
466 timestamp: "2026-08-25T11:00:00.000Z",
467 }),
468 tunnel({
469 id: "next",
470 suffix: "bbbbbbbbbbbbbbbb",
471 status: "down",
472 timestamp: "2026-08-25T11:00:00.000Z",
473 }),
474 ],
475 allocations: [
476 allocation({ tunnelId: "limited", recoveryEnabled: true }),
477 allocation({ tunnelId: "next", recoveryEnabled: true }),
478 ],
479 rateLimitedTunnelId: "limited",
480 });
481
482 return Effect.gen(function* () {
483 yield* TestClock.setTime(NOW_MILLIS);
484 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
485 expect(yield* reaper.sweep).toMatchObject({
486 attempted: 1,
487 deleted: 0,
488 failed: 1,
489 truncated: true,
490 });
491 expect(state.deleted).toEqual([]);
492 }).pipe(Effect.provide(state.layer));
493 });
494
495 it.effect("continues past a page of older hosts to find recoverable tunnels", () => {
496 const entries = Array.from({ length: 101 }, (_, index) =>
497 tunnel({
498 id: `tunnel-${index}`,
499 suffix: index.toString(16).padStart(16, "0"),
500 status: "down",
501 timestamp: "2026-08-25T11:00:00.000Z",
502 }),
503 );
504 const state = harness({
505 tunnels: entries,
506 allocations: entries.map((entry, index) =>
507 allocation({ tunnelId: entry.id!, recoveryEnabled: index === 100 }),
508 ),
509 });
510
511 return Effect.gen(function* () {
512 yield* TestClock.setTime(NOW_MILLIS);
513 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
514 expect(yield* reaper.sweep).toMatchObject({
515 scanned: 101,
516 deleted: 1,
517 skippedLegacy: 100,
518 failed: 0,
519 });
520 expect(state.deleted).toEqual(["tunnel-100"]);
521 expect(
522 state.listRequests
523 .filter((request) => request.status === "down")
524 .map((request) => request.page),
525 ).toEqual([1, 2]);
526 }).pipe(Effect.provide(state.layer));
527 });
528
529 it.effect("collects every page before deletions shift Cloudflare pagination", () => {
530 const entries = Array.from({ length: 120 }, (_, index) =>
531 tunnel({
532 id: `tunnel-${index}`,
533 suffix: index.toString(16).padStart(16, "0"),
534 status: "down",
535 timestamp: "2026-08-25T11:00:00.000Z",
536 }),
537 );
538 const state = harness({
539 tunnels: entries,
540 allocations: entries.map((entry, index) =>
541 allocation({ tunnelId: entry.id!, recoveryEnabled: index < 50 || index >= 100 }),
542 ),
543 });
544
545 return Effect.gen(function* () {
546 yield* TestClock.setTime(NOW_MILLIS);
547 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
548 expect(yield* reaper.sweep).toMatchObject({
549 scanned: 120,
550 deleted: 70,
551 skippedLegacy: 50,
552 failed: 0,
553 });
554 expect(state.deleted).toContain("tunnel-119");
555 }).pipe(Effect.provide(state.layer));
556 });
557
558 it.effect("limits each cleanup run to 100 tunnel deletions", () => {
559 const entries = Array.from({ length: 105 }, (_, index) =>
560 tunnel({
561 id: `tunnel-${index}`,
562 suffix: index.toString(16).padStart(16, "0"),
563 status: "down",
564 timestamp: "2026-08-25T11:00:00.000Z",
565 }),
566 );
567 const state = harness({ tunnels: entries, allocations: recoverableOwners(entries) });
568
569 return Effect.gen(function* () {
570 yield* TestClock.setTime(NOW_MILLIS);
571 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
572 expect((yield* reaper.sweep).deleted).toBe(100);
573 expect(state.deleted).toHaveLength(100);
574 }).pipe(Effect.provide(state.layer));
575 });
576
577 it.effect("does no Cloudflare work while cleanup is off", () => {
578 const state = harness({
579 cleanupMode: "off",
580 tunnels: [
581 tunnel({
582 id: "expired",
583 suffix: "aaaaaaaaaaaaaaaa",
584 status: "down",
585 timestamp: "2026-08-25T11:00:00.000Z",
586 }),
587 ],
588 allocations: [allocation({ tunnelId: "expired", recoveryEnabled: true })],
589 });
590
591 return Effect.gen(function* () {
592 yield* TestClock.setTime(NOW_MILLIS);
593 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
594 expect(yield* reaper.sweep).toEqual({
595 mode: "off",
596 listRequests: 0,
597 scanned: 0,
598 attempted: 0,
599 deleted: 0,
600 wouldDelete: 0,
601 skippedLegacy: 0,
602 skippedOrphan: 0,
603 failed: 0,
604 truncated: false,
605 });
606 expect(state.listRequests).toEqual([]);
607 expect(state.deleted).toEqual([]);
608 }).pipe(Effect.provide(state.layer));
609 });
610
611 it.effect("reports candidates without mutating them in dry-run mode", () => {
612 const state = harness({
613 cleanupMode: "dry-run",
614 tunnels: [
615 tunnel({
616 id: "expired",
617 suffix: "aaaaaaaaaaaaaaaa",
618 status: "down",
619 timestamp: "2026-08-25T11:00:00.000Z",
620 }),
621 ],
622 allocations: [allocation({ tunnelId: "expired", recoveryEnabled: true })],
623 });
624
625 return Effect.gen(function* () {
626 yield* TestClock.setTime(NOW_MILLIS);
627 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
628 expect(yield* reaper.sweep).toMatchObject({
629 mode: "dry-run",
630 scanned: 1,
631 attempted: 0,
632 deleted: 0,
633 wouldDelete: 1,
634 });
635 expect(state.deleted).toEqual([]);
636 expect(state.releases).toEqual([]);
637 }).pipe(Effect.provide(state.layer));
638 });
639
640 it.effect("caps failed deletion attempts and Cloudflare list pages", () => {
641 const entries = Array.from({ length: 250 }, (_, index) =>
642 tunnel({
643 id: `failed-${index}`,
644 suffix: index.toString(16).padStart(16, "0"),
645 status: "down",
646 timestamp: "2026-08-25T11:00:00.000Z",
647 }),
648 );
649 const state = harness({
650 tunnels: entries,
651 allocations: recoverableOwners(entries),
652 failAllDeletes: true,
653 });
654
655 return Effect.gen(function* () {
656 yield* TestClock.setTime(NOW_MILLIS);
657 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
658 expect(yield* reaper.sweep).toMatchObject({
659 attempted: 100,
660 deleted: 0,
661 failed: 100,
662 truncated: true,
663 });
664 expect(state.listRequests.length).toBeLessThanOrEqual(
665 ManagedEndpointReaper.MANAGED_ENDPOINT_SWEEP_LIST_REQUEST_LIMIT,
666 );
667 }).pipe(Effect.provide(state.layer));
668 });
669
670 it.effect("rotates the deletion order so persistent failures do not starve later tunnels", () => {
671 const entries = Array.from({ length: 200 }, (_, index) =>
672 tunnel({
673 id: `tunnel-${index}`,
674 suffix: index.toString(16).padStart(16, "0"),
675 status: "down",
676 timestamp: "2026-08-25T11:00:00.000Z",
677 }),
678 );
679 // The first 100 candidates always fail to delete.
680 const state = harness({
681 tunnels: entries,
682 allocations: recoverableOwners(entries),
683 failDeleteWhen: (id) => Number(id.split("-")[1]) < 100,
684 });
685
686 return Effect.gen(function* () {
687 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
688 yield* TestClock.setTime(NOW_MILLIS);
689 yield* reaper.sweep;
690 yield* TestClock.setTime(NOW_MILLIS + 5 * 60 * 1_000);
691 yield* reaper.sweep;
692 const later = state.deleted.filter((id) => Number(id.split("-")[1]) >= 100);
693 expect(later.length).toBeGreaterThan(0);
694 }).pipe(Effect.provide(state.layer));
695 });
696
697 it.effect("rotates bounded pages across a large legacy prefix", () => {
698 const entries = Array.from({ length: 1_000 }, (_, index) =>
699 tunnel({
700 id: `legacy-${index}`,
701 suffix: index.toString(16).padStart(16, "0"),
702 status: "down",
703 timestamp: "2026-08-25T11:00:00.000Z",
704 }),
705 );
706 const state = harness({
707 cleanupMode: "dry-run",
708 tunnels: entries,
709 allocations: entries.map((entry) =>
710 allocation({ tunnelId: entry.id!, recoveryEnabled: false }),
711 ),
712 });
713
714 return Effect.gen(function* () {
715 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
716 yield* TestClock.setTime(NOW_MILLIS);
717 yield* reaper.sweep;
718 const firstPages = state.listRequests
719 .filter((request) => request.status === "down")
720 .map((request) => request.page);
721 state.listRequests.length = 0;
722 yield* TestClock.setTime(NOW_MILLIS + 5 * 60 * 1_000);
723 yield* reaper.sweep;
724 const secondPages = state.listRequests
725 .filter((request) => request.status === "down")
726 .map((request) => request.page);
727 expect(firstPages).not.toEqual(secondPages);
728 expect(firstPages).toHaveLength(5);
729 expect(secondPages).toHaveLength(5);
730 }).pipe(Effect.provide(state.layer));
731 });
732
733 it.effect("rotates the attempt budget across both statuses over consecutive sweeps", () => {
734 const entries = (["down", "inactive"] as const).flatMap((status) =>
735 Array.from({ length: 100 }, (_, index) =>
736 tunnel({
737 id: `${status}-${index}`,
738 suffix: `${status === "down" ? "a" : "b"}${index.toString(16).padStart(15, "0")}`,
739 status,
740 timestamp: "2026-08-25T10:00:00.000Z",
741 }),
742 ),
743 );
744 const state = harness({ tunnels: entries, allocations: recoverableOwners(entries) });
745
746 return Effect.gen(function* () {
747 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
748 yield* TestClock.setTime(NOW_MILLIS);
749 yield* reaper.sweep;
750 const firstStatus = state.deleted[0]!.split("-")[0];
751 expect(state.deleted).toHaveLength(100);
752 expect(state.deleted.every((id) => id.startsWith(`${firstStatus}-`))).toBe(true);
753
754 state.deleted.length = 0;
755 yield* TestClock.setTime(NOW_MILLIS + 5 * 60 * 1_000);
756 yield* reaper.sweep;
757 expect(state.deleted).toHaveLength(100);
758 expect(state.deleted.every((id) => !id.startsWith(`${firstStatus}-`))).toBe(true);
759 }).pipe(Effect.provide(state.layer));
760 });
761});
infra/relay/src/environments/ManagedEndpointReaper.ts created+295
......@@ -0,0 +1,295 @@
1import * as Context from "effect/Context";
2import * as DateTime from "effect/DateTime";
3import * as Effect from "effect/Effect";
4import * as Layer from "effect/Layer";
5import * as Option from "effect/Option";
6
7import type { ManagedEndpointCleanupMode } from "../Config.ts";
8import * as RelayConfiguration from "../Config.ts";
9import { managedEndpointTunnelNamePrefix } from "../deploymentConfig.ts";
10import * as ManagedEndpointAllocations from "./ManagedEndpointAllocations.ts";
11import * as ManagedEndpointProvider from "./ManagedEndpointProvider.ts";
12
13export const MANAGED_ENDPOINT_GRACE_PERIOD_MINUTES = 5;
14// A tunnel that never connected is usually a link still being set up: a slow
15// cloudflared download or a user who walked away mid-pairing. Give it an hour.
16export const MANAGED_ENDPOINT_INACTIVE_GRACE_PERIOD_MINUTES = 60;
17export const MANAGED_ENDPOINT_SWEEP_PAGE_SIZE = 100;
18export const MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT = 100;
19export const MANAGED_ENDPOINT_SWEEP_LIST_REQUEST_LIMIT = 10;
20
21export interface ManagedEndpointSweepResult {
22 readonly mode: ManagedEndpointCleanupMode;
23 readonly listRequests: number;
24 readonly scanned: number;
25 readonly attempted: number;
26 readonly deleted: number;
27 readonly wouldDelete: number;
28 readonly skippedLegacy: number;
29 readonly skippedOrphan: number;
30 readonly failed: number;
31 readonly truncated: boolean;
32}
33
34export class ManagedEndpointReaper extends Context.Service<
35 ManagedEndpointReaper,
36 {
37 readonly sweep: Effect.Effect<
38 ManagedEndpointSweepResult,
39 | ManagedEndpointProvider.ManagedEndpointTunnelClientError
40 | ManagedEndpointAllocations.ManagedEndpointAllocationPersistenceError
41 >;
42 }
43>()("t3code-relay/environments/ManagedEndpointReaper") {}
44
45function isExpiredManagedTunnel(input: {
46 readonly tunnel: ManagedEndpointProvider.ManagedEndpointTunnel;
47 readonly status: "down" | "inactive";
48 readonly prefix: string;
49 readonly cutoff: DateTime.Utc;
50}): input is typeof input & {
51 readonly tunnel: ManagedEndpointProvider.ManagedEndpointTunnel & {
52 readonly id: string;
53 readonly name: string;
54 };
55} {
56 const { tunnel, status, prefix, cutoff } = input;
57 if (
58 typeof tunnel.id !== "string" ||
59 typeof tunnel.name !== "string" ||
60 tunnel.status !== status ||
61 !tunnel.name.startsWith(prefix) ||
62 !/^[a-f0-9]{16}$/u.test(tunnel.name.slice(prefix.length))
63 ) {
64 return false;
65 }
66 const inactiveAt = status === "down" ? tunnel.connsInactiveAt : tunnel.createdAt;
67 if (typeof inactiveAt !== "string") {
68 return false;
69 }
70 const timestamp = DateTime.make(inactiveAt);
71 return Option.isSome(timestamp) && timestamp.value.epochMilliseconds <= cutoff.epochMilliseconds;
72}
73
74function isRateLimited(cause: unknown): boolean {
75 if (typeof cause !== "object" || cause === null) {
76 return false;
77 }
78 if ("_tag" in cause && cause._tag === "TooManyRequests") {
79 return true;
80 }
81 if ("status" in cause && cause.status === 429) {
82 return true;
83 }
84 return "cause" in cause && isRateLimited(cause.cause);
85}
86
87function rotatedPages(input: {
88 readonly totalCount: number | undefined;
89 readonly slot: number;
90 readonly limit: number;
91}): ReadonlyArray<number> {
92 if (input.limit <= 0) return [];
93 if (input.totalCount === undefined) {
94 return Array.from({ length: input.limit }, (_, index) => index + 2);
95 }
96 const laterPageCount = Math.max(
97 0,
98 Math.ceil(input.totalCount / MANAGED_ENDPOINT_SWEEP_PAGE_SIZE) - 1,
99 );
100 if (laterPageCount === 0) return [];
101 const count = Math.min(input.limit, laterPageCount);
102 const start = input.slot % laterPageCount;
103 return Array.from({ length: count }, (_, index) => 2 + ((start + index) % laterPageCount));
104}
105
106const emptyResult = (mode: ManagedEndpointCleanupMode): ManagedEndpointSweepResult => ({
107 mode,
108 listRequests: 0,
109 scanned: 0,
110 attempted: 0,
111 deleted: 0,
112 wouldDelete: 0,
113 skippedLegacy: 0,
114 skippedOrphan: 0,
115 failed: 0,
116 truncated: false,
117});
118
119export const make = Effect.gen(function* () {
120 const config = yield* RelayConfiguration.RelayConfiguration;
121 const tunnels = yield* ManagedEndpointProvider.ManagedEndpointTunnelClient;
122 const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations;
123 const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
124
125 const sweep = Effect.gen(function* () {
126 const mode = config.managedEndpointCleanupMode ?? "off";
127 const namespace = config.managedEndpointNamespace;
128 if (mode === "off" || !namespace) return emptyResult(mode);
129
130 const now = yield* DateTime.now;
131 const cutoffFor = (status: "down" | "inactive") =>
132 DateTime.subtract(now, {
133 minutes:
134 status === "down"
135 ? MANAGED_ENDPOINT_GRACE_PERIOD_MINUTES
136 : MANAGED_ENDPOINT_INACTIVE_GRACE_PERIOD_MINUTES,
137 });
138 const prefix = managedEndpointTunnelNamePrefix(namespace);
139 const slot = Math.floor(
140 now.epochMilliseconds / (MANAGED_ENDPOINT_GRACE_PERIOD_MINUTES * 60 * 1_000),
141 );
142 let listRequests = 0;
143 let truncated = false;
144 const expired: Array<{
145 readonly tunnel: ManagedEndpointProvider.ManagedEndpointTunnel & {
146 readonly id: string;
147 readonly name: string;
148 };
149 readonly status: "down" | "inactive";
150 readonly cutoff: DateTime.Utc;
151 }> = [];
152
153 for (const status of ["down", "inactive"] as const) {
154 const cutoff = cutoffFor(status);
155 const cutoffIso = DateTime.formatIso(cutoff);
156 const listPage = (page: number) => {
157 listRequests += 1;
158 return tunnels.list({
159 isDeleted: false,
160 includePrefix: prefix,
161 status,
162 existedAt: cutoffIso,
163 ...(status === "down" ? { wasInactiveAt: cutoffIso } : {}),
164 page,
165 perPage: MANAGED_ENDPOINT_SWEEP_PAGE_SIZE,
166 });
167 };
168 const first = yield* listPage(1);
169 const totalCount =
170 typeof first.resultInfo?.totalCount === "number" ? first.resultInfo.totalCount : undefined;
171 const pages = rotatedPages({
172 totalCount,
173 slot,
174 limit: Math.floor(MANAGED_ENDPOINT_SWEEP_LIST_REQUEST_LIMIT / 2) - 1,
175 });
176 const responses = [first, ...(yield* Effect.forEach(pages, listPage, { concurrency: 1 }))];
177 if (
178 totalCount !== undefined &&
179 Math.ceil(totalCount / MANAGED_ENDPOINT_SWEEP_PAGE_SIZE) > responses.length
180 ) {
181 truncated = true;
182 } else if (
183 totalCount === undefined &&
184 responses.at(-1)?.result.length === MANAGED_ENDPOINT_SWEEP_PAGE_SIZE
185 ) {
186 truncated = true;
187 }
188 for (const response of responses) {
189 expired.push(
190 ...response.result
191 .map((tunnel) => ({ tunnel, status, prefix, cutoff }))
192 .filter(isExpiredManagedTunnel)
193 .map(({ tunnel }) => ({ tunnel, status, cutoff })),
194 );
195 }
196 }
197
198 const collected = [...new Map(expired.map((entry) => [entry.tunnel.id, entry])).values()];
199 // Start each sweep one attempt budget further along so a run of
200 // candidates whose deletes keep failing cannot hold the budget forever
201 // and starve everything listed after them.
202 const offset =
203 collected.length === 0 ? 0 : (slot * MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT) % collected.length;
204 const uniqueExpired = [...collected.slice(offset), ...collected.slice(0, offset)];
205 const recorded = yield* allocations.listByTunnelNames(
206 uniqueExpired.map(({ tunnel }) => tunnel.name),
207 );
208 const recordedByTunnelName = new Map(
209 recorded.map((allocation) => [allocation.tunnelName, allocation]),
210 );
211 let attempted = 0;
212 let deleted = 0;
213 let wouldDelete = 0;
214 let skippedLegacy = 0;
215 let skippedOrphan = 0;
216 let failed = 0;
217
218 for (const { tunnel, status, cutoff } of uniqueExpired) {
219 const cutoffIso = DateTime.formatIso(cutoff);
220 const allocation = recordedByTunnelName.get(tunnel.name);
221 if (
222 allocation !== undefined &&
223 allocation.tunnelId !== null &&
224 allocation.tunnelId !== tunnel.id
225 ) {
226 continue;
227 }
228 const owner = allocation?.tunnelId === tunnel.id ? allocation : undefined;
229 if (owner !== undefined && !owner.recoveryEnabled) {
230 skippedLegacy += 1;
231 continue;
232 }
233 if (allocation !== undefined && owner === undefined) continue;
234 // A tunnel with no allocation row cannot be claimed, so a relink that
235 // adopts it by name races any delete here. Count it and leave it for a
236 // manual sweep instead.
237 if (owner === undefined) {
238 skippedOrphan += 1;
239 continue;
240 }
241 wouldDelete += 1;
242 if (mode === "dry-run") continue;
243 if (attempted >= MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT) {
244 truncated = true;
245 break;
246 }
247 attempted += 1;
248 const result = yield* provider
249 .release({
250 userId: owner.userId,
251 environmentId: owner.environmentId,
252 expectedTunnelId: tunnel.id,
253 expectedInactiveBefore: cutoffIso,
254 expectedStatus: status,
255 })
256 .pipe(Effect.result);
257 if (result._tag === "Failure") {
258 failed += 1;
259 yield* Effect.logWarning("Failed to delete an inactive managed tunnel", {
260 tunnelId: tunnel.id,
261 tunnelName: tunnel.name,
262 cause: result.failure,
263 });
264 if (isRateLimited(result.failure)) {
265 truncated = true;
266 break;
267 }
268 } else if (result.success) {
269 deleted += 1;
270 yield* Effect.logInfo("Deleted an inactive managed tunnel", {
271 tunnelId: tunnel.id,
272 tunnelName: tunnel.name,
273 status,
274 });
275 }
276 }
277
278 return {
279 mode,
280 listRequests,
281 scanned: uniqueExpired.length,
282 attempted,
283 deleted,
284 wouldDelete,
285 skippedLegacy,
286 skippedOrphan,
287 failed,
288 truncated,
289 };
290 }).pipe(Effect.withSpan("relay.managed_endpoint_reaper.sweep"));
291
292 return ManagedEndpointReaper.of({ sweep });
293});
294
295export const layer = Layer.effect(ManagedEndpointReaper, make);
infra/relay/src/http/Api.test.ts+570-6
......@@ -7,13 +7,15 @@ import {
77import * as EnvironmentLinker from "../environments/EnvironmentLinker.ts";
88import * as RelayTokens from "../auth/RelayTokens.ts";
99import * as Devices from "../agentActivity/Devices.ts";
10import * as NodeCrypto from "node:crypto";
1011import { createClerkClient, verifyToken } from "@clerk/backend";
1112import * as NodeHttpPlatform from "@effect/platform-node/NodeHttpPlatform";
1213import * as NodeServices from "@effect/platform-node/NodeServices";
1314import { describe, expect, it } from "@effect/vitest";
1415import { vi } from "vite-plus/test";
1516import * as Context from "effect/Context";
16import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
17import * as NodeCryptoLayer from "@effect/platform-node/NodeCrypto";
18import * as DateTime from "effect/DateTime";
1719import * as Duration from "effect/Duration";
1820import * as Effect from "effect/Effect";
1921import * as Fiber from "effect/Fiber";
......@@ -35,6 +37,7 @@ import {
3537 RelayEnvironmentPrincipal,
3638 RelayApi,
3739} from "@t3tools/contracts/relay";
40import { RELAY_MANAGED_TUNNEL_RECOVERY_TYP, signRelayJwt } from "@t3tools/shared/relayJwt";
3841
3942import {
4043 RELAY_HTTP_ROUTER_CONFIG,
......@@ -44,18 +47,22 @@ import {
4447 relayDocsRedirectRoute,
4548 relayEnvironmentAuthLayer,
4649 relayNotFoundRoute,
50 recoverEnvironmentTunnelRecord,
51 registerEnvironmentTunnelRecovery,
4752 relayDpopFailureReason,
4853 revokeEnvironmentLinkRecord,
4954 serverApi,
5055 traceRelayHttpRequestWith,
5156 unlinkEnvironmentRecord,
5257 verifyRelayClientBearerToken,
58 verifyEnvironmentTunnelRecoveryProof,
5359 withoutCapturedParentSpan,
5460} from "./Api.ts";
5561import * as RelayConfiguration from "../Config.ts";
5662import * as RelayDb from "../db.ts";
5763import * as EnvironmentCredentials from "../environments/EnvironmentCredentials.ts";
5864import * as EnvironmentLinks from "../environments/EnvironmentLinks.ts";
65import * as ManagedEndpointAllocations from "../environments/ManagedEndpointAllocations.ts";
5966import * as ManagedEndpointProvider from "../environments/ManagedEndpointProvider.ts";
6067import * as AgentActivityPublisher from "../agentActivity/AgentActivityPublisher.ts";
6168import * as EnvironmentPublishSignatures from "../environments/EnvironmentPublishSignatures.ts";
......@@ -122,7 +129,7 @@ describe("device listing compatibility", () => {
122129 Layer.provide(
123130 Layer.mergeAll(
124131 Layer.succeed(RelayConfiguration.RelayConfiguration, relaySettings),
125 NodeCrypto.layer,
132 NodeCryptoLayer.layer,
126133 Layer.mock(RelayTokens.RelayTokens, { resolveDpopAccessTokenScopes: () => null }),
127134 Layer.mock(EnvironmentLinker.EnvironmentLinker, {}),
128135 Layer.mock(EnvironmentLinks.EnvironmentLinks, {}),
......@@ -307,6 +314,9 @@ function relayUnlinkTestLayer(input?: {
307314 readonly revokeCredential?: EnvironmentCredentials.EnvironmentCredentials["Service"]["revokeForEnvironmentPublicKey"];
308315 readonly prepareDeprovision?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["prepareDeprovision"];
309316 readonly deprovision?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["deprovision"];
317 readonly provision?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["provision"];
318 readonly reconcileOrigin?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["reconcileOrigin"];
319 readonly release?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["release"];
310320}) {
311321 return Layer.mergeAll(
312322 Layer.succeed(
......@@ -336,10 +346,11 @@ function relayUnlinkTestLayer(input?: {
336346 Layer.succeed(
337347 ManagedEndpointProvider.ManagedEndpointProvider,
338348 ManagedEndpointProvider.ManagedEndpointProvider.of({
339 provision: () => Effect.die("unused provision"),
349 provision: input?.provision ?? (() => Effect.die("unused provision")),
350 reconcileOrigin: input?.reconcileOrigin ?? (() => Effect.succeed("ready")),
340351 prepareDeprovision: input?.prepareDeprovision ?? (() => Effect.succeed(null)),
341 deprovision: input?.deprovision ?? (() => Effect.void),
342 release: () => Effect.die("unused release"),
352 deprovision: input?.deprovision ?? (() => Effect.succeed(true)),
353 release: input?.release ?? (() => Effect.die("unused release")),
343354 }),
344355 ),
345356 );
......@@ -357,6 +368,502 @@ const linkedEnvironmentRecord = {
357368 linkedAt: "2026-07-28T00:00:00.000Z",
358369} as const;
359370
371describe("relay managed tunnel recovery", () => {
372 it.effect("binds recovery requests to the host, cloud user, and T3 service origin", () =>
373 Effect.gen(function* () {
374 const keyPair = NodeCrypto.generateKeyPairSync("ed25519", {
375 privateKeyEncoding: { format: "pem", type: "pkcs8" },
376 publicKeyEncoding: { format: "pem", type: "spki" },
377 });
378 const now = yield* DateTime.now;
379 const issuedAt = Math.floor(now.epochMilliseconds / 1_000);
380 const proof = yield* signRelayJwt({
381 privateKey: keyPair.privateKey,
382 typ: RELAY_MANAGED_TUNNEL_RECOVERY_TYP,
383 payload: {
384 iss: "t3-env:environment-1",
385 aud: "https://relay.example.test",
386 sub: "environment-1",
387 jti: "recovery-proof",
388 iat: issuedAt,
389 exp: issuedAt + 60,
390 action: "recover",
391 environmentId: "environment-1",
392 cloudUserId: "user-1",
393 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
394 },
395 });
396 const request = {
397 action: "recover" as const,
398 proof,
399 userId: "user-1",
400 environmentId: "environment-1",
401 environmentPublicKey: keyPair.publicKey,
402 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
403 };
404
405 yield* verifyEnvironmentTunnelRecoveryProof(request);
406
407 const wrongOwner = yield* Effect.flip(
408 verifyEnvironmentTunnelRecoveryProof({ ...request, userId: "user-2" }),
409 );
410 expect(wrongOwner).toMatchObject({ _tag: "Unauthorized" });
411
412 const wrongOrigin = yield* Effect.flip(
413 verifyEnvironmentTunnelRecoveryProof({
414 ...request,
415 origin: { localHttpHost: "127.0.0.1", localHttpPort: 5432 },
416 }),
417 );
418 expect(wrongOrigin).toMatchObject({ _tag: "Unauthorized" });
419
420 const wrongAction = yield* Effect.flip(
421 verifyEnvironmentTunnelRecoveryProof({
422 action: "register",
423 proof,
424 userId: "user-1",
425 environmentId: "environment-1",
426 environmentPublicKey: keyPair.publicKey,
427 tunnelId: "existing-tunnel",
428 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
429 }),
430 );
431 expect(wrongAction).toMatchObject({ _tag: "Unauthorized" });
432 }).pipe(Effect.provideService(RelayConfiguration.RelayConfiguration, relaySettings)),
433 );
434
435 it.effect("rejects a signed registration proof for a different origin", () =>
436 Effect.gen(function* () {
437 const keyPair = NodeCrypto.generateKeyPairSync("ed25519", {
438 privateKeyEncoding: { format: "pem", type: "pkcs8" },
439 publicKeyEncoding: { format: "pem", type: "spki" },
440 });
441 const now = yield* DateTime.now;
442 const issuedAt = Math.floor(now.epochMilliseconds / 1_000);
443 const proof = yield* signRelayJwt({
444 privateKey: keyPair.privateKey,
445 typ: RELAY_MANAGED_TUNNEL_RECOVERY_TYP,
446 payload: {
447 iss: "t3-env:environment-1",
448 aud: "https://relay.example.test",
449 sub: "environment-1",
450 jti: "registration-origin-proof",
451 iat: issuedAt,
452 exp: issuedAt + 60,
453 action: "register",
454 environmentId: "environment-1",
455 cloudUserId: "user-1",
456 tunnelId: "existing-tunnel",
457 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
458 },
459 });
460
461 const error = yield* Effect.flip(
462 verifyEnvironmentTunnelRecoveryProof({
463 action: "register",
464 proof,
465 userId: "user-1",
466 environmentId: "environment-1",
467 environmentPublicKey: keyPair.publicKey,
468 tunnelId: "existing-tunnel",
469 origin: { localHttpHost: "127.0.0.1", localHttpPort: 5432 },
470 }),
471 );
472
473 expect(error).toMatchObject({ _tag: "Unauthorized" });
474 }).pipe(Effect.provideService(RelayConfiguration.RelayConfiguration, relaySettings)),
475 );
476
477 it.effect("registers recovery for an existing tunnel without provisioning it", () => {
478 let recoveryEnabledFor: {
479 readonly userId: string;
480 readonly environmentId: string;
481 readonly tunnelId: string;
482 readonly environmentPublicKey: string;
483 readonly origin: { readonly localHttpHost: string; readonly localHttpPort: number };
484 } | null = null;
485
486 return Effect.gen(function* () {
487 expect(
488 yield* registerEnvironmentTunnelRecovery({
489 userId: "user-1",
490 environmentId: "environment-1",
491 environmentPublicKey: "public-key",
492 tunnelId: "existing-tunnel",
493 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
494 }),
495 ).toEqual({ status: "ready" });
496 expect(recoveryEnabledFor).toEqual({
497 userId: "user-1",
498 environmentId: "environment-1",
499 tunnelId: "existing-tunnel",
500 environmentPublicKey: "public-key",
501 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
502 });
503 }).pipe(
504 Effect.provide(
505 Layer.merge(
506 relayUnlinkTestLayer({
507 getForUser: () => Effect.succeed(linkedEnvironmentRecord),
508 provision: () => Effect.die("registration must not provision a tunnel"),
509 }),
510 Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({
511 enableRecovery: (input) =>
512 Effect.sync(() => {
513 recoveryEnabledFor = input;
514 return true;
515 }),
516 }),
517 ),
518 ),
519 );
520 });
521
522 it.effect("requests recovery without enabling a stale tunnel", () => {
523 let recoveryEnabled = false;
524
525 return Effect.gen(function* () {
526 expect(
527 yield* registerEnvironmentTunnelRecovery({
528 userId: "user-1",
529 environmentId: "environment-1",
530 environmentPublicKey: "public-key",
531 tunnelId: "deleted-tunnel",
532 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
533 }),
534 ).toEqual({ status: "recovery_required" });
535 expect(recoveryEnabled).toBe(false);
536 }).pipe(
537 Effect.provide(
538 Layer.merge(
539 relayUnlinkTestLayer({
540 getForUser: () => Effect.succeed(linkedEnvironmentRecord),
541 reconcileOrigin: () => Effect.succeed("recovery_required"),
542 }),
543 Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({
544 enableRecovery: () =>
545 Effect.sync(() => {
546 recoveryEnabled = true;
547 return true;
548 }),
549 }),
550 ),
551 ),
552 );
553 });
554
555 it.effect("rejects recovery registration for a different environment key", () => {
556 let recoveryEnabled = false;
557
558 return Effect.gen(function* () {
559 const error = yield* Effect.flip(
560 registerEnvironmentTunnelRecovery({
561 userId: "user-1",
562 environmentId: "environment-1",
563 environmentPublicKey: "different-public-key",
564 tunnelId: "existing-tunnel",
565 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
566 }),
567 );
568
569 expect(error).toMatchObject({ _tag: "Unauthorized" });
570 expect(recoveryEnabled).toBe(false);
571 }).pipe(
572 Effect.provide(
573 Layer.merge(
574 relayUnlinkTestLayer({
575 getForUser: () => Effect.succeed(linkedEnvironmentRecord),
576 }),
577 Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({
578 enableRecovery: () =>
579 Effect.sync(() => {
580 recoveryEnabled = true;
581 return true;
582 }),
583 }),
584 ),
585 ),
586 );
587 });
588
589 it.effect("rejects recovery registration when the recorded tunnel changed", () =>
590 Effect.gen(function* () {
591 const error = yield* Effect.flip(
592 registerEnvironmentTunnelRecovery({
593 userId: "user-1",
594 environmentId: "environment-1",
595 environmentPublicKey: "public-key",
596 tunnelId: "stale-tunnel",
597 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
598 }),
599 );
600
601 expect(error).toMatchObject({ _tag: "Unauthorized" });
602 }).pipe(
603 Effect.provide(
604 Layer.merge(
605 relayUnlinkTestLayer({
606 getForUser: () => Effect.succeed(linkedEnvironmentRecord),
607 }),
608 Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({
609 enableRecovery: () => Effect.succeed(false),
610 }),
611 ),
612 ),
613 ),
614 );
615
616 it.effect("recovers a linked environment and marks its tunnel as recoverable", () => {
617 let recoveryEnabledFor: {
618 readonly userId: string;
619 readonly environmentId: string;
620 readonly tunnelId: string;
621 readonly environmentPublicKey: string;
622 } | null = null;
623 const runtime = {
624 providerKind: "cloudflare_tunnel" as const,
625 connectorToken: "replacement-token",
626 tunnelId: "replacement-tunnel",
627 };
628
629 return Effect.gen(function* () {
630 expect(
631 yield* recoverEnvironmentTunnelRecord({
632 userId: "user-1",
633 environmentId: "environment-1",
634 environmentPublicKey: "public-key",
635 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
636 }),
637 ).toEqual({
638 endpoint: linkedEnvironmentRecord.endpoint,
639 endpointRuntime: runtime,
640 });
641 expect(recoveryEnabledFor).toEqual({
642 userId: "user-1",
643 environmentId: "environment-1",
644 tunnelId: "replacement-tunnel",
645 environmentPublicKey: "public-key",
646 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
647 });
648 }).pipe(
649 Effect.provide(
650 Layer.merge(
651 relayUnlinkTestLayer({
652 getForUser: () => Effect.succeed(linkedEnvironmentRecord),
653 provision: () =>
654 Effect.succeed({
655 endpoint: linkedEnvironmentRecord.endpoint,
656 runtime,
657 }),
658 }),
659 Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({
660 enableRecovery: (input) =>
661 Effect.sync(() => {
662 recoveryEnabledFor = input;
663 return true;
664 }),
665 }),
666 ),
667 ),
668 );
669 });
670
671 it.effect("rejects a credential from a different environment owner", () => {
672 let provisioned = false;
673
674 return Effect.gen(function* () {
675 const error = yield* Effect.flip(
676 recoverEnvironmentTunnelRecord({
677 userId: "user-1",
678 environmentId: "environment-1",
679 environmentPublicKey: "different-public-key",
680 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
681 }),
682 );
683 expect(error).toMatchObject({ _tag: "Unauthorized" });
684 expect(provisioned).toBe(false);
685 }).pipe(
686 Effect.provide(
687 Layer.merge(
688 relayUnlinkTestLayer({
689 getForUser: () => Effect.succeed(linkedEnvironmentRecord),
690 provision: () =>
691 Effect.sync(() => {
692 provisioned = true;
693 return {
694 endpoint: linkedEnvironmentRecord.endpoint,
695 runtime: { providerKind: "cloudflare_tunnel", connectorToken: "token" },
696 };
697 }),
698 }),
699 Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({
700 enableRecovery: () => Effect.die("unused"),
701 }),
702 ),
703 ),
704 );
705 });
706
707 it.effect("does not recover a publish-only environment", () =>
708 Effect.gen(function* () {
709 const error = yield* Effect.flip(
710 recoverEnvironmentTunnelRecord({
711 userId: "user-1",
712 environmentId: "environment-1",
713 environmentPublicKey: "public-key",
714 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
715 }),
716 );
717 expect(error).toMatchObject({ _tag: "Unauthorized" });
718 }).pipe(
719 Effect.provide(
720 Layer.merge(
721 relayUnlinkTestLayer({
722 getForUser: () =>
723 Effect.succeed({
724 ...linkedEnvironmentRecord,
725 endpoint: {
726 ...linkedEnvironmentRecord.endpoint,
727 providerKind: "manual" as const,
728 },
729 }),
730 }),
731 Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({
732 enableRecovery: () => Effect.die("unused"),
733 }),
734 ),
735 ),
736 ),
737 );
738
739 it.effect("rejects a recovered tunnel that changes the linked endpoint", () => {
740 let recoveryEnabled = false;
741 const cleaned: Array<string> = [];
742
743 return Effect.gen(function* () {
744 const error = yield* Effect.flip(
745 recoverEnvironmentTunnelRecord({
746 userId: "user-1",
747 environmentId: "environment-1",
748 environmentPublicKey: "public-key",
749 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
750 }),
751 );
752 expect(error).toMatchObject({ _tag: "Unauthorized" });
753 expect(recoveryEnabled).toBe(false);
754 expect(cleaned).toEqual(["replacement-tunnel"]);
755 }).pipe(
756 Effect.provide(
757 Layer.merge(
758 relayUnlinkTestLayer({
759 getForUser: () => Effect.succeed(linkedEnvironmentRecord),
760 provision: () =>
761 Effect.succeed({
762 endpoint: {
763 httpBaseUrl: "https://different.example.test/",
764 wsBaseUrl: "wss://different.example.test/ws",
765 providerKind: "cloudflare_tunnel",
766 },
767 runtime: {
768 providerKind: "cloudflare_tunnel",
769 connectorToken: "token",
770 tunnelId: "replacement-tunnel",
771 },
772 }),
773 prepareDeprovision: () => Effect.die("must keep the active allocation"),
774 deprovision: () => Effect.die("must keep the active link DNS"),
775 release: ({ expectedTunnelId }) =>
776 Effect.sync(() => {
777 if (expectedTunnelId) {
778 cleaned.push(expectedTunnelId);
779 }
780 return true;
781 }),
782 }),
783 Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({
784 enableRecovery: () =>
785 Effect.sync(() => {
786 recoveryEnabled = true;
787 return true;
788 }),
789 }),
790 ),
791 ),
792 );
793 });
794
795 it.effect.each([
796 { state: "removed", currentLink: null },
797 {
798 state: "publish-only",
799 currentLink: {
800 ...linkedEnvironmentRecord,
801 endpoint: {
802 ...linkedEnvironmentRecord.endpoint,
803 providerKind: "manual" as const,
804 },
805 },
806 },
807 ])("removes a recovered tunnel when its link becomes $state", ({ currentLink }) => {
808 let lookups = 0;
809 const cleaned: Array<string> = [];
810 const target = {
811 userId: "user-1",
812 environmentId: "environment-1",
813 hostname: "environment-1.example.test",
814 tunnelId: "replacement-tunnel",
815 tunnelName: "environment-1-tunnel",
816 dnsRecordId: "dns-1",
817 readyAt: "2026-07-28T00:00:00.000Z",
818 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
819 updatedAt: "replacement-generation",
820 generation: 3,
821 } satisfies ManagedEndpointProvider.ManagedEndpointDeprovisionTarget;
822
823 return Effect.gen(function* () {
824 const error = yield* Effect.flip(
825 recoverEnvironmentTunnelRecord({
826 userId: "user-1",
827 environmentId: "environment-1",
828 environmentPublicKey: "public-key",
829 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
830 }),
831 );
832 expect(error).toMatchObject({ _tag: "Unauthorized" });
833 expect(cleaned).toEqual(["replacement-tunnel"]);
834 }).pipe(
835 Effect.provide(
836 Layer.merge(
837 relayUnlinkTestLayer({
838 getForUser: () =>
839 Effect.sync(() => (++lookups === 1 ? linkedEnvironmentRecord : currentLink)),
840 provision: () =>
841 Effect.succeed({
842 endpoint: linkedEnvironmentRecord.endpoint,
843 runtime: {
844 providerKind: "cloudflare_tunnel",
845 connectorToken: "replacement-token",
846 tunnelId: "replacement-tunnel",
847 },
848 }),
849 prepareDeprovision: () => Effect.succeed(target),
850 deprovision: ({ target: captured }) =>
851 Effect.sync(() => {
852 if (captured?.tunnelId) {
853 cleaned.push(captured.tunnelId);
854 }
855 return true;
856 }),
857 }),
858 Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({
859 enableRecovery: () => Effect.succeed(false),
860 }),
861 ),
862 ),
863 );
864 });
865});
866
360867describe("relay environment unlink", () => {
361868 it.effect("revokes the link and its credentials in one database transaction", () => {
362869 const calls: Array<string> = [];
......@@ -401,7 +908,9 @@ describe("relay environment unlink", () => {
401908 tunnelName: "environment-1-tunnel",
402909 dnsRecordId: "dns-1",
403910 readyAt: "2026-07-28T00:00:00.000Z",
911 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
404912 updatedAt: "generation-before-unlink",
913 generation: 1,
405914 } satisfies ManagedEndpointProvider.ManagedEndpointDeprovisionTarget;
406915
407916 return Effect.gen(function* () {
......@@ -450,6 +959,7 @@ describe("relay environment unlink", () => {
450959 Effect.sync(() => {
451960 expect(request.target).toBe(deprovisionTarget);
452961 calls.push("deprovision");
962 return true;
453963 }),
454964 }),
455965 ),
......@@ -498,6 +1008,7 @@ describe("relay environment unlink", () => {
4981008 deprovision: () =>
4991009 Effect.sync(() => {
5001010 calls.push("deprovision");
1011 return true;
5011012 }),
5021013 }),
5031014 ),
......@@ -525,6 +1036,47 @@ describe("relay environment unlink", () => {
5251036 deprovision: () =>
5261037 Effect.sync(() => {
5271038 calls.push("deprovision");
1039 return true;
1040 }),
1041 }),
1042 ),
1043 );
1044 });
1045
1046 it.effect("retries unlink cleanup when a concurrent tunnel release wins the first claim", () => {
1047 let lookups = 0;
1048 const targets: Array<ManagedEndpointProvider.ManagedEndpointDeprovisionTarget | undefined> = [];
1049 const target = {
1050 userId: "user-1",
1051 environmentId: "environment-1",
1052 hostname: "environment-1.example.test",
1053 tunnelId: "tunnel-1",
1054 tunnelName: "environment-1-tunnel",
1055 dnsRecordId: "dns-1",
1056 readyAt: "2026-07-28T00:00:00.000Z",
1057 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
1058 updatedAt: "original-generation",
1059 generation: 1,
1060 } satisfies ManagedEndpointProvider.ManagedEndpointDeprovisionTarget;
1061
1062 return Effect.gen(function* () {
1063 expect(
1064 yield* unlinkEnvironmentRecord({
1065 userId: "user-1",
1066 environmentId: "environment-1",
1067 }),
1068 ).toBe(true);
1069 expect(targets).toEqual([target, target]);
1070 }).pipe(
1071 Effect.provide(
1072 relayUnlinkTestLayer({
1073 getForUser: () => Effect.sync(() => (++lookups === 1 ? linkedEnvironmentRecord : null)),
1074 revokeForUser: () => Effect.succeed(true),
1075 prepareDeprovision: () => Effect.succeed(target),
1076 deprovision: ({ target: captured }) =>
1077 Effect.sync(() => {
1078 targets.push(captured ?? undefined);
1079 return targets.length > 1;
5281080 }),
5291081 }),
5301082 ),
......@@ -652,7 +1204,19 @@ describe("relay routing fallback", () => {
6521204 const routes = HttpApiBuilder.layer(
6531205 HttpApi.make("RelayApi").add(RelayApi.groups.server),
6541206 ).pipe(
655 Layer.provide(serverApi.pipe(Layer.provide([publisher, signatures]))),
1207 Layer.provide(
1208 serverApi.pipe(
1209 HttpRouter.provideRequest(
1210 Layer.mergeAll(
1211 Layer.succeed(RelayConfiguration.RelayConfiguration, relaySettings),
1212 Layer.mock(EnvironmentLinks.EnvironmentLinks, {}),
1213 Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations, {}),
1214 Layer.mock(ManagedEndpointProvider.ManagedEndpointProvider, {}),
1215 ),
1216 ),
1217 Layer.provide([publisher, signatures]),
1218 ),
1219 ),
6561220 Layer.provide(auth),
6571221 Layer.provide([NodeServices.layer, NodeHttpPlatform.layer, Etag.layerWeak]),
6581222 );
infra/relay/src/http/Api.ts+281-3
......@@ -47,10 +47,16 @@ import {
4747 RelayEnvironmentLinkLimitExceededError,
4848 RelayEnvironmentPrincipal,
4949 type RelayEnvironmentConnectRequest,
50 type RelayManagedEndpointOrigin,
51 RelayManagedEndpointRecoveryProofPayload,
5052 type RelayDpopAccessTokenScope,
5153 RelayInternalError,
5254} from "@t3tools/contracts/relay";
53import { normalizeRelayIssuer } from "@t3tools/shared/relayJwt";
55import {
56 normalizeRelayIssuer,
57 RELAY_MANAGED_TUNNEL_RECOVERY_TYP,
58 verifyRelayJwt,
59} from "@t3tools/shared/relayJwt";
5460
5561import * as DeliveryAttempts from "../agentActivity/DeliveryAttempts.ts";
5662import * as AgentActivityRows from "../agentActivity/AgentActivityRows.ts";
......@@ -99,6 +105,10 @@ const relayCorsPreflightHeaders = {
99105 "access-control-max-age": "86400",
100106} as const;
101107
108const decodeManagedTunnelRecoveryProof = Schema.decodeUnknownEffect(
109 RelayManagedEndpointRecoveryProofPayload,
110);
111
102112const appendRelayCredentialResponseHeaders = HttpEffect.appendPreResponseHandler(
103113 (_request, response) =>
104114 Effect.succeed(
......@@ -462,15 +472,207 @@ export const unlinkEnvironmentRecord = Effect.fn("relay.api.client.unlinkEnviron
462472 // revocation commits so a database failure leaves a fully usable active
463473 // link. Still run teardown when the link is already revoked, allowing a
464474 // retry to finish cleanup after an earlier Cloudflare failure.
465 yield* managedEndpointProvider.deprovision({
475 const deprovisioned = yield* managedEndpointProvider.deprovision({
466476 userId: input.userId,
467477 environmentId: input.environmentId,
468478 target: deprovisionTarget,
469479 });
480 if (!deprovisioned) {
481 const retryTarget = yield* managedEndpointProvider.prepareDeprovision(input);
482 if (retryTarget !== null && (yield* links.getForUser(input)) === null) {
483 yield* managedEndpointProvider.deprovision({ ...input, target: retryTarget });
484 }
485 }
470486 return unlinked;
471487 },
472488);
473489
490type EnvironmentTunnelRecoveryProofInput = {
491 readonly proof: string;
492 readonly userId: string;
493 readonly environmentId: string;
494 readonly environmentPublicKey: string;
495} & (
496 | {
497 readonly action: "register";
498 readonly tunnelId: string;
499 readonly origin: RelayManagedEndpointOrigin;
500 }
501 | { readonly action: "recover"; readonly origin: RelayManagedEndpointOrigin }
502);
503
504export const verifyEnvironmentTunnelRecoveryProof = Effect.fn(
505 "relay.api.server.verifyEnvironmentTunnelRecoveryProof",
506)(function* (input: EnvironmentTunnelRecoveryProofInput) {
507 const config = yield* RelayConfiguration.RelayConfiguration;
508 const now = yield* DateTime.now;
509 const verified = yield* verifyRelayJwt({
510 publicKey: input.environmentPublicKey,
511 token: input.proof,
512 typ: RELAY_MANAGED_TUNNEL_RECOVERY_TYP,
513 issuer: `t3-env:${input.environmentId}`,
514 audience: normalizeRelayIssuer(config.relayIssuer),
515 nowEpochSeconds: Math.floor(now.epochMilliseconds / 1_000),
516 }).pipe(
517 Effect.flatMap(decodeManagedTunnelRecoveryProof),
518 Effect.mapError(() => new HttpApiError.Unauthorized({})),
519 );
520
521 if (
522 verified.environmentId !== input.environmentId ||
523 verified.sub !== input.environmentId ||
524 verified.cloudUserId !== input.userId ||
525 verified.action !== input.action
526 ) {
527 return yield* new HttpApiError.Unauthorized({});
528 }
529 if (input.action === "register") {
530 if (
531 verified.action !== "register" ||
532 verified.tunnelId !== input.tunnelId ||
533 verified.origin.localHttpHost !== input.origin.localHttpHost ||
534 verified.origin.localHttpPort !== input.origin.localHttpPort
535 ) {
536 return yield* new HttpApiError.Unauthorized({});
537 }
538 return;
539 }
540 if (
541 verified.action !== "recover" ||
542 verified.origin.localHttpHost !== input.origin.localHttpHost ||
543 verified.origin.localHttpPort !== input.origin.localHttpPort
544 ) {
545 return yield* new HttpApiError.Unauthorized({});
546 }
547});
548
549export const registerEnvironmentTunnelRecovery = Effect.fn(
550 "relay.api.server.registerEnvironmentTunnelRecovery",
551)(function* (input: {
552 readonly userId: string;
553 readonly environmentId: string;
554 readonly environmentPublicKey: string;
555 readonly tunnelId: string;
556 readonly origin: RelayManagedEndpointOrigin;
557}) {
558 const links = yield* EnvironmentLinks.EnvironmentLinks;
559 const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations;
560 const managedEndpointProvider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
561 const link = yield* links.getForUser({
562 userId: input.userId,
563 environmentId: input.environmentId,
564 });
565 if (
566 link === null ||
567 link.environmentPublicKey !== input.environmentPublicKey ||
568 link.endpoint.providerKind !== "cloudflare_tunnel"
569 ) {
570 return yield* new HttpApiError.Unauthorized({});
571 }
572 const status = yield* managedEndpointProvider.reconcileOrigin({
573 userId: input.userId,
574 environmentId: input.environmentId,
575 tunnelId: input.tunnelId,
576 origin: input.origin,
577 endpoint: link.endpoint,
578 });
579 if (status === "recovery_required") {
580 return { status };
581 }
582 if (!(yield* allocations.enableRecovery(input))) {
583 return yield* new HttpApiError.Unauthorized({});
584 }
585 return { status };
586});
587
588export const recoverEnvironmentTunnelRecord = Effect.fn(
589 "relay.api.server.recoverEnvironmentTunnelRecord",
590)(function* (input: {
591 readonly userId: string;
592 readonly environmentId: string;
593 readonly environmentPublicKey: string;
594 readonly origin: RelayManagedEndpointOrigin;
595}) {
596 const links = yield* EnvironmentLinks.EnvironmentLinks;
597 const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations;
598 const managedEndpointProvider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
599 const link = yield* links.getForUser({
600 userId: input.userId,
601 environmentId: input.environmentId,
602 });
603 if (
604 link === null ||
605 link.environmentPublicKey !== input.environmentPublicKey ||
606 link.endpoint.providerKind !== "cloudflare_tunnel"
607 ) {
608 return yield* new HttpApiError.Unauthorized({});
609 }
610
611 const recovered = yield* managedEndpointProvider.provision({
612 userId: input.userId,
613 environmentId: input.environmentId,
614 origin: input.origin,
615 });
616 const recoveredTunnelId = recovered.runtime.tunnelId;
617 if (
618 recoveredTunnelId === undefined ||
619 recovered.endpoint.httpBaseUrl !== link.endpoint.httpBaseUrl ||
620 recovered.endpoint.wsBaseUrl !== link.endpoint.wsBaseUrl
621 ) {
622 if (recoveredTunnelId !== undefined) {
623 yield* managedEndpointProvider
624 .release({
625 userId: input.userId,
626 environmentId: input.environmentId,
627 expectedTunnelId: recoveredTunnelId,
628 })
629 .pipe(
630 Effect.catch((cause) =>
631 Effect.logWarning("Failed to clean up a tunnel with a mismatched endpoint", {
632 userId: input.userId,
633 environmentId: input.environmentId,
634 tunnelId: recoveredTunnelId,
635 cause,
636 }),
637 ),
638 );
639 }
640 return yield* new HttpApiError.Unauthorized({});
641 }
642
643 const enabled = yield* allocations.enableRecovery({
644 userId: input.userId,
645 environmentId: input.environmentId,
646 tunnelId: recoveredTunnelId,
647 environmentPublicKey: input.environmentPublicKey,
648 origin: input.origin,
649 });
650 if (!enabled) {
651 const owner = { userId: input.userId, environmentId: input.environmentId };
652 const target = yield* managedEndpointProvider.prepareDeprovision(owner);
653 const currentLink = target === null ? null : yield* links.getForUser(input);
654 if (
655 target !== null &&
656 (currentLink === null || currentLink.endpoint.providerKind !== "cloudflare_tunnel")
657 ) {
658 yield* managedEndpointProvider.deprovision({ ...owner, target }).pipe(
659 Effect.catch((cause) =>
660 Effect.logWarning("Failed to clean up a tunnel after its managed link was removed", {
661 userId: input.userId,
662 environmentId: input.environmentId,
663 cause,
664 }),
665 ),
666 );
667 }
668 return yield* new HttpApiError.Unauthorized({});
669 }
670 return {
671 endpoint: recovered.endpoint,
672 endpointRuntime: recovered.runtime,
673 };
674});
675
474676export const mobileApi = HttpApiBuilder.group(
475677 RelayApi,
476678 "mobile",
......@@ -879,7 +1081,7 @@ export const serverApi = HttpApiBuilder.group(
8791081 Effect.fnUntraced(function* (handlers) {
8801082 const publisher = yield* AgentActivityPublisher.AgentActivityPublisher;
8811083 const publishSignatures = yield* EnvironmentPublishSignatures.EnvironmentPublishSignatures;
882 return handlers.handle(
1084 const activityHandlers = handlers.handle(
8831085 "publishAgentActivity",
8841086 Effect.fn("relay.api.server.publishAgentActivity")(
8851087 function* (args) {
......@@ -1013,6 +1215,82 @@ export const serverApi = HttpApiBuilder.group(
10131215 mapRelayCommonApiErrors("not_authorized"),
10141216 ),
10151217 );
1218
1219 return activityHandlers
1220 .handle(
1221 "registerManagedEndpointRecovery",
1222 Effect.fn("relay.api.server.registerManagedEndpointRecovery")(
1223 function* ({ params, payload }) {
1224 const principal = yield* RelayEnvironmentPrincipal;
1225 if (principal.environmentId !== params.environmentId) {
1226 return yield* new HttpApiError.Unauthorized({});
1227 }
1228 yield* verifyEnvironmentTunnelRecoveryProof({
1229 action: "register",
1230 proof: payload.proof,
1231 userId: payload.cloudUserId,
1232 environmentId: params.environmentId,
1233 environmentPublicKey: principal.environmentPublicKey,
1234 tunnelId: payload.tunnelId,
1235 origin: payload.origin,
1236 });
1237 yield* appendRelayCredentialResponseHeaders;
1238 return yield* registerEnvironmentTunnelRecovery({
1239 userId: payload.cloudUserId,
1240 environmentId: params.environmentId,
1241 environmentPublicKey: principal.environmentPublicKey,
1242 tunnelId: payload.tunnelId,
1243 origin: payload.origin,
1244 });
1245 },
1246 Effect.catchTags({
1247 ManagedEndpointOriginNotAllowed: () => Effect.fail(new HttpApiError.Unauthorized({})),
1248 ManagedEndpointProvisioningNotConfigured: () =>
1249 relayInternalErrorResponse("upstream_unavailable"),
1250 ManagedEndpointProvisioningFailed: () =>
1251 relayInternalErrorResponse("upstream_unavailable"),
1252 ManagedTunnelLimitExceeded: () => relayInternalErrorResponse("upstream_unavailable"),
1253 }),
1254 mapRelayCommonApiErrors("not_authorized"),
1255 ),
1256 )
1257 .handle(
1258 "recoverManagedEndpoint",
1259 Effect.fn("relay.api.server.recoverManagedEndpoint")(
1260 function* ({ params, payload }) {
1261 const principal = yield* RelayEnvironmentPrincipal;
1262 if (principal.environmentId !== params.environmentId) {
1263 return yield* new HttpApiError.Unauthorized({});
1264 }
1265 yield* verifyEnvironmentTunnelRecoveryProof({
1266 action: "recover",
1267 proof: payload.proof,
1268 userId: payload.cloudUserId,
1269 environmentId: params.environmentId,
1270 environmentPublicKey: principal.environmentPublicKey,
1271 origin: payload.origin,
1272 });
1273 yield* appendRelayCredentialResponseHeaders;
1274 return yield* recoverEnvironmentTunnelRecord({
1275 userId: payload.cloudUserId,
1276 environmentId: params.environmentId,
1277 environmentPublicKey: principal.environmentPublicKey,
1278 origin: payload.origin,
1279 });
1280 },
1281 Effect.catchTags({
1282 ManagedEndpointOriginNotAllowed: () => Effect.fail(new HttpApiError.Unauthorized({})),
1283 ManagedEndpointProvisioningNotConfigured: () =>
1284 relayInternalErrorResponse("upstream_unavailable"),
1285 ManagedEndpointProvisioningFailed: () =>
1286 relayInternalErrorResponse("upstream_unavailable"),
1287 ManagedEndpointDeprovisioningFailed: () =>
1288 relayInternalErrorResponse("upstream_unavailable"),
1289 ManagedTunnelLimitExceeded: () => relayInternalErrorResponse("upstream_unavailable"),
1290 }),
1291 mapRelayCommonApiErrors("not_authorized"),
1292 ),
1293 );
10161294 }),
10171295);
10181296
infra/relay/src/persistence/schema.ts+5
......@@ -2,6 +2,7 @@ import type {
22 RelayAgentActivityAggregateState,
33 RelayAgentActivityState,
44 RelayAgentAwarenessPreferences,
5 RelayManagedEndpointOrigin,
56} from "@t3tools/contracts/relay";
67import {
78 boolean,
......@@ -94,6 +95,10 @@ export const relayManagedEndpointAllocations = pgTable(
9495 tunnelName: text("tunnel_name").notNull(),
9596 dnsRecordId: varchar("dns_record_id", { length: 191 }),
9697 readyAt: varchar("ready_at", { length: 64 }),
98 recoveryEnabledAt: varchar("recovery_enabled_at", { length: 64 }),
99 recoveryEnvironmentPublicKey: text("recovery_environment_public_key"),
100 origin: jsonb("origin").$type<RelayManagedEndpointOrigin>(),
101 generation: integer("generation").notNull().default(0),
97102 createdAt: varchar("created_at", { length: 64 }).notNull(),
98103 updatedAt: varchar("updated_at", { length: 64 }).notNull(),
99104 },
infra/relay/src/worker.ts+41-15
......@@ -2,6 +2,7 @@ import * as Alchemy from "alchemy";
22import * as Cloudflare from "alchemy/Cloudflare";
33import * as Drizzle from "alchemy/Drizzle/Postgres";
44import * as Config from "effect/Config";
5import * as Cause from "effect/Cause";
56import * as DateTime from "effect/DateTime";
67import * as Crypto from "effect/Crypto";
78import * as Effect from "effect/Effect";
......@@ -69,6 +70,7 @@ import * as EnvironmentConnector from "./environments/EnvironmentConnector.ts";
6970import * as EnvironmentLinker from "./environments/EnvironmentLinker.ts";
7071import * as EnvironmentPublishSignatures from "./environments/EnvironmentPublishSignatures.ts";
7172import * as ManagedEndpointProvider from "./environments/ManagedEndpointProvider.ts";
73import * as ManagedEndpointReaper from "./environments/ManagedEndpointReaper.ts";
7274import * as ManagedTunnelLimits from "./environments/ManagedTunnelLimits.ts";
7375import * as MobileRegistrations from "./agentActivity/MobileRegistrations.ts";
7476
......@@ -180,6 +182,7 @@ export const ApiLive = Api.make(
180182 yield* yield* relayApiZone.zoneId;
181183 const managedEndpointDnsBinding = yield* Cloudflare.DNS.ReadWriteDns(managedEndpointZone);
182184 const managedEndpointZoneName = yield* managedEndpointZone.name;
185 const managedEndpointCleanupMode = yield* RelayConfiguration.managedEndpointCleanupModeConfig;
183186
184187 //
185188 // 3. Runtime layers and app construction
......@@ -199,6 +202,7 @@ export const ApiLive = Api.make(
199202 cloudMintPublicKey: yield* cloudMintPublicKey,
200203 managedEndpointBaseDomain: yield* managedEndpointZoneName,
201204 managedEndpointNamespace: stage,
205 managedEndpointCleanupMode,
202206 });
203207 });
204208
......@@ -215,7 +219,9 @@ export const ApiLive = Api.make(
215219 Layer.provideMerge(AgentActivityPublisher.layer),
216220 Layer.provideMerge(EnvironmentConnector.layer),
217221 Layer.provideMerge(EnvironmentLinker.layer),
218 Layer.provideMerge(EnvironmentPublishSignatures.layer),
222 Layer.provideMerge(
223 Layer.merge(EnvironmentPublishSignatures.layer, ManagedEndpointReaper.layer),
224 ),
219225 Layer.provideMerge(
220226 ManagedEndpointProvider.layerCloudflareBindings(
221227 managedEndpointTunnelBinding,
......@@ -317,22 +323,42 @@ export const ApiLive = Api.make(
317323 );
318324
319325 yield* Cloudflare.Workers.cron("*/5 * * * *", () =>
320 DpopProofs.DpopProofReplay.pipe(
321 Effect.flatMap((dpopProofs) => dpopProofs.pruneExpired),
322 // Terminal thread rows are kept briefly so finished agents show as
323 // Done/Failed in the Live Activity; sweep them once they age out.
324 Effect.andThen(
325 Effect.all([AgentActivityRows.AgentActivityRows, DateTime.now]).pipe(
326 Effect.flatMap(([activityRows, now]) =>
327 activityRows.pruneTerminal({
328 updatedBefore: DateTime.formatIso(DateTime.subtract(now, { minutes: 30 })),
329 }),
326 Effect.all(
327 [
328 DpopProofs.DpopProofReplay.pipe(
329 Effect.flatMap((dpopProofs) => dpopProofs.pruneExpired),
330 // Keep completed thread rows long enough to show their final state.
331 Effect.andThen(
332 Effect.all([AgentActivityRows.AgentActivityRows, DateTime.now]).pipe(
333 Effect.flatMap(([activityRows, now]) =>
334 activityRows.pruneTerminal({
335 updatedBefore: DateTime.formatIso(DateTime.subtract(now, { minutes: 30 })),
336 }),
337 ),
338 ),
339 ),
340 Effect.catchCause((cause) =>
341 Cause.hasInterrupts(cause)
342 ? Effect.interrupt
343 : Effect.logWarning("Failed to prune expired relay state", { cause }),
330344 ),
331345 ),
332 ),
333 Effect.withSpan("relay.cron.prune_expired_state"),
334 Effect.provide(runtimeLayer),
335 ),
346 ManagedEndpointReaper.ManagedEndpointReaper.pipe(
347 Effect.flatMap((reaper) => reaper.sweep.pipe(Effect.timeout("2 minutes"))),
348 Effect.tap((result) =>
349 result.scanned > 0
350 ? Effect.logInfo("Finished managed tunnel cleanup", result)
351 : Effect.void,
352 ),
353 Effect.catchCause((cause) =>
354 Cause.hasInterrupts(cause)
355 ? Effect.interrupt
356 : Effect.logWarning("Failed to clean up inactive managed tunnels", { cause }),
357 ),
358 ),
359 ],
360 { concurrency: 2, discard: true },
361 ).pipe(Effect.withSpan("relay.cron.prune_expired_state"), Effect.provide(runtimeLayer)),
336362 );
337363
338364 const fetch = Layer.merge(
packages/contracts/src/relay.ts+68
......@@ -186,6 +186,34 @@ export const RelayManagedEndpointRuntimeConfig = Schema.Struct({
186186});
187187export type RelayManagedEndpointRuntimeConfig = typeof RelayManagedEndpointRuntimeConfig.Type;
188188
189export const RelayManagedEndpointRecoveryRequest = Schema.Struct({
190 cloudUserId: TrimmedNonEmptyString,
191 origin: RelayManagedEndpointOrigin,
192 proof: TrimmedNonEmptyString,
193});
194export type RelayManagedEndpointRecoveryRequest = typeof RelayManagedEndpointRecoveryRequest.Type;
195
196export const RelayManagedEndpointRecoveryRegistrationRequest = Schema.Struct({
197 cloudUserId: TrimmedNonEmptyString,
198 tunnelId: TrimmedNonEmptyString,
199 origin: RelayManagedEndpointOrigin,
200 proof: TrimmedNonEmptyString,
201});
202export type RelayManagedEndpointRecoveryRegistrationRequest =
203 typeof RelayManagedEndpointRecoveryRegistrationRequest.Type;
204
205export const RelayManagedEndpointRecoveryRegistrationResponse = Schema.Struct({
206 status: Schema.Literals(["ready", "recovery_required"]),
207});
208export type RelayManagedEndpointRecoveryRegistrationResponse =
209 typeof RelayManagedEndpointRecoveryRegistrationResponse.Type;
210
211export const RelayManagedEndpointRecoveryResponse = Schema.Struct({
212 endpoint: RelayManagedEndpoint,
213 endpointRuntime: RelayManagedEndpointRuntimeConfig,
214});
215export type RelayManagedEndpointRecoveryResponse = typeof RelayManagedEndpointRecoveryResponse.Type;
216
189217export const RelayLinkProofRequest = Schema.Struct({
190218 challenge: Schema.String,
191219 relayIssuer: Schema.String,
......@@ -213,6 +241,26 @@ const RelaySignedJwtRegisteredClaims = {
213241 exp: Schema.Int,
214242} as const;
215243
244export const RelayManagedEndpointRecoveryProofPayload = Schema.Union([
245 Schema.Struct({
246 ...RelaySignedJwtRegisteredClaims,
247 action: Schema.Literal("register"),
248 environmentId: EnvironmentId,
249 cloudUserId: TrimmedNonEmptyString,
250 tunnelId: TrimmedNonEmptyString,
251 origin: RelayManagedEndpointOrigin,
252 }),
253 Schema.Struct({
254 ...RelaySignedJwtRegisteredClaims,
255 action: Schema.Literal("recover"),
256 environmentId: EnvironmentId,
257 cloudUserId: TrimmedNonEmptyString,
258 origin: RelayManagedEndpointOrigin,
259 }),
260]);
261export type RelayManagedEndpointRecoveryProofPayload =
262 typeof RelayManagedEndpointRecoveryProofPayload.Type;
263
216264export const RelayAgentActivityPublishProofPayload = Schema.Struct({
217265 ...RelaySignedJwtRegisteredClaims,
218266 environmentId: EnvironmentId,
......@@ -1090,6 +1138,26 @@ const RelayDpopClientGroup = HttpApiGroup.make("dpopClient")
10901138
10911139const RelayServerGroup = HttpApiGroup.make("server")
10921140 .add(
1141 HttpApiEndpoint.post(
1142 "registerManagedEndpointRecovery",
1143 "/v1/environments/:environmentId/tunnel/recovery",
1144 {
1145 params: Schema.Struct({
1146 environmentId: EnvironmentId,
1147 }),
1148 payload: RelayManagedEndpointRecoveryRegistrationRequest,
1149 success: RelayManagedEndpointRecoveryRegistrationResponse,
1150 error: RelayAuthAndInternalErrors,
1151 },
1152 ).annotate(OpenApi.Summary, "Register managed tunnel recovery without provisioning"),
1153 HttpApiEndpoint.post("recoverManagedEndpoint", "/v1/environments/:environmentId/tunnel", {
1154 params: Schema.Struct({
1155 environmentId: EnvironmentId,
1156 }),
1157 payload: RelayManagedEndpointRecoveryRequest,
1158 success: RelayManagedEndpointRecoveryResponse,
1159 error: RelayAuthAndInternalErrors,
1160 }).annotate(OpenApi.Summary, "Recover an environment's managed tunnel"),
10931161 HttpApiEndpoint.post(
10941162 "publishAgentActivity",
10951163 "/v1/environments/:environmentId/threads/:threadId/agent-activity",
packages/shared/src/relayJwt.ts+1
......@@ -10,6 +10,7 @@ export const RELAY_HEALTH_REQUEST_TYP = "t3-cloud-health+jwt";
1010export const RELAY_MINT_RESPONSE_TYP = "t3-env-mint+jwt";
1111export const RELAY_HEALTH_RESPONSE_TYP = "t3-env-health+jwt";
1212export const RELAY_ACTIVITY_PUBLISH_TYP = "t3-env-activity+jwt";
13export const RELAY_MANAGED_TUNNEL_RECOVERY_TYP = "t3-env-managed-tunnel-recovery+jwt";
1314
1415export class RelayJwtError extends Schema.TaggedError<RelayJwtError>()("RelayJwtError", {
1516 operation: Schema.Literals(["sign", "verify"]),