authorgravatar for me@t3.ggTheo Browne <me@t3.gg> 2026-09-24 17:17:41-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-09-24 17:17:41-07:00
log8d7b5e998c72b4674326cbac7d23bafe9b222668
treefcbfc93d4f6343e71e840c3de3cc4903508e93a7
parentd4a33457cb0da797728f4846a9da8592d7d81d36
signature Signed by PGP key B5690EEEBB952194

fix(connect): remove tunnels after hosts go offline (#9386)

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: Julius Marminge <julius0216@outlook.com> Co-authored-by: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com>

36 files changed, 7563 insertions(+), 388 deletions(-)

.github/workflows/deploy-relay.yml+1
...@@ -28,6 +28,7 @@ jobs:...@@ -28,6 +28,7 @@ jobs:
28 RELAY_DOMAIN: ${{ vars.RELAY_DOMAIN }}28 RELAY_DOMAIN: ${{ vars.RELAY_DOMAIN }}
29 RELAY_API_ZONE_NAME: ${{ vars.RELAY_API_ZONE_NAME }}29 RELAY_API_ZONE_NAME: ${{ vars.RELAY_API_ZONE_NAME }}
30 RELAY_TUNNEL_ZONE_NAME: ${{ vars.RELAY_TUNNEL_ZONE_NAME }}30 RELAY_TUNNEL_ZONE_NAME: ${{ vars.RELAY_TUNNEL_ZONE_NAME }}
31 RELAY_TUNNEL_CLEANUP_MODE: ${{ vars.RELAY_TUNNEL_CLEANUP_MODE }}
31 CLERK_PUBLISHABLE_KEY: ${{ vars.CLERK_PUBLISHABLE_KEY }}32 CLERK_PUBLISHABLE_KEY: ${{ vars.CLERK_PUBLISHABLE_KEY }}
32 CLERK_JWT_AUDIENCE: ${{ vars.CLERK_JWT_AUDIENCE }}33 CLERK_JWT_AUDIENCE: ${{ vars.CLERK_JWT_AUDIENCE }}
33 APNS_ENVIRONMENT: ${{ vars.APNS_ENVIRONMENT }}34 APNS_ENVIRONMENT: ${{ vars.APNS_ENVIRONMENT }}
apps/server/src/cloud/CliState.test.ts+2
...@@ -8,6 +8,7 @@ import * as ServerSecretStore from "../auth/ServerSecretStore.ts";...@@ -8,6 +8,7 @@ import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
8import { ServerConfig } from "../config.ts";8import { ServerConfig } from "../config.ts";
9import * as CliState from "./CliState.ts";9import * as CliState from "./CliState.ts";
10import {10import {
11 CLOUD_ENDPOINT_CONFIRMED_ORIGIN,
11 CLOUD_ENDPOINT_RUNTIME_CONFIG,12 CLOUD_ENDPOINT_RUNTIME_CONFIG,
12 CLOUD_LINKED_USER_ID,13 CLOUD_LINKED_USER_ID,
13 CLOUD_MINT_PUBLIC_KEY,14 CLOUD_MINT_PUBLIC_KEY,
...@@ -24,6 +25,7 @@ const persistedCloudLinkSecrets = [...@@ -24,6 +25,7 @@ const persistedCloudLinkSecrets = [
24 RELAY_ENVIRONMENT_CREDENTIAL_SECRET,25 RELAY_ENVIRONMENT_CREDENTIAL_SECRET,
25 CLOUD_MINT_PUBLIC_KEY,26 CLOUD_MINT_PUBLIC_KEY,
26 CLOUD_ENDPOINT_RUNTIME_CONFIG,27 CLOUD_ENDPOINT_RUNTIME_CONFIG,
28 CLOUD_ENDPOINT_CONFIRMED_ORIGIN,
27 PUBLISH_AGENT_ACTIVITY_SECRET,29 PUBLISH_AGENT_ACTIVITY_SECRET,
28] as const;30] as const;
2931
apps/server/src/cloud/CliState.ts+2
...@@ -3,6 +3,7 @@ import * as Option from "effect/Option";...@@ -3,6 +3,7 @@ import * as Option from "effect/Option";
33
4import * as ServerSecretStore from "../auth/ServerSecretStore.ts";4import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
5import {5import {
6 CLOUD_ENDPOINT_CONFIRMED_ORIGIN,
6 CLOUD_ENDPOINT_RUNTIME_CONFIG,7 CLOUD_ENDPOINT_RUNTIME_CONFIG,
7 CLOUD_LINKED_USER_ID,8 CLOUD_LINKED_USER_ID,
8 CLOUD_MINT_PUBLIC_KEY,9 CLOUD_MINT_PUBLIC_KEY,
...@@ -67,6 +68,7 @@ export const clearPersistedCloudLink = Effect.gen(function* () {...@@ -67,6 +68,7 @@ export const clearPersistedCloudLink = Effect.gen(function* () {
67 secrets.remove(RELAY_ENVIRONMENT_CREDENTIAL_SECRET),68 secrets.remove(RELAY_ENVIRONMENT_CREDENTIAL_SECRET),
68 secrets.remove(CLOUD_MINT_PUBLIC_KEY),69 secrets.remove(CLOUD_MINT_PUBLIC_KEY),
69 secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG),70 secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG),
71 secrets.remove(CLOUD_ENDPOINT_CONFIRMED_ORIGIN),
70 secrets.remove(PUBLISH_AGENT_ACTIVITY_SECRET),72 secrets.remove(PUBLISH_AGENT_ACTIVITY_SECRET),
71 ],73 ],
72 { concurrency: "unbounded" },74 { concurrency: "unbounded" },
apps/server/src/cloud/ManagedEndpointRuntime.test.ts+214-3
...@@ -7,10 +7,12 @@ import * as Fiber from "effect/Fiber";...@@ -7,10 +7,12 @@ import * as Fiber from "effect/Fiber";
7import * as Layer from "effect/Layer";7import * as Layer from "effect/Layer";
8import * as Option from "effect/Option";8import * as Option from "effect/Option";
9import * as PlatformError from "effect/PlatformError";9import * as PlatformError from "effect/PlatformError";
10import * as Queue from "effect/Queue";
10import * as Sink from "effect/Sink";11import * as Sink from "effect/Sink";
11import * as Stream from "effect/Stream";12import * as Stream from "effect/Stream";
12import * as TestClock from "effect/testing/TestClock";13import * as TestClock from "effect/testing/TestClock";
13import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process";14import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process";
15import type { RelayManagedEndpointRuntimeConfig } from "@t3tools/contracts/relay";
14import * as RelayClient from "@t3tools/shared/relayClient";16import * as RelayClient from "@t3tools/shared/relayClient";
1517
16import * as ServerSecretStore from "../auth/ServerSecretStore.ts";18import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
...@@ -62,6 +64,7 @@ function makeHandle(input: {...@@ -62,6 +64,7 @@ function makeHandle(input: {
62 readonly onKill: () => void;64 readonly onKill: () => void;
63 readonly isRunning?: () => boolean;65 readonly isRunning?: () => boolean;
64 readonly exitCode?: Effect.Effect<ChildProcessSpawner.ExitCode>;66 readonly exitCode?: Effect.Effect<ChildProcessSpawner.ExitCode>;
67 readonly output?: Stream.Stream<Uint8Array>;
65}) {68}) {
66 return ChildProcessSpawner.makeHandle({69 return ChildProcessSpawner.makeHandle({
67 pid: ChildProcessSpawner.ProcessId(input.pid),70 pid: ChildProcessSpawner.ProcessId(input.pid),
...@@ -75,13 +78,70 @@ function makeHandle(input: {...@@ -75,13 +78,70 @@ function makeHandle(input: {
75 stdin: Sink.drain,78 stdin: Sink.drain,
76 stdout: Stream.empty,79 stdout: Stream.empty,
77 stderr: Stream.empty,80 stderr: Stream.empty,
78 all: Stream.empty,81 all: input.output ?? Stream.empty,
79 getInputFd: () => Sink.drain,82 getInputFd: () => Sink.drain,
80 getOutputFd: () => Stream.empty,83 getOutputFd: () => Stream.empty,
81 });84 });
82}85}
8386
84describe("CloudManagedEndpointRuntime", () => {87describe("CloudManagedEndpointRuntime", () => {
88 it("retries connector startup failures but stops for unsupported runtimes", () => {
89 expect(
90 ManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus({
91 status: "failed",
92 failure: "not-installed",
93 reason: "The relay client is not installed.",
94 }),
95 ).toBe(true);
96 expect(
97 ManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus({
98 status: "failed",
99 failure: "spawn-failed",
100 reason: "spawn failed",
101 }),
102 ).toBe(true);
103 expect(
104 ManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus({
105 status: "failed",
106 failure: "unsupported-platform",
107 reason: "Relay client is unsupported on linux-arm.",
108 }),
109 ).toBe(false);
110 expect(
111 ManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus({ status: "unsupported" }),
112 ).toBe(false);
113 });
114
115 it.effect("serializes updates to persisted cloud link state", () =>
116 Effect.gen(function* () {
117 const firstEntered = yield* Deferred.make<void>();
118 const releaseFirst = yield* Deferred.make<void>();
119 const secondEntered = yield* Deferred.make<void>();
120 const runtime = yield* buildCloudManagedEndpointRuntime(
121 ChildProcessSpawner.make(() => Effect.die("unused")),
122 );
123
124 const first = yield* runtime
125 .withLinkStateLock(
126 Deferred.succeed(firstEntered, undefined).pipe(
127 Effect.andThen(Deferred.await(releaseFirst)),
128 ),
129 )
130 .pipe(Effect.forkChild);
131 yield* Deferred.await(firstEntered);
132
133 const second = yield* runtime
134 .withLinkStateLock(Deferred.succeed(secondEntered, undefined))
135 .pipe(Effect.forkChild);
136 expect(yield* Deferred.isDone(secondEntered)).toBe(false);
137
138 yield* Deferred.succeed(releaseFirst, undefined);
139 yield* Fiber.join(first);
140 yield* Fiber.join(second);
141 expect(yield* Deferred.isDone(secondEntered)).toBe(true);
142 }),
143 );
144
85 it("classifies Cloudflare connection and warning output", () => {145 it("classifies Cloudflare connection and warning output", () => {
86 expect(146 expect(
87 ManagedEndpointRuntime.classifyRelayClientOutput(147 ManagedEndpointRuntime.classifyRelayClientOutput(
...@@ -109,6 +169,125 @@ describe("CloudManagedEndpointRuntime", () => {...@@ -109,6 +169,125 @@ describe("CloudManagedEndpointRuntime", () => {
109 ).toBe("warning");169 ).toBe("warning");
110 });170 });
111171
172 it("recognizes tunnel authorization failures without matching ordinary transport errors", () => {
173 expect(
174 ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput(
175 '2026-09-15T06:30:43Z ERR Register tunnel error from server side error="Failed to get tunnel" connIndex=0 event=0 ip=198.41.200.23',
176 ),
177 ).toBe(true);
178 expect(
179 ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput(
180 '2026-06-17T02:00:00Z ERR Register tunnel error from server side error="Unauthorized: Record for tunnel not found" connIndex=0',
181 ),
182 ).toBe(true);
183 expect(
184 ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput(
185 '2026-06-17T02:00:00Z ERR Register tunnel error from server side error="Unauthorized: Invalid tunnel secret" connIndex=0',
186 ),
187 ).toBe(true);
188 expect(
189 ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput(
190 '2026-06-17T02:00:00Z ERR Register tunnel error from server side error="connection timed out" connIndex=0',
191 ),
192 ).toBe(false);
193 });
194
195 it.effect("keeps recovery requests sent before the server starts consuming them", () =>
196 Effect.gen(function* () {
197 const runtime = yield* buildCloudManagedEndpointRuntime(
198 ChildProcessSpawner.make(() => Effect.die("unused")),
199 );
200 const config = {
201 providerKind: "cloudflare_tunnel" as const,
202 connectorToken: "token",
203 tunnelId: "tunnel-1",
204 };
205
206 yield* runtime.requestRecovery(config);
207
208 expect(Option.getOrNull(yield* Stream.runHead(runtime.recoveryRequests))).toEqual(config);
209 }),
210 );
211
212 it.effect("recovers a rejected tunnel without waiting for the connector to exit", () =>
213 Effect.gen(function* () {
214 const output = yield* Queue.unbounded<Uint8Array>();
215 const firstBatchObserved = yield* Deferred.make<void>();
216 const secondBatchObserved = yield* Deferred.make<void>();
217 const recoveryRequested = yield* Deferred.make<RelayManagedEndpointRuntimeConfig>();
218 const recoveryRetried = yield* Deferred.make<RelayManagedEndpointRuntimeConfig>();
219 let recoveryRequestCount = 0;
220 const spawned: Array<number> = [];
221 const encoder = new TextEncoder();
222 const connectorOutput = Stream.fromQueue(output).pipe(
223 Stream.tap((chunk) => {
224 const line = new TextDecoder().decode(chunk);
225 if (line === "first checkpoint\n") {
226 return Deferred.succeed(firstBatchObserved, undefined).pipe(Effect.asVoid);
227 }
228 if (line === "second checkpoint\n") {
229 return Deferred.succeed(secondBatchObserved, undefined).pipe(Effect.asVoid);
230 }
231 return Effect.void;
232 }),
233 );
234 const spawner = ChildProcessSpawner.make(() =>
235 Effect.gen(function* () {
236 const pid = 600;
237 spawned.push(pid);
238 const handle = makeHandle({ pid, onKill: () => {}, output: connectorOutput });
239 yield* Effect.addFinalizer(() => handle.kill().pipe(Effect.ignore));
240 return handle;
241 }),
242 );
243 const runtime = yield* buildCloudManagedEndpointRuntime(spawner);
244 const config = {
245 providerKind: "cloudflare_tunnel" as const,
246 connectorToken: "token",
247 tunnelId: "deleted-tunnel",
248 };
249 const rejectedLine =
250 '2026-09-15T06:30:43Z ERR Register tunnel error from server side error="Failed to get tunnel" connIndex=0 event=0 ip=198.41.200.23\n';
251
252 yield* runtime.recoveryRequests.pipe(
253 Stream.runForEach((requested) => {
254 recoveryRequestCount += 1;
255 return Deferred.succeed(
256 recoveryRequestCount === 1 ? recoveryRequested : recoveryRetried,
257 requested,
258 ).pipe(Effect.asVoid);
259 }),
260 Effect.forkChild,
261 );
262 yield* runtime.applyConfig(config);
263
264 yield* Queue.offer(output, encoder.encode(rejectedLine.repeat(3)));
265 yield* Queue.offer(output, encoder.encode("first checkpoint\n"));
266 yield* Deferred.await(firstBatchObserved);
267 expect(yield* Deferred.isDone(recoveryRequested)).toBe(false);
268
269 yield* Queue.offer(
270 output,
271 encoder.encode(
272 "2026-06-17T02:00:00Z INF Registered tunnel connection connIndex=0\n" +
273 rejectedLine.repeat(3),
274 ),
275 );
276 yield* Queue.offer(output, encoder.encode("second checkpoint\n"));
277 yield* Deferred.await(secondBatchObserved);
278 expect(yield* Deferred.isDone(recoveryRequested)).toBe(false);
279
280 yield* Queue.offer(output, encoder.encode(rejectedLine));
281
282 expect(yield* Deferred.await(recoveryRequested)).toEqual(config);
283
284 yield* Queue.offer(output, encoder.encode(rejectedLine.repeat(4)));
285
286 expect(yield* Deferred.await(recoveryRetried)).toEqual(config);
287 expect(spawned).toEqual([600]);
288 }),
289 );
290
112 it.effect("starts, deduplicates, rotates, and stops the Cloudflare connector", () =>291 it.effect("starts, deduplicates, rotates, and stops the Cloudflare connector", () =>
113 Effect.gen(function* () {292 Effect.gen(function* () {
114 const spawned: Array<ChildProcess.StandardCommand> = [];293 const spawned: Array<ChildProcess.StandardCommand> = [];
...@@ -156,8 +335,8 @@ describe("CloudManagedEndpointRuntime", () => {...@@ -156,8 +335,8 @@ describe("CloudManagedEndpointRuntime", () => {
156335
157 expect(spawned.map((command) => command.command)).toEqual(["cloudflared", "cloudflared"]);336 expect(spawned.map((command) => command.command)).toEqual(["cloudflared", "cloudflared"]);
158 expect(spawned.map((command) => command.args)).toEqual([337 expect(spawned.map((command) => command.args)).toEqual([
159 ["tunnel", "run"],338 ["tunnel", "--no-autoupdate", "--loglevel", "info", "--output", "default", "run"],
160 ["tunnel", "run"],339 ["tunnel", "--no-autoupdate", "--loglevel", "info", "--output", "default", "run"],
161 ]);340 ]);
162 expect(spawned.map((command) => command.options.env?.TUNNEL_TOKEN)).toEqual([341 expect(spawned.map((command) => command.options.env?.TUNNEL_TOKEN)).toEqual([
163 "token-1",342 "token-1",
...@@ -378,6 +557,37 @@ describe("CloudManagedEndpointRuntime", () => {...@@ -378,6 +557,37 @@ describe("CloudManagedEndpointRuntime", () => {
378 }).pipe(Effect.provide(TestClock.layer())),557 }).pipe(Effect.provide(TestClock.layer())),
379 );558 );
380559
560 it.effect("a recovery that returns the same config keeps the crash backoff", () =>
561 Effect.gen(function* () {
562 const { spawner, spawned, exits, spawnSignals } = yield* makeCrashLoopSpawner(900, 4);
563 const runtime = yield* buildCloudManagedEndpointRuntime(spawner);
564 const config = {
565 providerKind: "cloudflare_tunnel" as const,
566 connectorToken: "same-token",
567 tunnelId: "same-tunnel",
568 };
569 // The startup consumer re-applies whatever the relay hands back. When the
570 // relay confirms the current tunnel, that must not look like a config change.
571 yield* runtime.recoveryRequests.pipe(
572 Stream.runForEach((requested) => runtime.applyConfig(requested).pipe(Effect.asVoid)),
573 Effect.forkChild,
574 );
575
576 yield* runtime.applyConfig(config);
577 yield* Deferred.succeed(exits[0]!, ChildProcessSpawner.ExitCode(1));
578 yield* Deferred.await(spawnSignals[1]!);
579 expect(spawned).toEqual([900, 901]);
580
581 // Second rapid crash still waits out the base delay.
582 yield* Deferred.succeed(exits[1]!, ChildProcessSpawner.ExitCode(1));
583 yield* TestClock.adjust(Duration.millis(999));
584 expect(spawned).toEqual([900, 901]);
585 yield* TestClock.adjust(Duration.millis(1));
586 yield* Deferred.await(spawnSignals[2]!);
587 expect(spawned).toEqual([900, 901, 902]);
588 }).pipe(Effect.provide(TestClock.layer())),
589 );
590
381 it.effect("an explicit config change clears the backoff and preempts a delayed restart", () =>591 it.effect("an explicit config change clears the backoff and preempts a delayed restart", () =>
382 Effect.gen(function* () {592 Effect.gen(function* () {
383 const { spawner, spawned, exits, spawnSignals } = yield* makeCrashLoopSpawner(800, 3);593 const { spawner, spawned, exits, spawnSignals } = yield* makeCrashLoopSpawner(800, 3);
...@@ -510,6 +720,7 @@ describe("CloudManagedEndpointRuntime", () => {...@@ -510,6 +720,7 @@ describe("CloudManagedEndpointRuntime", () => {
510 expect(status).toEqual({720 expect(status).toEqual({
511 status: "failed",721 status: "failed",
512 providerKind: "cloudflare_tunnel",722 providerKind: "cloudflare_tunnel",
723 failure: "not-installed",
513 reason: "The relay client is not installed.",724 reason: "The relay client is not installed.",
514 });725 });
515 expect(spawn).not.toHaveBeenCalled();726 expect(spawn).not.toHaveBeenCalled();
apps/server/src/cloud/ManagedEndpointRuntime.ts+89-41
...@@ -6,7 +6,7 @@ import * as Duration from "effect/Duration";...@@ -6,7 +6,7 @@ import * as Duration from "effect/Duration";
6import * as Effect from "effect/Effect";6import * as Effect from "effect/Effect";
7import * as Exit from "effect/Exit";7import * as Exit from "effect/Exit";
8import * as Layer from "effect/Layer";8import * as Layer from "effect/Layer";
9import * as Option from "effect/Option";9import * as Queue from "effect/Queue";
10import * as Ref from "effect/Ref";10import * as Ref from "effect/Ref";
11import * as Result from "effect/Result";11import * as Result from "effect/Result";
12import * as Semaphore from "effect/Semaphore";12import * as Semaphore from "effect/Semaphore";
...@@ -15,22 +15,6 @@ import * as Stream from "effect/Stream";...@@ -15,22 +15,6 @@ import * as Stream from "effect/Stream";
15import * as ChildProcess from "effect/unstable/process/ChildProcess";15import * as ChildProcess from "effect/unstable/process/ChildProcess";
16import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner";16import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner";
1717
18import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
19import { CLOUD_ENDPOINT_RUNTIME_CONFIG, decodeRuntimeConfig } from "./config.ts";
20
21function bytesToString(bytes: Uint8Array): string {
22 return new TextDecoder().decode(bytes);
23}
24
25const readRuntimeConfig = Effect.gen(function* () {
26 const secrets = yield* ServerSecretStore.ServerSecretStore;
27 const bytes = yield* secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG);
28 if (Option.isNone(bytes)) {
29 return null;
30 }
31 return Option.getOrNull(decodeRuntimeConfig(bytesToString(bytes.value)));
32});
33
34export type CloudManagedEndpointRuntimeStatus =18export type CloudManagedEndpointRuntimeStatus =
35 | {19 | {
36 readonly status: "disabled";20 readonly status: "disabled";
...@@ -38,6 +22,7 @@ export type CloudManagedEndpointRuntimeStatus =...@@ -38,6 +22,7 @@ export type CloudManagedEndpointRuntimeStatus =
38 | {22 | {
39 readonly status: "failed";23 readonly status: "failed";
40 readonly providerKind: RelayManagedEndpointRuntimeConfig["providerKind"];24 readonly providerKind: RelayManagedEndpointRuntimeConfig["providerKind"];
25 readonly failure: "unsupported-platform" | "not-installed" | "spawn-failed";
41 readonly reason: string;26 readonly reason: string;
42 readonly tunnelId?: string;27 readonly tunnelId?: string;
43 readonly tunnelName?: string;28 readonly tunnelName?: string;
...@@ -60,6 +45,9 @@ export class CloudManagedEndpointRuntime extends Context.Service<...@@ -60,6 +45,9 @@ export class CloudManagedEndpointRuntime extends Context.Service<
60 readonly applyConfig: (45 readonly applyConfig: (
61 config: RelayManagedEndpointRuntimeConfig | null,46 config: RelayManagedEndpointRuntimeConfig | null,
62 ) => Effect.Effect<CloudManagedEndpointRuntimeStatus>;47 ) => Effect.Effect<CloudManagedEndpointRuntimeStatus>;
48 readonly recoveryRequests: Stream.Stream<RelayManagedEndpointRuntimeConfig>;
49 readonly requestRecovery: (config: RelayManagedEndpointRuntimeConfig) => Effect.Effect<void>;
50 readonly withLinkStateLock: <A, E, R>(effect: Effect.Effect<A, E, R>) => Effect.Effect<A, E, R>;
63 }51 }
64>()("t3/cloud/ManagedEndpointRuntime/CloudManagedEndpointRuntime") {}52>()("t3/cloud/ManagedEndpointRuntime/CloudManagedEndpointRuntime") {}
6553
...@@ -79,6 +67,8 @@ interface ActiveConnector {...@@ -79,6 +67,8 @@ interface ActiveConnector {
79const RELAY_RESTART_STABLE_UPTIME_MS = 30_000;67const RELAY_RESTART_STABLE_UPTIME_MS = 30_000;
80const RELAY_RESTART_BACKOFF_BASE_MS = 1_000;68const RELAY_RESTART_BACKOFF_BASE_MS = 1_000;
81const RELAY_RESTART_BACKOFF_MAX_MS = 60_000;69const RELAY_RESTART_BACKOFF_MAX_MS = 60_000;
70// Newly created tunnels can fail authorization briefly while Cloudflare propagates their token.
71const TUNNEL_AUTHORIZATION_FAILURES_BEFORE_RECOVERY = 4;
8272
83export function classifyRelayClientOutput(line: string): "connected" | "warning" | "debug" {73export function classifyRelayClientOutput(line: string): "connected" | "warning" | "debug" {
84 if (/\bRegistered tunnel connection\b/iu.test(line)) {74 if (/\bRegistered tunnel connection\b/iu.test(line)) {
...@@ -90,6 +80,32 @@ export function classifyRelayClientOutput(line: string): "connected" | "warning"...@@ -90,6 +80,32 @@ export function classifyRelayClientOutput(line: string): "connected" | "warning"
90 return /\b(?:ERR|WRN|FTL|PNC)\b/u.test(line) ? "warning" : "debug";80 return /\b(?:ERR|WRN|FTL|PNC)\b/u.test(line) ? "warning" : "debug";
91}81}
9282
83/**
84 * Cloudflare's edge rejects a connector whose tunnel was deleted or whose
85 * token no longer matches. Current edge output is
86 * `error="Failed to get tunnel"` with no prefix; older edges prefixed the
87 * same messages with `Unauthorized:`. Match both so recovery fires on either.
88 */
89export function isRejectedRelayClientTunnelOutput(line: string): boolean {
90 return (
91 /\bRegister tunnel error from server side\b/iu.test(line) &&
92 /error="(?:Unauthorized:\s*)?(?:Failed to get tunnel|Record for tunnel not found|Invalid tunnel secret)"/iu.test(
93 line,
94 )
95 );
96}
97
98/** Connector startup failures can clear after installation or a later spawn attempt. */
99export function isRetryableManagedEndpointRuntimeStatus(status: unknown): boolean {
100 if (typeof status !== "object" || status === null || !("status" in status)) {
101 return false;
102 }
103 if (status.status !== "failed" || !("failure" in status)) {
104 return false;
105 }
106 return status.failure === "not-installed" || status.failure === "spawn-failed";
107}
108
93function runtimeConfigKey(config: RelayManagedEndpointRuntimeConfig): string {109function runtimeConfigKey(config: RelayManagedEndpointRuntimeConfig): string {
94 return JSON.stringify({110 return JSON.stringify({
95 providerKind: config.providerKind,111 providerKind: config.providerKind,
...@@ -117,8 +133,10 @@ export const make = Effect.gen(function* () {...@@ -117,8 +133,10 @@ export const make = Effect.gen(function* () {
117 const relayClient = yield* RelayClient.RelayClient;133 const relayClient = yield* RelayClient.RelayClient;
118 const activeRef = yield* Ref.make<ActiveConnector | null>(null);134 const activeRef = yield* Ref.make<ActiveConnector | null>(null);
119 const desiredConfigRef = yield* Ref.make<RelayManagedEndpointRuntimeConfig | null>(null);135 const desiredConfigRef = yield* Ref.make<RelayManagedEndpointRuntimeConfig | null>(null);
136 const recoveryRequests = yield* Queue.sliding<RelayManagedEndpointRuntimeConfig>(1);
120 const reconcileSemaphore = yield* Semaphore.make(1);137 const reconcileSemaphore = yield* Semaphore.make(1);
121 const restartDelayRef = yield* Ref.make(0);138 const restartDelayRef = yield* Ref.make(0);
139 const linkStateSemaphore = yield* Semaphore.make(1);
122 let reconcileConfig: CloudManagedEndpointRuntime["Service"]["applyConfig"];140 let reconcileConfig: CloudManagedEndpointRuntime["Service"]["applyConfig"];
123141
124 const stopActive = Effect.gen(function* () {142 const stopActive = Effect.gen(function* () {
...@@ -191,6 +209,7 @@ export const make = Effect.gen(function* () {...@@ -191,6 +209,7 @@ export const make = Effect.gen(function* () {
191 tunnelId: connector.config.tunnelId,209 tunnelId: connector.config.tunnelId,
192 tunnelName: connector.config.tunnelName,210 tunnelName: connector.config.tunnelName,
193 });211 });
212 yield* Queue.offer(recoveryRequests, connector.config);
194 yield* reconcileConfig(desiredConfig);213 yield* reconcileConfig(desiredConfig);
195 }),214 }),
196 );215 );
...@@ -198,8 +217,10 @@ export const make = Effect.gen(function* () {...@@ -198,8 +217,10 @@ export const make = Effect.gen(function* () {
198 Effect.catchCause((cause) => Effect.logWarning("Relay client supervisor failed", { cause })),217 Effect.catchCause((cause) => Effect.logWarning("Relay client supervisor failed", { cause })),
199 );218 );
200219
201 const observeConnectorOutput = (connector: ActiveConnector) =>220 const observeConnectorOutput = (connector: ActiveConnector) => {
202 connector.child.all.pipe(221 let rejectedRegistrations = 0;
222
223 return connector.child.all.pipe(
203 Stream.decodeText(),224 Stream.decodeText(),
204 Stream.splitLines,225 Stream.splitLines,
205 Stream.map((line) => line.trim()),226 Stream.map((line) => line.trim()),
...@@ -214,8 +235,22 @@ export const make = Effect.gen(function* () {...@@ -214,8 +235,22 @@ export const make = Effect.gen(function* () {
214 };235 };
215 switch (classifyRelayClientOutput(line)) {236 switch (classifyRelayClientOutput(line)) {
216 case "connected":237 case "connected":
238 rejectedRegistrations = 0;
217 return Effect.logInfo("Relay client tunnel connection registered", attributes);239 return Effect.logInfo("Relay client tunnel connection registered", attributes);
218 case "warning":240 case "warning":
241 if (isRejectedRelayClientTunnelOutput(line)) {
242 rejectedRegistrations += 1;
243 if (rejectedRegistrations >= TUNNEL_AUTHORIZATION_FAILURES_BEFORE_RECOVERY) {
244 rejectedRegistrations = 0;
245 return Effect.logWarning(
246 "Relay client tunnel was rejected; requesting recovery",
247 attributes,
248 ).pipe(
249 Effect.andThen(Queue.offer(recoveryRequests, connector.config)),
250 Effect.asVoid,
251 );
252 }
253 }
219 return Effect.logWarning("Relay client reported a transport warning", attributes);254 return Effect.logWarning("Relay client reported a transport warning", attributes);
220 case "debug":255 case "debug":
221 return Effect.logDebug("Relay client output", attributes);256 return Effect.logDebug("Relay client output", attributes);
...@@ -230,6 +265,7 @@ export const make = Effect.gen(function* () {...@@ -230,6 +265,7 @@ export const make = Effect.gen(function* () {
230 }),265 }),
231 ),266 ),
232 );267 );
268 };
233269
234 reconcileConfig = Effect.fn("CloudManagedEndpointRuntime.reconcileConfig")(function* (config) {270 reconcileConfig = Effect.fn("CloudManagedEndpointRuntime.reconcileConfig")(function* (config) {
235 if (!config || config.providerKind !== "cloudflare_tunnel") {271 if (!config || config.providerKind !== "cloudflare_tunnel") {
...@@ -261,6 +297,7 @@ export const make = Effect.gen(function* () {...@@ -261,6 +297,7 @@ export const make = Effect.gen(function* () {
261 return {297 return {
262 status: "failed",298 status: "failed",
263 providerKind: "cloudflare_tunnel",299 providerKind: "cloudflare_tunnel",
300 failure: executable.status === "unsupported" ? "unsupported-platform" : "not-installed",
264 reason:301 reason:
265 executable.status === "unsupported"302 executable.status === "unsupported"
266 ? `Relay client is unsupported on ${executable.platform}-${executable.arch}.`303 ? `Relay client is unsupported on ${executable.platform}-${executable.arch}.`
...@@ -273,16 +310,20 @@ export const make = Effect.gen(function* () {...@@ -273,16 +310,20 @@ export const make = Effect.gen(function* () {
273 const connectorScope = yield* Scope.make("sequential");310 const connectorScope = yield* Scope.make("sequential");
274 const child = yield* spawner311 const child = yield* spawner
275 .spawn(312 .spawn(
276 ChildProcess.make(executable.executablePath, ["tunnel", "run"], {313 ChildProcess.make(
277 detached: false,314 executable.executablePath,
278 env: {315 ["tunnel", "--no-autoupdate", "--loglevel", "info", "--output", "default", "run"],
279 ...process.env,316 {
280 TUNNEL_TOKEN: config.connectorToken,317 detached: false,
318 env: {
319 ...process.env,
320 TUNNEL_TOKEN: config.connectorToken,
321 },
322 shell: false,
323 stderr: "pipe",
324 stdout: "pipe",
281 },325 },
282 shell: false,326 ),
283 stderr: "pipe",
284 stdout: "pipe",
285 }),
286 )327 )
287 .pipe(328 .pipe(
288 Effect.provideService(Scope.Scope, connectorScope),329 Effect.provideService(Scope.Scope, connectorScope),
...@@ -303,6 +344,7 @@ export const make = Effect.gen(function* () {...@@ -303,6 +344,7 @@ export const make = Effect.gen(function* () {
303 Effect.as({344 Effect.as({
304 status: "failed",345 status: "failed",
305 providerKind: "cloudflare_tunnel",346 providerKind: "cloudflare_tunnel",
347 failure: "spawn-failed",
306 reason: String(cause),348 reason: String(cause),
307 ...(config.tunnelId ? { tunnelId: config.tunnelId } : {}),349 ...(config.tunnelId ? { tunnelId: config.tunnelId } : {}),
308 ...(config.tunnelName ? { tunnelName: config.tunnelName } : {}),350 ...(config.tunnelName ? { tunnelName: config.tunnelName } : {}),
...@@ -338,6 +380,7 @@ export const make = Effect.gen(function* () {...@@ -338,6 +380,7 @@ export const make = Effect.gen(function* () {
338 return {380 return {
339 status: "failed",381 status: "failed",
340 providerKind: "cloudflare_tunnel",382 providerKind: "cloudflare_tunnel",
383 failure: "spawn-failed",
341 reason: "Relay client did not start.",384 reason: "Relay client did not start.",
342 ...(config.tunnelId ? { tunnelId: config.tunnelId } : {}),385 ...(config.tunnelId ? { tunnelId: config.tunnelId } : {}),
343 ...(config.tunnelName ? { tunnelName: config.tunnelName } : {}),386 ...(config.tunnelName ? { tunnelName: config.tunnelName } : {}),
...@@ -347,26 +390,31 @@ export const make = Effect.gen(function* () {...@@ -347,26 +390,31 @@ export const make = Effect.gen(function* () {
347 const applyConfig = Effect.fn("CloudManagedEndpointRuntime.applyConfig")(390 const applyConfig = Effect.fn("CloudManagedEndpointRuntime.applyConfig")(
348 (config: RelayManagedEndpointRuntimeConfig | null) =>391 (config: RelayManagedEndpointRuntimeConfig | null) =>
349 reconcileSemaphore.withPermits(1)(392 reconcileSemaphore.withPermits(1)(
350 // An explicit config change starts over with a fresh backoff.393 Effect.gen(function* () {
351 Ref.set(restartDelayRef, 0).pipe(394 // A real config change starts over with a fresh backoff. Recovery
352 Effect.andThen(Ref.set(desiredConfigRef, config)),395 // that hands back the same tunnel and token must keep the delay, or
353 Effect.andThen(reconcileConfig(config)),396 // a crash-looping connector respawns on every recovery round trip.
354 ),397 const desired = yield* Ref.get(desiredConfigRef);
398 const unchanged =
399 desired !== null &&
400 config !== null &&
401 runtimeConfigKey(desired) === runtimeConfigKey(config);
402 if (!unchanged) {
403 yield* Ref.set(restartDelayRef, 0);
404 }
405 yield* Ref.set(desiredConfigRef, config);
406 return yield* reconcileConfig(config);
407 }),
355 ),408 ),
356 );409 );
357410
358 const runtime = CloudManagedEndpointRuntime.of({411 const runtime = CloudManagedEndpointRuntime.of({
359 applyConfig,412 applyConfig,
413 recoveryRequests: Stream.fromQueue(recoveryRequests),
414 requestRecovery: (config) => Queue.offer(recoveryRequests, config).pipe(Effect.asVoid),
415 withLinkStateLock: linkStateSemaphore.withPermits(1),
360 });416 });
361417
362 const initialConfig = yield* readRuntimeConfig.pipe(
363 Effect.catch((cause) =>
364 Effect.logWarning("Failed to read managed endpoint runtime config", { cause }).pipe(
365 Effect.as(null),
366 ),
367 ),
368 );
369 yield* runtime.applyConfig(initialConfig);
370 yield* Effect.addFinalizer(() => runtime.applyConfig(null));418 yield* Effect.addFinalizer(() => runtime.applyConfig(null));
371 return runtime;419 return runtime;
372});420});
apps/server/src/cloud/config.ts+18-1
...@@ -1,4 +1,7 @@...@@ -1,4 +1,7 @@
1import { RelayManagedEndpointRuntimeConfig } from "@t3tools/contracts/relay";1import {
2 RelayManagedEndpointOrigin,
3 RelayManagedEndpointRuntimeConfig,
4} from "@t3tools/contracts/relay";
2import * as Effect from "effect/Effect";5import * as Effect from "effect/Effect";
3import * as Option from "effect/Option";6import * as Option from "effect/Option";
4import * as Schema from "effect/Schema";7import * as Schema from "effect/Schema";
...@@ -7,6 +10,7 @@ import type * as ServerSecretStore from "../auth/ServerSecretStore.ts";...@@ -7,6 +10,7 @@ import type * as ServerSecretStore from "../auth/ServerSecretStore.ts";
710
8export const CLOUD_MINT_PUBLIC_KEY = "cloud-mint-ed25519-public-key";11export const CLOUD_MINT_PUBLIC_KEY = "cloud-mint-ed25519-public-key";
9export const CLOUD_ENDPOINT_RUNTIME_CONFIG = "cloud-endpoint-runtime-config";12export const CLOUD_ENDPOINT_RUNTIME_CONFIG = "cloud-endpoint-runtime-config";
13export const CLOUD_ENDPOINT_CONFIRMED_ORIGIN = "cloud-endpoint-confirmed-origin";
10export const CLOUD_LINKED_USER_ID = "cloud-linked-user-id";14export const CLOUD_LINKED_USER_ID = "cloud-linked-user-id";
11export const RELAY_URL_SECRET = "cloud-relay-url";15export const RELAY_URL_SECRET = "cloud-relay-url";
12export const RELAY_ISSUER_SECRET = "cloud-relay-issuer";16export const RELAY_ISSUER_SECRET = "cloud-relay-issuer";
...@@ -21,6 +25,19 @@ export const decodeRuntimeConfig = Schema.decodeUnknownOption(...@@ -21,6 +25,19 @@ export const decodeRuntimeConfig = Schema.decodeUnknownOption(
21 Schema.fromJsonString(RelayManagedEndpointRuntimeConfig),25 Schema.fromJsonString(RelayManagedEndpointRuntimeConfig),
22);26);
2327
28export const ManagedEndpointConfirmedOrigin = Schema.Struct({
29 config: RelayManagedEndpointRuntimeConfig,
30 origin: RelayManagedEndpointOrigin,
31});
32
33export const encodeConfirmedOriginJson = Schema.encodeEffect(
34 Schema.fromJsonString(ManagedEndpointConfirmedOrigin),
35);
36
37export const decodeConfirmedOrigin = Schema.decodeUnknownOption(
38 Schema.fromJsonString(ManagedEndpointConfirmedOrigin),
39);
40
24export function isAgentActivityPublishingEnabledValue(value: string | null): boolean {41export function isAgentActivityPublishingEnabledValue(value: string | null): boolean {
25 return value === "true";42 return value === "true";
26}43}
apps/server/src/cloud/http.test.ts+605-11
...@@ -1,14 +1,18 @@...@@ -1,14 +1,18 @@
1import * as NodeServices from "@effect/platform-node/NodeServices";1import * as NodeServices from "@effect/platform-node/NodeServices";
2import { describe, expect, it } from "@effect/vitest";2import { describe, expect, it } from "@effect/vitest";
3import * as Deferred from "effect/Deferred";
3import * as Duration from "effect/Duration";4import * as Duration from "effect/Duration";
4import * as Effect from "effect/Effect";5import * as Effect from "effect/Effect";
6import * as Fiber from "effect/Fiber";
5import * as FileSystem from "effect/FileSystem";7import * as FileSystem from "effect/FileSystem";
6import * as Layer from "effect/Layer";8import * as Layer from "effect/Layer";
7import * as Option from "effect/Option";9import * as Option from "effect/Option";
8import * as Path from "effect/Path";10import * as Path from "effect/Path";
9import * as PlatformError from "effect/PlatformError";11import * as PlatformError from "effect/PlatformError";
12import * as Schema from "effect/Schema";
10import * as TestClock from "effect/testing/TestClock";13import * as TestClock from "effect/testing/TestClock";
11import * as Tracer from "effect/Tracer";14import * as Tracer from "effect/Tracer";
15import * as Stream from "effect/Stream";
12import {16import {
13 HttpClient,17 HttpClient,
14 HttpClientResponse,18 HttpClientResponse,
...@@ -31,16 +35,37 @@ import {...@@ -31,16 +35,37 @@ import {
31import * as ServerEnvironment from "../environment/ServerEnvironment.ts";35import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
32import { CLOUD_CLI_DESIRED_LINK_SECRET } from "./CliState.ts";36import { CLOUD_CLI_DESIRED_LINK_SECRET } from "./CliState.ts";
33import * as CliTokenManager from "./CliTokenManager.ts";37import * as CliTokenManager from "./CliTokenManager.ts";
34import type { RelayLinkProofRequest } from "@t3tools/contracts/relay";38import {
35import { CLOUD_ENDPOINT_RUNTIME_CONFIG, RELAY_URL_SECRET } from "./config.ts";39 RelayManagedEndpointRecoveryRegistrationRequest,
40 type RelayLinkProofRequest,
41} from "@t3tools/contracts/relay";
42import {
43 CLOUD_ENDPOINT_CONFIRMED_ORIGIN,
44 CLOUD_ENDPOINT_RUNTIME_CONFIG,
45 CLOUD_LINKED_USER_ID,
46 decodeConfirmedOrigin,
47 decodeRuntimeConfig,
48 RELAY_ENVIRONMENT_CREDENTIAL_SECRET,
49 RELAY_URL_SECRET,
50} from "./config.ts";
36import {51import {
37 consumeCloudReplayGuards,52 consumeCloudReplayGuards,
38 isSupportedLinkProviderKind,53 isSupportedLinkProviderKind,
39 linkProofScopes,54 linkProofScopes,
40 pendingServiceUpdateExists,55 pendingServiceUpdateExists,
56 parseManagedEndpointLocalOrigin,
41 reconcileDesiredCloudLink,57 reconcileDesiredCloudLink,
58 reconcileDesiredCloudLinkIfStillDesired,
59 recoverManagedCloudTunnel,
60 registerManagedCloudTunnelRecovery,
42 releaseManagedTunnelOnShutdown,61 releaseManagedTunnelOnShutdown,
62 startManagedCloudTunnelIfOriginConfirmed,
43} from "./http.ts";63} from "./http.ts";
64import {
65 managedTunnelStartupAction,
66 retryManagedTunnelRegistration,
67} from "./managedTunnelStartup.ts";
68import { shouldRetryCloudLink } from "./relayResponse.ts";
44import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts";69import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts";
45import { traceAuthenticatedRelayRequest, traceRelayRequest } from "./traceRelayRequest.ts";70import { traceAuthenticatedRelayRequest, traceRelayRequest } from "./traceRelayRequest.ts";
4671
...@@ -56,6 +81,9 @@ const storeFailure = (tag: "AlreadyExists" | "PermissionDenied") =>...@@ -56,6 +81,9 @@ const storeFailure = (tag: "AlreadyExists" | "PermissionDenied") =>
56 });81 });
5782
58const unusedSecretStoreOperation = () => Effect.die("unused secret-store operation");83const unusedSecretStoreOperation = () => Effect.die("unused secret-store operation");
84const decodeManagedTunnelRecoveryRegistration = Schema.decodeUnknownEffect(
85 Schema.fromJsonString(RelayManagedEndpointRecoveryRegistrationRequest),
86);
5987
60function makeSecretStore(88function makeSecretStore(
61 create: ServerSecretStore.ServerSecretStore["Service"]["create"],89 create: ServerSecretStore.ServerSecretStore["Service"]["create"],
...@@ -211,6 +239,9 @@ describe("reconcileDesiredCloudLink", () => {...@@ -211,6 +239,9 @@ describe("reconcileDesiredCloudLink", () => {
211 ManagedEndpointRuntime.CloudManagedEndpointRuntime,239 ManagedEndpointRuntime.CloudManagedEndpointRuntime,
212 ManagedEndpointRuntime.CloudManagedEndpointRuntime.of({240 ManagedEndpointRuntime.CloudManagedEndpointRuntime.of({
213 applyConfig: unusedSecretStoreOperation,241 applyConfig: unusedSecretStoreOperation,
242 recoveryRequests: Stream.empty,
243 requestRecovery: () => Effect.void,
244 withLinkStateLock: (effect) => effect,
214 } satisfies ManagedEndpointRuntime.CloudManagedEndpointRuntime["Service"]),245 } satisfies ManagedEndpointRuntime.CloudManagedEndpointRuntime["Service"]),
215 ),246 ),
216 Effect.provideService(247 Effect.provideService(
...@@ -236,6 +267,39 @@ describe("reconcileDesiredCloudLink", () => {...@@ -236,6 +267,39 @@ describe("reconcileDesiredCloudLink", () => {
236 );267 );
237});268});
238269
270describe("parseManagedEndpointLocalOrigin", () => {
271 it.each([
272 {
273 input: "http://127.0.0.1:80",
274 httpBaseUrl: "http://127.0.0.1",
275 wsBaseUrl: "ws://127.0.0.1",
276 port: 80,
277 },
278 {
279 input: "https://127.0.0.1:443",
280 httpBaseUrl: "https://127.0.0.1",
281 wsBaseUrl: "wss://127.0.0.1",
282 port: 443,
283 },
284 ])("accepts an explicit default port in $input", ({ input, httpBaseUrl, wsBaseUrl, port }) => {
285 expect(parseManagedEndpointLocalOrigin(input)).toEqual({
286 httpBaseUrl,
287 wsBaseUrl,
288 origin: { localHttpHost: "127.0.0.1", localHttpPort: port },
289 });
290 });
291
292 it.each([
293 "ftp://127.0.0.1:3773",
294 "http://user:password@127.0.0.1:3773",
295 "http://127.0.0.1:3773/api",
296 "http://127.0.0.1:3773?mode=test",
297 "http://127.0.0.1:3773#fragment",
298 ])("rejects non-origin URL %s", (input) => {
299 expect(() => parseManagedEndpointLocalOrigin(input)).toThrow("Invalid local origin");
300 });
301});
302
239describe("releaseManagedTunnelOnShutdown", () => {303describe("releaseManagedTunnelOnShutdown", () => {
240 const cliToken: CliTokenManager.PersistedToken = {304 const cliToken: CliTokenManager.PersistedToken = {
241 accessToken: "cli-access-token",305 accessToken: "cli-access-token",
...@@ -253,7 +317,10 @@ describe("releaseManagedTunnelOnShutdown", () => {...@@ -253,7 +317,10 @@ describe("releaseManagedTunnelOnShutdown", () => {
253 Effect.sync(() => {317 Effect.sync(() => {
254 values.set(name, value);318 values.set(name, value);
255 }),319 }),
256 create: unusedSecretStoreOperation,320 create: (name, value) =>
321 Effect.sync(() => {
322 values.set(name, value);
323 }),
257 getOrCreateRandom: unusedSecretStoreOperation,324 getOrCreateRandom: unusedSecretStoreOperation,
258 remove: (name) =>325 remove: (name) =>
259 Effect.sync(() => {326 Effect.sync(() => {
...@@ -267,7 +334,9 @@ describe("releaseManagedTunnelOnShutdown", () => {...@@ -267,7 +334,9 @@ describe("releaseManagedTunnelOnShutdown", () => {
267 readonly store: ServerSecretStore.ServerSecretStore["Service"];334 readonly store: ServerSecretStore.ServerSecretStore["Service"];
268 readonly applyConfigCalls: Array<unknown>;335 readonly applyConfigCalls: Array<unknown>;
269 readonly requests: Array<HttpClientRequest.HttpClientRequest>;336 readonly requests: Array<HttpClientRequest.HttpClientRequest>;
337 readonly onRequest?: (request: HttpClientRequest.HttpClientRequest) => Effect.Effect<void>;
270 readonly respond?: () => Response;338 readonly respond?: () => Response;
339 readonly respondEffect?: Effect.Effect<Response>;
271 }340 }
272341
273 // Writes the launcher's durable state file into this test's baseDir with342 // Writes the launcher's durable state file into this test's baseDir with
...@@ -319,10 +388,19 @@ describe("releaseManagedTunnelOnShutdown", () => {...@@ -319,10 +388,19 @@ describe("releaseManagedTunnelOnShutdown", () => {
319 applyConfig: (config) =>388 applyConfig: (config) =>
320 Effect.sync(() => {389 Effect.sync(() => {
321 harness.applyConfigCalls.push(config);390 harness.applyConfigCalls.push(config);
322 return {391 return config === null
323 status: "disabled",392 ? ({
324 } satisfies ManagedEndpointRuntime.CloudManagedEndpointRuntimeStatus;393 status: "disabled",
394 } satisfies ManagedEndpointRuntime.CloudManagedEndpointRuntimeStatus)
395 : ({
396 status: "running",
397 providerKind: "cloudflare_tunnel",
398 pid: 123,
399 } satisfies ManagedEndpointRuntime.CloudManagedEndpointRuntimeStatus);
325 }),400 }),
401 recoveryRequests: Stream.empty,
402 requestRecovery: () => Effect.void,
403 withLinkStateLock: (effect) => effect,
326 }),404 }),
327 ),405 ),
328 Effect.provideService(406 Effect.provideService(
...@@ -344,11 +422,14 @@ describe("releaseManagedTunnelOnShutdown", () => {...@@ -344,11 +422,14 @@ describe("releaseManagedTunnelOnShutdown", () => {
344 HttpClient.make((request) =>422 HttpClient.make((request) =>
345 Effect.sync(() => {423 Effect.sync(() => {
346 harness.requests.push(request);424 harness.requests.push(request);
347 return HttpClientResponse.fromWeb(425 }).pipe(
348 request,426 Effect.andThen(harness.onRequest?.(request) ?? Effect.void),
349 (harness.respond ?? (() => Response.json({ ok: true })))(),427 Effect.andThen(
350 );428 harness.respondEffect ??
351 }),429 Effect.sync(() => (harness.respond ?? (() => Response.json({ ok: true })))()),
430 ),
431 Effect.map((response) => HttpClientResponse.fromWeb(request, response)),
432 ),
352 ),433 ),
353 ),434 ),
354 // The release consults the launcher state file under the configured435 // The release consults the launcher state file under the configured
...@@ -364,10 +445,27 @@ describe("releaseManagedTunnelOnShutdown", () => {...@@ -364,10 +445,27 @@ describe("releaseManagedTunnelOnShutdown", () => {
364 // The persisted state of a CLI-managed link whose tunnel is releasable.445 // The persisted state of a CLI-managed link whose tunnel is releasable.
365 const managedLinkSecrets = [446 const managedLinkSecrets = [
366 [CLOUD_ENDPOINT_RUNTIME_CONFIG, "runtime-config"],447 [CLOUD_ENDPOINT_RUNTIME_CONFIG, "runtime-config"],
448 [CLOUD_ENDPOINT_CONFIRMED_ORIGIN, "confirmed-origin"],
367 [RELAY_URL_SECRET, "https://relay.example.test"],449 [RELAY_URL_SECRET, "https://relay.example.test"],
368 [CLOUD_CLI_DESIRED_LINK_SECRET, "managed"],450 [CLOUD_CLI_DESIRED_LINK_SECRET, "managed"],
369 ] as const;451 ] as const;
370452
453 it.effect("does not recreate a link that was unlinked while startup registration retried", () => {
454 const { store, values } = makeMemorySecretStore(managedLinkSecrets);
455 const applyConfigCalls: Array<unknown> = [];
456 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
457
458 return Effect.gen(function* () {
459 // Registration started while this marker existed. Unlink removes it
460 // before startup receives the relay's final not_linked response.
461 values.delete(CLOUD_CLI_DESIRED_LINK_SECRET);
462
463 expect(yield* reconcileDesiredCloudLinkIfStillDesired("http://127.0.0.1:3773")).toBeNull();
464 expect(requests).toEqual([]);
465 expect(applyConfigCalls).toEqual([]);
466 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
467 });
468
371 it.effect("stops the connector, releases the relay tunnel, and drops the dead token", () => {469 it.effect("stops the connector, releases the relay tunnel, and drops the dead token", () => {
372 const { store, values } = makeMemorySecretStore(managedLinkSecrets);470 const { store, values } = makeMemorySecretStore(managedLinkSecrets);
373 const applyConfigCalls: Array<unknown> = [];471 const applyConfigCalls: Array<unknown> = [];
...@@ -386,6 +484,7 @@ describe("releaseManagedTunnelOnShutdown", () => {...@@ -386,6 +484,7 @@ describe("releaseManagedTunnelOnShutdown", () => {
386 );484 );
387 expect(request.headers.authorization).toBe("Bearer cli-access-token");485 expect(request.headers.authorization).toBe("Bearer cli-access-token");
388 expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(false);486 expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(false);
487 expect(values.has(CLOUD_ENDPOINT_CONFIRMED_ORIGIN)).toBe(false);
389 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));488 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
390 });489 });
391490
...@@ -627,6 +726,501 @@ describe("releaseManagedTunnelOnShutdown", () => {...@@ -627,6 +726,501 @@ describe("releaseManagedTunnelOnShutdown", () => {
627 }),726 }),
628 );727 );
629 });728 });
729
730 it.effect("registers an existing tunnel and starts the confirmed connector", () => {
731 const { store } = makeMemorySecretStore([
732 [
733 CLOUD_ENDPOINT_RUNTIME_CONFIG,
734 '{"providerKind":"cloudflare_tunnel","connectorToken":"existing-token","tunnelId":"existing-tunnel"}',
735 ],
736 [RELAY_URL_SECRET, "https://relay.example.test"],
737 [CLOUD_LINKED_USER_ID, "user-123"],
738 [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"],
739 ]);
740 const applyConfigCalls: Array<unknown> = [];
741 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
742
743 return Effect.gen(function* () {
744 expect(yield* registerManagedCloudTunnelRecovery("http://127.0.0.1:3773")).toMatchObject({
745 status: "ready",
746 });
747 expect(requests).toHaveLength(1);
748 expect(requests[0]?.method).toBe("POST");
749 expect(requests[0]?.url).toBe(
750 "https://relay.example.test/v1/environments/env_123/tunnel/recovery",
751 );
752 expect(requests[0]?.headers.authorization).toBe("Bearer environment-credential");
753 const body = requests[0]?.body;
754 expect(body?._tag).toBe("Uint8Array");
755 if (body?._tag === "Uint8Array") {
756 expect(
757 yield* decodeManagedTunnelRecoveryRegistration(new TextDecoder().decode(body.body)),
758 ).toMatchObject({
759 cloudUserId: "user-123",
760 tunnelId: "existing-tunnel",
761 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
762 });
763 }
764 expect(applyConfigCalls).toHaveLength(1);
765 }).pipe(
766 provideReleaseHarness({
767 store,
768 applyConfigCalls,
769 requests,
770 respond: () => Response.json({ status: "ready" }),
771 }),
772 );
773 });
774
775 it.effect("reconciles a changed port after a relay outage outlasts the startup fallback", () => {
776 const config = {
777 providerKind: "cloudflare_tunnel" as const,
778 connectorToken: "existing-token",
779 tunnelId: "existing-tunnel",
780 };
781 const { store } = makeMemorySecretStore([
782 [CLOUD_ENDPOINT_RUNTIME_CONFIG, JSON.stringify(config)],
783 [
784 CLOUD_ENDPOINT_CONFIRMED_ORIGIN,
785 JSON.stringify({
786 config,
787 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
788 }),
789 ],
790 [RELAY_URL_SECRET, "https://relay.example.test"],
791 [CLOUD_LINKED_USER_ID, "user-123"],
792 [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"],
793 ]);
794 const applyConfigCalls: Array<unknown> = [];
795 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
796 let relayAvailable = false;
797 const localOrigin = "http://127.0.0.1:4884";
798
799 return Effect.gen(function* () {
800 const fallbackStarted = yield* Deferred.make<void>();
801 const firstFailure = yield* Deferred.make<void>();
802 expect(yield* startManagedCloudTunnelIfOriginConfirmed(localOrigin)).toBe(false);
803 const registration = yield* Effect.forkChild(
804 retryManagedTunnelRegistration(
805 registerManagedCloudTunnelRecovery(localOrigin).pipe(
806 Effect.tapError(() => Deferred.succeed(firstFailure, undefined)),
807 ),
808 shouldRetryCloudLink,
809 startManagedCloudTunnelIfOriginConfirmed(localOrigin, {
810 requireConfirmedOrigin: false,
811 }).pipe(
812 Effect.orDie,
813 Effect.tap((started) => {
814 expect(started).toBe(true);
815 return Deferred.succeed(fallbackStarted, undefined);
816 }),
817 Effect.asVoid,
818 ),
819 ),
820 { startImmediately: true },
821 );
822 yield* Deferred.await(firstFailure);
823 yield* TestClock.adjust("15 minutes");
824 yield* Effect.raceFirst(
825 Deferred.await(fallbackStarted),
826 Fiber.join(registration).pipe(
827 Effect.andThen(Effect.die("Registration ended before starting the fallback")),
828 ),
829 );
830 expect(applyConfigCalls).toEqual([config]);
831 const attemptsBeforeRecovery = requests.length;
832
833 relayAvailable = true;
834 yield* TestClock.adjust("1 minute");
835 expect(yield* Fiber.join(registration)).toMatchObject({ status: "ready" });
836 expect(requests.length).toBeGreaterThan(attemptsBeforeRecovery);
837 const marker = yield* store.get(CLOUD_ENDPOINT_CONFIRMED_ORIGIN);
838 expect(Option.isSome(marker)).toBe(true);
839 if (Option.isSome(marker)) {
840 expect(
841 Option.getOrThrow(decodeConfirmedOrigin(new TextDecoder().decode(marker.value))),
842 ).toEqual({
843 config,
844 origin: { localHttpHost: "127.0.0.1", localHttpPort: 4884 },
845 });
846 }
847 }).pipe(
848 provideReleaseHarness({
849 store,
850 applyConfigCalls,
851 requests,
852 respond: () =>
853 relayAvailable
854 ? Response.json({ status: "ready" })
855 : Response.json({ message: "relay unavailable" }, { status: 503 }),
856 }),
857 );
858 });
859
860 it.effect(
861 "starts a connector with a marker for the current origin without contacting relay",
862 () => {
863 const configJson =
864 '{"providerKind":"cloudflare_tunnel","connectorToken":"existing-token","tunnelId":"existing-tunnel"}';
865 const config = {
866 providerKind: "cloudflare_tunnel" as const,
867 connectorToken: "existing-token",
868 tunnelId: "existing-tunnel",
869 };
870 const { store } = makeMemorySecretStore([
871 [CLOUD_ENDPOINT_RUNTIME_CONFIG, configJson],
872 [
873 CLOUD_ENDPOINT_CONFIRMED_ORIGIN,
874 `{"config":${configJson},"origin":{"localHttpHost":"127.0.0.1","localHttpPort":3773}}`,
875 ],
876 ]);
877 const applyConfigCalls: Array<unknown> = [];
878 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
879
880 return Effect.gen(function* () {
881 expect(yield* startManagedCloudTunnelIfOriginConfirmed("http://127.0.0.1:3773")).toBe(true);
882 expect(applyConfigCalls).toEqual([config]);
883 expect(requests).toEqual([]);
884 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
885 },
886 );
887
888 it.effect.each([
889 { name: "missing", marker: undefined, origin: "http://127.0.0.1:3773" },
890 {
891 name: "stale",
892 marker:
893 '{"config":{"providerKind":"cloudflare_tunnel","connectorToken":"existing-token","tunnelId":"existing-tunnel"},"origin":{"localHttpHost":"127.0.0.1","localHttpPort":3773}}',
894 origin: "http://127.0.0.1:4884",
895 },
896 ])("does not start a connector with a $name origin marker", ({ marker, origin }) => {
897 const entries: Array<readonly [string, string]> = [
898 [
899 CLOUD_ENDPOINT_RUNTIME_CONFIG,
900 '{"providerKind":"cloudflare_tunnel","connectorToken":"existing-token","tunnelId":"existing-tunnel"}',
901 ],
902 ];
903 if (marker !== undefined) entries.push([CLOUD_ENDPOINT_CONFIRMED_ORIGIN, marker]);
904 const { store } = makeMemorySecretStore(entries);
905 const applyConfigCalls: Array<unknown> = [];
906 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
907
908 return Effect.gen(function* () {
909 expect(yield* startManagedCloudTunnelIfOriginConfirmed(origin)).toBe(false);
910 expect(applyConfigCalls).toEqual([]);
911 expect(requests).toEqual([]);
912 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
913 });
914
915 it.effect(
916 "starts the stored connector without a marker when confirmation is not required",
917 () => {
918 const config = {
919 providerKind: "cloudflare_tunnel" as const,
920 connectorToken: "existing-token",
921 tunnelId: "existing-tunnel",
922 };
923 const { store } = makeMemorySecretStore([
924 [CLOUD_ENDPOINT_RUNTIME_CONFIG, JSON.stringify(config)],
925 ]);
926 const applyConfigCalls: Array<unknown> = [];
927 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
928
929 return Effect.gen(function* () {
930 expect(
931 yield* startManagedCloudTunnelIfOriginConfirmed("http://127.0.0.1:3773", {
932 requireConfirmedOrigin: false,
933 }),
934 ).toBe(true);
935 expect(applyConfigCalls).toEqual([config]);
936 expect(requests).toEqual([]);
937 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
938 },
939 );
940
941 it.effect.each(["replaced", "removed"] as const)(
942 "does not activate a tunnel when its runtime config is %s during registration",
943 (mutation) => {
944 const originalConfig =
945 '{"providerKind":"cloudflare_tunnel","connectorToken":"existing-token","tunnelId":"existing-tunnel"}';
946 const { store, values } = makeMemorySecretStore([
947 [CLOUD_ENDPOINT_RUNTIME_CONFIG, originalConfig],
948 [RELAY_URL_SECRET, "https://relay.example.test"],
949 [CLOUD_LINKED_USER_ID, "user-123"],
950 [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"],
951 ]);
952 const applyConfigCalls: Array<unknown> = [];
953 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
954
955 return Effect.gen(function* () {
956 expect(yield* registerManagedCloudTunnelRecovery("http://127.0.0.1:3773")).toEqual({
957 status: "superseded",
958 });
959 expect(applyConfigCalls).toEqual([]);
960 expect(values.has(CLOUD_ENDPOINT_CONFIRMED_ORIGIN)).toBe(false);
961 }).pipe(
962 provideReleaseHarness({
963 store,
964 applyConfigCalls,
965 requests,
966 respond: () => {
967 if (mutation === "replaced") {
968 values.set(
969 CLOUD_ENDPOINT_RUNTIME_CONFIG,
970 new TextEncoder().encode(
971 '{"providerKind":"cloudflare_tunnel","connectorToken":"fresh-token","tunnelId":"fresh-tunnel"}',
972 ),
973 );
974 } else {
975 values.delete(CLOUD_ENDPOINT_RUNTIME_CONFIG);
976 }
977 return Response.json({ status: "ready" });
978 },
979 }),
980 );
981 },
982 );
983
984 it.effect("requests startup recovery for a legacy config without a recorded tunnel ID", () => {
985 const { store } = makeMemorySecretStore([
986 [
987 CLOUD_ENDPOINT_RUNTIME_CONFIG,
988 '{"providerKind":"cloudflare_tunnel","connectorToken":"token"}',
989 ],
990 [RELAY_URL_SECRET, "https://relay.example.test"],
991 [CLOUD_LINKED_USER_ID, "user-123"],
992 [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"],
993 ]);
994 const applyConfigCalls: Array<unknown> = [];
995 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
996
997 return Effect.gen(function* () {
998 const registration = yield* registerManagedCloudTunnelRecovery("http://127.0.0.1:3773");
999 expect(registration).toEqual({
1000 status: "recovery_required",
1001 config: { providerKind: "cloudflare_tunnel", connectorToken: "token" },
1002 });
1003 expect(
1004 managedTunnelStartupAction({
1005 wantsCliLink: false,
1006 registration,
1007 }),
1008 ).toEqual({
1009 action: "request_recovery",
1010 config: { providerKind: "cloudflare_tunnel", connectorToken: "token" },
1011 });
1012 expect(requests).toEqual([]);
1013 expect(applyConfigCalls).toEqual([]);
1014 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
1015 });
1016
1017 it.effect("recovers a web-linked tunnel with its environment credential", () => {
1018 const oldConfig =
1019 '{"providerKind":"cloudflare_tunnel","connectorToken":"old-token","tunnelId":"old-tunnel"}';
1020 const nextConfig = {
1021 providerKind: "cloudflare_tunnel",
1022 connectorToken: "new-token",
1023 tunnelId: "new-tunnel",
1024 } as const;
1025 const { store, values } = makeMemorySecretStore([
1026 [CLOUD_ENDPOINT_RUNTIME_CONFIG, oldConfig],
1027 [RELAY_URL_SECRET, "https://relay.example.test"],
1028 [CLOUD_LINKED_USER_ID, "user-123"],
1029 [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"],
1030 ]);
1031 const applyConfigCalls: Array<unknown> = [];
1032 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
1033
1034 return Effect.gen(function* () {
1035 expect(yield* recoverManagedCloudTunnel("http://127.0.0.1:3773")).toBe(true);
1036 expect(requests).toHaveLength(1);
1037 expect(requests[0]?.method).toBe("POST");
1038 expect(requests[0]?.url).toBe("https://relay.example.test/v1/environments/env_123/tunnel");
1039 expect(requests[0]?.headers.authorization).toBe("Bearer environment-credential");
1040 expect(applyConfigCalls).toEqual([nextConfig]);
1041 expect(
1042 Option.getOrNull(
1043 decodeRuntimeConfig(new TextDecoder().decode(values.get(CLOUD_ENDPOINT_RUNTIME_CONFIG))),
1044 ),
1045 ).toEqual(nextConfig);
1046 }).pipe(
1047 provideReleaseHarness({
1048 store,
1049 applyConfigCalls,
1050 requests,
1051 respond: () =>
1052 Response.json({
1053 endpoint: {
1054 httpBaseUrl: "https://environment.example.test/",
1055 wsBaseUrl: "wss://environment.example.test/ws",
1056 providerKind: "cloudflare_tunnel",
1057 },
1058 endpointRuntime: nextConfig,
1059 }),
1060 }),
1061 );
1062 });
1063
1064 it.effect("allows managed tunnel provisioning to take longer than ten seconds", () =>
1065 Effect.gen(function* () {
1066 const oldConfig =
1067 '{"providerKind":"cloudflare_tunnel","connectorToken":"old-token","tunnelId":"old-tunnel"}';
1068 const nextConfig = {
1069 providerKind: "cloudflare_tunnel" as const,
1070 connectorToken: "new-token",
1071 tunnelId: "new-tunnel",
1072 };
1073 const { store } = makeMemorySecretStore([
1074 [CLOUD_ENDPOINT_RUNTIME_CONFIG, oldConfig],
1075 [RELAY_URL_SECRET, "https://relay.example.test"],
1076 [CLOUD_LINKED_USER_ID, "user-123"],
1077 [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"],
1078 ]);
1079 const applyConfigCalls: Array<unknown> = [];
1080 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
1081 const requestStarted = yield* Deferred.make<void>();
1082 const response = yield* Deferred.make<Response>();
1083 const recovery = yield* recoverManagedCloudTunnel("http://127.0.0.1:3773").pipe(
1084 provideReleaseHarness({
1085 store,
1086 applyConfigCalls,
1087 requests,
1088 onRequest: () => Deferred.succeed(requestStarted, undefined),
1089 respondEffect: Deferred.await(response),
1090 }),
1091 Effect.forkChild({ startImmediately: true }),
1092 );
1093
1094 yield* Deferred.await(requestStarted);
1095 expect(requests).toHaveLength(1);
1096 yield* TestClock.adjust("11 seconds");
1097 yield* Effect.yieldNow;
1098 yield* Deferred.succeed(
1099 response,
1100 Response.json({
1101 endpoint: {
1102 httpBaseUrl: "https://environment.example.test/",
1103 wsBaseUrl: "wss://environment.example.test/ws",
1104 providerKind: "cloudflare_tunnel",
1105 },
1106 endpointRuntime: nextConfig,
1107 }),
1108 );
1109
1110 expect(yield* Fiber.join(recovery)).toBe(true);
1111 expect(requests).toHaveLength(1);
1112 expect(applyConfigCalls).toEqual([nextConfig]);
1113 }),
1114 );
1115
1116 it.effect("does not recover an environment without a managed tunnel credential", () => {
1117 const { store } = makeMemorySecretStore([
1118 [CLOUD_ENDPOINT_RUNTIME_CONFIG, "old-config"],
1119 [RELAY_URL_SECRET, "https://relay.example.test"],
1120 ]);
1121 const applyConfigCalls: Array<unknown> = [];
1122 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
1123
1124 return Effect.gen(function* () {
1125 expect(yield* recoverManagedCloudTunnel("http://127.0.0.1:3773")).toBe(false);
1126 expect(applyConfigCalls).toEqual([]);
1127 expect(requests).toEqual([]);
1128 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
1129 });
1130
1131 it.effect("ignores recovery requests for a tunnel that has already been replaced", () => {
1132 const { store } = makeMemorySecretStore([
1133 [
1134 CLOUD_ENDPOINT_RUNTIME_CONFIG,
1135 '{"providerKind":"cloudflare_tunnel","connectorToken":"current-token","tunnelId":"current-tunnel"}',
1136 ],
1137 [RELAY_URL_SECRET, "https://relay.example.test"],
1138 [CLOUD_LINKED_USER_ID, "user-123"],
1139 [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"],
1140 ]);
1141 const applyConfigCalls: Array<unknown> = [];
1142 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
1143
1144 return Effect.gen(function* () {
1145 expect(
1146 yield* recoverManagedCloudTunnel("http://127.0.0.1:3773", {
1147 providerKind: "cloudflare_tunnel",
1148 connectorToken: "old-token",
1149 tunnelId: "old-tunnel",
1150 }),
1151 ).toBe(false);
1152 expect(requests).toEqual([]);
1153 expect(applyConfigCalls).toEqual([]);
1154 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
1155 });
1156
1157 it.effect.each([
1158 { status: 401, errorTag: "EnvironmentHttpUnauthorizedError" },
1159 { status: 403, errorTag: "EnvironmentHttpForbiddenError" },
1160 { status: 409, errorTag: "EnvironmentHttpBadRequestError" },
1161 ])("preserves a permanent $status relay recovery failure", ({ status, errorTag }) => {
1162 const { store } = makeMemorySecretStore([
1163 [CLOUD_ENDPOINT_RUNTIME_CONFIG, "old-config"],
1164 [RELAY_URL_SECRET, "https://relay.example.test"],
1165 [CLOUD_LINKED_USER_ID, "user-123"],
1166 [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"],
1167 ]);
1168 const applyConfigCalls: Array<unknown> = [];
1169 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
1170
1171 return Effect.gen(function* () {
1172 const error = yield* Effect.flip(recoverManagedCloudTunnel("http://127.0.0.1:3773"));
1173
1174 expect(error._tag).toBe(errorTag);
1175 expect(requests).toHaveLength(1);
1176 expect(applyConfigCalls).toEqual([]);
1177 }).pipe(
1178 provideReleaseHarness({
1179 store,
1180 applyConfigCalls,
1181 requests,
1182 respond: () => Response.json({}, { status }),
1183 }),
1184 );
1185 });
1186
1187 it.effect("keeps a tunnel configuration replaced during recovery", () => {
1188 const { store, values } = makeMemorySecretStore([
1189 [CLOUD_ENDPOINT_RUNTIME_CONFIG, "old-config"],
1190 [RELAY_URL_SECRET, "https://relay.example.test"],
1191 [CLOUD_LINKED_USER_ID, "user-123"],
1192 [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"],
1193 ]);
1194 const applyConfigCalls: Array<unknown> = [];
1195 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
1196 const freshConfig = new TextEncoder().encode("fresh-config");
1197
1198 return Effect.gen(function* () {
1199 expect(yield* recoverManagedCloudTunnel("http://127.0.0.1:3773")).toBe(false);
1200 expect(values.get(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(freshConfig);
1201 expect(applyConfigCalls).toEqual([]);
1202 }).pipe(
1203 provideReleaseHarness({
1204 store,
1205 applyConfigCalls,
1206 requests,
1207 respond: () => {
1208 values.set(CLOUD_ENDPOINT_RUNTIME_CONFIG, freshConfig);
1209 return Response.json({
1210 endpoint: {
1211 httpBaseUrl: "https://environment.example.test/",
1212 wsBaseUrl: "wss://environment.example.test/ws",
1213 providerKind: "cloudflare_tunnel",
1214 },
1215 endpointRuntime: {
1216 providerKind: "cloudflare_tunnel",
1217 connectorToken: "replacement-token",
1218 },
1219 });
1220 },
1221 }),
1222 );
1223 });
630});1224});
6311225
632describe("link proof provider kinds", () => {1226describe("link proof provider kinds", () => {
apps/server/src/cloud/http.ts+583-84
...@@ -29,6 +29,10 @@ import {...@@ -29,6 +29,10 @@ import {
29 RelayEnvironmentLinkProofPayload,29 RelayEnvironmentLinkProofPayload,
30 RelayLinkProofRequest,30 RelayLinkProofRequest,
31 RelayManagedEndpointOrigin,31 RelayManagedEndpointOrigin,
32 RelayManagedEndpointRecoveryProofPayload,
33 RelayManagedEndpointRecoveryRegistrationResponse,
34 RelayManagedEndpointRecoveryResponse,
35 type RelayManagedEndpointRuntimeConfig,
32 RelayOkResponse,36 RelayOkResponse,
33} from "@t3tools/contracts/relay";37} from "@t3tools/contracts/relay";
34import { withRelayClientTracing } from "@t3tools/shared/relayTracing";38import { withRelayClientTracing } from "@t3tools/shared/relayTracing";
...@@ -37,6 +41,7 @@ import {...@@ -37,6 +41,7 @@ import {
37 RELAY_HEALTH_REQUEST_TYP,41 RELAY_HEALTH_REQUEST_TYP,
38 RELAY_HEALTH_RESPONSE_TYP,42 RELAY_HEALTH_RESPONSE_TYP,
39 RELAY_LINK_PROOF_TYP,43 RELAY_LINK_PROOF_TYP,
44 RELAY_MANAGED_TUNNEL_RECOVERY_TYP,
40 RELAY_MINT_REQUEST_TYP,45 RELAY_MINT_REQUEST_TYP,
41 RELAY_MINT_RESPONSE_TYP,46 RELAY_MINT_RESPONSE_TYP,
42 signRelayJwt,47 signRelayJwt,
...@@ -52,10 +57,12 @@ import * as FileSystem from "effect/FileSystem";...@@ -52,10 +57,12 @@ import * as FileSystem from "effect/FileSystem";
52import * as Option from "effect/Option";57import * as Option from "effect/Option";
53import * as Path from "effect/Path";58import * as Path from "effect/Path";
54import * as Schema from "effect/Schema";59import * as Schema from "effect/Schema";
60import * as Schedule from "effect/Schedule";
55import * as HttpEffect from "effect/unstable/http/HttpEffect";61import * as HttpEffect from "effect/unstable/http/HttpEffect";
56import { HttpServerRequest, HttpServerResponse } from "effect/unstable/http";62import { HttpServerRequest, HttpServerResponse } from "effect/unstable/http";
57import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http";63import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http";
58import * as HttpApiBuilder from "effect/unstable/httpapi/HttpApiBuilder";64import * as HttpApiBuilder from "effect/unstable/httpapi/HttpApiBuilder";
65import * as HttpServer from "effect/unstable/http/HttpServer";
5966
60import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts";67import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts";
61import * as ServerSecretStore from "../auth/ServerSecretStore.ts";68import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
...@@ -70,9 +77,13 @@ import {...@@ -70,9 +77,13 @@ import {
70} from "./serviceProtocol.ts";77} from "./serviceProtocol.ts";
71import {78import {
72 CLOUD_ENDPOINT_RUNTIME_CONFIG,79 CLOUD_ENDPOINT_RUNTIME_CONFIG,
80 CLOUD_ENDPOINT_CONFIRMED_ORIGIN,
81 decodeConfirmedOrigin,
73 CLOUD_LINKED_USER_ID,82 CLOUD_LINKED_USER_ID,
74 CLOUD_MINT_PUBLIC_KEY,83 CLOUD_MINT_PUBLIC_KEY,
84 decodeRuntimeConfig,
75 encodeEndpointRuntimeConfigJson,85 encodeEndpointRuntimeConfigJson,
86 encodeConfirmedOriginJson,
76 PUBLISH_AGENT_ACTIVITY_SECRET,87 PUBLISH_AGENT_ACTIVITY_SECRET,
77 RELAY_ENVIRONMENT_CREDENTIAL_SECRET,88 RELAY_ENVIRONMENT_CREDENTIAL_SECRET,
78 RELAY_ISSUER_SECRET,89 RELAY_ISSUER_SECRET,
...@@ -87,7 +98,7 @@ import {...@@ -87,7 +98,7 @@ import {
87import * as CliTokenManager from "./CliTokenManager.ts";98import * as CliTokenManager from "./CliTokenManager.ts";
88import { getOrCreateEnvironmentKeyPairFromSecretStore } from "./environmentKeys.ts";99import { getOrCreateEnvironmentKeyPairFromSecretStore } from "./environmentKeys.ts";
89import { traceRelayRequest } from "./traceRelayRequest.ts";100import { traceRelayRequest } from "./traceRelayRequest.ts";
90import { filterRelayResponse, relayRequestError } from "./relayResponse.ts";101import { filterRelayResponse, relayRequestError, shouldRetryCloudLink } from "./relayResponse.ts";
91102
92const CLOUD_MINT_NONCE_PREFIX = "cloud-mint-nonce-";103const CLOUD_MINT_NONCE_PREFIX = "cloud-mint-nonce-";
93const CLOUD_MINT_JTI_PREFIX = "cloud-mint-jti-";104const CLOUD_MINT_JTI_PREFIX = "cloud-mint-jti-";
...@@ -97,6 +108,7 @@ const CLOUD_PROOF_MAX_LIFETIME_SECONDS = 5 * 60;...@@ -97,6 +108,7 @@ const CLOUD_PROOF_MAX_LIFETIME_SECONDS = 5 * 60;
97const CLOUD_PROOF_CLOCK_SKEW_SECONDS = 60;108const CLOUD_PROOF_CLOCK_SKEW_SECONDS = 60;
98// The desktop app stops its backends within seconds of writing the marker.109// The desktop app stops its backends within seconds of writing the marker.
99const DESKTOP_UPDATE_RESTART_MARKER_TTL = Duration.minutes(1);110const DESKTOP_UPDATE_RESTART_MARKER_TTL = Duration.minutes(1);
111const MANAGED_ENDPOINT_PROVISION_REQUEST_TIMEOUT = Duration.minutes(2);
100const LOOPBACK_HOSTNAMES = new Set(["127.0.0.1", "::1", "localhost"]);112const LOOPBACK_HOSTNAMES = new Set(["127.0.0.1", "::1", "localhost"]);
101const CLOUD_CREDENTIAL_RESPONSE_HEADERS = {113const CLOUD_CREDENTIAL_RESPONSE_HEADERS = {
102 "cache-control": "no-store",114 "cache-control": "no-store",
...@@ -301,6 +313,33 @@ function endpointRequestPort(url: URL): number {...@@ -301,6 +313,33 @@ function endpointRequestPort(url: URL): number {
301 return Number(url.port || (url.protocol === "https:" ? 443 : 80));313 return Number(url.port || (url.protocol === "https:" ? 443 : 80));
302}314}
303315
316export function parseManagedEndpointLocalOrigin(localOrigin: string) {
317 const url = new URL(localOrigin);
318 if (
319 localOrigin !== localOrigin.trim() ||
320 (url.protocol !== "http:" && url.protocol !== "https:") ||
321 url.username !== "" ||
322 url.password !== "" ||
323 url.pathname !== "/" ||
324 url.search !== "" ||
325 url.hash !== "" ||
326 localOrigin.includes("?") ||
327 localOrigin.includes("#")
328 ) {
329 throw new Error("Invalid local origin");
330 }
331 const wsUrl = new URL(url.origin);
332 wsUrl.protocol = url.protocol === "https:" ? "wss:" : "ws:";
333 return {
334 httpBaseUrl: url.origin,
335 wsBaseUrl: wsUrl.origin,
336 origin: {
337 localHttpHost: url.hostname,
338 localHttpPort: endpointRequestPort(url),
339 } satisfies RelayManagedEndpointOrigin,
340 };
341}
342
304function isAllowedEndpointOrigin(input: {343function isAllowedEndpointOrigin(input: {
305 readonly origin: RelayManagedEndpointOrigin;344 readonly origin: RelayManagedEndpointOrigin;
306 readonly requestUrl: string;345 readonly requestUrl: string;
...@@ -455,55 +494,249 @@ const cloudLinkProofHandler = Effect.fn("environment.cloud.linkProof")(...@@ -455,55 +494,249 @@ const cloudLinkProofHandler = Effect.fn("environment.cloud.linkProof")(
455 ),494 ),
456);495);
457496
458const applyCloudRelayConfig = Effect.fn("environment.cloud.applyRelayConfig")(function* (497function managedEndpointRuntimeConfigsMatch(
498 left: RelayManagedEndpointRuntimeConfig,
499 right: RelayManagedEndpointRuntimeConfig,
500): boolean {
501 return (
502 left.providerKind === right.providerKind &&
503 left.connectorToken === right.connectorToken &&
504 left.tunnelId === right.tunnelId &&
505 left.tunnelName === right.tunnelName
506 );
507}
508
509const activateManagedTunnel = Effect.fn("environment.cloud.activateManagedTunnel")(function* (
459 dependencies: CloudHttpDependencies,510 dependencies: CloudHttpDependencies,
460 payload: RelayEnvironmentConfigRequest,511 input: {
512 readonly config: RelayManagedEndpointRuntimeConfig;
513 readonly configJson: string;
514 readonly origin: RelayManagedEndpointOrigin;
515 },
461) {516) {
462 yield* validateRelayConfigPayload(payload);517 return yield* dependencies.endpointRuntime.withLinkStateLock(
463 yield* validateLinkedCloudUser({518 Effect.gen(function* () {
464 secrets: dependencies.secrets,519 const currentConfig = yield* dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG);
465 cloudUserId: payload.cloudUserId,520 if (Option.isNone(currentConfig) || bytesToString(currentConfig.value) !== input.configJson) {
521 return null;
522 }
523 const status = yield* dependencies.endpointRuntime.applyConfig(input.config);
524 if (status.status !== "running") {
525 return yield* new EnvironmentCloudEndpointUnavailableError({
526 message: "Managed endpoint runtime could not be started.",
527 endpointRuntimeStatus: status,
528 });
529 }
530 const marker = yield* encodeConfirmedOriginJson({
531 config: input.config,
532 origin: input.origin,
533 });
534 yield* dependencies.secrets.set(CLOUD_ENDPOINT_CONFIRMED_ORIGIN, stringToBytes(marker));
535 return status;
536 }),
537 );
538});
539
540const activateManagedTunnelWithRetry = (
541 dependencies: CloudHttpDependencies,
542 input: {
543 readonly config: RelayManagedEndpointRuntimeConfig;
544 readonly configJson: string;
545 readonly origin: RelayManagedEndpointOrigin;
546 },
547 retryRuntimeFailures: boolean,
548) => {
549 const activate = activateManagedTunnel(dependencies, input);
550 return retryRuntimeFailures
551 ? activate.pipe(
552 Effect.retry({
553 while: (error) =>
554 error._tag === "EnvironmentCloudEndpointUnavailableError" &&
555 ManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus(
556 error.endpointRuntimeStatus,
557 ),
558 schedule: Schedule.exponential("1 second").pipe(
559 Schedule.modifyDelay(({ duration }) =>
560 Effect.succeed(Duration.min(duration, Duration.seconds(30))),
561 ),
562 Schedule.jittered,
563 ),
564 }),
565 )
566 : activate;
567};
568
569export const startManagedCloudTunnelIfOriginConfirmed = Effect.fn(
570 "environment.cloud.startManagedCloudTunnelIfOriginConfirmed",
571)(function* (localOrigin: string, options?: { readonly requireConfirmedOrigin?: boolean }) {
572 const dependencies = yield* cloudHttpDependencies;
573 const requireConfirmedOrigin = options?.requireConfirmedOrigin ?? true;
574 const parsedOrigin = yield* Effect.try({
575 try: () => parseManagedEndpointLocalOrigin(localOrigin),
576 catch: () =>
577 new EnvironmentHttpBadRequestError({
578 message: "Could not resolve local environment origin.",
579 }),
466 });580 });
467 yield* validateCloudMintPublicKey(payload.cloudMintPublicKey);581 return yield* dependencies.endpointRuntime.withLinkStateLock(
468 const endpointRuntimeStatus = yield* dependencies.endpointRuntime.applyConfig(582 Effect.gen(function* () {
469 payload.endpointRuntime,583 const [runtimeBytes, markerBytes] = yield* Effect.all([
584 dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG),
585 dependencies.secrets.get(CLOUD_ENDPOINT_CONFIRMED_ORIGIN),
586 ]);
587 if (Option.isNone(runtimeBytes)) return false;
588 const config = Option.getOrNull(decodeRuntimeConfig(bytesToString(runtimeBytes.value)));
589 if (config === null || config.providerKind !== "cloudflare_tunnel") return false;
590 // With the marker required, only a config the relay already confirmed on
591 // this port may start. Without it, startup is falling back after the
592 // relay stayed unreachable: an unconfirmed origin may send traffic to a
593 // stale port, but that beats no remote access at all.
594 if (requireConfirmedOrigin) {
595 if (Option.isNone(markerBytes)) return false;
596 const marker = Option.getOrNull(decodeConfirmedOrigin(bytesToString(markerBytes.value)));
597 if (
598 marker === null ||
599 !managedEndpointRuntimeConfigsMatch(marker.config, config) ||
600 marker.origin.localHttpHost !== parsedOrigin.origin.localHttpHost ||
601 marker.origin.localHttpPort !== parsedOrigin.origin.localHttpPort
602 ) {
603 return false;
604 }
605 }
606 const status = yield* dependencies.endpointRuntime.applyConfig(config);
607 if (status.status !== "running") {
608 return yield* new EnvironmentCloudEndpointUnavailableError({
609 message: "Managed endpoint runtime could not be started.",
610 endpointRuntimeStatus: status,
611 });
612 }
613 return true;
614 }),
470 );615 );
471 const ok =616});
472 endpointRuntimeStatus.status === "disabled" || endpointRuntimeStatus.status === "running";617
473 if (!ok) {618const applyCloudRelayConfig = Effect.fn("environment.cloud.applyRelayConfig")(function* (
474 return yield* new EnvironmentCloudEndpointUnavailableError({619 dependencies: CloudHttpDependencies,
475 message: "Managed endpoint runtime could not be started.",620 payload: RelayEnvironmentConfigRequest,
476 endpointRuntimeStatus,621 options?: {
622 readonly lockHeld?: boolean;
623 readonly confirmedOrigin?: RelayManagedEndpointOrigin;
624 },
625) {
626 const apply = Effect.gen(function* () {
627 yield* validateRelayConfigPayload(payload);
628 yield* validateLinkedCloudUser({
629 secrets: dependencies.secrets,
630 cloudUserId: payload.cloudUserId,
477 });631 });
478 }632 yield* validateCloudMintPublicKey(payload.cloudMintPublicKey);
633 // Reject unsupported runtimes before touching the connector so a bad
634 // payload cannot stop a healthy tunnel on its way to a 503.
635 if (
636 payload.endpointRuntime !== null &&
637 payload.endpointRuntime.providerKind !== "cloudflare_tunnel"
638 ) {
639 return yield* new EnvironmentCloudEndpointUnavailableError({
640 message: "Managed endpoint runtime could not be started.",
641 endpointRuntimeStatus: {
642 status: "unsupported",
643 providerKind: payload.endpointRuntime.providerKind,
644 },
645 });
646 }
647 yield* dependencies.endpointRuntime.applyConfig(null);
648 yield* dependencies.secrets.remove(CLOUD_ENDPOINT_CONFIRMED_ORIGIN);
479649
480 yield* dependencies.secrets.set(RELAY_URL_SECRET, stringToBytes(payload.relayUrl));650 yield* dependencies.secrets.set(RELAY_URL_SECRET, stringToBytes(payload.relayUrl));
481 yield* dependencies.secrets.set(
482 RELAY_ISSUER_SECRET,
483 stringToBytes(payload.relayIssuer ?? payload.relayUrl),
484 );
485 yield* dependencies.secrets.set(CLOUD_LINKED_USER_ID, stringToBytes(payload.cloudUserId));
486 yield* dependencies.secrets.set(
487 RELAY_ENVIRONMENT_CREDENTIAL_SECRET,
488 stringToBytes(payload.environmentCredential),
489 );
490 yield* dependencies.secrets.set(CLOUD_MINT_PUBLIC_KEY, stringToBytes(payload.cloudMintPublicKey));
491 if (payload.endpointRuntime) {
492 const endpointRuntimeJson = yield* encodeEndpointRuntimeConfigJson(payload.endpointRuntime);
493 yield* dependencies.secrets.set(651 yield* dependencies.secrets.set(
494 CLOUD_ENDPOINT_RUNTIME_CONFIG,652 RELAY_ISSUER_SECRET,
495 stringToBytes(endpointRuntimeJson),653 stringToBytes(payload.relayIssuer ?? payload.relayUrl),
496 );654 );
497 } else {655 yield* dependencies.secrets.set(CLOUD_LINKED_USER_ID, stringToBytes(payload.cloudUserId));
498 yield* dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG);656 yield* dependencies.secrets.set(
499 }657 RELAY_ENVIRONMENT_CREDENTIAL_SECRET,
500 return { ok, endpointRuntimeStatus } satisfies EnvironmentCloudRelayConfigResult;658 stringToBytes(payload.environmentCredential),
659 );
660 yield* dependencies.secrets.set(
661 CLOUD_MINT_PUBLIC_KEY,
662 stringToBytes(payload.cloudMintPublicKey),
663 );
664 if (payload.endpointRuntime) {
665 const endpointRuntimeJson = yield* encodeEndpointRuntimeConfigJson(payload.endpointRuntime);
666 yield* dependencies.secrets.set(
667 CLOUD_ENDPOINT_RUNTIME_CONFIG,
668 stringToBytes(endpointRuntimeJson),
669 );
670 } else {
671 yield* dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG);
672 }
673 if (payload.endpointRuntime === null || options?.confirmedOrigin === undefined) {
674 return {
675 ok: true,
676 endpointRuntimeStatus: { status: "disabled" },
677 } satisfies EnvironmentCloudRelayConfigResult;
678 }
679 const endpointRuntimeStatus = yield* dependencies.endpointRuntime.applyConfig(
680 payload.endpointRuntime,
681 );
682 if (endpointRuntimeStatus.status !== "running") {
683 return yield* new EnvironmentCloudEndpointUnavailableError({
684 message: "Managed endpoint runtime could not be started.",
685 endpointRuntimeStatus,
686 });
687 }
688 const marker = yield* encodeConfirmedOriginJson({
689 config: payload.endpointRuntime,
690 origin: options.confirmedOrigin,
691 });
692 yield* dependencies.secrets.set(CLOUD_ENDPOINT_CONFIRMED_ORIGIN, stringToBytes(marker));
693 return { ok: true, endpointRuntimeStatus } satisfies EnvironmentCloudRelayConfigResult;
694 });
695 return yield* options?.lockHeld ? apply : dependencies.endpointRuntime.withLinkStateLock(apply);
501});696});
502697
503const cloudRelayConfigHandler = Effect.fn("environment.cloud.relayConfig")(698const cloudRelayConfigHandler = Effect.fn("environment.cloud.relayConfig")(
504 function* (dependencies: CloudHttpDependencies, payload: RelayEnvironmentConfigRequest) {699 function* (dependencies: CloudHttpDependencies, payload: RelayEnvironmentConfigRequest) {
505 yield* requireEnvironmentScope(AuthRelayWriteScope);700 yield* requireEnvironmentScope(AuthRelayWriteScope);
506 return yield* applyCloudRelayConfig(dependencies, payload);701 const result = yield* applyCloudRelayConfig(dependencies, payload);
702 if (payload.endpointRuntime?.providerKind === "cloudflare_tunnel") {
703 const server = yield* HttpServer.HttpServer;
704 const address = server.address;
705 if (typeof address === "string" || !("port" in address)) {
706 return yield* new EnvironmentHttpInternalServerError({
707 message: "Could not resolve the local server origin.",
708 });
709 }
710 const registration = yield* registerManagedCloudTunnelRecovery(
711 `http://127.0.0.1:${address.port}`,
712 ).pipe(
713 Effect.retry({
714 times: 2,
715 while: (error) =>
716 shouldRetryCloudLink(error) &&
717 error._tag !== "EnvironmentCloudEndpointUnavailableError",
718 }),
719 );
720 if (registration.status === "superseded") {
721 return yield* new EnvironmentHttpConflictError({
722 message: "The managed tunnel configuration changed during registration.",
723 });
724 }
725 if (registration.status === "recovery_required") {
726 yield* dependencies.endpointRuntime.requestRecovery(registration.config);
727 }
728 if (registration.status !== "ready") {
729 return yield* new EnvironmentCloudEndpointUnavailableError({
730 message: "Managed endpoint origin could not be confirmed.",
731 endpointRuntimeStatus: { status: "disabled" },
732 });
733 }
734 return {
735 ok: true,
736 endpointRuntimeStatus: registration.endpointRuntimeStatus,
737 } satisfies EnvironmentCloudRelayConfigResult;
738 }
739 return result;
507 },740 },
508 Effect.catchIf(EnvironmentAuth.isServerAuthInternalError, (error) =>741 Effect.catchIf(EnvironmentAuth.isServerAuthInternalError, (error) =>
509 failEnvironmentCloudInternalError(error.message)(error),742 failEnvironmentCloudInternalError(error.message)(error),
...@@ -512,10 +745,14 @@ const cloudRelayConfigHandler = Effect.fn("environment.cloud.relayConfig")(...@@ -512,10 +745,14 @@ const cloudRelayConfigHandler = Effect.fn("environment.cloud.relayConfig")(
512 ServerSecretStore.isSecretStoreError,745 ServerSecretStore.isSecretStoreError,
513 failEnvironmentCloudInternalError("Could not persist environment relay configuration."),746 failEnvironmentCloudInternalError("Could not persist environment relay configuration."),
514 ),747 ),
515 Effect.catchTag(748 Effect.catchTags({
516 "SchemaError",749 SchemaError: failEnvironmentCloudInternalError(
517 failEnvironmentCloudInternalError("Could not persist environment relay configuration."),750 "Could not persist environment relay configuration.",
518 ),751 ),
752 PlatformError: failEnvironmentCloudInternalError(
753 "Could not register the managed endpoint origin.",
754 ),
755 }),
519);756);
520757
521const relayClientRequest = <A>(758const relayClientRequest = <A>(
...@@ -525,6 +762,7 @@ const relayClientRequest = <A>(...@@ -525,6 +762,7 @@ const relayClientRequest = <A>(
525 readonly token: string;762 readonly token: string;
526 readonly payload: unknown;763 readonly payload: unknown;
527 readonly schema: Schema.Decoder<A>;764 readonly schema: Schema.Decoder<A>;
765 readonly timeout?: Duration.Input;
528 },766 },
529) =>767) =>
530 HttpClientRequest.post(input.url).pipe(768 HttpClientRequest.post(input.url).pipe(
...@@ -533,25 +771,20 @@ const relayClientRequest = <A>(...@@ -533,25 +771,20 @@ const relayClientRequest = <A>(
533 Effect.flatMap(dependencies.httpClient.execute),771 Effect.flatMap(dependencies.httpClient.execute),
534 Effect.flatMap(filterRelayResponse),772 Effect.flatMap(filterRelayResponse),
535 Effect.flatMap(HttpClientResponse.schemaBodyJson(input.schema)),773 Effect.flatMap(HttpClientResponse.schemaBodyJson(input.schema)),
774 Effect.timeout(input.timeout ?? "10 seconds"),
536 Effect.mapError(relayRequestError),775 Effect.mapError(relayRequestError),
537 withRelayClientTracing,776 withRelayClientTracing,
538 );777 );
539778
540const reconcileDesiredCloudLinkWith = Effect.fn("environment.cloud.reconcileDesiredLinkWith")(779const reconcileDesiredCloudLinkWith = Effect.fn("environment.cloud.reconcileDesiredLinkWith")(
541 function* (dependencies: CloudHttpDependencies, localOrigin: string) {780 function* (dependencies: CloudHttpDependencies, localOrigin: string) {
542 const localUrl = yield* Effect.try({781 const parsedOrigin = yield* Effect.try({
543 try: () => new URL(localOrigin),782 try: () => parseManagedEndpointLocalOrigin(localOrigin),
544 catch: () =>783 catch: () =>
545 new EnvironmentHttpBadRequestError({784 new EnvironmentHttpBadRequestError({
546 message: "Could not resolve local environment origin.",785 message: "Could not resolve local environment origin.",
547 }),786 }),
548 });787 });
549 if (localUrl.origin !== localOrigin) {
550 return yield* new EnvironmentHttpBadRequestError({
551 message: "Could not resolve local environment origin.",
552 });
553 }
554 const localWsOrigin = localOrigin.replace(/^http/u, "ws");
555 const token = yield* dependencies.cliTokenManager.getExisting.pipe(788 const token = yield* dependencies.cliTokenManager.getExisting.pipe(
556 Effect.flatMap(789 Effect.flatMap(
557 Option.match({790 Option.match({
...@@ -584,16 +817,13 @@ const reconcileDesiredCloudLinkWith = Effect.fn("environment.cloud.reconcileDesi...@@ -584,16 +817,13 @@ const reconcileDesiredCloudLinkWith = Effect.fn("environment.cloud.reconcileDesi
584 challenge: challenge.challenge,817 challenge: challenge.challenge,
585 relayIssuer: relayUrl,818 relayIssuer: relayUrl,
586 endpoint: {819 endpoint: {
587 httpBaseUrl: localOrigin,820 httpBaseUrl: parsedOrigin.httpBaseUrl,
588 wsBaseUrl: localWsOrigin,821 wsBaseUrl: parsedOrigin.wsBaseUrl,
589 providerKind: managedTunnelsEnabled ? "cloudflare_tunnel" : "manual",822 providerKind: managedTunnelsEnabled ? "cloudflare_tunnel" : "manual",
590 },823 },
591 origin: {824 origin: parsedOrigin.origin,
592 localHttpHost: localUrl.hostname,
593 localHttpPort: endpointRequestPort(localUrl),
594 },
595 },825 },
596 localOrigin,826 parsedOrigin.httpBaseUrl,
597 );827 );
598 const link = yield* relayClientRequest(dependencies, {828 const link = yield* relayClientRequest(dependencies, {
599 url: `${relayUrl}/v1/client/environment-links`,829 url: `${relayUrl}/v1/client/environment-links`,
...@@ -605,16 +835,27 @@ const reconcileDesiredCloudLinkWith = Effect.fn("environment.cloud.reconcileDesi...@@ -605,16 +835,27 @@ const reconcileDesiredCloudLinkWith = Effect.fn("environment.cloud.reconcileDesi
605 managedTunnelsEnabled,835 managedTunnelsEnabled,
606 },836 },
607 schema: RelayEnvironmentLinkResponse,837 schema: RelayEnvironmentLinkResponse,
838 timeout: MANAGED_ENDPOINT_PROVISION_REQUEST_TIMEOUT,
608 });839 });
609 yield* setCliDesiredCloudLink(true, mode);840 yield* setCliDesiredCloudLink(true, mode);
610 return yield* applyCloudRelayConfig(dependencies, {841 yield* applyCloudRelayConfig(
611 relayUrl,842 dependencies,
612 relayIssuer: link.relayIssuer,843 {
613 cloudUserId: link.cloudUserId,844 relayUrl,
614 environmentCredential: link.environmentCredential,845 relayIssuer: link.relayIssuer,
615 cloudMintPublicKey: link.cloudMintPublicKey,846 cloudUserId: link.cloudUserId,
616 endpointRuntime: link.endpointRuntime,847 environmentCredential: link.environmentCredential,
617 });848 cloudMintPublicKey: link.cloudMintPublicKey,
849 endpointRuntime: link.endpointRuntime,
850 },
851 {
852 lockHeld: true,
853 confirmedOrigin: parsedOrigin.origin,
854 },
855 );
856 // Callers decide on managed tunnel recovery from the mode this link
857 // actually used, not from a value read before the relay round trip.
858 return mode;
618 },859 },
619 Effect.catchIf(860 Effect.catchIf(
620 ServerSecretStore.isSecretStoreError,861 ServerSecretStore.isSecretStoreError,
...@@ -631,7 +872,260 @@ const reconcileDesiredCloudLinkWith = Effect.fn("environment.cloud.reconcileDesi...@@ -631,7 +872,260 @@ const reconcileDesiredCloudLinkWith = Effect.fn("environment.cloud.reconcileDesi
631872
632export const reconcileDesiredCloudLink = Effect.fn("environment.cloud.reconcileDesiredLink")(873export const reconcileDesiredCloudLink = Effect.fn("environment.cloud.reconcileDesiredLink")(
633 function* (localOrigin: string) {874 function* (localOrigin: string) {
634 return yield* reconcileDesiredCloudLinkWith(yield* cloudHttpDependencies, localOrigin);875 const dependencies = yield* cloudHttpDependencies;
876 return yield* dependencies.endpointRuntime.withLinkStateLock(
877 reconcileDesiredCloudLinkWith(dependencies, localOrigin),
878 );
879 },
880);
881
882export const reconcileDesiredCloudLinkIfStillDesired = Effect.fn(
883 "environment.cloud.reconcileDesiredLinkIfStillDesired",
884)(function* (localOrigin: string) {
885 const dependencies = yield* cloudHttpDependencies;
886 return yield* dependencies.endpointRuntime.withLinkStateLock(
887 Effect.gen(function* () {
888 if (!(yield* readCliDesiredCloudLink)) {
889 return null;
890 }
891 return yield* reconcileDesiredCloudLinkWith(dependencies, localOrigin);
892 }),
893 );
894});
895
896type ManagedTunnelRecoveryProofInput = {
897 readonly environmentId: RelayManagedEndpointRecoveryProofPayload["environmentId"];
898 readonly cloudUserId: string;
899 readonly relayUrl: string;
900} & (
901 | {
902 readonly action: "register";
903 readonly tunnelId: string;
904 readonly origin: RelayManagedEndpointOrigin;
905 }
906 | { readonly action: "recover"; readonly origin: RelayManagedEndpointOrigin }
907);
908
909const makeManagedTunnelRecoveryProof = Effect.fn(
910 "environment.cloud.makeManagedTunnelRecoveryProof",
911)(function* (dependencies: CloudHttpDependencies, input: ManagedTunnelRecoveryProofInput) {
912 const keyPair = yield* getOrCreateEnvironmentKeyPairFromSecretStore(dependencies.secrets);
913 const configuredIssuer = yield* dependencies.secrets.get(RELAY_ISSUER_SECRET);
914 const now = yield* DateTime.now;
915 const issuedAt = Math.floor(now.epochMilliseconds / 1_000);
916 const claims = {
917 iss: `t3-env:${input.environmentId}`,
918 aud: normalizeRelayIssuer(
919 Option.isSome(configuredIssuer) ? bytesToString(configuredIssuer.value) : input.relayUrl,
920 ),
921 sub: input.environmentId,
922 jti: yield* Crypto.Crypto.pipe(Effect.flatMap((crypto) => crypto.randomUUIDv4)),
923 iat: issuedAt,
924 exp: issuedAt + 60,
925 environmentId: input.environmentId,
926 cloudUserId: input.cloudUserId,
927 };
928 const payload =
929 input.action === "register"
930 ? {
931 ...claims,
932 action: "register" as const,
933 tunnelId: input.tunnelId,
934 origin: input.origin,
935 }
936 : { ...claims, action: "recover" as const, origin: input.origin };
937
938 return yield* signRelayJwt({
939 privateKey: keyPair.privateKey,
940 typ: RELAY_MANAGED_TUNNEL_RECOVERY_TYP,
941 payload,
942 }).pipe(
943 Effect.mapError(
944 () =>
945 new EnvironmentHttpInternalServerError({
946 message: "Could not sign the managed tunnel recovery request.",
947 }),
948 ),
949 );
950});
951
952export const registerManagedCloudTunnelRecovery = Effect.fn(
953 "environment.cloud.registerManagedCloudTunnelRecovery",
954)(function* (localOrigin: string, options?: { readonly retryRuntimeFailures?: boolean }) {
955 const dependencies = yield* cloudHttpDependencies;
956 const [runtimeConfig, relayUrl, cloudUserId, environmentCredential] = yield* Effect.all([
957 dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG),
958 dependencies.secrets.get(RELAY_URL_SECRET),
959 dependencies.secrets.get(CLOUD_LINKED_USER_ID),
960 dependencies.secrets.get(RELAY_ENVIRONMENT_CREDENTIAL_SECRET),
961 ]);
962 if (
963 Option.isNone(runtimeConfig) ||
964 Option.isNone(relayUrl) ||
965 Option.isNone(cloudUserId) ||
966 Option.isNone(environmentCredential)
967 ) {
968 return { status: "not_linked" as const };
969 }
970
971 const config = Option.getOrNull(decodeRuntimeConfig(bytesToString(runtimeConfig.value)));
972 if (config?.providerKind !== "cloudflare_tunnel") {
973 return { status: "not_linked" as const };
974 }
975
976 const parsedOrigin = yield* Effect.try({
977 try: () => parseManagedEndpointLocalOrigin(localOrigin),
978 catch: () =>
979 new EnvironmentHttpBadRequestError({
980 message: "Could not resolve local environment origin.",
981 }),
982 });
983 if (config.tunnelId === undefined) {
984 return { status: "recovery_required" as const, config };
985 }
986 const origin = parsedOrigin.origin;
987 const environmentId = yield* dependencies.environment.getEnvironmentId;
988 const relayUrlValue = bytesToString(relayUrl.value);
989 const cloudUserIdValue = bytesToString(cloudUserId.value);
990 const proof = yield* makeManagedTunnelRecoveryProof(dependencies, {
991 action: "register",
992 environmentId,
993 cloudUserId: cloudUserIdValue,
994 relayUrl: relayUrlValue,
995 tunnelId: config.tunnelId,
996 origin,
997 });
998 const registered = yield* relayClientRequest(dependencies, {
999 url: `${relayUrlValue}/v1/environments/${encodeURIComponent(environmentId)}/tunnel/recovery`,
1000 token: bytesToString(environmentCredential.value),
1001 payload: {
1002 cloudUserId: cloudUserIdValue,
1003 tunnelId: config.tunnelId,
1004 origin,
1005 proof,
1006 },
1007 schema: RelayManagedEndpointRecoveryRegistrationResponse,
1008 });
1009 if (registered.status === "recovery_required") {
1010 return { status: registered.status, config };
1011 }
1012 const endpointRuntimeStatus = yield* activateManagedTunnelWithRetry(
1013 dependencies,
1014 {
1015 config,
1016 configJson: bytesToString(runtimeConfig.value),
1017 origin,
1018 },
1019 options?.retryRuntimeFailures === true,
1020 );
1021 return endpointRuntimeStatus === null
1022 ? { status: "superseded" as const }
1023 : { status: "ready" as const, endpointRuntimeStatus };
1024});
1025
1026export const recoverManagedCloudTunnel = Effect.fn("environment.cloud.recoverManagedCloudTunnel")(
1027 function* (
1028 localOrigin: string,
1029 expectedConfig?: RelayManagedEndpointRuntimeConfig,
1030 options?: { readonly retryRuntimeFailures?: boolean },
1031 ) {
1032 const dependencies = yield* cloudHttpDependencies;
1033 const [runtimeConfig, relayUrl, cloudUserId, environmentCredential] = yield* Effect.all([
1034 dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG),
1035 dependencies.secrets.get(RELAY_URL_SECRET),
1036 dependencies.secrets.get(CLOUD_LINKED_USER_ID),
1037 dependencies.secrets.get(RELAY_ENVIRONMENT_CREDENTIAL_SECRET),
1038 ]);
1039 if (
1040 Option.isNone(runtimeConfig) ||
1041 Option.isNone(relayUrl) ||
1042 Option.isNone(cloudUserId) ||
1043 Option.isNone(environmentCredential)
1044 ) {
1045 return false;
1046 }
1047 if (expectedConfig !== undefined) {
1048 const current = Option.getOrNull(decodeRuntimeConfig(bytesToString(runtimeConfig.value)));
1049 if (
1050 current === null ||
1051 current.providerKind !== expectedConfig.providerKind ||
1052 current.connectorToken !== expectedConfig.connectorToken ||
1053 current.tunnelId !== expectedConfig.tunnelId ||
1054 current.tunnelName !== expectedConfig.tunnelName
1055 ) {
1056 return false;
1057 }
1058 }
1059
1060 const parsedOrigin = yield* Effect.try({
1061 try: () => parseManagedEndpointLocalOrigin(localOrigin),
1062 catch: () =>
1063 new EnvironmentHttpBadRequestError({
1064 message: "Could not resolve local environment origin.",
1065 }),
1066 });
1067
1068 const environmentId = yield* dependencies.environment.getEnvironmentId;
1069 const relayUrlValue = bytesToString(relayUrl.value);
1070 const cloudUserIdValue = bytesToString(cloudUserId.value);
1071 const origin = parsedOrigin.origin;
1072 const proof = yield* makeManagedTunnelRecoveryProof(dependencies, {
1073 action: "recover",
1074 environmentId,
1075 cloudUserId: cloudUserIdValue,
1076 relayUrl: relayUrlValue,
1077 origin,
1078 });
1079 const recovered = yield* relayClientRequest(dependencies, {
1080 url: `${relayUrlValue}/v1/environments/${encodeURIComponent(environmentId)}/tunnel`,
1081 token: bytesToString(environmentCredential.value),
1082 payload: {
1083 cloudUserId: cloudUserIdValue,
1084 origin,
1085 proof,
1086 },
1087 schema: RelayManagedEndpointRecoveryResponse,
1088 timeout: MANAGED_ENDPOINT_PROVISION_REQUEST_TIMEOUT,
1089 });
1090 if (recovered.endpointRuntime.providerKind !== "cloudflare_tunnel") {
1091 return yield* new EnvironmentHttpInternalServerError({
1092 message: "T3 Connect returned an unsupported managed tunnel configuration.",
1093 });
1094 }
1095
1096 const encoded = yield* encodeEndpointRuntimeConfigJson(recovered.endpointRuntime).pipe(
1097 Effect.mapError(
1098 () =>
1099 new EnvironmentHttpInternalServerError({
1100 message: "Could not persist the recovered managed tunnel configuration.",
1101 }),
1102 ),
1103 );
1104 const stored = yield* dependencies.endpointRuntime.withLinkStateLock(
1105 Effect.gen(function* () {
1106 const currentConfig = yield* dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG);
1107 if (
1108 Option.isNone(currentConfig) ||
1109 bytesToString(currentConfig.value) !== bytesToString(runtimeConfig.value)
1110 ) {
1111 return false;
1112 }
1113 yield* dependencies.secrets.set(CLOUD_ENDPOINT_RUNTIME_CONFIG, stringToBytes(encoded));
1114 yield* dependencies.secrets.remove(CLOUD_ENDPOINT_CONFIRMED_ORIGIN);
1115 return true;
1116 }),
1117 );
1118 if (!stored) return false;
1119 const status = yield* activateManagedTunnelWithRetry(
1120 dependencies,
1121 {
1122 config: recovered.endpointRuntime,
1123 configJson: encoded,
1124 origin,
1125 },
1126 options?.retryRuntimeFailures === true,
1127 );
1128 return status !== null;
635 },1129 },
636);1130);
6371131
...@@ -704,11 +1198,10 @@ export const releaseManagedTunnelOnShutdown = Effect.fn(...@@ -704,11 +1198,10 @@ export const releaseManagedTunnelOnShutdown = Effect.fn(
704 if (Option.isNone(runtimeConfig)) {1198 if (Option.isNone(runtimeConfig)) {
705 return false;1199 return false;
706 }1200 }
707 // Only CLI-desired managed links release on shutdown, because the startup1201 // Only CLI-desired managed links release eagerly because this request uses
708 // reconcile that provisions the replacement tunnel only runs for them. A1202 // CLI authorization. Web/mobile links register startup recovery with their
709 // link installed by a web/mobile client comes back after a restart by1203 // environment credential, and the relay reaper removes them after they are
710 // reapplying the stored connector token — it has no boot-time re-provision1204 // down for the configured grace period. Unlink still deletes either kind.
711 // path — so its tunnel must survive the restart. (Unlink still deletes it.)
712 if (!(yield* readCliDesiredCloudLink) || (yield* readCliDesiredLinkMode) !== "managed") {1205 if (!(yield* readCliDesiredCloudLink) || (yield* readCliDesiredLinkMode) !== "managed") {
713 return false;1206 return false;
714 }1207 }
...@@ -765,6 +1258,7 @@ export const releaseManagedTunnelOnShutdown = Effect.fn(...@@ -765,6 +1258,7 @@ export const releaseManagedTunnelOnShutdown = Effect.fn(
765 bytesToString(storedConfig.value) === bytesToString(runtimeConfig.value)1258 bytesToString(storedConfig.value) === bytesToString(runtimeConfig.value)
766 ) {1259 ) {
767 yield* dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG);1260 yield* dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG);
1261 yield* dependencies.secrets.remove(CLOUD_ENDPOINT_CONFIRMED_ORIGIN);
768 }1262 }
769 return true;1263 return true;
770});1264});
...@@ -811,21 +1305,26 @@ const cloudLinkStateHandler = Effect.fn("environment.cloud.linkState")(...@@ -811,21 +1305,26 @@ const cloudLinkStateHandler = Effect.fn("environment.cloud.linkState")(
811const cloudUnlinkHandler = Effect.fn("environment.cloud.unlink")(1305const cloudUnlinkHandler = Effect.fn("environment.cloud.unlink")(
812 function* (dependencies: CloudHttpDependencies) {1306 function* (dependencies: CloudHttpDependencies) {
813 yield* requireEnvironmentScope(AuthRelayWriteScope);1307 yield* requireEnvironmentScope(AuthRelayWriteScope);
814 const endpointRuntimeStatus = yield* dependencies.endpointRuntime.applyConfig(null);1308 return yield* dependencies.endpointRuntime.withLinkStateLock(
815 yield* Effect.all(1309 Effect.gen(function* () {
816 [1310 const endpointRuntimeStatus = yield* dependencies.endpointRuntime.applyConfig(null);
817 dependencies.secrets.remove(CLOUD_LINKED_USER_ID),1311 yield* Effect.all(
818 dependencies.secrets.remove(RELAY_URL_SECRET),1312 [
819 dependencies.secrets.remove(RELAY_ISSUER_SECRET),1313 dependencies.secrets.remove(CLOUD_LINKED_USER_ID),
820 dependencies.secrets.remove(RELAY_ENVIRONMENT_CREDENTIAL_SECRET),1314 dependencies.secrets.remove(RELAY_URL_SECRET),
821 dependencies.secrets.remove(CLOUD_MINT_PUBLIC_KEY),1315 dependencies.secrets.remove(RELAY_ISSUER_SECRET),
822 dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG),1316 dependencies.secrets.remove(RELAY_ENVIRONMENT_CREDENTIAL_SECRET),
823 dependencies.secrets.remove(PUBLISH_AGENT_ACTIVITY_SECRET),1317 dependencies.secrets.remove(CLOUD_MINT_PUBLIC_KEY),
824 ],1318 dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG),
825 { concurrency: 7 },1319 dependencies.secrets.remove(CLOUD_ENDPOINT_CONFIRMED_ORIGIN),
1320 dependencies.secrets.remove(PUBLISH_AGENT_ACTIVITY_SECRET),
1321 ],
1322 { concurrency: 8 },
1323 );
1324 yield* setCliDesiredCloudLink(false);
1325 return { ok: true, endpointRuntimeStatus } satisfies EnvironmentCloudRelayConfigResult;
1326 }),
826 );1327 );
827 yield* setCliDesiredCloudLink(false);
828 return { ok: true, endpointRuntimeStatus } satisfies EnvironmentCloudRelayConfigResult;
829 },1328 },
830 Effect.catchIf(1329 Effect.catchIf(
831 ServerSecretStore.isSecretStoreError,1330 ServerSecretStore.isSecretStoreError,
apps/server/src/cloud/managedTunnelStartup.test.ts created+132
...@@ -0,0 +1,132 @@
1import { describe, expect, it } from "@effect/vitest";
2import * as Deferred from "effect/Deferred";
3import * as Effect from "effect/Effect";
4import * as Fiber from "effect/Fiber";
5import * as TestClock from "effect/testing/TestClock";
6
7import {
8 managedTunnelStartupAction,
9 retryManagedTunnelRegistration,
10} from "./managedTunnelStartup.ts";
11
12describe("managedTunnelStartupAction", () => {
13 const config = {
14 providerKind: "cloudflare_tunnel" as const,
15 connectorToken: "connector-token",
16 tunnelId: "tunnel-1",
17 };
18
19 it("requests tunnel recovery only when the relay proves it is needed", () => {
20 expect(
21 managedTunnelStartupAction({
22 wantsCliLink: true,
23 registration: { status: "recovery_required", config },
24 }),
25 ).toEqual({ action: "request_recovery", config });
26 });
27
28 it("creates a desired CLI link only when no local managed link exists", () => {
29 expect(
30 managedTunnelStartupAction({
31 wantsCliLink: true,
32 registration: { status: "not_linked" },
33 }),
34 ).toEqual({ action: "reconcile_link" });
35 });
36
37 it.each(["ready", "unavailable"] as const)(
38 "does not provision after a %s registration result",
39 (status) => {
40 expect(
41 managedTunnelStartupAction({
42 wantsCliLink: true,
43 registration: { status },
44 }),
45 ).toEqual({ action: "none" });
46 },
47 );
48});
49
50describe("retryManagedTunnelRegistration", () => {
51 it.effect("does not fall back or retry when registration is permanently rejected", () =>
52 Effect.gen(function* () {
53 let attempts = 0;
54 let fallbacks = 0;
55 const error = yield* Effect.flip(
56 retryManagedTunnelRegistration(
57 Effect.suspend(() => {
58 attempts += 1;
59 return Effect.fail("not authorized");
60 }),
61 () => false,
62 Effect.sync(() => {
63 fallbacks += 1;
64 }),
65 ),
66 );
67 expect(error).toBe("not authorized");
68 expect(attempts).toBe(1);
69 expect(fallbacks).toBe(0);
70 }),
71 );
72
73 it.effect("stops retrying after the retry window so startup can fall back", () =>
74 Effect.gen(function* () {
75 let attempts = 0;
76 const registration = Effect.suspend(() => {
77 attempts += 1;
78 return Effect.fail("relay unavailable" as const);
79 });
80 const fiber = yield* Effect.forkChild(
81 Effect.flip(retryManagedTunnelRegistration(registration, () => true)),
82 { startImmediately: true },
83 );
84 yield* TestClock.adjust("15 minutes");
85 expect(yield* Fiber.join(fiber)).toBe("relay unavailable");
86 // Capped at 30 seconds between attempts, ten minutes allows a bounded run.
87 expect(attempts).toBeGreaterThan(5);
88 expect(attempts).toBeLessThan(60);
89 }),
90 );
91
92 it.effect("waits for successful registration before it activates the connector", () =>
93 Effect.gen(function* () {
94 const firstAttempt = yield* Deferred.make<void>();
95 let attempts = 0;
96 let activations = 0;
97 let reconciliations = 0;
98 const registration = Effect.suspend(() => {
99 attempts += 1;
100 if (attempts === 1) {
101 return Deferred.succeed(firstAttempt, undefined).pipe(
102 Effect.andThen(Effect.fail("relay unavailable" as const)),
103 );
104 }
105 return Effect.succeed({ status: "ready" as const });
106 });
107 const startup = retryManagedTunnelRegistration(registration, () => true).pipe(
108 Effect.tap((result) =>
109 Effect.sync(() => {
110 const action = managedTunnelStartupAction({ wantsCliLink: true, registration: result });
111 if (action.action === "reconcile_link") {
112 reconciliations += 1;
113 }
114 activations += 1;
115 }),
116 ),
117 );
118
119 const fiber = yield* Effect.forkChild(startup, { startImmediately: true });
120 yield* Deferred.await(firstAttempt);
121 expect(attempts).toBe(1);
122 expect(activations).toBe(0);
123
124 yield* TestClock.adjust("2 seconds");
125 yield* Fiber.join(fiber);
126
127 expect(attempts).toBe(2);
128 expect(activations).toBe(1);
129 expect(reconciliations).toBe(0);
130 }),
131 );
132});
apps/server/src/cloud/managedTunnelStartup.ts created+77
...@@ -0,0 +1,77 @@
1import type { RelayManagedEndpointRuntimeConfig } from "@t3tools/contracts/relay";
2import * as Duration from "effect/Duration";
3import * as Effect from "effect/Effect";
4import * as Schedule from "effect/Schedule";
5
6export type ManagedTunnelRegistrationResult =
7 | { readonly status: "not_linked" | "ready" | "unavailable" | "superseded" }
8 | {
9 readonly status: "recovery_required";
10 readonly config: RelayManagedEndpointRuntimeConfig;
11 };
12
13export type ManagedTunnelStartupAction =
14 | { readonly action: "none" }
15 | { readonly action: "reconcile_link" }
16 | {
17 readonly action: "request_recovery";
18 readonly config: RelayManagedEndpointRuntimeConfig;
19 };
20
21export function managedTunnelStartupAction(input: {
22 readonly wantsCliLink: boolean;
23 readonly registration: ManagedTunnelRegistrationResult;
24}): ManagedTunnelStartupAction {
25 if (input.registration.status === "recovery_required") {
26 return {
27 action: "request_recovery",
28 config: input.registration.config,
29 };
30 }
31 if (input.wantsCliLink && input.registration.status === "not_linked") {
32 return { action: "reconcile_link" };
33 }
34 return { action: "none" };
35}
36
37// After this window the host can start its stored connector config while
38// registration keeps retrying to reconcile the origin when the relay returns.
39const MANAGED_TUNNEL_REGISTRATION_RETRY_WINDOW = Duration.minutes(10);
40
41export const retryManagedTunnelRegistration = <A, E, R>(
42 registration: Effect.Effect<A, E, R>,
43 isRetryable: (error: E) => boolean,
44 onRetryWindowExhausted?: Effect.Effect<void, never, R>,
45) => {
46 const schedule = Schedule.exponential("1 second").pipe(
47 Schedule.modifyDelay(({ duration }) =>
48 Effect.succeed(Duration.min(duration, Duration.seconds(30))),
49 ),
50 Schedule.jittered,
51 );
52 return registration.pipe(
53 Effect.retry({
54 while: isRetryable,
55 schedule: schedule.pipe(
56 Schedule.upTo({ duration: MANAGED_TUNNEL_REGISTRATION_RETRY_WINDOW }),
57 ),
58 }),
59 Effect.catch((error) =>
60 onRetryWindowExhausted !== undefined && isRetryable(error)
61 ? onRetryWindowExhausted.pipe(
62 Effect.andThen(registration.pipe(Effect.retry({ while: isRetryable, schedule }))),
63 )
64 : Effect.fail(error),
65 ),
66 );
67};
68
69// A host asks the relay for a replacement tunnel at most this often. Every
70// managed host shares one relay, so a host stuck in a bad loop must not turn
71// into a fleet-wide request storm.
72export const MANAGED_TUNNEL_RECOVERY_COOLDOWN = Duration.minutes(2);
73
74// Existing hosts register on their first boot after an upgrade, and desktop
75// auto-update delivers that boot to many hosts at once. Spread the first
76// registration so the relay and Cloudflare see a ramp instead of a spike.
77export const MANAGED_TUNNEL_FIRST_REGISTRATION_JITTER = Duration.seconds(30);
apps/server/src/server.test.ts+244-33
...@@ -562,6 +562,7 @@ const buildAppUnderTest = (options?: {...@@ -562,6 +562,7 @@ const buildAppUnderTest = (options?: {
562 >;562 >;
563 relayClient?: Partial<RelayClient.RelayClient["Service"]>;563 relayClient?: Partial<RelayClient.RelayClient["Service"]>;
564 cloudCliTokenManager?: Partial<CloudCliTokenManager.CloudCliTokenManager["Service"]>;564 cloudCliTokenManager?: Partial<CloudCliTokenManager.CloudCliTokenManager["Service"]>;
565 httpClient?: HttpClient.HttpClient;
565 nativeTelemetryClient?: Partial<NativeTelemetryClient.NativeTelemetryClient["Service"]>;566 nativeTelemetryClient?: Partial<NativeTelemetryClient.NativeTelemetryClient["Service"]>;
566 desktopTelemetryReceiver?: Partial<567 desktopTelemetryReceiver?: Partial<
567 DesktopTelemetryReceiver.DesktopTelemetryReceiver["Service"]568 DesktopTelemetryReceiver.DesktopTelemetryReceiver["Service"]
...@@ -1176,6 +1177,9 @@ const buildAppUnderTest = (options?: {...@@ -1176,6 +1177,9 @@ const buildAppUnderTest = (options?: {
1176 CloudManagedEndpointRuntime.CloudManagedEndpointRuntime,1177 CloudManagedEndpointRuntime.CloudManagedEndpointRuntime,
1177 CloudManagedEndpointRuntime.CloudManagedEndpointRuntime.of({1178 CloudManagedEndpointRuntime.CloudManagedEndpointRuntime.of({
1178 applyConfig: () => Effect.succeed({ status: "disabled" }),1179 applyConfig: () => Effect.succeed({ status: "disabled" }),
1180 recoveryRequests: Stream.empty,
1181 requestRecovery: () => Effect.void,
1182 withLinkStateLock: (effect) => effect,
1179 ...options?.layers?.cloudManagedEndpointRuntime,1183 ...options?.layers?.cloudManagedEndpointRuntime,
1180 }),1184 }),
1181 ),1185 ),
...@@ -1224,7 +1228,11 @@ const buildAppUnderTest = (options?: {...@@ -1224,7 +1228,11 @@ const buildAppUnderTest = (options?: {
1224 Layer.provideMerge(makeAuthTestLayer()),1228 Layer.provideMerge(makeAuthTestLayer()),
1225 Layer.provideMerge(ServerSecretStore.layer),1229 Layer.provideMerge(ServerSecretStore.layer),
1226 Layer.provide(workspaceAndProjectServicesLayer),1230 Layer.provide(workspaceAndProjectServicesLayer),
1227 Layer.provideMerge(FetchHttpClient.layer),1231 Layer.provideMerge(
1232 options?.layers?.httpClient === undefined
1233 ? FetchHttpClient.layer
1234 : Layer.succeed(HttpClient.HttpClient, options.layers.httpClient),
1235 ),
1228 Layer.provide(GitHubCli.layer.pipe(Layer.provideMerge(VcsProcess.layer))),1236 Layer.provide(GitHubCli.layer.pipe(Layer.provideMerge(VcsProcess.layer))),
1229 Layer.provide(layerConfig),1237 Layer.provide(layerConfig),
1230 );1238 );
...@@ -3223,6 +3231,68 @@ it.layer(NodeServices.layer)("server router seam", (it) => {...@@ -3223,6 +3231,68 @@ it.layer(NodeServices.layer)("server router seam", (it) => {
3223 }).pipe(Effect.provide(NodeHttpServer.layerTest)),3231 }).pipe(Effect.provide(NodeHttpServer.layerTest)),
3224 );3232 );
32253233
3234 it.effect("rejects a non-Cloudflare managed endpoint runtime without persisting the link", () =>
3235 Effect.gen(function* () {
3236 const appliedRuntimeConfigs: Array<unknown> = [];
3237 yield* buildAppUnderTest({
3238 layers: {
3239 cloudManagedEndpointRuntime: {
3240 applyConfig: (config) =>
3241 Effect.sync(() => {
3242 appliedRuntimeConfigs.push(config);
3243 return config === null
3244 ? ({ status: "disabled" } as const)
3245 : ({ status: "unsupported", providerKind: config.providerKind } as const);
3246 }),
3247 },
3248 },
3249 });
3250
3251 const cloudKeyPair = NodeCrypto.generateKeyPairSync("ed25519", {
3252 privateKeyEncoding: { format: "pem", type: "pkcs8" },
3253 publicKeyEncoding: { format: "pem", type: "spki" },
3254 });
3255 const ownerCookie = yield* getAuthenticatedSessionCookieHeader();
3256 const relayConfigUrl = yield* getHttpServerUrl("/api/connect/relay-config");
3257 const relayConfigResponse = yield* fetchEffect(relayConfigUrl, {
3258 method: "POST",
3259 headers: {
3260 cookie: ownerCookie,
3261 "content-type": "application/json",
3262 },
3263 body: jsonRequestBody({
3264 relayUrl: "https://relay.example.test",
3265 cloudUserId: "user_123",
3266 environmentCredential: "t3env_test_credential",
3267 cloudMintPublicKey: cloudKeyPair.publicKey,
3268 endpointRuntime: {
3269 providerKind: "manual",
3270 connectorToken: "manual-token",
3271 },
3272 }),
3273 });
3274 const relayConfigBody = yield* responseJsonEffect<{
3275 readonly _tag?: string;
3276 readonly endpointRuntimeStatus?: { readonly status?: string };
3277 }>(relayConfigResponse);
3278 const linkStateUrl = yield* getHttpServerUrl("/api/connect/link-state");
3279 const linkStateResponse = yield* fetchEffect(linkStateUrl, {
3280 headers: { cookie: ownerCookie },
3281 });
3282 const linkStateBody = yield* responseJsonEffect<{ readonly linked?: boolean }>(
3283 linkStateResponse,
3284 );
3285
3286 assert.equal(relayConfigResponse.status, 503);
3287 assert.equal(relayConfigBody._tag, "EnvironmentCloudEndpointUnavailableError");
3288 assert.equal(relayConfigBody.endpointRuntimeStatus?.status, "unsupported");
3289 // The connector is never touched for a rejected runtime.
3290 assert.deepEqual(appliedRuntimeConfigs, []);
3291 assert.equal(linkStateResponse.status, 200);
3292 assert.equal(linkStateBody.linked, false);
3293 }).pipe(Effect.provide(NodeHttpServer.layerTest)),
3294 );
3295
3226 it.effect("reports local cloud link state from persisted relay config", () =>3296 it.effect("reports local cloud link state from persisted relay config", () =>
3227 Effect.gen(function* () {3297 Effect.gen(function* () {
3228 yield* buildAppUnderTest();3298 yield* buildAppUnderTest();
...@@ -3301,6 +3371,7 @@ it.layer(NodeServices.layer)("server router seam", (it) => {...@@ -3301,6 +3371,7 @@ it.layer(NodeServices.layer)("server router seam", (it) => {
3301 it.effect("unlinks local cloud state and disables the managed endpoint runtime", () =>3371 it.effect("unlinks local cloud state and disables the managed endpoint runtime", () =>
3302 Effect.gen(function* () {3372 Effect.gen(function* () {
3303 const appliedRuntimeConfigs: Array<unknown> = [];3373 const appliedRuntimeConfigs: Array<unknown> = [];
3374 const requestedRecoveryConfigs: Array<unknown> = [];
3304 yield* buildAppUnderTest({3375 yield* buildAppUnderTest({
3305 layers: {3376 layers: {
3306 cloudManagedEndpointRuntime: {3377 cloudManagedEndpointRuntime: {
...@@ -3317,7 +3388,14 @@ it.layer(NodeServices.layer)("server router seam", (it) => {...@@ -3317,7 +3388,14 @@ it.layer(NodeServices.layer)("server router seam", (it) => {
3317 ...(config.tunnelName ? { tunnelName: config.tunnelName } : {}),3388 ...(config.tunnelName ? { tunnelName: config.tunnelName } : {}),
3318 });3389 });
3319 },3390 },
3391 requestRecovery: (config) =>
3392 Effect.sync(() => {
3393 requestedRecoveryConfigs.push(config);
3394 }),
3320 },3395 },
3396 httpClient: HttpClient.make((request) =>
3397 Effect.succeed(HttpClientResponse.fromWeb(request, Response.json({ status: "ready" }))),
3398 ),
3321 },3399 },
3322 });3400 });
33233401
...@@ -3383,6 +3461,7 @@ it.layer(NodeServices.layer)("server router seam", (it) => {...@@ -3383,6 +3461,7 @@ it.layer(NodeServices.layer)("server router seam", (it) => {
3383 assert.equal(linkStateBody.relayUrl, null);3461 assert.equal(linkStateBody.relayUrl, null);
3384 assert.equal(linkStateBody.relayIssuer, null);3462 assert.equal(linkStateBody.relayIssuer, null);
3385 assert.deepEqual(appliedRuntimeConfigs, [3463 assert.deepEqual(appliedRuntimeConfigs, [
3464 null,
3386 {3465 {
3387 providerKind: "cloudflare_tunnel",3466 providerKind: "cloudflare_tunnel",
3388 connectorToken: "connector-token",3467 connectorToken: "connector-token",
...@@ -3391,6 +3470,7 @@ it.layer(NodeServices.layer)("server router seam", (it) => {...@@ -3391,6 +3470,7 @@ it.layer(NodeServices.layer)("server router seam", (it) => {
3391 },3470 },
3392 null,3471 null,
3393 ]);3472 ]);
3473 assert.deepEqual(requestedRecoveryConfigs, []);
3394 }).pipe(Effect.provide(NodeHttpServer.layerTest)),3474 }).pipe(Effect.provide(NodeHttpServer.layerTest)),
3395 );3475 );
33963476
...@@ -3701,19 +3781,169 @@ it.layer(NodeServices.layer)("server router seam", (it) => {...@@ -3701,19 +3781,169 @@ it.layer(NodeServices.layer)("server router seam", (it) => {
3701 }).pipe(Effect.provide(NodeHttpServer.layerTest)),3781 }).pipe(Effect.provide(NodeHttpServer.layerTest)),
3702 );3782 );
37033783
3784 it.effect("keeps a managed connector stopped when relay registration fails", () =>
3785 Effect.gen(function* () {
3786 const appliedRuntimeConfigs: Array<unknown> = [];
3787 const relayRequests: Array<HttpClientRequest.HttpClientRequest> = [];
3788 yield* buildAppUnderTest({
3789 layers: {
3790 cloudManagedEndpointRuntime: {
3791 applyConfig: (config) =>
3792 Effect.sync(() => {
3793 appliedRuntimeConfigs.push(config);
3794 return config === null
3795 ? ({ status: "disabled" } as const)
3796 : ({ status: "running", providerKind: "cloudflare_tunnel", pid: 123 } as const);
3797 }),
3798 },
3799 httpClient: HttpClient.make((request) =>
3800 Effect.sync(() => {
3801 relayRequests.push(request);
3802 return HttpClientResponse.fromWeb(
3803 request,
3804 Response.json({ message: "relay unavailable" }, { status: 503 }),
3805 );
3806 }),
3807 ),
3808 },
3809 });
3810
3811 const cloudKeyPair = NodeCrypto.generateKeyPairSync("ed25519", {
3812 privateKeyEncoding: { format: "pem", type: "pkcs8" },
3813 publicKeyEncoding: { format: "pem", type: "spki" },
3814 });
3815 const ownerCookie = yield* getAuthenticatedSessionCookieHeader();
3816 const relayConfigUrl = yield* getHttpServerUrl("/api/connect/relay-config");
3817 const relayConfigResponse = yield* fetchEffect(relayConfigUrl, {
3818 method: "POST",
3819 headers: {
3820 cookie: ownerCookie,
3821 "content-type": "application/json",
3822 },
3823 body: jsonRequestBody({
3824 relayUrl: "https://relay.example.test",
3825 cloudUserId: "user_123",
3826 environmentCredential: "t3env_test_credential",
3827 cloudMintPublicKey: cloudKeyPair.publicKey,
3828 endpointRuntime: {
3829 providerKind: "cloudflare_tunnel",
3830 connectorToken: "connector-token",
3831 tunnelId: "tunnel-1",
3832 },
3833 }),
3834 });
3835 const relayConfigBody = yield* responseJsonEffect<{ readonly _tag?: string }>(
3836 relayConfigResponse,
3837 );
3838
3839 assert.equal(relayConfigResponse.status, 500);
3840 assert.equal(relayConfigBody._tag, "EnvironmentHttpInternalServerError");
3841 assert.equal(relayRequests.length, 3);
3842 assert.deepEqual(appliedRuntimeConfigs, [null]);
3843 }).pipe(Effect.provide(NodeHttpServer.layerTest)),
3844 );
3845
3846 it.effect(
3847 "queues recovery without starting a connector when relay registration requires it",
3848 () =>
3849 Effect.gen(function* () {
3850 const appliedRuntimeConfigs: Array<unknown> = [];
3851 const requestedRecoveryConfigs: Array<unknown> = [];
3852 const relayRequests: Array<HttpClientRequest.HttpClientRequest> = [];
3853 yield* buildAppUnderTest({
3854 layers: {
3855 cloudManagedEndpointRuntime: {
3856 applyConfig: (config) =>
3857 Effect.sync(() => {
3858 appliedRuntimeConfigs.push(config);
3859 return config === null
3860 ? ({ status: "disabled" } as const)
3861 : ({ status: "running", providerKind: "cloudflare_tunnel", pid: 123 } as const);
3862 }),
3863 requestRecovery: (config) =>
3864 Effect.sync(() => {
3865 requestedRecoveryConfigs.push(config);
3866 }),
3867 },
3868 httpClient: HttpClient.make((request) =>
3869 Effect.sync(() => {
3870 relayRequests.push(request);
3871 return HttpClientResponse.fromWeb(
3872 request,
3873 Response.json({ status: "recovery_required" }),
3874 );
3875 }),
3876 ),
3877 },
3878 });
3879
3880 const cloudKeyPair = NodeCrypto.generateKeyPairSync("ed25519", {
3881 privateKeyEncoding: { format: "pem", type: "pkcs8" },
3882 publicKeyEncoding: { format: "pem", type: "spki" },
3883 });
3884 const ownerCookie = yield* getAuthenticatedSessionCookieHeader();
3885 const relayConfigUrl = yield* getHttpServerUrl("/api/connect/relay-config");
3886 const relayConfigResponse = yield* fetchEffect(relayConfigUrl, {
3887 method: "POST",
3888 headers: {
3889 cookie: ownerCookie,
3890 "content-type": "application/json",
3891 },
3892 body: jsonRequestBody({
3893 relayUrl: "https://relay.example.test",
3894 cloudUserId: "user_123",
3895 environmentCredential: "t3env_test_credential",
3896 cloudMintPublicKey: cloudKeyPair.publicKey,
3897 endpointRuntime: {
3898 providerKind: "cloudflare_tunnel",
3899 connectorToken: "connector-token",
3900 tunnelId: "tunnel-1",
3901 },
3902 }),
3903 });
3904 const relayConfigBody = yield* responseJsonEffect<{
3905 readonly _tag?: string;
3906 readonly endpointRuntimeStatus?: { readonly status?: string };
3907 }>(relayConfigResponse);
3908
3909 assert.equal(relayConfigResponse.status, 503);
3910 assert.equal(relayConfigBody._tag, "EnvironmentCloudEndpointUnavailableError");
3911 assert.equal(relayConfigBody.endpointRuntimeStatus?.status, "disabled");
3912 assert.equal(relayRequests.length, 1);
3913 assert.deepEqual(appliedRuntimeConfigs, [null]);
3914 assert.deepEqual(requestedRecoveryConfigs, [
3915 {
3916 providerKind: "cloudflare_tunnel",
3917 connectorToken: "connector-token",
3918 tunnelId: "tunnel-1",
3919 },
3920 ]);
3921 }).pipe(Effect.provide(NodeHttpServer.layerTest)),
3922 );
3923
3704 it.effect("fails relay config when the managed endpoint connector cannot start", () =>3924 it.effect("fails relay config when the managed endpoint connector cannot start", () =>
3705 Effect.gen(function* () {3925 Effect.gen(function* () {
3926 const appliedRuntimeConfigs: Array<unknown> = [];
3706 yield* buildAppUnderTest({3927 yield* buildAppUnderTest({
3707 layers: {3928 layers: {
3708 cloudManagedEndpointRuntime: {3929 cloudManagedEndpointRuntime: {
3709 applyConfig: () =>3930 applyConfig: (config) =>
3710 Effect.succeed({3931 Effect.sync(() => {
3711 status: "failed",3932 appliedRuntimeConfigs.push(config);
3712 providerKind: "cloudflare_tunnel",3933 return config === null
3713 reason: "cloudflared missing",3934 ? ({ status: "disabled" } as const)
3714 tunnelId: "tunnel-1",3935 : ({
3936 status: "failed",
3937 providerKind: "cloudflare_tunnel",
3938 failure: "not-installed",
3939 reason: "cloudflared missing",
3940 tunnelId: "tunnel-1",
3941 } as const);
3715 }),3942 }),
3716 },3943 },
3944 httpClient: HttpClient.make((request) =>
3945 Effect.succeed(HttpClientResponse.fromWeb(request, Response.json({ status: "ready" }))),
3946 ),
3717 },3947 },
3718 });3948 });
37193949
...@@ -3752,33 +3982,14 @@ it.layer(NodeServices.layer)("server router seam", (it) => {...@@ -3752,33 +3982,14 @@ it.layer(NodeServices.layer)("server router seam", (it) => {
3752 assert.equal(relayConfigBody.message, "Managed endpoint runtime could not be started.");3982 assert.equal(relayConfigBody.message, "Managed endpoint runtime could not be started.");
3753 assert.equal(relayConfigBody.endpointRuntimeStatus?.status, "failed");3983 assert.equal(relayConfigBody.endpointRuntimeStatus?.status, "failed");
3754 assert.equal(relayConfigBody.endpointRuntimeStatus?.reason, "cloudflared missing");3984 assert.equal(relayConfigBody.endpointRuntimeStatus?.reason, "cloudflared missing");
37553985 assert.deepEqual(appliedRuntimeConfigs, [
3756 const now = yield* DateTime.now;3986 null,
3757 const healthRequest = makeCloudEnvironmentHealthRequest({3987 {
3758 privateKey: cloudKeyPair.privateKey,3988 providerKind: "cloudflare_tunnel",
3759 environmentId: testEnvironmentDescriptor.environmentId,3989 connectorToken: "connector-token",
3760 nonce: "cloud-health-after-failed-runtime",3990 tunnelId: "tunnel-1",
3761 issuedAt: DateTime.formatIso(now),
3762 expiresAt: DateTime.formatIso(DateTime.add(now, { minutes: 5 })),
3763 });
3764 const healthUrl = yield* getHttpServerUrl("/api/t3-connect/health");
3765 const healthResponse = yield* fetchEffect(healthUrl, {
3766 method: "POST",
3767 headers: {
3768 "content-type": "application/json",
3769 },3991 },
3770 body: jsonRequestBody(healthRequest),3992 ]);
3771 });
3772 const healthBody = yield* responseJsonEffect<{
3773 _tag?: string;
3774 message?: string;
3775 }>(healthResponse);
3776 assert.equal(healthResponse.status, 500);
3777 assert.equal(healthBody._tag, "EnvironmentHttpInternalServerError");
3778 assert.equal(
3779 healthBody.message,
3780 "Cloud mint public key is not installed for this environment.",
3781 );
3782 }).pipe(Effect.provide(NodeHttpServer.layerTest)),3993 }).pipe(Effect.provide(NodeHttpServer.layerTest)),
3783 );3994 );
37843995
apps/server/src/server.ts+179-19
...@@ -8,12 +8,16 @@ import {...@@ -8,12 +8,16 @@ import {
8 ProviderDriverKind,8 ProviderDriverKind,
9 type RepositoryIdentity,9 type RepositoryIdentity,
10} from "@t3tools/contracts";10} from "@t3tools/contracts";
11import type { RelayManagedEndpointRuntimeConfig } from "@t3tools/contracts/relay";
11import * as Cause from "effect/Cause";12import * as Cause from "effect/Cause";
13import * as Clock from "effect/Clock";
12import * as Duration from "effect/Duration";14import * as Duration from "effect/Duration";
13import * as Deferred from "effect/Deferred";15import * as Deferred from "effect/Deferred";
14import * as Effect from "effect/Effect";16import * as Effect from "effect/Effect";
15import * as Layer from "effect/Layer";17import * as Layer from "effect/Layer";
18import * as Random from "effect/Random";
16import * as Schedule from "effect/Schedule";19import * as Schedule from "effect/Schedule";
20import * as Semaphore from "effect/Semaphore";
17import * as Stream from "effect/Stream";21import * as Stream from "effect/Stream";
18import { FetchHttpClient, HttpRouter, HttpServer } from "effect/unstable/http";22import { FetchHttpClient, HttpRouter, HttpServer } from "effect/unstable/http";
19import * as HttpApiBuilder from "effect/unstable/httpapi/HttpApiBuilder";23import * as HttpApiBuilder from "effect/unstable/httpapi/HttpApiBuilder";
...@@ -123,12 +127,21 @@ import * as EnvironmentAuth from "./auth/EnvironmentAuth.ts";...@@ -123,12 +127,21 @@ import * as EnvironmentAuth from "./auth/EnvironmentAuth.ts";
123import {127import {
124 connectHttpApiLayer,128 connectHttpApiLayer,
125 pendingServiceUpdateExists,129 pendingServiceUpdateExists,
126 reconcileDesiredCloudLink,130 reconcileDesiredCloudLinkIfStillDesired,
131 recoverManagedCloudTunnel,
132 registerManagedCloudTunnelRecovery,
133 startManagedCloudTunnelIfOriginConfirmed,
127 releaseManagedTunnelOnShutdown,134 releaseManagedTunnelOnShutdown,
128} from "./cloud/http.ts";135} from "./cloud/http.ts";
129import { serverRelayBrokerTracingLayer } from "./cloud/relayTracing.ts";136import { serverRelayBrokerTracingLayer } from "./cloud/relayTracing.ts";
130import { shouldRetryCloudLink } from "./cloud/relayResponse.ts";137import { shouldRetryCloudLink } from "./cloud/relayResponse.ts";
131import * as CloudManagedEndpointRuntime from "./cloud/ManagedEndpointRuntime.ts";138import * as CloudManagedEndpointRuntime from "./cloud/ManagedEndpointRuntime.ts";
139import {
140 MANAGED_TUNNEL_FIRST_REGISTRATION_JITTER,
141 MANAGED_TUNNEL_RECOVERY_COOLDOWN,
142 managedTunnelStartupAction,
143 retryManagedTunnelRegistration,
144} from "./cloud/managedTunnelStartup.ts";
132import * as CloudCliTokenManager from "./cloud/CliTokenManager.ts";145import * as CloudCliTokenManager from "./cloud/CliTokenManager.ts";
133import * as CloudCliState from "./cloud/CliState.ts";146import * as CloudCliState from "./cloud/CliState.ts";
134import * as ServerSelfUpdate from "./cloud/selfUpdate.ts";147import * as ServerSelfUpdate from "./cloud/selfUpdate.ts";
...@@ -708,10 +721,6 @@ const makeServerLayer = Layer.unwrap(...@@ -708,10 +721,6 @@ const makeServerLayer = Layer.unwrap(
708 : Layer.empty;721 : Layer.empty;
709 const cloudDesiredLinkReconcileLayer = Layer.effectDiscard(722 const cloudDesiredLinkReconcileLayer = Layer.effectDiscard(
710 Effect.gen(function* () {723 Effect.gen(function* () {
711 if (!hasCloudPublicConfig) {
712 yield* Deferred.succeed(cloudLinkParked, undefined).pipe(Effect.orDie);
713 return;
714 }
715 const releaseManagedTunnel = releaseManagedTunnelOnShutdown().pipe(724 const releaseManagedTunnel = releaseManagedTunnelOnShutdown().pipe(
716 Effect.timeout("10 seconds"),725 Effect.timeout("10 seconds"),
717 Effect.tap((released) =>726 Effect.tap((released) =>
...@@ -740,33 +749,184 @@ const makeServerLayer = Layer.unwrap(...@@ -740,33 +749,184 @@ const makeServerLayer = Layer.unwrap(
740 if (!cleanupBeforeActivation) {749 if (!cleanupBeforeActivation) {
741 yield* Effect.addFinalizer(() => releaseManagedTunnel);750 yield* Effect.addFinalizer(() => releaseManagedTunnel);
742 }751 }
743 if (!(yield* CloudCliState.readCliDesiredCloudLink)) return;
744 const server = yield* HttpServer.HttpServer;752 const server = yield* HttpServer.HttpServer;
745 const address = server.address;753 const address = server.address;
746 if (typeof address === "string" || !("port" in address)) return;754 if (typeof address === "string" || !("port" in address)) return;
755 const localOrigin = `http://127.0.0.1:${address.port}`;
756 const endpointRuntime = yield* CloudManagedEndpointRuntime.CloudManagedEndpointRuntime;
757 const recoveryLock = yield* Semaphore.make(1);
758 let lastRecoveryAtMillis = 0;
759 const recoverManagedTunnel = (config: RelayManagedEndpointRuntimeConfig) =>
760 recoveryLock.withPermits(1)(
761 Effect.gen(function* () {
762 const elapsed = (yield* Clock.currentTimeMillis) - lastRecoveryAtMillis;
763 const wait = Duration.toMillis(MANAGED_TUNNEL_RECOVERY_COOLDOWN) - elapsed;
764 if (wait > 0) yield* Effect.sleep(Duration.millis(wait));
765 lastRecoveryAtMillis = yield* Clock.currentTimeMillis;
766 }).pipe(
767 Effect.andThen(
768 recoverManagedCloudTunnel(localOrigin, config, {
769 retryRuntimeFailures: true,
770 }),
771 ),
772 Effect.retry({
773 while: (error) =>
774 shouldRetryCloudLink(error) &&
775 error._tag !== "EnvironmentCloudEndpointUnavailableError",
776 schedule: Schedule.exponential("1 second").pipe(
777 Schedule.modifyDelay(({ duration }) =>
778 Effect.succeed(Duration.min(duration, Duration.seconds(30))),
779 ),
780 Schedule.jittered,
781 ),
782 }),
783 Effect.tap((recovered) =>
784 recovered ? Effect.logInfo("T3 Connect managed tunnel recovered") : Effect.void,
785 ),
786 Effect.catchCause((cause) =>
787 Cause.hasInterrupts(cause)
788 ? Effect.interrupt
789 : Effect.logWarning("Failed to recover the T3 Connect managed tunnel", {
790 cause,
791 }),
792 ),
793 ),
794 );
795 yield* endpointRuntime.recoveryRequests.pipe(
796 Stream.runForEach(recoverManagedTunnel),
797 Effect.forkScoped,
798 );
747 // No settling delay before the first attempt: routes are already799 // No settling delay before the first attempt: routes are already
748 // serving by the time activation opens this gate (the startup800 // serving by the time activation opens this gate (the startup
749 // sequence awaits routesReady), and the retry schedule below801 // sequence awaits routesReady), and the retry schedule below
750 // covers anything this sleep used to hedge against. Every802 // covers anything this sleep used to hedge against. Every
751 // millisecond here is dead time on the path to remote803 // millisecond here is dead time on the path to remote
752 // reachability after a restart.804 // reachability after a restart.
753 yield* reconcileDesiredCloudLink(`http://127.0.0.1:${address.port}`).pipe(805 const wantsCliLink = hasCloudPublicConfig
754 Effect.retry({806 ? yield* CloudCliState.readCliDesiredCloudLink.pipe(
755 while: shouldRetryCloudLink,807 Effect.catch((cause) =>
756 schedule: Schedule.exponential("1 second").pipe(808 Effect.logWarning("Failed to read the desired T3 Connect link", { cause }).pipe(
757 Schedule.modifyDelay(({ duration }) =>809 Effect.as(false),
758 Effect.succeed(Duration.min(duration, Duration.seconds(30))),810 ),
759 ),811 ),
760 Schedule.upTo({ duration: "10 minutes" }),812 )
761 ),813 : false;
762 }),814 // A failed read must not end this fiber before it registers
763 Effect.tap(() => Effect.logInfo("T3 Connect desired link reconciled on startup")),815 // recovery and starts consuming recovery requests. "managed" is
816 // what a missing value means, so it is the safe fallback.
817 const desiredCliLinkMode = wantsCliLink
818 ? yield* CloudCliState.readCliDesiredLinkMode.pipe(
819 Effect.catch((cause) =>
820 Effect.logWarning("Failed to read the desired T3 Connect link mode", {
821 cause,
822 }).pipe(Effect.as("managed" as const)),
823 ),
824 )
825 : null;
826 // A publish-only link must not expose the host, even if a managed
827 // config from an earlier link is still stored.
828 const startedConfirmed =
829 desiredCliLinkMode === "publish_only"
830 ? false
831 : yield* startManagedCloudTunnelIfOriginConfirmed(localOrigin).pipe(
832 Effect.catch((cause) =>
833 Effect.logWarning("Failed to start the confirmed T3 Connect tunnel", {
834 cause,
835 }).pipe(Effect.as(false)),
836 ),
837 );
838 const startStoredManagedTunnel = startManagedCloudTunnelIfOriginConfirmed(localOrigin, {
839 requireConfirmedOrigin: false,
840 }).pipe(
841 Effect.tap((started) =>
842 started
843 ? Effect.logWarning(
844 "T3 Connect started the stored tunnel without relay confirmation",
845 )
846 : Effect.void,
847 ),
764 Effect.catch((cause) =>848 Effect.catch((cause) =>
765 Effect.logWarning("Failed to reconcile T3 Connect desired link on startup", {849 Effect.logWarning("Failed to start the stored T3 Connect tunnel", { cause }),
766 message: cause.message,
767 }),
768 ),850 ),
851 Effect.asVoid,
769 );852 );
853 const registerManagedTunnel = retryManagedTunnelRegistration(
854 registerManagedCloudTunnelRecovery(localOrigin, {
855 retryRuntimeFailures: true,
856 }),
857 (error) =>
858 shouldRetryCloudLink(error) &&
859 error._tag !== "EnvironmentCloudEndpointUnavailableError",
860 startedConfirmed ? Effect.void : startStoredManagedTunnel,
861 ).pipe(
862 Effect.tap((result) =>
863 result.status === "ready"
864 ? Effect.logInfo("T3 Connect managed tunnel recovery registered")
865 : Effect.void,
866 ),
867 Effect.catchCause((cause) =>
868 Cause.hasInterrupts(cause)
869 ? Effect.interrupt
870 : Effect.logWarning("Failed to register T3 Connect managed tunnel recovery", {
871 cause,
872 }).pipe(Effect.as({ status: "unavailable" as const })),
873 ),
874 );
875 // A host without a confirmed marker is on its first boot after the
876 // upgrade. Spread those registrations so an auto-update wave does
877 // not hit the relay all at once.
878 if (!startedConfirmed && desiredCliLinkMode !== "publish_only") {
879 const jitter = yield* Random.nextIntBetween(
880 0,
881 Duration.toMillis(MANAGED_TUNNEL_FIRST_REGISTRATION_JITTER),
882 );
883 yield* Effect.sleep(Duration.millis(jitter));
884 }
885 const registration =
886 desiredCliLinkMode === "publish_only"
887 ? { status: "not_linked" as const }
888 : yield* registerManagedTunnel;
889 // A terminal registration failure also allows the stored config
890 // to start. Transient outages use the fallback above and keep
891 // registration retrying in this scoped startup fiber.
892 if (registration.status === "unavailable" && !startedConfirmed) {
893 yield* startStoredManagedTunnel;
894 }
895 const startupAction = managedTunnelStartupAction({ wantsCliLink, registration });
896 if (startupAction.action === "request_recovery") {
897 yield* endpointRuntime.requestRecovery(startupAction.config);
898 }
899 if (startupAction.action === "reconcile_link") {
900 const reconciledMode = yield* reconcileDesiredCloudLinkIfStillDesired(
901 localOrigin,
902 ).pipe(
903 Effect.retry({
904 while: shouldRetryCloudLink,
905 schedule: Schedule.exponential("1 second").pipe(
906 Schedule.modifyDelay(({ duration }) =>
907 Effect.succeed(Duration.min(duration, Duration.seconds(30))),
908 ),
909 Schedule.upTo({ duration: "10 minutes" }),
910 ),
911 }),
912 Effect.tap((mode) =>
913 mode === null
914 ? Effect.void
915 : Effect.logInfo("T3 Connect desired link reconciled on startup"),
916 ),
917 Effect.catch((cause) =>
918 Effect.logWarning("Failed to reconcile T3 Connect desired link on startup", {
919 cause,
920 }).pipe(Effect.as(null)),
921 ),
922 );
923 if (reconciledMode === "managed") {
924 const afterReconcile = yield* registerManagedTunnel;
925 if (afterReconcile.status === "recovery_required") {
926 yield* endpointRuntime.requestRecovery(afterReconcile.config);
927 }
928 }
929 }
770 }),930 }),
771 );931 );
772 yield* Deferred.succeed(cloudLinkParked, undefined).pipe(Effect.orDie);932 yield* Deferred.succeed(cloudLinkParked, undefined).pipe(Effect.orDie);
docs/internals/t3-connect.md+41
...@@ -65,6 +65,47 @@ teardown, because a database failure must leave the active link usable. Failed...@@ -65,6 +65,47 @@ teardown, because a database failure must leave the active link usable. Failed
65teardown retains enough state to retry. See the65teardown retains enough state to retry. See the
66[managed endpoint lifecycle](../../infra/relay/src/environments/ManagedEndpointProvider.ts).66[managed endpoint lifecycle](../../infra/relay/src/environments/ManagedEndpointProvider.ts).
6767
68## Idle tunnels are reclaimed and recovered
69
70Cloudflare bills a tunnel whether or not a connector is attached, so a laptop
71that sleeps with a linked environment leaves a paid tunnel behind. The relay's
72five-minute maintenance job can reclaim those tunnels. `RELAY_TUNNEL_CLEANUP_MODE`
73selects `off`, `dry-run`, or `enabled`, with `off` as the default. The mode is
74read at deploy time, so changing it means a relay deploy, not a variable flip.
75A candidate is a same-stage tunnel that Cloudflare reports down for at least
76five minutes, or one that never connected and is at least an hour old. The
77longer grace for never-connected tunnels covers a pairing still in progress.
78
79Cleanup deletes only tunnels whose host has registered recovery. Allocations
80without recovery registration belong to hosts that cannot replace a deleted
81tunnel and are left alone. Allocations with no recorded tunnel ID, or a
82different tunnel ID, are skipped because a provision may own them. A tunnel with
83no allocation row at all is counted as `skippedOrphan` and never deleted: there
84is no row to lock, so a relink that adopts it by name could race the delete.
85Clear those by hand. Each sweep is bounded: at most ten list requests, 100 deletions, a
86two-minute deadline, and an early stop on a Cloudflare rate limit. Each sweep
87starts one budget further along the candidate list, so a block of deletes that
88keep failing cannot starve the tunnels listed after them. See the
89[reaper](../../infra/relay/src/environments/ManagedEndpointReaper.ts).
90
91A host registers recovery at startup by sending its tunnel ID and loopback
92origin with a short-lived signature from the environment key. Registration
93touches Cloudflare only when the local host or port changed, and once per
94existing allocation on the first registration after the upgrade because the
95stored origin is empty. First registrations are jittered so an auto-update wave
96does not hit the relay at once. The host stores a confirmed-origin marker with
97the connector config, and a later boot starts the connector before registration
98only when that marker matches the current config and port. If registration
99cannot reach the relay for ten minutes, the host starts its stored config anyway
100and keeps registering in the background until it can reconcile the origin.
101If the connector exits, or `cloudflared` reports repeated tunnel
102rejections, the host asks the relay for a replacement, at most once every two
103minutes. The relay
104provisions under the same allocation, so the hostname and DNS record survive
105and clients keep their bindings. Every mutation on an allocation bumps its
106`generation`, and deletion locks the row at the generation it claimed, so a
107host that reconnects mid-sweep wins.
108
68## OAuth traps109## OAuth traps
69110
70Interactive clients and the headless CLI use the same Clerk application but111Interactive clients and the headless CLI use the same Clerk application but
docs/operations/release.md+51
...@@ -132,6 +132,8 @@ Required `production` environment variables:...@@ -132,6 +132,8 @@ Required `production` environment variables:
132Optional `production` environment variables:132Optional `production` environment variables:
133133
134- `RELAY_DOMAIN` when overriding the derived `relay.<RELAY_API_ZONE_NAME>` domain134- `RELAY_DOMAIN` when overriding the derived `relay.<RELAY_API_ZONE_NAME>` domain
135- `RELAY_TUNNEL_CLEANUP_MODE` with `off`, `dry-run`, or `enabled`. Missing and blank values use
136 `off`.
135137
136Required `production` environment secrets:138Required `production` environment secrets:
137139
...@@ -151,6 +153,55 @@ Developers deploy personal stages locally rather than through pull-request autom...@@ -151,6 +153,55 @@ Developers deploy personal stages locally rather than through pull-request autom
151vp run --filter t3code-relay deploy -- --stage "$USER" --env-file .env.local153vp run --filter t3code-relay deploy -- --stage "$USER" --env-file .env.local
152```154```
153155
156### Managed tunnel cleanup rollout
157
158Keep `RELAY_TUNNEL_CLEANUP_MODE=off` for the first production deploy. That deploy applies the
159nullable allocation migration and adds the recovery endpoints. Web and mobile clients need no
160coordinated release. CLI and desktop server builds must reach users before cleanup is enabled,
161because those builds register recovery and replace a deleted tunnel after wake.
162
1631. Deploy the relay and migration with cleanup `off`.
1642. Release the server build and confirm current hosts register recovery. Older hosts stay marked
165 legacy and are never candidates.
1663. Set `dry-run`, deploy, and read the sweep counters (`scanned`, `wouldDelete`, `skippedLegacy`,
167 `skippedOrphan`, `failed`, `truncated`) across several sweeps.
1684. Run the disposable-host canary below.
1695. Set `enabled` only after the canary recovers without a server restart.
170
171The job runs every five minutes with a five-minute grace period for tunnels that lost their
172connector, so a candidate is usually removed five to ten minutes after it goes down. Tunnels that
173never connected wait an hour. One sweep attempts at most 100 deletions, so a backlog takes longer.
174`RELAY_TUNNEL_CLEANUP_MODE` is read at deploy time. Changing it, including turning cleanup off during
175an incident, needs a relay deploy.
176
177To roll back, set cleanup to `off` and deploy the relay before downgrading any host. Keep the
178recovery endpoints deployed while current server builds are in use. The nullable columns can stay.
179
180### Disposable-host canary
181
182This test has not been run against a real Cloudflare account. Run it against a disposable relay
183stage, test Cloudflare account, disposable host, and disposable T3 home. Keep production cleanup at
184`off` or `dry-run` until it passes. Do not stop a daily-use T3 server.
185
1861. Deploy the disposable stage with cleanup `dry-run`. Link a first disposable environment through
187 web or mobile settings and confirm its tunnel is healthy and recovery is registered.
1882. Stop that host and restart the same T3 home on a different local port. Confirm the public
189 hostname reaches the new port and sends nothing to the old one.
1903. Link a second disposable environment with a server build that predates recovery registration.
191 Capture its managed `cloudflared` child PID, confirm it belongs to that host, and pause only that
192 child with `kill -STOP <legacy-pid>`. Wait until Cloudflare reports it down for over five minutes.
1934. Capture the first environment's `cloudflared` child PID from its server logs, confirm ownership,
194 and pause it with `kill -STOP <first-pid>`. Wait until Cloudflare reports it down for over five
195 minutes.
1965. Confirm dry-run counts the first tunnel in `wouldDelete` and the second in `skippedLegacy`.
1976. Set cleanup `enabled` on the disposable stage and deploy. Confirm in the test Cloudflare account
198 that the first tunnel is deleted and the legacy tunnel still exists.
1997. Resume the first child with `kill -CONT <first-pid>`. Confirm the running server detects the
200 repeated rejection, requests recovery, and becomes reachable at the same hostname without a
201 restart.
2028. Resume the legacy child with `kill -CONT <legacy-pid>` and confirm its tunnel reconnects.
2039. Repeat with a physical sleep and wake cycle on a disposable laptop before broad rollout.
204
154## Marketing site deployment205## Marketing site deployment
155206
156After a nightly release is published, the release workflow deploys the same commit207After a nightly release is published, the release workflow deploys the same commit
docs/user/remote-access.md+6
...@@ -158,6 +158,12 @@ page, or **Settings → T3 Connect** on mobile, and choose **Deregister**. This...@@ -158,6 +158,12 @@ page, or **Settings → T3 Connect** on mobile, and choose **Deregister**. This
158revokes its cloud access and frees its host space even when the environment is158revokes its cloud access and frees its host space even when the environment is
159offline or has been wiped.159offline or has been wiped.
160160
161When idle tunnel cleanup is enabled, T3 Connect removes a linked environment's
162tunnel after it stays offline for several minutes. The environment stays linked
163and keeps the same address. When the host starts again or wakes, T3 Connect
164creates a replacement tunnel on its own. You do not need to pair again. Cleanup
165usually runs five to ten minutes after the tunnel goes down.
166
161On a command-line host, `t3 connect unlink` disables exposure while retaining167On a command-line host, `t3 connect unlink` disables exposure while retaining
162your login; `t3 connect logout` also clears that login. Background-service168your login; `t3 connect logout` also clears that login. Background-service
163[removal](./background-service.md#manage-the-service) is separate.169[removal](./background-service.md#manage-the-service) is separate.
infra/relay/.env.example+4
...@@ -5,6 +5,10 @@...@@ -5,6 +5,10 @@
5RELAY_API_ZONE_NAME=example.com5RELAY_API_ZONE_NAME=example.com
6RELAY_TUNNEL_ZONE_NAME=tunnels.example.com6RELAY_TUNNEL_ZONE_NAME=tunnels.example.com
77
8# Optional: inactive tunnel cleanup. Start with dry-run, verify the cleanup
9# logs, then set enabled. Unset and off both disable cleanup.
10# RELAY_TUNNEL_CLEANUP_MODE=off
11
8# Optional: Relay domain override12# Optional: Relay domain override
9# Set this only when the derived relay hostname should not be used.13# Set this only when the derived relay hostname should not be used.
10# RELAY_DOMAIN=relay.example.com14# RELAY_DOMAIN=relay.example.com
infra/relay/migrations/postgres/20260919015455_managed_endpoint_recovery/migration.sql created+4
...@@ -0,0 +1,4 @@
1ALTER TABLE "relay_managed_endpoint_allocations" ADD COLUMN "recovery_enabled_at" varchar(64);--> statement-breakpoint
2ALTER TABLE "relay_managed_endpoint_allocations" ADD COLUMN "recovery_environment_public_key" text;--> statement-breakpoint
3ALTER TABLE "relay_managed_endpoint_allocations" ADD COLUMN "origin" jsonb;--> statement-breakpoint
4ALTER TABLE "relay_managed_endpoint_allocations" ADD COLUMN "generation" integer DEFAULT 0 NOT NULL;
infra/relay/migrations/postgres/20260919015455_managed_endpoint_recovery/snapshot.json created+1568
...@@ -0,0 +1,1568 @@
1{
2 "dialect": "postgres",
3 "id": "3809c51e-3821-4a08-818d-e5d28dd3e9b4",
4 "prevIds": ["4b6d0d21-8d78-4499-9dde-b42bcf633d05"],
5 "version": "8",
6 "ddl": [
7 {
8 "isRlsEnabled": false,
9 "name": "relay_agent_activity_rows",
10 "entityType": "tables",
11 "schema": "public"
12 },
13 {
14 "isRlsEnabled": false,
15 "name": "relay_delivery_attempts",
16 "entityType": "tables",
17 "schema": "public"
18 },
19 {
20 "isRlsEnabled": false,
21 "name": "relay_dpop_proofs",
22 "entityType": "tables",
23 "schema": "public"
24 },
25 {
26 "isRlsEnabled": false,
27 "name": "relay_environment_credentials",
28 "entityType": "tables",
29 "schema": "public"
30 },
31 {
32 "isRlsEnabled": false,
33 "name": "relay_environment_links",
34 "entityType": "tables",
35 "schema": "public"
36 },
37 {
38 "isRlsEnabled": false,
39 "name": "relay_live_activities",
40 "entityType": "tables",
41 "schema": "public"
42 },
43 {
44 "isRlsEnabled": false,
45 "name": "relay_managed_endpoint_allocations",
46 "entityType": "tables",
47 "schema": "public"
48 },
49 {
50 "isRlsEnabled": false,
51 "name": "relay_managed_tunnel_limits",
52 "entityType": "tables",
53 "schema": "public"
54 },
55 {
56 "isRlsEnabled": false,
57 "name": "relay_mobile_devices",
58 "entityType": "tables",
59 "schema": "public"
60 },
61 {
62 "type": "varchar(191)",
63 "typeSchema": null,
64 "notNull": true,
65 "dimensions": 0,
66 "default": null,
67 "generated": null,
68 "identity": null,
69 "name": "environment_id",
70 "entityType": "columns",
71 "schema": "public",
72 "table": "relay_agent_activity_rows"
73 },
74 {
75 "type": "text",
76 "typeSchema": null,
77 "notNull": true,
78 "dimensions": 0,
79 "default": null,
80 "generated": null,
81 "identity": null,
82 "name": "environment_public_key",
83 "entityType": "columns",
84 "schema": "public",
85 "table": "relay_agent_activity_rows"
86 },
87 {
88 "type": "varchar(512)",
89 "typeSchema": null,
90 "notNull": true,
91 "dimensions": 0,
92 "default": null,
93 "generated": null,
94 "identity": null,
95 "name": "thread_id",
96 "entityType": "columns",
97 "schema": "public",
98 "table": "relay_agent_activity_rows"
99 },
100 {
101 "type": "jsonb",
102 "typeSchema": null,
103 "notNull": true,
104 "dimensions": 0,
105 "default": null,
106 "generated": null,
107 "identity": null,
108 "name": "state_json",
109 "entityType": "columns",
110 "schema": "public",
111 "table": "relay_agent_activity_rows"
112 },
113 {
114 "type": "varchar(64)",
115 "typeSchema": null,
116 "notNull": true,
117 "dimensions": 0,
118 "default": null,
119 "generated": null,
120 "identity": null,
121 "name": "updated_at",
122 "entityType": "columns",
123 "schema": "public",
124 "table": "relay_agent_activity_rows"
125 },
126 {
127 "type": "varchar(64)",
128 "typeSchema": null,
129 "notNull": true,
130 "dimensions": 0,
131 "default": null,
132 "generated": null,
133 "identity": null,
134 "name": "created_at",
135 "entityType": "columns",
136 "schema": "public",
137 "table": "relay_agent_activity_rows"
138 },
139 {
140 "type": "varchar(36)",
141 "typeSchema": null,
142 "notNull": true,
143 "dimensions": 0,
144 "default": null,
145 "generated": null,
146 "identity": null,
147 "name": "id",
148 "entityType": "columns",
149 "schema": "public",
150 "table": "relay_delivery_attempts"
151 },
152 {
153 "type": "varchar(64)",
154 "typeSchema": null,
155 "notNull": true,
156 "dimensions": 0,
157 "default": null,
158 "generated": null,
159 "identity": null,
160 "name": "created_at",
161 "entityType": "columns",
162 "schema": "public",
163 "table": "relay_delivery_attempts"
164 },
165 {
166 "type": "varchar(255)",
167 "typeSchema": null,
168 "notNull": false,
169 "dimensions": 0,
170 "default": null,
171 "generated": null,
172 "identity": null,
173 "name": "user_id",
174 "entityType": "columns",
175 "schema": "public",
176 "table": "relay_delivery_attempts"
177 },
178 {
179 "type": "varchar(191)",
180 "typeSchema": null,
181 "notNull": false,
182 "dimensions": 0,
183 "default": null,
184 "generated": null,
185 "identity": null,
186 "name": "environment_id",
187 "entityType": "columns",
188 "schema": "public",
189 "table": "relay_delivery_attempts"
190 },
191 {
192 "type": "varchar(512)",
193 "typeSchema": null,
194 "notNull": false,
195 "dimensions": 0,
196 "default": null,
197 "generated": null,
198 "identity": null,
199 "name": "thread_id",
200 "entityType": "columns",
201 "schema": "public",
202 "table": "relay_delivery_attempts"
203 },
204 {
205 "type": "varchar(255)",
206 "typeSchema": null,
207 "notNull": false,
208 "dimensions": 0,
209 "default": null,
210 "generated": null,
211 "identity": null,
212 "name": "device_id",
213 "entityType": "columns",
214 "schema": "public",
215 "table": "relay_delivery_attempts"
216 },
217 {
218 "type": "varchar(64)",
219 "typeSchema": null,
220 "notNull": true,
221 "dimensions": 0,
222 "default": null,
223 "generated": null,
224 "identity": null,
225 "name": "kind",
226 "entityType": "columns",
227 "schema": "public",
228 "table": "relay_delivery_attempts"
229 },
230 {
231 "type": "varchar(64)",
232 "typeSchema": null,
233 "notNull": false,
234 "dimensions": 0,
235 "default": null,
236 "generated": null,
237 "identity": null,
238 "name": "source_job_id",
239 "entityType": "columns",
240 "schema": "public",
241 "table": "relay_delivery_attempts"
242 },
243 {
244 "type": "varchar(16)",
245 "typeSchema": null,
246 "notNull": false,
247 "dimensions": 0,
248 "default": null,
249 "generated": null,
250 "identity": null,
251 "name": "token_suffix",
252 "entityType": "columns",
253 "schema": "public",
254 "table": "relay_delivery_attempts"
255 },
256 {
257 "type": "integer",
258 "typeSchema": null,
259 "notNull": false,
260 "dimensions": 0,
261 "default": null,
262 "generated": null,
263 "identity": null,
264 "name": "apns_status",
265 "entityType": "columns",
266 "schema": "public",
267 "table": "relay_delivery_attempts"
268 },
269 {
270 "type": "text",
271 "typeSchema": null,
272 "notNull": false,
273 "dimensions": 0,
274 "default": null,
275 "generated": null,
276 "identity": null,
277 "name": "apns_reason",
278 "entityType": "columns",
279 "schema": "public",
280 "table": "relay_delivery_attempts"
281 },
282 {
283 "type": "varchar(128)",
284 "typeSchema": null,
285 "notNull": false,
286 "dimensions": 0,
287 "default": null,
288 "generated": null,
289 "identity": null,
290 "name": "apns_id",
291 "entityType": "columns",
292 "schema": "public",
293 "table": "relay_delivery_attempts"
294 },
295 {
296 "type": "text",
297 "typeSchema": null,
298 "notNull": false,
299 "dimensions": 0,
300 "default": null,
301 "generated": null,
302 "identity": null,
303 "name": "transport_error",
304 "entityType": "columns",
305 "schema": "public",
306 "table": "relay_delivery_attempts"
307 },
308 {
309 "type": "varchar(128)",
310 "typeSchema": null,
311 "notNull": true,
312 "dimensions": 0,
313 "default": null,
314 "generated": null,
315 "identity": null,
316 "name": "thumbprint",
317 "entityType": "columns",
318 "schema": "public",
319 "table": "relay_dpop_proofs"
320 },
321 {
322 "type": "varchar(255)",
323 "typeSchema": null,
324 "notNull": true,
325 "dimensions": 0,
326 "default": null,
327 "generated": null,
328 "identity": null,
329 "name": "jti",
330 "entityType": "columns",
331 "schema": "public",
332 "table": "relay_dpop_proofs"
333 },
334 {
335 "type": "integer",
336 "typeSchema": null,
337 "notNull": true,
338 "dimensions": 0,
339 "default": null,
340 "generated": null,
341 "identity": null,
342 "name": "iat",
343 "entityType": "columns",
344 "schema": "public",
345 "table": "relay_dpop_proofs"
346 },
347 {
348 "type": "varchar(64)",
349 "typeSchema": null,
350 "notNull": true,
351 "dimensions": 0,
352 "default": null,
353 "generated": null,
354 "identity": null,
355 "name": "expires_at",
356 "entityType": "columns",
357 "schema": "public",
358 "table": "relay_dpop_proofs"
359 },
360 {
361 "type": "varchar(64)",
362 "typeSchema": null,
363 "notNull": true,
364 "dimensions": 0,
365 "default": null,
366 "generated": null,
367 "identity": null,
368 "name": "created_at",
369 "entityType": "columns",
370 "schema": "public",
371 "table": "relay_dpop_proofs"
372 },
373 {
374 "type": "varchar(64)",
375 "typeSchema": null,
376 "notNull": true,
377 "dimensions": 0,
378 "default": null,
379 "generated": null,
380 "identity": null,
381 "name": "credential_id",
382 "entityType": "columns",
383 "schema": "public",
384 "table": "relay_environment_credentials"
385 },
386 {
387 "type": "varchar(191)",
388 "typeSchema": null,
389 "notNull": true,
390 "dimensions": 0,
391 "default": null,
392 "generated": null,
393 "identity": null,
394 "name": "environment_id",
395 "entityType": "columns",
396 "schema": "public",
397 "table": "relay_environment_credentials"
398 },
399 {
400 "type": "text",
401 "typeSchema": null,
402 "notNull": true,
403 "dimensions": 0,
404 "default": null,
405 "generated": null,
406 "identity": null,
407 "name": "environment_public_key",
408 "entityType": "columns",
409 "schema": "public",
410 "table": "relay_environment_credentials"
411 },
412 {
413 "type": "varchar(191)",
414 "typeSchema": null,
415 "notNull": true,
416 "dimensions": 0,
417 "default": null,
418 "generated": null,
419 "identity": null,
420 "name": "credential_hash",
421 "entityType": "columns",
422 "schema": "public",
423 "table": "relay_environment_credentials"
424 },
425 {
426 "type": "varchar(64)",
427 "typeSchema": null,
428 "notNull": false,
429 "dimensions": 0,
430 "default": null,
431 "generated": null,
432 "identity": null,
433 "name": "revoked_at",
434 "entityType": "columns",
435 "schema": "public",
436 "table": "relay_environment_credentials"
437 },
438 {
439 "type": "varchar(64)",
440 "typeSchema": null,
441 "notNull": true,
442 "dimensions": 0,
443 "default": null,
444 "generated": null,
445 "identity": null,
446 "name": "created_at",
447 "entityType": "columns",
448 "schema": "public",
449 "table": "relay_environment_credentials"
450 },
451 {
452 "type": "varchar(64)",
453 "typeSchema": null,
454 "notNull": true,
455 "dimensions": 0,
456 "default": null,
457 "generated": null,
458 "identity": null,
459 "name": "updated_at",
460 "entityType": "columns",
461 "schema": "public",
462 "table": "relay_environment_credentials"
463 },
464 {
465 "type": "varchar(191)",
466 "typeSchema": null,
467 "notNull": true,
468 "dimensions": 0,
469 "default": null,
470 "generated": null,
471 "identity": null,
472 "name": "user_id",
473 "entityType": "columns",
474 "schema": "public",
475 "table": "relay_environment_links"
476 },
477 {
478 "type": "varchar(191)",
479 "typeSchema": null,
480 "notNull": true,
481 "dimensions": 0,
482 "default": null,
483 "generated": null,
484 "identity": null,
485 "name": "environment_id",
486 "entityType": "columns",
487 "schema": "public",
488 "table": "relay_environment_links"
489 },
490 {
491 "type": "text",
492 "typeSchema": null,
493 "notNull": true,
494 "dimensions": 0,
495 "default": "'T3 Environment'",
496 "generated": null,
497 "identity": null,
498 "name": "environment_label",
499 "entityType": "columns",
500 "schema": "public",
501 "table": "relay_environment_links"
502 },
503 {
504 "type": "text",
505 "typeSchema": null,
506 "notNull": true,
507 "dimensions": 0,
508 "default": null,
509 "generated": null,
510 "identity": null,
511 "name": "environment_public_key",
512 "entityType": "columns",
513 "schema": "public",
514 "table": "relay_environment_links"
515 },
516 {
517 "type": "text",
518 "typeSchema": null,
519 "notNull": true,
520 "dimensions": 0,
521 "default": null,
522 "generated": null,
523 "identity": null,
524 "name": "endpoint_http_base_url",
525 "entityType": "columns",
526 "schema": "public",
527 "table": "relay_environment_links"
528 },
529 {
530 "type": "text",
531 "typeSchema": null,
532 "notNull": true,
533 "dimensions": 0,
534 "default": null,
535 "generated": null,
536 "identity": null,
537 "name": "endpoint_ws_base_url",
538 "entityType": "columns",
539 "schema": "public",
540 "table": "relay_environment_links"
541 },
542 {
543 "type": "varchar(32)",
544 "typeSchema": null,
545 "notNull": true,
546 "dimensions": 0,
547 "default": null,
548 "generated": null,
549 "identity": null,
550 "name": "endpoint_provider_kind",
551 "entityType": "columns",
552 "schema": "public",
553 "table": "relay_environment_links"
554 },
555 {
556 "type": "boolean",
557 "typeSchema": null,
558 "notNull": true,
559 "dimensions": 0,
560 "default": "true",
561 "generated": null,
562 "identity": null,
563 "name": "notifications_enabled",
564 "entityType": "columns",
565 "schema": "public",
566 "table": "relay_environment_links"
567 },
568 {
569 "type": "boolean",
570 "typeSchema": null,
571 "notNull": true,
572 "dimensions": 0,
573 "default": "true",
574 "generated": null,
575 "identity": null,
576 "name": "live_activities_enabled",
577 "entityType": "columns",
578 "schema": "public",
579 "table": "relay_environment_links"
580 },
581 {
582 "type": "boolean",
583 "typeSchema": null,
584 "notNull": true,
585 "dimensions": 0,
586 "default": "false",
587 "generated": null,
588 "identity": null,
589 "name": "managed_tunnels_enabled",
590 "entityType": "columns",
591 "schema": "public",
592 "table": "relay_environment_links"
593 },
594 {
595 "type": "varchar(191)",
596 "typeSchema": null,
597 "notNull": false,
598 "dimensions": 0,
599 "default": null,
600 "generated": null,
601 "identity": null,
602 "name": "created_by_device_id",
603 "entityType": "columns",
604 "schema": "public",
605 "table": "relay_environment_links"
606 },
607 {
608 "type": "varchar(64)",
609 "typeSchema": null,
610 "notNull": false,
611 "dimensions": 0,
612 "default": null,
613 "generated": null,
614 "identity": null,
615 "name": "revoked_at",
616 "entityType": "columns",
617 "schema": "public",
618 "table": "relay_environment_links"
619 },
620 {
621 "type": "varchar(64)",
622 "typeSchema": null,
623 "notNull": true,
624 "dimensions": 0,
625 "default": null,
626 "generated": null,
627 "identity": null,
628 "name": "created_at",
629 "entityType": "columns",
630 "schema": "public",
631 "table": "relay_environment_links"
632 },
633 {
634 "type": "varchar(64)",
635 "typeSchema": null,
636 "notNull": true,
637 "dimensions": 0,
638 "default": null,
639 "generated": null,
640 "identity": null,
641 "name": "updated_at",
642 "entityType": "columns",
643 "schema": "public",
644 "table": "relay_environment_links"
645 },
646 {
647 "type": "varchar(255)",
648 "typeSchema": null,
649 "notNull": true,
650 "dimensions": 0,
651 "default": null,
652 "generated": null,
653 "identity": null,
654 "name": "user_id",
655 "entityType": "columns",
656 "schema": "public",
657 "table": "relay_live_activities"
658 },
659 {
660 "type": "varchar(255)",
661 "typeSchema": null,
662 "notNull": true,
663 "dimensions": 0,
664 "default": null,
665 "generated": null,
666 "identity": null,
667 "name": "device_id",
668 "entityType": "columns",
669 "schema": "public",
670 "table": "relay_live_activities"
671 },
672 {
673 "type": "text",
674 "typeSchema": null,
675 "notNull": false,
676 "dimensions": 0,
677 "default": null,
678 "generated": null,
679 "identity": null,
680 "name": "activity_push_token",
681 "entityType": "columns",
682 "schema": "public",
683 "table": "relay_live_activities"
684 },
685 {
686 "type": "varchar(64)",
687 "typeSchema": null,
688 "notNull": false,
689 "dimensions": 0,
690 "default": null,
691 "generated": null,
692 "identity": null,
693 "name": "remote_start_queued_at",
694 "entityType": "columns",
695 "schema": "public",
696 "table": "relay_live_activities"
697 },
698 {
699 "type": "varchar(64)",
700 "typeSchema": null,
701 "notNull": false,
702 "dimensions": 0,
703 "default": null,
704 "generated": null,
705 "identity": null,
706 "name": "remote_started_at",
707 "entityType": "columns",
708 "schema": "public",
709 "table": "relay_live_activities"
710 },
711 {
712 "type": "varchar(64)",
713 "typeSchema": null,
714 "notNull": false,
715 "dimensions": 0,
716 "default": null,
717 "generated": null,
718 "identity": null,
719 "name": "ended_at",
720 "entityType": "columns",
721 "schema": "public",
722 "table": "relay_live_activities"
723 },
724 {
725 "type": "jsonb",
726 "typeSchema": null,
727 "notNull": false,
728 "dimensions": 0,
729 "default": null,
730 "generated": null,
731 "identity": null,
732 "name": "last_aggregate_json",
733 "entityType": "columns",
734 "schema": "public",
735 "table": "relay_live_activities"
736 },
737 {
738 "type": "varchar(64)",
739 "typeSchema": null,
740 "notNull": false,
741 "dimensions": 0,
742 "default": null,
743 "generated": null,
744 "identity": null,
745 "name": "last_live_activity_delivery_at",
746 "entityType": "columns",
747 "schema": "public",
748 "table": "relay_live_activities"
749 },
750 {
751 "type": "varchar(64)",
752 "typeSchema": null,
753 "notNull": true,
754 "dimensions": 0,
755 "default": null,
756 "generated": null,
757 "identity": null,
758 "name": "created_at",
759 "entityType": "columns",
760 "schema": "public",
761 "table": "relay_live_activities"
762 },
763 {
764 "type": "varchar(64)",
765 "typeSchema": null,
766 "notNull": true,
767 "dimensions": 0,
768 "default": null,
769 "generated": null,
770 "identity": null,
771 "name": "updated_at",
772 "entityType": "columns",
773 "schema": "public",
774 "table": "relay_live_activities"
775 },
776 {
777 "type": "varchar(191)",
778 "typeSchema": null,
779 "notNull": true,
780 "dimensions": 0,
781 "default": null,
782 "generated": null,
783 "identity": null,
784 "name": "user_id",
785 "entityType": "columns",
786 "schema": "public",
787 "table": "relay_managed_endpoint_allocations"
788 },
789 {
790 "type": "varchar(191)",
791 "typeSchema": null,
792 "notNull": true,
793 "dimensions": 0,
794 "default": null,
795 "generated": null,
796 "identity": null,
797 "name": "environment_id",
798 "entityType": "columns",
799 "schema": "public",
800 "table": "relay_managed_endpoint_allocations"
801 },
802 {
803 "type": "text",
804 "typeSchema": null,
805 "notNull": true,
806 "dimensions": 0,
807 "default": null,
808 "generated": null,
809 "identity": null,
810 "name": "hostname",
811 "entityType": "columns",
812 "schema": "public",
813 "table": "relay_managed_endpoint_allocations"
814 },
815 {
816 "type": "varchar(191)",
817 "typeSchema": null,
818 "notNull": false,
819 "dimensions": 0,
820 "default": null,
821 "generated": null,
822 "identity": null,
823 "name": "tunnel_id",
824 "entityType": "columns",
825 "schema": "public",
826 "table": "relay_managed_endpoint_allocations"
827 },
828 {
829 "type": "text",
830 "typeSchema": null,
831 "notNull": true,
832 "dimensions": 0,
833 "default": null,
834 "generated": null,
835 "identity": null,
836 "name": "tunnel_name",
837 "entityType": "columns",
838 "schema": "public",
839 "table": "relay_managed_endpoint_allocations"
840 },
841 {
842 "type": "varchar(191)",
843 "typeSchema": null,
844 "notNull": false,
845 "dimensions": 0,
846 "default": null,
847 "generated": null,
848 "identity": null,
849 "name": "dns_record_id",
850 "entityType": "columns",
851 "schema": "public",
852 "table": "relay_managed_endpoint_allocations"
853 },
854 {
855 "type": "varchar(64)",
856 "typeSchema": null,
857 "notNull": false,
858 "dimensions": 0,
859 "default": null,
860 "generated": null,
861 "identity": null,
862 "name": "ready_at",
863 "entityType": "columns",
864 "schema": "public",
865 "table": "relay_managed_endpoint_allocations"
866 },
867 {
868 "type": "varchar(64)",
869 "typeSchema": null,
870 "notNull": false,
871 "dimensions": 0,
872 "default": null,
873 "generated": null,
874 "identity": null,
875 "name": "recovery_enabled_at",
876 "entityType": "columns",
877 "schema": "public",
878 "table": "relay_managed_endpoint_allocations"
879 },
880 {
881 "type": "text",
882 "typeSchema": null,
883 "notNull": false,
884 "dimensions": 0,
885 "default": null,
886 "generated": null,
887 "identity": null,
888 "name": "recovery_environment_public_key",
889 "entityType": "columns",
890 "schema": "public",
891 "table": "relay_managed_endpoint_allocations"
892 },
893 {
894 "type": "jsonb",
895 "typeSchema": null,
896 "notNull": false,
897 "dimensions": 0,
898 "default": null,
899 "generated": null,
900 "identity": null,
901 "name": "origin",
902 "entityType": "columns",
903 "schema": "public",
904 "table": "relay_managed_endpoint_allocations"
905 },
906 {
907 "type": "integer",
908 "typeSchema": null,
909 "notNull": true,
910 "dimensions": 0,
911 "default": "0",
912 "generated": null,
913 "identity": null,
914 "name": "generation",
915 "entityType": "columns",
916 "schema": "public",
917 "table": "relay_managed_endpoint_allocations"
918 },
919 {
920 "type": "varchar(64)",
921 "typeSchema": null,
922 "notNull": true,
923 "dimensions": 0,
924 "default": null,
925 "generated": null,
926 "identity": null,
927 "name": "created_at",
928 "entityType": "columns",
929 "schema": "public",
930 "table": "relay_managed_endpoint_allocations"
931 },
932 {
933 "type": "varchar(64)",
934 "typeSchema": null,
935 "notNull": true,
936 "dimensions": 0,
937 "default": null,
938 "generated": null,
939 "identity": null,
940 "name": "updated_at",
941 "entityType": "columns",
942 "schema": "public",
943 "table": "relay_managed_endpoint_allocations"
944 },
945 {
946 "type": "varchar(191)",
947 "typeSchema": null,
948 "notNull": true,
949 "dimensions": 0,
950 "default": null,
951 "generated": null,
952 "identity": null,
953 "name": "user_id",
954 "entityType": "columns",
955 "schema": "public",
956 "table": "relay_managed_tunnel_limits"
957 },
958 {
959 "type": "integer",
960 "typeSchema": null,
961 "notNull": true,
962 "dimensions": 0,
963 "default": null,
964 "generated": null,
965 "identity": null,
966 "name": "max_tunnels",
967 "entityType": "columns",
968 "schema": "public",
969 "table": "relay_managed_tunnel_limits"
970 },
971 {
972 "type": "varchar(64)",
973 "typeSchema": null,
974 "notNull": true,
975 "dimensions": 0,
976 "default": null,
977 "generated": null,
978 "identity": null,
979 "name": "created_at",
980 "entityType": "columns",
981 "schema": "public",
982 "table": "relay_managed_tunnel_limits"
983 },
984 {
985 "type": "varchar(64)",
986 "typeSchema": null,
987 "notNull": true,
988 "dimensions": 0,
989 "default": null,
990 "generated": null,
991 "identity": null,
992 "name": "updated_at",
993 "entityType": "columns",
994 "schema": "public",
995 "table": "relay_managed_tunnel_limits"
996 },
997 {
998 "type": "varchar(255)",
999 "typeSchema": null,
1000 "notNull": true,
1001 "dimensions": 0,
1002 "default": null,
1003 "generated": null,
1004 "identity": null,
1005 "name": "user_id",
1006 "entityType": "columns",
1007 "schema": "public",
1008 "table": "relay_mobile_devices"
1009 },
1010 {
1011 "type": "varchar(255)",
1012 "typeSchema": null,
1013 "notNull": true,
1014 "dimensions": 0,
1015 "default": null,
1016 "generated": null,
1017 "identity": null,
1018 "name": "device_id",
1019 "entityType": "columns",
1020 "schema": "public",
1021 "table": "relay_mobile_devices"
1022 },
1023 {
1024 "type": "text",
1025 "typeSchema": null,
1026 "notNull": true,
1027 "dimensions": 0,
1028 "default": "'iOS device'",
1029 "generated": null,
1030 "identity": null,
1031 "name": "label",
1032 "entityType": "columns",
1033 "schema": "public",
1034 "table": "relay_mobile_devices"
1035 },
1036 {
1037 "type": "varchar(16)",
1038 "typeSchema": null,
1039 "notNull": true,
1040 "dimensions": 0,
1041 "default": null,
1042 "generated": null,
1043 "identity": null,
1044 "name": "platform",
1045 "entityType": "columns",
1046 "schema": "public",
1047 "table": "relay_mobile_devices"
1048 },
1049 {
1050 "type": "integer",
1051 "typeSchema": null,
1052 "notNull": false,
1053 "dimensions": 0,
1054 "default": null,
1055 "generated": null,
1056 "identity": null,
1057 "name": "ios_major_version",
1058 "entityType": "columns",
1059 "schema": "public",
1060 "table": "relay_mobile_devices"
1061 },
1062 {
1063 "type": "integer",
1064 "typeSchema": null,
1065 "notNull": false,
1066 "dimensions": 0,
1067 "default": null,
1068 "generated": null,
1069 "identity": null,
1070 "name": "android_api_level",
1071 "entityType": "columns",
1072 "schema": "public",
1073 "table": "relay_mobile_devices"
1074 },
1075 {
1076 "type": "varchar(64)",
1077 "typeSchema": null,
1078 "notNull": false,
1079 "dimensions": 0,
1080 "default": null,
1081 "generated": null,
1082 "identity": null,
1083 "name": "app_version",
1084 "entityType": "columns",
1085 "schema": "public",
1086 "table": "relay_mobile_devices"
1087 },
1088 {
1089 "type": "varchar(255)",
1090 "typeSchema": null,
1091 "notNull": false,
1092 "dimensions": 0,
1093 "default": null,
1094 "generated": null,
1095 "identity": null,
1096 "name": "bundle_id",
1097 "entityType": "columns",
1098 "schema": "public",
1099 "table": "relay_mobile_devices"
1100 },
1101 {
1102 "type": "varchar(16)",
1103 "typeSchema": null,
1104 "notNull": false,
1105 "dimensions": 0,
1106 "default": null,
1107 "generated": null,
1108 "identity": null,
1109 "name": "aps_environment",
1110 "entityType": "columns",
1111 "schema": "public",
1112 "table": "relay_mobile_devices"
1113 },
1114 {
1115 "type": "text",
1116 "typeSchema": null,
1117 "notNull": false,
1118 "dimensions": 0,
1119 "default": null,
1120 "generated": null,
1121 "identity": null,
1122 "name": "push_token",
1123 "entityType": "columns",
1124 "schema": "public",
1125 "table": "relay_mobile_devices"
1126 },
1127 {
1128 "type": "text",
1129 "typeSchema": null,
1130 "notNull": false,
1131 "dimensions": 0,
1132 "default": null,
1133 "generated": null,
1134 "identity": null,
1135 "name": "push_to_start_token",
1136 "entityType": "columns",
1137 "schema": "public",
1138 "table": "relay_mobile_devices"
1139 },
1140 {
1141 "type": "jsonb",
1142 "typeSchema": null,
1143 "notNull": true,
1144 "dimensions": 0,
1145 "default": null,
1146 "generated": null,
1147 "identity": null,
1148 "name": "preferences_json",
1149 "entityType": "columns",
1150 "schema": "public",
1151 "table": "relay_mobile_devices"
1152 },
1153 {
1154 "type": "varchar(64)",
1155 "typeSchema": null,
1156 "notNull": true,
1157 "dimensions": 0,
1158 "default": null,
1159 "generated": null,
1160 "identity": null,
1161 "name": "created_at",
1162 "entityType": "columns",
1163 "schema": "public",
1164 "table": "relay_mobile_devices"
1165 },
1166 {
1167 "type": "varchar(64)",
1168 "typeSchema": null,
1169 "notNull": true,
1170 "dimensions": 0,
1171 "default": null,
1172 "generated": null,
1173 "identity": null,
1174 "name": "updated_at",
1175 "entityType": "columns",
1176 "schema": "public",
1177 "table": "relay_mobile_devices"
1178 },
1179 {
1180 "nameExplicit": true,
1181 "columns": [
1182 {
1183 "value": "updated_at",
1184 "isExpression": false,
1185 "asc": true,
1186 "nullsFirst": false,
1187 "opclass": null
1188 }
1189 ],
1190 "isUnique": false,
1191 "where": null,
1192 "with": "",
1193 "method": "btree",
1194 "concurrently": false,
1195 "name": "idx_relay_agent_activity_rows_updated",
1196 "entityType": "indexes",
1197 "schema": "public",
1198 "table": "relay_agent_activity_rows"
1199 },
1200 {
1201 "nameExplicit": true,
1202 "columns": [
1203 {
1204 "value": "environment_id",
1205 "isExpression": false,
1206 "asc": true,
1207 "nullsFirst": false,
1208 "opclass": null
1209 },
1210 {
1211 "value": "thread_id",
1212 "isExpression": false,
1213 "asc": true,
1214 "nullsFirst": false,
1215 "opclass": null
1216 },
1217 {
1218 "value": "created_at",
1219 "isExpression": false,
1220 "asc": true,
1221 "nullsFirst": false,
1222 "opclass": null
1223 }
1224 ],
1225 "isUnique": false,
1226 "where": null,
1227 "with": "",
1228 "method": "btree",
1229 "concurrently": false,
1230 "name": "idx_relay_delivery_attempts_environment",
1231 "entityType": "indexes",
1232 "schema": "public",
1233 "table": "relay_delivery_attempts"
1234 },
1235 {
1236 "nameExplicit": true,
1237 "columns": [
1238 {
1239 "value": "source_job_id",
1240 "isExpression": false,
1241 "asc": true,
1242 "nullsFirst": false,
1243 "opclass": null
1244 }
1245 ],
1246 "isUnique": true,
1247 "where": null,
1248 "with": "",
1249 "method": "btree",
1250 "concurrently": false,
1251 "name": "idx_relay_delivery_attempts_source_job",
1252 "entityType": "indexes",
1253 "schema": "public",
1254 "table": "relay_delivery_attempts"
1255 },
1256 {
1257 "nameExplicit": true,
1258 "columns": [
1259 {
1260 "value": "expires_at",
1261 "isExpression": false,
1262 "asc": true,
1263 "nullsFirst": false,
1264 "opclass": null
1265 }
1266 ],
1267 "isUnique": false,
1268 "where": null,
1269 "with": "",
1270 "method": "btree",
1271 "concurrently": false,
1272 "name": "idx_relay_dpop_proofs_expires_at",
1273 "entityType": "indexes",
1274 "schema": "public",
1275 "table": "relay_dpop_proofs"
1276 },
1277 {
1278 "nameExplicit": true,
1279 "columns": [
1280 {
1281 "value": "credential_hash",
1282 "isExpression": false,
1283 "asc": true,
1284 "nullsFirst": false,
1285 "opclass": null
1286 }
1287 ],
1288 "isUnique": true,
1289 "where": null,
1290 "with": "",
1291 "method": "btree",
1292 "concurrently": false,
1293 "name": "idx_relay_environment_credentials_hash",
1294 "entityType": "indexes",
1295 "schema": "public",
1296 "table": "relay_environment_credentials"
1297 },
1298 {
1299 "nameExplicit": true,
1300 "columns": [
1301 {
1302 "value": "environment_id",
1303 "isExpression": false,
1304 "asc": true,
1305 "nullsFirst": false,
1306 "opclass": null
1307 },
1308 {
1309 "value": "revoked_at",
1310 "isExpression": false,
1311 "asc": true,
1312 "nullsFirst": false,
1313 "opclass": null
1314 }
1315 ],
1316 "isUnique": false,
1317 "where": null,
1318 "with": "",
1319 "method": "btree",
1320 "concurrently": false,
1321 "name": "idx_relay_environment_credentials_environment",
1322 "entityType": "indexes",
1323 "schema": "public",
1324 "table": "relay_environment_credentials"
1325 },
1326 {
1327 "nameExplicit": true,
1328 "columns": [
1329 {
1330 "value": "environment_id",
1331 "isExpression": false,
1332 "asc": true,
1333 "nullsFirst": false,
1334 "opclass": null
1335 },
1336 {
1337 "value": "environment_public_key",
1338 "isExpression": false,
1339 "asc": true,
1340 "nullsFirst": false,
1341 "opclass": null
1342 },
1343 {
1344 "value": "revoked_at",
1345 "isExpression": false,
1346 "asc": true,
1347 "nullsFirst": false,
1348 "opclass": null
1349 }
1350 ],
1351 "isUnique": false,
1352 "where": null,
1353 "with": "",
1354 "method": "btree",
1355 "concurrently": false,
1356 "name": "idx_relay_environment_credentials_environment_key",
1357 "entityType": "indexes",
1358 "schema": "public",
1359 "table": "relay_environment_credentials"
1360 },
1361 {
1362 "nameExplicit": true,
1363 "columns": [
1364 {
1365 "value": "environment_id",
1366 "isExpression": false,
1367 "asc": true,
1368 "nullsFirst": false,
1369 "opclass": null
1370 },
1371 {
1372 "value": "revoked_at",
1373 "isExpression": false,
1374 "asc": true,
1375 "nullsFirst": false,
1376 "opclass": null
1377 }
1378 ],
1379 "isUnique": false,
1380 "where": null,
1381 "with": "",
1382 "method": "btree",
1383 "concurrently": false,
1384 "name": "idx_relay_environment_links_environment",
1385 "entityType": "indexes",
1386 "schema": "public",
1387 "table": "relay_environment_links"
1388 },
1389 {
1390 "nameExplicit": true,
1391 "columns": [
1392 {
1393 "value": "activity_push_token",
1394 "isExpression": false,
1395 "asc": true,
1396 "nullsFirst": false,
1397 "opclass": null
1398 }
1399 ],
1400 "isUnique": true,
1401 "where": null,
1402 "with": "",
1403 "method": "btree",
1404 "concurrently": false,
1405 "name": "idx_relay_live_activities_activity_push_token",
1406 "entityType": "indexes",
1407 "schema": "public",
1408 "table": "relay_live_activities"
1409 },
1410 {
1411 "nameExplicit": true,
1412 "columns": [
1413 {
1414 "value": "hostname",
1415 "isExpression": false,
1416 "asc": true,
1417 "nullsFirst": false,
1418 "opclass": null
1419 }
1420 ],
1421 "isUnique": true,
1422 "where": null,
1423 "with": "",
1424 "method": "btree",
1425 "concurrently": false,
1426 "name": "idx_relay_managed_endpoint_allocations_hostname",
1427 "entityType": "indexes",
1428 "schema": "public",
1429 "table": "relay_managed_endpoint_allocations"
1430 },
1431 {
1432 "nameExplicit": true,
1433 "columns": [
1434 {
1435 "value": "tunnel_name",
1436 "isExpression": false,
1437 "asc": true,
1438 "nullsFirst": false,
1439 "opclass": null
1440 }
1441 ],
1442 "isUnique": true,
1443 "where": null,
1444 "with": "",
1445 "method": "btree",
1446 "concurrently": false,
1447 "name": "idx_relay_managed_endpoint_allocations_tunnel_name",
1448 "entityType": "indexes",
1449 "schema": "public",
1450 "table": "relay_managed_endpoint_allocations"
1451 },
1452 {
1453 "nameExplicit": true,
1454 "columns": [
1455 {
1456 "value": "push_token",
1457 "isExpression": false,
1458 "asc": true,
1459 "nullsFirst": false,
1460 "opclass": null
1461 }
1462 ],
1463 "isUnique": true,
1464 "where": null,
1465 "with": "",
1466 "method": "btree",
1467 "concurrently": false,
1468 "name": "idx_relay_mobile_devices_push_token",
1469 "entityType": "indexes",
1470 "schema": "public",
1471 "table": "relay_mobile_devices"
1472 },
1473 {
1474 "nameExplicit": true,
1475 "columns": [
1476 {
1477 "value": "push_to_start_token",
1478 "isExpression": false,
1479 "asc": true,
1480 "nullsFirst": false,
1481 "opclass": null
1482 }
1483 ],
1484 "isUnique": true,
1485 "where": null,
1486 "with": "",
1487 "method": "btree",
1488 "concurrently": false,
1489 "name": "idx_relay_mobile_devices_push_to_start_token",
1490 "entityType": "indexes",
1491 "schema": "public",
1492 "table": "relay_mobile_devices"
1493 },
1494 {
1495 "columns": ["environment_id", "environment_public_key", "thread_id"],
1496 "nameExplicit": false,
1497 "name": "relay_agent_activity_rows_pkey",
1498 "entityType": "pks",
1499 "schema": "public",
1500 "table": "relay_agent_activity_rows"
1501 },
1502 {
1503 "columns": ["thumbprint", "jti"],
1504 "nameExplicit": false,
1505 "name": "relay_dpop_proofs_pkey",
1506 "entityType": "pks",
1507 "schema": "public",
1508 "table": "relay_dpop_proofs"
1509 },
1510 {
1511 "columns": ["user_id", "environment_id"],
1512 "nameExplicit": false,
1513 "name": "relay_environment_links_pkey",
1514 "entityType": "pks",
1515 "schema": "public",
1516 "table": "relay_environment_links"
1517 },
1518 {
1519 "columns": ["user_id", "device_id"],
1520 "nameExplicit": false,
1521 "name": "relay_live_activities_pkey",
1522 "entityType": "pks",
1523 "schema": "public",
1524 "table": "relay_live_activities"
1525 },
1526 {
1527 "columns": ["user_id", "environment_id"],
1528 "nameExplicit": false,
1529 "name": "relay_managed_endpoint_allocations_pkey",
1530 "entityType": "pks",
1531 "schema": "public",
1532 "table": "relay_managed_endpoint_allocations"
1533 },
1534 {
1535 "columns": ["user_id", "device_id"],
1536 "nameExplicit": false,
1537 "name": "relay_mobile_devices_pkey",
1538 "entityType": "pks",
1539 "schema": "public",
1540 "table": "relay_mobile_devices"
1541 },
1542 {
1543 "columns": ["id"],
1544 "nameExplicit": false,
1545 "name": "relay_delivery_attempts_pkey",
1546 "schema": "public",
1547 "table": "relay_delivery_attempts",
1548 "entityType": "pks"
1549 },
1550 {
1551 "columns": ["credential_id"],
1552 "nameExplicit": false,
1553 "name": "relay_environment_credentials_pkey",
1554 "schema": "public",
1555 "table": "relay_environment_credentials",
1556 "entityType": "pks"
1557 },
1558 {
1559 "columns": ["user_id"],
1560 "nameExplicit": false,
1561 "name": "relay_managed_tunnel_limits_pkey",
1562 "schema": "public",
1563 "table": "relay_managed_tunnel_limits",
1564 "entityType": "pks"
1565 }
1566 ],
1567 "renames": []
1568}
infra/relay/src/Config.test.ts created+35
...@@ -0,0 +1,35 @@
1import { expect, it } from "@effect/vitest";
2import * as ConfigProvider from "effect/ConfigProvider";
3import * as Effect from "effect/Effect";
4
5import { managedEndpointCleanupModeConfig } from "./Config.ts";
6
7it.effect.each([
8 { name: "missing", env: {}, expected: "off" },
9 { name: "empty", env: { RELAY_TUNNEL_CLEANUP_MODE: "" }, expected: "off" },
10 { name: "whitespace", env: { RELAY_TUNNEL_CLEANUP_MODE: " \t" }, expected: "off" },
11 { name: "off", env: { RELAY_TUNNEL_CLEANUP_MODE: "off" }, expected: "off" },
12 {
13 name: "dry-run",
14 env: { RELAY_TUNNEL_CLEANUP_MODE: "dry-run" },
15 expected: "dry-run",
16 },
17 { name: "enabled", env: { RELAY_TUNNEL_CLEANUP_MODE: "enabled" }, expected: "enabled" },
18] as const)("loads $name cleanup mode as $expected", ({ env, expected }) =>
19 Effect.gen(function* () {
20 const provider = ConfigProvider.fromEnv({ env });
21 expect(yield* managedEndpointCleanupModeConfig.parse(provider)).toBe(expected);
22 }),
23);
24
25it.effect("rejects an invalid cleanup mode", () =>
26 Effect.gen(function* () {
27 const provider = ConfigProvider.fromEnv({
28 env: { RELAY_TUNNEL_CLEANUP_MODE: "delete-everything" },
29 });
30 const error = yield* Effect.flip(managedEndpointCleanupModeConfig.parse(provider));
31
32 expect(error._tag).toBe("ConfigError");
33 expect(error.message).toContain('Expected "off" | "dry-run" | "enabled"');
34 }),
35);
infra/relay/src/Config.ts+17
...@@ -1,4 +1,6 @@...@@ -1,4 +1,6 @@
1import * as Config from "effect/Config";
1import * as Context from "effect/Context";2import * as Context from "effect/Context";
3import * as Effect from "effect/Effect";
2import * as Layer from "effect/Layer";4import * as Layer from "effect/Layer";
3import * as Redacted from "effect/Redacted";5import * as Redacted from "effect/Redacted";
4import * as Schema from "effect/Schema";6import * as Schema from "effect/Schema";
...@@ -6,6 +8,20 @@ import * as Schema from "effect/Schema";...@@ -6,6 +8,20 @@ import * as Schema from "effect/Schema";
6export const ApnsEnvironment = Schema.Literals(["sandbox", "production"]);8export const ApnsEnvironment = Schema.Literals(["sandbox", "production"]);
7export type ApnsEnvironment = typeof ApnsEnvironment.Type;9export type ApnsEnvironment = typeof ApnsEnvironment.Type;
810
11export const ManagedEndpointCleanupMode = Schema.Literals(["off", "dry-run", "enabled"]);
12export type ManagedEndpointCleanupMode = typeof ManagedEndpointCleanupMode.Type;
13const decodeManagedEndpointCleanupMode = Schema.decodeUnknownEffect(ManagedEndpointCleanupMode);
14
15export const managedEndpointCleanupModeConfig = Config.String("RELAY_TUNNEL_CLEANUP_MODE").pipe(
16 Config.withDefault("off"),
17 Config.map((value) => value.trim() || "off"),
18 Config.mapEffect((value) =>
19 decodeManagedEndpointCleanupMode(value).pipe(
20 Effect.mapError((error) => new Config.ConfigError(error)),
21 ),
22 ),
23);
24
9export interface ApnsCredentials {25export interface ApnsCredentials {
10 readonly teamId: string;26 readonly teamId: string;
11 readonly keyId: string;27 readonly keyId: string;
...@@ -28,6 +44,7 @@ export class RelayConfiguration extends Context.Service<...@@ -28,6 +44,7 @@ export class RelayConfiguration extends Context.Service<
28 readonly cloudMintPublicKey: string;44 readonly cloudMintPublicKey: string;
29 readonly managedEndpointBaseDomain: string | undefined;45 readonly managedEndpointBaseDomain: string | undefined;
30 readonly managedEndpointNamespace: string | undefined;46 readonly managedEndpointNamespace: string | undefined;
47 readonly managedEndpointCleanupMode?: ManagedEndpointCleanupMode;
31 }48 }
32>()("t3code-relay/Config/RelayConfiguration") {}49>()("t3code-relay/Config/RelayConfiguration") {}
3350
infra/relay/src/deploymentConfig.test.ts+4
...@@ -7,6 +7,7 @@ import {...@@ -7,6 +7,7 @@ import {
7 managedEndpointHostname,7 managedEndpointHostname,
8 isManagedEndpointHostname,8 isManagedEndpointHostname,
9 managedEndpointTunnelName,9 managedEndpointTunnelName,
10 managedEndpointTunnelNamePrefix,
10 relayOwnsManagedEndpointZone,11 relayOwnsManagedEndpointZone,
11 RelayPublicDomainLabelTooLongError,12 RelayPublicDomainLabelTooLongError,
12 relayPublicDomainForStage,13 relayPublicDomainForStage,
...@@ -84,6 +85,9 @@ describe("managed endpoint names", () => {...@@ -84,6 +85,9 @@ describe("managed endpoint names", () => {
84 expect(managedEndpointTunnelName("dev_julius", hash)).toBe(85 expect(managedEndpointTunnelName("dev_julius", hash)).toBe(
85 "t3coderelay-managedendpoint-dev-julius-abcdef0123456789",86 "t3coderelay-managedendpoint-dev-julius-abcdef0123456789",
86 );87 );
88 expect(managedEndpointTunnelNamePrefix("dev_julius")).toBe(
89 "t3coderelay-managedendpoint-dev-julius-",
90 );
87 });91 });
8892
89 it("keeps the DNS label within the provider limit for long stage names", () => {93 it("keeps the DNS label within the provider limit for long stage names", () => {
infra/relay/src/deploymentConfig.ts+5-1
...@@ -117,6 +117,10 @@ export function managedEndpointForHostname(hostname: string): RelayManagedEndpoi...@@ -117,6 +117,10 @@ export function managedEndpointForHostname(hostname: string): RelayManagedEndpoi
117 };117 };
118}118}
119119
120export function managedEndpointTunnelNamePrefix(stage: string): string {
121 return `${MANAGED_ENDPOINT_TUNNEL_PREFIX}-${relayStageSlug(stage)}-`;
122}
123
120export function managedEndpointTunnelName(stage: string, hash: string): string {124export function managedEndpointTunnelName(stage: string, hash: string): string {
121 return `${MANAGED_ENDPOINT_TUNNEL_PREFIX}-${relayStageSlug(stage)}-${stableSuffix(hash)}`;125 return `${managedEndpointTunnelNamePrefix(stage)}${stableSuffix(hash)}`;
122}126}
infra/relay/src/environments/EnvironmentConnector.test.ts+7
...@@ -188,7 +188,9 @@ function makeAllocations(...@@ -188,7 +188,9 @@ function makeAllocations(
188 tunnelName: "tunnel-name",188 tunnelName: "tunnel-name",
189 dnsRecordId: "dns-record-id",189 dnsRecordId: "dns-record-id",
190 readyAt: "2026-05-25T00:00:00.000Z",190 readyAt: "2026-05-25T00:00:00.000Z",
191 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
191 updatedAt: "2026-05-25T00:00:00.000Z",192 updatedAt: "2026-05-25T00:00:00.000Z",
193 generation: 1,
192 },194 },
193): ManagedEndpointAllocations.ManagedEndpointAllocations["Service"] {195): ManagedEndpointAllocations.ManagedEndpointAllocations["Service"] {
194 return {196 return {
...@@ -197,7 +199,10 @@ function makeAllocations(...@@ -197,7 +199,10 @@ function makeAllocations(
197 recordTunnel: () => Effect.die("unused"),199 recordTunnel: () => Effect.die("unused"),
198 recordDns: () => Effect.die("unused"),200 recordDns: () => Effect.die("unused"),
199 markReady: () => Effect.die("unused"),201 markReady: () => Effect.die("unused"),
202 enableRecovery: () => Effect.die("unused"),
203 listByTunnelNames: () => Effect.die("unused"),
200 claimRelease: () => Effect.die("unused"),204 claimRelease: () => Effect.die("unused"),
205 withClaimedTunnel: () => Effect.die("unused"),
201 claimDeprovision: () => Effect.die("unused"),206 claimDeprovision: () => Effect.die("unused"),
202 remove: () => Effect.die("unused"),207 remove: () => Effect.die("unused"),
203 removeClaimed: () => Effect.die("unused"),208 removeClaimed: () => Effect.die("unused"),
...@@ -469,7 +474,9 @@ describe("EnvironmentConnector", () => {...@@ -469,7 +474,9 @@ describe("EnvironmentConnector", () => {
469 tunnelName: "tunnel-name",474 tunnelName: "tunnel-name",
470 dnsRecordId: "dns-record-id",475 dnsRecordId: "dns-record-id",
471 readyAt: null,476 readyAt: null,
477 origin: null,
472 updatedAt: "2026-05-25T00:00:00.000Z",478 updatedAt: "2026-05-25T00:00:00.000Z",
479 generation: 1,
473 }),480 }),
474 }),481 }),
475 ),482 ),
infra/relay/src/environments/EnvironmentLinker.test.ts+3-1
...@@ -134,8 +134,9 @@ function testLayer(input?: {...@@ -134,8 +134,9 @@ function testLayer(input?: {
134 revokeForEnvironmentPublicKey: () => Effect.succeed(false),134 revokeForEnvironmentPublicKey: () => Effect.succeed(false),
135 }),135 }),
136 Layer.succeed(ManagedEndpointProvider.ManagedEndpointProvider, {136 Layer.succeed(ManagedEndpointProvider.ManagedEndpointProvider, {
137 reconcileOrigin: () => Effect.succeed("ready"),
137 prepareDeprovision: () => Effect.succeed(null),138 prepareDeprovision: () => Effect.succeed(null),
138 deprovision: input?.deprovision ?? (() => Effect.void),139 deprovision: input?.deprovision ?? (() => Effect.succeed(true)),
139 release: () => Effect.succeed(true),140 release: () => Effect.succeed(true),
140 provision: () =>141 provision: () =>
141 Effect.succeed({142 Effect.succeed({
...@@ -241,6 +242,7 @@ describe("EnvironmentLinker", () => {...@@ -241,6 +242,7 @@ describe("EnvironmentLinker", () => {
241 deprovision: (input) =>242 deprovision: (input) =>
242 Effect.sync(() => {243 Effect.sync(() => {
243 deprovisionedEnvironmentId = input.environmentId;244 deprovisionedEnvironmentId = input.environmentId;
245 return true;
244 }),246 }),
245 }),247 }),
246 ),248 ),
infra/relay/src/environments/ManagedEndpointAllocations.test.ts+289-7
...@@ -1,6 +1,8 @@...@@ -1,6 +1,8 @@
1import { describe, expect, it } from "@effect/vitest";1import { describe, expect, it } from "@effect/vitest";
2import * as Effect from "effect/Effect";2import * as Effect from "effect/Effect";
3import * as Layer from "effect/Layer";3import * as Layer from "effect/Layer";
4import * as Option from "effect/Option";
5import { PgDialect } from "drizzle-orm/pg-core";
46
5import * as RelayDb from "../db.ts";7import * as RelayDb from "../db.ts";
6import { relayManagedEndpointAllocations } from "../persistence/schema.ts";8import { relayManagedEndpointAllocations } from "../persistence/schema.ts";
...@@ -10,17 +12,253 @@ const layerWithDb = (db: RelayDb.RelayDb["Service"]) =>...@@ -10,17 +12,253 @@ const layerWithDb = (db: RelayDb.RelayDb["Service"]) =>
10 ManagedEndpointAllocations.layer.pipe(Layer.provide(Layer.succeed(RelayDb.RelayDb, db)));12 ManagedEndpointAllocations.layer.pipe(Layer.provide(Layer.succeed(RelayDb.RelayDb, db)));
1113
12describe("ManagedEndpointAllocations", () => {14describe("ManagedEndpointAllocations", () => {
15 it.effect("clears endpoint readiness and recovery only when the recorded tunnel changes", () => {
16 let updated:
17 | {
18 readonly tunnelId: string;
19 readonly readyAt: unknown;
20 readonly recoveryEnabledAt: unknown;
21 readonly recoveryEnvironmentPublicKey: unknown;
22 }
23 | undefined;
24 const fakeDb = {
25 update: (table: unknown) => {
26 expect(table).toBe(relayManagedEndpointAllocations);
27 return {
28 set: (values: {
29 readonly tunnelId: string;
30 readonly readyAt: unknown;
31 readonly recoveryEnabledAt: unknown;
32 readonly recoveryEnvironmentPublicKey: unknown;
33 }) => {
34 updated = values;
35 return {
36 where: () => ({
37 returning: () => Effect.succeed([{ generation: 8 }]),
38 }),
39 };
40 },
41 };
42 },
43 } as unknown as RelayDb.RelayDb["Service"];
44
45 return Effect.gen(function* () {
46 const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations;
47 expect(
48 yield* allocations.recordTunnel({
49 userId: "user-1",
50 environmentId: "environment-1",
51 tunnelId: "replacement-tunnel",
52 generation: 7,
53 }),
54 ).toBe(8);
55 expect(updated?.tunnelId).toBe("replacement-tunnel");
56 const query = new PgDialect().sqlToQuery(updated?.readyAt as never);
57 expect(query.sql).toBe(
58 'case when "relay_managed_endpoint_allocations"."tunnel_id" = $1 then "relay_managed_endpoint_allocations"."ready_at" else null end',
59 );
60 expect(query.params).toEqual(["replacement-tunnel"]);
61 for (const [column, value] of [
62 ["recovery_enabled_at", updated?.recoveryEnabledAt],
63 ["recovery_environment_public_key", updated?.recoveryEnvironmentPublicKey],
64 ] as const) {
65 const recoveryQuery = new PgDialect().sqlToQuery(value as never);
66 expect(recoveryQuery.sql).toBe(
67 `case when "relay_managed_endpoint_allocations"."tunnel_id" = $1 then "relay_managed_endpoint_allocations"."${column}" else null end`,
68 );
69 expect(recoveryQuery.params).toEqual(["replacement-tunnel"]);
70 }
71 }).pipe(Effect.provide(layerWithDb(fakeDb)));
72 });
73
74 it.effect("records recovery support and advances the allocation generation", () => {
75 let updated:
76 | {
77 readonly recoveryEnabledAt: string;
78 readonly recoveryEnvironmentPublicKey: string;
79 readonly updatedAt: string;
80 }
81 | undefined;
82 let condition: unknown;
83 const fakeDb = {
84 update: (table: unknown) => {
85 expect(table).toBe(relayManagedEndpointAllocations);
86 return {
87 set: (values: {
88 readonly recoveryEnabledAt: string;
89 readonly recoveryEnvironmentPublicKey: string;
90 readonly updatedAt: string;
91 }) => {
92 updated = values;
93 return {
94 where: (where: unknown) => {
95 condition = where;
96 return {
97 returning: () => Effect.succeed([{ environmentId: "environment-1" }]),
98 };
99 },
100 };
101 },
102 };
103 },
104 } as unknown as RelayDb.RelayDb["Service"];
105
106 return Effect.gen(function* () {
107 const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations;
108 expect(
109 yield* allocations.enableRecovery({
110 userId: "user-1",
111 environmentId: "environment-1",
112 tunnelId: "tunnel-1",
113 environmentPublicKey: "public-key",
114 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
115 }),
116 ).toBe(true);
117
118 expect(updated?.recoveryEnabledAt).toBe(updated?.updatedAt);
119 expect(updated?.recoveryEnabledAt).toBeDefined();
120 expect(updated?.recoveryEnvironmentPublicKey).toBe("public-key");
121 const query = new PgDialect().sqlToQuery(condition as never);
122 expect(query.sql).toContain('"relay_managed_endpoint_allocations"."tunnel_id"');
123 expect(query.sql).toContain('"relay_environment_links"."environment_public_key"');
124 expect(query.sql).toContain('"relay_environment_links"."endpoint_provider_kind"');
125 expect(query.sql).toContain('"relay_environment_links"."revoked_at" is null');
126 expect(query.sql).toContain("for update");
127 expect(query.params).toContain("tunnel-1");
128 expect(query.params).toContain("public-key");
129 expect(query.params).toContain("cloudflare_tunnel");
130 }).pipe(Effect.provide(layerWithDb(fakeDb)));
131 });
132
133 it.effect("rejects recovery when the tunnel or active link no longer matches", () => {
134 const fakeDb = {
135 update: () => ({
136 set: () => ({
137 where: () => ({
138 returning: () => Effect.succeed([]),
139 }),
140 }),
141 }),
142 } as unknown as RelayDb.RelayDb["Service"];
143
144 return Effect.gen(function* () {
145 const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations;
146 expect(
147 yield* allocations.enableRecovery({
148 userId: "user-1",
149 environmentId: "environment-1",
150 tunnelId: "missing-tunnel",
151 environmentPublicKey: "public-key",
152 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
153 }),
154 ).toBe(false);
155 }).pipe(Effect.provide(layerWithDb(fakeDb)));
156 });
157
158 it.effect("returns recovery support with tunnel allocation lookups", () => {
159 const base = {
160 userId: "user-1",
161 hostname: "environment.example.test",
162 tunnelName: "managed-tunnel",
163 dnsRecordId: "dns-1",
164 readyAt: "2026-08-25T12:00:00.000Z",
165 updatedAt: "2026-08-25T12:00:00.000Z",
166 generation: 1,
167 };
168 const fakeDb = {
169 select: () => ({
170 from: (table: unknown) => {
171 expect(table).toBe(relayManagedEndpointAllocations);
172 return {
173 leftJoin: () => ({
174 where: () =>
175 Effect.succeed([
176 {
177 ...base,
178 environmentId: "environment-1",
179 tunnelId: "tunnel-1",
180 recoveryEnabledAt: "2026-08-25T12:00:00.000Z",
181 recoveryEnvironmentPublicKey: "current-key",
182 linkedEnvironmentPublicKey: "current-key",
183 },
184 {
185 ...base,
186 environmentId: "environment-2",
187 tunnelId: "tunnel-2",
188 recoveryEnabledAt: null,
189 recoveryEnvironmentPublicKey: null,
190 linkedEnvironmentPublicKey: "current-key",
191 },
192 {
193 ...base,
194 environmentId: "environment-3",
195 tunnelId: "tunnel-3",
196 recoveryEnabledAt: "2026-08-25T12:00:00.000Z",
197 recoveryEnvironmentPublicKey: "old-key",
198 linkedEnvironmentPublicKey: "new-key",
199 },
200 ]),
201 }),
202 };
203 },
204 }),
205 } as unknown as RelayDb.RelayDb["Service"];
206
207 return Effect.gen(function* () {
208 const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations;
209 const result = yield* allocations.listByTunnelNames([
210 "first-tunnel",
211 "second-tunnel",
212 "third-tunnel",
213 ]);
214
215 expect(result.map((entry) => [entry.tunnelId, entry.recoveryEnabled])).toEqual([
216 ["tunnel-1", true],
217 ["tunnel-2", false],
218 ["tunnel-3", false],
219 ]);
220 }).pipe(Effect.provide(layerWithDb(fakeDb)));
221 });
222
223 it.effect("skips the database for an empty tunnel lookup", () =>
224 Effect.gen(function* () {
225 const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations;
226 expect(yield* allocations.listByTunnelNames([])).toEqual([]);
227 }).pipe(Effect.provide(layerWithDb({} as RelayDb.RelayDb["Service"]))),
228 );
229
230 it.effect("splits large tunnel lookups into bounded database queries", () => {
231 const batchSizes: number[] = [];
232 const fakeDb = {
233 select: () => ({
234 from: () => ({
235 leftJoin: () => ({
236 where: (condition: unknown) => {
237 batchSizes.push(new PgDialect().sqlToQuery(condition as never).params.length);
238 return Effect.succeed([]);
239 },
240 }),
241 }),
242 }),
243 } as unknown as RelayDb.RelayDb["Service"];
244
245 return Effect.gen(function* () {
246 const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations;
247 const names = Array.from({ length: 1_001 }, (_, index) => `tunnel-${index}`);
248 expect(yield* allocations.listByTunnelNames(names)).toEqual([]);
249 expect(batchSizes).toEqual([500, 500, 1]);
250 }).pipe(Effect.provide(layerWithDb(fakeDb)));
251 });
252
13 it.effect("returns a claim generation only when deprovision wins the allocation CAS", () => {253 it.effect("returns a claim generation only when deprovision wins the allocation CAS", () => {
14 let claimedAt: string | undefined;
15 const fakeDb = {254 const fakeDb = {
16 update: (table: unknown) => {255 update: (table: unknown) => {
17 expect(table).toBe(relayManagedEndpointAllocations);256 expect(table).toBe(relayManagedEndpointAllocations);
18 return {257 return {
19 set: (values: { readonly updatedAt: string }) => {258 set: (_values: { readonly updatedAt: string }) => {
20 claimedAt = values.updatedAt;
21 return {259 return {
22 where: () => ({260 where: () => ({
23 returning: () => Effect.succeed([{ userId: "user-1" }]),261 returning: () => Effect.succeed([{ generation: 8 }]),
24 }),262 }),
25 };263 };
26 },264 },
...@@ -33,14 +271,58 @@ describe("ManagedEndpointAllocations", () => {...@@ -33,14 +271,58 @@ describe("ManagedEndpointAllocations", () => {
33 const generation = yield* allocations.claimDeprovision({271 const generation = yield* allocations.claimDeprovision({
34 userId: "user-1",272 userId: "user-1",
35 environmentId: "environment-1",273 environmentId: "environment-1",
36 updatedAt: "captured-generation",274 generation: 7,
37 });275 });
38276
39 expect(generation).toBe(claimedAt);277 expect(generation).toBe(8);
40 expect(generation).not.toBeNull();278 expect(generation).not.toBeNull();
41 }).pipe(Effect.provide(layerWithDb(fakeDb)));279 }).pipe(Effect.provide(layerWithDb(fakeDb)));
42 });280 });
43281
282 it.effect("holds the claimed allocation row while deleting its tunnel", () => {
283 const operations: string[] = [];
284 const fakeDb = {
285 $client: {
286 withTransaction: <A, E, R>(effect: Effect.Effect<A, E, R>) =>
287 Effect.sync(() => {
288 operations.push("transaction");
289 }).pipe(Effect.andThen(effect)),
290 },
291 select: () => ({
292 from: () => ({
293 where: () => ({
294 limit: () => ({
295 for: (strength: string) =>
296 Effect.sync(() => {
297 operations.push(`lock:${strength}`);
298 return [{ generation: 7 }];
299 }),
300 }),
301 }),
302 }),
303 }),
304 } as unknown as RelayDb.RelayDb["Service"];
305
306 return Effect.gen(function* () {
307 const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations;
308 const result = yield* allocations.withClaimedTunnel(
309 {
310 userId: "user-1",
311 environmentId: "environment-1",
312 tunnelId: "tunnel-1",
313 generation: 7,
314 },
315 Effect.sync(() => {
316 operations.push("delete");
317 return true;
318 }),
319 );
320
321 expect(Option.getOrNull(result)).toBe(true);
322 expect(operations).toEqual(["transaction", "lock:update", "delete"]);
323 }).pipe(Effect.provide(layerWithDb(fakeDb)));
324 });
325
44 it.effect("does not remove an allocation superseded after a deprovision claim", () => {326 it.effect("does not remove an allocation superseded after a deprovision claim", () => {
45 const fakeDb = {327 const fakeDb = {
46 delete: (table: unknown) => {328 delete: (table: unknown) => {
...@@ -59,7 +341,7 @@ describe("ManagedEndpointAllocations", () => {...@@ -59,7 +341,7 @@ describe("ManagedEndpointAllocations", () => {
59 yield* allocations.removeClaimed({341 yield* allocations.removeClaimed({
60 userId: "user-1",342 userId: "user-1",
61 environmentId: "environment-1",343 environmentId: "environment-1",
62 updatedAt: "outdated-claim-generation",344 generation: 7,
63 }),345 }),
64 ).toBe(false);346 ).toBe(false);
65 }).pipe(Effect.provide(layerWithDb(fakeDb)));347 }).pipe(Effect.provide(layerWithDb(fakeDb)));
infra/relay/src/environments/ManagedEndpointAllocations.ts+270-34
...@@ -1,14 +1,17 @@...@@ -1,14 +1,17 @@
1import type { RelayManagedEndpoint } from "@t3tools/contracts/relay";1import type { RelayManagedEndpoint, RelayManagedEndpointOrigin } from "@t3tools/contracts/relay";
2import { and, eq } from "drizzle-orm";2import { and, eq, exists, inArray, isNull, sql } from "drizzle-orm";
3import { QueryBuilder } from "drizzle-orm/pg-core";
3import * as Context from "effect/Context";4import * as Context from "effect/Context";
4import * as DateTime from "effect/DateTime";5import * as DateTime from "effect/DateTime";
5import * as Effect from "effect/Effect";6import * as Effect from "effect/Effect";
6import * as Layer from "effect/Layer";7import * as Layer from "effect/Layer";
8import * as Option from "effect/Option";
7import * as Schema from "effect/Schema";9import * as Schema from "effect/Schema";
10import * as SqlError from "effect/unstable/sql/SqlError";
811
9import * as RelayDb from "../db.ts";12import * as RelayDb from "../db.ts";
10import { isManagedEndpointHostname, managedEndpointForHostname } from "../deploymentConfig.ts";13import { isManagedEndpointHostname, managedEndpointForHostname } from "../deploymentConfig.ts";
11import { relayManagedEndpointAllocations } from "../persistence/schema.ts";14import { relayEnvironmentLinks, relayManagedEndpointAllocations } from "../persistence/schema.ts";
1215
13export interface ManagedEndpointAllocation {16export interface ManagedEndpointAllocation {
14 readonly userId: string;17 readonly userId: string;
...@@ -18,13 +21,17 @@ export interface ManagedEndpointAllocation {...@@ -18,13 +21,17 @@ export interface ManagedEndpointAllocation {
18 readonly tunnelName: string;21 readonly tunnelName: string;
19 readonly dnsRecordId: string | null;22 readonly dnsRecordId: string | null;
20 readonly readyAt: string | null;23 readonly readyAt: string | null;
21 /**24 readonly origin: RelayManagedEndpointOrigin | null;
22 * Doubles as the allocation's generation marker: every mutation rewrites it,
23 * so `claimRelease` can detect a provision that raced a release.
24 */
25 readonly updatedAt: string;25 readonly updatedAt: string;
26 readonly generation: number;
26}27}
2728
29export interface ManagedEndpointTunnelAllocation extends ManagedEndpointAllocation {
30 readonly recoveryEnabled: boolean;
31}
32
33export const MANAGED_ENDPOINT_ALLOCATION_LOOKUP_BATCH_SIZE = 500;
34
28export function resolveReadyManagedEndpoint(input: {35export function resolveReadyManagedEndpoint(input: {
29 readonly allocation: ManagedEndpointAllocation;36 readonly allocation: ManagedEndpointAllocation;
30 readonly baseDomain: string | undefined;37 readonly baseDomain: string | undefined;
...@@ -50,6 +57,9 @@ export class ManagedEndpointAllocationPersistenceError extends Schema.TaggedErro...@@ -50,6 +57,9 @@ export class ManagedEndpointAllocationPersistenceError extends Schema.TaggedErro
50 "record-tunnel",57 "record-tunnel",
51 "record-dns",58 "record-dns",
52 "mark-ready",59 "mark-ready",
60 "enable-recovery",
61 "list-tunnels",
62 "lock-tunnel",
53 "claim-release",63 "claim-release",
54 "claim-deprovision",64 "claim-deprovision",
55 "remove",65 "remove",
...@@ -82,23 +92,38 @@ interface ReserveManagedEndpointAllocationInput extends ManagedEndpointAllocatio...@@ -82,23 +92,38 @@ interface ReserveManagedEndpointAllocationInput extends ManagedEndpointAllocatio
8292
83interface RecordManagedEndpointTunnelInput extends ManagedEndpointAllocationKey {93interface RecordManagedEndpointTunnelInput extends ManagedEndpointAllocationKey {
84 readonly tunnelId: string;94 readonly tunnelId: string;
95 readonly generation: number;
85}96}
8697
87interface RecordManagedEndpointDnsInput extends ManagedEndpointAllocationKey {98interface RecordManagedEndpointDnsInput extends ManagedEndpointAllocationKey {
88 readonly dnsRecordId: string;99 readonly dnsRecordId: string;
100 readonly tunnelId: string;
101 readonly generation: number;
102}
103
104interface MarkManagedEndpointReadyInput extends ManagedEndpointAllocationKey {
105 readonly tunnelId: string;
106 readonly generation: number;
107 readonly origin: RelayManagedEndpointOrigin;
89}108}
90109
91interface ClaimManagedEndpointReleaseInput extends ManagedEndpointAllocationKey {110interface ClaimManagedEndpointReleaseInput extends ManagedEndpointAllocationKey {
92 readonly tunnelId: string;111 readonly tunnelId: string;
93 readonly updatedAt: string;112 readonly generation: number;
113}
114
115interface EnableManagedEndpointRecoveryInput extends ManagedEndpointAllocationKey {
116 readonly tunnelId: string;
117 readonly environmentPublicKey: string;
118 readonly origin: RelayManagedEndpointOrigin;
94}119}
95120
96interface ClaimManagedEndpointDeprovisionInput extends ManagedEndpointAllocationKey {121interface ClaimManagedEndpointDeprovisionInput extends ManagedEndpointAllocationKey {
97 readonly updatedAt: string;122 readonly generation: number;
98}123}
99124
100interface RemoveClaimedManagedEndpointAllocationInput extends ManagedEndpointAllocationKey {125interface RemoveClaimedManagedEndpointAllocationInput extends ManagedEndpointAllocationKey {
101 readonly updatedAt: string;126 readonly generation: number;
102}127}
103128
104export class ManagedEndpointAllocations extends Context.Service<129export class ManagedEndpointAllocations extends Context.Service<
...@@ -112,23 +137,36 @@ export class ManagedEndpointAllocations extends Context.Service<...@@ -112,23 +137,36 @@ export class ManagedEndpointAllocations extends Context.Service<
112 ) => Effect.Effect<ManagedEndpointAllocation, ManagedEndpointAllocationPersistenceError>;137 ) => Effect.Effect<ManagedEndpointAllocation, ManagedEndpointAllocationPersistenceError>;
113 readonly recordTunnel: (138 readonly recordTunnel: (
114 input: RecordManagedEndpointTunnelInput,139 input: RecordManagedEndpointTunnelInput,
115 ) => Effect.Effect<void, ManagedEndpointAllocationPersistenceError>;140 ) => Effect.Effect<number | null, ManagedEndpointAllocationPersistenceError>;
116 readonly recordDns: (141 readonly recordDns: (
117 input: RecordManagedEndpointDnsInput,142 input: RecordManagedEndpointDnsInput,
118 ) => Effect.Effect<void, ManagedEndpointAllocationPersistenceError>;143 ) => Effect.Effect<number | null, ManagedEndpointAllocationPersistenceError>;
119 readonly markReady: (144 readonly markReady: (
120 input: ManagedEndpointAllocationKey,145 input: MarkManagedEndpointReadyInput,
121 ) => Effect.Effect<void, ManagedEndpointAllocationPersistenceError>;146 ) => Effect.Effect<boolean, ManagedEndpointAllocationPersistenceError>;
147 readonly enableRecovery: (
148 input: EnableManagedEndpointRecoveryInput,
149 ) => Effect.Effect<boolean, ManagedEndpointAllocationPersistenceError>;
150 readonly listByTunnelNames: (
151 tunnelNames: ReadonlyArray<string>,
152 ) => Effect.Effect<
153 ReadonlyArray<ManagedEndpointTunnelAllocation>,
154 ManagedEndpointAllocationPersistenceError
155 >;
122 /**156 /**
123 * Atomically claims the right to delete the allocation's tunnel: succeeds157 * Atomically claims the right to delete the allocation's tunnel: succeeds
124 * only while the recorded tunnel and generation still match what the158 * only while the recorded tunnel and generation still match what the
125 * caller loaded. A concurrent provision rewrites `updatedAt` when it159 * caller loaded. A concurrent provision increments `generation` when it
126 * records its tunnel, which makes a stale claim fail and keeps the freshly160 * records its tunnel, which makes a stale claim fail and keeps the freshly
127 * issued tunnel alive.161 * issued tunnel alive.
128 */162 */
129 readonly claimRelease: (163 readonly claimRelease: (
130 input: ClaimManagedEndpointReleaseInput,164 input: ClaimManagedEndpointReleaseInput,
131 ) => Effect.Effect<boolean, ManagedEndpointAllocationPersistenceError>;165 ) => Effect.Effect<number | null, ManagedEndpointAllocationPersistenceError>;
166 readonly withClaimedTunnel: <A, E, R>(
167 input: ClaimManagedEndpointReleaseInput,
168 effect: Effect.Effect<A, E, R>,
169 ) => Effect.Effect<Option.Option<A>, E | ManagedEndpointAllocationPersistenceError, R>;
132 /**170 /**
133 * Claims the complete allocation for teardown only if its generation still171 * Claims the complete allocation for teardown only if its generation still
134 * matches the snapshot captured by the unlink operation.172 * matches the snapshot captured by the unlink operation.
...@@ -138,7 +176,7 @@ export class ManagedEndpointAllocations extends Context.Service<...@@ -138,7 +176,7 @@ export class ManagedEndpointAllocations extends Context.Service<
138 */176 */
139 readonly claimDeprovision: (177 readonly claimDeprovision: (
140 input: ClaimManagedEndpointDeprovisionInput,178 input: ClaimManagedEndpointDeprovisionInput,
141 ) => Effect.Effect<string | null, ManagedEndpointAllocationPersistenceError>;179 ) => Effect.Effect<number | null, ManagedEndpointAllocationPersistenceError>;
142 readonly remove: (180 readonly remove: (
143 input: ManagedEndpointAllocationKey,181 input: ManagedEndpointAllocationKey,
144 ) => Effect.Effect<void, ManagedEndpointAllocationPersistenceError>;182 ) => Effect.Effect<void, ManagedEndpointAllocationPersistenceError>;
...@@ -156,7 +194,9 @@ const allocationSelection = {...@@ -156,7 +194,9 @@ const allocationSelection = {
156 tunnelName: relayManagedEndpointAllocations.tunnelName,194 tunnelName: relayManagedEndpointAllocations.tunnelName,
157 dnsRecordId: relayManagedEndpointAllocations.dnsRecordId,195 dnsRecordId: relayManagedEndpointAllocations.dnsRecordId,
158 readyAt: relayManagedEndpointAllocations.readyAt,196 readyAt: relayManagedEndpointAllocations.readyAt,
197 origin: relayManagedEndpointAllocations.origin,
159 updatedAt: relayManagedEndpointAllocations.updatedAt,198 updatedAt: relayManagedEndpointAllocations.updatedAt,
199 generation: relayManagedEndpointAllocations.generation,
160};200};
161201
162const whereAllocation = (input: ManagedEndpointAllocationKey) =>202const whereAllocation = (input: ManagedEndpointAllocationKey) =>
...@@ -248,14 +288,28 @@ export const make = Effect.gen(function* () {...@@ -248,14 +288,28 @@ export const make = Effect.gen(function* () {
248 recordTunnel: Effect.fn("relay.managed_endpoint_allocations.record_tunnel")(function* (288 recordTunnel: Effect.fn("relay.managed_endpoint_allocations.record_tunnel")(function* (
249 input: RecordManagedEndpointTunnelInput,289 input: RecordManagedEndpointTunnelInput,
250 ) {290 ) {
251 yield* db291 return yield* db
252 .update(relayManagedEndpointAllocations)292 .update(relayManagedEndpointAllocations)
253 .set({293 .set({
254 tunnelId: input.tunnelId,294 tunnelId: input.tunnelId,
295 readyAt: sql`case when ${relayManagedEndpointAllocations.tunnelId} = ${input.tunnelId} then ${relayManagedEndpointAllocations.readyAt} else null end`,
296 origin: sql`case when ${relayManagedEndpointAllocations.tunnelId} = ${input.tunnelId} then ${relayManagedEndpointAllocations.origin} else null end`,
297 // Recovery registration is per tunnel: a replacement must register
298 // again before the reaper may treat it as recoverable.
299 recoveryEnabledAt: sql`case when ${relayManagedEndpointAllocations.tunnelId} = ${input.tunnelId} then ${relayManagedEndpointAllocations.recoveryEnabledAt} else null end`,
300 recoveryEnvironmentPublicKey: sql`case when ${relayManagedEndpointAllocations.tunnelId} = ${input.tunnelId} then ${relayManagedEndpointAllocations.recoveryEnvironmentPublicKey} else null end`,
255 updatedAt: DateTime.formatIso(yield* DateTime.now),301 updatedAt: DateTime.formatIso(yield* DateTime.now),
302 generation: sql`${relayManagedEndpointAllocations.generation} + 1`,
256 })303 })
257 .where(whereAllocation(input))304 .where(
305 and(
306 whereAllocation(input),
307 eq(relayManagedEndpointAllocations.generation, input.generation),
308 ),
309 )
310 .returning({ generation: relayManagedEndpointAllocations.generation })
258 .pipe(311 .pipe(
312 Effect.map((rows) => rows[0]?.generation ?? null),
259 Effect.mapError(313 Effect.mapError(
260 (cause) =>314 (cause) =>
261 new ManagedEndpointAllocationPersistenceError({315 new ManagedEndpointAllocationPersistenceError({
...@@ -270,14 +324,23 @@ export const make = Effect.gen(function* () {...@@ -270,14 +324,23 @@ export const make = Effect.gen(function* () {
270 recordDns: Effect.fn("relay.managed_endpoint_allocations.record_dns")(function* (324 recordDns: Effect.fn("relay.managed_endpoint_allocations.record_dns")(function* (
271 input: RecordManagedEndpointDnsInput,325 input: RecordManagedEndpointDnsInput,
272 ) {326 ) {
273 yield* db327 return yield* db
274 .update(relayManagedEndpointAllocations)328 .update(relayManagedEndpointAllocations)
275 .set({329 .set({
276 dnsRecordId: input.dnsRecordId,330 dnsRecordId: input.dnsRecordId,
277 updatedAt: DateTime.formatIso(yield* DateTime.now),331 updatedAt: DateTime.formatIso(yield* DateTime.now),
332 generation: sql`${relayManagedEndpointAllocations.generation} + 1`,
278 })333 })
279 .where(whereAllocation(input))334 .where(
335 and(
336 whereAllocation(input),
337 eq(relayManagedEndpointAllocations.tunnelId, input.tunnelId),
338 eq(relayManagedEndpointAllocations.generation, input.generation),
339 ),
340 )
341 .returning({ generation: relayManagedEndpointAllocations.generation })
280 .pipe(342 .pipe(
343 Effect.map((rows) => rows[0]?.generation ?? null),
281 Effect.mapError(344 Effect.mapError(
282 (cause) =>345 (cause) =>
283 new ManagedEndpointAllocationPersistenceError({346 new ManagedEndpointAllocationPersistenceError({
...@@ -290,17 +353,27 @@ export const make = Effect.gen(function* () {...@@ -290,17 +353,27 @@ export const make = Effect.gen(function* () {
290 );353 );
291 }),354 }),
292 markReady: Effect.fn("relay.managed_endpoint_allocations.mark_ready")(function* (355 markReady: Effect.fn("relay.managed_endpoint_allocations.mark_ready")(function* (
293 input: ManagedEndpointAllocationKey,356 input: MarkManagedEndpointReadyInput,
294 ) {357 ) {
295 const now = DateTime.formatIso(yield* DateTime.now);358 const now = DateTime.formatIso(yield* DateTime.now);
296 yield* db359 return yield* db
297 .update(relayManagedEndpointAllocations)360 .update(relayManagedEndpointAllocations)
298 .set({361 .set({
299 readyAt: now,362 readyAt: now,
363 origin: input.origin,
300 updatedAt: now,364 updatedAt: now,
365 generation: sql`${relayManagedEndpointAllocations.generation} + 1`,
301 })366 })
302 .where(whereAllocation(input))367 .where(
368 and(
369 whereAllocation(input),
370 eq(relayManagedEndpointAllocations.tunnelId, input.tunnelId),
371 eq(relayManagedEndpointAllocations.generation, input.generation),
372 ),
373 )
374 .returning({ environmentId: relayManagedEndpointAllocations.environmentId })
303 .pipe(375 .pipe(
376 Effect.map((rows) => rows.length > 0),
304 Effect.mapError(377 Effect.mapError(
305 (cause) =>378 (cause) =>
306 new ManagedEndpointAllocationPersistenceError({379 new ManagedEndpointAllocationPersistenceError({
...@@ -312,6 +385,124 @@ export const make = Effect.gen(function* () {...@@ -312,6 +385,124 @@ export const make = Effect.gen(function* () {
312 ),385 ),
313 );386 );
314 }),387 }),
388 enableRecovery: Effect.fn("relay.managed_endpoint_allocations.enable_recovery")(function* (
389 input: EnableManagedEndpointRecoveryInput,
390 ) {
391 const now = DateTime.formatIso(yield* DateTime.now);
392 return yield* db
393 .update(relayManagedEndpointAllocations)
394 .set({
395 recoveryEnabledAt: now,
396 recoveryEnvironmentPublicKey: input.environmentPublicKey,
397 updatedAt: now,
398 generation: sql`${relayManagedEndpointAllocations.generation} + 1`,
399 })
400 .where(
401 and(
402 whereAllocation(input),
403 eq(relayManagedEndpointAllocations.tunnelId, input.tunnelId),
404 eq(relayManagedEndpointAllocations.origin, input.origin),
405 exists(
406 new QueryBuilder()
407 .select({ userId: relayEnvironmentLinks.userId })
408 .from(relayEnvironmentLinks)
409 .where(
410 and(
411 eq(relayEnvironmentLinks.userId, input.userId),
412 eq(relayEnvironmentLinks.environmentId, input.environmentId),
413 eq(relayEnvironmentLinks.environmentPublicKey, input.environmentPublicKey),
414 eq(relayEnvironmentLinks.endpointProviderKind, "cloudflare_tunnel"),
415 isNull(relayEnvironmentLinks.revokedAt),
416 ),
417 )
418 .for("update"),
419 ),
420 ),
421 )
422 .returning({ environmentId: relayManagedEndpointAllocations.environmentId })
423 .pipe(
424 Effect.map((rows) => rows.length > 0),
425 Effect.mapError(
426 (cause) =>
427 new ManagedEndpointAllocationPersistenceError({
428 operation: "enable-recovery",
429 stage: "database-request",
430 ...input,
431 cause,
432 }),
433 ),
434 );
435 }),
436 listByTunnelNames: Effect.fn("relay.managed_endpoint_allocations.list_by_tunnel_names")(
437 function* (tunnelNames: ReadonlyArray<string>) {
438 if (tunnelNames.length === 0) {
439 return [];
440 }
441 const batches = Array.from(
442 { length: Math.ceil(tunnelNames.length / MANAGED_ENDPOINT_ALLOCATION_LOOKUP_BATCH_SIZE) },
443 (_, index) =>
444 tunnelNames.slice(
445 index * MANAGED_ENDPOINT_ALLOCATION_LOOKUP_BATCH_SIZE,
446 (index + 1) * MANAGED_ENDPOINT_ALLOCATION_LOOKUP_BATCH_SIZE,
447 ),
448 );
449 const results = yield* Effect.forEach(
450 batches,
451 (batch) =>
452 db
453 .select({
454 ...allocationSelection,
455 recoveryEnabledAt: relayManagedEndpointAllocations.recoveryEnabledAt,
456 recoveryEnvironmentPublicKey:
457 relayManagedEndpointAllocations.recoveryEnvironmentPublicKey,
458 linkedEnvironmentPublicKey: relayEnvironmentLinks.environmentPublicKey,
459 })
460 .from(relayManagedEndpointAllocations)
461 .leftJoin(
462 relayEnvironmentLinks,
463 and(
464 eq(relayEnvironmentLinks.userId, relayManagedEndpointAllocations.userId),
465 eq(
466 relayEnvironmentLinks.environmentId,
467 relayManagedEndpointAllocations.environmentId,
468 ),
469 isNull(relayEnvironmentLinks.revokedAt),
470 ),
471 )
472 .where(inArray(relayManagedEndpointAllocations.tunnelName, batch))
473 .pipe(
474 Effect.map((rows) =>
475 rows.map(
476 ({
477 recoveryEnabledAt,
478 recoveryEnvironmentPublicKey,
479 linkedEnvironmentPublicKey,
480 ...allocation
481 }) => ({
482 ...allocation,
483 recoveryEnabled:
484 recoveryEnabledAt !== null &&
485 recoveryEnvironmentPublicKey !== null &&
486 recoveryEnvironmentPublicKey === linkedEnvironmentPublicKey,
487 }),
488 ),
489 ),
490 Effect.mapError(
491 (cause) =>
492 new ManagedEndpointAllocationPersistenceError({
493 operation: "list-tunnels",
494 stage: "database-request",
495 userId: "*",
496 environmentId: "*",
497 cause,
498 }),
499 ),
500 ),
501 { concurrency: 1 },
502 );
503 return results.flat();
504 },
505 ),
315 claimRelease: Effect.fn("relay.managed_endpoint_allocations.claim_release")(function* (506 claimRelease: Effect.fn("relay.managed_endpoint_allocations.claim_release")(function* (
316 input: ClaimManagedEndpointReleaseInput,507 input: ClaimManagedEndpointReleaseInput,
317 ) {508 ) {
...@@ -319,17 +510,18 @@ export const make = Effect.gen(function* () {...@@ -319,17 +510,18 @@ export const make = Effect.gen(function* () {
319 .update(relayManagedEndpointAllocations)510 .update(relayManagedEndpointAllocations)
320 .set({511 .set({
321 updatedAt: DateTime.formatIso(yield* DateTime.now),512 updatedAt: DateTime.formatIso(yield* DateTime.now),
513 generation: sql`${relayManagedEndpointAllocations.generation} + 1`,
322 })514 })
323 .where(515 .where(
324 and(516 and(
325 whereAllocation(input),517 whereAllocation(input),
326 eq(relayManagedEndpointAllocations.tunnelId, input.tunnelId),518 eq(relayManagedEndpointAllocations.tunnelId, input.tunnelId),
327 eq(relayManagedEndpointAllocations.updatedAt, input.updatedAt),519 eq(relayManagedEndpointAllocations.generation, input.generation),
328 ),520 ),
329 )521 )
330 .returning({ userId: relayManagedEndpointAllocations.userId })522 .returning({ generation: relayManagedEndpointAllocations.generation })
331 .pipe(523 .pipe(
332 Effect.map((rows) => rows.length > 0),524 Effect.map((rows) => rows[0]?.generation ?? null),
333 Effect.mapError(525 Effect.mapError(
334 (cause) =>526 (cause) =>
335 new ManagedEndpointAllocationPersistenceError({527 new ManagedEndpointAllocationPersistenceError({
...@@ -344,22 +536,66 @@ export const make = Effect.gen(function* () {...@@ -344,22 +536,66 @@ export const make = Effect.gen(function* () {
344 );536 );
345 return claimed;537 return claimed;
346 }),538 }),
539 withClaimedTunnel: Effect.fn("relay.managed_endpoint_allocations.with_claimed_tunnel")(
540 function* <A, E, R>(
541 input: ClaimManagedEndpointReleaseInput,
542 effect: Effect.Effect<A, E, R>,
543 ): Effect.fn.Return<Option.Option<A>, E | ManagedEndpointAllocationPersistenceError, R> {
544 const lockError = (cause: unknown) =>
545 new ManagedEndpointAllocationPersistenceError({
546 operation: "lock-tunnel",
547 stage: "database-request",
548 userId: input.userId,
549 environmentId: input.environmentId,
550 tunnelId: input.tunnelId,
551 cause,
552 });
553 return yield* db.$client
554 .withTransaction(
555 db
556 .select({ generation: relayManagedEndpointAllocations.generation })
557 .from(relayManagedEndpointAllocations)
558 .where(
559 and(
560 whereAllocation(input),
561 eq(relayManagedEndpointAllocations.tunnelId, input.tunnelId),
562 eq(relayManagedEndpointAllocations.generation, input.generation),
563 ),
564 )
565 .limit(1)
566 .for("update")
567 .pipe(
568 Effect.mapError(lockError),
569 Effect.flatMap((rows) =>
570 rows.length === 0
571 ? Effect.succeed(Option.none())
572 : effect.pipe(Effect.map(Option.some)),
573 ),
574 ),
575 )
576 .pipe(
577 Effect.mapError((cause) => (SqlError.isSqlError(cause) ? lockError(cause) : cause)),
578 );
579 },
580 ),
347 claimDeprovision: Effect.fn("relay.managed_endpoint_allocations.claim_deprovision")(function* (581 claimDeprovision: Effect.fn("relay.managed_endpoint_allocations.claim_deprovision")(function* (
348 input: ClaimManagedEndpointDeprovisionInput,582 input: ClaimManagedEndpointDeprovisionInput,
349 ) {583 ) {
350 const claimedAt = DateTime.formatIso(yield* DateTime.now);
351 const claimed = yield* db584 const claimed = yield* db
352 .update(relayManagedEndpointAllocations)585 .update(relayManagedEndpointAllocations)
353 .set({ updatedAt: claimedAt })586 .set({
587 updatedAt: DateTime.formatIso(yield* DateTime.now),
588 generation: sql`${relayManagedEndpointAllocations.generation} + 1`,
589 })
354 .where(590 .where(
355 and(591 and(
356 whereAllocation(input),592 whereAllocation(input),
357 eq(relayManagedEndpointAllocations.updatedAt, input.updatedAt),593 eq(relayManagedEndpointAllocations.generation, input.generation),
358 ),594 ),
359 )595 )
360 .returning({ userId: relayManagedEndpointAllocations.userId })596 .returning({ generation: relayManagedEndpointAllocations.generation })
361 .pipe(597 .pipe(
362 Effect.map((rows) => rows.length > 0),598 Effect.map((rows) => rows[0]?.generation ?? null),
363 Effect.mapError(599 Effect.mapError(
364 (cause) =>600 (cause) =>
365 new ManagedEndpointAllocationPersistenceError({601 new ManagedEndpointAllocationPersistenceError({
...@@ -371,7 +607,7 @@ export const make = Effect.gen(function* () {...@@ -371,7 +607,7 @@ export const make = Effect.gen(function* () {
371 }),607 }),
372 ),608 ),
373 );609 );
374 return claimed ? claimedAt : null;610 return claimed;
375 }),611 }),
376 remove: Effect.fn("relay.managed_endpoint_allocations.remove")(function* (612 remove: Effect.fn("relay.managed_endpoint_allocations.remove")(function* (
377 input: ManagedEndpointAllocationKey,613 input: ManagedEndpointAllocationKey,
...@@ -399,7 +635,7 @@ export const make = Effect.gen(function* () {...@@ -399,7 +635,7 @@ export const make = Effect.gen(function* () {
399 .where(635 .where(
400 and(636 and(
401 whereAllocation(input),637 whereAllocation(input),
402 eq(relayManagedEndpointAllocations.updatedAt, input.updatedAt),638 eq(relayManagedEndpointAllocations.generation, input.generation),
403 ),639 ),
404 )640 )
405 .returning({ userId: relayManagedEndpointAllocations.userId })641 .returning({ userId: relayManagedEndpointAllocations.userId })
infra/relay/src/environments/ManagedEndpointProvider.test.ts+546-10
...@@ -6,6 +6,7 @@ import * as Alchemy from "alchemy";...@@ -6,6 +6,7 @@ import * as Alchemy from "alchemy";
6import * as Cloudflare from "alchemy/Cloudflare";6import * as Cloudflare from "alchemy/Cloudflare";
7import * as Effect from "effect/Effect";7import * as Effect from "effect/Effect";
8import * as Layer from "effect/Layer";8import * as Layer from "effect/Layer";
9import * as Option from "effect/Option";
9import * as Redacted from "effect/Redacted";10import * as Redacted from "effect/Redacted";
1011
11import * as RelayConfiguration from "../Config.ts";12import * as RelayConfiguration from "../Config.ts";
...@@ -33,7 +34,7 @@ const config = RelayConfiguration.RelayConfiguration.of({...@@ -33,7 +34,7 @@ const config = RelayConfiguration.RelayConfiguration.of({
33});34});
3435
35interface TunnelCall {36interface TunnelCall {
36 readonly operation: "list" | "create" | "putConfiguration" | "getToken" | "delete";37 readonly operation: "get" | "list" | "create" | "putConfiguration" | "getToken" | "delete";
37 readonly input: unknown;38 readonly input: unknown;
38}39}
3940
...@@ -62,6 +63,16 @@ function allocationKey(input: { readonly userId: string; readonly environmentId:...@@ -62,6 +63,16 @@ function allocationKey(input: { readonly userId: string; readonly environmentId:
6263
63function makeTunnelClient(calls: TunnelCall[] = []) {64function makeTunnelClient(calls: TunnelCall[] = []) {
64 return ManagedEndpointProvider.ManagedEndpointTunnelClient.of({65 return ManagedEndpointProvider.ManagedEndpointTunnelClient.of({
66 get: (tunnelId) =>
67 Effect.sync(() => {
68 calls.push({ operation: "get", input: tunnelId });
69 return {
70 id: tunnelId,
71 name: "managed-tunnel",
72 status: "down",
73 connsInactiveAt: "2026-06-01T00:00:00.000Z",
74 };
75 }),
65 list: (request) =>76 list: (request) =>
66 Effect.sync(() => {77 Effect.sync(() => {
67 calls.push({ operation: "list", input: request });78 calls.push({ operation: "list", input: request });
...@@ -91,6 +102,23 @@ function makeTunnelClient(calls: TunnelCall[] = []) {...@@ -91,6 +102,23 @@ function makeTunnelClient(calls: TunnelCall[] = []) {
91function makePersistentTunnelClient(calls: TunnelCall[] = []) {102function makePersistentTunnelClient(calls: TunnelCall[] = []) {
92 let tunnel: { readonly id: string; readonly name: string } | null = null;103 let tunnel: { readonly id: string; readonly name: string } | null = null;
93 return ManagedEndpointProvider.ManagedEndpointTunnelClient.of({104 return ManagedEndpointProvider.ManagedEndpointTunnelClient.of({
105 get: (tunnelId) =>
106 Effect.suspend(() => {
107 calls.push({ operation: "get", input: tunnelId });
108 return tunnel === null
109 ? Effect.fail(
110 new ManagedEndpointProvider.ManagedEndpointTunnelClientError({
111 operation: "get",
112 tunnelId,
113 cause: { _tag: "NotFound" },
114 }),
115 )
116 : Effect.succeed({
117 ...tunnel,
118 status: "down",
119 connsInactiveAt: "2026-06-01T00:00:00.000Z",
120 });
121 }),
94 list: (request) =>122 list: (request) =>
95 Effect.sync(() => {123 Effect.sync(() => {
96 calls.push({ operation: "list", input: request });124 calls.push({ operation: "list", input: request });
...@@ -159,6 +187,7 @@ function makeDnsClient(...@@ -159,6 +187,7 @@ function makeDnsClient(
159187
160function makeAllocations(calls: AllocationCall[] = []) {188function makeAllocations(calls: AllocationCall[] = []) {
161 const allocations = new Map<string, ManagedEndpointAllocations.ManagedEndpointAllocation>();189 const allocations = new Map<string, ManagedEndpointAllocations.ManagedEndpointAllocation>();
190 const recoveryEnabled = new Set<string>();
162 let generation = 0;191 let generation = 0;
163 const mutate = (192 const mutate = (
164 key: string,193 key: string,
...@@ -168,7 +197,11 @@ function makeAllocations(calls: AllocationCall[] = []) {...@@ -168,7 +197,11 @@ function makeAllocations(calls: AllocationCall[] = []) {
168 ) => {197 ) => {
169 const allocation = allocations.get(key);198 const allocation = allocations.get(key);
170 if (allocation !== undefined) {199 if (allocation !== undefined) {
171 allocations.set(key, { ...change(allocation), updatedAt: `generation-${++generation}` });200 allocations.set(key, {
201 ...change(allocation),
202 generation: allocation.generation + 1,
203 updatedAt: `generation-${++generation}`,
204 });
172 }205 }
173 };206 };
174 return ManagedEndpointAllocations.ManagedEndpointAllocations.of({207 return ManagedEndpointAllocations.ManagedEndpointAllocations.of({
...@@ -185,7 +218,9 @@ function makeAllocations(calls: AllocationCall[] = []) {...@@ -185,7 +218,9 @@ function makeAllocations(calls: AllocationCall[] = []) {
185 tunnelId: null,218 tunnelId: null,
186 dnsRecordId: null,219 dnsRecordId: null,
187 readyAt: null,220 readyAt: null,
221 origin: null,
188 updatedAt: `generation-${++generation}`,222 updatedAt: `generation-${++generation}`,
223 generation: 0,
189 };224 };
190 allocations.set(allocationKey(input), allocation);225 allocations.set(allocationKey(input), allocation);
191 return allocation;226 return allocation;
...@@ -193,27 +228,62 @@ function makeAllocations(calls: AllocationCall[] = []) {...@@ -193,27 +228,62 @@ function makeAllocations(calls: AllocationCall[] = []) {
193 recordTunnel: (input) =>228 recordTunnel: (input) =>
194 Effect.sync(() => {229 Effect.sync(() => {
195 calls.push({ operation: "recordTunnel", input });230 calls.push({ operation: "recordTunnel", input });
231 const current = allocations.get(allocationKey(input));
232 if (current?.generation !== input.generation) {
233 return null;
234 }
196 mutate(allocationKey(input), (allocation) => ({235 mutate(allocationKey(input), (allocation) => ({
197 ...allocation,236 ...allocation,
198 tunnelId: input.tunnelId,237 tunnelId: input.tunnelId,
238 readyAt: allocation.tunnelId === input.tunnelId ? allocation.readyAt : null,
199 }));239 }));
240 return allocations.get(allocationKey(input))?.generation ?? null;
200 }),241 }),
201 recordDns: (input) =>242 recordDns: (input) =>
202 Effect.sync(() => {243 Effect.sync(() => {
203 calls.push({ operation: "recordDns", input });244 calls.push({ operation: "recordDns", input });
245 const current = allocations.get(allocationKey(input));
246 if (current?.generation !== input.generation || current.tunnelId !== input.tunnelId) {
247 return null;
248 }
204 mutate(allocationKey(input), (allocation) => ({249 mutate(allocationKey(input), (allocation) => ({
205 ...allocation,250 ...allocation,
206 dnsRecordId: input.dnsRecordId,251 dnsRecordId: input.dnsRecordId,
207 }));252 }));
253 return allocations.get(allocationKey(input))?.generation ?? null;
208 }),254 }),
209 markReady: (input) =>255 markReady: (input) =>
210 Effect.sync(() => {256 Effect.sync(() => {
211 calls.push({ operation: "markReady", input });257 calls.push({ operation: "markReady", input });
258 const current = allocations.get(allocationKey(input));
259 if (current?.generation !== input.generation || current.tunnelId !== input.tunnelId) {
260 return false;
261 }
212 mutate(allocationKey(input), (allocation) => ({262 mutate(allocationKey(input), (allocation) => ({
213 ...allocation,263 ...allocation,
214 readyAt: "2026-06-02T00:00:00.000Z",264 readyAt: "2026-06-02T00:00:00.000Z",
265 origin: input.origin,
215 }));266 }));
267 return true;
268 }),
269 enableRecovery: (input) =>
270 Effect.sync(() => {
271 const allocation = allocations.get(allocationKey(input));
272 if (allocation?.tunnelId !== input.tunnelId) {
273 return false;
274 }
275 recoveryEnabled.add(allocationKey(input));
276 return true;
216 }),277 }),
278 listByTunnelNames: (tunnelNames) =>
279 Effect.sync(() =>
280 [...allocations.values()]
281 .filter((allocation) => tunnelNames.includes(allocation.tunnelName))
282 .map((allocation) => ({
283 ...allocation,
284 recoveryEnabled: recoveryEnabled.has(allocationKey(allocation)),
285 })),
286 ),
217 claimRelease: (input) =>287 claimRelease: (input) =>
218 Effect.sync(() => {288 Effect.sync(() => {
219 calls.push({ operation: "claimRelease", input });289 calls.push({ operation: "claimRelease", input });
...@@ -221,22 +291,29 @@ function makeAllocations(calls: AllocationCall[] = []) {...@@ -221,22 +291,29 @@ function makeAllocations(calls: AllocationCall[] = []) {
221 if (291 if (
222 allocation === undefined ||292 allocation === undefined ||
223 allocation.tunnelId !== input.tunnelId ||293 allocation.tunnelId !== input.tunnelId ||
224 allocation.updatedAt !== input.updatedAt294 allocation.generation !== input.generation
225 ) {295 ) {
226 return false;296 return null;
227 }297 }
228 mutate(allocationKey(input), (current) => current);298 mutate(allocationKey(input), (current) => current);
229 return true;299 return allocations.get(allocationKey(input))?.generation ?? null;
300 }),
301 withClaimedTunnel: (input, effect) =>
302 Effect.suspend(() => {
303 const current = allocations.get(allocationKey(input));
304 return current?.tunnelId === input.tunnelId && current.generation === input.generation
305 ? effect.pipe(Effect.map(Option.some))
306 : Effect.succeed(Option.none());
230 }),307 }),
231 claimDeprovision: (input) =>308 claimDeprovision: (input) =>
232 Effect.sync(() => {309 Effect.sync(() => {
233 calls.push({ operation: "claimDeprovision", input });310 calls.push({ operation: "claimDeprovision", input });
234 const allocation = allocations.get(allocationKey(input));311 const allocation = allocations.get(allocationKey(input));
235 if (allocation === undefined || allocation.updatedAt !== input.updatedAt) {312 if (allocation === undefined || allocation.generation !== input.generation) {
236 return null;313 return null;
237 }314 }
238 mutate(allocationKey(input), (current) => current);315 mutate(allocationKey(input), (current) => current);
239 return allocations.get(allocationKey(input))?.updatedAt ?? null;316 return allocations.get(allocationKey(input))?.generation ?? null;
240 }),317 }),
241 remove: (input) =>318 remove: (input) =>
242 Effect.sync(() => {319 Effect.sync(() => {
...@@ -247,7 +324,7 @@ function makeAllocations(calls: AllocationCall[] = []) {...@@ -247,7 +324,7 @@ function makeAllocations(calls: AllocationCall[] = []) {
247 Effect.sync(() => {324 Effect.sync(() => {
248 calls.push({ operation: "removeClaimed", input });325 calls.push({ operation: "removeClaimed", input });
249 const allocation = allocations.get(allocationKey(input));326 const allocation = allocations.get(allocationKey(input));
250 if (allocation === undefined || allocation.updatedAt !== input.updatedAt) {327 if (allocation === undefined || allocation.generation !== input.generation) {
251 return false;328 return false;
252 }329 }
253 allocations.delete(allocationKey(input));330 allocations.delete(allocationKey(input));
...@@ -306,6 +383,7 @@ function expectedManagedTunnelName(environmentId: string, userId = "user_ABC"):...@@ -306,6 +383,7 @@ function expectedManagedTunnelName(environmentId: string, userId = "user_ABC"):
306describe("ManagedEndpointProvider", () => {383describe("ManagedEndpointProvider", () => {
307 it.effect("does not require the deployment RuntimeContext when building the Worker layer", () => {384 it.effect("does not require the deployment RuntimeContext when building the Worker layer", () => {
308 const tunnelClient = {385 const tunnelClient = {
386 get: () => Effect.succeed({ id: "tunnel-id", name: "managed-tunnel" }),
309 list: () => Effect.succeed({ result: [] }),387 list: () => Effect.succeed({ result: [] }),
310 create: (request: { readonly name: string }) =>388 create: (request: { readonly name: string }) =>
311 Effect.succeed({ id: "tunnel-id", name: request.name }),389 Effect.succeed({ id: "tunnel-id", name: request.name }),
...@@ -731,7 +809,8 @@ describe("ManagedEndpointProvider", () => {...@@ -731,7 +809,8 @@ describe("ManagedEndpointProvider", () => {
731 }).pipe(Effect.andThen(Effect.fail(failure))),809 }).pipe(Effect.andThen(Effect.fail(failure))),
732 deleteRecord: () => Effect.void,810 deleteRecord: () => Effect.void,
733 });811 });
734 const layer = providerLayer(makePersistentTunnelClient(), dnsClient, makeAllocations());812 const allocations = makeAllocations();
813 const layer = providerLayer(makePersistentTunnelClient(), dnsClient, allocations);
735814
736 return Effect.gen(function* () {815 return Effect.gen(function* () {
737 const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;816 const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
...@@ -754,6 +833,10 @@ describe("ManagedEndpointProvider", () => {...@@ -754,6 +833,10 @@ describe("ManagedEndpointProvider", () => {
754 "createRecord",833 "createRecord",
755 "updateRecord",834 "updateRecord",
756 ]);835 ]);
836 expect(yield* allocations.get(request)).toMatchObject({
837 tunnelId: "tunnel-id",
838 readyAt: "2026-06-02T00:00:00.000Z",
839 });
757 }).pipe(Effect.provide(layer));840 }).pipe(Effect.provide(layer));
758 });841 });
759842
...@@ -914,7 +997,7 @@ describe("ManagedEndpointProvider", () => {...@@ -914,7 +997,7 @@ describe("ManagedEndpointProvider", () => {
914 // longer matches what the release loaded, so the claim fails.997 // longer matches what the release loaded, so the claim fails.
915 const outdated = ManagedEndpointAllocations.ManagedEndpointAllocations.of({998 const outdated = ManagedEndpointAllocations.ManagedEndpointAllocations.of({
916 ...allocations,999 ...allocations,
917 claimRelease: () => Effect.succeed(false),1000 claimRelease: () => Effect.succeed(null),
918 });1001 });
919 const layer = providerLayer(makePersistentTunnelClient(tunnelCalls), makeDnsClient(), outdated);1002 const layer = providerLayer(makePersistentTunnelClient(tunnelCalls), makeDnsClient(), outdated);
9201003
...@@ -939,6 +1022,459 @@ describe("ManagedEndpointProvider", () => {...@@ -939,6 +1022,459 @@ describe("ManagedEndpointProvider", () => {
939 }).pipe(Effect.provide(layer));1022 }).pipe(Effect.provide(layer));
940 });1023 });
9411024
1025 it.effect("does not release a tunnel when the requested tunnel id is outdated", () => {
1026 const tunnelCalls: TunnelCall[] = [];
1027 const layer = providerLayer(
1028 makePersistentTunnelClient(tunnelCalls),
1029 makeDnsClient(),
1030 makeAllocations(),
1031 );
1032
1033 return Effect.gen(function* () {
1034 const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
1035 const key = { userId: "user_ABC", environmentId: "env_ABC" } as const;
1036 yield* provider.provision({
1037 ...key,
1038 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
1039 });
1040
1041 expect(yield* provider.release({ ...key, expectedTunnelId: "old-tunnel-id" })).toBe(false);
1042 expect(tunnelCalls.map((call) => call.operation)).not.toContain("delete");
1043 }).pipe(Effect.provide(layer));
1044 });
1045
1046 it.effect("keeps a tunnel when a provision replaces an ordinary release generation", () => {
1047 const tunnelCalls: TunnelCall[] = [];
1048 const allocations = makeAllocations();
1049 let replaceAfterClaim = false;
1050 const replaced = ManagedEndpointAllocations.ManagedEndpointAllocations.of({
1051 ...allocations,
1052 claimRelease: (input) =>
1053 allocations.claimRelease(input).pipe(
1054 Effect.tap((claimedGeneration) => {
1055 if (claimedGeneration === null || replaceAfterClaim) return Effect.void;
1056 replaceAfterClaim = true;
1057 return allocations
1058 .recordTunnel({
1059 userId: input.userId,
1060 environmentId: input.environmentId,
1061 tunnelId: "replacement-tunnel",
1062 generation: claimedGeneration,
1063 })
1064 .pipe(Effect.asVoid);
1065 }),
1066 ),
1067 });
1068 const layer = providerLayer(makePersistentTunnelClient(tunnelCalls), makeDnsClient(), replaced);
1069
1070 return Effect.gen(function* () {
1071 const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
1072 const key = { userId: "user_ABC", environmentId: "env_ABC" } as const;
1073 yield* provider.provision({
1074 ...key,
1075 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
1076 });
1077
1078 expect(yield* provider.release(key)).toBe(false);
1079 expect(tunnelCalls.map((call) => call.operation)).not.toContain("delete");
1080 }).pipe(Effect.provide(layer));
1081 });
1082
1083 it.effect("does no Cloudflare work when the registered origin is unchanged", () => {
1084 const tunnelCalls: TunnelCall[] = [];
1085 const layer = providerLayer(makePersistentTunnelClient(tunnelCalls));
1086
1087 return Effect.gen(function* () {
1088 const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
1089 const key = { userId: "user_ABC", environmentId: "env_ABC" } as const;
1090 const origin = { localHttpHost: "127.0.0.1", localHttpPort: 3773 } as const;
1091 const provisioned = yield* provider.provision({ ...key, origin });
1092 tunnelCalls.length = 0;
1093
1094 expect(
1095 yield* provider.reconcileOrigin({
1096 ...key,
1097 tunnelId: provisioned.runtime.tunnelId!,
1098 origin,
1099 endpoint: provisioned.endpoint,
1100 }),
1101 ).toBe("ready");
1102 expect(tunnelCalls).toEqual([]);
1103 }).pipe(Effect.provide(layer));
1104 });
1105
1106 it.effect("updates Cloudflare ingress once when the registered port changes", () => {
1107 const tunnelCalls: TunnelCall[] = [];
1108 const layer = providerLayer(makePersistentTunnelClient(tunnelCalls));
1109
1110 return Effect.gen(function* () {
1111 const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
1112 const key = { userId: "user_ABC", environmentId: "env_ABC" } as const;
1113 const provisioned = yield* provider.provision({
1114 ...key,
1115 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
1116 });
1117 tunnelCalls.length = 0;
1118
1119 expect(
1120 yield* provider.reconcileOrigin({
1121 ...key,
1122 tunnelId: provisioned.runtime.tunnelId!,
1123 origin: { localHttpHost: "127.0.0.1", localHttpPort: 4884 },
1124 endpoint: provisioned.endpoint,
1125 }),
1126 ).toBe("ready");
1127 expect(tunnelCalls).toEqual([
1128 {
1129 operation: "putConfiguration",
1130 input: {
1131 tunnelId: "tunnel-id",
1132 tunnelConfig: {
1133 ingress: [
1134 {
1135 hostname: expectedManagedHostname("env_ABC"),
1136 service: "http://127.0.0.1:4884",
1137 },
1138 { service: "http_status:404" },
1139 ],
1140 },
1141 },
1142 },
1143 ]);
1144 }).pipe(Effect.provide(layer));
1145 });
1146
1147 it.effect("requires recovery when Cloudflare reports the registered tunnel is missing", () => {
1148 const tunnelCalls: TunnelCall[] = [];
1149 const baseTunnelClient = makePersistentTunnelClient(tunnelCalls);
1150 let tunnelMissing = false;
1151 const tunnelClient = ManagedEndpointProvider.ManagedEndpointTunnelClient.of({
1152 ...baseTunnelClient,
1153 putConfiguration: (tunnelId, tunnelConfig) =>
1154 tunnelMissing
1155 ? Effect.fail(
1156 new ManagedEndpointProvider.ManagedEndpointTunnelClientError({
1157 operation: "put-configuration",
1158 tunnelId,
1159 cause: { _tag: "TunnelNotFound" },
1160 }),
1161 )
1162 : baseTunnelClient.putConfiguration(tunnelId, tunnelConfig),
1163 });
1164 const layer = providerLayer(tunnelClient);
1165
1166 return Effect.gen(function* () {
1167 const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
1168 const key = { userId: "user_ABC", environmentId: "env_ABC" } as const;
1169 const provisioned = yield* provider.provision({
1170 ...key,
1171 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
1172 });
1173 tunnelMissing = true;
1174
1175 expect(
1176 yield* provider.reconcileOrigin({
1177 ...key,
1178 tunnelId: provisioned.runtime.tunnelId!,
1179 origin: { localHttpHost: "127.0.0.1", localHttpPort: 4884 },
1180 endpoint: provisioned.endpoint,
1181 }),
1182 ).toBe("recovery_required");
1183 }).pipe(Effect.provide(layer));
1184 });
1185
1186 it.effect("fails origin sync when the allocation generation changes", () => {
1187 const allocations = makeAllocations();
1188 let loseClaim = false;
1189 const changed = ManagedEndpointAllocations.ManagedEndpointAllocations.of({
1190 ...allocations,
1191 withClaimedTunnel: (input, effect) =>
1192 loseClaim ? Effect.succeed(Option.none()) : allocations.withClaimedTunnel(input, effect),
1193 });
1194 const layer = providerLayer(makePersistentTunnelClient(), makeDnsClient(), changed);
1195
1196 return Effect.gen(function* () {
1197 const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
1198 const key = { userId: "user_ABC", environmentId: "env_ABC" } as const;
1199 const provisioned = yield* provider.provision({
1200 ...key,
1201 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
1202 });
1203 loseClaim = true;
1204
1205 const error = yield* Effect.flip(
1206 provider.reconcileOrigin({
1207 ...key,
1208 tunnelId: provisioned.runtime.tunnelId!,
1209 origin: { localHttpHost: "127.0.0.1", localHttpPort: 4884 },
1210 endpoint: provisioned.endpoint,
1211 }),
1212 );
1213 expect(error).toMatchObject({
1214 _tag: "ManagedEndpointProvisioningFailed",
1215 stage: "sync-origin",
1216 });
1217 }).pipe(Effect.provide(layer));
1218 });
1219
1220 it.effect("rejects an active endpoint that does not match the allocation hostname", () => {
1221 const layer = providerLayer(makePersistentTunnelClient());
1222
1223 return Effect.gen(function* () {
1224 const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
1225 const key = { userId: "user_ABC", environmentId: "env_ABC" } as const;
1226 const provisioned = yield* provider.provision({
1227 ...key,
1228 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
1229 });
1230
1231 const error = yield* Effect.flip(
1232 provider.reconcileOrigin({
1233 ...key,
1234 tunnelId: provisioned.runtime.tunnelId!,
1235 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
1236 endpoint: {
1237 ...provisioned.endpoint,
1238 httpBaseUrl: "https://different-host.t3code.test/",
1239 },
1240 }),
1241 );
1242 expect(error).toMatchObject({
1243 _tag: "ManagedEndpointProvisioningFailed",
1244 stage: "verify-endpoint",
1245 });
1246 }).pipe(Effect.provide(layer));
1247 });
1248
1249 it.effect(
1250 "keeps a newly created tunnel available for retry after losing its allocation claim",
1251 () => {
1252 const tunnelCalls: TunnelCall[] = [];
1253 const allocations = makeAllocations();
1254 let changeGeneration = true;
1255 const changed = ManagedEndpointAllocations.ManagedEndpointAllocations.of({
1256 ...allocations,
1257 recordTunnel: (input) =>
1258 Effect.gen(function* () {
1259 if (changeGeneration) {
1260 changeGeneration = false;
1261 yield* allocations.claimDeprovision(input);
1262 }
1263 return yield* allocations.recordTunnel(input);
1264 }),
1265 });
1266 const layer = providerLayer(
1267 makePersistentTunnelClient(tunnelCalls),
1268 makeDnsClient(),
1269 changed,
1270 );
1271
1272 return Effect.gen(function* () {
1273 const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
1274 const input = {
1275 userId: "user_ABC",
1276 environmentId: "env_ABC",
1277 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
1278 };
1279 const error = yield* Effect.flip(provider.provision(input));
1280
1281 expect(error).toMatchObject({
1282 _tag: "ManagedEndpointProvisioningFailed",
1283 stage: "record-tunnel",
1284 });
1285 expect(tunnelCalls.map((call) => call.operation)).toEqual(["list", "create"]);
1286 expect((yield* provider.provision(input)).runtime.tunnelId).toBe("tunnel-id");
1287 expect(tunnelCalls.filter((call) => call.operation === "create")).toHaveLength(1);
1288 expect(tunnelCalls.filter((call) => call.operation === "delete")).toEqual([]);
1289 }).pipe(Effect.provide(layer));
1290 },
1291 );
1292
1293 it.effect("does not overwrite DNS when tunnel ownership changes during provisioning", () => {
1294 const allocations = makeAllocations();
1295 const changed = ManagedEndpointAllocations.ManagedEndpointAllocations.of({
1296 ...allocations,
1297 recordDns: () => Effect.succeed(null),
1298 });
1299 const layer = providerLayer(makePersistentTunnelClient(), makeDnsClient(), changed);
1300
1301 return Effect.gen(function* () {
1302 const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
1303 const error = yield* Effect.flip(
1304 provider.provision({
1305 userId: "user_ABC",
1306 environmentId: "env_ABC",
1307 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
1308 }),
1309 );
1310
1311 expect(error).toMatchObject({
1312 _tag: "ManagedEndpointProvisioningFailed",
1313 stage: "record-dns",
1314 });
1315 }).pipe(Effect.provide(layer));
1316 });
1317
1318 it.effect("does not change tunnel ingress after another provision takes ownership", () => {
1319 const tunnelCalls: TunnelCall[] = [];
1320 const allocations = makeAllocations();
1321 const changed = ManagedEndpointAllocations.ManagedEndpointAllocations.of({
1322 ...allocations,
1323 withClaimedTunnel: () => Effect.succeed(Option.none()),
1324 });
1325 const layer = providerLayer(makePersistentTunnelClient(tunnelCalls), makeDnsClient(), changed);
1326
1327 return Effect.gen(function* () {
1328 const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
1329 const error = yield* Effect.flip(
1330 provider.provision({
1331 userId: "user_ABC",
1332 environmentId: "env_ABC",
1333 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
1334 }),
1335 );
1336
1337 expect(error).toMatchObject({
1338 _tag: "ManagedEndpointProvisioningFailed",
1339 stage: "configure-tunnel",
1340 });
1341 expect(tunnelCalls.map((call) => call.operation)).not.toContain("putConfiguration");
1342 }).pipe(Effect.provide(layer));
1343 });
1344
1345 it.effect("does not change DNS after another provision takes ownership", () => {
1346 const dnsCalls: DnsCall[] = [];
1347 const allocations = makeAllocations();
1348 let lockCount = 0;
1349 const changed = ManagedEndpointAllocations.ManagedEndpointAllocations.of({
1350 ...allocations,
1351 withClaimedTunnel: (input, effect) =>
1352 ++lockCount === 1
1353 ? allocations.withClaimedTunnel(input, effect)
1354 : Effect.succeed(Option.none()),
1355 });
1356 const layer = providerLayer(makePersistentTunnelClient(), makeDnsClient(dnsCalls), changed);
1357
1358 return Effect.gen(function* () {
1359 const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
1360 const error = yield* Effect.flip(
1361 provider.provision({
1362 userId: "user_ABC",
1363 environmentId: "env_ABC",
1364 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
1365 }),
1366 );
1367
1368 expect(error).toMatchObject({
1369 _tag: "ManagedEndpointProvisioningFailed",
1370 stage: "record-dns",
1371 });
1372 expect(dnsCalls).toEqual([]);
1373 }).pipe(Effect.provide(layer));
1374 });
1375
1376 it.effect("does not mark a superseded tunnel allocation as ready", () => {
1377 const allocations = makeAllocations();
1378 const changed = ManagedEndpointAllocations.ManagedEndpointAllocations.of({
1379 ...allocations,
1380 markReady: () => Effect.succeed(false),
1381 });
1382 const layer = providerLayer(makePersistentTunnelClient(), makeDnsClient(), changed);
1383
1384 return Effect.gen(function* () {
1385 const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
1386 const error = yield* Effect.flip(
1387 provider.provision({
1388 userId: "user_ABC",
1389 environmentId: "env_ABC",
1390 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
1391 }),
1392 );
1393
1394 expect(error).toMatchObject({
1395 _tag: "ManagedEndpointProvisioningFailed",
1396 stage: "mark-allocation-ready",
1397 });
1398 }).pipe(Effect.provide(layer));
1399 });
1400
1401 it.effect("keeps a tunnel that reconnects before scheduled deletion", () => {
1402 const tunnelCalls: TunnelCall[] = [];
1403 const tunnelClient = ManagedEndpointProvider.ManagedEndpointTunnelClient.of({
1404 ...makePersistentTunnelClient(tunnelCalls),
1405 get: (tunnelId) =>
1406 Effect.succeed({
1407 id: tunnelId,
1408 name: expectedManagedTunnelName("env_ABC"),
1409 status: "healthy",
1410 connsInactiveAt: null,
1411 }),
1412 });
1413 const layer = providerLayer(tunnelClient, makeDnsClient(), makeAllocations());
1414
1415 return Effect.gen(function* () {
1416 const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
1417 const key = { userId: "user_ABC", environmentId: "env_ABC" } as const;
1418 yield* provider.provision({
1419 ...key,
1420 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
1421 });
1422
1423 expect(
1424 yield* provider.release({
1425 ...key,
1426 expectedTunnelId: "tunnel-id",
1427 expectedStatus: "down",
1428 expectedInactiveBefore: "2026-06-01T00:05:00.000Z",
1429 }),
1430 ).toBe(false);
1431 expect(tunnelCalls.map((call) => call.operation)).not.toContain("delete");
1432 }).pipe(Effect.provide(layer));
1433 });
1434
1435 it.effect("keeps a tunnel when a new provision replaces the release generation", () => {
1436 const tunnelCalls: TunnelCall[] = [];
1437 const allocations = makeAllocations();
1438 const replaced = ManagedEndpointAllocations.ManagedEndpointAllocations.of({
1439 ...allocations,
1440 claimRelease: (input) =>
1441 allocations.claimRelease(input).pipe(
1442 Effect.tap((claimedGeneration) =>
1443 claimedGeneration === null
1444 ? Effect.void
1445 : allocations
1446 .recordTunnel({
1447 userId: input.userId,
1448 environmentId: input.environmentId,
1449 tunnelId: "replacement-tunnel",
1450 generation: claimedGeneration,
1451 })
1452 .pipe(Effect.asVoid),
1453 ),
1454 ),
1455 });
1456 const layer = providerLayer(makePersistentTunnelClient(tunnelCalls), makeDnsClient(), replaced);
1457
1458 return Effect.gen(function* () {
1459 const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
1460 const key = { userId: "user_ABC", environmentId: "env_ABC" } as const;
1461 yield* provider.provision({
1462 ...key,
1463 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
1464 });
1465
1466 expect(
1467 yield* provider.release({
1468 ...key,
1469 expectedTunnelId: "tunnel-id",
1470 expectedStatus: "down",
1471 expectedInactiveBefore: "2026-06-01T00:05:00.000Z",
1472 }),
1473 ).toBe(false);
1474 expect(tunnelCalls.map((call) => call.operation)).not.toContain("delete");
1475 }).pipe(Effect.provide(layer));
1476 });
1477
942 it.effect("treats an already deleted tunnel as successfully released", () => {1478 it.effect("treats an already deleted tunnel as successfully released", () => {
943 const notFound = { _tag: "NotFound" } as const;1479 const notFound = { _tag: "NotFound" } as const;
944 const tunnelClient = ManagedEndpointProvider.ManagedEndpointTunnelClient.of({1480 const tunnelClient = ManagedEndpointProvider.ManagedEndpointTunnelClient.of({
infra/relay/src/environments/ManagedEndpointProvider.ts+545-119
...@@ -3,6 +3,7 @@ import * as Cloudflare from "alchemy/Cloudflare";...@@ -3,6 +3,7 @@ import * as Cloudflare from "alchemy/Cloudflare";
3import * as Arr from "effect/Array";3import * as Arr from "effect/Array";
4import * as Context from "effect/Context";4import * as Context from "effect/Context";
5import * as Crypto from "effect/Crypto";5import * as Crypto from "effect/Crypto";
6import * as DateTime from "effect/DateTime";
6import * as Effect from "effect/Effect";7import * as Effect from "effect/Effect";
7import * as Encoding from "effect/Encoding";8import * as Encoding from "effect/Encoding";
8import * as Layer from "effect/Layer";9import * as Layer from "effect/Layer";
...@@ -52,6 +53,9 @@ const ManagedEndpointProvisioningStage = Schema.Literals([...@@ -52,6 +53,9 @@ const ManagedEndpointProvisioningStage = Schema.Literals([
52 "record-dns",53 "record-dns",
53 "get-tunnel-token",54 "get-tunnel-token",
54 "mark-allocation-ready",55 "mark-allocation-ready",
56 "load-allocation",
57 "verify-endpoint",
58 "sync-origin",
55]);59]);
5660
57export class ManagedEndpointProvisioningFailed extends Schema.TaggedError<ManagedEndpointProvisioningFailed>()(61export class ManagedEndpointProvisioningFailed extends Schema.TaggedError<ManagedEndpointProvisioningFailed>()(
...@@ -76,6 +80,7 @@ export class ManagedEndpointProvisioningFailed extends Schema.TaggedError<Manage...@@ -76,6 +80,7 @@ export class ManagedEndpointProvisioningFailed extends Schema.TaggedError<Manage
7680
77const ManagedEndpointDeprovisioningStage = Schema.Literals([81const ManagedEndpointDeprovisioningStage = Schema.Literals([
78 "load-allocation",82 "load-allocation",
83 "load-tunnel",
79 "claim-release",84 "claim-release",
80 "claim-deprovision",85 "claim-deprovision",
81 "delete-dns-record",86 "delete-dns-record",
...@@ -124,6 +129,8 @@ export interface ManagedEndpointProvisioningResult {...@@ -124,6 +129,8 @@ export interface ManagedEndpointProvisioningResult {
124 readonly runtime: RelayManagedEndpointRuntimeConfig;129 readonly runtime: RelayManagedEndpointRuntimeConfig;
125}130}
126131
132export type ManagedEndpointOriginSyncResult = "ready" | "recovery_required";
133
127export type ManagedEndpointDeprovisionTarget = ManagedEndpointAllocations.ManagedEndpointAllocation;134export type ManagedEndpointDeprovisionTarget = ManagedEndpointAllocations.ManagedEndpointAllocation;
128135
129export class ManagedEndpointProvider extends Context.Service<136export class ManagedEndpointProvider extends Context.Service<
...@@ -134,6 +141,13 @@ export class ManagedEndpointProvider extends Context.Service<...@@ -134,6 +141,13 @@ export class ManagedEndpointProvider extends Context.Service<
134 readonly environmentId: string;141 readonly environmentId: string;
135 readonly origin: RelayManagedEndpointOrigin;142 readonly origin: RelayManagedEndpointOrigin;
136 }) => Effect.Effect<ManagedEndpointProvisioningResult, ManagedEndpointProviderError>;143 }) => Effect.Effect<ManagedEndpointProvisioningResult, ManagedEndpointProviderError>;
144 readonly reconcileOrigin: (input: {
145 readonly userId: string;
146 readonly environmentId: string;
147 readonly tunnelId: string;
148 readonly origin: RelayManagedEndpointOrigin;
149 readonly endpoint: RelayManagedEndpoint;
150 }) => Effect.Effect<ManagedEndpointOriginSyncResult, ManagedEndpointProviderError>;
137 /**151 /**
138 * Captures the allocation generation owned by an unlink before its link152 * Captures the allocation generation owned by an unlink before its link
139 * revocation commits. Passing this target to `deprovision` prevents a153 * revocation commits. Passing this target to `deprovision` prevents a
...@@ -150,7 +164,7 @@ export class ManagedEndpointProvider extends Context.Service<...@@ -150,7 +164,7 @@ export class ManagedEndpointProvider extends Context.Service<
150 readonly userId: string;164 readonly userId: string;
151 readonly environmentId: string;165 readonly environmentId: string;
152 readonly target?: ManagedEndpointDeprovisionTarget | null;166 readonly target?: ManagedEndpointDeprovisionTarget | null;
153 }) => Effect.Effect<void, ManagedEndpointDeprovisioningFailed>;167 }) => Effect.Effect<boolean, ManagedEndpointDeprovisioningFailed>;
154 /**168 /**
155 * Deletes the provisioned Cloudflare tunnel while keeping the allocation169 * Deletes the provisioned Cloudflare tunnel while keeping the allocation
156 * (hostname + tunnel name reservation) and DNS record. Cloudflare bills per170 * (hostname + tunnel name reservation) and DNS record. Cloudflare bills per
...@@ -166,16 +180,34 @@ export class ManagedEndpointProvider extends Context.Service<...@@ -166,16 +180,34 @@ export class ManagedEndpointProvider extends Context.Service<
166 readonly release: (input: {180 readonly release: (input: {
167 readonly userId: string;181 readonly userId: string;
168 readonly environmentId: string;182 readonly environmentId: string;
183 readonly expectedTunnelId?: string;
184 readonly expectedInactiveBefore?: string;
185 readonly expectedStatus?: "inactive" | "down";
169 }) => Effect.Effect<boolean, ManagedEndpointDeprovisioningFailed>;186 }) => Effect.Effect<boolean, ManagedEndpointDeprovisioningFailed>;
170 }187 }
171>()("t3code-relay/environments/ManagedEndpointProvider") {}188>()("t3code-relay/environments/ManagedEndpointProvider") {}
172189
173interface ManagedEndpointTunnel {190export interface ManagedEndpointTunnel {
174 readonly id?: string | null;191 readonly id?: string | null;
175 readonly name?: string | null;192 readonly name?: string | null;
193 readonly status?: string | null;
194 readonly createdAt?: string | null;
195 readonly connsInactiveAt?: string | null;
196}
197
198export interface ManagedEndpointTunnelListRequest {
199 readonly isDeleted: false;
200 readonly name?: string;
201 readonly includePrefix?: string;
202 readonly status?: "inactive" | "down";
203 readonly existedAt?: string;
204 readonly wasInactiveAt?: string;
205 readonly page?: number;
206 readonly perPage?: number;
176}207}
177208
178const ManagedEndpointTunnelClientOperation = Schema.Literals([209const ManagedEndpointTunnelClientOperation = Schema.Literals([
210 "get",
179 "list",211 "list",
180 "create",212 "create",
181 "put-configuration",213 "put-configuration",
...@@ -201,11 +233,18 @@ export class ManagedEndpointTunnelClientError extends Schema.TaggedError<Managed...@@ -201,11 +233,18 @@ export class ManagedEndpointTunnelClientError extends Schema.TaggedError<Managed
201export class ManagedEndpointTunnelClient extends Context.Service<233export class ManagedEndpointTunnelClient extends Context.Service<
202 ManagedEndpointTunnelClient,234 ManagedEndpointTunnelClient,
203 {235 {
204 readonly list: (request: {236 readonly get: (
205 readonly name: string;237 tunnelId: string,
206 readonly isDeleted: false;238 ) => Effect.Effect<ManagedEndpointTunnel, ManagedEndpointTunnelClientError>;
207 }) => Effect.Effect<239 readonly list: (request: ManagedEndpointTunnelListRequest) => Effect.Effect<
208 { readonly result: ReadonlyArray<ManagedEndpointTunnel> },240 {
241 readonly result: ReadonlyArray<ManagedEndpointTunnel>;
242 readonly resultInfo?: {
243 readonly page?: number | null;
244 readonly perPage?: number | null;
245 readonly totalCount?: number | null;
246 } | null;
247 },
209 ManagedEndpointTunnelClientError248 ManagedEndpointTunnelClientError
210 >;249 >;
211 readonly create: (request: {250 readonly create: (request: {
...@@ -333,17 +372,17 @@ function isLoopbackOrigin(origin: RelayManagedEndpointOrigin): boolean {...@@ -333,17 +372,17 @@ function isLoopbackOrigin(origin: RelayManagedEndpointOrigin): boolean {
333 );372 );
334}373}
335374
336function isNotFoundCause(cause: unknown): boolean {375export function isManagedEndpointNotFound(cause: unknown): boolean {
337 if (typeof cause !== "object" || cause === null) {376 if (typeof cause !== "object" || cause === null) {
338 return false;377 return false;
339 }378 }
340 if ("_tag" in cause && cause._tag === "NotFound") {379 if ("_tag" in cause && (cause._tag === "NotFound" || cause._tag === "TunnelNotFound")) {
341 return true;380 return true;
342 }381 }
343 if ("status" in cause && cause.status === 404) {382 if ("status" in cause && cause.status === 404) {
344 return true;383 return true;
345 }384 }
346 return "cause" in cause && isNotFoundCause(cause.cause);385 return "cause" in cause && isManagedEndpointNotFound(cause.cause);
347}386}
348387
349type ManagedEndpointClientError = ManagedEndpointTunnelClientError | ManagedEndpointDnsClientError;388type ManagedEndpointClientError = ManagedEndpointTunnelClientError | ManagedEndpointDnsClientError;
...@@ -355,9 +394,9 @@ const ignoreNotFound = <A>(...@@ -355,9 +394,9 @@ const ignoreNotFound = <A>(
355 Effect.asVoid,394 Effect.asVoid,
356 Effect.catchTags({395 Effect.catchTags({
357 ManagedEndpointTunnelClientError: (error) =>396 ManagedEndpointTunnelClientError: (error) =>
358 isNotFoundCause(error.cause) ? Effect.void : Effect.fail(error),397 isManagedEndpointNotFound(error.cause) ? Effect.void : Effect.fail(error),
359 ManagedEndpointDnsClientError: (error) =>398 ManagedEndpointDnsClientError: (error) =>
360 isNotFoundCause(error.cause) ? Effect.void : Effect.fail(error),399 isManagedEndpointNotFound(error.cause) ? Effect.void : Effect.fail(error),
361 }),400 }),
362 );401 );
363402
...@@ -399,7 +438,7 @@ export const make = Effect.gen(function* () {...@@ -399,7 +438,7 @@ export const make = Effect.gen(function* () {
399 Effect.as(true),438 Effect.as(true),
400 Effect.catchTags({439 Effect.catchTags({
401 ManagedEndpointDnsClientError: (error) =>440 ManagedEndpointDnsClientError: (error) =>
402 isNotFoundCause(error.cause) ? Effect.succeed(false) : Effect.fail(error),441 isManagedEndpointNotFound(error.cause) ? Effect.succeed(false) : Effect.fail(error),
403 }),442 }),
404 );443 );
405 if (checkpointedRecordUpdated) {444 if (checkpointedRecordUpdated) {
...@@ -455,8 +494,129 @@ export const make = Effect.gen(function* () {...@@ -455,8 +494,129 @@ export const make = Effect.gen(function* () {
455 },494 },
456 );495 );
457496
497 const reconcileOrigin = Effect.fn("relay.managed_endpoint_provider.reconcile_origin")(
498 function* (input: {
499 readonly userId: string;
500 readonly environmentId: string;
501 readonly tunnelId: string;
502 readonly origin: RelayManagedEndpointOrigin;
503 readonly endpoint: RelayManagedEndpoint;
504 }) {
505 if (!isLoopbackOrigin(input.origin)) {
506 return yield* new ManagedEndpointOriginNotAllowed({
507 userId: input.userId,
508 environmentId: input.environmentId,
509 host: input.origin.localHttpHost,
510 port: input.origin.localHttpPort,
511 });
512 }
513 const allocation = yield* allocations.get(input).pipe(
514 Effect.mapError(
515 (cause) =>
516 new ManagedEndpointProvisioningFailed({
517 ...input,
518 stage: "load-allocation",
519 cause,
520 }),
521 ),
522 );
523 if (
524 allocation === null ||
525 allocation.tunnelId !== input.tunnelId ||
526 allocation.dnsRecordId === null ||
527 allocation.readyAt === null
528 ) {
529 return "recovery_required";
530 }
531 const cf = yield* requireCloudflareSettings(config, input);
532 const recordedEndpoint = ManagedEndpointAllocations.resolveReadyManagedEndpoint({
533 allocation,
534 baseDomain: cf.baseDomain,
535 });
536 if (
537 recordedEndpoint === null ||
538 recordedEndpoint.httpBaseUrl !== input.endpoint.httpBaseUrl ||
539 recordedEndpoint.wsBaseUrl !== input.endpoint.wsBaseUrl ||
540 recordedEndpoint.providerKind !== input.endpoint.providerKind
541 ) {
542 return yield* new ManagedEndpointProvisioningFailed({
543 ...input,
544 stage: "verify-endpoint",
545 hostname: allocation.hostname,
546 });
547 }
548 if (
549 allocation.origin?.localHttpHost === input.origin.localHttpHost &&
550 allocation.origin.localHttpPort === input.origin.localHttpPort
551 ) {
552 return "ready";
553 }
554
555 const updated = yield* allocations
556 .withClaimedTunnel(
557 {
558 userId: input.userId,
559 environmentId: input.environmentId,
560 tunnelId: input.tunnelId,
561 generation: allocation.generation,
562 },
563 tunnels
564 .putConfiguration(input.tunnelId, {
565 ingress: [
566 {
567 hostname: allocation.hostname,
568 service: formatOriginService(input.origin),
569 },
570 { service: "http_status:404" },
571 ],
572 })
573 .pipe(
574 Effect.as("configured" as const),
575 Effect.catchTags({
576 ManagedEndpointTunnelClientError: (error) =>
577 isManagedEndpointNotFound(error.cause)
578 ? Effect.succeed("missing" as const)
579 : Effect.fail(error),
580 }),
581 Effect.flatMap((result) =>
582 result === "missing"
583 ? Effect.succeed(result)
584 : allocations
585 .markReady({
586 ...input,
587 generation: allocation.generation,
588 })
589 .pipe(
590 Effect.map((updated) =>
591 updated ? ("configured" as const) : ("stale" as const),
592 ),
593 ),
594 ),
595 ),
596 )
597 .pipe(
598 Effect.mapError(
599 (cause) =>
600 new ManagedEndpointProvisioningFailed({
601 ...input,
602 stage: "sync-origin",
603 cause,
604 }),
605 ),
606 );
607 if (Option.isNone(updated) || updated.value === "stale") {
608 return yield* new ManagedEndpointProvisioningFailed({
609 ...input,
610 stage: "sync-origin",
611 });
612 }
613 return updated.value === "configured" ? "ready" : "recovery_required";
614 },
615 );
616
458 return ManagedEndpointProvider.of({617 return ManagedEndpointProvider.of({
459 prepareDeprovision,618 prepareDeprovision,
619 reconcileOrigin,
460 deprovision: Effect.fn("relay.managed_endpoint_provider.deprovision")(function* (input) {620 deprovision: Effect.fn("relay.managed_endpoint_provider.deprovision")(function* (input) {
461 yield* Effect.annotateCurrentSpan({621 yield* Effect.annotateCurrentSpan({
462 "relay.user_id": input.userId,622 "relay.user_id": input.userId,
...@@ -465,13 +625,13 @@ export const make = Effect.gen(function* () {...@@ -465,13 +625,13 @@ export const make = Effect.gen(function* () {
465 const allocation =625 const allocation =
466 input.target === undefined ? yield* prepareDeprovision(input) : input.target;626 input.target === undefined ? yield* prepareDeprovision(input) : input.target;
467 if (allocation === null) {627 if (allocation === null) {
468 return;628 return true;
469 }629 }
470 const claimedAt = yield* allocations630 const claimedGeneration = yield* allocations
471 .claimDeprovision({631 .claimDeprovision({
472 userId: input.userId,632 userId: input.userId,
473 environmentId: input.environmentId,633 environmentId: input.environmentId,
474 updatedAt: allocation.updatedAt,634 generation: allocation.generation,
475 })635 })
476 .pipe(636 .pipe(
477 Effect.mapError(637 Effect.mapError(
...@@ -485,55 +645,86 @@ export const make = Effect.gen(function* () {...@@ -485,55 +645,86 @@ export const make = Effect.gen(function* () {
485 }),645 }),
486 ),646 ),
487 );647 );
488 if (claimedAt === null) {648 if (claimedGeneration === null) {
489 return;649 return false;
490 }
491 const dnsRecordId = allocation.dnsRecordId;
492 if (dnsRecordId !== null) {
493 yield* ignoreNotFound(dns.deleteRecord(dnsRecordId)).pipe(
494 Effect.mapError(
495 (cause) =>
496 new ManagedEndpointDeprovisioningFailed({
497 ...input,
498 stage: "delete-dns-record",
499 dnsRecordId,
500 cause,
501 }),
502 ),
503 );
504 }650 }
505 const tunnelId = allocation.tunnelId;651 const tunnelId = allocation.tunnelId;
506 if (tunnelId !== null) {652 const deprovision = Effect.gen(function* () {
507 yield* ignoreNotFound(tunnels.delete(tunnelId)).pipe(653 const dnsRecordId = allocation.dnsRecordId;
508 Effect.mapError(654 if (dnsRecordId !== null) {
509 (cause) =>655 yield* ignoreNotFound(dns.deleteRecord(dnsRecordId)).pipe(
510 new ManagedEndpointDeprovisioningFailed({656 Effect.mapError(
511 ...input,657 (cause) =>
512 stage: "delete-tunnel",658 new ManagedEndpointDeprovisioningFailed({
513 tunnelId,659 ...input,
514 cause,660 stage: "delete-dns-record",
515 }),661 dnsRecordId,
516 ),662 cause,
517 );663 }),
664 ),
665 );
666 }
667 if (tunnelId !== null) {
668 yield* ignoreNotFound(tunnels.delete(tunnelId)).pipe(
669 Effect.mapError(
670 (cause) =>
671 new ManagedEndpointDeprovisioningFailed({
672 ...input,
673 stage: "delete-tunnel",
674 tunnelId,
675 cause,
676 }),
677 ),
678 );
679 }
680 return yield* allocations
681 .removeClaimed({
682 userId: input.userId,
683 environmentId: input.environmentId,
684 generation: claimedGeneration,
685 })
686 .pipe(
687 Effect.mapError(
688 (cause) =>
689 new ManagedEndpointDeprovisioningFailed({
690 ...input,
691 stage: "remove-allocation",
692 ...(allocation.tunnelId === null ? {} : { tunnelId: allocation.tunnelId }),
693 ...(allocation.dnsRecordId === null
694 ? {}
695 : { dnsRecordId: allocation.dnsRecordId }),
696 cause,
697 }),
698 ),
699 );
700 });
701 if (tunnelId === null) {
702 return yield* deprovision;
518 }703 }
519 yield* allocations704 const removed = yield* allocations
520 .removeClaimed({705 .withClaimedTunnel(
521 userId: input.userId,706 {
522 environmentId: input.environmentId,707 userId: input.userId,
523 updatedAt: claimedAt,708 environmentId: input.environmentId,
524 })709 tunnelId,
710 generation: claimedGeneration,
711 },
712 deprovision,
713 )
525 .pipe(714 .pipe(
526 Effect.mapError(715 Effect.catchTags({
527 (cause) =>716 ManagedEndpointAllocationPersistenceError: (cause) =>
528 new ManagedEndpointDeprovisioningFailed({717 Effect.fail(
529 ...input,718 new ManagedEndpointDeprovisioningFailed({
530 stage: "remove-allocation",719 ...input,
531 ...(allocation.tunnelId === null ? {} : { tunnelId: allocation.tunnelId }),720 stage: "claim-deprovision",
532 ...(allocation.dnsRecordId === null ? {} : { dnsRecordId: allocation.dnsRecordId }),721 tunnelId,
533 cause,722 cause,
534 }),723 }),
535 ),724 ),
725 }),
536 );726 );
727 return Option.getOrElse(removed, () => false);
537 }),728 }),
538 release: Effect.fn("relay.managed_endpoint_provider.release")(function* (input) {729 release: Effect.fn("relay.managed_endpoint_provider.release")(function* (input) {
539 yield* Effect.annotateCurrentSpan({730 yield* Effect.annotateCurrentSpan({
...@@ -554,19 +745,22 @@ export const make = Effect.gen(function* () {...@@ -554,19 +745,22 @@ export const make = Effect.gen(function* () {
554 if (allocation === null || tunnelId === null) {745 if (allocation === null || tunnelId === null) {
555 return true;746 return true;
556 }747 }
748 if (input.expectedTunnelId !== undefined && input.expectedTunnelId !== tunnelId) {
749 return false;
750 }
557 // Claim the release against the allocation's current generation before751 // Claim the release against the allocation's current generation before
558 // touching Cloudflare. A provision racing this release (fast environment752 // touching Cloudflare. A provision racing this release (fast environment
559 // restart) rewrites updatedAt when it records its tunnel, so a stale753 // restart) increments the generation when it records its tunnel, so a stale
560 // claim means the recorded tunnel may already back a fresh connector and754 // claim means the recorded tunnel may already back a fresh connector and
561 // must be left alive. A provision that starts after the claim instead755 // must be left alive. A provision that starts after the claim instead
562 // fails loudly on the deleted tunnel and the client-side retry756 // fails loudly on the deleted tunnel and the client-side retry
563 // provisions a replacement.757 // provisions a replacement.
564 const claimed = yield* allocations758 const claimedGeneration = yield* allocations
565 .claimRelease({759 .claimRelease({
566 userId: input.userId,760 userId: input.userId,
567 environmentId: input.environmentId,761 environmentId: input.environmentId,
568 tunnelId,762 tunnelId,
569 updatedAt: allocation.updatedAt,763 generation: allocation.generation,
570 })764 })
571 .pipe(765 .pipe(
572 Effect.mapError(766 Effect.mapError(
...@@ -579,10 +773,10 @@ export const make = Effect.gen(function* () {...@@ -579,10 +773,10 @@ export const make = Effect.gen(function* () {
579 }),773 }),
580 ),774 ),
581 );775 );
582 if (!claimed) {776 if (claimedGeneration === null) {
583 return false;777 return false;
584 }778 }
585 yield* ignoreNotFound(tunnels.delete(tunnelId)).pipe(779 const deleteTunnel = ignoreNotFound(tunnels.delete(tunnelId)).pipe(
586 Effect.mapError(780 Effect.mapError(
587 (cause) =>781 (cause) =>
588 new ManagedEndpointDeprovisioningFailed({782 new ManagedEndpointDeprovisioningFailed({
...@@ -593,13 +787,103 @@ export const make = Effect.gen(function* () {...@@ -593,13 +787,103 @@ export const make = Effect.gen(function* () {
593 }),787 }),
594 ),788 ),
595 );789 );
790 if (input.expectedInactiveBefore !== undefined && input.expectedStatus !== undefined) {
791 const expectedStatus = input.expectedStatus;
792 const inactiveBefore = input.expectedInactiveBefore;
793 const currentTunnel = yield* tunnels.get(tunnelId).pipe(
794 Effect.map(Option.some),
795 Effect.catchTags({
796 ManagedEndpointTunnelClientError: (cause) =>
797 isManagedEndpointNotFound(cause.cause)
798 ? Effect.succeed(Option.none())
799 : Effect.fail(
800 new ManagedEndpointDeprovisioningFailed({
801 ...input,
802 stage: "load-tunnel",
803 tunnelId,
804 cause,
805 }),
806 ),
807 }),
808 );
809 if (Option.isNone(currentTunnel)) {
810 return true;
811 }
812 const inactiveAt =
813 expectedStatus === "down"
814 ? currentTunnel.value.connsInactiveAt
815 : currentTunnel.value.createdAt;
816 if (
817 currentTunnel.value.id !== tunnelId ||
818 currentTunnel.value.status !== expectedStatus ||
819 typeof inactiveAt !== "string"
820 ) {
821 return false;
822 }
823 const inactiveTime = DateTime.make(inactiveAt);
824 const cutoff = DateTime.make(inactiveBefore);
825 if (
826 Option.isNone(inactiveTime) ||
827 Option.isNone(cutoff) ||
828 inactiveTime.value.epochMilliseconds > cutoff.value.epochMilliseconds
829 ) {
830 return false;
831 }
832 }
833 const released = yield* allocations
834 .withClaimedTunnel(
835 {
836 userId: input.userId,
837 environmentId: input.environmentId,
838 tunnelId,
839 generation: claimedGeneration,
840 },
841 Effect.gen(function* () {
842 const finalGeneration = yield* allocations
843 .claimRelease({
844 userId: input.userId,
845 environmentId: input.environmentId,
846 tunnelId,
847 generation: claimedGeneration,
848 })
849 .pipe(
850 Effect.mapError(
851 (cause) =>
852 new ManagedEndpointDeprovisioningFailed({
853 ...input,
854 stage: "claim-release",
855 tunnelId,
856 cause,
857 }),
858 ),
859 );
860 if (finalGeneration === null) {
861 return false;
862 }
863 yield* deleteTunnel;
864 return true;
865 }),
866 )
867 .pipe(
868 Effect.catchTags({
869 ManagedEndpointAllocationPersistenceError: (cause) =>
870 Effect.fail(
871 new ManagedEndpointDeprovisioningFailed({
872 ...input,
873 stage: "claim-release",
874 tunnelId,
875 cause,
876 }),
877 ),
878 }),
879 );
596 // The recorded tunnelId is now stale, but the allocation row is left880 // The recorded tunnelId is now stale, but the allocation row is left
597 // untouched deliberately: connect/status authorization requires a fully881 // untouched deliberately: connect/status authorization requires a fully
598 // recorded allocation, and an offline environment must keep reporting882 // recorded allocation, and an offline environment must keep reporting
599 // "offline" (health probe fails) rather than "not authorized". The next883 // "offline" (health probe fails) rather than "not authorized". The next
600 // provision lists tunnels by name, finds none, creates a replacement and884 // provision lists tunnels by name, finds none, creates a replacement and
601 // re-records the fresh id.885 // re-records the fresh id.
602 return true;886 return Option.getOrElse(released, () => false);
603 }),887 }),
604 provision: Effect.fn("relay.managed_endpoint_provider.provision")(function* (input) {888 provision: Effect.fn("relay.managed_endpoint_provider.provision")(function* (input) {
605 yield* Effect.annotateCurrentSpan({889 yield* Effect.annotateCurrentSpan({
...@@ -717,11 +1001,12 @@ export const make = Effect.gen(function* () {...@@ -717,11 +1001,12 @@ export const make = Effect.gen(function* () {
717 });1001 });
718 }1002 }
719 const tunnel = { id: tunnelResponse.id, name: tunnelResponse.name };1003 const tunnel = { id: tunnelResponse.id, name: tunnelResponse.name };
720 yield* allocations1004 const tunnelGeneration = yield* allocations
721 .recordTunnel({1005 .recordTunnel({
722 userId: input.userId,1006 userId: input.userId,
723 environmentId: input.environmentId,1007 environmentId: input.environmentId,
724 tunnelId: tunnel.id,1008 tunnelId: tunnel.id,
1009 generation: allocation.generation,
725 })1010 })
726 .pipe(1011 .pipe(
727 Effect.mapError(1012 Effect.mapError(
...@@ -737,31 +1022,78 @@ export const make = Effect.gen(function* () {...@@ -737,31 +1022,78 @@ export const make = Effect.gen(function* () {
737 }),1022 }),
738 ),1023 ),
739 );1024 );
1025 if (tunnelGeneration === null) {
1026 // A newer provision can adopt this tunnel by name at any point after
1027 // our claim fails. Leave it available for that provision or a retry.
1028 return yield* new ManagedEndpointProvisioningFailed({
1029 userId: input.userId,
1030 environmentId: input.environmentId,
1031 stage: "record-tunnel",
1032 hostname,
1033 tunnelName,
1034 tunnelId: tunnel.id,
1035 });
1036 }
7401037
741 yield* tunnels1038 const configured = yield* allocations
742 .putConfiguration(tunnel.id, {1039 .withClaimedTunnel(
743 ingress: [1040 {
744 {1041 userId: input.userId,
745 hostname,1042 environmentId: input.environmentId,
746 service: formatOriginService(input.origin),1043 tunnelId: tunnel.id,
747 },1044 generation: tunnelGeneration,
748 { service: "http_status:404" },1045 },
749 ],1046 tunnels
750 })1047 .putConfiguration(tunnel.id, {
1048 ingress: [
1049 {
1050 hostname,
1051 service: formatOriginService(input.origin),
1052 },
1053 { service: "http_status:404" },
1054 ],
1055 })
1056 .pipe(
1057 Effect.mapError(
1058 (cause) =>
1059 new ManagedEndpointProvisioningFailed({
1060 userId: input.userId,
1061 environmentId: input.environmentId,
1062 stage: "configure-tunnel",
1063 hostname,
1064 tunnelName,
1065 tunnelId: tunnel.id,
1066 cause,
1067 }),
1068 ),
1069 ),
1070 )
751 .pipe(1071 .pipe(
752 Effect.mapError(1072 Effect.catchTags({
753 (cause) =>1073 ManagedEndpointAllocationPersistenceError: (cause) =>
754 new ManagedEndpointProvisioningFailed({1074 Effect.fail(
755 userId: input.userId,1075 new ManagedEndpointProvisioningFailed({
756 environmentId: input.environmentId,1076 userId: input.userId,
757 stage: "configure-tunnel",1077 environmentId: input.environmentId,
758 hostname,1078 stage: "configure-tunnel",
759 tunnelName,1079 hostname,
760 tunnelId: tunnel.id,1080 tunnelName,
761 cause,1081 tunnelId: tunnel.id,
762 }),1082 cause,
763 ),1083 }),
1084 ),
1085 }),
764 );1086 );
1087 if (Option.isNone(configured)) {
1088 return yield* new ManagedEndpointProvisioningFailed({
1089 userId: input.userId,
1090 environmentId: input.environmentId,
1091 stage: "configure-tunnel",
1092 hostname,
1093 tunnelName,
1094 tunnelId: tunnel.id,
1095 });
1096 }
7651097
766 const dnsRecord = {1098 const dnsRecord = {
767 type: "CNAME",1099 type: "CNAME",
...@@ -771,31 +1103,61 @@ export const make = Effect.gen(function* () {...@@ -771,31 +1103,61 @@ export const make = Effect.gen(function* () {
771 proxied: true,1103 proxied: true,
772 } as const;1104 } as const;
7731105
774 const dnsRecordId = yield* ensureDnsRecord(hostname, allocation.dnsRecordId, dnsRecord).pipe(1106 const recordedDns = yield* allocations
775 Effect.mapError(1107 .withClaimedTunnel(
776 (cause) =>1108 {
777 new ManagedEndpointProvisioningFailed({1109 userId: input.userId,
778 userId: input.userId,1110 environmentId: input.environmentId,
779 environmentId: input.environmentId,1111 tunnelId: tunnel.id,
780 stage: "ensure-dns-record",1112 generation: tunnelGeneration,
1113 },
1114 Effect.gen(function* () {
1115 const dnsRecordId = yield* ensureDnsRecord(
781 hostname,1116 hostname,
782 tunnelName,1117 allocation.dnsRecordId,
783 tunnelId: tunnel.id,1118 dnsRecord,
784 ...(allocation.dnsRecordId === null ? {} : { dnsRecordId: allocation.dnsRecordId }),1119 ).pipe(
785 cause,1120 Effect.mapError(
786 }),1121 (cause) =>
787 ),1122 new ManagedEndpointProvisioningFailed({
788 );1123 userId: input.userId,
789 yield* allocations1124 environmentId: input.environmentId,
790 .recordDns({1125 stage: "ensure-dns-record",
791 userId: input.userId,1126 hostname,
792 environmentId: input.environmentId,1127 tunnelName,
793 dnsRecordId,1128 tunnelId: tunnel.id,
794 })1129 ...(allocation.dnsRecordId === null
795 .pipe(1130 ? {}
796 Effect.mapError(1131 : { dnsRecordId: allocation.dnsRecordId }),
797 (cause) =>1132 cause,
798 new ManagedEndpointProvisioningFailed({1133 }),
1134 ),
1135 );
1136 const dnsGeneration = yield* allocations
1137 .recordDns({
1138 userId: input.userId,
1139 environmentId: input.environmentId,
1140 dnsRecordId,
1141 tunnelId: tunnel.id,
1142 generation: tunnelGeneration,
1143 })
1144 .pipe(
1145 Effect.mapError(
1146 (cause) =>
1147 new ManagedEndpointProvisioningFailed({
1148 userId: input.userId,
1149 environmentId: input.environmentId,
1150 stage: "record-dns",
1151 hostname,
1152 tunnelName,
1153 tunnelId: tunnel.id,
1154 dnsRecordId,
1155 cause,
1156 }),
1157 ),
1158 );
1159 if (dnsGeneration === null) {
1160 return yield* new ManagedEndpointProvisioningFailed({
799 userId: input.userId,1161 userId: input.userId,
800 environmentId: input.environmentId,1162 environmentId: input.environmentId,
801 stage: "record-dns",1163 stage: "record-dns",
...@@ -803,10 +1165,38 @@ export const make = Effect.gen(function* () {...@@ -803,10 +1165,38 @@ export const make = Effect.gen(function* () {
803 tunnelName,1165 tunnelName,
804 tunnelId: tunnel.id,1166 tunnelId: tunnel.id,
805 dnsRecordId,1167 dnsRecordId,
806 cause,1168 });
807 }),1169 }
808 ),1170 return { dnsRecordId, dnsGeneration };
1171 }),
1172 )
1173 .pipe(
1174 Effect.catchTags({
1175 ManagedEndpointAllocationPersistenceError: (cause) =>
1176 Effect.fail(
1177 new ManagedEndpointProvisioningFailed({
1178 userId: input.userId,
1179 environmentId: input.environmentId,
1180 stage: "record-dns",
1181 hostname,
1182 tunnelName,
1183 tunnelId: tunnel.id,
1184 cause,
1185 }),
1186 ),
1187 }),
809 );1188 );
1189 if (Option.isNone(recordedDns)) {
1190 return yield* new ManagedEndpointProvisioningFailed({
1191 userId: input.userId,
1192 environmentId: input.environmentId,
1193 stage: "record-dns",
1194 hostname,
1195 tunnelName,
1196 tunnelId: tunnel.id,
1197 });
1198 }
1199 const { dnsRecordId, dnsGeneration } = recordedDns.value;
8101200
811 const connectorToken = yield* tunnels.getToken(tunnel.id).pipe(1201 const connectorToken = yield* tunnels.getToken(tunnel.id).pipe(
812 Effect.mapError(1202 Effect.mapError(
...@@ -823,10 +1213,13 @@ export const make = Effect.gen(function* () {...@@ -823,10 +1213,13 @@ export const make = Effect.gen(function* () {
823 }),1213 }),
824 ),1214 ),
825 );1215 );
826 yield* allocations1216 const ready = yield* allocations
827 .markReady({1217 .markReady({
828 userId: input.userId,1218 userId: input.userId,
829 environmentId: input.environmentId,1219 environmentId: input.environmentId,
1220 tunnelId: tunnel.id,
1221 generation: dnsGeneration,
1222 origin: input.origin,
830 })1223 })
831 .pipe(1224 .pipe(
832 Effect.mapError(1225 Effect.mapError(
...@@ -843,6 +1236,17 @@ export const make = Effect.gen(function* () {...@@ -843,6 +1236,17 @@ export const make = Effect.gen(function* () {
843 }),1236 }),
844 ),1237 ),
845 );1238 );
1239 if (!ready) {
1240 return yield* new ManagedEndpointProvisioningFailed({
1241 userId: input.userId,
1242 environmentId: input.environmentId,
1243 stage: "mark-allocation-ready",
1244 hostname,
1245 tunnelName,
1246 tunnelId: tunnel.id,
1247 dnsRecordId,
1248 });
1249 }
8461250
847 return {1251 return {
848 endpoint: managedEndpointForHostname(hostname),1252 endpoint: managedEndpointForHostname(hostname),
...@@ -865,16 +1269,30 @@ export const layerCloudflareBindings = (...@@ -865,16 +1269,30 @@ export const layerCloudflareBindings = (
865 alchemyRuntimeContext: Alchemy.BaseRuntimeContext,1269 alchemyRuntimeContext: Alchemy.BaseRuntimeContext,
866) =>1270) =>
867 layer.pipe(1271 layer.pipe(
868 Layer.provide(1272 Layer.provideMerge(
869 Layer.mergeAll(1273 Layer.mergeAll(
870 layerTunnelClient({1274 layerTunnelClient({
1275 get: (tunnelId) =>
1276 tunnelClient.get(tunnelId).pipe(
1277 Effect.timeout("8 seconds"),
1278 Effect.mapError(
1279 (cause) =>
1280 new ManagedEndpointTunnelClientError({
1281 operation: "get",
1282 tunnelId,
1283 cause,
1284 }),
1285 ),
1286 Effect.provideService(Alchemy.RuntimeContext, alchemyRuntimeContext),
1287 ),
871 list: (request) =>1288 list: (request) =>
872 tunnelClient.list(request).pipe(1289 tunnelClient.list(request).pipe(
1290 Effect.timeout("8 seconds"),
873 Effect.mapError(1291 Effect.mapError(
874 (cause) =>1292 (cause) =>
875 new ManagedEndpointTunnelClientError({1293 new ManagedEndpointTunnelClientError({
876 operation: "list",1294 operation: "list",
877 tunnelName: request.name,1295 ...(request.name === undefined ? {} : { tunnelName: request.name }),
878 cause,1296 cause,
879 }),1297 }),
880 ),1298 ),
...@@ -882,6 +1300,7 @@ export const layerCloudflareBindings = (...@@ -882,6 +1300,7 @@ export const layerCloudflareBindings = (
882 ),1300 ),
883 create: (request) =>1301 create: (request) =>
884 tunnelClient.create(request).pipe(1302 tunnelClient.create(request).pipe(
1303 Effect.timeout("8 seconds"),
885 Effect.mapError(1304 Effect.mapError(
886 (cause) =>1305 (cause) =>
887 new ManagedEndpointTunnelClientError({1306 new ManagedEndpointTunnelClientError({
...@@ -894,6 +1313,7 @@ export const layerCloudflareBindings = (...@@ -894,6 +1313,7 @@ export const layerCloudflareBindings = (
894 ),1313 ),
895 putConfiguration: (tunnelId, config) =>1314 putConfiguration: (tunnelId, config) =>
896 tunnelClient.putConfiguration(tunnelId, config).pipe(1315 tunnelClient.putConfiguration(tunnelId, config).pipe(
1316 Effect.timeout("8 seconds"),
897 Effect.mapError(1317 Effect.mapError(
898 (cause) =>1318 (cause) =>
899 new ManagedEndpointTunnelClientError({1319 new ManagedEndpointTunnelClientError({
...@@ -906,6 +1326,7 @@ export const layerCloudflareBindings = (...@@ -906,6 +1326,7 @@ export const layerCloudflareBindings = (
906 ),1326 ),
907 getToken: (tunnelId) =>1327 getToken: (tunnelId) =>
908 tunnelClient.getToken(tunnelId).pipe(1328 tunnelClient.getToken(tunnelId).pipe(
1329 Effect.timeout("8 seconds"),
909 Effect.mapError(1330 Effect.mapError(
910 (cause) =>1331 (cause) =>
911 new ManagedEndpointTunnelClientError({1332 new ManagedEndpointTunnelClientError({
...@@ -918,6 +1339,7 @@ export const layerCloudflareBindings = (...@@ -918,6 +1339,7 @@ export const layerCloudflareBindings = (
918 ),1339 ),
919 delete: (tunnelId) =>1340 delete: (tunnelId) =>
920 tunnelClient.delete(tunnelId).pipe(1341 tunnelClient.delete(tunnelId).pipe(
1342 Effect.timeout("8 seconds"),
921 Effect.mapError(1343 Effect.mapError(
922 (cause) =>1344 (cause) =>
923 new ManagedEndpointTunnelClientError({1345 new ManagedEndpointTunnelClientError({
...@@ -932,6 +1354,7 @@ export const layerCloudflareBindings = (...@@ -932,6 +1354,7 @@ export const layerCloudflareBindings = (
932 layerDnsClient({1354 layerDnsClient({
933 listRecords: (hostname) =>1355 listRecords: (hostname) =>
934 dnsClient.listDnsRecords({ search: hostname }).pipe(1356 dnsClient.listDnsRecords({ search: hostname }).pipe(
1357 Effect.timeout("8 seconds"),
935 Effect.map((response) =>1358 Effect.map((response) =>
936 response.result.filter(1359 response.result.filter(
937 (record): record is typeof record & { readonly id: string } =>1360 (record): record is typeof record & { readonly id: string } =>
...@@ -951,6 +1374,7 @@ export const layerCloudflareBindings = (...@@ -951,6 +1374,7 @@ export const layerCloudflareBindings = (
951 ),1374 ),
952 createRecord: (request) =>1375 createRecord: (request) =>
953 dnsClient.createDnsRecord(request).pipe(1376 dnsClient.createDnsRecord(request).pipe(
1377 Effect.timeout("8 seconds"),
954 Effect.map((response) => ({ id: response.id })),1378 Effect.map((response) => ({ id: response.id })),
955 Effect.mapError(1379 Effect.mapError(
956 (cause) =>1380 (cause) =>
...@@ -964,6 +1388,7 @@ export const layerCloudflareBindings = (...@@ -964,6 +1388,7 @@ export const layerCloudflareBindings = (
964 ),1388 ),
965 updateRecord: (dnsRecordId, request) =>1389 updateRecord: (dnsRecordId, request) =>
966 dnsClient.updateDnsRecord(dnsRecordId, request).pipe(1390 dnsClient.updateDnsRecord(dnsRecordId, request).pipe(
1391 Effect.timeout("8 seconds"),
967 Effect.mapError(1392 Effect.mapError(
968 (cause) =>1393 (cause) =>
969 new ManagedEndpointDnsClientError({1394 new ManagedEndpointDnsClientError({
...@@ -977,6 +1402,7 @@ export const layerCloudflareBindings = (...@@ -977,6 +1402,7 @@ export const layerCloudflareBindings = (
977 ),1402 ),
978 deleteRecord: (dnsRecordId) =>1403 deleteRecord: (dnsRecordId) =>
979 dnsClient.deleteDnsRecord(dnsRecordId).pipe(1404 dnsClient.deleteDnsRecord(dnsRecordId).pipe(
1405 Effect.timeout("8 seconds"),
980 Effect.mapError(1406 Effect.mapError(
981 (cause) =>1407 (cause) =>
982 new ManagedEndpointDnsClientError({1408 new ManagedEndpointDnsClientError({
infra/relay/src/environments/ManagedEndpointReaper.test.ts created+761
...@@ -0,0 +1,761 @@
1import { describe, expect, it } from "@effect/vitest";
2import * as DateTime from "effect/DateTime";
3import * as Effect from "effect/Effect";
4import * as Layer from "effect/Layer";
5import * as Redacted from "effect/Redacted";
6import * as TestClock from "effect/testing/TestClock";
7
8import * as RelayConfiguration from "../Config.ts";
9import * as ManagedEndpointAllocations from "./ManagedEndpointAllocations.ts";
10import * as ManagedEndpointProvider from "./ManagedEndpointProvider.ts";
11import * as ManagedEndpointReaper from "./ManagedEndpointReaper.ts";
12
13const NOW = "2026-08-25T12:00:00.000Z";
14const NOW_MILLIS = DateTime.makeUnsafe(NOW).epochMilliseconds;
15const PREFIX = "t3coderelay-managedendpoint-prod-";
16
17function tunnel(input: {
18 readonly id: string;
19 readonly suffix: string;
20 readonly status: "down" | "inactive" | "healthy" | "degraded";
21 readonly timestamp?: string | null;
22 readonly prefix?: string;
23}): ManagedEndpointProvider.ManagedEndpointTunnel {
24 return {
25 id: input.id,
26 name: `${input.prefix ?? PREFIX}${input.suffix}`,
27 status: input.status,
28 ...(input.timestamp === undefined
29 ? {}
30 : input.status === "inactive"
31 ? { createdAt: input.timestamp }
32 : { connsInactiveAt: input.timestamp }),
33 };
34}
35
36function recoverableOwners(
37 tunnels: ReadonlyArray<ManagedEndpointProvider.ManagedEndpointTunnel>,
38): ReadonlyArray<ManagedEndpointAllocations.ManagedEndpointTunnelAllocation> {
39 return tunnels.map((entry) => allocation({ tunnelId: entry.id!, recoveryEnabled: true }));
40}
41
42function allocation(input: {
43 readonly tunnelId: string | null;
44 readonly recoveryEnabled: boolean;
45}): ManagedEndpointAllocations.ManagedEndpointTunnelAllocation {
46 return {
47 userId: "user-1",
48 environmentId: `environment-${input.tunnelId ?? "pending"}`,
49 hostname: `${input.tunnelId ?? "pending"}.example.test`,
50 tunnelId: input.tunnelId,
51 tunnelName: `${PREFIX}aaaaaaaaaaaaaaaa`,
52 dnsRecordId: "dns-1",
53 readyAt: "2026-08-25T11:00:00.000Z",
54 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
55 updatedAt: "2026-08-25T11:00:00.000Z",
56 generation: 1,
57 recoveryEnabled: input.recoveryEnabled,
58 };
59}
60
61function harness(input?: {
62 readonly tunnels?: ReadonlyArray<ManagedEndpointProvider.ManagedEndpointTunnel>;
63 readonly allocations?: ReadonlyArray<ManagedEndpointAllocations.ManagedEndpointTunnelAllocation>;
64 readonly namespace?: string;
65 readonly failTunnelId?: string;
66 readonly rateLimitedTunnelId?: string;
67 readonly failAllDeletes?: boolean;
68 readonly failDeleteWhen?: (tunnelId: string) => boolean;
69 readonly missingOnDeleteTunnelId?: string;
70 readonly missingOnGetTunnelId?: string;
71 readonly reserveOnGetTunnelId?: string;
72 readonly refreshedTunnels?: ReadonlyMap<string, ManagedEndpointProvider.ManagedEndpointTunnel>;
73 readonly skipTunnelId?: string;
74 readonly cleanupMode?: RelayConfiguration.ManagedEndpointCleanupMode;
75}) {
76 const listRequests: ManagedEndpointProvider.ManagedEndpointTunnelListRequest[] = [];
77 const deleted: string[] = [];
78 const releases: Array<
79 Parameters<ManagedEndpointProvider.ManagedEndpointProvider["Service"]["release"]>[0]
80 > = [];
81 const remaining = [...(input?.tunnels ?? [])];
82 const recorded = (input?.allocations ?? []).map((entry) => {
83 const matching = remaining.find((candidate) => candidate.id === entry.tunnelId);
84 return typeof matching?.name === "string" ? { ...entry, tunnelName: matching.name } : entry;
85 });
86 const tunnelClient = ManagedEndpointProvider.ManagedEndpointTunnelClient.of({
87 get: (tunnelId) =>
88 Effect.suspend(() => {
89 if (tunnelId === input?.missingOnGetTunnelId) {
90 return Effect.fail(
91 new ManagedEndpointProvider.ManagedEndpointTunnelClientError({
92 operation: "get",
93 tunnelId,
94 cause: { _tag: "NotFound" },
95 }),
96 );
97 }
98 const found =
99 input?.refreshedTunnels?.get(tunnelId) ??
100 remaining.find((candidate) => candidate.id === tunnelId);
101 if (found === undefined) {
102 return Effect.fail(
103 new ManagedEndpointProvider.ManagedEndpointTunnelClientError({
104 operation: "get",
105 tunnelId,
106 cause: { _tag: "NotFound" },
107 }),
108 );
109 }
110 if (tunnelId === input?.reserveOnGetTunnelId && typeof found.name === "string") {
111 recorded.push({
112 ...allocation({ tunnelId, recoveryEnabled: false }),
113 tunnelName: found.name,
114 });
115 }
116 return Effect.succeed(found);
117 }),
118 list: (request) =>
119 Effect.sync(() => {
120 listRequests.push(request);
121 const matching = remaining.filter((entry) => entry.status === request.status);
122 const start = ((request.page ?? 1) - 1) * (request.perPage ?? 100);
123 return {
124 result: matching.slice(start, start + (request.perPage ?? 100)),
125 resultInfo: {
126 page: request.page ?? 1,
127 perPage: request.perPage ?? 100,
128 totalCount: matching.length,
129 },
130 };
131 }),
132 create: () => Effect.die("unused"),
133 putConfiguration: () => Effect.die("unused"),
134 getToken: () => Effect.die("unused"),
135 delete: (tunnelId) =>
136 input?.failAllDeletes === true ||
137 input?.failDeleteWhen?.(tunnelId) === true ||
138 tunnelId === input?.failTunnelId ||
139 tunnelId === input?.rateLimitedTunnelId ||
140 tunnelId === input?.missingOnDeleteTunnelId
141 ? Effect.fail(
142 new ManagedEndpointProvider.ManagedEndpointTunnelClientError({
143 operation: "delete",
144 tunnelId,
145 cause:
146 tunnelId === input?.missingOnDeleteTunnelId
147 ? { _tag: "NotFound" }
148 : tunnelId === input?.rateLimitedTunnelId
149 ? {
150 cause: {
151 _tag: "TooManyRequests",
152 message: "Cloudflare rate limit exceeded",
153 retryAfter: 60,
154 },
155 }
156 : "Cloudflare refused the deletion",
157 }),
158 )
159 : Effect.sync(() => {
160 deleted.push(tunnelId);
161 const index = remaining.findIndex((candidate) => candidate.id === tunnelId);
162 if (index !== -1) {
163 remaining.splice(index, 1);
164 }
165 }),
166 });
167 const allocationService = ManagedEndpointAllocations.ManagedEndpointAllocations.of({
168 get: () => Effect.die("unused"),
169 reserve: () => Effect.die("unused"),
170 recordTunnel: () => Effect.die("unused"),
171 recordDns: () => Effect.die("unused"),
172 markReady: () => Effect.die("unused"),
173 enableRecovery: () => Effect.die("unused"),
174 listByTunnelNames: (tunnelNames) =>
175 Effect.succeed(recorded.filter((entry) => tunnelNames.includes(entry.tunnelName))),
176 claimRelease: () => Effect.die("unused"),
177 withClaimedTunnel: () => Effect.die("unused"),
178 claimDeprovision: () => Effect.die("unused"),
179 remove: () => Effect.die("unused"),
180 removeClaimed: () => Effect.die("unused"),
181 });
182 const provider = ManagedEndpointProvider.ManagedEndpointProvider.of({
183 provision: () => Effect.die("unused"),
184 reconcileOrigin: () => Effect.die("unused"),
185 prepareDeprovision: () => Effect.die("unused"),
186 deprovision: () => Effect.die("unused"),
187 release: (request) =>
188 Effect.gen(function* () {
189 releases.push(request);
190 if (request.expectedTunnelId === input?.skipTunnelId) {
191 return false;
192 }
193 if (request.expectedTunnelId !== undefined) {
194 // Surface Cloudflare failures the same way the real release does.
195 yield* tunnelClient.delete(request.expectedTunnelId).pipe(
196 Effect.mapError(
197 (cause) =>
198 new ManagedEndpointProvider.ManagedEndpointDeprovisioningFailed({
199 stage: "delete-tunnel",
200 userId: request.userId,
201 environmentId: request.environmentId,
202 tunnelId: request.expectedTunnelId!,
203 cause,
204 }),
205 ),
206 );
207 }
208 return true;
209 }),
210 });
211 const config = RelayConfiguration.RelayConfiguration.of({
212 relayIssuer: "https://relay.example.test",
213 apns: {
214 environment: "sandbox",
215 teamId: "team-id",
216 keyId: "key-id",
217 privateKey: Redacted.make("private-key"),
218 bundleId: "com.t3tools.t3code.dev",
219 },
220 apnsDeliveryJobSigningSecret: Redacted.make("job-secret"),
221 clerkSecretKey: Redacted.make("clerk-secret"),
222 clerkPublishableKey: "pk_test_test",
223 clerkJwtAudience: "t3-code-relay",
224 cloudMintPrivateKey: Redacted.make("cloud-private-key"),
225 cloudMintPublicKey: "cloud-public-key",
226 managedEndpointBaseDomain: "example.test",
227 managedEndpointNamespace: input?.namespace ?? "prod",
228 managedEndpointCleanupMode: input?.cleanupMode ?? "enabled",
229 });
230
231 return {
232 listRequests,
233 deleted,
234 releases,
235 layer: ManagedEndpointReaper.layer.pipe(
236 Layer.provide(
237 Layer.mergeAll(
238 RelayConfiguration.layer(config),
239 ManagedEndpointProvider.layerTunnelClient(tunnelClient),
240 Layer.succeed(ManagedEndpointProvider.ManagedEndpointProvider, provider),
241 Layer.succeed(ManagedEndpointAllocations.ManagedEndpointAllocations, allocationService),
242 ),
243 ),
244 ),
245 };
246}
247
248describe("ManagedEndpointReaper", () => {
249 it.effect("removes expired down and inactive tunnels from recoverable environments", () => {
250 const state = harness({
251 tunnels: [
252 tunnel({
253 id: "down-1",
254 suffix: "aaaaaaaaaaaaaaaa",
255 status: "down",
256 timestamp: "2026-08-25T11:55:00.000Z",
257 }),
258 tunnel({
259 id: "inactive-1",
260 suffix: "bbbbbbbbbbbbbbbb",
261 status: "inactive",
262 timestamp: "2026-08-25T10:59:00.000Z",
263 }),
264 ],
265 allocations: [
266 allocation({ tunnelId: "down-1", recoveryEnabled: true }),
267 allocation({ tunnelId: "inactive-1", recoveryEnabled: true }),
268 ],
269 });
270
271 return Effect.gen(function* () {
272 yield* TestClock.setTime(NOW_MILLIS);
273 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
274 expect(yield* reaper.sweep).toMatchObject({
275 scanned: 2,
276 deleted: 2,
277 skippedLegacy: 0,
278 failed: 0,
279 });
280 expect(state.deleted).toEqual(["down-1", "inactive-1"]);
281 expect(state.releases.map((request) => request.expectedTunnelId)).toEqual([
282 "down-1",
283 "inactive-1",
284 ]);
285 expect(state.listRequests).toEqual([
286 {
287 isDeleted: false,
288 includePrefix: PREFIX,
289 status: "down",
290 existedAt: "2026-08-25T11:55:00.000Z",
291 wasInactiveAt: "2026-08-25T11:55:00.000Z",
292 page: 1,
293 perPage: 100,
294 },
295 {
296 isDeleted: false,
297 includePrefix: PREFIX,
298 status: "inactive",
299 existedAt: "2026-08-25T11:00:00.000Z",
300 page: 1,
301 perPage: 100,
302 },
303 ]);
304 }).pipe(Effect.provide(state.layer));
305 });
306
307 it.effect("keeps a tunnel that never connected until it is an hour old", () => {
308 const state = harness({
309 tunnels: [
310 tunnel({
311 id: "pairing",
312 suffix: "cccccccccccccccc",
313 status: "inactive",
314 timestamp: "2026-08-25T11:30:00.000Z",
315 }),
316 ],
317 allocations: [allocation({ tunnelId: "pairing", recoveryEnabled: true })],
318 });
319
320 return Effect.gen(function* () {
321 yield* TestClock.setTime(NOW_MILLIS);
322 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
323 expect((yield* reaper.sweep).deleted).toBe(0);
324 expect(state.deleted).toEqual([]);
325 }).pipe(Effect.provide(state.layer));
326 });
327
328 it.effect("keeps recent tunnels, other stages, and tunnels without valid timestamps", () => {
329 const state = harness({
330 tunnels: [
331 tunnel({
332 id: "recent",
333 suffix: "aaaaaaaaaaaaaaaa",
334 status: "down",
335 timestamp: "2026-08-25T11:55:01.000Z",
336 }),
337 tunnel({
338 id: "other-stage",
339 prefix: `${PREFIX}julius-`,
340 suffix: "bbbbbbbbbbbbbbbb",
341 status: "down",
342 timestamp: "2026-08-25T11:00:00.000Z",
343 }),
344 tunnel({
345 id: "missing-time",
346 suffix: "cccccccccccccccc",
347 status: "inactive",
348 timestamp: null,
349 }),
350 ],
351 });
352
353 return Effect.gen(function* () {
354 yield* TestClock.setTime(NOW_MILLIS);
355 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
356 expect(yield* reaper.sweep).toMatchObject({
357 scanned: 0,
358 deleted: 0,
359 skippedLegacy: 0,
360 failed: 0,
361 });
362 expect(state.deleted).toEqual([]);
363 }).pipe(Effect.provide(state.layer));
364 });
365
366 it.effect("keeps tunnels owned by environments that cannot recover yet", () => {
367 const state = harness({
368 tunnels: [
369 tunnel({
370 id: "legacy",
371 suffix: "aaaaaaaaaaaaaaaa",
372 status: "down",
373 timestamp: "2026-08-25T11:00:00.000Z",
374 }),
375 ],
376 allocations: [allocation({ tunnelId: "legacy", recoveryEnabled: false })],
377 });
378
379 return Effect.gen(function* () {
380 yield* TestClock.setTime(NOW_MILLIS);
381 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
382 expect(yield* reaper.sweep).toMatchObject({
383 scanned: 1,
384 deleted: 0,
385 skippedLegacy: 1,
386 failed: 0,
387 });
388 expect(state.deleted).toEqual([]);
389 }).pipe(Effect.provide(state.layer));
390 });
391
392 it.effect("counts an expired tunnel with no allocation instead of deleting it", () => {
393 const state = harness({
394 tunnels: [
395 tunnel({
396 id: "orphan",
397 suffix: "cccccccccccccccc",
398 status: "down",
399 timestamp: "2026-08-25T11:00:00.000Z",
400 }),
401 ],
402 });
403
404 return Effect.gen(function* () {
405 yield* TestClock.setTime(NOW_MILLIS);
406 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
407 expect(yield* reaper.sweep).toMatchObject({
408 scanned: 1,
409 wouldDelete: 0,
410 deleted: 0,
411 skippedOrphan: 1,
412 });
413 expect(state.deleted).toEqual([]);
414 }).pipe(Effect.provide(state.layer));
415 });
416 it.effect("keeps an expired tunnel while its allocation is incomplete", () => {
417 const state = harness({
418 tunnels: [
419 tunnel({
420 id: "unrecorded",
421 suffix: "aaaaaaaaaaaaaaaa",
422 status: "inactive",
423 timestamp: "2026-08-25T11:00:00.000Z",
424 }),
425 ],
426 allocations: [allocation({ tunnelId: null, recoveryEnabled: false })],
427 });
428
429 return Effect.gen(function* () {
430 yield* TestClock.setTime(NOW_MILLIS);
431 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
432 expect((yield* reaper.sweep).deleted).toBe(0);
433 expect(state.deleted).toEqual([]);
434 }).pipe(Effect.provide(state.layer));
435 });
436
437 it.effect("does not count a tunnel that was replaced before its release", () => {
438 const state = harness({
439 tunnels: [
440 tunnel({
441 id: "replaced",
442 suffix: "aaaaaaaaaaaaaaaa",
443 status: "down",
444 timestamp: "2026-08-25T11:00:00.000Z",
445 }),
446 ],
447 allocations: [allocation({ tunnelId: "replaced", recoveryEnabled: true })],
448 skipTunnelId: "replaced",
449 });
450
451 return Effect.gen(function* () {
452 yield* TestClock.setTime(NOW_MILLIS);
453 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
454 expect((yield* reaper.sweep).deleted).toBe(0);
455 expect(state.deleted).toEqual([]);
456 }).pipe(Effect.provide(state.layer));
457 });
458
459 it.effect("stops the sweep after a structured Cloudflare rate limit error", () => {
460 const state = harness({
461 tunnels: [
462 tunnel({
463 id: "limited",
464 suffix: "aaaaaaaaaaaaaaaa",
465 status: "down",
466 timestamp: "2026-08-25T11:00:00.000Z",
467 }),
468 tunnel({
469 id: "next",
470 suffix: "bbbbbbbbbbbbbbbb",
471 status: "down",
472 timestamp: "2026-08-25T11:00:00.000Z",
473 }),
474 ],
475 allocations: [
476 allocation({ tunnelId: "limited", recoveryEnabled: true }),
477 allocation({ tunnelId: "next", recoveryEnabled: true }),
478 ],
479 rateLimitedTunnelId: "limited",
480 });
481
482 return Effect.gen(function* () {
483 yield* TestClock.setTime(NOW_MILLIS);
484 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
485 expect(yield* reaper.sweep).toMatchObject({
486 attempted: 1,
487 deleted: 0,
488 failed: 1,
489 truncated: true,
490 });
491 expect(state.deleted).toEqual([]);
492 }).pipe(Effect.provide(state.layer));
493 });
494
495 it.effect("continues past a page of older hosts to find recoverable tunnels", () => {
496 const entries = Array.from({ length: 101 }, (_, index) =>
497 tunnel({
498 id: `tunnel-${index}`,
499 suffix: index.toString(16).padStart(16, "0"),
500 status: "down",
501 timestamp: "2026-08-25T11:00:00.000Z",
502 }),
503 );
504 const state = harness({
505 tunnels: entries,
506 allocations: entries.map((entry, index) =>
507 allocation({ tunnelId: entry.id!, recoveryEnabled: index === 100 }),
508 ),
509 });
510
511 return Effect.gen(function* () {
512 yield* TestClock.setTime(NOW_MILLIS);
513 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
514 expect(yield* reaper.sweep).toMatchObject({
515 scanned: 101,
516 deleted: 1,
517 skippedLegacy: 100,
518 failed: 0,
519 });
520 expect(state.deleted).toEqual(["tunnel-100"]);
521 expect(
522 state.listRequests
523 .filter((request) => request.status === "down")
524 .map((request) => request.page),
525 ).toEqual([1, 2]);
526 }).pipe(Effect.provide(state.layer));
527 });
528
529 it.effect("collects every page before deletions shift Cloudflare pagination", () => {
530 const entries = Array.from({ length: 120 }, (_, index) =>
531 tunnel({
532 id: `tunnel-${index}`,
533 suffix: index.toString(16).padStart(16, "0"),
534 status: "down",
535 timestamp: "2026-08-25T11:00:00.000Z",
536 }),
537 );
538 const state = harness({
539 tunnels: entries,
540 allocations: entries.map((entry, index) =>
541 allocation({ tunnelId: entry.id!, recoveryEnabled: index < 50 || index >= 100 }),
542 ),
543 });
544
545 return Effect.gen(function* () {
546 yield* TestClock.setTime(NOW_MILLIS);
547 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
548 expect(yield* reaper.sweep).toMatchObject({
549 scanned: 120,
550 deleted: 70,
551 skippedLegacy: 50,
552 failed: 0,
553 });
554 expect(state.deleted).toContain("tunnel-119");
555 }).pipe(Effect.provide(state.layer));
556 });
557
558 it.effect("limits each cleanup run to 100 tunnel deletions", () => {
559 const entries = Array.from({ length: 105 }, (_, index) =>
560 tunnel({
561 id: `tunnel-${index}`,
562 suffix: index.toString(16).padStart(16, "0"),
563 status: "down",
564 timestamp: "2026-08-25T11:00:00.000Z",
565 }),
566 );
567 const state = harness({ tunnels: entries, allocations: recoverableOwners(entries) });
568
569 return Effect.gen(function* () {
570 yield* TestClock.setTime(NOW_MILLIS);
571 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
572 expect((yield* reaper.sweep).deleted).toBe(100);
573 expect(state.deleted).toHaveLength(100);
574 }).pipe(Effect.provide(state.layer));
575 });
576
577 it.effect("does no Cloudflare work while cleanup is off", () => {
578 const state = harness({
579 cleanupMode: "off",
580 tunnels: [
581 tunnel({
582 id: "expired",
583 suffix: "aaaaaaaaaaaaaaaa",
584 status: "down",
585 timestamp: "2026-08-25T11:00:00.000Z",
586 }),
587 ],
588 allocations: [allocation({ tunnelId: "expired", recoveryEnabled: true })],
589 });
590
591 return Effect.gen(function* () {
592 yield* TestClock.setTime(NOW_MILLIS);
593 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
594 expect(yield* reaper.sweep).toEqual({
595 mode: "off",
596 listRequests: 0,
597 scanned: 0,
598 attempted: 0,
599 deleted: 0,
600 wouldDelete: 0,
601 skippedLegacy: 0,
602 skippedOrphan: 0,
603 failed: 0,
604 truncated: false,
605 });
606 expect(state.listRequests).toEqual([]);
607 expect(state.deleted).toEqual([]);
608 }).pipe(Effect.provide(state.layer));
609 });
610
611 it.effect("reports candidates without mutating them in dry-run mode", () => {
612 const state = harness({
613 cleanupMode: "dry-run",
614 tunnels: [
615 tunnel({
616 id: "expired",
617 suffix: "aaaaaaaaaaaaaaaa",
618 status: "down",
619 timestamp: "2026-08-25T11:00:00.000Z",
620 }),
621 ],
622 allocations: [allocation({ tunnelId: "expired", recoveryEnabled: true })],
623 });
624
625 return Effect.gen(function* () {
626 yield* TestClock.setTime(NOW_MILLIS);
627 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
628 expect(yield* reaper.sweep).toMatchObject({
629 mode: "dry-run",
630 scanned: 1,
631 attempted: 0,
632 deleted: 0,
633 wouldDelete: 1,
634 });
635 expect(state.deleted).toEqual([]);
636 expect(state.releases).toEqual([]);
637 }).pipe(Effect.provide(state.layer));
638 });
639
640 it.effect("caps failed deletion attempts and Cloudflare list pages", () => {
641 const entries = Array.from({ length: 250 }, (_, index) =>
642 tunnel({
643 id: `failed-${index}`,
644 suffix: index.toString(16).padStart(16, "0"),
645 status: "down",
646 timestamp: "2026-08-25T11:00:00.000Z",
647 }),
648 );
649 const state = harness({
650 tunnels: entries,
651 allocations: recoverableOwners(entries),
652 failAllDeletes: true,
653 });
654
655 return Effect.gen(function* () {
656 yield* TestClock.setTime(NOW_MILLIS);
657 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
658 expect(yield* reaper.sweep).toMatchObject({
659 attempted: 100,
660 deleted: 0,
661 failed: 100,
662 truncated: true,
663 });
664 expect(state.listRequests.length).toBeLessThanOrEqual(
665 ManagedEndpointReaper.MANAGED_ENDPOINT_SWEEP_LIST_REQUEST_LIMIT,
666 );
667 }).pipe(Effect.provide(state.layer));
668 });
669
670 it.effect("rotates the deletion order so persistent failures do not starve later tunnels", () => {
671 const entries = Array.from({ length: 200 }, (_, index) =>
672 tunnel({
673 id: `tunnel-${index}`,
674 suffix: index.toString(16).padStart(16, "0"),
675 status: "down",
676 timestamp: "2026-08-25T11:00:00.000Z",
677 }),
678 );
679 // The first 100 candidates always fail to delete.
680 const state = harness({
681 tunnels: entries,
682 allocations: recoverableOwners(entries),
683 failDeleteWhen: (id) => Number(id.split("-")[1]) < 100,
684 });
685
686 return Effect.gen(function* () {
687 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
688 yield* TestClock.setTime(NOW_MILLIS);
689 yield* reaper.sweep;
690 yield* TestClock.setTime(NOW_MILLIS + 5 * 60 * 1_000);
691 yield* reaper.sweep;
692 const later = state.deleted.filter((id) => Number(id.split("-")[1]) >= 100);
693 expect(later.length).toBeGreaterThan(0);
694 }).pipe(Effect.provide(state.layer));
695 });
696
697 it.effect("rotates bounded pages across a large legacy prefix", () => {
698 const entries = Array.from({ length: 1_000 }, (_, index) =>
699 tunnel({
700 id: `legacy-${index}`,
701 suffix: index.toString(16).padStart(16, "0"),
702 status: "down",
703 timestamp: "2026-08-25T11:00:00.000Z",
704 }),
705 );
706 const state = harness({
707 cleanupMode: "dry-run",
708 tunnels: entries,
709 allocations: entries.map((entry) =>
710 allocation({ tunnelId: entry.id!, recoveryEnabled: false }),
711 ),
712 });
713
714 return Effect.gen(function* () {
715 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
716 yield* TestClock.setTime(NOW_MILLIS);
717 yield* reaper.sweep;
718 const firstPages = state.listRequests
719 .filter((request) => request.status === "down")
720 .map((request) => request.page);
721 state.listRequests.length = 0;
722 yield* TestClock.setTime(NOW_MILLIS + 5 * 60 * 1_000);
723 yield* reaper.sweep;
724 const secondPages = state.listRequests
725 .filter((request) => request.status === "down")
726 .map((request) => request.page);
727 expect(firstPages).not.toEqual(secondPages);
728 expect(firstPages).toHaveLength(5);
729 expect(secondPages).toHaveLength(5);
730 }).pipe(Effect.provide(state.layer));
731 });
732
733 it.effect("rotates the attempt budget across both statuses over consecutive sweeps", () => {
734 const entries = (["down", "inactive"] as const).flatMap((status) =>
735 Array.from({ length: 100 }, (_, index) =>
736 tunnel({
737 id: `${status}-${index}`,
738 suffix: `${status === "down" ? "a" : "b"}${index.toString(16).padStart(15, "0")}`,
739 status,
740 timestamp: "2026-08-25T10:00:00.000Z",
741 }),
742 ),
743 );
744 const state = harness({ tunnels: entries, allocations: recoverableOwners(entries) });
745
746 return Effect.gen(function* () {
747 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
748 yield* TestClock.setTime(NOW_MILLIS);
749 yield* reaper.sweep;
750 const firstStatus = state.deleted[0]!.split("-")[0];
751 expect(state.deleted).toHaveLength(100);
752 expect(state.deleted.every((id) => id.startsWith(`${firstStatus}-`))).toBe(true);
753
754 state.deleted.length = 0;
755 yield* TestClock.setTime(NOW_MILLIS + 5 * 60 * 1_000);
756 yield* reaper.sweep;
757 expect(state.deleted).toHaveLength(100);
758 expect(state.deleted.every((id) => !id.startsWith(`${firstStatus}-`))).toBe(true);
759 }).pipe(Effect.provide(state.layer));
760 });
761});
infra/relay/src/environments/ManagedEndpointReaper.ts created+295
...@@ -0,0 +1,295 @@
1import * as Context from "effect/Context";
2import * as DateTime from "effect/DateTime";
3import * as Effect from "effect/Effect";
4import * as Layer from "effect/Layer";
5import * as Option from "effect/Option";
6
7import type { ManagedEndpointCleanupMode } from "../Config.ts";
8import * as RelayConfiguration from "../Config.ts";
9import { managedEndpointTunnelNamePrefix } from "../deploymentConfig.ts";
10import * as ManagedEndpointAllocations from "./ManagedEndpointAllocations.ts";
11import * as ManagedEndpointProvider from "./ManagedEndpointProvider.ts";
12
13export const MANAGED_ENDPOINT_GRACE_PERIOD_MINUTES = 5;
14// A tunnel that never connected is usually a link still being set up: a slow
15// cloudflared download or a user who walked away mid-pairing. Give it an hour.
16export const MANAGED_ENDPOINT_INACTIVE_GRACE_PERIOD_MINUTES = 60;
17export const MANAGED_ENDPOINT_SWEEP_PAGE_SIZE = 100;
18export const MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT = 100;
19export const MANAGED_ENDPOINT_SWEEP_LIST_REQUEST_LIMIT = 10;
20
21export interface ManagedEndpointSweepResult {
22 readonly mode: ManagedEndpointCleanupMode;
23 readonly listRequests: number;
24 readonly scanned: number;
25 readonly attempted: number;
26 readonly deleted: number;
27 readonly wouldDelete: number;
28 readonly skippedLegacy: number;
29 readonly skippedOrphan: number;
30 readonly failed: number;
31 readonly truncated: boolean;
32}
33
34export class ManagedEndpointReaper extends Context.Service<
35 ManagedEndpointReaper,
36 {
37 readonly sweep: Effect.Effect<
38 ManagedEndpointSweepResult,
39 | ManagedEndpointProvider.ManagedEndpointTunnelClientError
40 | ManagedEndpointAllocations.ManagedEndpointAllocationPersistenceError
41 >;
42 }
43>()("t3code-relay/environments/ManagedEndpointReaper") {}
44
45function isExpiredManagedTunnel(input: {
46 readonly tunnel: ManagedEndpointProvider.ManagedEndpointTunnel;
47 readonly status: "down" | "inactive";
48 readonly prefix: string;
49 readonly cutoff: DateTime.Utc;
50}): input is typeof input & {
51 readonly tunnel: ManagedEndpointProvider.ManagedEndpointTunnel & {
52 readonly id: string;
53 readonly name: string;
54 };
55} {
56 const { tunnel, status, prefix, cutoff } = input;
57 if (
58 typeof tunnel.id !== "string" ||
59 typeof tunnel.name !== "string" ||
60 tunnel.status !== status ||
61 !tunnel.name.startsWith(prefix) ||
62 !/^[a-f0-9]{16}$/u.test(tunnel.name.slice(prefix.length))
63 ) {
64 return false;
65 }
66 const inactiveAt = status === "down" ? tunnel.connsInactiveAt : tunnel.createdAt;
67 if (typeof inactiveAt !== "string") {
68 return false;
69 }
70 const timestamp = DateTime.make(inactiveAt);
71 return Option.isSome(timestamp) && timestamp.value.epochMilliseconds <= cutoff.epochMilliseconds;
72}
73
74function isRateLimited(cause: unknown): boolean {
75 if (typeof cause !== "object" || cause === null) {
76 return false;
77 }
78 if ("_tag" in cause && cause._tag === "TooManyRequests") {
79 return true;
80 }
81 if ("status" in cause && cause.status === 429) {
82 return true;
83 }
84 return "cause" in cause && isRateLimited(cause.cause);
85}
86
87function rotatedPages(input: {
88 readonly totalCount: number | undefined;
89 readonly slot: number;
90 readonly limit: number;
91}): ReadonlyArray<number> {
92 if (input.limit <= 0) return [];
93 if (input.totalCount === undefined) {
94 return Array.from({ length: input.limit }, (_, index) => index + 2);
95 }
96 const laterPageCount = Math.max(
97 0,
98 Math.ceil(input.totalCount / MANAGED_ENDPOINT_SWEEP_PAGE_SIZE) - 1,
99 );
100 if (laterPageCount === 0) return [];
101 const count = Math.min(input.limit, laterPageCount);
102 const start = input.slot % laterPageCount;
103 return Array.from({ length: count }, (_, index) => 2 + ((start + index) % laterPageCount));
104}
105
106const emptyResult = (mode: ManagedEndpointCleanupMode): ManagedEndpointSweepResult => ({
107 mode,
108 listRequests: 0,
109 scanned: 0,
110 attempted: 0,
111 deleted: 0,
112 wouldDelete: 0,
113 skippedLegacy: 0,
114 skippedOrphan: 0,
115 failed: 0,
116 truncated: false,
117});
118
119export const make = Effect.gen(function* () {
120 const config = yield* RelayConfiguration.RelayConfiguration;
121 const tunnels = yield* ManagedEndpointProvider.ManagedEndpointTunnelClient;
122 const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations;
123 const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
124
125 const sweep = Effect.gen(function* () {
126 const mode = config.managedEndpointCleanupMode ?? "off";
127 const namespace = config.managedEndpointNamespace;
128 if (mode === "off" || !namespace) return emptyResult(mode);
129
130 const now = yield* DateTime.now;
131 const cutoffFor = (status: "down" | "inactive") =>
132 DateTime.subtract(now, {
133 minutes:
134 status === "down"
135 ? MANAGED_ENDPOINT_GRACE_PERIOD_MINUTES
136 : MANAGED_ENDPOINT_INACTIVE_GRACE_PERIOD_MINUTES,
137 });
138 const prefix = managedEndpointTunnelNamePrefix(namespace);
139 const slot = Math.floor(
140 now.epochMilliseconds / (MANAGED_ENDPOINT_GRACE_PERIOD_MINUTES * 60 * 1_000),
141 );
142 let listRequests = 0;
143 let truncated = false;
144 const expired: Array<{
145 readonly tunnel: ManagedEndpointProvider.ManagedEndpointTunnel & {
146 readonly id: string;
147 readonly name: string;
148 };
149 readonly status: "down" | "inactive";
150 readonly cutoff: DateTime.Utc;
151 }> = [];
152
153 for (const status of ["down", "inactive"] as const) {
154 const cutoff = cutoffFor(status);
155 const cutoffIso = DateTime.formatIso(cutoff);
156 const listPage = (page: number) => {
157 listRequests += 1;
158 return tunnels.list({
159 isDeleted: false,
160 includePrefix: prefix,
161 status,
162 existedAt: cutoffIso,
163 ...(status === "down" ? { wasInactiveAt: cutoffIso } : {}),
164 page,
165 perPage: MANAGED_ENDPOINT_SWEEP_PAGE_SIZE,
166 });
167 };
168 const first = yield* listPage(1);
169 const totalCount =
170 typeof first.resultInfo?.totalCount === "number" ? first.resultInfo.totalCount : undefined;
171 const pages = rotatedPages({
172 totalCount,
173 slot,
174 limit: Math.floor(MANAGED_ENDPOINT_SWEEP_LIST_REQUEST_LIMIT / 2) - 1,
175 });
176 const responses = [first, ...(yield* Effect.forEach(pages, listPage, { concurrency: 1 }))];
177 if (
178 totalCount !== undefined &&
179 Math.ceil(totalCount / MANAGED_ENDPOINT_SWEEP_PAGE_SIZE) > responses.length
180 ) {
181 truncated = true;
182 } else if (
183 totalCount === undefined &&
184 responses.at(-1)?.result.length === MANAGED_ENDPOINT_SWEEP_PAGE_SIZE
185 ) {
186 truncated = true;
187 }
188 for (const response of responses) {
189 expired.push(
190 ...response.result
191 .map((tunnel) => ({ tunnel, status, prefix, cutoff }))
192 .filter(isExpiredManagedTunnel)
193 .map(({ tunnel }) => ({ tunnel, status, cutoff })),
194 );
195 }
196 }
197
198 const collected = [...new Map(expired.map((entry) => [entry.tunnel.id, entry])).values()];
199 // Start each sweep one attempt budget further along so a run of
200 // candidates whose deletes keep failing cannot hold the budget forever
201 // and starve everything listed after them.
202 const offset =
203 collected.length === 0 ? 0 : (slot * MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT) % collected.length;
204 const uniqueExpired = [...collected.slice(offset), ...collected.slice(0, offset)];
205 const recorded = yield* allocations.listByTunnelNames(
206 uniqueExpired.map(({ tunnel }) => tunnel.name),
207 );
208 const recordedByTunnelName = new Map(
209 recorded.map((allocation) => [allocation.tunnelName, allocation]),
210 );
211 let attempted = 0;
212 let deleted = 0;
213 let wouldDelete = 0;
214 let skippedLegacy = 0;
215 let skippedOrphan = 0;
216 let failed = 0;
217
218 for (const { tunnel, status, cutoff } of uniqueExpired) {
219 const cutoffIso = DateTime.formatIso(cutoff);
220 const allocation = recordedByTunnelName.get(tunnel.name);
221 if (
222 allocation !== undefined &&
223 allocation.tunnelId !== null &&
224 allocation.tunnelId !== tunnel.id
225 ) {
226 continue;
227 }
228 const owner = allocation?.tunnelId === tunnel.id ? allocation : undefined;
229 if (owner !== undefined && !owner.recoveryEnabled) {
230 skippedLegacy += 1;
231 continue;
232 }
233 if (allocation !== undefined && owner === undefined) continue;
234 // A tunnel with no allocation row cannot be claimed, so a relink that
235 // adopts it by name races any delete here. Count it and leave it for a
236 // manual sweep instead.
237 if (owner === undefined) {
238 skippedOrphan += 1;
239 continue;
240 }
241 wouldDelete += 1;
242 if (mode === "dry-run") continue;
243 if (attempted >= MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT) {
244 truncated = true;
245 break;
246 }
247 attempted += 1;
248 const result = yield* provider
249 .release({
250 userId: owner.userId,
251 environmentId: owner.environmentId,
252 expectedTunnelId: tunnel.id,
253 expectedInactiveBefore: cutoffIso,
254 expectedStatus: status,
255 })
256 .pipe(Effect.result);
257 if (result._tag === "Failure") {
258 failed += 1;
259 yield* Effect.logWarning("Failed to delete an inactive managed tunnel", {
260 tunnelId: tunnel.id,
261 tunnelName: tunnel.name,
262 cause: result.failure,
263 });
264 if (isRateLimited(result.failure)) {
265 truncated = true;
266 break;
267 }
268 } else if (result.success) {
269 deleted += 1;
270 yield* Effect.logInfo("Deleted an inactive managed tunnel", {
271 tunnelId: tunnel.id,
272 tunnelName: tunnel.name,
273 status,
274 });
275 }
276 }
277
278 return {
279 mode,
280 listRequests,
281 scanned: uniqueExpired.length,
282 attempted,
283 deleted,
284 wouldDelete,
285 skippedLegacy,
286 skippedOrphan,
287 failed,
288 truncated,
289 };
290 }).pipe(Effect.withSpan("relay.managed_endpoint_reaper.sweep"));
291
292 return ManagedEndpointReaper.of({ sweep });
293});
294
295export const layer = Layer.effect(ManagedEndpointReaper, make);
infra/relay/src/http/Api.test.ts+570-6
...@@ -7,13 +7,15 @@ import {...@@ -7,13 +7,15 @@ import {
7import * as EnvironmentLinker from "../environments/EnvironmentLinker.ts";7import * as EnvironmentLinker from "../environments/EnvironmentLinker.ts";
8import * as RelayTokens from "../auth/RelayTokens.ts";8import * as RelayTokens from "../auth/RelayTokens.ts";
9import * as Devices from "../agentActivity/Devices.ts";9import * as Devices from "../agentActivity/Devices.ts";
10import * as NodeCrypto from "node:crypto";
10import { createClerkClient, verifyToken } from "@clerk/backend";11import { createClerkClient, verifyToken } from "@clerk/backend";
11import * as NodeHttpPlatform from "@effect/platform-node/NodeHttpPlatform";12import * as NodeHttpPlatform from "@effect/platform-node/NodeHttpPlatform";
12import * as NodeServices from "@effect/platform-node/NodeServices";13import * as NodeServices from "@effect/platform-node/NodeServices";
13import { describe, expect, it } from "@effect/vitest";14import { describe, expect, it } from "@effect/vitest";
14import { vi } from "vite-plus/test";15import { vi } from "vite-plus/test";
15import * as Context from "effect/Context";16import * as Context from "effect/Context";
16import * as NodeCrypto from "@effect/platform-node/NodeCrypto";17import * as NodeCryptoLayer from "@effect/platform-node/NodeCrypto";
18import * as DateTime from "effect/DateTime";
17import * as Duration from "effect/Duration";19import * as Duration from "effect/Duration";
18import * as Effect from "effect/Effect";20import * as Effect from "effect/Effect";
19import * as Fiber from "effect/Fiber";21import * as Fiber from "effect/Fiber";
...@@ -35,6 +37,7 @@ import {...@@ -35,6 +37,7 @@ import {
35 RelayEnvironmentPrincipal,37 RelayEnvironmentPrincipal,
36 RelayApi,38 RelayApi,
37} from "@t3tools/contracts/relay";39} from "@t3tools/contracts/relay";
40import { RELAY_MANAGED_TUNNEL_RECOVERY_TYP, signRelayJwt } from "@t3tools/shared/relayJwt";
3841
39import {42import {
40 RELAY_HTTP_ROUTER_CONFIG,43 RELAY_HTTP_ROUTER_CONFIG,
...@@ -44,18 +47,22 @@ import {...@@ -44,18 +47,22 @@ import {
44 relayDocsRedirectRoute,47 relayDocsRedirectRoute,
45 relayEnvironmentAuthLayer,48 relayEnvironmentAuthLayer,
46 relayNotFoundRoute,49 relayNotFoundRoute,
50 recoverEnvironmentTunnelRecord,
51 registerEnvironmentTunnelRecovery,
47 relayDpopFailureReason,52 relayDpopFailureReason,
48 revokeEnvironmentLinkRecord,53 revokeEnvironmentLinkRecord,
49 serverApi,54 serverApi,
50 traceRelayHttpRequestWith,55 traceRelayHttpRequestWith,
51 unlinkEnvironmentRecord,56 unlinkEnvironmentRecord,
52 verifyRelayClientBearerToken,57 verifyRelayClientBearerToken,
58 verifyEnvironmentTunnelRecoveryProof,
53 withoutCapturedParentSpan,59 withoutCapturedParentSpan,
54} from "./Api.ts";60} from "./Api.ts";
55import * as RelayConfiguration from "../Config.ts";61import * as RelayConfiguration from "../Config.ts";
56import * as RelayDb from "../db.ts";62import * as RelayDb from "../db.ts";
57import * as EnvironmentCredentials from "../environments/EnvironmentCredentials.ts";63import * as EnvironmentCredentials from "../environments/EnvironmentCredentials.ts";
58import * as EnvironmentLinks from "../environments/EnvironmentLinks.ts";64import * as EnvironmentLinks from "../environments/EnvironmentLinks.ts";
65import * as ManagedEndpointAllocations from "../environments/ManagedEndpointAllocations.ts";
59import * as ManagedEndpointProvider from "../environments/ManagedEndpointProvider.ts";66import * as ManagedEndpointProvider from "../environments/ManagedEndpointProvider.ts";
60import * as AgentActivityPublisher from "../agentActivity/AgentActivityPublisher.ts";67import * as AgentActivityPublisher from "../agentActivity/AgentActivityPublisher.ts";
61import * as EnvironmentPublishSignatures from "../environments/EnvironmentPublishSignatures.ts";68import * as EnvironmentPublishSignatures from "../environments/EnvironmentPublishSignatures.ts";
...@@ -122,7 +129,7 @@ describe("device listing compatibility", () => {...@@ -122,7 +129,7 @@ describe("device listing compatibility", () => {
122 Layer.provide(129 Layer.provide(
123 Layer.mergeAll(130 Layer.mergeAll(
124 Layer.succeed(RelayConfiguration.RelayConfiguration, relaySettings),131 Layer.succeed(RelayConfiguration.RelayConfiguration, relaySettings),
125 NodeCrypto.layer,132 NodeCryptoLayer.layer,
126 Layer.mock(RelayTokens.RelayTokens, { resolveDpopAccessTokenScopes: () => null }),133 Layer.mock(RelayTokens.RelayTokens, { resolveDpopAccessTokenScopes: () => null }),
127 Layer.mock(EnvironmentLinker.EnvironmentLinker, {}),134 Layer.mock(EnvironmentLinker.EnvironmentLinker, {}),
128 Layer.mock(EnvironmentLinks.EnvironmentLinks, {}),135 Layer.mock(EnvironmentLinks.EnvironmentLinks, {}),
...@@ -307,6 +314,9 @@ function relayUnlinkTestLayer(input?: {...@@ -307,6 +314,9 @@ function relayUnlinkTestLayer(input?: {
307 readonly revokeCredential?: EnvironmentCredentials.EnvironmentCredentials["Service"]["revokeForEnvironmentPublicKey"];314 readonly revokeCredential?: EnvironmentCredentials.EnvironmentCredentials["Service"]["revokeForEnvironmentPublicKey"];
308 readonly prepareDeprovision?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["prepareDeprovision"];315 readonly prepareDeprovision?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["prepareDeprovision"];
309 readonly deprovision?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["deprovision"];316 readonly deprovision?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["deprovision"];
317 readonly provision?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["provision"];
318 readonly reconcileOrigin?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["reconcileOrigin"];
319 readonly release?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["release"];
310}) {320}) {
311 return Layer.mergeAll(321 return Layer.mergeAll(
312 Layer.succeed(322 Layer.succeed(
...@@ -336,10 +346,11 @@ function relayUnlinkTestLayer(input?: {...@@ -336,10 +346,11 @@ function relayUnlinkTestLayer(input?: {
336 Layer.succeed(346 Layer.succeed(
337 ManagedEndpointProvider.ManagedEndpointProvider,347 ManagedEndpointProvider.ManagedEndpointProvider,
338 ManagedEndpointProvider.ManagedEndpointProvider.of({348 ManagedEndpointProvider.ManagedEndpointProvider.of({
339 provision: () => Effect.die("unused provision"),349 provision: input?.provision ?? (() => Effect.die("unused provision")),
350 reconcileOrigin: input?.reconcileOrigin ?? (() => Effect.succeed("ready")),
340 prepareDeprovision: input?.prepareDeprovision ?? (() => Effect.succeed(null)),351 prepareDeprovision: input?.prepareDeprovision ?? (() => Effect.succeed(null)),
341 deprovision: input?.deprovision ?? (() => Effect.void),352 deprovision: input?.deprovision ?? (() => Effect.succeed(true)),
342 release: () => Effect.die("unused release"),353 release: input?.release ?? (() => Effect.die("unused release")),
343 }),354 }),
344 ),355 ),
345 );356 );
...@@ -357,6 +368,502 @@ const linkedEnvironmentRecord = {...@@ -357,6 +368,502 @@ const linkedEnvironmentRecord = {
357 linkedAt: "2026-07-28T00:00:00.000Z",368 linkedAt: "2026-07-28T00:00:00.000Z",
358} as const;369} as const;
359370
371describe("relay managed tunnel recovery", () => {
372 it.effect("binds recovery requests to the host, cloud user, and T3 service origin", () =>
373 Effect.gen(function* () {
374 const keyPair = NodeCrypto.generateKeyPairSync("ed25519", {
375 privateKeyEncoding: { format: "pem", type: "pkcs8" },
376 publicKeyEncoding: { format: "pem", type: "spki" },
377 });
378 const now = yield* DateTime.now;
379 const issuedAt = Math.floor(now.epochMilliseconds / 1_000);
380 const proof = yield* signRelayJwt({
381 privateKey: keyPair.privateKey,
382 typ: RELAY_MANAGED_TUNNEL_RECOVERY_TYP,
383 payload: {
384 iss: "t3-env:environment-1",
385 aud: "https://relay.example.test",
386 sub: "environment-1",
387 jti: "recovery-proof",
388 iat: issuedAt,
389 exp: issuedAt + 60,
390 action: "recover",
391 environmentId: "environment-1",
392 cloudUserId: "user-1",
393 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
394 },
395 });
396 const request = {
397 action: "recover" as const,
398 proof,
399 userId: "user-1",
400 environmentId: "environment-1",
401 environmentPublicKey: keyPair.publicKey,
402 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
403 };
404
405 yield* verifyEnvironmentTunnelRecoveryProof(request);
406
407 const wrongOwner = yield* Effect.flip(
408 verifyEnvironmentTunnelRecoveryProof({ ...request, userId: "user-2" }),
409 );
410 expect(wrongOwner).toMatchObject({ _tag: "Unauthorized" });
411
412 const wrongOrigin = yield* Effect.flip(
413 verifyEnvironmentTunnelRecoveryProof({
414 ...request,
415 origin: { localHttpHost: "127.0.0.1", localHttpPort: 5432 },
416 }),
417 );
418 expect(wrongOrigin).toMatchObject({ _tag: "Unauthorized" });
419
420 const wrongAction = yield* Effect.flip(
421 verifyEnvironmentTunnelRecoveryProof({
422 action: "register",
423 proof,
424 userId: "user-1",
425 environmentId: "environment-1",
426 environmentPublicKey: keyPair.publicKey,
427 tunnelId: "existing-tunnel",
428 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
429 }),
430 );
431 expect(wrongAction).toMatchObject({ _tag: "Unauthorized" });
432 }).pipe(Effect.provideService(RelayConfiguration.RelayConfiguration, relaySettings)),
433 );
434
435 it.effect("rejects a signed registration proof for a different origin", () =>
436 Effect.gen(function* () {
437 const keyPair = NodeCrypto.generateKeyPairSync("ed25519", {
438 privateKeyEncoding: { format: "pem", type: "pkcs8" },
439 publicKeyEncoding: { format: "pem", type: "spki" },
440 });
441 const now = yield* DateTime.now;
442 const issuedAt = Math.floor(now.epochMilliseconds / 1_000);
443 const proof = yield* signRelayJwt({
444 privateKey: keyPair.privateKey,
445 typ: RELAY_MANAGED_TUNNEL_RECOVERY_TYP,
446 payload: {
447 iss: "t3-env:environment-1",
448 aud: "https://relay.example.test",
449 sub: "environment-1",
450 jti: "registration-origin-proof",
451 iat: issuedAt,
452 exp: issuedAt + 60,
453 action: "register",
454 environmentId: "environment-1",
455 cloudUserId: "user-1",
456 tunnelId: "existing-tunnel",
457 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
458 },
459 });
460
461 const error = yield* Effect.flip(
462 verifyEnvironmentTunnelRecoveryProof({
463 action: "register",
464 proof,
465 userId: "user-1",
466 environmentId: "environment-1",
467 environmentPublicKey: keyPair.publicKey,
468 tunnelId: "existing-tunnel",
469 origin: { localHttpHost: "127.0.0.1", localHttpPort: 5432 },
470 }),
471 );
472
473 expect(error).toMatchObject({ _tag: "Unauthorized" });
474 }).pipe(Effect.provideService(RelayConfiguration.RelayConfiguration, relaySettings)),
475 );
476
477 it.effect("registers recovery for an existing tunnel without provisioning it", () => {
478 let recoveryEnabledFor: {
479 readonly userId: string;
480 readonly environmentId: string;
481 readonly tunnelId: string;
482 readonly environmentPublicKey: string;
483 readonly origin: { readonly localHttpHost: string; readonly localHttpPort: number };
484 } | null = null;
485
486 return Effect.gen(function* () {
487 expect(
488 yield* registerEnvironmentTunnelRecovery({
489 userId: "user-1",
490 environmentId: "environment-1",
491 environmentPublicKey: "public-key",
492 tunnelId: "existing-tunnel",
493 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
494 }),
495 ).toEqual({ status: "ready" });
496 expect(recoveryEnabledFor).toEqual({
497 userId: "user-1",
498 environmentId: "environment-1",
499 tunnelId: "existing-tunnel",
500 environmentPublicKey: "public-key",
501 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
502 });
503 }).pipe(
504 Effect.provide(
505 Layer.merge(
506 relayUnlinkTestLayer({
507 getForUser: () => Effect.succeed(linkedEnvironmentRecord),
508 provision: () => Effect.die("registration must not provision a tunnel"),
509 }),
510 Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({
511 enableRecovery: (input) =>
512 Effect.sync(() => {
513 recoveryEnabledFor = input;
514 return true;
515 }),
516 }),
517 ),
518 ),
519 );
520 });
521
522 it.effect("requests recovery without enabling a stale tunnel", () => {
523 let recoveryEnabled = false;
524
525 return Effect.gen(function* () {
526 expect(
527 yield* registerEnvironmentTunnelRecovery({
528 userId: "user-1",
529 environmentId: "environment-1",
530 environmentPublicKey: "public-key",
531 tunnelId: "deleted-tunnel",
532 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
533 }),
534 ).toEqual({ status: "recovery_required" });
535 expect(recoveryEnabled).toBe(false);
536 }).pipe(
537 Effect.provide(
538 Layer.merge(
539 relayUnlinkTestLayer({
540 getForUser: () => Effect.succeed(linkedEnvironmentRecord),
541 reconcileOrigin: () => Effect.succeed("recovery_required"),
542 }),
543 Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({
544 enableRecovery: () =>
545 Effect.sync(() => {
546 recoveryEnabled = true;
547 return true;
548 }),
549 }),
550 ),
551 ),
552 );
553 });
554
555 it.effect("rejects recovery registration for a different environment key", () => {
556 let recoveryEnabled = false;
557
558 return Effect.gen(function* () {
559 const error = yield* Effect.flip(
560 registerEnvironmentTunnelRecovery({
561 userId: "user-1",
562 environmentId: "environment-1",
563 environmentPublicKey: "different-public-key",
564 tunnelId: "existing-tunnel",
565 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
566 }),
567 );
568
569 expect(error).toMatchObject({ _tag: "Unauthorized" });
570 expect(recoveryEnabled).toBe(false);
571 }).pipe(
572 Effect.provide(
573 Layer.merge(
574 relayUnlinkTestLayer({
575 getForUser: () => Effect.succeed(linkedEnvironmentRecord),
576 }),
577 Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({
578 enableRecovery: () =>
579 Effect.sync(() => {
580 recoveryEnabled = true;
581 return true;
582 }),
583 }),
584 ),
585 ),
586 );
587 });
588
589 it.effect("rejects recovery registration when the recorded tunnel changed", () =>
590 Effect.gen(function* () {
591 const error = yield* Effect.flip(
592 registerEnvironmentTunnelRecovery({
593 userId: "user-1",
594 environmentId: "environment-1",
595 environmentPublicKey: "public-key",
596 tunnelId: "stale-tunnel",
597 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
598 }),
599 );
600
601 expect(error).toMatchObject({ _tag: "Unauthorized" });
602 }).pipe(
603 Effect.provide(
604 Layer.merge(
605 relayUnlinkTestLayer({
606 getForUser: () => Effect.succeed(linkedEnvironmentRecord),
607 }),
608 Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({
609 enableRecovery: () => Effect.succeed(false),
610 }),
611 ),
612 ),
613 ),
614 );
615
616 it.effect("recovers a linked environment and marks its tunnel as recoverable", () => {
617 let recoveryEnabledFor: {
618 readonly userId: string;
619 readonly environmentId: string;
620 readonly tunnelId: string;
621 readonly environmentPublicKey: string;
622 } | null = null;
623 const runtime = {
624 providerKind: "cloudflare_tunnel" as const,
625 connectorToken: "replacement-token",
626 tunnelId: "replacement-tunnel",
627 };
628
629 return Effect.gen(function* () {
630 expect(
631 yield* recoverEnvironmentTunnelRecord({
632 userId: "user-1",
633 environmentId: "environment-1",
634 environmentPublicKey: "public-key",
635 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
636 }),
637 ).toEqual({
638 endpoint: linkedEnvironmentRecord.endpoint,
639 endpointRuntime: runtime,
640 });
641 expect(recoveryEnabledFor).toEqual({
642 userId: "user-1",
643 environmentId: "environment-1",
644 tunnelId: "replacement-tunnel",
645 environmentPublicKey: "public-key",
646 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
647 });
648 }).pipe(
649 Effect.provide(
650 Layer.merge(
651 relayUnlinkTestLayer({
652 getForUser: () => Effect.succeed(linkedEnvironmentRecord),
653 provision: () =>
654 Effect.succeed({
655 endpoint: linkedEnvironmentRecord.endpoint,
656 runtime,
657 }),
658 }),
659 Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({
660 enableRecovery: (input) =>
661 Effect.sync(() => {
662 recoveryEnabledFor = input;
663 return true;
664 }),
665 }),
666 ),
667 ),
668 );
669 });
670
671 it.effect("rejects a credential from a different environment owner", () => {
672 let provisioned = false;
673
674 return Effect.gen(function* () {
675 const error = yield* Effect.flip(
676 recoverEnvironmentTunnelRecord({
677 userId: "user-1",
678 environmentId: "environment-1",
679 environmentPublicKey: "different-public-key",
680 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
681 }),
682 );
683 expect(error).toMatchObject({ _tag: "Unauthorized" });
684 expect(provisioned).toBe(false);
685 }).pipe(
686 Effect.provide(
687 Layer.merge(
688 relayUnlinkTestLayer({
689 getForUser: () => Effect.succeed(linkedEnvironmentRecord),
690 provision: () =>
691 Effect.sync(() => {
692 provisioned = true;
693 return {
694 endpoint: linkedEnvironmentRecord.endpoint,
695 runtime: { providerKind: "cloudflare_tunnel", connectorToken: "token" },
696 };
697 }),
698 }),
699 Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({
700 enableRecovery: () => Effect.die("unused"),
701 }),
702 ),
703 ),
704 );
705 });
706
707 it.effect("does not recover a publish-only environment", () =>
708 Effect.gen(function* () {
709 const error = yield* Effect.flip(
710 recoverEnvironmentTunnelRecord({
711 userId: "user-1",
712 environmentId: "environment-1",
713 environmentPublicKey: "public-key",
714 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
715 }),
716 );
717 expect(error).toMatchObject({ _tag: "Unauthorized" });
718 }).pipe(
719 Effect.provide(
720 Layer.merge(
721 relayUnlinkTestLayer({
722 getForUser: () =>
723 Effect.succeed({
724 ...linkedEnvironmentRecord,
725 endpoint: {
726 ...linkedEnvironmentRecord.endpoint,
727 providerKind: "manual" as const,
728 },
729 }),
730 }),
731 Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({
732 enableRecovery: () => Effect.die("unused"),
733 }),
734 ),
735 ),
736 ),
737 );
738
739 it.effect("rejects a recovered tunnel that changes the linked endpoint", () => {
740 let recoveryEnabled = false;
741 const cleaned: Array<string> = [];
742
743 return Effect.gen(function* () {
744 const error = yield* Effect.flip(
745 recoverEnvironmentTunnelRecord({
746 userId: "user-1",
747 environmentId: "environment-1",
748 environmentPublicKey: "public-key",
749 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
750 }),
751 );
752 expect(error).toMatchObject({ _tag: "Unauthorized" });
753 expect(recoveryEnabled).toBe(false);
754 expect(cleaned).toEqual(["replacement-tunnel"]);
755 }).pipe(
756 Effect.provide(
757 Layer.merge(
758 relayUnlinkTestLayer({
759 getForUser: () => Effect.succeed(linkedEnvironmentRecord),
760 provision: () =>
761 Effect.succeed({
762 endpoint: {
763 httpBaseUrl: "https://different.example.test/",
764 wsBaseUrl: "wss://different.example.test/ws",
765 providerKind: "cloudflare_tunnel",
766 },
767 runtime: {
768 providerKind: "cloudflare_tunnel",
769 connectorToken: "token",
770 tunnelId: "replacement-tunnel",
771 },
772 }),
773 prepareDeprovision: () => Effect.die("must keep the active allocation"),
774 deprovision: () => Effect.die("must keep the active link DNS"),
775 release: ({ expectedTunnelId }) =>
776 Effect.sync(() => {
777 if (expectedTunnelId) {
778 cleaned.push(expectedTunnelId);
779 }
780 return true;
781 }),
782 }),
783 Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({
784 enableRecovery: () =>
785 Effect.sync(() => {
786 recoveryEnabled = true;
787 return true;
788 }),
789 }),
790 ),
791 ),
792 );
793 });
794
795 it.effect.each([
796 { state: "removed", currentLink: null },
797 {
798 state: "publish-only",
799 currentLink: {
800 ...linkedEnvironmentRecord,
801 endpoint: {
802 ...linkedEnvironmentRecord.endpoint,
803 providerKind: "manual" as const,
804 },
805 },
806 },
807 ])("removes a recovered tunnel when its link becomes $state", ({ currentLink }) => {
808 let lookups = 0;
809 const cleaned: Array<string> = [];
810 const target = {
811 userId: "user-1",
812 environmentId: "environment-1",
813 hostname: "environment-1.example.test",
814 tunnelId: "replacement-tunnel",
815 tunnelName: "environment-1-tunnel",
816 dnsRecordId: "dns-1",
817 readyAt: "2026-07-28T00:00:00.000Z",
818 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
819 updatedAt: "replacement-generation",
820 generation: 3,
821 } satisfies ManagedEndpointProvider.ManagedEndpointDeprovisionTarget;
822
823 return Effect.gen(function* () {
824 const error = yield* Effect.flip(
825 recoverEnvironmentTunnelRecord({
826 userId: "user-1",
827 environmentId: "environment-1",
828 environmentPublicKey: "public-key",
829 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
830 }),
831 );
832 expect(error).toMatchObject({ _tag: "Unauthorized" });
833 expect(cleaned).toEqual(["replacement-tunnel"]);
834 }).pipe(
835 Effect.provide(
836 Layer.merge(
837 relayUnlinkTestLayer({
838 getForUser: () =>
839 Effect.sync(() => (++lookups === 1 ? linkedEnvironmentRecord : currentLink)),
840 provision: () =>
841 Effect.succeed({
842 endpoint: linkedEnvironmentRecord.endpoint,
843 runtime: {
844 providerKind: "cloudflare_tunnel",
845 connectorToken: "replacement-token",
846 tunnelId: "replacement-tunnel",
847 },
848 }),
849 prepareDeprovision: () => Effect.succeed(target),
850 deprovision: ({ target: captured }) =>
851 Effect.sync(() => {
852 if (captured?.tunnelId) {
853 cleaned.push(captured.tunnelId);
854 }
855 return true;
856 }),
857 }),
858 Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({
859 enableRecovery: () => Effect.succeed(false),
860 }),
861 ),
862 ),
863 );
864 });
865});
866
360describe("relay environment unlink", () => {867describe("relay environment unlink", () => {
361 it.effect("revokes the link and its credentials in one database transaction", () => {868 it.effect("revokes the link and its credentials in one database transaction", () => {
362 const calls: Array<string> = [];869 const calls: Array<string> = [];
...@@ -401,7 +908,9 @@ describe("relay environment unlink", () => {...@@ -401,7 +908,9 @@ describe("relay environment unlink", () => {
401 tunnelName: "environment-1-tunnel",908 tunnelName: "environment-1-tunnel",
402 dnsRecordId: "dns-1",909 dnsRecordId: "dns-1",
403 readyAt: "2026-07-28T00:00:00.000Z",910 readyAt: "2026-07-28T00:00:00.000Z",
911 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
404 updatedAt: "generation-before-unlink",912 updatedAt: "generation-before-unlink",
913 generation: 1,
405 } satisfies ManagedEndpointProvider.ManagedEndpointDeprovisionTarget;914 } satisfies ManagedEndpointProvider.ManagedEndpointDeprovisionTarget;
406915
407 return Effect.gen(function* () {916 return Effect.gen(function* () {
...@@ -450,6 +959,7 @@ describe("relay environment unlink", () => {...@@ -450,6 +959,7 @@ describe("relay environment unlink", () => {
450 Effect.sync(() => {959 Effect.sync(() => {
451 expect(request.target).toBe(deprovisionTarget);960 expect(request.target).toBe(deprovisionTarget);
452 calls.push("deprovision");961 calls.push("deprovision");
962 return true;
453 }),963 }),
454 }),964 }),
455 ),965 ),
...@@ -498,6 +1008,7 @@ describe("relay environment unlink", () => {...@@ -498,6 +1008,7 @@ describe("relay environment unlink", () => {
498 deprovision: () =>1008 deprovision: () =>
499 Effect.sync(() => {1009 Effect.sync(() => {
500 calls.push("deprovision");1010 calls.push("deprovision");
1011 return true;
501 }),1012 }),
502 }),1013 }),
503 ),1014 ),
...@@ -525,6 +1036,47 @@ describe("relay environment unlink", () => {...@@ -525,6 +1036,47 @@ describe("relay environment unlink", () => {
525 deprovision: () =>1036 deprovision: () =>
526 Effect.sync(() => {1037 Effect.sync(() => {
527 calls.push("deprovision");1038 calls.push("deprovision");
1039 return true;
1040 }),
1041 }),
1042 ),
1043 );
1044 });
1045
1046 it.effect("retries unlink cleanup when a concurrent tunnel release wins the first claim", () => {
1047 let lookups = 0;
1048 const targets: Array<ManagedEndpointProvider.ManagedEndpointDeprovisionTarget | undefined> = [];
1049 const target = {
1050 userId: "user-1",
1051 environmentId: "environment-1",
1052 hostname: "environment-1.example.test",
1053 tunnelId: "tunnel-1",
1054 tunnelName: "environment-1-tunnel",
1055 dnsRecordId: "dns-1",
1056 readyAt: "2026-07-28T00:00:00.000Z",
1057 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
1058 updatedAt: "original-generation",
1059 generation: 1,
1060 } satisfies ManagedEndpointProvider.ManagedEndpointDeprovisionTarget;
1061
1062 return Effect.gen(function* () {
1063 expect(
1064 yield* unlinkEnvironmentRecord({
1065 userId: "user-1",
1066 environmentId: "environment-1",
1067 }),
1068 ).toBe(true);
1069 expect(targets).toEqual([target, target]);
1070 }).pipe(
1071 Effect.provide(
1072 relayUnlinkTestLayer({
1073 getForUser: () => Effect.sync(() => (++lookups === 1 ? linkedEnvironmentRecord : null)),
1074 revokeForUser: () => Effect.succeed(true),
1075 prepareDeprovision: () => Effect.succeed(target),
1076 deprovision: ({ target: captured }) =>
1077 Effect.sync(() => {
1078 targets.push(captured ?? undefined);
1079 return targets.length > 1;
528 }),1080 }),
529 }),1081 }),
530 ),1082 ),
...@@ -652,7 +1204,19 @@ describe("relay routing fallback", () => {...@@ -652,7 +1204,19 @@ describe("relay routing fallback", () => {
652 const routes = HttpApiBuilder.layer(1204 const routes = HttpApiBuilder.layer(
653 HttpApi.make("RelayApi").add(RelayApi.groups.server),1205 HttpApi.make("RelayApi").add(RelayApi.groups.server),
654 ).pipe(1206 ).pipe(
655 Layer.provide(serverApi.pipe(Layer.provide([publisher, signatures]))),1207 Layer.provide(
1208 serverApi.pipe(
1209 HttpRouter.provideRequest(
1210 Layer.mergeAll(
1211 Layer.succeed(RelayConfiguration.RelayConfiguration, relaySettings),
1212 Layer.mock(EnvironmentLinks.EnvironmentLinks, {}),
1213 Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations, {}),
1214 Layer.mock(ManagedEndpointProvider.ManagedEndpointProvider, {}),
1215 ),
1216 ),
1217 Layer.provide([publisher, signatures]),
1218 ),
1219 ),
656 Layer.provide(auth),1220 Layer.provide(auth),
657 Layer.provide([NodeServices.layer, NodeHttpPlatform.layer, Etag.layerWeak]),1221 Layer.provide([NodeServices.layer, NodeHttpPlatform.layer, Etag.layerWeak]),
658 );1222 );
infra/relay/src/http/Api.ts+281-3
...@@ -47,10 +47,16 @@ import {...@@ -47,10 +47,16 @@ import {
47 RelayEnvironmentLinkLimitExceededError,47 RelayEnvironmentLinkLimitExceededError,
48 RelayEnvironmentPrincipal,48 RelayEnvironmentPrincipal,
49 type RelayEnvironmentConnectRequest,49 type RelayEnvironmentConnectRequest,
50 type RelayManagedEndpointOrigin,
51 RelayManagedEndpointRecoveryProofPayload,
50 type RelayDpopAccessTokenScope,52 type RelayDpopAccessTokenScope,
51 RelayInternalError,53 RelayInternalError,
52} from "@t3tools/contracts/relay";54} from "@t3tools/contracts/relay";
53import { normalizeRelayIssuer } from "@t3tools/shared/relayJwt";55import {
56 normalizeRelayIssuer,
57 RELAY_MANAGED_TUNNEL_RECOVERY_TYP,
58 verifyRelayJwt,
59} from "@t3tools/shared/relayJwt";
5460
55import * as DeliveryAttempts from "../agentActivity/DeliveryAttempts.ts";61import * as DeliveryAttempts from "../agentActivity/DeliveryAttempts.ts";
56import * as AgentActivityRows from "../agentActivity/AgentActivityRows.ts";62import * as AgentActivityRows from "../agentActivity/AgentActivityRows.ts";
...@@ -99,6 +105,10 @@ const relayCorsPreflightHeaders = {...@@ -99,6 +105,10 @@ const relayCorsPreflightHeaders = {
99 "access-control-max-age": "86400",105 "access-control-max-age": "86400",
100} as const;106} as const;
101107
108const decodeManagedTunnelRecoveryProof = Schema.decodeUnknownEffect(
109 RelayManagedEndpointRecoveryProofPayload,
110);
111
102const appendRelayCredentialResponseHeaders = HttpEffect.appendPreResponseHandler(112const appendRelayCredentialResponseHeaders = HttpEffect.appendPreResponseHandler(
103 (_request, response) =>113 (_request, response) =>
104 Effect.succeed(114 Effect.succeed(
...@@ -462,15 +472,207 @@ export const unlinkEnvironmentRecord = Effect.fn("relay.api.client.unlinkEnviron...@@ -462,15 +472,207 @@ export const unlinkEnvironmentRecord = Effect.fn("relay.api.client.unlinkEnviron
462 // revocation commits so a database failure leaves a fully usable active472 // revocation commits so a database failure leaves a fully usable active
463 // link. Still run teardown when the link is already revoked, allowing a473 // link. Still run teardown when the link is already revoked, allowing a
464 // retry to finish cleanup after an earlier Cloudflare failure.474 // retry to finish cleanup after an earlier Cloudflare failure.
465 yield* managedEndpointProvider.deprovision({475 const deprovisioned = yield* managedEndpointProvider.deprovision({
466 userId: input.userId,476 userId: input.userId,
467 environmentId: input.environmentId,477 environmentId: input.environmentId,
468 target: deprovisionTarget,478 target: deprovisionTarget,
469 });479 });
480 if (!deprovisioned) {
481 const retryTarget = yield* managedEndpointProvider.prepareDeprovision(input);
482 if (retryTarget !== null && (yield* links.getForUser(input)) === null) {
483 yield* managedEndpointProvider.deprovision({ ...input, target: retryTarget });
484 }
485 }
470 return unlinked;486 return unlinked;
471 },487 },
472);488);
473489
490type EnvironmentTunnelRecoveryProofInput = {
491 readonly proof: string;
492 readonly userId: string;
493 readonly environmentId: string;
494 readonly environmentPublicKey: string;
495} & (
496 | {
497 readonly action: "register";
498 readonly tunnelId: string;
499 readonly origin: RelayManagedEndpointOrigin;
500 }
501 | { readonly action: "recover"; readonly origin: RelayManagedEndpointOrigin }
502);
503
504export const verifyEnvironmentTunnelRecoveryProof = Effect.fn(
505 "relay.api.server.verifyEnvironmentTunnelRecoveryProof",
506)(function* (input: EnvironmentTunnelRecoveryProofInput) {
507 const config = yield* RelayConfiguration.RelayConfiguration;
508 const now = yield* DateTime.now;
509 const verified = yield* verifyRelayJwt({
510 publicKey: input.environmentPublicKey,
511 token: input.proof,
512 typ: RELAY_MANAGED_TUNNEL_RECOVERY_TYP,
513 issuer: `t3-env:${input.environmentId}`,
514 audience: normalizeRelayIssuer(config.relayIssuer),
515 nowEpochSeconds: Math.floor(now.epochMilliseconds / 1_000),
516 }).pipe(
517 Effect.flatMap(decodeManagedTunnelRecoveryProof),
518 Effect.mapError(() => new HttpApiError.Unauthorized({})),
519 );
520
521 if (
522 verified.environmentId !== input.environmentId ||
523 verified.sub !== input.environmentId ||
524 verified.cloudUserId !== input.userId ||
525 verified.action !== input.action
526 ) {
527 return yield* new HttpApiError.Unauthorized({});
528 }
529 if (input.action === "register") {
530 if (
531 verified.action !== "register" ||
532 verified.tunnelId !== input.tunnelId ||
533 verified.origin.localHttpHost !== input.origin.localHttpHost ||
534 verified.origin.localHttpPort !== input.origin.localHttpPort
535 ) {
536 return yield* new HttpApiError.Unauthorized({});
537 }
538 return;
539 }
540 if (
541 verified.action !== "recover" ||
542 verified.origin.localHttpHost !== input.origin.localHttpHost ||
543 verified.origin.localHttpPort !== input.origin.localHttpPort
544 ) {
545 return yield* new HttpApiError.Unauthorized({});
546 }
547});
548
549export const registerEnvironmentTunnelRecovery = Effect.fn(
550 "relay.api.server.registerEnvironmentTunnelRecovery",
551)(function* (input: {
552 readonly userId: string;
553 readonly environmentId: string;
554 readonly environmentPublicKey: string;
555 readonly tunnelId: string;
556 readonly origin: RelayManagedEndpointOrigin;
557}) {
558 const links = yield* EnvironmentLinks.EnvironmentLinks;
559 const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations;
560 const managedEndpointProvider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
561 const link = yield* links.getForUser({
562 userId: input.userId,
563 environmentId: input.environmentId,
564 });
565 if (
566 link === null ||
567 link.environmentPublicKey !== input.environmentPublicKey ||
568 link.endpoint.providerKind !== "cloudflare_tunnel"
569 ) {
570 return yield* new HttpApiError.Unauthorized({});
571 }
572 const status = yield* managedEndpointProvider.reconcileOrigin({
573 userId: input.userId,
574 environmentId: input.environmentId,
575 tunnelId: input.tunnelId,
576 origin: input.origin,
577 endpoint: link.endpoint,
578 });
579 if (status === "recovery_required") {
580 return { status };
581 }
582 if (!(yield* allocations.enableRecovery(input))) {
583 return yield* new HttpApiError.Unauthorized({});
584 }
585 return { status };
586});
587
588export const recoverEnvironmentTunnelRecord = Effect.fn(
589 "relay.api.server.recoverEnvironmentTunnelRecord",
590)(function* (input: {
591 readonly userId: string;
592 readonly environmentId: string;
593 readonly environmentPublicKey: string;
594 readonly origin: RelayManagedEndpointOrigin;
595}) {
596 const links = yield* EnvironmentLinks.EnvironmentLinks;
597 const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations;
598 const managedEndpointProvider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
599 const link = yield* links.getForUser({
600 userId: input.userId,
601 environmentId: input.environmentId,
602 });
603 if (
604 link === null ||
605 link.environmentPublicKey !== input.environmentPublicKey ||
606 link.endpoint.providerKind !== "cloudflare_tunnel"
607 ) {
608 return yield* new HttpApiError.Unauthorized({});
609 }
610
611 const recovered = yield* managedEndpointProvider.provision({
612 userId: input.userId,
613 environmentId: input.environmentId,
614 origin: input.origin,
615 });
616 const recoveredTunnelId = recovered.runtime.tunnelId;
617 if (
618 recoveredTunnelId === undefined ||
619 recovered.endpoint.httpBaseUrl !== link.endpoint.httpBaseUrl ||
620 recovered.endpoint.wsBaseUrl !== link.endpoint.wsBaseUrl
621 ) {
622 if (recoveredTunnelId !== undefined) {
623 yield* managedEndpointProvider
624 .release({
625 userId: input.userId,
626 environmentId: input.environmentId,
627 expectedTunnelId: recoveredTunnelId,
628 })
629 .pipe(
630 Effect.catch((cause) =>
631 Effect.logWarning("Failed to clean up a tunnel with a mismatched endpoint", {
632 userId: input.userId,
633 environmentId: input.environmentId,
634 tunnelId: recoveredTunnelId,
635 cause,
636 }),
637 ),
638 );
639 }
640 return yield* new HttpApiError.Unauthorized({});
641 }
642
643 const enabled = yield* allocations.enableRecovery({
644 userId: input.userId,
645 environmentId: input.environmentId,
646 tunnelId: recoveredTunnelId,
647 environmentPublicKey: input.environmentPublicKey,
648 origin: input.origin,
649 });
650 if (!enabled) {
651 const owner = { userId: input.userId, environmentId: input.environmentId };
652 const target = yield* managedEndpointProvider.prepareDeprovision(owner);
653 const currentLink = target === null ? null : yield* links.getForUser(input);
654 if (
655 target !== null &&
656 (currentLink === null || currentLink.endpoint.providerKind !== "cloudflare_tunnel")
657 ) {
658 yield* managedEndpointProvider.deprovision({ ...owner, target }).pipe(
659 Effect.catch((cause) =>
660 Effect.logWarning("Failed to clean up a tunnel after its managed link was removed", {
661 userId: input.userId,
662 environmentId: input.environmentId,
663 cause,
664 }),
665 ),
666 );
667 }
668 return yield* new HttpApiError.Unauthorized({});
669 }
670 return {
671 endpoint: recovered.endpoint,
672 endpointRuntime: recovered.runtime,
673 };
674});
675
474export const mobileApi = HttpApiBuilder.group(676export const mobileApi = HttpApiBuilder.group(
475 RelayApi,677 RelayApi,
476 "mobile",678 "mobile",
...@@ -879,7 +1081,7 @@ export const serverApi = HttpApiBuilder.group(...@@ -879,7 +1081,7 @@ export const serverApi = HttpApiBuilder.group(
879 Effect.fnUntraced(function* (handlers) {1081 Effect.fnUntraced(function* (handlers) {
880 const publisher = yield* AgentActivityPublisher.AgentActivityPublisher;1082 const publisher = yield* AgentActivityPublisher.AgentActivityPublisher;
881 const publishSignatures = yield* EnvironmentPublishSignatures.EnvironmentPublishSignatures;1083 const publishSignatures = yield* EnvironmentPublishSignatures.EnvironmentPublishSignatures;
882 return handlers.handle(1084 const activityHandlers = handlers.handle(
883 "publishAgentActivity",1085 "publishAgentActivity",
884 Effect.fn("relay.api.server.publishAgentActivity")(1086 Effect.fn("relay.api.server.publishAgentActivity")(
885 function* (args) {1087 function* (args) {
...@@ -1013,6 +1215,82 @@ export const serverApi = HttpApiBuilder.group(...@@ -1013,6 +1215,82 @@ export const serverApi = HttpApiBuilder.group(
1013 mapRelayCommonApiErrors("not_authorized"),1215 mapRelayCommonApiErrors("not_authorized"),
1014 ),1216 ),
1015 );1217 );
1218
1219 return activityHandlers
1220 .handle(
1221 "registerManagedEndpointRecovery",
1222 Effect.fn("relay.api.server.registerManagedEndpointRecovery")(
1223 function* ({ params, payload }) {
1224 const principal = yield* RelayEnvironmentPrincipal;
1225 if (principal.environmentId !== params.environmentId) {
1226 return yield* new HttpApiError.Unauthorized({});
1227 }
1228 yield* verifyEnvironmentTunnelRecoveryProof({
1229 action: "register",
1230 proof: payload.proof,
1231 userId: payload.cloudUserId,
1232 environmentId: params.environmentId,
1233 environmentPublicKey: principal.environmentPublicKey,
1234 tunnelId: payload.tunnelId,
1235 origin: payload.origin,
1236 });
1237 yield* appendRelayCredentialResponseHeaders;
1238 return yield* registerEnvironmentTunnelRecovery({
1239 userId: payload.cloudUserId,
1240 environmentId: params.environmentId,
1241 environmentPublicKey: principal.environmentPublicKey,
1242 tunnelId: payload.tunnelId,
1243 origin: payload.origin,
1244 });
1245 },
1246 Effect.catchTags({
1247 ManagedEndpointOriginNotAllowed: () => Effect.fail(new HttpApiError.Unauthorized({})),
1248 ManagedEndpointProvisioningNotConfigured: () =>
1249 relayInternalErrorResponse("upstream_unavailable"),
1250 ManagedEndpointProvisioningFailed: () =>
1251 relayInternalErrorResponse("upstream_unavailable"),
1252 ManagedTunnelLimitExceeded: () => relayInternalErrorResponse("upstream_unavailable"),
1253 }),
1254 mapRelayCommonApiErrors("not_authorized"),
1255 ),
1256 )
1257 .handle(
1258 "recoverManagedEndpoint",
1259 Effect.fn("relay.api.server.recoverManagedEndpoint")(
1260 function* ({ params, payload }) {
1261 const principal = yield* RelayEnvironmentPrincipal;
1262 if (principal.environmentId !== params.environmentId) {
1263 return yield* new HttpApiError.Unauthorized({});
1264 }
1265 yield* verifyEnvironmentTunnelRecoveryProof({
1266 action: "recover",
1267 proof: payload.proof,
1268 userId: payload.cloudUserId,
1269 environmentId: params.environmentId,
1270 environmentPublicKey: principal.environmentPublicKey,
1271 origin: payload.origin,
1272 });
1273 yield* appendRelayCredentialResponseHeaders;
1274 return yield* recoverEnvironmentTunnelRecord({
1275 userId: payload.cloudUserId,
1276 environmentId: params.environmentId,
1277 environmentPublicKey: principal.environmentPublicKey,
1278 origin: payload.origin,
1279 });
1280 },
1281 Effect.catchTags({
1282 ManagedEndpointOriginNotAllowed: () => Effect.fail(new HttpApiError.Unauthorized({})),
1283 ManagedEndpointProvisioningNotConfigured: () =>
1284 relayInternalErrorResponse("upstream_unavailable"),
1285 ManagedEndpointProvisioningFailed: () =>
1286 relayInternalErrorResponse("upstream_unavailable"),
1287 ManagedEndpointDeprovisioningFailed: () =>
1288 relayInternalErrorResponse("upstream_unavailable"),
1289 ManagedTunnelLimitExceeded: () => relayInternalErrorResponse("upstream_unavailable"),
1290 }),
1291 mapRelayCommonApiErrors("not_authorized"),
1292 ),
1293 );
1016 }),1294 }),
1017);1295);
10181296
infra/relay/src/persistence/schema.ts+5
...@@ -2,6 +2,7 @@ import type {...@@ -2,6 +2,7 @@ import type {
2 RelayAgentActivityAggregateState,2 RelayAgentActivityAggregateState,
3 RelayAgentActivityState,3 RelayAgentActivityState,
4 RelayAgentAwarenessPreferences,4 RelayAgentAwarenessPreferences,
5 RelayManagedEndpointOrigin,
5} from "@t3tools/contracts/relay";6} from "@t3tools/contracts/relay";
6import {7import {
7 boolean,8 boolean,
...@@ -94,6 +95,10 @@ export const relayManagedEndpointAllocations = pgTable(...@@ -94,6 +95,10 @@ export const relayManagedEndpointAllocations = pgTable(
94 tunnelName: text("tunnel_name").notNull(),95 tunnelName: text("tunnel_name").notNull(),
95 dnsRecordId: varchar("dns_record_id", { length: 191 }),96 dnsRecordId: varchar("dns_record_id", { length: 191 }),
96 readyAt: varchar("ready_at", { length: 64 }),97 readyAt: varchar("ready_at", { length: 64 }),
98 recoveryEnabledAt: varchar("recovery_enabled_at", { length: 64 }),
99 recoveryEnvironmentPublicKey: text("recovery_environment_public_key"),
100 origin: jsonb("origin").$type<RelayManagedEndpointOrigin>(),
101 generation: integer("generation").notNull().default(0),
97 createdAt: varchar("created_at", { length: 64 }).notNull(),102 createdAt: varchar("created_at", { length: 64 }).notNull(),
98 updatedAt: varchar("updated_at", { length: 64 }).notNull(),103 updatedAt: varchar("updated_at", { length: 64 }).notNull(),
99 },104 },
infra/relay/src/worker.ts+41-15
...@@ -2,6 +2,7 @@ import * as Alchemy from "alchemy";...@@ -2,6 +2,7 @@ import * as Alchemy from "alchemy";
2import * as Cloudflare from "alchemy/Cloudflare";2import * as Cloudflare from "alchemy/Cloudflare";
3import * as Drizzle from "alchemy/Drizzle/Postgres";3import * as Drizzle from "alchemy/Drizzle/Postgres";
4import * as Config from "effect/Config";4import * as Config from "effect/Config";
5import * as Cause from "effect/Cause";
5import * as DateTime from "effect/DateTime";6import * as DateTime from "effect/DateTime";
6import * as Crypto from "effect/Crypto";7import * as Crypto from "effect/Crypto";
7import * as Effect from "effect/Effect";8import * as Effect from "effect/Effect";
...@@ -69,6 +70,7 @@ import * as EnvironmentConnector from "./environments/EnvironmentConnector.ts";...@@ -69,6 +70,7 @@ import * as EnvironmentConnector from "./environments/EnvironmentConnector.ts";
69import * as EnvironmentLinker from "./environments/EnvironmentLinker.ts";70import * as EnvironmentLinker from "./environments/EnvironmentLinker.ts";
70import * as EnvironmentPublishSignatures from "./environments/EnvironmentPublishSignatures.ts";71import * as EnvironmentPublishSignatures from "./environments/EnvironmentPublishSignatures.ts";
71import * as ManagedEndpointProvider from "./environments/ManagedEndpointProvider.ts";72import * as ManagedEndpointProvider from "./environments/ManagedEndpointProvider.ts";
73import * as ManagedEndpointReaper from "./environments/ManagedEndpointReaper.ts";
72import * as ManagedTunnelLimits from "./environments/ManagedTunnelLimits.ts";74import * as ManagedTunnelLimits from "./environments/ManagedTunnelLimits.ts";
73import * as MobileRegistrations from "./agentActivity/MobileRegistrations.ts";75import * as MobileRegistrations from "./agentActivity/MobileRegistrations.ts";
7476
...@@ -180,6 +182,7 @@ export const ApiLive = Api.make(...@@ -180,6 +182,7 @@ export const ApiLive = Api.make(
180 yield* yield* relayApiZone.zoneId;182 yield* yield* relayApiZone.zoneId;
181 const managedEndpointDnsBinding = yield* Cloudflare.DNS.ReadWriteDns(managedEndpointZone);183 const managedEndpointDnsBinding = yield* Cloudflare.DNS.ReadWriteDns(managedEndpointZone);
182 const managedEndpointZoneName = yield* managedEndpointZone.name;184 const managedEndpointZoneName = yield* managedEndpointZone.name;
185 const managedEndpointCleanupMode = yield* RelayConfiguration.managedEndpointCleanupModeConfig;
183186
184 //187 //
185 // 3. Runtime layers and app construction188 // 3. Runtime layers and app construction
...@@ -199,6 +202,7 @@ export const ApiLive = Api.make(...@@ -199,6 +202,7 @@ export const ApiLive = Api.make(
199 cloudMintPublicKey: yield* cloudMintPublicKey,202 cloudMintPublicKey: yield* cloudMintPublicKey,
200 managedEndpointBaseDomain: yield* managedEndpointZoneName,203 managedEndpointBaseDomain: yield* managedEndpointZoneName,
201 managedEndpointNamespace: stage,204 managedEndpointNamespace: stage,
205 managedEndpointCleanupMode,
202 });206 });
203 });207 });
204208
...@@ -215,7 +219,9 @@ export const ApiLive = Api.make(...@@ -215,7 +219,9 @@ export const ApiLive = Api.make(
215 Layer.provideMerge(AgentActivityPublisher.layer),219 Layer.provideMerge(AgentActivityPublisher.layer),
216 Layer.provideMerge(EnvironmentConnector.layer),220 Layer.provideMerge(EnvironmentConnector.layer),
217 Layer.provideMerge(EnvironmentLinker.layer),221 Layer.provideMerge(EnvironmentLinker.layer),
218 Layer.provideMerge(EnvironmentPublishSignatures.layer),222 Layer.provideMerge(
223 Layer.merge(EnvironmentPublishSignatures.layer, ManagedEndpointReaper.layer),
224 ),
219 Layer.provideMerge(225 Layer.provideMerge(
220 ManagedEndpointProvider.layerCloudflareBindings(226 ManagedEndpointProvider.layerCloudflareBindings(
221 managedEndpointTunnelBinding,227 managedEndpointTunnelBinding,
...@@ -317,22 +323,42 @@ export const ApiLive = Api.make(...@@ -317,22 +323,42 @@ export const ApiLive = Api.make(
317 );323 );
318324
319 yield* Cloudflare.Workers.cron("*/5 * * * *", () =>325 yield* Cloudflare.Workers.cron("*/5 * * * *", () =>
320 DpopProofs.DpopProofReplay.pipe(326 Effect.all(
321 Effect.flatMap((dpopProofs) => dpopProofs.pruneExpired),327 [
322 // Terminal thread rows are kept briefly so finished agents show as328 DpopProofs.DpopProofReplay.pipe(
323 // Done/Failed in the Live Activity; sweep them once they age out.329 Effect.flatMap((dpopProofs) => dpopProofs.pruneExpired),
324 Effect.andThen(330 // Keep completed thread rows long enough to show their final state.
325 Effect.all([AgentActivityRows.AgentActivityRows, DateTime.now]).pipe(331 Effect.andThen(
326 Effect.flatMap(([activityRows, now]) =>332 Effect.all([AgentActivityRows.AgentActivityRows, DateTime.now]).pipe(
327 activityRows.pruneTerminal({333 Effect.flatMap(([activityRows, now]) =>
328 updatedBefore: DateTime.formatIso(DateTime.subtract(now, { minutes: 30 })),334 activityRows.pruneTerminal({
329 }),335 updatedBefore: DateTime.formatIso(DateTime.subtract(now, { minutes: 30 })),
336 }),
337 ),
338 ),
339 ),
340 Effect.catchCause((cause) =>
341 Cause.hasInterrupts(cause)
342 ? Effect.interrupt
343 : Effect.logWarning("Failed to prune expired relay state", { cause }),
330 ),344 ),
331 ),345 ),
332 ),346 ManagedEndpointReaper.ManagedEndpointReaper.pipe(
333 Effect.withSpan("relay.cron.prune_expired_state"),347 Effect.flatMap((reaper) => reaper.sweep.pipe(Effect.timeout("2 minutes"))),
334 Effect.provide(runtimeLayer),348 Effect.tap((result) =>
335 ),349 result.scanned > 0
350 ? Effect.logInfo("Finished managed tunnel cleanup", result)
351 : Effect.void,
352 ),
353 Effect.catchCause((cause) =>
354 Cause.hasInterrupts(cause)
355 ? Effect.interrupt
356 : Effect.logWarning("Failed to clean up inactive managed tunnels", { cause }),
357 ),
358 ),
359 ],
360 { concurrency: 2, discard: true },
361 ).pipe(Effect.withSpan("relay.cron.prune_expired_state"), Effect.provide(runtimeLayer)),
336 );362 );
337363
338 const fetch = Layer.merge(364 const fetch = Layer.merge(
packages/contracts/src/relay.ts+68
...@@ -186,6 +186,34 @@ export const RelayManagedEndpointRuntimeConfig = Schema.Struct({...@@ -186,6 +186,34 @@ export const RelayManagedEndpointRuntimeConfig = Schema.Struct({
186});186});
187export type RelayManagedEndpointRuntimeConfig = typeof RelayManagedEndpointRuntimeConfig.Type;187export type RelayManagedEndpointRuntimeConfig = typeof RelayManagedEndpointRuntimeConfig.Type;
188188
189export const RelayManagedEndpointRecoveryRequest = Schema.Struct({
190 cloudUserId: TrimmedNonEmptyString,
191 origin: RelayManagedEndpointOrigin,
192 proof: TrimmedNonEmptyString,
193});
194export type RelayManagedEndpointRecoveryRequest = typeof RelayManagedEndpointRecoveryRequest.Type;
195
196export const RelayManagedEndpointRecoveryRegistrationRequest = Schema.Struct({
197 cloudUserId: TrimmedNonEmptyString,
198 tunnelId: TrimmedNonEmptyString,
199 origin: RelayManagedEndpointOrigin,
200 proof: TrimmedNonEmptyString,
201});
202export type RelayManagedEndpointRecoveryRegistrationRequest =
203 typeof RelayManagedEndpointRecoveryRegistrationRequest.Type;
204
205export const RelayManagedEndpointRecoveryRegistrationResponse = Schema.Struct({
206 status: Schema.Literals(["ready", "recovery_required"]),
207});
208export type RelayManagedEndpointRecoveryRegistrationResponse =
209 typeof RelayManagedEndpointRecoveryRegistrationResponse.Type;
210
211export const RelayManagedEndpointRecoveryResponse = Schema.Struct({
212 endpoint: RelayManagedEndpoint,
213 endpointRuntime: RelayManagedEndpointRuntimeConfig,
214});
215export type RelayManagedEndpointRecoveryResponse = typeof RelayManagedEndpointRecoveryResponse.Type;
216
189export const RelayLinkProofRequest = Schema.Struct({217export const RelayLinkProofRequest = Schema.Struct({
190 challenge: Schema.String,218 challenge: Schema.String,
191 relayIssuer: Schema.String,219 relayIssuer: Schema.String,
...@@ -213,6 +241,26 @@ const RelaySignedJwtRegisteredClaims = {...@@ -213,6 +241,26 @@ const RelaySignedJwtRegisteredClaims = {
213 exp: Schema.Int,241 exp: Schema.Int,
214} as const;242} as const;
215243
244export const RelayManagedEndpointRecoveryProofPayload = Schema.Union([
245 Schema.Struct({
246 ...RelaySignedJwtRegisteredClaims,
247 action: Schema.Literal("register"),
248 environmentId: EnvironmentId,
249 cloudUserId: TrimmedNonEmptyString,
250 tunnelId: TrimmedNonEmptyString,
251 origin: RelayManagedEndpointOrigin,
252 }),
253 Schema.Struct({
254 ...RelaySignedJwtRegisteredClaims,
255 action: Schema.Literal("recover"),
256 environmentId: EnvironmentId,
257 cloudUserId: TrimmedNonEmptyString,
258 origin: RelayManagedEndpointOrigin,
259 }),
260]);
261export type RelayManagedEndpointRecoveryProofPayload =
262 typeof RelayManagedEndpointRecoveryProofPayload.Type;
263
216export const RelayAgentActivityPublishProofPayload = Schema.Struct({264export const RelayAgentActivityPublishProofPayload = Schema.Struct({
217 ...RelaySignedJwtRegisteredClaims,265 ...RelaySignedJwtRegisteredClaims,
218 environmentId: EnvironmentId,266 environmentId: EnvironmentId,
...@@ -1090,6 +1138,26 @@ const RelayDpopClientGroup = HttpApiGroup.make("dpopClient")...@@ -1090,6 +1138,26 @@ const RelayDpopClientGroup = HttpApiGroup.make("dpopClient")
10901138
1091const RelayServerGroup = HttpApiGroup.make("server")1139const RelayServerGroup = HttpApiGroup.make("server")
1092 .add(1140 .add(
1141 HttpApiEndpoint.post(
1142 "registerManagedEndpointRecovery",
1143 "/v1/environments/:environmentId/tunnel/recovery",
1144 {
1145 params: Schema.Struct({
1146 environmentId: EnvironmentId,
1147 }),
1148 payload: RelayManagedEndpointRecoveryRegistrationRequest,
1149 success: RelayManagedEndpointRecoveryRegistrationResponse,
1150 error: RelayAuthAndInternalErrors,
1151 },
1152 ).annotate(OpenApi.Summary, "Register managed tunnel recovery without provisioning"),
1153 HttpApiEndpoint.post("recoverManagedEndpoint", "/v1/environments/:environmentId/tunnel", {
1154 params: Schema.Struct({
1155 environmentId: EnvironmentId,
1156 }),
1157 payload: RelayManagedEndpointRecoveryRequest,
1158 success: RelayManagedEndpointRecoveryResponse,
1159 error: RelayAuthAndInternalErrors,
1160 }).annotate(OpenApi.Summary, "Recover an environment's managed tunnel"),
1093 HttpApiEndpoint.post(1161 HttpApiEndpoint.post(
1094 "publishAgentActivity",1162 "publishAgentActivity",
1095 "/v1/environments/:environmentId/threads/:threadId/agent-activity",1163 "/v1/environments/:environmentId/threads/:threadId/agent-activity",
packages/shared/src/relayJwt.ts+1
...@@ -10,6 +10,7 @@ export const RELAY_HEALTH_REQUEST_TYP = "t3-cloud-health+jwt";...@@ -10,6 +10,7 @@ export const RELAY_HEALTH_REQUEST_TYP = "t3-cloud-health+jwt";
10export const RELAY_MINT_RESPONSE_TYP = "t3-env-mint+jwt";10export const RELAY_MINT_RESPONSE_TYP = "t3-env-mint+jwt";
11export const RELAY_HEALTH_RESPONSE_TYP = "t3-env-health+jwt";11export const RELAY_HEALTH_RESPONSE_TYP = "t3-env-health+jwt";
12export const RELAY_ACTIVITY_PUBLISH_TYP = "t3-env-activity+jwt";12export const RELAY_ACTIVITY_PUBLISH_TYP = "t3-env-activity+jwt";
13export const RELAY_MANAGED_TUNNEL_RECOVERY_TYP = "t3-env-managed-tunnel-recovery+jwt";
1314
14export class RelayJwtError extends Schema.TaggedError<RelayJwtError>()("RelayJwtError", {15export class RelayJwtError extends Schema.TaggedError<RelayJwtError>()("RelayJwtError", {
15 operation: Schema.Literals(["sign", "verify"]),16 operation: Schema.Literals(["sign", "verify"]),