authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-10-05 20:46:19-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-10-05 20:46:19-07:00
logff873b2ec1972b701c3f9edd6a0f727c91604dc4
tree06475c122df4dfef36a10df5d14623e0bf87c543
parent3a9c1a6df1b71d8ba73d287be90443e8e874802f
signature Signed by PGP key B5690EEEBB952194

refactor(server): CloudLink keeps only the link lifecycle; pure checks live beside it (#16340)

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

9 files changed, 1604 insertions(+), 1528 deletions(-)

apps/server/src/auth/replayMarkers.ts+1-1
......@@ -7,7 +7,7 @@ import * as Option from "effect/Option";
77import * as Path from "effect/Path";
88import * as Schedule from "effect/Schedule";
99
10import { CLOUD_REPLAY_MARKER_PREFIXES } from "../cloud/CloudLink.ts";
10import { CLOUD_REPLAY_MARKER_PREFIXES } from "../cloud/linkChecks.ts";
1111import * as ServerConfig from "../config.ts";
1212import { forkParked } from "../serverActivation.ts";
1313import { DPOP_REPLAY_MARKER_PREFIX } from "./dpop.ts";
apps/server/src/cloud/CloudLink.lifecycle.test.ts created+1122
......@@ -0,0 +1,1122 @@
1import * as NodeServices from "@effect/platform-node/NodeServices";
2import { describe, expect, it } from "@effect/vitest";
3import * as Deferred from "effect/Deferred";
4import * as Duration from "effect/Duration";
5import * as Effect from "effect/Effect";
6import * as Fiber from "effect/Fiber";
7import * as FileSystem from "effect/FileSystem";
8import * as Layer from "effect/Layer";
9import * as Option from "effect/Option";
10import * as Path from "effect/Path";
11import * as Schema from "effect/Schema";
12import * as TestClock from "effect/testing/TestClock";
13import * as Stream from "effect/Stream";
14import { HttpClient, HttpClientResponse, HttpServer, type HttpClientRequest } from "effect/http";
15import * as NetAddress from "effect/net/NetAddress";
16
17import { DESKTOP_UPDATE_RESTART_MARKER_FILE, EnvironmentId } from "@t3tools/contracts";
18import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts";
19import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
20import * as ServerConfigModule from "../config.ts";
21import { writeServiceState } from "../serviceLauncher.ts";
22import {
23 SERVICE_LAUNCHER_PROTOCOL,
24 SERVICE_STATE_FILE,
25 SERVICE_STOP_MARKER_FILE,
26 type ServiceUpdateRecord,
27} from "./serviceProtocol.ts";
28import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
29import * as AgentAwarenessRelay from "../relay/AgentAwarenessRelay.ts";
30import { CLOUD_CLI_DESIRED_LINK_SECRET } from "./CliState.ts";
31import * as CliTokenManager from "./CliTokenManager.ts";
32import { RelayManagedEndpointRecoveryRegistrationRequest } from "@t3tools/contracts/relay";
33import {
34 CLOUD_ENDPOINT_CONFIRMED_ORIGIN,
35 CLOUD_ENDPOINT_RUNTIME_CONFIG,
36 CLOUD_LINKED_USER_ID,
37 decodeConfirmedOrigin,
38 decodeRuntimeConfig,
39 RELAY_ENVIRONMENT_CREDENTIAL_SECRET,
40 RELAY_URL_SECRET,
41} from "./config.ts";
42import * as CloudLink from "./CloudLink.ts";
43import { pendingServiceUpdateExists } from "./updateHandoff.ts";
44import {
45 managedTunnelStartupAction,
46 retryManagedTunnelRegistration,
47} from "./managedTunnelStartup.ts";
48import { shouldRetryCloudLink } from "./relayResponse.ts";
49import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts";
50
51const unusedSecretStoreOperation = () => Effect.die("unused secret-store operation");
52// Linking wakes the awareness relay; these tests do not run it.
53const idleAwarenessRelay = AgentAwarenessRelay.AgentAwarenessRelay.of({
54 publishThread: () => Effect.void,
55 drain: Effect.void,
56 requestCatchUp: () => Effect.void,
57 start: () => Effect.void,
58});
59// The connect routes register recovery against the listening port; these tests
60// call the service directly.
61const idleHttpServer = HttpServer.HttpServer.of({
62 address: NetAddress.inetAddressFromIpStringUnsafe("127.0.0.1", 3773),
63 serve: (() => Effect.void) as HttpServer.HttpServer["Service"]["serve"],
64});
65
66const decodeManagedTunnelRecoveryRegistration = Schema.decodeUnknownEffect(
67 Schema.fromJsonString(RelayManagedEndpointRecoveryRegistrationRequest),
68);
69
70function makeSecretStore(
71 create: ServerSecretStore.ServerSecretStore["Service"]["create"],
72): ServerSecretStore.ServerSecretStore["Service"] {
73 return {
74 get: unusedSecretStoreOperation,
75 set: unusedSecretStoreOperation,
76 create,
77 getOrCreateRandom: unusedSecretStoreOperation,
78 remove: unusedSecretStoreOperation,
79 };
80}
81
82it("preserves messages surfaced by cloud 500 responses", () => {
83 const cause = new Error("cloud operation failed");
84
85 expect([
86 new EnvironmentAuth.ServerAuthLinkedCloudAccountVerificationError({ cause }).message,
87 new EnvironmentAuth.ServerAuthLinkedCloudAccountReadError({ cause }).message,
88 new EnvironmentAuth.ServerAuthLinkedCloudAccountMissingError({}).message,
89 new EnvironmentAuth.ServerAuthCloudLinkJwtSigningError({ cause }).message,
90 new EnvironmentAuth.ServerAuthCloudMintPublicKeyMissingError({}).message,
91 new EnvironmentAuth.ServerAuthCloudRelayIssuerMissingError({}).message,
92 new EnvironmentAuth.ServerAuthCloudHealthJwtSigningError({ cause }).message,
93 new EnvironmentAuth.ServerAuthCloudMintJwtSigningError({ cause }).message,
94 ]).toEqual([
95 "Could not verify the linked cloud account.",
96 "Could not read the linked cloud account.",
97 "Cloud linked user is not installed for this environment.",
98 "Failed to sign cloud link JWT.",
99 "Cloud mint public key is not installed for this environment.",
100 "Cloud relay issuer is not installed for this environment.",
101 "Failed to sign cloud health JWT.",
102 "Failed to sign cloud mint JWT.",
103 ]);
104});
105
106describe("reconcileDesiredCloudLink", () => {
107 it.effect("requires stored CLI authorization without exposing an HTTP endpoint", () =>
108 Effect.gen(function* () {
109 const link = yield* CloudLink.CloudLink;
110 const error = yield* Effect.flip(link.reconcileDesiredLink("http://127.0.0.1:3774"));
111
112 expect(error).toMatchObject({
113 _tag: "EnvironmentHttpUnauthorizedError",
114 message: "Run `t3 connect link` to authorize this environment.",
115 });
116 }).pipe(
117 Effect.provide(CloudLink.layer),
118 Effect.provideService(HttpServer.HttpServer, idleHttpServer),
119 Effect.provideService(
120 ServerConfigModule.ServerConfig,
121 ServerConfigModule.ServerConfig.of({} as ServerConfigModule.ServerConfig["Service"]),
122 ),
123 Effect.provideService(
124 ServerSecretStore.ServerSecretStore,
125 makeSecretStore(unusedSecretStoreOperation),
126 ),
127 Effect.provideService(
128 ServerEnvironment.ServerEnvironment,
129 ServerEnvironment.ServerEnvironment.of({
130 getEnvironmentId: unusedSecretStoreOperation(),
131 getDescriptor: unusedSecretStoreOperation(),
132 }),
133 ),
134 Effect.provideService(
135 ManagedEndpointRuntime.CloudManagedEndpointRuntime,
136 ManagedEndpointRuntime.CloudManagedEndpointRuntime.of({
137 applyConfig: unusedSecretStoreOperation,
138 recoveryRequests: Stream.empty,
139 requestRecovery: () => Effect.void,
140 tunnelConnected: Stream.empty,
141 withLinkStateLock: (effect) => effect,
142 } satisfies ManagedEndpointRuntime.CloudManagedEndpointRuntime["Service"]),
143 ),
144 Effect.provideService(
145 EnvironmentAuth.EnvironmentAuth,
146 EnvironmentAuth.EnvironmentAuth.of({} as EnvironmentAuth.EnvironmentAuth["Service"]),
147 ),
148 Effect.provideService(
149 CliTokenManager.CloudCliTokenManager,
150 CliTokenManager.CloudCliTokenManager.of({
151 get: unusedSecretStoreOperation(),
152 getExisting: Effect.succeedNone,
153 hasCredential: unusedSecretStoreOperation(),
154 store: () => unusedSecretStoreOperation(),
155 clear: unusedSecretStoreOperation(),
156 }),
157 ),
158 Effect.provideService(
159 HttpClient.HttpClient,
160 HttpClient.make(() => unusedSecretStoreOperation()),
161 ),
162 Effect.provideService(AgentAwarenessRelay.AgentAwarenessRelay, idleAwarenessRelay),
163 Effect.provide(NodeServices.layer),
164 ),
165 );
166});
167
168describe("releaseManagedTunnelOnShutdown", () => {
169 const cliToken: CliTokenManager.PersistedToken = {
170 accessToken: "cli-access-token",
171 refreshToken: "cli-refresh-token",
172 expiresAtEpochMs: Number.MAX_SAFE_INTEGER,
173 };
174
175 function makeMemorySecretStore(initial: Iterable<readonly [string, string]> = []) {
176 const values = new Map<string, Uint8Array>(
177 Array.from(initial, ([name, value]) => [name, new TextEncoder().encode(value)] as const),
178 );
179 const store: ServerSecretStore.ServerSecretStore["Service"] = {
180 get: (name) => Effect.sync(() => Option.fromNullishOr(values.get(name))),
181 set: (name, value) =>
182 Effect.sync(() => {
183 values.set(name, value);
184 }),
185 create: (name, value) =>
186 Effect.sync(() => {
187 values.set(name, value);
188 }),
189 getOrCreateRandom: unusedSecretStoreOperation,
190 remove: (name) =>
191 Effect.sync(() => {
192 values.delete(name);
193 }),
194 };
195 return { store, values };
196 }
197
198 interface ReleaseHarness {
199 readonly store: ServerSecretStore.ServerSecretStore["Service"];
200 readonly applyConfigCalls: Array<unknown>;
201 readonly requests: Array<HttpClientRequest.HttpClientRequest>;
202 readonly onRequest?: (request: HttpClientRequest.HttpClientRequest) => Effect.Effect<void>;
203 readonly respond?: () => Response;
204 readonly respondEffect?: Effect.Effect<Response>;
205 }
206
207 // Writes the launcher's durable state file into this test's baseDir with
208 // the launcher's own writer; the release reads it to detect an in-flight
209 // update handoff.
210 const writeLauncherState = (update: ServiceUpdateRecord) =>
211 Effect.gen(function* () {
212 const path = yield* Path.Path;
213 const config = yield* ServerConfigModule.ServerConfig;
214 const statePath = path.join(config.baseDir, "runtime", SERVICE_STATE_FILE);
215 yield* Effect.promise(() =>
216 writeServiceState(statePath, {
217 protocol: SERVICE_LAUNCHER_PROTOCOL,
218 activeVersion: "0.0.30",
219 update,
220 }),
221 );
222 });
223
224 // Writes the marker the desktop app leaves just before it stops its backend
225 // to install an update, and returns when it was written.
226 const writeDesktopUpdateRestartMarker = Effect.gen(function* () {
227 const fs = yield* FileSystem.FileSystem;
228 const path = yield* Path.Path;
229 const config = yield* ServerConfigModule.ServerConfig;
230 const runtimeDir = path.join(config.baseDir, "runtime");
231 const markerPath = path.join(runtimeDir, DESKTOP_UPDATE_RESTART_MARKER_FILE);
232 yield* fs.makeDirectory(runtimeDir, { recursive: true });
233 yield* fs.writeFileString(markerPath, "");
234 const { mtime } = yield* fs.stat(markerPath);
235 return Option.getOrThrow(mtime).getTime();
236 });
237
238 const provideReleaseHarness =
239 (harness: ReleaseHarness) =>
240 <A, E, R>(effect: Effect.Effect<A, E, R>) =>
241 effect.pipe(
242 Effect.provide(CloudLink.layer),
243 Effect.provideService(HttpServer.HttpServer, idleHttpServer),
244 Effect.provideService(ServerSecretStore.ServerSecretStore, harness.store),
245 Effect.provideService(AgentAwarenessRelay.AgentAwarenessRelay, idleAwarenessRelay),
246 Effect.provideService(
247 ServerEnvironment.ServerEnvironment,
248 ServerEnvironment.ServerEnvironment.of({
249 getEnvironmentId: Effect.succeed(EnvironmentId.make("env_123")),
250 getDescriptor: Effect.die("unused"),
251 }),
252 ),
253 Effect.provideService(
254 ManagedEndpointRuntime.CloudManagedEndpointRuntime,
255 ManagedEndpointRuntime.CloudManagedEndpointRuntime.of({
256 applyConfig: (config) =>
257 Effect.sync(() => {
258 harness.applyConfigCalls.push(config);
259 return config === null
260 ? ({
261 status: "disabled",
262 } satisfies ManagedEndpointRuntime.CloudManagedEndpointRuntimeStatus)
263 : ({
264 status: "running",
265 providerKind: "cloudflare_tunnel",
266 pid: 123,
267 } satisfies ManagedEndpointRuntime.CloudManagedEndpointRuntimeStatus);
268 }),
269 recoveryRequests: Stream.empty,
270 requestRecovery: () => Effect.void,
271 tunnelConnected: Stream.empty,
272 withLinkStateLock: (effect) => effect,
273 }),
274 ),
275 Effect.provideService(
276 EnvironmentAuth.EnvironmentAuth,
277 EnvironmentAuth.EnvironmentAuth.of({} as EnvironmentAuth.EnvironmentAuth["Service"]),
278 ),
279 Effect.provideService(
280 CliTokenManager.CloudCliTokenManager,
281 CliTokenManager.CloudCliTokenManager.of({
282 get: unusedSecretStoreOperation(),
283 getExisting: Effect.succeedSome(cliToken),
284 hasCredential: unusedSecretStoreOperation(),
285 store: () => unusedSecretStoreOperation(),
286 clear: unusedSecretStoreOperation(),
287 }),
288 ),
289 Effect.provideService(
290 HttpClient.HttpClient,
291 HttpClient.make((request) =>
292 Effect.sync(() => {
293 harness.requests.push(request);
294 }).pipe(
295 Effect.andThen(harness.onRequest?.(request) ?? Effect.void),
296 Effect.andThen(
297 harness.respondEffect ??
298 Effect.sync(() => (harness.respond ?? (() => Response.json({ ok: true })))()),
299 ),
300 Effect.map((response) => HttpClientResponse.fromWeb(request, response)),
301 ),
302 ),
303 ),
304 // The release consults the launcher state file under the configured
305 // baseDir, so every harness run gets a scoped temp baseDir.
306 Effect.provide(
307 ServerConfigModule.layerTest("/", { prefix: "t3-http-release-test-" }).pipe(
308 Layer.provideMerge(NodeServices.layer),
309 ),
310 ),
311 Effect.scoped,
312 );
313
314 // The persisted state of a CLI-managed link whose tunnel is releasable.
315 const managedLinkSecrets = [
316 [CLOUD_ENDPOINT_RUNTIME_CONFIG, "runtime-config"],
317 [CLOUD_ENDPOINT_CONFIRMED_ORIGIN, "confirmed-origin"],
318 [RELAY_URL_SECRET, "https://relay.example.test"],
319 [CLOUD_CLI_DESIRED_LINK_SECRET, "managed"],
320 ] as const;
321
322 it.effect("does not recreate a link that was unlinked while startup registration retried", () => {
323 const { store, values } = makeMemorySecretStore(managedLinkSecrets);
324 const applyConfigCalls: Array<unknown> = [];
325 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
326
327 return Effect.gen(function* () {
328 const link = yield* CloudLink.CloudLink;
329 // Registration started while this marker existed. Unlink removes it
330 // before startup receives the relay's final not_linked response.
331 values.delete(CLOUD_CLI_DESIRED_LINK_SECRET);
332
333 expect(yield* link.reconcileDesiredLinkIfStillDesired("http://127.0.0.1:3773")).toBeNull();
334 expect(requests).toEqual([]);
335 expect(applyConfigCalls).toEqual([]);
336 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
337 });
338
339 it.effect("stops the connector, releases the relay tunnel, and drops the dead token", () => {
340 const { store, values } = makeMemorySecretStore(managedLinkSecrets);
341 const applyConfigCalls: Array<unknown> = [];
342 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
343
344 return Effect.gen(function* () {
345 const link = yield* CloudLink.CloudLink;
346 const released = yield* link.releaseManagedTunnelOnShutdown();
347
348 expect(released).toBe(true);
349 expect(applyConfigCalls).toEqual([null]);
350 expect(requests).toHaveLength(1);
351 const request = requests[0]!;
352 expect(request.method).toBe("DELETE");
353 expect(request.url).toBe(
354 "https://relay.example.test/v1/client/environment-links/env_123/tunnel",
355 );
356 expect(request.headers.authorization).toBe("Bearer cli-access-token");
357 expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(false);
358 expect(values.has(CLOUD_ENDPOINT_CONFIRMED_ORIGIN)).toBe(false);
359 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
360 });
361
362 it.effect("does nothing for links without a stored managed tunnel runtime config", () => {
363 const { store } = makeMemorySecretStore();
364 const applyConfigCalls: Array<unknown> = [];
365 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
366
367 return Effect.gen(function* () {
368 const link = yield* CloudLink.CloudLink;
369 const released = yield* link.releaseManagedTunnelOnShutdown();
370
371 expect(released).toBe(false);
372 expect(applyConfigCalls).toEqual([]);
373 expect(requests).toEqual([]);
374 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
375 });
376
377 it.effect("leaves the tunnel of a web/mobile-installed link untouched", () => {
378 // A managed runtime config without a CLI-desired link: the environment was
379 // linked by a web/mobile client, and nothing re-provisions the tunnel on
380 // the next boot, so shutdown must not release it.
381 const { store, values } = makeMemorySecretStore([
382 [CLOUD_ENDPOINT_RUNTIME_CONFIG, "runtime-config"],
383 [RELAY_URL_SECRET, "https://relay.example.test"],
384 ]);
385 const applyConfigCalls: Array<unknown> = [];
386 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
387
388 return Effect.gen(function* () {
389 const link = yield* CloudLink.CloudLink;
390 const released = yield* link.releaseManagedTunnelOnShutdown();
391
392 expect(released).toBe(false);
393 expect(applyConfigCalls).toEqual([]);
394 expect(requests).toEqual([]);
395 expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(true);
396 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
397 });
398
399 it.effect("leaves the tunnel of a publish-only desired link untouched", () => {
400 const { store, values } = makeMemorySecretStore([
401 [CLOUD_ENDPOINT_RUNTIME_CONFIG, "runtime-config"],
402 [RELAY_URL_SECRET, "https://relay.example.test"],
403 [CLOUD_CLI_DESIRED_LINK_SECRET, "publish_only"],
404 ]);
405 const applyConfigCalls: Array<unknown> = [];
406 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
407
408 return Effect.gen(function* () {
409 const link = yield* CloudLink.CloudLink;
410 const released = yield* link.releaseManagedTunnelOnShutdown();
411
412 expect(released).toBe(false);
413 expect(applyConfigCalls).toEqual([]);
414 expect(requests).toEqual([]);
415 expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(true);
416 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
417 });
418
419 it.effect("keeps the tunnel when shutdown hands off to a pending update", () => {
420 const { store, values } = makeMemorySecretStore(managedLinkSecrets);
421 const applyConfigCalls: Array<unknown> = [];
422 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
423
424 return Effect.gen(function* () {
425 const link = yield* CloudLink.CloudLink;
426 yield* writeLauncherState({
427 id: "update-1",
428 fromVersion: "0.0.30",
429 targetVersion: "0.0.31",
430 dbPath: "/tmp/state.sqlite",
431 status: "pending",
432 });
433
434 const released = yield* link.releaseManagedTunnelOnShutdown();
435
436 // The launcher restarts a server immediately, so the tunnel is not
437 // orphaned; keeping it avoids the hostname route re-propagation that
438 // dominates update downtime. The stored config must survive so the
439 // next boot respawns the connector against the same tunnel.
440 expect(released).toBe(false);
441 expect(applyConfigCalls).toEqual([]);
442 expect(requests).toEqual([]);
443 expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(true);
444 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
445 });
446
447 it.effect("keeps the tunnel once when the desktop app restarts it for an update", () => {
448 const { store, values } = makeMemorySecretStore(managedLinkSecrets);
449 const applyConfigCalls: Array<unknown> = [];
450 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
451
452 return Effect.gen(function* () {
453 const link = yield* CloudLink.CloudLink;
454 yield* TestClock.setTime(yield* writeDesktopUpdateRestartMarker);
455
456 expect(yield* link.releaseManagedTunnelOnShutdown()).toBe(false);
457 expect(requests).toEqual([]);
458 expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(true);
459
460 // The shutdown consumed the marker, so a later quit releases the tunnel.
461 expect(yield* link.releaseManagedTunnelOnShutdown()).toBe(true);
462 expect(requests).toHaveLength(1);
463 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
464 });
465
466 it.effect("releases the tunnel when the desktop update marker is stale", () => {
467 const { store } = makeMemorySecretStore(managedLinkSecrets);
468 const applyConfigCalls: Array<unknown> = [];
469 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
470
471 return Effect.gen(function* () {
472 const link = yield* CloudLink.CloudLink;
473 const writtenAt = yield* writeDesktopUpdateRestartMarker;
474 yield* TestClock.setTime(writtenAt + Duration.toMillis(Duration.minutes(2)));
475
476 expect(yield* link.releaseManagedTunnelOnShutdown()).toBe(true);
477 expect(requests).toHaveLength(1);
478 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
479 });
480
481 it.effect("still releases a pending update when the launcher is stopping", () => {
482 // `t3 service uninstall` or `systemctl stop` during the pending window:
483 // the launcher writes its stop marker before signalling the child, so no
484 // replacement server is coming and the tunnel must not be kept.
485 const { store, values } = makeMemorySecretStore(managedLinkSecrets);
486 const applyConfigCalls: Array<unknown> = [];
487 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
488
489 return Effect.gen(function* () {
490 const link = yield* CloudLink.CloudLink;
491 yield* writeLauncherState({
492 id: "update-1",
493 fromVersion: "0.0.30",
494 targetVersion: "0.0.31",
495 dbPath: "/tmp/state.sqlite",
496 status: "pending",
497 });
498 const fs = yield* FileSystem.FileSystem;
499 const path = yield* Path.Path;
500 const config = yield* ServerConfigModule.ServerConfig;
501 yield* fs.writeFileString(path.join(config.baseDir, "runtime", SERVICE_STOP_MARKER_FILE), "");
502
503 expect(yield* pendingServiceUpdateExists).toBe(true);
504 const released = yield* link.releaseManagedTunnelOnShutdown();
505
506 expect(released).toBe(true);
507 expect(requests).toHaveLength(1);
508 expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(false);
509 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
510 });
511
512 it.effect("still releases when the recorded update already settled", () => {
513 const { store, values } = makeMemorySecretStore(managedLinkSecrets);
514 const applyConfigCalls: Array<unknown> = [];
515 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
516
517 return Effect.gen(function* () {
518 const link = yield* CloudLink.CloudLink;
519 yield* writeLauncherState({
520 id: "update-1",
521 fromVersion: "0.0.30",
522 targetVersion: "0.0.31",
523 status: "committed",
524 });
525
526 const released = yield* link.releaseManagedTunnelOnShutdown();
527
528 expect(released).toBe(true);
529 expect(requests).toHaveLength(1);
530 expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(false);
531 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
532 });
533
534 it.effect("keeps a runtime config that a fast restart replaced mid-release", () => {
535 const { store, values } = makeMemorySecretStore(managedLinkSecrets);
536 const applyConfigCalls: Array<unknown> = [];
537 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
538 const freshConfig = new TextEncoder().encode("fresh-runtime-config");
539
540 return Effect.gen(function* () {
541 const link = yield* CloudLink.CloudLink;
542 const released = yield* link.releaseManagedTunnelOnShutdown();
543
544 expect(released).toBe(true);
545 // The finalizer only drops the config it released; the one written by
546 // the restarted process while the DELETE was in flight stays.
547 expect(values.get(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(freshConfig);
548 }).pipe(
549 provideReleaseHarness({
550 store,
551 applyConfigCalls,
552 requests,
553 respond: () => {
554 // A restarted process reconciled and stored a fresh connector config
555 // while this shutdown's release request was in flight.
556 values.set(CLOUD_ENDPOINT_RUNTIME_CONFIG, freshConfig);
557 return Response.json({ ok: true });
558 },
559 }),
560 );
561 });
562
563 it.effect("keeps the stored connector token when the relay skipped the release", () => {
564 // ok:false means a concurrent provision owns the recorded tunnel, so the
565 // stored runtime config (possibly freshly written by that provision) must
566 // survive.
567 const { store, values } = makeMemorySecretStore(managedLinkSecrets);
568 const applyConfigCalls: Array<unknown> = [];
569 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
570
571 return Effect.gen(function* () {
572 const link = yield* CloudLink.CloudLink;
573 const released = yield* link.releaseManagedTunnelOnShutdown();
574
575 expect(released).toBe(false);
576 expect(requests).toHaveLength(1);
577 expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(true);
578 }).pipe(
579 provideReleaseHarness({
580 store,
581 applyConfigCalls,
582 requests,
583 respond: () => Response.json({ ok: false }),
584 }),
585 );
586 });
587
588 it.effect("keeps the stored connector token when the relay release request fails", () => {
589 const { store, values } = makeMemorySecretStore(managedLinkSecrets);
590 const applyConfigCalls: Array<unknown> = [];
591 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
592
593 return Effect.gen(function* () {
594 const link = yield* CloudLink.CloudLink;
595 const result = yield* Effect.result(link.releaseManagedTunnelOnShutdown());
596
597 expect(result._tag).toBe("Failure");
598 expect(requests).toHaveLength(1);
599 // The tunnel still exists, so the stored token stays valid across the
600 // restart and the next boot can bring the connector back immediately.
601 expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(true);
602 }).pipe(
603 provideReleaseHarness({
604 store,
605 applyConfigCalls,
606 requests,
607 respond: () => Response.json({ ok: false }, { status: 503 }),
608 }),
609 );
610 });
611
612 it.effect("registers an existing tunnel and starts the confirmed connector", () => {
613 const { store } = makeMemorySecretStore([
614 [
615 CLOUD_ENDPOINT_RUNTIME_CONFIG,
616 '{"providerKind":"cloudflare_tunnel","connectorToken":"existing-token","tunnelId":"existing-tunnel"}',
617 ],
618 [RELAY_URL_SECRET, "https://relay.example.test"],
619 [CLOUD_LINKED_USER_ID, "user-123"],
620 [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"],
621 ]);
622 const applyConfigCalls: Array<unknown> = [];
623 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
624
625 return Effect.gen(function* () {
626 const link = yield* CloudLink.CloudLink;
627 expect(yield* link.registerManagedTunnelRecovery("http://127.0.0.1:3773")).toMatchObject({
628 status: "ready",
629 });
630 expect(requests).toHaveLength(1);
631 expect(requests[0]?.method).toBe("POST");
632 expect(requests[0]?.url).toBe(
633 "https://relay.example.test/v1/environments/env_123/tunnel/recovery",
634 );
635 expect(requests[0]?.headers.authorization).toBe("Bearer environment-credential");
636 const body = requests[0]?.body;
637 expect(body?._tag).toBe("Uint8Array");
638 if (body?._tag === "Uint8Array") {
639 expect(
640 yield* decodeManagedTunnelRecoveryRegistration(new TextDecoder().decode(body.body)),
641 ).toMatchObject({
642 cloudUserId: "user-123",
643 tunnelId: "existing-tunnel",
644 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
645 });
646 }
647 expect(applyConfigCalls).toHaveLength(1);
648 }).pipe(
649 provideReleaseHarness({
650 store,
651 applyConfigCalls,
652 requests,
653 respond: () => Response.json({ status: "ready" }),
654 }),
655 );
656 });
657
658 it.effect("reconciles a changed port after a relay outage outlasts the startup fallback", () => {
659 const config = {
660 providerKind: "cloudflare_tunnel" as const,
661 connectorToken: "existing-token",
662 tunnelId: "existing-tunnel",
663 };
664 const { store } = makeMemorySecretStore([
665 [CLOUD_ENDPOINT_RUNTIME_CONFIG, JSON.stringify(config)],
666 [
667 CLOUD_ENDPOINT_CONFIRMED_ORIGIN,
668 JSON.stringify({
669 config,
670 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
671 }),
672 ],
673 [RELAY_URL_SECRET, "https://relay.example.test"],
674 [CLOUD_LINKED_USER_ID, "user-123"],
675 [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"],
676 ]);
677 const applyConfigCalls: Array<unknown> = [];
678 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
679 let relayAvailable = false;
680 const localOrigin = "http://127.0.0.1:4884";
681
682 return Effect.gen(function* () {
683 const link = yield* CloudLink.CloudLink;
684 const fallbackStarted = yield* Deferred.make<void>();
685 const firstFailure = yield* Deferred.make<void>();
686 expect(yield* link.startManagedTunnelIfOriginConfirmed(localOrigin)).toBe(false);
687 const registration = yield* Effect.forkChild(
688 retryManagedTunnelRegistration(
689 link
690 .registerManagedTunnelRecovery(localOrigin)
691 .pipe(Effect.tapError(() => Deferred.succeed(firstFailure, undefined))),
692 shouldRetryCloudLink,
693 link
694 .startManagedTunnelIfOriginConfirmed(localOrigin, {
695 requireConfirmedOrigin: false,
696 })
697 .pipe(
698 Effect.orDie,
699 Effect.tap((started) => {
700 expect(started).toBe(true);
701 return Deferred.succeed(fallbackStarted, undefined);
702 }),
703 Effect.asVoid,
704 ),
705 ),
706 { startImmediately: true },
707 );
708 yield* Deferred.await(firstFailure);
709 yield* TestClock.adjust("15 minutes");
710 yield* Effect.raceFirst(
711 Deferred.await(fallbackStarted),
712 Fiber.join(registration).pipe(
713 Effect.andThen(Effect.die("Registration ended before starting the fallback")),
714 ),
715 );
716 expect(applyConfigCalls).toEqual([config]);
717 const attemptsBeforeRecovery = requests.length;
718
719 relayAvailable = true;
720 yield* TestClock.adjust("1 minute");
721 expect(yield* Fiber.join(registration)).toMatchObject({ status: "ready" });
722 expect(requests.length).toBeGreaterThan(attemptsBeforeRecovery);
723 const marker = yield* store.get(CLOUD_ENDPOINT_CONFIRMED_ORIGIN);
724 expect(Option.isSome(marker)).toBe(true);
725 if (Option.isSome(marker)) {
726 expect(
727 Option.getOrThrow(decodeConfirmedOrigin(new TextDecoder().decode(marker.value))),
728 ).toEqual({
729 config,
730 origin: { localHttpHost: "127.0.0.1", localHttpPort: 4884 },
731 });
732 }
733 }).pipe(
734 provideReleaseHarness({
735 store,
736 applyConfigCalls,
737 requests,
738 respond: () =>
739 relayAvailable
740 ? Response.json({ status: "ready" })
741 : Response.json({ message: "relay unavailable" }, { status: 503 }),
742 }),
743 );
744 });
745
746 it.effect(
747 "starts a connector with a marker for the current origin without contacting relay",
748 () => {
749 const configJson =
750 '{"providerKind":"cloudflare_tunnel","connectorToken":"existing-token","tunnelId":"existing-tunnel"}';
751 const config = {
752 providerKind: "cloudflare_tunnel" as const,
753 connectorToken: "existing-token",
754 tunnelId: "existing-tunnel",
755 };
756 const { store } = makeMemorySecretStore([
757 [CLOUD_ENDPOINT_RUNTIME_CONFIG, configJson],
758 [
759 CLOUD_ENDPOINT_CONFIRMED_ORIGIN,
760 `{"config":${configJson},"origin":{"localHttpHost":"127.0.0.1","localHttpPort":3773}}`,
761 ],
762 ]);
763 const applyConfigCalls: Array<unknown> = [];
764 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
765
766 return Effect.gen(function* () {
767 const link = yield* CloudLink.CloudLink;
768 expect(yield* link.startManagedTunnelIfOriginConfirmed("http://127.0.0.1:3773")).toBe(true);
769 expect(applyConfigCalls).toEqual([config]);
770 expect(requests).toEqual([]);
771 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
772 },
773 );
774
775 it.effect.each([
776 { name: "missing", marker: undefined, origin: "http://127.0.0.1:3773" },
777 {
778 name: "stale",
779 marker:
780 '{"config":{"providerKind":"cloudflare_tunnel","connectorToken":"existing-token","tunnelId":"existing-tunnel"},"origin":{"localHttpHost":"127.0.0.1","localHttpPort":3773}}',
781 origin: "http://127.0.0.1:4884",
782 },
783 ])("does not start a connector with a $name origin marker", ({ marker, origin }) => {
784 const entries: Array<readonly [string, string]> = [
785 [
786 CLOUD_ENDPOINT_RUNTIME_CONFIG,
787 '{"providerKind":"cloudflare_tunnel","connectorToken":"existing-token","tunnelId":"existing-tunnel"}',
788 ],
789 ];
790 if (marker !== undefined) entries.push([CLOUD_ENDPOINT_CONFIRMED_ORIGIN, marker]);
791 const { store } = makeMemorySecretStore(entries);
792 const applyConfigCalls: Array<unknown> = [];
793 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
794
795 return Effect.gen(function* () {
796 const link = yield* CloudLink.CloudLink;
797 expect(yield* link.startManagedTunnelIfOriginConfirmed(origin)).toBe(false);
798 expect(applyConfigCalls).toEqual([]);
799 expect(requests).toEqual([]);
800 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
801 });
802
803 it.effect(
804 "starts the stored connector without a marker when confirmation is not required",
805 () => {
806 const config = {
807 providerKind: "cloudflare_tunnel" as const,
808 connectorToken: "existing-token",
809 tunnelId: "existing-tunnel",
810 };
811 const { store } = makeMemorySecretStore([
812 [CLOUD_ENDPOINT_RUNTIME_CONFIG, JSON.stringify(config)],
813 ]);
814 const applyConfigCalls: Array<unknown> = [];
815 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
816
817 return Effect.gen(function* () {
818 const link = yield* CloudLink.CloudLink;
819 expect(
820 yield* link.startManagedTunnelIfOriginConfirmed("http://127.0.0.1:3773", {
821 requireConfirmedOrigin: false,
822 }),
823 ).toBe(true);
824 expect(applyConfigCalls).toEqual([config]);
825 expect(requests).toEqual([]);
826 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
827 },
828 );
829
830 it.effect.each(["replaced", "removed"] as const)(
831 "does not activate a tunnel when its runtime config is %s during registration",
832 (mutation) => {
833 const originalConfig =
834 '{"providerKind":"cloudflare_tunnel","connectorToken":"existing-token","tunnelId":"existing-tunnel"}';
835 const { store, values } = makeMemorySecretStore([
836 [CLOUD_ENDPOINT_RUNTIME_CONFIG, originalConfig],
837 [RELAY_URL_SECRET, "https://relay.example.test"],
838 [CLOUD_LINKED_USER_ID, "user-123"],
839 [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"],
840 ]);
841 const applyConfigCalls: Array<unknown> = [];
842 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
843
844 return Effect.gen(function* () {
845 const link = yield* CloudLink.CloudLink;
846 expect(yield* link.registerManagedTunnelRecovery("http://127.0.0.1:3773")).toEqual({
847 status: "superseded",
848 });
849 expect(applyConfigCalls).toEqual([]);
850 expect(values.has(CLOUD_ENDPOINT_CONFIRMED_ORIGIN)).toBe(false);
851 }).pipe(
852 provideReleaseHarness({
853 store,
854 applyConfigCalls,
855 requests,
856 respond: () => {
857 if (mutation === "replaced") {
858 values.set(
859 CLOUD_ENDPOINT_RUNTIME_CONFIG,
860 new TextEncoder().encode(
861 '{"providerKind":"cloudflare_tunnel","connectorToken":"fresh-token","tunnelId":"fresh-tunnel"}',
862 ),
863 );
864 } else {
865 values.delete(CLOUD_ENDPOINT_RUNTIME_CONFIG);
866 }
867 return Response.json({ status: "ready" });
868 },
869 }),
870 );
871 },
872 );
873
874 it.effect("requests startup recovery for a legacy config without a recorded tunnel ID", () => {
875 const { store } = makeMemorySecretStore([
876 [
877 CLOUD_ENDPOINT_RUNTIME_CONFIG,
878 '{"providerKind":"cloudflare_tunnel","connectorToken":"token"}',
879 ],
880 [RELAY_URL_SECRET, "https://relay.example.test"],
881 [CLOUD_LINKED_USER_ID, "user-123"],
882 [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"],
883 ]);
884 const applyConfigCalls: Array<unknown> = [];
885 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
886
887 return Effect.gen(function* () {
888 const link = yield* CloudLink.CloudLink;
889 const registration = yield* link.registerManagedTunnelRecovery("http://127.0.0.1:3773");
890 expect(registration).toEqual({
891 status: "recovery_required",
892 config: { providerKind: "cloudflare_tunnel", connectorToken: "token" },
893 });
894 expect(
895 managedTunnelStartupAction({
896 wantsCliLink: false,
897 registration,
898 }),
899 ).toEqual({
900 action: "request_recovery",
901 config: { providerKind: "cloudflare_tunnel", connectorToken: "token" },
902 });
903 expect(requests).toEqual([]);
904 expect(applyConfigCalls).toEqual([]);
905 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
906 });
907
908 it.effect("recovers a web-linked tunnel with its environment credential", () => {
909 const oldConfig =
910 '{"providerKind":"cloudflare_tunnel","connectorToken":"old-token","tunnelId":"old-tunnel"}';
911 const nextConfig = {
912 providerKind: "cloudflare_tunnel",
913 connectorToken: "new-token",
914 tunnelId: "new-tunnel",
915 } as const;
916 const { store, values } = makeMemorySecretStore([
917 [CLOUD_ENDPOINT_RUNTIME_CONFIG, oldConfig],
918 [RELAY_URL_SECRET, "https://relay.example.test"],
919 [CLOUD_LINKED_USER_ID, "user-123"],
920 [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"],
921 ]);
922 const applyConfigCalls: Array<unknown> = [];
923 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
924
925 return Effect.gen(function* () {
926 const link = yield* CloudLink.CloudLink;
927 expect(yield* link.recoverManagedTunnel("http://127.0.0.1:3773")).toBe(true);
928 expect(requests).toHaveLength(1);
929 expect(requests[0]?.method).toBe("POST");
930 expect(requests[0]?.url).toBe("https://relay.example.test/v1/environments/env_123/tunnel");
931 expect(requests[0]?.headers.authorization).toBe("Bearer environment-credential");
932 expect(applyConfigCalls).toEqual([nextConfig]);
933 expect(
934 Option.getOrNull(
935 decodeRuntimeConfig(new TextDecoder().decode(values.get(CLOUD_ENDPOINT_RUNTIME_CONFIG))),
936 ),
937 ).toEqual(nextConfig);
938 }).pipe(
939 provideReleaseHarness({
940 store,
941 applyConfigCalls,
942 requests,
943 respond: () =>
944 Response.json({
945 endpoint: {
946 httpBaseUrl: "https://environment.example.test/",
947 wsBaseUrl: "wss://environment.example.test/ws",
948 providerKind: "cloudflare_tunnel",
949 },
950 endpointRuntime: nextConfig,
951 }),
952 }),
953 );
954 });
955
956 it.effect("allows managed tunnel provisioning to take longer than ten seconds", () =>
957 Effect.gen(function* () {
958 const oldConfig =
959 '{"providerKind":"cloudflare_tunnel","connectorToken":"old-token","tunnelId":"old-tunnel"}';
960 const nextConfig = {
961 providerKind: "cloudflare_tunnel" as const,
962 connectorToken: "new-token",
963 tunnelId: "new-tunnel",
964 };
965 const { store } = makeMemorySecretStore([
966 [CLOUD_ENDPOINT_RUNTIME_CONFIG, oldConfig],
967 [RELAY_URL_SECRET, "https://relay.example.test"],
968 [CLOUD_LINKED_USER_ID, "user-123"],
969 [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"],
970 ]);
971 const applyConfigCalls: Array<unknown> = [];
972 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
973 const requestStarted = yield* Deferred.make<void>();
974 const response = yield* Deferred.make<Response>();
975 const recovery = yield* CloudLink.CloudLink.use((link) =>
976 link.recoverManagedTunnel("http://127.0.0.1:3773"),
977 ).pipe(
978 provideReleaseHarness({
979 store,
980 applyConfigCalls,
981 requests,
982 onRequest: () => Deferred.succeed(requestStarted, undefined),
983 respondEffect: Deferred.await(response),
984 }),
985 Effect.forkChild({ startImmediately: true }),
986 );
987
988 yield* Deferred.await(requestStarted);
989 expect(requests).toHaveLength(1);
990 yield* TestClock.adjust("11 seconds");
991 yield* Effect.yieldNow;
992 yield* Deferred.succeed(
993 response,
994 Response.json({
995 endpoint: {
996 httpBaseUrl: "https://environment.example.test/",
997 wsBaseUrl: "wss://environment.example.test/ws",
998 providerKind: "cloudflare_tunnel",
999 },
1000 endpointRuntime: nextConfig,
1001 }),
1002 );
1003
1004 expect(yield* Fiber.join(recovery)).toBe(true);
1005 expect(requests).toHaveLength(1);
1006 expect(applyConfigCalls).toEqual([nextConfig]);
1007 }),
1008 );
1009
1010 it.effect("does not recover an environment without a managed tunnel credential", () => {
1011 const { store } = makeMemorySecretStore([
1012 [CLOUD_ENDPOINT_RUNTIME_CONFIG, "old-config"],
1013 [RELAY_URL_SECRET, "https://relay.example.test"],
1014 ]);
1015 const applyConfigCalls: Array<unknown> = [];
1016 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
1017
1018 return Effect.gen(function* () {
1019 const link = yield* CloudLink.CloudLink;
1020 expect(yield* link.recoverManagedTunnel("http://127.0.0.1:3773")).toBe(false);
1021 expect(applyConfigCalls).toEqual([]);
1022 expect(requests).toEqual([]);
1023 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
1024 });
1025
1026 it.effect("ignores recovery requests for a tunnel that has already been replaced", () => {
1027 const { store } = makeMemorySecretStore([
1028 [
1029 CLOUD_ENDPOINT_RUNTIME_CONFIG,
1030 '{"providerKind":"cloudflare_tunnel","connectorToken":"current-token","tunnelId":"current-tunnel"}',
1031 ],
1032 [RELAY_URL_SECRET, "https://relay.example.test"],
1033 [CLOUD_LINKED_USER_ID, "user-123"],
1034 [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"],
1035 ]);
1036 const applyConfigCalls: Array<unknown> = [];
1037 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
1038
1039 return Effect.gen(function* () {
1040 const link = yield* CloudLink.CloudLink;
1041 expect(
1042 yield* link.recoverManagedTunnel("http://127.0.0.1:3773", {
1043 providerKind: "cloudflare_tunnel",
1044 connectorToken: "old-token",
1045 tunnelId: "old-tunnel",
1046 }),
1047 ).toBe(false);
1048 expect(requests).toEqual([]);
1049 expect(applyConfigCalls).toEqual([]);
1050 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
1051 });
1052
1053 it.effect.each([
1054 { status: 401, errorTag: "EnvironmentHttpUnauthorizedError" },
1055 { status: 403, errorTag: "EnvironmentHttpForbiddenError" },
1056 { status: 409, errorTag: "EnvironmentHttpBadRequestError" },
1057 ])("preserves a permanent $status relay recovery failure", ({ status, errorTag }) => {
1058 const { store } = makeMemorySecretStore([
1059 [CLOUD_ENDPOINT_RUNTIME_CONFIG, "old-config"],
1060 [RELAY_URL_SECRET, "https://relay.example.test"],
1061 [CLOUD_LINKED_USER_ID, "user-123"],
1062 [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"],
1063 ]);
1064 const applyConfigCalls: Array<unknown> = [];
1065 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
1066
1067 return Effect.gen(function* () {
1068 const link = yield* CloudLink.CloudLink;
1069 const error = yield* Effect.flip(link.recoverManagedTunnel("http://127.0.0.1:3773"));
1070
1071 expect(error._tag).toBe(errorTag);
1072 expect(requests).toHaveLength(1);
1073 expect(applyConfigCalls).toEqual([]);
1074 }).pipe(
1075 provideReleaseHarness({
1076 store,
1077 applyConfigCalls,
1078 requests,
1079 respond: () => Response.json({}, { status }),
1080 }),
1081 );
1082 });
1083
1084 it.effect("keeps a tunnel configuration replaced during recovery", () => {
1085 const { store, values } = makeMemorySecretStore([
1086 [CLOUD_ENDPOINT_RUNTIME_CONFIG, "old-config"],
1087 [RELAY_URL_SECRET, "https://relay.example.test"],
1088 [CLOUD_LINKED_USER_ID, "user-123"],
1089 [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"],
1090 ]);
1091 const applyConfigCalls: Array<unknown> = [];
1092 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
1093 const freshConfig = new TextEncoder().encode("fresh-config");
1094
1095 return Effect.gen(function* () {
1096 const link = yield* CloudLink.CloudLink;
1097 expect(yield* link.recoverManagedTunnel("http://127.0.0.1:3773")).toBe(false);
1098 expect(values.get(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(freshConfig);
1099 expect(applyConfigCalls).toEqual([]);
1100 }).pipe(
1101 provideReleaseHarness({
1102 store,
1103 applyConfigCalls,
1104 requests,
1105 respond: () => {
1106 values.set(CLOUD_ENDPOINT_RUNTIME_CONFIG, freshConfig);
1107 return Response.json({
1108 endpoint: {
1109 httpBaseUrl: "https://environment.example.test/",
1110 wsBaseUrl: "wss://environment.example.test/ws",
1111 providerKind: "cloudflare_tunnel",
1112 },
1113 endpointRuntime: {
1114 providerKind: "cloudflare_tunnel",
1115 connectorToken: "replacement-token",
1116 },
1117 });
1118 },
1119 }),
1120 );
1121 });
1122});
apps/server/src/cloud/CloudLink.ts+18-233
......@@ -7,7 +7,6 @@
77import * as NodeCrypto from "node:crypto";
88import {
99 AuthStandardClientScopes,
10 DESKTOP_UPDATE_RESTART_MARKER_FILE,
1110 EnvironmentCloudEndpointUnavailableError,
1211 type EnvironmentCloudLinkStateResult,
1312 type EnvironmentCloudPreferencesRequest,
......@@ -53,7 +52,6 @@ import {
5352 verifyRelayJwt,
5453} from "@t3tools/shared/relayJwt";
5554import { isSecureRelayUrl } from "@t3tools/shared/relayUrl";
56import * as Clock from "effect/Clock";
5755import * as Context from "effect/Context";
5856import * as Crypto from "effect/Crypto";
5957import * as DateTime from "effect/DateTime";
......@@ -110,29 +108,24 @@ import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts";
110108import { relayUrlConfig } from "./publicConfig.ts";
111109import { filterRelayResponse, relayRequestError, shouldRetryCloudLink } from "./relayResponse.ts";
112110import {
113 SERVICE_STATE_FILE,
114 SERVICE_STOP_MARKER_FILE,
115 serviceStateHasPendingUpdate,
116} from "./serviceProtocol.ts";
117
118const CLOUD_MINT_NONCE_PREFIX = "cloud-mint-nonce-";
119const CLOUD_MINT_JTI_PREFIX = "cloud-mint-jti-";
120const CLOUD_HEALTH_NONCE_PREFIX = "cloud-health-nonce-";
121const CLOUD_HEALTH_JTI_PREFIX = "cloud-health-jti-";
122/** Secret store name prefixes of cloud replay markers. The server prunes expired ones. */
123export const CLOUD_REPLAY_MARKER_PREFIXES = [
124 CLOUD_MINT_NONCE_PREFIX,
125 CLOUD_MINT_JTI_PREFIX,
126 CLOUD_HEALTH_NONCE_PREFIX,
127111 CLOUD_HEALTH_JTI_PREFIX,
128] as const;
129const CLOUD_PROOF_MAX_LIFETIME_SECONDS = 5 * 60;
130const CLOUD_PROOF_CLOCK_SKEW_SECONDS = 60;
131// The desktop app stops its backends within seconds of writing the marker.
132const DESKTOP_UPDATE_RESTART_MARKER_TTL = Duration.minutes(1);
133const MANAGED_ENDPOINT_PROVISION_REQUEST_TIMEOUT = Duration.minutes(2);
134const LOOPBACK_HOSTNAMES = new Set(["127.0.0.1", "::1", "localhost"]);
112 CLOUD_HEALTH_NONCE_PREFIX,
113 CLOUD_MINT_JTI_PREFIX,
114 CLOUD_MINT_NONCE_PREFIX,
115 consumeCloudReplayGuards,
116 hasBoundedCloudProofLifetime,
117 hasExactScope,
118 hasForwardedAuthorityHeaders,
119 isAllowedEndpointOrigin,
120 isSupportedLinkProviderKind,
121 linkProofScopes,
122 managedEndpointRuntimeConfigsMatch,
123 parseManagedEndpointLocalOrigin,
124 requestAbsoluteUrl,
125} from "./linkChecks.ts";
126import { desktopUpdateRestartPending, pendingUpdateHandoffExists } from "./updateHandoff.ts";
135127
128const MANAGED_ENDPOINT_PROVISION_REQUEST_TIMEOUT = Duration.minutes(2);
136129const failEnvironmentCloudInternalError =
137130 (message: string) =>
138131 (cause: unknown): Effect.Effect<never, EnvironmentHttpInternalServerError> =>
......@@ -164,37 +157,6 @@ function stringToBytes(value: string): Uint8Array {
164157 return new TextEncoder().encode(value);
165158}
166159
167export function consumeCloudReplayGuards(input: {
168 readonly secrets: ServerSecretStore.ServerSecretStore["Service"];
169 readonly names: ReadonlyArray<string>;
170 readonly value: Uint8Array;
171}) {
172 return Effect.forEach(
173 input.names,
174 (name) =>
175 input.secrets.create(name, input.value).pipe(
176 Effect.as(true),
177 Effect.catchIf(ServerSecretStore.isSecretStoreError, (error) =>
178 ServerSecretStore.isSecretAlreadyExistsError(error)
179 ? Effect.succeed(false)
180 : Effect.fail(error),
181 ),
182 ),
183 { concurrency: input.names.length },
184 ).pipe(Effect.map((created) => created.every(Boolean)));
185}
186
187function normalizePemForSignedPayload(value: string): string {
188 return value.trim();
189}
190
191function normalizeHostname(hostname: string): string {
192 return hostname
193 .trim()
194 .toLowerCase()
195 .replace(/^\[(.*)\]$/, "$1");
196}
197
198160function validateCloudMintPublicKey(
199161 publicKey: string,
200162): Effect.Effect<void, EnvironmentHttpBadRequestError> {
......@@ -251,190 +213,13 @@ function validateRelayConfigPayload(
251213 return Effect.void;
252214}
253215
254function isLoopbackHostname(hostname: string): boolean {
255 return LOOPBACK_HOSTNAMES.has(normalizeHostname(hostname));
256}
257
258function firstForwardedHeaderValue(value: string | undefined): string | undefined {
259 const first = value?.split(",")[0]?.trim();
260 return first && first.length > 0 ? first : undefined;
261}
262
263function requestAbsoluteUrl(request: HttpServerRequest.HttpServerRequest): string | null {
264 try {
265 return new URL(request.originalUrl).href;
266 } catch {
267 const host = firstForwardedHeaderValue(request.headers.host) ?? "127.0.0.1";
268 try {
269 return new URL(request.originalUrl, `http://${host}`).href;
270 } catch {
271 return null;
272 }
273 }
274}
275
276function hasForwardedAuthorityHeaders(request: HttpServerRequest.HttpServerRequest): boolean {
277 return (
278 firstForwardedHeaderValue(request.headers["x-forwarded-host"]) !== undefined ||
279 firstForwardedHeaderValue(request.headers["x-forwarded-proto"]) !== undefined
280 );
281}
282
283function endpointRequestPort(url: URL): number {
284 return Number(url.port || (url.protocol === "https:" ? 443 : 80));
285}
286
287export function parseManagedEndpointLocalOrigin(localOrigin: string) {
288 const url = new URL(localOrigin);
289 if (
290 localOrigin !== localOrigin.trim() ||
291 (url.protocol !== "http:" && url.protocol !== "https:") ||
292 url.username !== "" ||
293 url.password !== "" ||
294 url.pathname !== "/" ||
295 url.search !== "" ||
296 url.hash !== "" ||
297 localOrigin.includes("?") ||
298 localOrigin.includes("#")
299 ) {
300 throw new Error("Invalid local origin");
301 }
302 const wsUrl = new URL(url.origin);
303 wsUrl.protocol = url.protocol === "https:" ? "wss:" : "ws:";
304 return {
305 httpBaseUrl: url.origin,
306 wsBaseUrl: wsUrl.origin,
307 origin: {
308 localHttpHost: url.hostname,
309 localHttpPort: endpointRequestPort(url),
310 } satisfies RelayManagedEndpointOrigin,
311 };
312}
313
314function isAllowedEndpointOrigin(input: {
315 readonly origin: RelayManagedEndpointOrigin;
316 readonly requestUrl: string;
317}): boolean {
318 if (!isLoopbackHostname(input.origin.localHttpHost)) {
319 return false;
320 }
321
322 const url = new URL(input.requestUrl);
323 if (!isLoopbackHostname(url.hostname)) {
324 return false;
325 }
326
327 return input.origin.localHttpPort === endpointRequestPort(url);
328}
329
330// A managed (Cloudflare tunnel) endpoint is provisioned by the relay and must
331// point at a loopback origin. A manual endpoint is reached out of band (e.g.
332// Tailscale) or not advertised at all for publish-only links, so it is not
333// tied to the managed-tunnel scope.
334export function isSupportedLinkProviderKind(request: RelayLinkProofRequest): boolean {
335 return (
336 request.endpoint.providerKind === "cloudflare_tunnel" ||
337 request.endpoint.providerKind === "manual"
338 );
339}
340
341export function linkProofScopes(
342 request: RelayLinkProofRequest,
343): RelayEnvironmentLinkProofPayload["scopes"] {
344 return request.endpoint.providerKind === "cloudflare_tunnel"
345 ? ["agent_activity_notifications", "managed_tunnels"]
346 : ["agent_activity_notifications"];
347}
348
349function hasExactScope(input: {
350 readonly scopes: ReadonlyArray<string>;
351 readonly expected: string;
352}): boolean {
353 return input.scopes.length === 1 && input.scopes[0] === input.expected;
354}
355
356function hasBoundedCloudProofLifetime(input: {
357 readonly iat: number;
358 readonly exp: number;
359 readonly nowSeconds: number;
360}): boolean {
361 return (
362 input.exp > input.iat &&
363 input.exp - input.iat <= CLOUD_PROOF_MAX_LIFETIME_SECONDS &&
364 input.iat <= input.nowSeconds + CLOUD_PROOF_CLOCK_SKEW_SECONDS
365 );
366}
367
368function managedEndpointRuntimeConfigsMatch(
369 left: RelayManagedEndpointRuntimeConfig,
370 right: RelayManagedEndpointRuntimeConfig,
371): boolean {
372 return (
373 left.providerKind === right.providerKind &&
374 left.connectorToken === right.connectorToken &&
375 left.tunnelId === right.tunnelId &&
376 left.tunnelName === right.tunnelName
377 );
216function normalizePemForSignedPayload(value: string): string {
217 return value.trim();
378218}
379219
380220const decodeCloudHealthProof = Schema.decodeUnknownEffect(RelayCloudEnvironmentHealthProofPayload);
381221const decodeCloudMintProof = Schema.decodeUnknownEffect(RelayCloudMintCredentialProofPayload);
382222
383// The launcher owns this durable state, so read it directly both when a trial
384// decides whether it owns pre-activation cleanup and while a server tears down.
385export const pendingServiceUpdateExists = Effect.gen(function* () {
386 const config = yield* ServerConfig.ServerConfig;
387 const fs = yield* FileSystem.FileSystem;
388 const path = yield* Path.Path;
389 const runtimeDir = path.join(config.baseDir, "runtime");
390 const stateText = yield* fs
391 .readFileString(path.join(runtimeDir, SERVICE_STATE_FILE))
392 .pipe(Effect.option);
393 return Option.isSome(stateText) && serviceStateHasPendingUpdate(stateText.value);
394});
395
396// A pending update alone is not proof a replacement server is coming: an
397// explicit launcher stop (`t3 service uninstall`, `systemctl stop`,
398// `launchctl bootout`) during
399// the pending window also tears this server down. The launcher marks that case
400// just before it signals the child, so pending + no marker is the handoff.
401const pendingUpdateHandoffExists = Effect.gen(function* () {
402 if (!(yield* pendingServiceUpdateExists)) {
403 return false;
404 }
405 const config = yield* ServerConfig.ServerConfig;
406 const fs = yield* FileSystem.FileSystem;
407 const path = yield* Path.Path;
408 const runtimeDir = path.join(config.baseDir, "runtime");
409 const stopping = yield* fs
410 .exists(path.join(runtimeDir, SERVICE_STOP_MARKER_FILE))
411 .pipe(Effect.orElseSucceed(() => false));
412 return !stopping;
413});
414
415// The desktop app writes its marker right before it stops this server to
416// install an update, whether a remote client or the local app started it.
417// Reading consumes it, so shutdown checks it first. Only a fresh marker counts,
418// so a marker the server never read (a hard kill) cannot keep the tunnel on a
419// later quit.
420const desktopUpdateRestartPending = Effect.gen(function* () {
421 const config = yield* ServerConfig.ServerConfig;
422 const fs = yield* FileSystem.FileSystem;
423 const path = yield* Path.Path;
424 const markerPath = path.join(config.baseDir, "runtime", DESKTOP_UPDATE_RESTART_MARKER_FILE);
425 const marker = yield* fs.stat(markerPath).pipe(Effect.option);
426 if (Option.isNone(marker)) {
427 return false;
428 }
429 yield* fs.remove(markerPath).pipe(Effect.ignore);
430 const now = yield* Clock.currentTimeMillis;
431 return Option.match(marker.value.mtime, {
432 onNone: () => false,
433 onSome: (writtenAt) =>
434 now - writtenAt.getTime() < Duration.toMillis(DESKTOP_UPDATE_RESTART_MARKER_TTL),
435 });
436});
437
438223type ManagedTunnelRecoveryProofInput = {
439224 readonly environmentId: RelayManagedEndpointRecoveryProofPayload["environmentId"];
440225 readonly cloudUserId: string;
apps/server/src/cloud/http.test.ts deleted-1293
......@@ -1,1293 +0,0 @@
1import * as NodeServices from "@effect/platform-node/NodeServices";
2import { describe, expect, it } from "@effect/vitest";
3import * as Deferred from "effect/Deferred";
4import * as Duration from "effect/Duration";
5import * as Effect from "effect/Effect";
6import * as Fiber from "effect/Fiber";
7import * as FileSystem from "effect/FileSystem";
8import * as Layer from "effect/Layer";
9import * as Option from "effect/Option";
10import * as Path from "effect/Path";
11import * as PlatformError from "effect/PlatformError";
12import * as Schema from "effect/Schema";
13import * as TestClock from "effect/testing/TestClock";
14import * as Tracer from "effect/Tracer";
15import * as Stream from "effect/Stream";
16import {
17 HttpClient,
18 HttpClientResponse,
19 HttpServer,
20 HttpServerRequest,
21 type HttpClientRequest,
22} from "effect/http";
23import * as NetAddress from "effect/net/NetAddress";
24
25import { DESKTOP_UPDATE_RESTART_MARKER_FILE, EnvironmentId } from "@t3tools/contracts";
26import { RelayClientTracer } from "@t3tools/shared/relayTracing";
27import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts";
28import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
29import * as ServerConfigModule from "../config.ts";
30import { writeServiceState } from "../serviceLauncher.ts";
31import {
32 SERVICE_LAUNCHER_PROTOCOL,
33 SERVICE_STATE_FILE,
34 SERVICE_STOP_MARKER_FILE,
35 type ServiceUpdateRecord,
36} from "./serviceProtocol.ts";
37import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
38import * as AgentAwarenessRelay from "../relay/AgentAwarenessRelay.ts";
39import { CLOUD_CLI_DESIRED_LINK_SECRET } from "./CliState.ts";
40import * as CliTokenManager from "./CliTokenManager.ts";
41import {
42 RelayManagedEndpointRecoveryRegistrationRequest,
43 type RelayLinkProofRequest,
44} from "@t3tools/contracts/relay";
45import {
46 CLOUD_ENDPOINT_CONFIRMED_ORIGIN,
47 CLOUD_ENDPOINT_RUNTIME_CONFIG,
48 CLOUD_LINKED_USER_ID,
49 decodeConfirmedOrigin,
50 decodeRuntimeConfig,
51 RELAY_ENVIRONMENT_CREDENTIAL_SECRET,
52 RELAY_URL_SECRET,
53} from "./config.ts";
54import * as CloudLink from "./CloudLink.ts";
55import {
56 consumeCloudReplayGuards,
57 isSupportedLinkProviderKind,
58 linkProofScopes,
59 pendingServiceUpdateExists,
60 parseManagedEndpointLocalOrigin,
61} from "./CloudLink.ts";
62import {
63 managedTunnelStartupAction,
64 retryManagedTunnelRegistration,
65} from "./managedTunnelStartup.ts";
66import { shouldRetryCloudLink } from "./relayResponse.ts";
67import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts";
68import { traceAuthenticatedRelayRequest, traceRelayRequest } from "./traceRelayRequest.ts";
69
70const storeFailure = (tag: "AlreadyExists" | "PermissionDenied") =>
71 new ServerSecretStore.SecretStorePersistError({
72 resource: "cloud replay guard",
73 cause: PlatformError.systemError({
74 _tag: tag,
75 module: "FileSystem",
76 method: "open",
77 pathOrDescriptor: "cloud-replay-guard.bin",
78 }),
79 });
80
81const unusedSecretStoreOperation = () => Effect.die("unused secret-store operation");
82// Linking wakes the awareness relay; these tests do not run it.
83const idleAwarenessRelay = AgentAwarenessRelay.AgentAwarenessRelay.of({
84 publishThread: () => Effect.void,
85 drain: Effect.void,
86 requestCatchUp: () => Effect.void,
87 start: () => Effect.void,
88});
89// The connect routes register recovery against the listening port; these tests
90// call the service directly.
91const idleHttpServer = HttpServer.HttpServer.of({
92 address: NetAddress.inetAddressFromIpStringUnsafe("127.0.0.1", 3773),
93 serve: (() => Effect.void) as HttpServer.HttpServer["Service"]["serve"],
94});
95
96const reconcileDesiredCloudLink = (localOrigin: string) =>
97 CloudLink.CloudLink.use((link) => link.reconcileDesiredLink(localOrigin));
98const reconcileDesiredCloudLinkIfStillDesired = (localOrigin: string) =>
99 CloudLink.CloudLink.use((link) => link.reconcileDesiredLinkIfStillDesired(localOrigin));
100const recoverManagedCloudTunnel = (
101 ...args: Parameters<CloudLink.CloudLink["Service"]["recoverManagedTunnel"]>
102) => CloudLink.CloudLink.use((link) => link.recoverManagedTunnel(...args));
103const registerManagedCloudTunnelRecovery = (
104 ...args: Parameters<CloudLink.CloudLink["Service"]["registerManagedTunnelRecovery"]>
105) => CloudLink.CloudLink.use((link) => link.registerManagedTunnelRecovery(...args));
106const releaseManagedTunnelOnShutdown = () =>
107 CloudLink.CloudLink.use((link) => link.releaseManagedTunnelOnShutdown());
108const startManagedCloudTunnelIfOriginConfirmed = (
109 ...args: Parameters<CloudLink.CloudLink["Service"]["startManagedTunnelIfOriginConfirmed"]>
110) => CloudLink.CloudLink.use((link) => link.startManagedTunnelIfOriginConfirmed(...args));
111
112const decodeManagedTunnelRecoveryRegistration = Schema.decodeUnknownEffect(
113 Schema.fromJsonString(RelayManagedEndpointRecoveryRegistrationRequest),
114);
115
116function makeSecretStore(
117 create: ServerSecretStore.ServerSecretStore["Service"]["create"],
118): ServerSecretStore.ServerSecretStore["Service"] {
119 return {
120 get: unusedSecretStoreOperation,
121 set: unusedSecretStoreOperation,
122 create,
123 getOrCreateRandom: unusedSecretStoreOperation,
124 remove: unusedSecretStoreOperation,
125 };
126}
127
128it("preserves messages surfaced by cloud 500 responses", () => {
129 const cause = new Error("cloud operation failed");
130
131 expect([
132 new EnvironmentAuth.ServerAuthLinkedCloudAccountVerificationError({ cause }).message,
133 new EnvironmentAuth.ServerAuthLinkedCloudAccountReadError({ cause }).message,
134 new EnvironmentAuth.ServerAuthLinkedCloudAccountMissingError({}).message,
135 new EnvironmentAuth.ServerAuthCloudLinkJwtSigningError({ cause }).message,
136 new EnvironmentAuth.ServerAuthCloudMintPublicKeyMissingError({}).message,
137 new EnvironmentAuth.ServerAuthCloudRelayIssuerMissingError({}).message,
138 new EnvironmentAuth.ServerAuthCloudHealthJwtSigningError({ cause }).message,
139 new EnvironmentAuth.ServerAuthCloudMintJwtSigningError({ cause }).message,
140 ]).toEqual([
141 "Could not verify the linked cloud account.",
142 "Could not read the linked cloud account.",
143 "Cloud linked user is not installed for this environment.",
144 "Failed to sign cloud link JWT.",
145 "Cloud mint public key is not installed for this environment.",
146 "Cloud relay issuer is not installed for this environment.",
147 "Failed to sign cloud health JWT.",
148 "Failed to sign cloud mint JWT.",
149 ]);
150});
151
152describe("consumeCloudReplayGuards", () => {
153 it.effect("reports already-created guards as replay conflicts", () =>
154 Effect.gen(function* () {
155 const consumed = yield* consumeCloudReplayGuards({
156 secrets: makeSecretStore(() => Effect.fail(storeFailure("AlreadyExists"))),
157 names: ["cloud-jti", "cloud-nonce"],
158 value: new Uint8Array(),
159 });
160
161 expect(consumed).toBe(false);
162 }),
163 );
164
165 it.effect("preserves replay-store availability failures", () =>
166 Effect.gen(function* () {
167 const failure = storeFailure("PermissionDenied");
168 const error = yield* Effect.flip(
169 consumeCloudReplayGuards({
170 secrets: makeSecretStore(() => Effect.fail(failure)),
171 names: ["cloud-jti", "cloud-nonce"],
172 value: new Uint8Array(),
173 }),
174 );
175
176 expect(error).toBe(failure);
177 }),
178 );
179});
180
181describe("relay request tracing", () => {
182 it.effect("does not accept an unauthenticated request trace parent", () =>
183 Effect.gen(function* () {
184 const spans: Array<Tracer.Span> = [];
185 const productTracer = Tracer.make({
186 span: (options) => {
187 const span = new Tracer.NativeSpan(options);
188 spans.push(span);
189 return span;
190 },
191 });
192 const request = HttpServerRequest.fromWeb(
193 new Request("https://environment.example.test/api/t3-cloud/mint-credential", {
194 headers: {
195 traceparent: "00-0123456789abcdef0123456789abcdef-0123456789abcdef-01",
196 },
197 }),
198 );
199
200 yield* traceRelayRequest(Effect.void.pipe(Effect.withSpan("relay.mint.handler"))).pipe(
201 Effect.provideService(HttpServerRequest.HttpServerRequest, request),
202 Effect.provideService(RelayClientTracer, Option.some(productTracer)),
203 );
204
205 expect(spans).toHaveLength(1);
206 const span = spans[0]!;
207 expect(span.traceId).not.toBe("0123456789abcdef0123456789abcdef");
208 expect(Option.isNone(span.parent)).toBe(true);
209 }),
210 );
211
212 it.effect("continues an authenticated relay trace with the product tracer", () =>
213 Effect.gen(function* () {
214 const spans: Array<Tracer.Span> = [];
215 const productTracer = Tracer.make({
216 span: (options) => {
217 const span = new Tracer.NativeSpan(options);
218 spans.push(span);
219 return span;
220 },
221 });
222 const request = HttpServerRequest.fromWeb(
223 new Request("https://environment.example.test/api/t3-cloud/mint-credential", {
224 headers: {
225 traceparent: "00-0123456789abcdef0123456789abcdef-0123456789abcdef-01",
226 },
227 }),
228 );
229
230 yield* traceAuthenticatedRelayRequest(
231 Effect.void.pipe(Effect.withSpan("relay.mint.handler")),
232 ).pipe(
233 Effect.provideService(HttpServerRequest.HttpServerRequest, request),
234 Effect.provideService(RelayClientTracer, Option.some(productTracer)),
235 );
236
237 expect(spans).toHaveLength(1);
238 const span = spans[0]!;
239 expect(span.traceId).toBe("0123456789abcdef0123456789abcdef");
240 expect(Option.getOrUndefined(span.parent)?.spanId).toBe("0123456789abcdef");
241 }),
242 );
243});
244
245describe("reconcileDesiredCloudLink", () => {
246 it.effect("requires stored CLI authorization without exposing an HTTP endpoint", () =>
247 Effect.gen(function* () {
248 const error = yield* Effect.flip(reconcileDesiredCloudLink("http://127.0.0.1:3774"));
249
250 expect(error).toMatchObject({
251 _tag: "EnvironmentHttpUnauthorizedError",
252 message: "Run `t3 connect link` to authorize this environment.",
253 });
254 }).pipe(
255 Effect.provide(CloudLink.layer),
256 Effect.provideService(HttpServer.HttpServer, idleHttpServer),
257 Effect.provideService(
258 ServerConfigModule.ServerConfig,
259 ServerConfigModule.ServerConfig.of({} as ServerConfigModule.ServerConfig["Service"]),
260 ),
261 Effect.provideService(
262 ServerSecretStore.ServerSecretStore,
263 makeSecretStore(unusedSecretStoreOperation),
264 ),
265 Effect.provideService(
266 ServerEnvironment.ServerEnvironment,
267 ServerEnvironment.ServerEnvironment.of({
268 getEnvironmentId: unusedSecretStoreOperation(),
269 getDescriptor: unusedSecretStoreOperation(),
270 }),
271 ),
272 Effect.provideService(
273 ManagedEndpointRuntime.CloudManagedEndpointRuntime,
274 ManagedEndpointRuntime.CloudManagedEndpointRuntime.of({
275 applyConfig: unusedSecretStoreOperation,
276 recoveryRequests: Stream.empty,
277 requestRecovery: () => Effect.void,
278 tunnelConnected: Stream.empty,
279 withLinkStateLock: (effect) => effect,
280 } satisfies ManagedEndpointRuntime.CloudManagedEndpointRuntime["Service"]),
281 ),
282 Effect.provideService(
283 EnvironmentAuth.EnvironmentAuth,
284 EnvironmentAuth.EnvironmentAuth.of({} as EnvironmentAuth.EnvironmentAuth["Service"]),
285 ),
286 Effect.provideService(
287 CliTokenManager.CloudCliTokenManager,
288 CliTokenManager.CloudCliTokenManager.of({
289 get: unusedSecretStoreOperation(),
290 getExisting: Effect.succeedNone,
291 hasCredential: unusedSecretStoreOperation(),
292 store: () => unusedSecretStoreOperation(),
293 clear: unusedSecretStoreOperation(),
294 }),
295 ),
296 Effect.provideService(
297 HttpClient.HttpClient,
298 HttpClient.make(() => unusedSecretStoreOperation()),
299 ),
300 Effect.provideService(AgentAwarenessRelay.AgentAwarenessRelay, idleAwarenessRelay),
301 Effect.provide(NodeServices.layer),
302 ),
303 );
304});
305
306describe("parseManagedEndpointLocalOrigin", () => {
307 it.each([
308 {
309 input: "http://127.0.0.1:80",
310 httpBaseUrl: "http://127.0.0.1",
311 wsBaseUrl: "ws://127.0.0.1",
312 port: 80,
313 },
314 {
315 input: "https://127.0.0.1:443",
316 httpBaseUrl: "https://127.0.0.1",
317 wsBaseUrl: "wss://127.0.0.1",
318 port: 443,
319 },
320 ])("accepts an explicit default port in $input", ({ input, httpBaseUrl, wsBaseUrl, port }) => {
321 expect(parseManagedEndpointLocalOrigin(input)).toEqual({
322 httpBaseUrl,
323 wsBaseUrl,
324 origin: { localHttpHost: "127.0.0.1", localHttpPort: port },
325 });
326 });
327
328 it.each([
329 "ftp://127.0.0.1:3773",
330 "http://user:password@127.0.0.1:3773",
331 "http://127.0.0.1:3773/api",
332 "http://127.0.0.1:3773?mode=test",
333 "http://127.0.0.1:3773#fragment",
334 ])("rejects non-origin URL %s", (input) => {
335 expect(() => parseManagedEndpointLocalOrigin(input)).toThrow("Invalid local origin");
336 });
337});
338
339describe("releaseManagedTunnelOnShutdown", () => {
340 const cliToken: CliTokenManager.PersistedToken = {
341 accessToken: "cli-access-token",
342 refreshToken: "cli-refresh-token",
343 expiresAtEpochMs: Number.MAX_SAFE_INTEGER,
344 };
345
346 function makeMemorySecretStore(initial: Iterable<readonly [string, string]> = []) {
347 const values = new Map<string, Uint8Array>(
348 Array.from(initial, ([name, value]) => [name, new TextEncoder().encode(value)] as const),
349 );
350 const store: ServerSecretStore.ServerSecretStore["Service"] = {
351 get: (name) => Effect.sync(() => Option.fromNullishOr(values.get(name))),
352 set: (name, value) =>
353 Effect.sync(() => {
354 values.set(name, value);
355 }),
356 create: (name, value) =>
357 Effect.sync(() => {
358 values.set(name, value);
359 }),
360 getOrCreateRandom: unusedSecretStoreOperation,
361 remove: (name) =>
362 Effect.sync(() => {
363 values.delete(name);
364 }),
365 };
366 return { store, values };
367 }
368
369 interface ReleaseHarness {
370 readonly store: ServerSecretStore.ServerSecretStore["Service"];
371 readonly applyConfigCalls: Array<unknown>;
372 readonly requests: Array<HttpClientRequest.HttpClientRequest>;
373 readonly onRequest?: (request: HttpClientRequest.HttpClientRequest) => Effect.Effect<void>;
374 readonly respond?: () => Response;
375 readonly respondEffect?: Effect.Effect<Response>;
376 }
377
378 // Writes the launcher's durable state file into this test's baseDir with
379 // the launcher's own writer; the release reads it to detect an in-flight
380 // update handoff.
381 const writeLauncherState = (update: ServiceUpdateRecord) =>
382 Effect.gen(function* () {
383 const path = yield* Path.Path;
384 const config = yield* ServerConfigModule.ServerConfig;
385 const statePath = path.join(config.baseDir, "runtime", SERVICE_STATE_FILE);
386 yield* Effect.promise(() =>
387 writeServiceState(statePath, {
388 protocol: SERVICE_LAUNCHER_PROTOCOL,
389 activeVersion: "0.0.30",
390 update,
391 }),
392 );
393 });
394
395 // Writes the marker the desktop app leaves just before it stops its backend
396 // to install an update, and returns when it was written.
397 const writeDesktopUpdateRestartMarker = Effect.gen(function* () {
398 const fs = yield* FileSystem.FileSystem;
399 const path = yield* Path.Path;
400 const config = yield* ServerConfigModule.ServerConfig;
401 const runtimeDir = path.join(config.baseDir, "runtime");
402 const markerPath = path.join(runtimeDir, DESKTOP_UPDATE_RESTART_MARKER_FILE);
403 yield* fs.makeDirectory(runtimeDir, { recursive: true });
404 yield* fs.writeFileString(markerPath, "");
405 const { mtime } = yield* fs.stat(markerPath);
406 return Option.getOrThrow(mtime).getTime();
407 });
408
409 const provideReleaseHarness =
410 (harness: ReleaseHarness) =>
411 <A, E, R>(effect: Effect.Effect<A, E, R>) =>
412 effect.pipe(
413 Effect.provide(CloudLink.layer),
414 Effect.provideService(HttpServer.HttpServer, idleHttpServer),
415 Effect.provideService(ServerSecretStore.ServerSecretStore, harness.store),
416 Effect.provideService(AgentAwarenessRelay.AgentAwarenessRelay, idleAwarenessRelay),
417 Effect.provideService(
418 ServerEnvironment.ServerEnvironment,
419 ServerEnvironment.ServerEnvironment.of({
420 getEnvironmentId: Effect.succeed(EnvironmentId.make("env_123")),
421 getDescriptor: Effect.die("unused"),
422 }),
423 ),
424 Effect.provideService(
425 ManagedEndpointRuntime.CloudManagedEndpointRuntime,
426 ManagedEndpointRuntime.CloudManagedEndpointRuntime.of({
427 applyConfig: (config) =>
428 Effect.sync(() => {
429 harness.applyConfigCalls.push(config);
430 return config === null
431 ? ({
432 status: "disabled",
433 } satisfies ManagedEndpointRuntime.CloudManagedEndpointRuntimeStatus)
434 : ({
435 status: "running",
436 providerKind: "cloudflare_tunnel",
437 pid: 123,
438 } satisfies ManagedEndpointRuntime.CloudManagedEndpointRuntimeStatus);
439 }),
440 recoveryRequests: Stream.empty,
441 requestRecovery: () => Effect.void,
442 tunnelConnected: Stream.empty,
443 withLinkStateLock: (effect) => effect,
444 }),
445 ),
446 Effect.provideService(
447 EnvironmentAuth.EnvironmentAuth,
448 EnvironmentAuth.EnvironmentAuth.of({} as EnvironmentAuth.EnvironmentAuth["Service"]),
449 ),
450 Effect.provideService(
451 CliTokenManager.CloudCliTokenManager,
452 CliTokenManager.CloudCliTokenManager.of({
453 get: unusedSecretStoreOperation(),
454 getExisting: Effect.succeedSome(cliToken),
455 hasCredential: unusedSecretStoreOperation(),
456 store: () => unusedSecretStoreOperation(),
457 clear: unusedSecretStoreOperation(),
458 }),
459 ),
460 Effect.provideService(
461 HttpClient.HttpClient,
462 HttpClient.make((request) =>
463 Effect.sync(() => {
464 harness.requests.push(request);
465 }).pipe(
466 Effect.andThen(harness.onRequest?.(request) ?? Effect.void),
467 Effect.andThen(
468 harness.respondEffect ??
469 Effect.sync(() => (harness.respond ?? (() => Response.json({ ok: true })))()),
470 ),
471 Effect.map((response) => HttpClientResponse.fromWeb(request, response)),
472 ),
473 ),
474 ),
475 // The release consults the launcher state file under the configured
476 // baseDir, so every harness run gets a scoped temp baseDir.
477 Effect.provide(
478 ServerConfigModule.layerTest("/", { prefix: "t3-http-release-test-" }).pipe(
479 Layer.provideMerge(NodeServices.layer),
480 ),
481 ),
482 Effect.scoped,
483 );
484
485 // The persisted state of a CLI-managed link whose tunnel is releasable.
486 const managedLinkSecrets = [
487 [CLOUD_ENDPOINT_RUNTIME_CONFIG, "runtime-config"],
488 [CLOUD_ENDPOINT_CONFIRMED_ORIGIN, "confirmed-origin"],
489 [RELAY_URL_SECRET, "https://relay.example.test"],
490 [CLOUD_CLI_DESIRED_LINK_SECRET, "managed"],
491 ] as const;
492
493 it.effect("does not recreate a link that was unlinked while startup registration retried", () => {
494 const { store, values } = makeMemorySecretStore(managedLinkSecrets);
495 const applyConfigCalls: Array<unknown> = [];
496 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
497
498 return Effect.gen(function* () {
499 // Registration started while this marker existed. Unlink removes it
500 // before startup receives the relay's final not_linked response.
501 values.delete(CLOUD_CLI_DESIRED_LINK_SECRET);
502
503 expect(yield* reconcileDesiredCloudLinkIfStillDesired("http://127.0.0.1:3773")).toBeNull();
504 expect(requests).toEqual([]);
505 expect(applyConfigCalls).toEqual([]);
506 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
507 });
508
509 it.effect("stops the connector, releases the relay tunnel, and drops the dead token", () => {
510 const { store, values } = makeMemorySecretStore(managedLinkSecrets);
511 const applyConfigCalls: Array<unknown> = [];
512 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
513
514 return Effect.gen(function* () {
515 const released = yield* releaseManagedTunnelOnShutdown();
516
517 expect(released).toBe(true);
518 expect(applyConfigCalls).toEqual([null]);
519 expect(requests).toHaveLength(1);
520 const request = requests[0]!;
521 expect(request.method).toBe("DELETE");
522 expect(request.url).toBe(
523 "https://relay.example.test/v1/client/environment-links/env_123/tunnel",
524 );
525 expect(request.headers.authorization).toBe("Bearer cli-access-token");
526 expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(false);
527 expect(values.has(CLOUD_ENDPOINT_CONFIRMED_ORIGIN)).toBe(false);
528 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
529 });
530
531 it.effect("does nothing for links without a stored managed tunnel runtime config", () => {
532 const { store } = makeMemorySecretStore();
533 const applyConfigCalls: Array<unknown> = [];
534 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
535
536 return Effect.gen(function* () {
537 const released = yield* releaseManagedTunnelOnShutdown();
538
539 expect(released).toBe(false);
540 expect(applyConfigCalls).toEqual([]);
541 expect(requests).toEqual([]);
542 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
543 });
544
545 it.effect("leaves the tunnel of a web/mobile-installed link untouched", () => {
546 // A managed runtime config without a CLI-desired link: the environment was
547 // linked by a web/mobile client, and nothing re-provisions the tunnel on
548 // the next boot, so shutdown must not release it.
549 const { store, values } = makeMemorySecretStore([
550 [CLOUD_ENDPOINT_RUNTIME_CONFIG, "runtime-config"],
551 [RELAY_URL_SECRET, "https://relay.example.test"],
552 ]);
553 const applyConfigCalls: Array<unknown> = [];
554 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
555
556 return Effect.gen(function* () {
557 const released = yield* releaseManagedTunnelOnShutdown();
558
559 expect(released).toBe(false);
560 expect(applyConfigCalls).toEqual([]);
561 expect(requests).toEqual([]);
562 expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(true);
563 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
564 });
565
566 it.effect("leaves the tunnel of a publish-only desired link untouched", () => {
567 const { store, values } = makeMemorySecretStore([
568 [CLOUD_ENDPOINT_RUNTIME_CONFIG, "runtime-config"],
569 [RELAY_URL_SECRET, "https://relay.example.test"],
570 [CLOUD_CLI_DESIRED_LINK_SECRET, "publish_only"],
571 ]);
572 const applyConfigCalls: Array<unknown> = [];
573 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
574
575 return Effect.gen(function* () {
576 const released = yield* releaseManagedTunnelOnShutdown();
577
578 expect(released).toBe(false);
579 expect(applyConfigCalls).toEqual([]);
580 expect(requests).toEqual([]);
581 expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(true);
582 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
583 });
584
585 it.effect("keeps the tunnel when shutdown hands off to a pending update", () => {
586 const { store, values } = makeMemorySecretStore(managedLinkSecrets);
587 const applyConfigCalls: Array<unknown> = [];
588 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
589
590 return Effect.gen(function* () {
591 yield* writeLauncherState({
592 id: "update-1",
593 fromVersion: "0.0.30",
594 targetVersion: "0.0.31",
595 dbPath: "/tmp/state.sqlite",
596 status: "pending",
597 });
598
599 const released = yield* releaseManagedTunnelOnShutdown();
600
601 // The launcher restarts a server immediately, so the tunnel is not
602 // orphaned; keeping it avoids the hostname route re-propagation that
603 // dominates update downtime. The stored config must survive so the
604 // next boot respawns the connector against the same tunnel.
605 expect(released).toBe(false);
606 expect(applyConfigCalls).toEqual([]);
607 expect(requests).toEqual([]);
608 expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(true);
609 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
610 });
611
612 it.effect("keeps the tunnel once when the desktop app restarts it for an update", () => {
613 const { store, values } = makeMemorySecretStore(managedLinkSecrets);
614 const applyConfigCalls: Array<unknown> = [];
615 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
616
617 return Effect.gen(function* () {
618 yield* TestClock.setTime(yield* writeDesktopUpdateRestartMarker);
619
620 expect(yield* releaseManagedTunnelOnShutdown()).toBe(false);
621 expect(requests).toEqual([]);
622 expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(true);
623
624 // The shutdown consumed the marker, so a later quit releases the tunnel.
625 expect(yield* releaseManagedTunnelOnShutdown()).toBe(true);
626 expect(requests).toHaveLength(1);
627 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
628 });
629
630 it.effect("releases the tunnel when the desktop update marker is stale", () => {
631 const { store } = makeMemorySecretStore(managedLinkSecrets);
632 const applyConfigCalls: Array<unknown> = [];
633 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
634
635 return Effect.gen(function* () {
636 const writtenAt = yield* writeDesktopUpdateRestartMarker;
637 yield* TestClock.setTime(writtenAt + Duration.toMillis(Duration.minutes(2)));
638
639 expect(yield* releaseManagedTunnelOnShutdown()).toBe(true);
640 expect(requests).toHaveLength(1);
641 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
642 });
643
644 it.effect("still releases a pending update when the launcher is stopping", () => {
645 // `t3 service uninstall` or `systemctl stop` during the pending window:
646 // the launcher writes its stop marker before signalling the child, so no
647 // replacement server is coming and the tunnel must not be kept.
648 const { store, values } = makeMemorySecretStore(managedLinkSecrets);
649 const applyConfigCalls: Array<unknown> = [];
650 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
651
652 return Effect.gen(function* () {
653 yield* writeLauncherState({
654 id: "update-1",
655 fromVersion: "0.0.30",
656 targetVersion: "0.0.31",
657 dbPath: "/tmp/state.sqlite",
658 status: "pending",
659 });
660 const fs = yield* FileSystem.FileSystem;
661 const path = yield* Path.Path;
662 const config = yield* ServerConfigModule.ServerConfig;
663 yield* fs.writeFileString(path.join(config.baseDir, "runtime", SERVICE_STOP_MARKER_FILE), "");
664
665 expect(yield* pendingServiceUpdateExists).toBe(true);
666 const released = yield* releaseManagedTunnelOnShutdown();
667
668 expect(released).toBe(true);
669 expect(requests).toHaveLength(1);
670 expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(false);
671 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
672 });
673
674 it.effect("still releases when the recorded update already settled", () => {
675 const { store, values } = makeMemorySecretStore(managedLinkSecrets);
676 const applyConfigCalls: Array<unknown> = [];
677 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
678
679 return Effect.gen(function* () {
680 yield* writeLauncherState({
681 id: "update-1",
682 fromVersion: "0.0.30",
683 targetVersion: "0.0.31",
684 status: "committed",
685 });
686
687 const released = yield* releaseManagedTunnelOnShutdown();
688
689 expect(released).toBe(true);
690 expect(requests).toHaveLength(1);
691 expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(false);
692 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
693 });
694
695 it.effect("keeps a runtime config that a fast restart replaced mid-release", () => {
696 const { store, values } = makeMemorySecretStore(managedLinkSecrets);
697 const applyConfigCalls: Array<unknown> = [];
698 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
699 const freshConfig = new TextEncoder().encode("fresh-runtime-config");
700
701 return Effect.gen(function* () {
702 const released = yield* releaseManagedTunnelOnShutdown();
703
704 expect(released).toBe(true);
705 // The finalizer only drops the config it released; the one written by
706 // the restarted process while the DELETE was in flight stays.
707 expect(values.get(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(freshConfig);
708 }).pipe(
709 provideReleaseHarness({
710 store,
711 applyConfigCalls,
712 requests,
713 respond: () => {
714 // A restarted process reconciled and stored a fresh connector config
715 // while this shutdown's release request was in flight.
716 values.set(CLOUD_ENDPOINT_RUNTIME_CONFIG, freshConfig);
717 return Response.json({ ok: true });
718 },
719 }),
720 );
721 });
722
723 it.effect("keeps the stored connector token when the relay skipped the release", () => {
724 // ok:false means a concurrent provision owns the recorded tunnel, so the
725 // stored runtime config (possibly freshly written by that provision) must
726 // survive.
727 const { store, values } = makeMemorySecretStore(managedLinkSecrets);
728 const applyConfigCalls: Array<unknown> = [];
729 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
730
731 return Effect.gen(function* () {
732 const released = yield* releaseManagedTunnelOnShutdown();
733
734 expect(released).toBe(false);
735 expect(requests).toHaveLength(1);
736 expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(true);
737 }).pipe(
738 provideReleaseHarness({
739 store,
740 applyConfigCalls,
741 requests,
742 respond: () => Response.json({ ok: false }),
743 }),
744 );
745 });
746
747 it.effect("keeps the stored connector token when the relay release request fails", () => {
748 const { store, values } = makeMemorySecretStore(managedLinkSecrets);
749 const applyConfigCalls: Array<unknown> = [];
750 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
751
752 return Effect.gen(function* () {
753 const result = yield* Effect.result(releaseManagedTunnelOnShutdown());
754
755 expect(result._tag).toBe("Failure");
756 expect(requests).toHaveLength(1);
757 // The tunnel still exists, so the stored token stays valid across the
758 // restart and the next boot can bring the connector back immediately.
759 expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(true);
760 }).pipe(
761 provideReleaseHarness({
762 store,
763 applyConfigCalls,
764 requests,
765 respond: () => Response.json({ ok: false }, { status: 503 }),
766 }),
767 );
768 });
769
770 it.effect("registers an existing tunnel and starts the confirmed connector", () => {
771 const { store } = makeMemorySecretStore([
772 [
773 CLOUD_ENDPOINT_RUNTIME_CONFIG,
774 '{"providerKind":"cloudflare_tunnel","connectorToken":"existing-token","tunnelId":"existing-tunnel"}',
775 ],
776 [RELAY_URL_SECRET, "https://relay.example.test"],
777 [CLOUD_LINKED_USER_ID, "user-123"],
778 [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"],
779 ]);
780 const applyConfigCalls: Array<unknown> = [];
781 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
782
783 return Effect.gen(function* () {
784 expect(yield* registerManagedCloudTunnelRecovery("http://127.0.0.1:3773")).toMatchObject({
785 status: "ready",
786 });
787 expect(requests).toHaveLength(1);
788 expect(requests[0]?.method).toBe("POST");
789 expect(requests[0]?.url).toBe(
790 "https://relay.example.test/v1/environments/env_123/tunnel/recovery",
791 );
792 expect(requests[0]?.headers.authorization).toBe("Bearer environment-credential");
793 const body = requests[0]?.body;
794 expect(body?._tag).toBe("Uint8Array");
795 if (body?._tag === "Uint8Array") {
796 expect(
797 yield* decodeManagedTunnelRecoveryRegistration(new TextDecoder().decode(body.body)),
798 ).toMatchObject({
799 cloudUserId: "user-123",
800 tunnelId: "existing-tunnel",
801 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
802 });
803 }
804 expect(applyConfigCalls).toHaveLength(1);
805 }).pipe(
806 provideReleaseHarness({
807 store,
808 applyConfigCalls,
809 requests,
810 respond: () => Response.json({ status: "ready" }),
811 }),
812 );
813 });
814
815 it.effect("reconciles a changed port after a relay outage outlasts the startup fallback", () => {
816 const config = {
817 providerKind: "cloudflare_tunnel" as const,
818 connectorToken: "existing-token",
819 tunnelId: "existing-tunnel",
820 };
821 const { store } = makeMemorySecretStore([
822 [CLOUD_ENDPOINT_RUNTIME_CONFIG, JSON.stringify(config)],
823 [
824 CLOUD_ENDPOINT_CONFIRMED_ORIGIN,
825 JSON.stringify({
826 config,
827 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
828 }),
829 ],
830 [RELAY_URL_SECRET, "https://relay.example.test"],
831 [CLOUD_LINKED_USER_ID, "user-123"],
832 [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"],
833 ]);
834 const applyConfigCalls: Array<unknown> = [];
835 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
836 let relayAvailable = false;
837 const localOrigin = "http://127.0.0.1:4884";
838
839 return Effect.gen(function* () {
840 const fallbackStarted = yield* Deferred.make<void>();
841 const firstFailure = yield* Deferred.make<void>();
842 expect(yield* startManagedCloudTunnelIfOriginConfirmed(localOrigin)).toBe(false);
843 const registration = yield* Effect.forkChild(
844 retryManagedTunnelRegistration(
845 registerManagedCloudTunnelRecovery(localOrigin).pipe(
846 Effect.tapError(() => Deferred.succeed(firstFailure, undefined)),
847 ),
848 shouldRetryCloudLink,
849 startManagedCloudTunnelIfOriginConfirmed(localOrigin, {
850 requireConfirmedOrigin: false,
851 }).pipe(
852 Effect.orDie,
853 Effect.tap((started) => {
854 expect(started).toBe(true);
855 return Deferred.succeed(fallbackStarted, undefined);
856 }),
857 Effect.asVoid,
858 ),
859 ),
860 { startImmediately: true },
861 );
862 yield* Deferred.await(firstFailure);
863 yield* TestClock.adjust("15 minutes");
864 yield* Effect.raceFirst(
865 Deferred.await(fallbackStarted),
866 Fiber.join(registration).pipe(
867 Effect.andThen(Effect.die("Registration ended before starting the fallback")),
868 ),
869 );
870 expect(applyConfigCalls).toEqual([config]);
871 const attemptsBeforeRecovery = requests.length;
872
873 relayAvailable = true;
874 yield* TestClock.adjust("1 minute");
875 expect(yield* Fiber.join(registration)).toMatchObject({ status: "ready" });
876 expect(requests.length).toBeGreaterThan(attemptsBeforeRecovery);
877 const marker = yield* store.get(CLOUD_ENDPOINT_CONFIRMED_ORIGIN);
878 expect(Option.isSome(marker)).toBe(true);
879 if (Option.isSome(marker)) {
880 expect(
881 Option.getOrThrow(decodeConfirmedOrigin(new TextDecoder().decode(marker.value))),
882 ).toEqual({
883 config,
884 origin: { localHttpHost: "127.0.0.1", localHttpPort: 4884 },
885 });
886 }
887 }).pipe(
888 provideReleaseHarness({
889 store,
890 applyConfigCalls,
891 requests,
892 respond: () =>
893 relayAvailable
894 ? Response.json({ status: "ready" })
895 : Response.json({ message: "relay unavailable" }, { status: 503 }),
896 }),
897 );
898 });
899
900 it.effect(
901 "starts a connector with a marker for the current origin without contacting relay",
902 () => {
903 const configJson =
904 '{"providerKind":"cloudflare_tunnel","connectorToken":"existing-token","tunnelId":"existing-tunnel"}';
905 const config = {
906 providerKind: "cloudflare_tunnel" as const,
907 connectorToken: "existing-token",
908 tunnelId: "existing-tunnel",
909 };
910 const { store } = makeMemorySecretStore([
911 [CLOUD_ENDPOINT_RUNTIME_CONFIG, configJson],
912 [
913 CLOUD_ENDPOINT_CONFIRMED_ORIGIN,
914 `{"config":${configJson},"origin":{"localHttpHost":"127.0.0.1","localHttpPort":3773}}`,
915 ],
916 ]);
917 const applyConfigCalls: Array<unknown> = [];
918 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
919
920 return Effect.gen(function* () {
921 expect(yield* startManagedCloudTunnelIfOriginConfirmed("http://127.0.0.1:3773")).toBe(true);
922 expect(applyConfigCalls).toEqual([config]);
923 expect(requests).toEqual([]);
924 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
925 },
926 );
927
928 it.effect.each([
929 { name: "missing", marker: undefined, origin: "http://127.0.0.1:3773" },
930 {
931 name: "stale",
932 marker:
933 '{"config":{"providerKind":"cloudflare_tunnel","connectorToken":"existing-token","tunnelId":"existing-tunnel"},"origin":{"localHttpHost":"127.0.0.1","localHttpPort":3773}}',
934 origin: "http://127.0.0.1:4884",
935 },
936 ])("does not start a connector with a $name origin marker", ({ marker, origin }) => {
937 const entries: Array<readonly [string, string]> = [
938 [
939 CLOUD_ENDPOINT_RUNTIME_CONFIG,
940 '{"providerKind":"cloudflare_tunnel","connectorToken":"existing-token","tunnelId":"existing-tunnel"}',
941 ],
942 ];
943 if (marker !== undefined) entries.push([CLOUD_ENDPOINT_CONFIRMED_ORIGIN, marker]);
944 const { store } = makeMemorySecretStore(entries);
945 const applyConfigCalls: Array<unknown> = [];
946 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
947
948 return Effect.gen(function* () {
949 expect(yield* startManagedCloudTunnelIfOriginConfirmed(origin)).toBe(false);
950 expect(applyConfigCalls).toEqual([]);
951 expect(requests).toEqual([]);
952 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
953 });
954
955 it.effect(
956 "starts the stored connector without a marker when confirmation is not required",
957 () => {
958 const config = {
959 providerKind: "cloudflare_tunnel" as const,
960 connectorToken: "existing-token",
961 tunnelId: "existing-tunnel",
962 };
963 const { store } = makeMemorySecretStore([
964 [CLOUD_ENDPOINT_RUNTIME_CONFIG, JSON.stringify(config)],
965 ]);
966 const applyConfigCalls: Array<unknown> = [];
967 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
968
969 return Effect.gen(function* () {
970 expect(
971 yield* startManagedCloudTunnelIfOriginConfirmed("http://127.0.0.1:3773", {
972 requireConfirmedOrigin: false,
973 }),
974 ).toBe(true);
975 expect(applyConfigCalls).toEqual([config]);
976 expect(requests).toEqual([]);
977 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
978 },
979 );
980
981 it.effect.each(["replaced", "removed"] as const)(
982 "does not activate a tunnel when its runtime config is %s during registration",
983 (mutation) => {
984 const originalConfig =
985 '{"providerKind":"cloudflare_tunnel","connectorToken":"existing-token","tunnelId":"existing-tunnel"}';
986 const { store, values } = makeMemorySecretStore([
987 [CLOUD_ENDPOINT_RUNTIME_CONFIG, originalConfig],
988 [RELAY_URL_SECRET, "https://relay.example.test"],
989 [CLOUD_LINKED_USER_ID, "user-123"],
990 [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"],
991 ]);
992 const applyConfigCalls: Array<unknown> = [];
993 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
994
995 return Effect.gen(function* () {
996 expect(yield* registerManagedCloudTunnelRecovery("http://127.0.0.1:3773")).toEqual({
997 status: "superseded",
998 });
999 expect(applyConfigCalls).toEqual([]);
1000 expect(values.has(CLOUD_ENDPOINT_CONFIRMED_ORIGIN)).toBe(false);
1001 }).pipe(
1002 provideReleaseHarness({
1003 store,
1004 applyConfigCalls,
1005 requests,
1006 respond: () => {
1007 if (mutation === "replaced") {
1008 values.set(
1009 CLOUD_ENDPOINT_RUNTIME_CONFIG,
1010 new TextEncoder().encode(
1011 '{"providerKind":"cloudflare_tunnel","connectorToken":"fresh-token","tunnelId":"fresh-tunnel"}',
1012 ),
1013 );
1014 } else {
1015 values.delete(CLOUD_ENDPOINT_RUNTIME_CONFIG);
1016 }
1017 return Response.json({ status: "ready" });
1018 },
1019 }),
1020 );
1021 },
1022 );
1023
1024 it.effect("requests startup recovery for a legacy config without a recorded tunnel ID", () => {
1025 const { store } = makeMemorySecretStore([
1026 [
1027 CLOUD_ENDPOINT_RUNTIME_CONFIG,
1028 '{"providerKind":"cloudflare_tunnel","connectorToken":"token"}',
1029 ],
1030 [RELAY_URL_SECRET, "https://relay.example.test"],
1031 [CLOUD_LINKED_USER_ID, "user-123"],
1032 [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"],
1033 ]);
1034 const applyConfigCalls: Array<unknown> = [];
1035 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
1036
1037 return Effect.gen(function* () {
1038 const registration = yield* registerManagedCloudTunnelRecovery("http://127.0.0.1:3773");
1039 expect(registration).toEqual({
1040 status: "recovery_required",
1041 config: { providerKind: "cloudflare_tunnel", connectorToken: "token" },
1042 });
1043 expect(
1044 managedTunnelStartupAction({
1045 wantsCliLink: false,
1046 registration,
1047 }),
1048 ).toEqual({
1049 action: "request_recovery",
1050 config: { providerKind: "cloudflare_tunnel", connectorToken: "token" },
1051 });
1052 expect(requests).toEqual([]);
1053 expect(applyConfigCalls).toEqual([]);
1054 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
1055 });
1056
1057 it.effect("recovers a web-linked tunnel with its environment credential", () => {
1058 const oldConfig =
1059 '{"providerKind":"cloudflare_tunnel","connectorToken":"old-token","tunnelId":"old-tunnel"}';
1060 const nextConfig = {
1061 providerKind: "cloudflare_tunnel",
1062 connectorToken: "new-token",
1063 tunnelId: "new-tunnel",
1064 } as const;
1065 const { store, values } = makeMemorySecretStore([
1066 [CLOUD_ENDPOINT_RUNTIME_CONFIG, oldConfig],
1067 [RELAY_URL_SECRET, "https://relay.example.test"],
1068 [CLOUD_LINKED_USER_ID, "user-123"],
1069 [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"],
1070 ]);
1071 const applyConfigCalls: Array<unknown> = [];
1072 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
1073
1074 return Effect.gen(function* () {
1075 expect(yield* recoverManagedCloudTunnel("http://127.0.0.1:3773")).toBe(true);
1076 expect(requests).toHaveLength(1);
1077 expect(requests[0]?.method).toBe("POST");
1078 expect(requests[0]?.url).toBe("https://relay.example.test/v1/environments/env_123/tunnel");
1079 expect(requests[0]?.headers.authorization).toBe("Bearer environment-credential");
1080 expect(applyConfigCalls).toEqual([nextConfig]);
1081 expect(
1082 Option.getOrNull(
1083 decodeRuntimeConfig(new TextDecoder().decode(values.get(CLOUD_ENDPOINT_RUNTIME_CONFIG))),
1084 ),
1085 ).toEqual(nextConfig);
1086 }).pipe(
1087 provideReleaseHarness({
1088 store,
1089 applyConfigCalls,
1090 requests,
1091 respond: () =>
1092 Response.json({
1093 endpoint: {
1094 httpBaseUrl: "https://environment.example.test/",
1095 wsBaseUrl: "wss://environment.example.test/ws",
1096 providerKind: "cloudflare_tunnel",
1097 },
1098 endpointRuntime: nextConfig,
1099 }),
1100 }),
1101 );
1102 });
1103
1104 it.effect("allows managed tunnel provisioning to take longer than ten seconds", () =>
1105 Effect.gen(function* () {
1106 const oldConfig =
1107 '{"providerKind":"cloudflare_tunnel","connectorToken":"old-token","tunnelId":"old-tunnel"}';
1108 const nextConfig = {
1109 providerKind: "cloudflare_tunnel" as const,
1110 connectorToken: "new-token",
1111 tunnelId: "new-tunnel",
1112 };
1113 const { store } = makeMemorySecretStore([
1114 [CLOUD_ENDPOINT_RUNTIME_CONFIG, oldConfig],
1115 [RELAY_URL_SECRET, "https://relay.example.test"],
1116 [CLOUD_LINKED_USER_ID, "user-123"],
1117 [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"],
1118 ]);
1119 const applyConfigCalls: Array<unknown> = [];
1120 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
1121 const requestStarted = yield* Deferred.make<void>();
1122 const response = yield* Deferred.make<Response>();
1123 const recovery = yield* recoverManagedCloudTunnel("http://127.0.0.1:3773").pipe(
1124 provideReleaseHarness({
1125 store,
1126 applyConfigCalls,
1127 requests,
1128 onRequest: () => Deferred.succeed(requestStarted, undefined),
1129 respondEffect: Deferred.await(response),
1130 }),
1131 Effect.forkChild({ startImmediately: true }),
1132 );
1133
1134 yield* Deferred.await(requestStarted);
1135 expect(requests).toHaveLength(1);
1136 yield* TestClock.adjust("11 seconds");
1137 yield* Effect.yieldNow;
1138 yield* Deferred.succeed(
1139 response,
1140 Response.json({
1141 endpoint: {
1142 httpBaseUrl: "https://environment.example.test/",
1143 wsBaseUrl: "wss://environment.example.test/ws",
1144 providerKind: "cloudflare_tunnel",
1145 },
1146 endpointRuntime: nextConfig,
1147 }),
1148 );
1149
1150 expect(yield* Fiber.join(recovery)).toBe(true);
1151 expect(requests).toHaveLength(1);
1152 expect(applyConfigCalls).toEqual([nextConfig]);
1153 }),
1154 );
1155
1156 it.effect("does not recover an environment without a managed tunnel credential", () => {
1157 const { store } = makeMemorySecretStore([
1158 [CLOUD_ENDPOINT_RUNTIME_CONFIG, "old-config"],
1159 [RELAY_URL_SECRET, "https://relay.example.test"],
1160 ]);
1161 const applyConfigCalls: Array<unknown> = [];
1162 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
1163
1164 return Effect.gen(function* () {
1165 expect(yield* recoverManagedCloudTunnel("http://127.0.0.1:3773")).toBe(false);
1166 expect(applyConfigCalls).toEqual([]);
1167 expect(requests).toEqual([]);
1168 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
1169 });
1170
1171 it.effect("ignores recovery requests for a tunnel that has already been replaced", () => {
1172 const { store } = makeMemorySecretStore([
1173 [
1174 CLOUD_ENDPOINT_RUNTIME_CONFIG,
1175 '{"providerKind":"cloudflare_tunnel","connectorToken":"current-token","tunnelId":"current-tunnel"}',
1176 ],
1177 [RELAY_URL_SECRET, "https://relay.example.test"],
1178 [CLOUD_LINKED_USER_ID, "user-123"],
1179 [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"],
1180 ]);
1181 const applyConfigCalls: Array<unknown> = [];
1182 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
1183
1184 return Effect.gen(function* () {
1185 expect(
1186 yield* recoverManagedCloudTunnel("http://127.0.0.1:3773", {
1187 providerKind: "cloudflare_tunnel",
1188 connectorToken: "old-token",
1189 tunnelId: "old-tunnel",
1190 }),
1191 ).toBe(false);
1192 expect(requests).toEqual([]);
1193 expect(applyConfigCalls).toEqual([]);
1194 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
1195 });
1196
1197 it.effect.each([
1198 { status: 401, errorTag: "EnvironmentHttpUnauthorizedError" },
1199 { status: 403, errorTag: "EnvironmentHttpForbiddenError" },
1200 { status: 409, errorTag: "EnvironmentHttpBadRequestError" },
1201 ])("preserves a permanent $status relay recovery failure", ({ status, errorTag }) => {
1202 const { store } = makeMemorySecretStore([
1203 [CLOUD_ENDPOINT_RUNTIME_CONFIG, "old-config"],
1204 [RELAY_URL_SECRET, "https://relay.example.test"],
1205 [CLOUD_LINKED_USER_ID, "user-123"],
1206 [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"],
1207 ]);
1208 const applyConfigCalls: Array<unknown> = [];
1209 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
1210
1211 return Effect.gen(function* () {
1212 const error = yield* Effect.flip(recoverManagedCloudTunnel("http://127.0.0.1:3773"));
1213
1214 expect(error._tag).toBe(errorTag);
1215 expect(requests).toHaveLength(1);
1216 expect(applyConfigCalls).toEqual([]);
1217 }).pipe(
1218 provideReleaseHarness({
1219 store,
1220 applyConfigCalls,
1221 requests,
1222 respond: () => Response.json({}, { status }),
1223 }),
1224 );
1225 });
1226
1227 it.effect("keeps a tunnel configuration replaced during recovery", () => {
1228 const { store, values } = makeMemorySecretStore([
1229 [CLOUD_ENDPOINT_RUNTIME_CONFIG, "old-config"],
1230 [RELAY_URL_SECRET, "https://relay.example.test"],
1231 [CLOUD_LINKED_USER_ID, "user-123"],
1232 [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"],
1233 ]);
1234 const applyConfigCalls: Array<unknown> = [];
1235 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
1236 const freshConfig = new TextEncoder().encode("fresh-config");
1237
1238 return Effect.gen(function* () {
1239 expect(yield* recoverManagedCloudTunnel("http://127.0.0.1:3773")).toBe(false);
1240 expect(values.get(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(freshConfig);
1241 expect(applyConfigCalls).toEqual([]);
1242 }).pipe(
1243 provideReleaseHarness({
1244 store,
1245 applyConfigCalls,
1246 requests,
1247 respond: () => {
1248 values.set(CLOUD_ENDPOINT_RUNTIME_CONFIG, freshConfig);
1249 return Response.json({
1250 endpoint: {
1251 httpBaseUrl: "https://environment.example.test/",
1252 wsBaseUrl: "wss://environment.example.test/ws",
1253 providerKind: "cloudflare_tunnel",
1254 },
1255 endpointRuntime: {
1256 providerKind: "cloudflare_tunnel",
1257 connectorToken: "replacement-token",
1258 },
1259 });
1260 },
1261 }),
1262 );
1263 });
1264});
1265
1266describe("link proof provider kinds", () => {
1267 const proofRequest = (
1268 providerKind: RelayLinkProofRequest["endpoint"]["providerKind"],
1269 ): RelayLinkProofRequest => ({
1270 challenge: "challenge",
1271 relayIssuer: "https://relay.example.test",
1272 endpoint: {
1273 httpBaseUrl: "http://127.0.0.1:7331",
1274 wsBaseUrl: "ws://127.0.0.1:7331",
1275 providerKind,
1276 },
1277 origin: { localHttpHost: "127.0.0.1", localHttpPort: 7331 },
1278 });
1279
1280 it("accepts managed and manual endpoints but not t3_relay", () => {
1281 expect(isSupportedLinkProviderKind(proofRequest("cloudflare_tunnel"))).toBe(true);
1282 expect(isSupportedLinkProviderKind(proofRequest("manual"))).toBe(true);
1283 expect(isSupportedLinkProviderKind(proofRequest("t3_relay"))).toBe(false);
1284 });
1285
1286 it("only claims the managed-tunnel scope for tunnel links", () => {
1287 expect(linkProofScopes(proofRequest("cloudflare_tunnel"))).toEqual([
1288 "agent_activity_notifications",
1289 "managed_tunnels",
1290 ]);
1291 expect(linkProofScopes(proofRequest("manual"))).toEqual(["agent_activity_notifications"]);
1292 });
1293});
apps/server/src/cloud/linkChecks.test.ts created+128
......@@ -0,0 +1,128 @@
1import { describe, expect, it } from "@effect/vitest";
2import * as Effect from "effect/Effect";
3import * as PlatformError from "effect/PlatformError";
4import type { RelayLinkProofRequest } from "@t3tools/contracts/relay";
5
6import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
7import {
8 consumeCloudReplayGuards,
9 isSupportedLinkProviderKind,
10 linkProofScopes,
11 parseManagedEndpointLocalOrigin,
12} from "./linkChecks.ts";
13
14const storeFailure = (tag: "AlreadyExists" | "PermissionDenied") =>
15 new ServerSecretStore.SecretStorePersistError({
16 resource: "cloud replay guard",
17 cause: PlatformError.systemError({
18 _tag: tag,
19 module: "FileSystem",
20 method: "open",
21 pathOrDescriptor: "cloud-replay-guard.bin",
22 }),
23 });
24
25const unusedSecretStoreOperation = () => Effect.die("unused secret-store operation");
26
27function makeSecretStore(
28 create: ServerSecretStore.ServerSecretStore["Service"]["create"],
29): ServerSecretStore.ServerSecretStore["Service"] {
30 return {
31 get: unusedSecretStoreOperation,
32 set: unusedSecretStoreOperation,
33 create,
34 getOrCreateRandom: unusedSecretStoreOperation,
35 remove: unusedSecretStoreOperation,
36 };
37}
38
39describe("consumeCloudReplayGuards", () => {
40 it.effect("reports already-created guards as replay conflicts", () =>
41 Effect.gen(function* () {
42 const consumed = yield* consumeCloudReplayGuards({
43 secrets: makeSecretStore(() => Effect.fail(storeFailure("AlreadyExists"))),
44 names: ["cloud-jti", "cloud-nonce"],
45 value: new Uint8Array(),
46 });
47
48 expect(consumed).toBe(false);
49 }),
50 );
51
52 it.effect("preserves replay-store availability failures", () =>
53 Effect.gen(function* () {
54 const failure = storeFailure("PermissionDenied");
55 const error = yield* Effect.flip(
56 consumeCloudReplayGuards({
57 secrets: makeSecretStore(() => Effect.fail(failure)),
58 names: ["cloud-jti", "cloud-nonce"],
59 value: new Uint8Array(),
60 }),
61 );
62
63 expect(error).toBe(failure);
64 }),
65 );
66});
67
68describe("parseManagedEndpointLocalOrigin", () => {
69 it.each([
70 {
71 input: "http://127.0.0.1:80",
72 httpBaseUrl: "http://127.0.0.1",
73 wsBaseUrl: "ws://127.0.0.1",
74 port: 80,
75 },
76 {
77 input: "https://127.0.0.1:443",
78 httpBaseUrl: "https://127.0.0.1",
79 wsBaseUrl: "wss://127.0.0.1",
80 port: 443,
81 },
82 ])("accepts an explicit default port in $input", ({ input, httpBaseUrl, wsBaseUrl, port }) => {
83 expect(parseManagedEndpointLocalOrigin(input)).toEqual({
84 httpBaseUrl,
85 wsBaseUrl,
86 origin: { localHttpHost: "127.0.0.1", localHttpPort: port },
87 });
88 });
89
90 it.each([
91 "ftp://127.0.0.1:3773",
92 "http://user:password@127.0.0.1:3773",
93 "http://127.0.0.1:3773/api",
94 "http://127.0.0.1:3773?mode=test",
95 "http://127.0.0.1:3773#fragment",
96 ])("rejects non-origin URL %s", (input) => {
97 expect(() => parseManagedEndpointLocalOrigin(input)).toThrow("Invalid local origin");
98 });
99});
100
101describe("link proof provider kinds", () => {
102 const proofRequest = (
103 providerKind: RelayLinkProofRequest["endpoint"]["providerKind"],
104 ): RelayLinkProofRequest => ({
105 challenge: "challenge",
106 relayIssuer: "https://relay.example.test",
107 endpoint: {
108 httpBaseUrl: "http://127.0.0.1:7331",
109 wsBaseUrl: "ws://127.0.0.1:7331",
110 providerKind,
111 },
112 origin: { localHttpHost: "127.0.0.1", localHttpPort: 7331 },
113 });
114
115 it("accepts managed and manual endpoints but not t3_relay", () => {
116 expect(isSupportedLinkProviderKind(proofRequest("cloudflare_tunnel"))).toBe(true);
117 expect(isSupportedLinkProviderKind(proofRequest("manual"))).toBe(true);
118 expect(isSupportedLinkProviderKind(proofRequest("t3_relay"))).toBe(false);
119 });
120
121 it("only claims the managed-tunnel scope for tunnel links", () => {
122 expect(linkProofScopes(proofRequest("cloudflare_tunnel"))).toEqual([
123 "agent_activity_notifications",
124 "managed_tunnels",
125 ]);
126 expect(linkProofScopes(proofRequest("manual"))).toEqual(["agent_activity_notifications"]);
127 });
128});
apps/server/src/cloud/linkChecks.ts created+185
......@@ -0,0 +1,185 @@
1/**
2 * Pure checks behind T3 Connect link proofs and relay requests: which local
3 * origin a link may point at, which scopes a proof claims and accepts, and the
4 * one-shot replay guards for signed relay requests.
5 */
6import type {
7 RelayEnvironmentLinkProofPayload,
8 RelayLinkProofRequest,
9 RelayManagedEndpointOrigin,
10 RelayManagedEndpointRuntimeConfig,
11} from "@t3tools/contracts/relay";
12import * as Effect from "effect/Effect";
13import type { HttpServerRequest } from "effect/http";
14
15import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
16
17export const CLOUD_MINT_NONCE_PREFIX = "cloud-mint-nonce-";
18export const CLOUD_MINT_JTI_PREFIX = "cloud-mint-jti-";
19export const CLOUD_HEALTH_NONCE_PREFIX = "cloud-health-nonce-";
20export const CLOUD_HEALTH_JTI_PREFIX = "cloud-health-jti-";
21/** Secret store name prefixes of cloud replay markers. The server prunes expired ones. */
22export const CLOUD_REPLAY_MARKER_PREFIXES = [
23 CLOUD_MINT_NONCE_PREFIX,
24 CLOUD_MINT_JTI_PREFIX,
25 CLOUD_HEALTH_NONCE_PREFIX,
26 CLOUD_HEALTH_JTI_PREFIX,
27] as const;
28const CLOUD_PROOF_MAX_LIFETIME_SECONDS = 5 * 60;
29const CLOUD_PROOF_CLOCK_SKEW_SECONDS = 60;
30const LOOPBACK_HOSTNAMES = new Set(["127.0.0.1", "::1", "localhost"]);
31
32export function consumeCloudReplayGuards(input: {
33 readonly secrets: ServerSecretStore.ServerSecretStore["Service"];
34 readonly names: ReadonlyArray<string>;
35 readonly value: Uint8Array;
36}) {
37 return Effect.forEach(
38 input.names,
39 (name) =>
40 input.secrets.create(name, input.value).pipe(
41 Effect.as(true),
42 Effect.catchIf(ServerSecretStore.isSecretStoreError, (error) =>
43 ServerSecretStore.isSecretAlreadyExistsError(error)
44 ? Effect.succeed(false)
45 : Effect.fail(error),
46 ),
47 ),
48 { concurrency: input.names.length },
49 ).pipe(Effect.map((created) => created.every(Boolean)));
50}
51
52function normalizeHostname(hostname: string): string {
53 return hostname
54 .trim()
55 .toLowerCase()
56 .replace(/^\[(.*)\]$/, "$1");
57}
58
59function isLoopbackHostname(hostname: string): boolean {
60 return LOOPBACK_HOSTNAMES.has(normalizeHostname(hostname));
61}
62
63function firstForwardedHeaderValue(value: string | undefined): string | undefined {
64 const first = value?.split(",")[0]?.trim();
65 return first && first.length > 0 ? first : undefined;
66}
67
68export function requestAbsoluteUrl(request: HttpServerRequest.HttpServerRequest): string | null {
69 try {
70 return new URL(request.originalUrl).href;
71 } catch {
72 const host = firstForwardedHeaderValue(request.headers.host) ?? "127.0.0.1";
73 try {
74 return new URL(request.originalUrl, `http://${host}`).href;
75 } catch {
76 return null;
77 }
78 }
79}
80
81export function hasForwardedAuthorityHeaders(
82 request: HttpServerRequest.HttpServerRequest,
83): boolean {
84 return (
85 firstForwardedHeaderValue(request.headers["x-forwarded-host"]) !== undefined ||
86 firstForwardedHeaderValue(request.headers["x-forwarded-proto"]) !== undefined
87 );
88}
89
90function endpointRequestPort(url: URL): number {
91 return Number(url.port || (url.protocol === "https:" ? 443 : 80));
92}
93
94export function parseManagedEndpointLocalOrigin(localOrigin: string) {
95 const url = new URL(localOrigin);
96 if (
97 localOrigin !== localOrigin.trim() ||
98 (url.protocol !== "http:" && url.protocol !== "https:") ||
99 url.username !== "" ||
100 url.password !== "" ||
101 url.pathname !== "/" ||
102 url.search !== "" ||
103 url.hash !== "" ||
104 localOrigin.includes("?") ||
105 localOrigin.includes("#")
106 ) {
107 throw new Error("Invalid local origin");
108 }
109 const wsUrl = new URL(url.origin);
110 wsUrl.protocol = url.protocol === "https:" ? "wss:" : "ws:";
111 return {
112 httpBaseUrl: url.origin,
113 wsBaseUrl: wsUrl.origin,
114 origin: {
115 localHttpHost: url.hostname,
116 localHttpPort: endpointRequestPort(url),
117 } satisfies RelayManagedEndpointOrigin,
118 };
119}
120
121export function isAllowedEndpointOrigin(input: {
122 readonly origin: RelayManagedEndpointOrigin;
123 readonly requestUrl: string;
124}): boolean {
125 if (!isLoopbackHostname(input.origin.localHttpHost)) {
126 return false;
127 }
128
129 const url = new URL(input.requestUrl);
130 if (!isLoopbackHostname(url.hostname)) {
131 return false;
132 }
133
134 return input.origin.localHttpPort === endpointRequestPort(url);
135}
136
137// A managed (Cloudflare tunnel) endpoint is provisioned by the relay and must
138// point at a loopback origin. A manual endpoint is reached out of band (e.g.
139// Tailscale) or not advertised at all for publish-only links, so it is not
140// tied to the managed-tunnel scope.
141export function isSupportedLinkProviderKind(request: RelayLinkProofRequest): boolean {
142 return (
143 request.endpoint.providerKind === "cloudflare_tunnel" ||
144 request.endpoint.providerKind === "manual"
145 );
146}
147
148export function linkProofScopes(
149 request: RelayLinkProofRequest,
150): RelayEnvironmentLinkProofPayload["scopes"] {
151 return request.endpoint.providerKind === "cloudflare_tunnel"
152 ? ["agent_activity_notifications", "managed_tunnels"]
153 : ["agent_activity_notifications"];
154}
155
156export function hasExactScope(input: {
157 readonly scopes: ReadonlyArray<string>;
158 readonly expected: string;
159}): boolean {
160 return input.scopes.length === 1 && input.scopes[0] === input.expected;
161}
162
163export function hasBoundedCloudProofLifetime(input: {
164 readonly iat: number;
165 readonly exp: number;
166 readonly nowSeconds: number;
167}): boolean {
168 return (
169 input.exp > input.iat &&
170 input.exp - input.iat <= CLOUD_PROOF_MAX_LIFETIME_SECONDS &&
171 input.iat <= input.nowSeconds + CLOUD_PROOF_CLOCK_SKEW_SECONDS
172 );
173}
174
175export function managedEndpointRuntimeConfigsMatch(
176 left: RelayManagedEndpointRuntimeConfig,
177 right: RelayManagedEndpointRuntimeConfig,
178): boolean {
179 return (
180 left.providerKind === right.providerKind &&
181 left.connectorToken === right.connectorToken &&
182 left.tunnelId === right.tunnelId &&
183 left.tunnelName === right.tunnelName
184 );
185}
apps/server/src/cloud/traceRelayRequest.test.ts created+72
......@@ -0,0 +1,72 @@
1import { describe, expect, it } from "@effect/vitest";
2import * as Effect from "effect/Effect";
3import * as Option from "effect/Option";
4import * as Tracer from "effect/Tracer";
5import { HttpServerRequest } from "effect/http";
6import { RelayClientTracer } from "@t3tools/shared/relayTracing";
7
8import { traceAuthenticatedRelayRequest, traceRelayRequest } from "./traceRelayRequest.ts";
9
10describe("relay request tracing", () => {
11 it.effect("does not accept an unauthenticated request trace parent", () =>
12 Effect.gen(function* () {
13 const spans: Array<Tracer.Span> = [];
14 const productTracer = Tracer.make({
15 span: (options) => {
16 const span = new Tracer.NativeSpan(options);
17 spans.push(span);
18 return span;
19 },
20 });
21 const request = HttpServerRequest.fromWeb(
22 new Request("https://environment.example.test/api/t3-cloud/mint-credential", {
23 headers: {
24 traceparent: "00-0123456789abcdef0123456789abcdef-0123456789abcdef-01",
25 },
26 }),
27 );
28
29 yield* traceRelayRequest(Effect.void.pipe(Effect.withSpan("relay.mint.handler"))).pipe(
30 Effect.provideService(HttpServerRequest.HttpServerRequest, request),
31 Effect.provideService(RelayClientTracer, Option.some(productTracer)),
32 );
33
34 expect(spans).toHaveLength(1);
35 const span = spans[0]!;
36 expect(span.traceId).not.toBe("0123456789abcdef0123456789abcdef");
37 expect(Option.isNone(span.parent)).toBe(true);
38 }),
39 );
40
41 it.effect("continues an authenticated relay trace with the product tracer", () =>
42 Effect.gen(function* () {
43 const spans: Array<Tracer.Span> = [];
44 const productTracer = Tracer.make({
45 span: (options) => {
46 const span = new Tracer.NativeSpan(options);
47 spans.push(span);
48 return span;
49 },
50 });
51 const request = HttpServerRequest.fromWeb(
52 new Request("https://environment.example.test/api/t3-cloud/mint-credential", {
53 headers: {
54 traceparent: "00-0123456789abcdef0123456789abcdef-0123456789abcdef-01",
55 },
56 }),
57 );
58
59 yield* traceAuthenticatedRelayRequest(
60 Effect.void.pipe(Effect.withSpan("relay.mint.handler")),
61 ).pipe(
62 Effect.provideService(HttpServerRequest.HttpServerRequest, request),
63 Effect.provideService(RelayClientTracer, Option.some(productTracer)),
64 );
65
66 expect(spans).toHaveLength(1);
67 const span = spans[0]!;
68 expect(span.traceId).toBe("0123456789abcdef0123456789abcdef");
69 expect(Option.getOrUndefined(span.parent)?.spanId).toBe("0123456789abcdef");
70 }),
71 );
72});
apps/server/src/cloud/updateHandoff.ts created+76
......@@ -0,0 +1,76 @@
1/**
2 * Whether this server is going down only to be replaced by an update. Shutdown
3 * keeps the managed tunnel across those restarts instead of releasing it.
4 */
5import { DESKTOP_UPDATE_RESTART_MARKER_FILE } from "@t3tools/contracts";
6import * as Clock from "effect/Clock";
7import * as Duration from "effect/Duration";
8import * as Effect from "effect/Effect";
9import * as FileSystem from "effect/FileSystem";
10import * as Option from "effect/Option";
11import * as Path from "effect/Path";
12
13import * as ServerConfig from "../config.ts";
14import {
15 SERVICE_STATE_FILE,
16 SERVICE_STOP_MARKER_FILE,
17 serviceStateHasPendingUpdate,
18} from "./serviceProtocol.ts";
19
20// The desktop app stops its backends within seconds of writing the marker.
21const DESKTOP_UPDATE_RESTART_MARKER_TTL = Duration.minutes(1);
22
23// The launcher owns this durable state, so read it directly both when a trial
24// decides whether it owns pre-activation cleanup and while a server tears down.
25export const pendingServiceUpdateExists = Effect.gen(function* () {
26 const config = yield* ServerConfig.ServerConfig;
27 const fs = yield* FileSystem.FileSystem;
28 const path = yield* Path.Path;
29 const runtimeDir = path.join(config.baseDir, "runtime");
30 const stateText = yield* fs
31 .readFileString(path.join(runtimeDir, SERVICE_STATE_FILE))
32 .pipe(Effect.option);
33 return Option.isSome(stateText) && serviceStateHasPendingUpdate(stateText.value);
34});
35
36// A pending update alone is not proof a replacement server is coming: an
37// explicit launcher stop (`t3 service uninstall`, `systemctl stop`,
38// `launchctl bootout`) during
39// the pending window also tears this server down. The launcher marks that case
40// just before it signals the child, so pending + no marker is the handoff.
41export const pendingUpdateHandoffExists = Effect.gen(function* () {
42 if (!(yield* pendingServiceUpdateExists)) {
43 return false;
44 }
45 const config = yield* ServerConfig.ServerConfig;
46 const fs = yield* FileSystem.FileSystem;
47 const path = yield* Path.Path;
48 const runtimeDir = path.join(config.baseDir, "runtime");
49 const stopping = yield* fs
50 .exists(path.join(runtimeDir, SERVICE_STOP_MARKER_FILE))
51 .pipe(Effect.orElseSucceed(() => false));
52 return !stopping;
53});
54
55// The desktop app writes its marker right before it stops this server to
56// install an update, whether a remote client or the local app started it.
57// Reading consumes it, so shutdown checks it first. Only a fresh marker counts,
58// so a marker the server never read (a hard kill) cannot keep the tunnel on a
59// later quit.
60export const desktopUpdateRestartPending = Effect.gen(function* () {
61 const config = yield* ServerConfig.ServerConfig;
62 const fs = yield* FileSystem.FileSystem;
63 const path = yield* Path.Path;
64 const markerPath = path.join(config.baseDir, "runtime", DESKTOP_UPDATE_RESTART_MARKER_FILE);
65 const marker = yield* fs.stat(markerPath).pipe(Effect.option);
66 if (Option.isNone(marker)) {
67 return false;
68 }
69 yield* fs.remove(markerPath).pipe(Effect.ignore);
70 const now = yield* Clock.currentTimeMillis;
71 return Option.match(marker.value.mtime, {
72 onNone: () => false,
73 onSome: (writtenAt) =>
74 now - writtenAt.getTime() < Duration.toMillis(DESKTOP_UPDATE_RESTART_MARKER_TTL),
75 });
76});
apps/server/src/server.ts+2-1
......@@ -124,6 +124,7 @@ import {
124124import * as EnvironmentAuth from "./auth/EnvironmentAuth.ts";
125125import * as CloudHttp from "./cloud/http.ts";
126126import * as CloudLink from "./cloud/CloudLink.ts";
127import { pendingServiceUpdateExists } from "./cloud/updateHandoff.ts";
127128import * as RelayTracing from "./cloud/relayTracing.ts";
128129import { shouldRetryCloudLink } from "./cloud/relayResponse.ts";
129130import * as CloudManagedEndpointRuntime from "./cloud/ManagedEndpointRuntime.ts";
......@@ -813,7 +814,7 @@ const layerMakeServer = Layer.unwrap(
813814 // while the launcher's explicit-stop marker allows it to be released.
814815 // Other runtimes wait for activation so a failed standby cannot tear
815816 // down the active runtime's tunnel.
816 const cleanupBeforeActivation = yield* CloudLink.pendingServiceUpdateExists;
817 const cleanupBeforeActivation = yield* pendingServiceUpdateExists;
817818 if (cleanupBeforeActivation) {
818819 yield* Effect.addFinalizer(() => releaseManagedTunnel);
819820 }