From 740bda40c92e1ded65dd2b65ef7c583357b1d258 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 6 Oct 2026 18:11:46 -0700 Subject: [PATCH] fix(server): recover a deleted tunnel when Cloudflare says "Tunnel not found" (#16648) Co-authored-by: Claude Opus 5.5 (1M context) --- .../src/cloud/ManagedEndpointRuntime.test.ts | 91 +++++++++++++++++++ .../src/cloud/ManagedEndpointRuntime.ts | 41 ++++++++- 2 files changed, 127 insertions(+), 5 deletions(-) diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts index 42c5d9b20e1dd0bae751e170e99d481422964df7..4b06ae2124ff5019d14f68fd5ed1877a7b4d8042 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts @@ -185,11 +185,27 @@ describe("CloudManagedEndpointRuntime", () => { '2026-06-17T02:00:00Z ERR Register tunnel error from server side error="Unauthorized: Invalid tunnel secret" connIndex=0', ), ).toBe(true); + // Seen in production on 2026-10-06 after the relay deleted an idle tunnel. + expect( + ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput( + '2026-10-06T12:00:00Z ERR Register tunnel error from server side error="Unauthorized: Tunnel not found" connIndex=0 event=0 ip=198.41.200.23', + ), + ).toBe(true); + expect( + ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput( + '2026-10-06T12:00:00Z ERR Register tunnel error from server side error="Tunnel not found" connIndex=0', + ), + ).toBe(true); expect( ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput( '2026-06-17T02:00:00Z ERR Register tunnel error from server side error="connection timed out" connIndex=0', ), ).toBe(false); + expect( + ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput( + '2026-06-17T02:00:00Z ERR Failed to serve tunnel connection error="Unauthorized: Tunnel not found" connIndex=0', + ), + ).toBe(false); }); it.effect("keeps recovery requests sent before the server starts consuming them", () => @@ -318,6 +334,81 @@ describe("CloudManagedEndpointRuntime", () => { }), ); + it.effect("requests recovery while the connector never registers a connection", () => + Effect.gen(function* () { + const output = yield* Queue.unbounded(); + const spawner = ChildProcessSpawner.make(() => + Effect.gen(function* () { + const handle = makeHandle({ + pid: 800, + onKill: () => {}, + output: Stream.fromQueue(output), + }); + yield* Effect.addFinalizer(() => handle.kill().pipe(Effect.ignore)); + return handle; + }), + ); + const runtime = yield* buildCloudManagedEndpointRuntime(spawner); + const requests = yield* Queue.unbounded(); + yield* runtime.recoveryRequests.pipe( + Stream.runForEach((config) => Queue.offer(requests, config)), + Effect.forkChild, + ); + const config = { + providerKind: "cloudflare_tunnel" as const, + connectorToken: "token", + tunnelId: "silently-deleted", + }; + yield* runtime.applyConfig(config); + + yield* TestClock.adjust(Duration.minutes(2)); + expect(yield* Queue.size(requests)).toBe(0); + yield* TestClock.adjust(Duration.minutes(1)); + expect(yield* Queue.take(requests)).toEqual(config); + // Still unconnected, so it asks again. + yield* TestClock.adjust(Duration.minutes(3)); + expect(yield* Queue.take(requests)).toEqual(config); + }).pipe(Effect.provide(TestClock.layer())), + ); + + it.effect("does not request recovery once the connector has connected", () => + Effect.gen(function* () { + const output = yield* Queue.unbounded(); + const spawner = ChildProcessSpawner.make(() => + Effect.gen(function* () { + const handle = makeHandle({ + pid: 801, + onKill: () => {}, + output: Stream.fromQueue(output), + }); + yield* Effect.addFinalizer(() => handle.kill().pipe(Effect.ignore)); + return handle; + }), + ); + const runtime = yield* buildCloudManagedEndpointRuntime(spawner); + const requests = yield* Queue.unbounded(); + yield* runtime.recoveryRequests.pipe( + Stream.runForEach((config) => Queue.offer(requests, config)), + Effect.forkChild, + ); + yield* runtime.applyConfig({ + providerKind: "cloudflare_tunnel", + connectorToken: "token", + tunnelId: "tunnel-1", + }); + yield* Queue.offer( + output, + new TextEncoder().encode( + "2026-10-06T00:00:00Z INF Registered tunnel connection connIndex=0\n", + ), + ); + yield* Stream.runHead(runtime.tunnelConnected); + + yield* TestClock.adjust(Duration.minutes(10)); + expect(yield* Queue.size(requests)).toBe(0); + }).pipe(Effect.provide(TestClock.layer())), + ); + it.effect("starts, deduplicates, rotates, and stops the Cloudflare connector", () => Effect.gen(function* () { const spawned: Array = []; diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.ts b/apps/server/src/cloud/ManagedEndpointRuntime.ts index 6b54b7ee3bdfdbbd0ca4c276dfd1a020621bab98..6c601fad80f38815129b378a8af802a476161925 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.ts @@ -59,6 +59,7 @@ interface ActiveConnector { readonly configKey: string; readonly config: RelayManagedEndpointRuntimeConfig; readonly startedAtMillis: number; + readonly connected: Ref.Ref; } // A connector that exits before running this long is treated as part of a @@ -71,6 +72,11 @@ const RELAY_RESTART_BACKOFF_BASE_MS = 1_000; const RELAY_RESTART_BACKOFF_MAX_MS = 60_000; // Newly created tunnels can fail authorization briefly while Cloudflare propagates their token. const TUNNEL_AUTHORIZATION_FAILURES_BEFORE_RECOVERY = 4; +// A connector that never registers a connection may hold a token for a tunnel +// that no longer exists, rejected in wording the output check does not know. +// Ask for recovery after this long, and again at this interval while it stays +// unconnected; the relay hands back the same tunnel when it is still live. +const CONNECTOR_REGISTRATION_TIMEOUT = Duration.minutes(3); export function classifyRelayClientOutput(line: string): "connected" | "warning" | "debug" { if (/\bRegistered tunnel connection\b/iu.test(line)) { @@ -84,14 +90,16 @@ export function classifyRelayClientOutput(line: string): "connected" | "warning" /** * Cloudflare's edge rejects a connector whose tunnel was deleted or whose - * token no longer matches. Current edge output is - * `error="Failed to get tunnel"` with no prefix; older edges prefixed the - * same messages with `Unauthorized:`. Match both so recovery fires on either. + * token no longer matches. The edge words this differently over time + * (`Failed to get tunnel`, `Tunnel not found`, ...), sometimes prefixed with + * `Unauthorized:`. Treat any `Unauthorized:` registration error as a rejection, + * plus the unprefixed messages seen so far. Transient rejections while a new + * tunnel's token propagates are absorbed by requiring several in a row. */ export function isRejectedRelayClientTunnelOutput(line: string): boolean { return ( /\bRegister tunnel error from server side\b/iu.test(line) && - /error="(?:Unauthorized:\s*)?(?:Failed to get tunnel|Record for tunnel not found|Invalid tunnel secret)"/iu.test( + /error="(?:Unauthorized:[^"]*|Failed to get tunnel|Tunnel not found|Record for tunnel not found|Invalid tunnel secret)"/iu.test( line, ) ); @@ -220,6 +228,24 @@ export const make = Effect.gen(function* () { Effect.catchCause((cause) => Effect.logWarning("Relay client supervisor failed", { cause })), ); + // Requests recovery while the connector has not registered a connection, + // once per timeout, until it connects or is replaced. + const watchConnectorRegistration = (connector: ActiveConnector) => + Effect.gen(function* () { + yield* Effect.sleep(CONNECTOR_REGISTRATION_TIMEOUT); + if (yield* Ref.get(connector.connected)) return true; + yield* Effect.logWarning( + "Relay client has not registered a tunnel connection; requesting recovery", + { + pid: Number(connector.child.pid), + tunnelId: connector.config.tunnelId, + tunnelName: connector.config.tunnelName, + }, + ); + yield* Queue.offer(recoveryRequests, connector.config); + return false; + }).pipe(Effect.repeat({ until: (connected) => connected })); + const observeConnectorOutput = (connector: ActiveConnector) => { let rejectedRegistrations = 0; @@ -239,7 +265,10 @@ export const make = Effect.gen(function* () { switch (classifyRelayClientOutput(line)) { case "connected": rejectedRegistrations = 0; - return Effect.logInfo("Relay client tunnel connection registered", attributes).pipe( + return Ref.set(connector.connected, true).pipe( + Effect.andThen( + Effect.logInfo("Relay client tunnel connection registered", attributes), + ), Effect.andThen(Queue.offer(tunnelConnections, undefined)), Effect.asVoid, ); @@ -370,10 +399,12 @@ export const make = Effect.gen(function* () { configKey: nextConfigKey, config, startedAtMillis: yield* Clock.currentTimeMillis, + connected: yield* Ref.make(false), } satisfies ActiveConnector; yield* Ref.set(activeRef, connector); yield* Effect.forkIn(observeConnectorOutput(connector), connectorScope); yield* Effect.forkIn(superviseConnector(connector), connectorScope); + yield* Effect.forkIn(watchConnectorRegistration(connector), connectorScope); return { status: "running", providerKind: "cloudflare_tunnel", -- 2.54.0