authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-10-06 18:11:46-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-10-06 18:11:46-07:00
log740bda40c92e1ded65dd2b65ef7c583357b1d258
tree9ed3257fa593eabe3cebdc4b450f3340acd3eb03
parentc4481b9b07d9108d3aa93494a0ec5dd28ed8fd80
signature Signed by PGP key B5690EEEBB952194

fix(server): recover a deleted tunnel when Cloudflare says "Tunnel not found" (#16648)

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

2 files changed, 127 insertions(+), 5 deletions(-)

apps/server/src/cloud/ManagedEndpointRuntime.test.ts+91
......@@ -185,11 +185,27 @@ describe("CloudManagedEndpointRuntime", () => {
185185 '2026-06-17T02:00:00Z ERR Register tunnel error from server side error="Unauthorized: Invalid tunnel secret" connIndex=0',
186186 ),
187187 ).toBe(true);
188 // Seen in production on 2026-10-06 after the relay deleted an idle tunnel.
189 expect(
190 ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput(
191 '2026-10-06T12:00:00Z ERR Register tunnel error from server side error="Unauthorized: Tunnel not found" connIndex=0 event=0 ip=198.41.200.23',
192 ),
193 ).toBe(true);
194 expect(
195 ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput(
196 '2026-10-06T12:00:00Z ERR Register tunnel error from server side error="Tunnel not found" connIndex=0',
197 ),
198 ).toBe(true);
188199 expect(
189200 ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput(
190201 '2026-06-17T02:00:00Z ERR Register tunnel error from server side error="connection timed out" connIndex=0',
191202 ),
192203 ).toBe(false);
204 expect(
205 ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput(
206 '2026-06-17T02:00:00Z ERR Failed to serve tunnel connection error="Unauthorized: Tunnel not found" connIndex=0',
207 ),
208 ).toBe(false);
193209 });
194210
195211 it.effect("keeps recovery requests sent before the server starts consuming them", () =>
......@@ -318,6 +334,81 @@ describe("CloudManagedEndpointRuntime", () => {
318334 }),
319335 );
320336
337 it.effect("requests recovery while the connector never registers a connection", () =>
338 Effect.gen(function* () {
339 const output = yield* Queue.unbounded<Uint8Array>();
340 const spawner = ChildProcessSpawner.make(() =>
341 Effect.gen(function* () {
342 const handle = makeHandle({
343 pid: 800,
344 onKill: () => {},
345 output: Stream.fromQueue(output),
346 });
347 yield* Effect.addFinalizer(() => handle.kill().pipe(Effect.ignore));
348 return handle;
349 }),
350 );
351 const runtime = yield* buildCloudManagedEndpointRuntime(spawner);
352 const requests = yield* Queue.unbounded<RelayManagedEndpointRuntimeConfig>();
353 yield* runtime.recoveryRequests.pipe(
354 Stream.runForEach((config) => Queue.offer(requests, config)),
355 Effect.forkChild,
356 );
357 const config = {
358 providerKind: "cloudflare_tunnel" as const,
359 connectorToken: "token",
360 tunnelId: "silently-deleted",
361 };
362 yield* runtime.applyConfig(config);
363
364 yield* TestClock.adjust(Duration.minutes(2));
365 expect(yield* Queue.size(requests)).toBe(0);
366 yield* TestClock.adjust(Duration.minutes(1));
367 expect(yield* Queue.take(requests)).toEqual(config);
368 // Still unconnected, so it asks again.
369 yield* TestClock.adjust(Duration.minutes(3));
370 expect(yield* Queue.take(requests)).toEqual(config);
371 }).pipe(Effect.provide(TestClock.layer())),
372 );
373
374 it.effect("does not request recovery once the connector has connected", () =>
375 Effect.gen(function* () {
376 const output = yield* Queue.unbounded<Uint8Array>();
377 const spawner = ChildProcessSpawner.make(() =>
378 Effect.gen(function* () {
379 const handle = makeHandle({
380 pid: 801,
381 onKill: () => {},
382 output: Stream.fromQueue(output),
383 });
384 yield* Effect.addFinalizer(() => handle.kill().pipe(Effect.ignore));
385 return handle;
386 }),
387 );
388 const runtime = yield* buildCloudManagedEndpointRuntime(spawner);
389 const requests = yield* Queue.unbounded<RelayManagedEndpointRuntimeConfig>();
390 yield* runtime.recoveryRequests.pipe(
391 Stream.runForEach((config) => Queue.offer(requests, config)),
392 Effect.forkChild,
393 );
394 yield* runtime.applyConfig({
395 providerKind: "cloudflare_tunnel",
396 connectorToken: "token",
397 tunnelId: "tunnel-1",
398 });
399 yield* Queue.offer(
400 output,
401 new TextEncoder().encode(
402 "2026-10-06T00:00:00Z INF Registered tunnel connection connIndex=0\n",
403 ),
404 );
405 yield* Stream.runHead(runtime.tunnelConnected);
406
407 yield* TestClock.adjust(Duration.minutes(10));
408 expect(yield* Queue.size(requests)).toBe(0);
409 }).pipe(Effect.provide(TestClock.layer())),
410 );
411
321412 it.effect("starts, deduplicates, rotates, and stops the Cloudflare connector", () =>
322413 Effect.gen(function* () {
323414 const spawned: Array<ChildProcess.StandardCommand> = [];
apps/server/src/cloud/ManagedEndpointRuntime.ts+36-5
......@@ -59,6 +59,7 @@ interface ActiveConnector {
5959 readonly configKey: string;
6060 readonly config: RelayManagedEndpointRuntimeConfig;
6161 readonly startedAtMillis: number;
62 readonly connected: Ref.Ref<boolean>;
6263}
6364
6465// A connector that exits before running this long is treated as part of a
......@@ -71,6 +72,11 @@ const RELAY_RESTART_BACKOFF_BASE_MS = 1_000;
7172const RELAY_RESTART_BACKOFF_MAX_MS = 60_000;
7273// Newly created tunnels can fail authorization briefly while Cloudflare propagates their token.
7374const TUNNEL_AUTHORIZATION_FAILURES_BEFORE_RECOVERY = 4;
75// A connector that never registers a connection may hold a token for a tunnel
76// that no longer exists, rejected in wording the output check does not know.
77// Ask for recovery after this long, and again at this interval while it stays
78// unconnected; the relay hands back the same tunnel when it is still live.
79const CONNECTOR_REGISTRATION_TIMEOUT = Duration.minutes(3);
7480
7581export function classifyRelayClientOutput(line: string): "connected" | "warning" | "debug" {
7682 if (/\bRegistered tunnel connection\b/iu.test(line)) {
......@@ -84,14 +90,16 @@ export function classifyRelayClientOutput(line: string): "connected" | "warning"
8490
8591/**
8692 * Cloudflare's edge rejects a connector whose tunnel was deleted or whose
87 * token no longer matches. Current edge output is
88 * `error="Failed to get tunnel"` with no prefix; older edges prefixed the
89 * same messages with `Unauthorized:`. Match both so recovery fires on either.
93 * token no longer matches. The edge words this differently over time
94 * (`Failed to get tunnel`, `Tunnel not found`, ...), sometimes prefixed with
95 * `Unauthorized:`. Treat any `Unauthorized:` registration error as a rejection,
96 * plus the unprefixed messages seen so far. Transient rejections while a new
97 * tunnel's token propagates are absorbed by requiring several in a row.
9098 */
9199export function isRejectedRelayClientTunnelOutput(line: string): boolean {
92100 return (
93101 /\bRegister tunnel error from server side\b/iu.test(line) &&
94 /error="(?:Unauthorized:\s*)?(?:Failed to get tunnel|Record for tunnel not found|Invalid tunnel secret)"/iu.test(
102 /error="(?:Unauthorized:[^"]*|Failed to get tunnel|Tunnel not found|Record for tunnel not found|Invalid tunnel secret)"/iu.test(
95103 line,
96104 )
97105 );
......@@ -220,6 +228,24 @@ export const make = Effect.gen(function* () {
220228 Effect.catchCause((cause) => Effect.logWarning("Relay client supervisor failed", { cause })),
221229 );
222230
231 // Requests recovery while the connector has not registered a connection,
232 // once per timeout, until it connects or is replaced.
233 const watchConnectorRegistration = (connector: ActiveConnector) =>
234 Effect.gen(function* () {
235 yield* Effect.sleep(CONNECTOR_REGISTRATION_TIMEOUT);
236 if (yield* Ref.get(connector.connected)) return true;
237 yield* Effect.logWarning(
238 "Relay client has not registered a tunnel connection; requesting recovery",
239 {
240 pid: Number(connector.child.pid),
241 tunnelId: connector.config.tunnelId,
242 tunnelName: connector.config.tunnelName,
243 },
244 );
245 yield* Queue.offer(recoveryRequests, connector.config);
246 return false;
247 }).pipe(Effect.repeat({ until: (connected) => connected }));
248
223249 const observeConnectorOutput = (connector: ActiveConnector) => {
224250 let rejectedRegistrations = 0;
225251
......@@ -239,7 +265,10 @@ export const make = Effect.gen(function* () {
239265 switch (classifyRelayClientOutput(line)) {
240266 case "connected":
241267 rejectedRegistrations = 0;
242 return Effect.logInfo("Relay client tunnel connection registered", attributes).pipe(
268 return Ref.set(connector.connected, true).pipe(
269 Effect.andThen(
270 Effect.logInfo("Relay client tunnel connection registered", attributes),
271 ),
243272 Effect.andThen(Queue.offer(tunnelConnections, undefined)),
244273 Effect.asVoid,
245274 );
......@@ -370,10 +399,12 @@ export const make = Effect.gen(function* () {
370399 configKey: nextConfigKey,
371400 config,
372401 startedAtMillis: yield* Clock.currentTimeMillis,
402 connected: yield* Ref.make(false),
373403 } satisfies ActiveConnector;
374404 yield* Ref.set(activeRef, connector);
375405 yield* Effect.forkIn(observeConnectorOutput(connector), connectorScope);
376406 yield* Effect.forkIn(superviseConnector(connector), connectorScope);
407 yield* Effect.forkIn(watchConnectorRegistration(connector), connectorScope);
377408 return {
378409 status: "running",
379410 providerKind: "cloudflare_tunnel",