authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-10-06 18:11:46-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-10-06 18:11:46-07:00
log740bda40c92e1ded65dd2b65ef7c583357b1d258
tree9ed3257fa593eabe3cebdc4b450f3340acd3eb03
parentc4481b9b07d9108d3aa93494a0ec5dd28ed8fd80
signature Signed by PGP key B5690EEEBB952194

fix(server): recover a deleted tunnel when Cloudflare says "Tunnel not found" (#16648)

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

2 files changed, 127 insertions(+), 5 deletions(-)

apps/server/src/cloud/ManagedEndpointRuntime.test.ts+91
...@@ -185,11 +185,27 @@ describe("CloudManagedEndpointRuntime", () => {...@@ -185,11 +185,27 @@ describe("CloudManagedEndpointRuntime", () => {
185 '2026-06-17T02:00:00Z ERR Register tunnel error from server side error="Unauthorized: Invalid tunnel secret" connIndex=0',185 '2026-06-17T02:00:00Z ERR Register tunnel error from server side error="Unauthorized: Invalid tunnel secret" connIndex=0',
186 ),186 ),
187 ).toBe(true);187 ).toBe(true);
188 // Seen in production on 2026-10-06 after the relay deleted an idle tunnel.
189 expect(
190 ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput(
191 '2026-10-06T12:00:00Z ERR Register tunnel error from server side error="Unauthorized: Tunnel not found" connIndex=0 event=0 ip=198.41.200.23',
192 ),
193 ).toBe(true);
194 expect(
195 ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput(
196 '2026-10-06T12:00:00Z ERR Register tunnel error from server side error="Tunnel not found" connIndex=0',
197 ),
198 ).toBe(true);
188 expect(199 expect(
189 ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput(200 ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput(
190 '2026-06-17T02:00:00Z ERR Register tunnel error from server side error="connection timed out" connIndex=0',201 '2026-06-17T02:00:00Z ERR Register tunnel error from server side error="connection timed out" connIndex=0',
191 ),202 ),
192 ).toBe(false);203 ).toBe(false);
204 expect(
205 ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput(
206 '2026-06-17T02:00:00Z ERR Failed to serve tunnel connection error="Unauthorized: Tunnel not found" connIndex=0',
207 ),
208 ).toBe(false);
193 });209 });
194210
195 it.effect("keeps recovery requests sent before the server starts consuming them", () =>211 it.effect("keeps recovery requests sent before the server starts consuming them", () =>
...@@ -318,6 +334,81 @@ describe("CloudManagedEndpointRuntime", () => {...@@ -318,6 +334,81 @@ describe("CloudManagedEndpointRuntime", () => {
318 }),334 }),
319 );335 );
320336
337 it.effect("requests recovery while the connector never registers a connection", () =>
338 Effect.gen(function* () {
339 const output = yield* Queue.unbounded<Uint8Array>();
340 const spawner = ChildProcessSpawner.make(() =>
341 Effect.gen(function* () {
342 const handle = makeHandle({
343 pid: 800,
344 onKill: () => {},
345 output: Stream.fromQueue(output),
346 });
347 yield* Effect.addFinalizer(() => handle.kill().pipe(Effect.ignore));
348 return handle;
349 }),
350 );
351 const runtime = yield* buildCloudManagedEndpointRuntime(spawner);
352 const requests = yield* Queue.unbounded<RelayManagedEndpointRuntimeConfig>();
353 yield* runtime.recoveryRequests.pipe(
354 Stream.runForEach((config) => Queue.offer(requests, config)),
355 Effect.forkChild,
356 );
357 const config = {
358 providerKind: "cloudflare_tunnel" as const,
359 connectorToken: "token",
360 tunnelId: "silently-deleted",
361 };
362 yield* runtime.applyConfig(config);
363
364 yield* TestClock.adjust(Duration.minutes(2));
365 expect(yield* Queue.size(requests)).toBe(0);
366 yield* TestClock.adjust(Duration.minutes(1));
367 expect(yield* Queue.take(requests)).toEqual(config);
368 // Still unconnected, so it asks again.
369 yield* TestClock.adjust(Duration.minutes(3));
370 expect(yield* Queue.take(requests)).toEqual(config);
371 }).pipe(Effect.provide(TestClock.layer())),
372 );
373
374 it.effect("does not request recovery once the connector has connected", () =>
375 Effect.gen(function* () {
376 const output = yield* Queue.unbounded<Uint8Array>();
377 const spawner = ChildProcessSpawner.make(() =>
378 Effect.gen(function* () {
379 const handle = makeHandle({
380 pid: 801,
381 onKill: () => {},
382 output: Stream.fromQueue(output),
383 });
384 yield* Effect.addFinalizer(() => handle.kill().pipe(Effect.ignore));
385 return handle;
386 }),
387 );
388 const runtime = yield* buildCloudManagedEndpointRuntime(spawner);
389 const requests = yield* Queue.unbounded<RelayManagedEndpointRuntimeConfig>();
390 yield* runtime.recoveryRequests.pipe(
391 Stream.runForEach((config) => Queue.offer(requests, config)),
392 Effect.forkChild,
393 );
394 yield* runtime.applyConfig({
395 providerKind: "cloudflare_tunnel",
396 connectorToken: "token",
397 tunnelId: "tunnel-1",
398 });
399 yield* Queue.offer(
400 output,
401 new TextEncoder().encode(
402 "2026-10-06T00:00:00Z INF Registered tunnel connection connIndex=0\n",
403 ),
404 );
405 yield* Stream.runHead(runtime.tunnelConnected);
406
407 yield* TestClock.adjust(Duration.minutes(10));
408 expect(yield* Queue.size(requests)).toBe(0);
409 }).pipe(Effect.provide(TestClock.layer())),
410 );
411
321 it.effect("starts, deduplicates, rotates, and stops the Cloudflare connector", () =>412 it.effect("starts, deduplicates, rotates, and stops the Cloudflare connector", () =>
322 Effect.gen(function* () {413 Effect.gen(function* () {
323 const spawned: Array<ChildProcess.StandardCommand> = [];414 const spawned: Array<ChildProcess.StandardCommand> = [];
apps/server/src/cloud/ManagedEndpointRuntime.ts+36-5
...@@ -59,6 +59,7 @@ interface ActiveConnector {...@@ -59,6 +59,7 @@ interface ActiveConnector {
59 readonly configKey: string;59 readonly configKey: string;
60 readonly config: RelayManagedEndpointRuntimeConfig;60 readonly config: RelayManagedEndpointRuntimeConfig;
61 readonly startedAtMillis: number;61 readonly startedAtMillis: number;
62 readonly connected: Ref.Ref<boolean>;
62}63}
6364
64// A connector that exits before running this long is treated as part of a65// A connector that exits before running this long is treated as part of a
...@@ -71,6 +72,11 @@ const RELAY_RESTART_BACKOFF_BASE_MS = 1_000;...@@ -71,6 +72,11 @@ const RELAY_RESTART_BACKOFF_BASE_MS = 1_000;
71const RELAY_RESTART_BACKOFF_MAX_MS = 60_000;72const RELAY_RESTART_BACKOFF_MAX_MS = 60_000;
72// Newly created tunnels can fail authorization briefly while Cloudflare propagates their token.73// Newly created tunnels can fail authorization briefly while Cloudflare propagates their token.
73const TUNNEL_AUTHORIZATION_FAILURES_BEFORE_RECOVERY = 4;74const TUNNEL_AUTHORIZATION_FAILURES_BEFORE_RECOVERY = 4;
75// A connector that never registers a connection may hold a token for a tunnel
76// that no longer exists, rejected in wording the output check does not know.
77// Ask for recovery after this long, and again at this interval while it stays
78// unconnected; the relay hands back the same tunnel when it is still live.
79const CONNECTOR_REGISTRATION_TIMEOUT = Duration.minutes(3);
7480
75export function classifyRelayClientOutput(line: string): "connected" | "warning" | "debug" {81export function classifyRelayClientOutput(line: string): "connected" | "warning" | "debug" {
76 if (/\bRegistered tunnel connection\b/iu.test(line)) {82 if (/\bRegistered tunnel connection\b/iu.test(line)) {
...@@ -84,14 +90,16 @@ export function classifyRelayClientOutput(line: string): "connected" | "warning"...@@ -84,14 +90,16 @@ export function classifyRelayClientOutput(line: string): "connected" | "warning"
8490
85/**91/**
86 * Cloudflare's edge rejects a connector whose tunnel was deleted or whose92 * Cloudflare's edge rejects a connector whose tunnel was deleted or whose
87 * token no longer matches. Current edge output is93 * token no longer matches. The edge words this differently over time
88 * `error="Failed to get tunnel"` with no prefix; older edges prefixed the94 * (`Failed to get tunnel`, `Tunnel not found`, ...), sometimes prefixed with
89 * same messages with `Unauthorized:`. Match both so recovery fires on either.95 * `Unauthorized:`. Treat any `Unauthorized:` registration error as a rejection,
96 * plus the unprefixed messages seen so far. Transient rejections while a new
97 * tunnel's token propagates are absorbed by requiring several in a row.
90 */98 */
91export function isRejectedRelayClientTunnelOutput(line: string): boolean {99export function isRejectedRelayClientTunnelOutput(line: string): boolean {
92 return (100 return (
93 /\bRegister tunnel error from server side\b/iu.test(line) &&101 /\bRegister tunnel error from server side\b/iu.test(line) &&
94 /error="(?:Unauthorized:\s*)?(?:Failed to get tunnel|Record for tunnel not found|Invalid tunnel secret)"/iu.test(102 /error="(?:Unauthorized:[^"]*|Failed to get tunnel|Tunnel not found|Record for tunnel not found|Invalid tunnel secret)"/iu.test(
95 line,103 line,
96 )104 )
97 );105 );
...@@ -220,6 +228,24 @@ export const make = Effect.gen(function* () {...@@ -220,6 +228,24 @@ export const make = Effect.gen(function* () {
220 Effect.catchCause((cause) => Effect.logWarning("Relay client supervisor failed", { cause })),228 Effect.catchCause((cause) => Effect.logWarning("Relay client supervisor failed", { cause })),
221 );229 );
222230
231 // Requests recovery while the connector has not registered a connection,
232 // once per timeout, until it connects or is replaced.
233 const watchConnectorRegistration = (connector: ActiveConnector) =>
234 Effect.gen(function* () {
235 yield* Effect.sleep(CONNECTOR_REGISTRATION_TIMEOUT);
236 if (yield* Ref.get(connector.connected)) return true;
237 yield* Effect.logWarning(
238 "Relay client has not registered a tunnel connection; requesting recovery",
239 {
240 pid: Number(connector.child.pid),
241 tunnelId: connector.config.tunnelId,
242 tunnelName: connector.config.tunnelName,
243 },
244 );
245 yield* Queue.offer(recoveryRequests, connector.config);
246 return false;
247 }).pipe(Effect.repeat({ until: (connected) => connected }));
248
223 const observeConnectorOutput = (connector: ActiveConnector) => {249 const observeConnectorOutput = (connector: ActiveConnector) => {
224 let rejectedRegistrations = 0;250 let rejectedRegistrations = 0;
225251
...@@ -239,7 +265,10 @@ export const make = Effect.gen(function* () {...@@ -239,7 +265,10 @@ export const make = Effect.gen(function* () {
239 switch (classifyRelayClientOutput(line)) {265 switch (classifyRelayClientOutput(line)) {
240 case "connected":266 case "connected":
241 rejectedRegistrations = 0;267 rejectedRegistrations = 0;
242 return Effect.logInfo("Relay client tunnel connection registered", attributes).pipe(268 return Ref.set(connector.connected, true).pipe(
269 Effect.andThen(
270 Effect.logInfo("Relay client tunnel connection registered", attributes),
271 ),
243 Effect.andThen(Queue.offer(tunnelConnections, undefined)),272 Effect.andThen(Queue.offer(tunnelConnections, undefined)),
244 Effect.asVoid,273 Effect.asVoid,
245 );274 );
...@@ -370,10 +399,12 @@ export const make = Effect.gen(function* () {...@@ -370,10 +399,12 @@ export const make = Effect.gen(function* () {
370 configKey: nextConfigKey,399 configKey: nextConfigKey,
371 config,400 config,
372 startedAtMillis: yield* Clock.currentTimeMillis,401 startedAtMillis: yield* Clock.currentTimeMillis,
402 connected: yield* Ref.make(false),
373 } satisfies ActiveConnector;403 } satisfies ActiveConnector;
374 yield* Ref.set(activeRef, connector);404 yield* Ref.set(activeRef, connector);
375 yield* Effect.forkIn(observeConnectorOutput(connector), connectorScope);405 yield* Effect.forkIn(observeConnectorOutput(connector), connectorScope);
376 yield* Effect.forkIn(superviseConnector(connector), connectorScope);406 yield* Effect.forkIn(superviseConnector(connector), connectorScope);
407 yield* Effect.forkIn(watchConnectorRegistration(connector), connectorScope);
377 return {408 return {
378 status: "running",409 status: "running",
379 providerKind: "cloudflare_tunnel",410 providerKind: "cloudflare_tunnel",