authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-10-05 19:53:50-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-10-05 19:53:50-07:00
log7b32b2ca39892bc59e1d8cfff7aea2a8fbefa73a
tree857b9827d18d5c603e7806548f13dc97750ffe56
parent18c1210810cbdb798d28edda1826fa4de91992d1
signature Signed by PGP key B5690EEEBB952194

feat(relay): measure the managed tunnel backlog (#13564)

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

4 files changed, 218 insertions(+), 1 deletions(-)

infra/relay/package.json+1
...@@ -6,6 +6,7 @@...@@ -6,6 +6,7 @@
6 "deploy": "alchemy deploy",6 "deploy": "alchemy deploy",
7 "push:android:smoke": "node scripts/android-push-smoke.ts",7 "push:android:smoke": "node scripts/android-push-smoke.ts",
8 "push:android:watch": "node scripts/android-push-watch.ts",8 "push:android:watch": "node scripts/android-push-watch.ts",
9 "tunnels:census": "node scripts/tunnel-census.ts",
9 "destroy": "alchemy destroy",10 "destroy": "alchemy destroy",
10 "test": "vp test run --config ../../vite.config.ts --dir .",11 "test": "vp test run --config ../../vite.config.ts --dir .",
11 "typecheck": "tsc --noEmit"12 "typecheck": "tsc --noEmit"
infra/relay/scripts/tunnel-census.ts created+97
...@@ -0,0 +1,97 @@
1// Read-only count of Cloudflare tunnels in the managed-endpoint account,
2// grouped by relay stage, status, and how long they have been idle. The reaper
3// only sees its own stage's prefix, so this shows how much of the backlog
4// belongs to other stages.
5//
6// CLOUDFLARE_ACCOUNT_ID=... CLOUDFLARE_API_TOKEN=... node scripts/tunnel-census.ts
7//
8// The token needs Cloudflare Tunnel read access. Nothing is modified.
9import * as NodeRuntime from "@effect/platform-node/NodeRuntime";
10import * as Clock from "effect/Clock";
11import * as Config from "effect/Config";
12import * as Console from "effect/Console";
13import * as Effect from "effect/Effect";
14import * as Redacted from "effect/Redacted";
15import * as Schema from "effect/Schema";
16import * as FetchHttpClient from "effect/http/FetchHttpClient";
17import * as HttpClient from "effect/http/HttpClient";
18import * as HttpClientRequest from "effect/http/HttpClientRequest";
19import * as HttpClientResponse from "effect/http/HttpClientResponse";
20
21const PREFIX = "t3coderelay-managedendpoint-";
22const PAGE_SIZE = 1_000;
23const DAY_MS = 86_400_000;
24
25// Cloudflare documents every field here as optional, so a tunnel missing one
26// must not fail the whole page.
27const Tunnel = Schema.Struct({
28 name: Schema.optional(Schema.NullOr(Schema.String)),
29 status: Schema.optional(Schema.NullOr(Schema.String)),
30 created_at: Schema.optional(Schema.NullOr(Schema.String)),
31 conns_inactive_at: Schema.optional(Schema.NullOr(Schema.String)),
32});
33type Tunnel = typeof Tunnel.Type;
34const TunnelPage = Schema.Struct({ result: Schema.Array(Tunnel) });
35
36// "t3coderelay-managedendpoint-<stage>-<16 hex>"; stages may contain hyphens.
37const stageOf = (name: string | null | undefined) =>
38 name ? name.slice(PREFIX.length).replace(/-[a-f0-9]{16}$/u, "") : "unknown";
39
40// Like the reaper: a down tunnel is aged from when it lost its connector, and a
41// never-connected (inactive) one from when it was created.
42const ageBucket = (tunnel: Tunnel, now: number) => {
43 const since = Date.parse(
44 (tunnel.status === "down" ? tunnel.conns_inactive_at : tunnel.created_at) ?? "",
45 );
46 if (Number.isNaN(since)) return "unknown";
47 const days = (now - since) / DAY_MS;
48 if (days > 90) return ">90d";
49 if (days > 30) return ">30d";
50 if (days > 7) return ">7d";
51 return "<=7d";
52};
53
54const main = Effect.gen(function* () {
55 const accountId = yield* Config.String("CLOUDFLARE_ACCOUNT_ID");
56 const token = yield* Config.Redacted("CLOUDFLARE_API_TOKEN");
57 const client = yield* HttpClient.HttpClient;
58 const now = yield* Clock.currentTimeMillis;
59
60 const listPage = (page: number) =>
61 HttpClientRequest.get(`https://api.cloudflare.com/client/v4/accounts/${accountId}/cfd_tunnel`, {
62 urlParams: {
63 is_deleted: "false",
64 include_prefix: PREFIX,
65 per_page: String(PAGE_SIZE),
66 page: String(page),
67 },
68 }).pipe(
69 HttpClientRequest.bearerToken(Redacted.value(token)),
70 client.execute,
71 Effect.flatMap(HttpClientResponse.filterStatusOk),
72 Effect.flatMap(HttpClientResponse.schemaBodyJson(TunnelPage)),
73 Effect.map((body) => body.result),
74 );
75
76 const counts = new Map<string, number>();
77 let total = 0;
78 for (let page = 1; ; page += 1) {
79 const tunnels = yield* listPage(page);
80 for (const tunnel of tunnels) {
81 total += 1;
82 const idle =
83 tunnel.status === "down" || tunnel.status === "inactive" ? ageBucket(tunnel, now) : "-";
84 const key = `${stageOf(tunnel.name)}\t${tunnel.status ?? "unknown"}\t${idle}`;
85 counts.set(key, (counts.get(key) ?? 0) + 1);
86 }
87 if (tunnels.length < PAGE_SIZE) break;
88 }
89
90 yield* Console.log("stage\tstatus\tidle\tcount");
91 for (const [key, count] of [...counts].sort(([a], [b]) => a.localeCompare(b))) {
92 yield* Console.log(`${key}\t${count}`);
93 }
94 yield* Console.log(`total\t\t\t${total}`);
95}).pipe(Effect.provide(FetchHttpClient.layer));
96
97NodeRuntime.runMain(main);
infra/relay/src/environments/ManagedEndpointReaper.test.ts+61
...@@ -436,6 +436,60 @@ describe("ManagedEndpointReaper", () => {...@@ -436,6 +436,60 @@ describe("ManagedEndpointReaper", () => {
436 }).pipe(Effect.provide(state.layer));436 }).pipe(Effect.provide(state.layer));
437 });437 });
438438
439 it.effect("measures legacy age, uncounted skips, and Cloudflare totals", () => {
440 const down = (id: string, suffix: string, timestamp: string) =>
441 tunnel({ id, suffix, status: "down", timestamp });
442 const tunnels = [
443 // Legacy owners, down for 1, 10, 40, and 100 days, plus one just past
444 // seven days, which must already count as over seven.
445 down("legacy-1d", "1111111111111111", "2026-08-24T11:00:00.000Z"),
446 down("legacy-7d1h", "7777777777777777", "2026-08-18T11:00:00.000Z"),
447 down("legacy-10d", "2222222222222222", "2026-08-15T11:00:00.000Z"),
448 down("legacy-40d", "3333333333333333", "2026-07-16T11:00:00.000Z"),
449 down("legacy-100d", "4444444444444444", "2026-05-17T11:00:00.000Z"),
450 // The allocation now records a different tunnel under this name.
451 down("stale", "5555555555555555", "2026-08-25T11:00:00.000Z"),
452 // The allocation has not recorded a tunnel yet.
453 down("pending", "6666666666666666", "2026-08-25T11:00:00.000Z"),
454 ];
455 const state = harness({
456 tunnels,
457 allocations: [
458 allocation({ tunnelId: "legacy-1d", recoveryEnabled: false }),
459 allocation({ tunnelId: "legacy-7d1h", recoveryEnabled: false }),
460 allocation({ tunnelId: "legacy-10d", recoveryEnabled: false }),
461 allocation({ tunnelId: "legacy-40d", recoveryEnabled: false }),
462 allocation({ tunnelId: "legacy-100d", recoveryEnabled: false }),
463 {
464 ...allocation({ tunnelId: "current", recoveryEnabled: true }),
465 tunnelName: `${PREFIX}5555555555555555`,
466 },
467 {
468 ...allocation({ tunnelId: null, recoveryEnabled: false }),
469 tunnelName: `${PREFIX}6666666666666666`,
470 },
471 ],
472 cleanupMode: "dry-run",
473 });
474
475 return Effect.gen(function* () {
476 yield* TestClock.setTime(NOW_MILLIS);
477 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
478 expect(yield* reaper.sweep).toMatchObject({
479 scanned: 7,
480 skippedLegacy: 5,
481 legacyOver7Days: 4,
482 legacyOver30Days: 2,
483 legacyOver90Days: 1,
484 skippedReplaced: 1,
485 skippedUnrecorded: 1,
486 totalDown: 7,
487 totalInactive: 0,
488 wouldDelete: 0,
489 });
490 }).pipe(Effect.provide(state.layer));
491 });
492
439 it.effect("does not count a tunnel that was replaced before its release", () => {493 it.effect("does not count a tunnel that was replaced before its release", () => {
440 const state = harness({494 const state = harness({
441 tunnels: [495 tunnels: [
...@@ -649,7 +703,14 @@ describe("ManagedEndpointReaper", () => {...@@ -649,7 +703,14 @@ describe("ManagedEndpointReaper", () => {
649 deleted: 0,703 deleted: 0,
650 wouldDelete: 0,704 wouldDelete: 0,
651 skippedLegacy: 0,705 skippedLegacy: 0,
706 legacyOver7Days: 0,
707 legacyOver30Days: 0,
708 legacyOver90Days: 0,
652 skippedOrphan: 0,709 skippedOrphan: 0,
710 skippedReplaced: 0,
711 skippedUnrecorded: 0,
712 totalDown: null,
713 totalInactive: null,
653 failed: 0,714 failed: 0,
654 truncated: false,715 truncated: false,
655 });716 });
infra/relay/src/environments/ManagedEndpointReaper.ts+59-1
...@@ -16,6 +16,9 @@ export const MANAGED_ENDPOINT_INACTIVE_GRACE_PERIOD_MINUTES = 60;...@@ -16,6 +16,9 @@ export const MANAGED_ENDPOINT_INACTIVE_GRACE_PERIOD_MINUTES = 60;
16export const MANAGED_ENDPOINT_SWEEP_PAGE_SIZE = 100;16export const MANAGED_ENDPOINT_SWEEP_PAGE_SIZE = 100;
17export const MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT = 100;17export const MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT = 100;
18export const MANAGED_ENDPOINT_SWEEP_LIST_REQUEST_LIMIT = 10;18export const MANAGED_ENDPOINT_SWEEP_LIST_REQUEST_LIMIT = 10;
19// Age buckets for legacy candidates, in days since the tunnel went down (or
20// was created, for one that never connected).
21const MANAGED_ENDPOINT_LEGACY_AGE_BUCKET_DAYS = [7, 30, 90] as const;
1922
20export interface ManagedEndpointSweepResult {23export interface ManagedEndpointSweepResult {
21 readonly mode: RelayConfiguration.ManagedEndpointCleanupMode;24 readonly mode: RelayConfiguration.ManagedEndpointCleanupMode;
...@@ -25,7 +28,18 @@ export interface ManagedEndpointSweepResult {...@@ -25,7 +28,18 @@ export interface ManagedEndpointSweepResult {
25 readonly deleted: number;28 readonly deleted: number;
26 readonly wouldDelete: number;29 readonly wouldDelete: number;
27 readonly skippedLegacy: number;30 readonly skippedLegacy: number;
31 /** Legacy candidates, by days since they went down: over 7, 30, and 90. */
32 readonly legacyOver7Days: number;
33 readonly legacyOver30Days: number;
34 readonly legacyOver90Days: number;
28 readonly skippedOrphan: number;35 readonly skippedOrphan: number;
36 /** The allocation row records a different tunnel under this name. */
37 readonly skippedReplaced: number;
38 /** The allocation row has not recorded a tunnel yet. */
39 readonly skippedUnrecorded: number;
40 /** Cloudflare's count of matching tunnels, before page limits. */
41 readonly totalDown: number | null;
42 readonly totalInactive: number | null;
29 readonly failed: number;43 readonly failed: number;
30 readonly truncated: boolean;44 readonly truncated: boolean;
31}45}
...@@ -70,6 +84,19 @@ function isExpiredManagedTunnel(input: {...@@ -70,6 +84,19 @@ function isExpiredManagedTunnel(input: {
70 return Option.isSome(timestamp) && timestamp.value.epochMilliseconds <= cutoff.epochMilliseconds;84 return Option.isSome(timestamp) && timestamp.value.epochMilliseconds <= cutoff.epochMilliseconds;
71}85}
7286
87/** Days (fractional) since the tunnel went down, or was created if it never connected. */
88function inactiveDaysAt(
89 tunnel: ManagedEndpointProvider.ManagedEndpointTunnel,
90 status: "down" | "inactive",
91 now: DateTime.Utc,
92): number | null {
93 const since = status === "down" ? tunnel.connsInactiveAt : tunnel.createdAt;
94 if (typeof since !== "string") return null;
95 const timestamp = DateTime.make(since);
96 if (Option.isNone(timestamp)) return null;
97 return (now.epochMilliseconds - timestamp.value.epochMilliseconds) / 86_400_000;
98}
99
73function isRateLimited(cause: unknown): boolean {100function isRateLimited(cause: unknown): boolean {
74 if (typeof cause !== "object" || cause === null) {101 if (typeof cause !== "object" || cause === null) {
75 return false;102 return false;
...@@ -112,7 +139,14 @@ const emptyResult = (...@@ -112,7 +139,14 @@ const emptyResult = (
112 deleted: 0,139 deleted: 0,
113 wouldDelete: 0,140 wouldDelete: 0,
114 skippedLegacy: 0,141 skippedLegacy: 0,
142 legacyOver7Days: 0,
143 legacyOver30Days: 0,
144 legacyOver90Days: 0,
115 skippedOrphan: 0,145 skippedOrphan: 0,
146 skippedReplaced: 0,
147 skippedUnrecorded: 0,
148 totalDown: null,
149 totalInactive: null,
116 failed: 0,150 failed: 0,
117 truncated: false,151 truncated: false,
118});152});
...@@ -142,6 +176,7 @@ export const make = Effect.gen(function* () {...@@ -142,6 +176,7 @@ export const make = Effect.gen(function* () {
142 );176 );
143 let listRequests = 0;177 let listRequests = 0;
144 let truncated = false;178 let truncated = false;
179 const totals: Record<"down" | "inactive", number | null> = { down: null, inactive: null };
145 const expired: Array<{180 const expired: Array<{
146 readonly tunnel: ManagedEndpointProvider.ManagedEndpointTunnel & {181 readonly tunnel: ManagedEndpointProvider.ManagedEndpointTunnel & {
147 readonly id: string;182 readonly id: string;
...@@ -169,6 +204,7 @@ export const make = Effect.gen(function* () {...@@ -169,6 +204,7 @@ export const make = Effect.gen(function* () {
169 const first = yield* listPage(1);204 const first = yield* listPage(1);
170 const totalCount =205 const totalCount =
171 typeof first.resultInfo?.totalCount === "number" ? first.resultInfo.totalCount : undefined;206 typeof first.resultInfo?.totalCount === "number" ? first.resultInfo.totalCount : undefined;
207 totals[status] = totalCount ?? null;
172 const pages = rotatedPages({208 const pages = rotatedPages({
173 totalCount,209 totalCount,
174 slot,210 slot,
...@@ -213,7 +249,12 @@ export const make = Effect.gen(function* () {...@@ -213,7 +249,12 @@ export const make = Effect.gen(function* () {
213 let deleted = 0;249 let deleted = 0;
214 let wouldDelete = 0;250 let wouldDelete = 0;
215 let skippedLegacy = 0;251 let skippedLegacy = 0;
252 const legacyOverDays = new Map<number, number>(
253 MANAGED_ENDPOINT_LEGACY_AGE_BUCKET_DAYS.map((days) => [days, 0]),
254 );
216 let skippedOrphan = 0;255 let skippedOrphan = 0;
256 let skippedReplaced = 0;
257 let skippedUnrecorded = 0;
217 let failed = 0;258 let failed = 0;
218259
219 for (const { tunnel, status, cutoff } of uniqueExpired) {260 for (const { tunnel, status, cutoff } of uniqueExpired) {
...@@ -224,14 +265,24 @@ export const make = Effect.gen(function* () {...@@ -224,14 +265,24 @@ export const make = Effect.gen(function* () {
224 allocation.tunnelId !== null &&265 allocation.tunnelId !== null &&
225 allocation.tunnelId !== tunnel.id266 allocation.tunnelId !== tunnel.id
226 ) {267 ) {
268 skippedReplaced += 1;
227 continue;269 continue;
228 }270 }
229 const owner = allocation?.tunnelId === tunnel.id ? allocation : undefined;271 const owner = allocation?.tunnelId === tunnel.id ? allocation : undefined;
230 if (owner !== undefined && !owner.recoveryEnabled) {272 if (owner !== undefined && !owner.recoveryEnabled) {
231 skippedLegacy += 1;273 skippedLegacy += 1;
274 const inactiveDays = inactiveDaysAt(tunnel, status, now);
275 for (const days of MANAGED_ENDPOINT_LEGACY_AGE_BUCKET_DAYS) {
276 if (inactiveDays !== null && inactiveDays > days) {
277 legacyOverDays.set(days, (legacyOverDays.get(days) ?? 0) + 1);
278 }
279 }
280 continue;
281 }
282 if (allocation !== undefined && owner === undefined) {
283 skippedUnrecorded += 1;
232 continue;284 continue;
233 }285 }
234 if (allocation !== undefined && owner === undefined) continue;
235 // A tunnel with no allocation row cannot be claimed, so a relink that286 // A tunnel with no allocation row cannot be claimed, so a relink that
236 // adopts it by name races any delete here. Count it and leave it for a287 // adopts it by name races any delete here. Count it and leave it for a
237 // manual sweep instead.288 // manual sweep instead.
...@@ -284,7 +335,14 @@ export const make = Effect.gen(function* () {...@@ -284,7 +335,14 @@ export const make = Effect.gen(function* () {
284 deleted,335 deleted,
285 wouldDelete,336 wouldDelete,
286 skippedLegacy,337 skippedLegacy,
338 legacyOver7Days: legacyOverDays.get(7) ?? 0,
339 legacyOver30Days: legacyOverDays.get(30) ?? 0,
340 legacyOver90Days: legacyOverDays.get(90) ?? 0,
287 skippedOrphan,341 skippedOrphan,
342 skippedReplaced,
343 skippedUnrecorded,
344 totalDown: totals.down,
345 totalInactive: totals.inactive,
288 failed,346 failed,
289 truncated,347 truncated,
290 };348 };