authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-10-05 19:53:51-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-10-05 19:53:51-07:00
log6bfc8baeb8f280e57ef6d0d514680b0bce8538a2
tree39dd52130322cdcb0cf70304dededf2238cb24f1
parent41d2535a90fdfb49b2b9a7a7c8fb4bce09c4170f
signature Signed by PGP key B5690EEEBB952194

perf(relay): delete expired tunnels four at a time within a time budget (#13566)

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

3 files changed, 186 insertions(+), 38 deletions(-)

docs/operations/release.md+6-2
...@@ -210,8 +210,12 @@ A deleted legacy tunnel keeps its allocation, so its hostname is kept. When the...@@ -210,8 +210,12 @@ A deleted legacy tunnel keeps its allocation, so its hostname is kept. When the
2103. Run the legacy steps of the disposable-host canary below.2103. Run the legacy steps of the disposable-host canary below.
2114. Before enabling, confirm the web and mobile builds that show the "update T3 Code on that computer"2114. Before enabling, confirm the web and mobile builds that show the "update T3 Code on that computer"
212 message are live. Without them, a user whose older host lost its tunnel only sees it as offline.212 message are live. Without them, a user whose older host lost its tunnel only sees it as offline.
2135. Set the legacy mode to `enabled`. One sweep deletes at most 100 tunnels, so a backlog of2135. Set the legacy mode to `enabled`. One sweep deletes at most 100 tunnels, four at a time, and
214 20,000 takes about 17 hours. Watch `deletedLegacy`, `failed`, and `truncated`.214 stops starting new deletions after 90 seconds. A backlog of 20,000 takes about 17 hours if each
215 sweep finishes its 100. Watch `deletedLegacy`, `attempted`, `failed`, and `truncated`. An
216 `attempted` well under 100 with `truncated` set means the sweep stopped early: either the time
217 budget ran out or Cloudflare rate-limited a deletion. The counters don't say which; the relay
218 logs a warning with the Cloudflare error for each failed deletion.
215219
216### Disposable-host canary220### Disposable-host canary
217221
infra/relay/src/environments/ManagedEndpointReaper.test.ts+101-1
...@@ -75,6 +75,10 @@ function harness(input?: {...@@ -75,6 +75,10 @@ function harness(input?: {
75 readonly cleanupMode?: RelayConfiguration.ManagedEndpointCleanupMode;75 readonly cleanupMode?: RelayConfiguration.ManagedEndpointCleanupMode;
76 readonly legacyCleanupMode?: RelayConfiguration.ManagedEndpointCleanupMode;76 readonly legacyCleanupMode?: RelayConfiguration.ManagedEndpointCleanupMode;
77 readonly legacyTunnelGraceMinutes?: number;77 readonly legacyTunnelGraceMinutes?: number;
78 /** Simulated time each release takes, advanced on the test clock. */
79 readonly releaseDelayMs?: number;
80 /** Simulated time each Cloudflare list request takes. */
81 readonly listDelayMs?: number;
78}) {82}) {
79 const listRequests: ManagedEndpointProvider.ManagedEndpointTunnelListRequest[] = [];83 const listRequests: ManagedEndpointProvider.ManagedEndpointTunnelListRequest[] = [];
80 const deleted: string[] = [];84 const deleted: string[] = [];
...@@ -119,7 +123,10 @@ function harness(input?: {...@@ -119,7 +123,10 @@ function harness(input?: {
119 return Effect.succeed(found);123 return Effect.succeed(found);
120 }),124 }),
121 list: (request) =>125 list: (request) =>
122 Effect.sync(() => {126 Effect.gen(function* () {
127 if (input?.listDelayMs !== undefined) {
128 yield* TestClock.adjust(input.listDelayMs);
129 }
123 listRequests.push(request);130 listRequests.push(request);
124 const matching = remaining.filter((entry) => entry.status === request.status);131 const matching = remaining.filter((entry) => entry.status === request.status);
125 const start = ((request.page ?? 1) - 1) * (request.perPage ?? 100);132 const start = ((request.page ?? 1) - 1) * (request.perPage ?? 100);
...@@ -191,6 +198,9 @@ function harness(input?: {...@@ -191,6 +198,9 @@ function harness(input?: {
191 release: (request) =>198 release: (request) =>
192 Effect.gen(function* () {199 Effect.gen(function* () {
193 releases.push(request);200 releases.push(request);
201 if (input?.releaseDelayMs !== undefined) {
202 yield* TestClock.adjust(input.releaseDelayMs);
203 }
194 if (request.expectedTunnelId === input?.skipTunnelId) {204 if (request.expectedTunnelId === input?.skipTunnelId) {
195 return false;205 return false;
196 }206 }
...@@ -689,6 +699,96 @@ describe("ManagedEndpointReaper", () => {...@@ -689,6 +699,96 @@ describe("ManagedEndpointReaper", () => {
689 }).pipe(Effect.provide(state.layer));699 }).pipe(Effect.provide(state.layer));
690 });700 });
691701
702 it.effect("starts no new deletion after a rate limit, even with deletions in flight", () => {
703 // Enough candidates to fill every concurrent slot several times over.
704 const entries = Array.from({ length: 12 }, (_, index) =>
705 tunnel({
706 id: index === 0 ? "limited" : `ok-${index}`,
707 suffix: index.toString(16).padStart(16, "0"),
708 status: "down",
709 timestamp: "2026-08-25T11:00:00.000Z",
710 }),
711 );
712 const state = harness({
713 tunnels: entries,
714 allocations: recoverableOwners(entries),
715 rateLimitedTunnelId: "limited",
716 });
717
718 return Effect.gen(function* () {
719 yield* TestClock.setTime(NOW_MILLIS);
720 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
721 const result = yield* reaper.sweep;
722 expect(result.truncated).toBe(true);
723 expect(result.failed).toBe(1);
724 // Releases already running may finish, but none start afterwards.
725 expect(result.attempted).toBeLessThanOrEqual(
726 ManagedEndpointReaper.MANAGED_ENDPOINT_SWEEP_DELETE_CONCURRENCY,
727 );
728 expect(state.releases.length).toBe(result.attempted);
729 }).pipe(Effect.provide(state.layer));
730 });
731
732 it.effect("stops starting deletions when the sweep's time budget runs out", () => {
733 const entries = Array.from({ length: 40 }, (_, index) =>
734 tunnel({
735 id: `slow-${index}`,
736 suffix: index.toString(16).padStart(16, "0"),
737 status: "down",
738 timestamp: "2026-08-25T11:00:00.000Z",
739 }),
740 );
741 const state = harness({
742 tunnels: entries,
743 allocations: recoverableOwners(entries),
744 // Ten seconds each: the 90-second budget allows about 9 rounds.
745 releaseDelayMs: 10_000,
746 });
747
748 return Effect.gen(function* () {
749 yield* TestClock.setTime(NOW_MILLIS);
750 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
751 const result = yield* reaper.sweep;
752 expect(result.truncated).toBe(true);
753 expect(result.attempted).toBeGreaterThan(0);
754 expect(result.attempted).toBeLessThan(entries.length);
755 expect(result.deleted).toBe(result.attempted);
756 }).pipe(Effect.provide(state.layer));
757 });
758
759 it.effect("counts listing time against the deletion budget", () => {
760 const entries = Array.from({ length: 40 }, (_, index) =>
761 tunnel({
762 id: `slow-${index}`,
763 suffix: index.toString(16).padStart(16, "0"),
764 status: "down",
765 timestamp: "2026-08-25T11:00:00.000Z",
766 }),
767 );
768 const sweepWith = (listDelayMs: number) =>
769 Effect.gen(function* () {
770 yield* TestClock.setTime(NOW_MILLIS);
771 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
772 return (yield* reaper.sweep).attempted;
773 }).pipe(
774 Effect.provide(
775 harness({
776 tunnels: entries,
777 allocations: recoverableOwners(entries),
778 releaseDelayMs: 10_000,
779 listDelayMs,
780 }).layer,
781 ),
782 );
783
784 return Effect.gen(function* () {
785 const fastListing = yield* sweepWith(0);
786 // Two list requests of 20 seconds each use 40 of the 90-second budget.
787 const slowListing = yield* sweepWith(20_000);
788 expect(slowListing).toBeLessThan(fastListing);
789 });
790 });
791
692 it.effect("continues past a page of older hosts to find recoverable tunnels", () => {792 it.effect("continues past a page of older hosts to find recoverable tunnels", () => {
693 const entries = Array.from({ length: 101 }, (_, index) =>793 const entries = Array.from({ length: 101 }, (_, index) =>
694 tunnel({794 tunnel({
infra/relay/src/environments/ManagedEndpointReaper.ts+79-35
...@@ -1,3 +1,4 @@...@@ -1,3 +1,4 @@
1import * as Clock from "effect/Clock";
1import * as Context from "effect/Context";2import * as Context from "effect/Context";
2import * as DateTime from "effect/DateTime";3import * as DateTime from "effect/DateTime";
3import * as Effect from "effect/Effect";4import * as Effect from "effect/Effect";
...@@ -27,6 +28,16 @@ const MANAGED_ENDPOINT_LEGACY_AGE_BUCKET_DAYS = [7, 30, 90] as const;...@@ -27,6 +28,16 @@ const MANAGED_ENDPOINT_LEGACY_AGE_BUCKET_DAYS = [7, 30, 90] as const;
27// returning host that has updated recovers the tunnel at the same hostname;28// returning host that has updated recovers the tunnel at the same hostname;
28// one that has not sees the client's "update T3 Code" message instead.29// one that has not sees the client's "update T3 Code" message instead.
29const MANAGED_ENDPOINT_LEGACY_GRACE_PERIOD_DAYS = 7;30const MANAGED_ENDPOINT_LEGACY_GRACE_PERIOD_DAYS = 7;
31// Deletions run a few at a time: each one is a row-locked database
32// transaction plus two Cloudflare calls, so one at a time cannot finish a
33// full attempt budget inside the cron's timeout. Each holds a Hyperdrive
34// connection while it runs; keep this well under the 20-connection origin
35// limit that request handlers share.
36export const MANAGED_ENDPOINT_SWEEP_DELETE_CONCURRENCY = 4;
37// Stop starting deletions this long after the sweep starts, leaving room under
38// the cron's two-minute timeout to finish in-flight ones and record the
39// counters. Counted from sweep start so slow listing eats into it.
40const MANAGED_ENDPOINT_SWEEP_DELETE_BUDGET_MS = 90_000;
3041
31export interface ManagedEndpointSweepResult {42export interface ManagedEndpointSweepResult {
32 readonly mode: RelayConfiguration.ManagedEndpointCleanupMode;43 readonly mode: RelayConfiguration.ManagedEndpointCleanupMode;
...@@ -180,6 +191,7 @@ export const make = Effect.gen(function* () {...@@ -180,6 +191,7 @@ export const make = Effect.gen(function* () {
180 if ((mode === "off" && legacyMode === "off") || !namespace) {191 if ((mode === "off" && legacyMode === "off") || !namespace) {
181 return emptyResult(mode, legacyMode);192 return emptyResult(mode, legacyMode);
182 }193 }
194 const sweepStartedAtMillis = yield* Clock.currentTimeMillis;
183 // The override exists for the disposable canary stage; prod always195 // The override exists for the disposable canary stage; prod always
184 // waits the full grace period.196 // waits the full grace period.
185 const legacyGraceMinutes =197 const legacyGraceMinutes =
...@@ -275,6 +287,16 @@ export const make = Effect.gen(function* () {...@@ -275,6 +287,16 @@ export const make = Effect.gen(function* () {
275 let attempted = 0;287 let attempted = 0;
276 let deleted = 0;288 let deleted = 0;
277 let wouldDelete = 0;289 let wouldDelete = 0;
290 const candidates: Array<{
291 readonly owner: ManagedEndpointAllocations.ManagedEndpointTunnelAllocation;
292 readonly tunnel: ManagedEndpointProvider.ManagedEndpointTunnel & {
293 readonly id: string;
294 readonly name: string;
295 };
296 readonly status: "down" | "inactive";
297 readonly legacy: boolean;
298 readonly inactiveBefore: string;
299 }> = [];
278 let wouldDeleteLegacy = 0;300 let wouldDeleteLegacy = 0;
279 let deletedLegacy = 0;301 let deletedLegacy = 0;
280 let skippedLegacy = 0;302 let skippedLegacy = 0;
...@@ -330,46 +352,68 @@ export const make = Effect.gen(function* () {...@@ -330,46 +352,68 @@ export const make = Effect.gen(function* () {
330 wouldDelete += 1;352 wouldDelete += 1;
331 if (mode === "dry-run") continue;353 if (mode === "dry-run") continue;
332 }354 }
333 if (attempted >= MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT) {355 if (candidates.length >= MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT) {
334 truncated = true;356 truncated = true;
335 break;357 break;
336 }358 }
337 attempted += 1;359 candidates.push({
338 // The release re-reads the tunnel and deletes only if it is still in360 owner,
339 // this status and inactive since before this tunnel's cutoff.361 tunnel,
340 const result = yield* provider362 status,
341 .release({363 legacy,
342 userId: owner.userId,364 // The release re-reads the tunnel and deletes only if it is still in
343 environmentId: owner.environmentId,365 // this status and inactive since before this cutoff.
344 expectedTunnelId: tunnel.id,366 inactiveBefore: DateTime.formatIso(legacy ? legacyCutoff : cutoff),
345 expectedInactiveBefore: DateTime.formatIso(legacy ? legacyCutoff : cutoff),367 });
346 expectedStatus: status,
347 })
348 .pipe(Effect.result);
349 if (result._tag === "Failure") {
350 failed += 1;
351 yield* Effect.logWarning("Failed to delete an inactive managed tunnel", {
352 tunnelId: tunnel.id,
353 tunnelName: tunnel.name,
354 legacy,
355 cause: result.failure,
356 });
357 if (isRateLimited(result.failure)) {
358 truncated = true;
359 break;
360 }
361 } else if (result.success) {
362 deleted += 1;
363 if (legacy) deletedLegacy += 1;
364 yield* Effect.logInfo("Deleted an inactive managed tunnel", {
365 tunnelId: tunnel.id,
366 tunnelName: tunnel.name,
367 status,
368 legacy,
369 });
370 }
371 }368 }
372369
370 const deleteDeadline = sweepStartedAtMillis + MANAGED_ENDPOINT_SWEEP_DELETE_BUDGET_MS;
371 let stopDeleting = false;
372 yield* Effect.forEach(
373 candidates,
374 (candidate) =>
375 Effect.gen(function* () {
376 if (stopDeleting || (yield* Clock.currentTimeMillis) >= deleteDeadline) {
377 stopDeleting = true;
378 truncated = true;
379 return;
380 }
381 attempted += 1;
382 const result = yield* provider
383 .release({
384 userId: candidate.owner.userId,
385 environmentId: candidate.owner.environmentId,
386 expectedTunnelId: candidate.tunnel.id,
387 expectedInactiveBefore: candidate.inactiveBefore,
388 expectedStatus: candidate.status,
389 })
390 .pipe(Effect.result);
391 if (result._tag === "Failure") {
392 failed += 1;
393 yield* Effect.logWarning("Failed to delete an inactive managed tunnel", {
394 tunnelId: candidate.tunnel.id,
395 tunnelName: candidate.tunnel.name,
396 legacy: candidate.legacy,
397 cause: result.failure,
398 });
399 if (isRateLimited(result.failure)) {
400 stopDeleting = true;
401 truncated = true;
402 }
403 } else if (result.success) {
404 deleted += 1;
405 if (candidate.legacy) deletedLegacy += 1;
406 yield* Effect.logInfo("Deleted an inactive managed tunnel", {
407 tunnelId: candidate.tunnel.id,
408 tunnelName: candidate.tunnel.name,
409 status: candidate.status,
410 legacy: candidate.legacy,
411 });
412 }
413 }),
414 { concurrency: MANAGED_ENDPOINT_SWEEP_DELETE_CONCURRENCY, discard: true },
415 );
416
373 return {417 return {
374 mode,418 mode,
375 legacyMode,419 legacyMode,