1import { describe, expect, it } from "@effect/vitest";
2import * as DateTime from "effect/DateTime";
3import * as Effect from "effect/Effect";
4import * as Layer from "effect/Layer";
5import * as Redacted from "effect/Redacted";
6import * as TestClock from "effect/testing/TestClock";
7import * as Tracer from "effect/Tracer";
8
9import * as RelayConfiguration from "../Config.ts";
10import * as ManagedEndpointAllocations from "./ManagedEndpointAllocations.ts";
11import * as ManagedEndpointProvider from "./ManagedEndpointProvider.ts";
12import * as ManagedEndpointReaper from "./ManagedEndpointReaper.ts";
13
14const NOW = "2026-08-25T12:00:00.000Z";
15const NOW_MILLIS = DateTime.makeUnsafe(NOW).epochMilliseconds;
16const PREFIX = "t3coderelay-managedendpoint-prod-";
17
18function tunnel(input: {
19 readonly id: string;
20 readonly suffix: string;
21 readonly status: "down" | "inactive" | "healthy" | "degraded";
22 readonly timestamp?: string | null;
23 readonly prefix?: string;
24}): ManagedEndpointProvider.ManagedEndpointTunnel {
25 return {
26 id: input.id,
27 name: `${input.prefix ?? PREFIX}${input.suffix}`,
28 status: input.status,
29 ...(input.timestamp === undefined
30 ? {}
31 : input.status === "inactive"
32 ? { createdAt: input.timestamp }
33 : { connsInactiveAt: input.timestamp }),
34 };
35}
36
37function recoverableOwners(
38 tunnels: ReadonlyArray<ManagedEndpointProvider.ManagedEndpointTunnel>,
39): ReadonlyArray<ManagedEndpointAllocations.ManagedEndpointTunnelAllocation> {
40 return tunnels.map((entry) => allocation({ tunnelId: entry.id!, recoveryEnabled: true }));
41}
42
43function allocation(input: {
44 readonly tunnelId: string | null;
45 readonly recoveryEnabled: boolean;
46}): ManagedEndpointAllocations.ManagedEndpointTunnelAllocation {
47 return {
48 userId: "user-1",
49 environmentId: `environment-${input.tunnelId ?? "pending"}`,
50 hostname: `${input.tunnelId ?? "pending"}.example.test`,
51 tunnelId: input.tunnelId,
52 tunnelName: `${PREFIX}aaaaaaaaaaaaaaaa`,
53 dnsRecordId: "dns-1",
54 readyAt: "2026-08-25T11:00:00.000Z",
55 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
56 updatedAt: "2026-08-25T11:00:00.000Z",
57 generation: 1,
58 tunnelReleasedAt: null,
59 recoveryEnabled: input.recoveryEnabled,
60 };
61}
62
63function harness(input?: {
64 readonly tunnels?: ReadonlyArray<ManagedEndpointProvider.ManagedEndpointTunnel>;
65 readonly allocations?: ReadonlyArray<ManagedEndpointAllocations.ManagedEndpointTunnelAllocation>;
66 readonly namespace?: string;
67 readonly failTunnelId?: string;
68 readonly rateLimitedTunnelId?: string;
69 readonly failAllDeletes?: boolean;
70 readonly failDeleteWhen?: (tunnelId: string) => boolean;
71 readonly missingOnDeleteTunnelId?: string;
72 readonly missingOnGetTunnelId?: string;
73 readonly reserveOnGetTunnelId?: string;
74 readonly refreshedTunnels?: ReadonlyMap<string, ManagedEndpointProvider.ManagedEndpointTunnel>;
75 readonly skipTunnelId?: string;
76 readonly cleanupMode?: RelayConfiguration.ManagedEndpointCleanupMode;
77 readonly legacyCleanupMode?: RelayConfiguration.ManagedEndpointCleanupMode;
78 readonly legacyTunnelGraceMinutes?: number;
79 /** Simulated time each release takes, advanced on the test clock. */
80 readonly releaseDelayMs?: number;
81 /** Simulated time each Cloudflare list request takes. */
82 readonly listDelayMs?: number;
83}) {
84 const listRequests: ManagedEndpointProvider.ManagedEndpointTunnelListRequest[] = [];
85 const deleted: string[] = [];
86 const releases: Array<
87 Parameters<ManagedEndpointProvider.ManagedEndpointProvider["Service"]["release"]>[0]
88 > = [];
89 const remaining = [...(input?.tunnels ?? [])];
90 const recorded = (input?.allocations ?? []).map((entry) => {
91 const matching = remaining.find((candidate) => candidate.id === entry.tunnelId);
92 return typeof matching?.name === "string" ? { ...entry, tunnelName: matching.name } : entry;
93 });
94 const tunnelClient = ManagedEndpointProvider.ManagedEndpointTunnelClient.of({
95 get: (tunnelId) =>
96 Effect.suspend(() => {
97 if (tunnelId === input?.missingOnGetTunnelId) {
98 return Effect.fail(
99 new ManagedEndpointProvider.ManagedEndpointTunnelClientError({
100 operation: "get",
101 tunnelId,
102 cause: { _tag: "NotFound" },
103 }),
104 );
105 }
106 const found =
107 input?.refreshedTunnels?.get(tunnelId) ??
108 remaining.find((candidate) => candidate.id === tunnelId);
109 if (found === undefined) {
110 return Effect.fail(
111 new ManagedEndpointProvider.ManagedEndpointTunnelClientError({
112 operation: "get",
113 tunnelId,
114 cause: { _tag: "NotFound" },
115 }),
116 );
117 }
118 if (tunnelId === input?.reserveOnGetTunnelId && typeof found.name === "string") {
119 recorded.push({
120 ...allocation({ tunnelId, recoveryEnabled: false }),
121 tunnelName: found.name,
122 });
123 }
124 return Effect.succeed(found);
125 }),
126 list: (request) =>
127 Effect.gen(function* () {
128 if (input?.listDelayMs !== undefined) {
129 yield* TestClock.adjust(input.listDelayMs);
130 }
131 listRequests.push(request);
132 const matching = remaining.filter((entry) => entry.status === request.status);
133 const start = ((request.page ?? 1) - 1) * (request.perPage ?? 100);
134 return {
135 result: matching.slice(start, start + (request.perPage ?? 100)),
136 resultInfo: {
137 page: request.page ?? 1,
138 perPage: request.perPage ?? 100,
139 totalCount: matching.length,
140 },
141 };
142 }),
143 create: () => Effect.die("unused"),
144 putConfiguration: () => Effect.die("unused"),
145 getToken: () => Effect.die("unused"),
146 delete: (tunnelId) =>
147 input?.failAllDeletes === true ||
148 input?.failDeleteWhen?.(tunnelId) === true ||
149 tunnelId === input?.failTunnelId ||
150 tunnelId === input?.rateLimitedTunnelId ||
151 tunnelId === input?.missingOnDeleteTunnelId
152 ? Effect.fail(
153 new ManagedEndpointProvider.ManagedEndpointTunnelClientError({
154 operation: "delete",
155 tunnelId,
156 cause:
157 tunnelId === input?.missingOnDeleteTunnelId
158 ? { _tag: "NotFound" }
159 : tunnelId === input?.rateLimitedTunnelId
160 ? {
161 cause: {
162 _tag: "TooManyRequests",
163 message: "Cloudflare rate limit exceeded",
164 retryAfter: 60,
165 },
166 }
167 : "Cloudflare refused the deletion",
168 }),
169 )
170 : Effect.sync(() => {
171 deleted.push(tunnelId);
172 const index = remaining.findIndex((candidate) => candidate.id === tunnelId);
173 if (index !== -1) {
174 remaining.splice(index, 1);
175 }
176 }),
177 });
178 const allocationService = ManagedEndpointAllocations.ManagedEndpointAllocations.of({
179 getByTunnelName: () => Effect.die("unused"),
180 get: () => Effect.die("unused"),
181 reserve: () => Effect.die("unused"),
182 recordTunnel: () => Effect.die("unused"),
183 recordDns: () => Effect.die("unused"),
184 markReady: () => Effect.die("unused"),
185 enableRecovery: () => Effect.die("unused"),
186 listByTunnelNames: (tunnelNames) =>
187 Effect.succeed(recorded.filter((entry) => tunnelNames.includes(entry.tunnelName))),
188 claimRelease: () => Effect.die("unused"),
189 withClaimedTunnel: () => Effect.die("unused"),
190 claimDeprovision: () => Effect.die("unused"),
191 remove: () => Effect.die("unused"),
192 removeClaimed: () => Effect.die("unused"),
193 });
194 const provider = ManagedEndpointProvider.ManagedEndpointProvider.of({
195 provision: () => Effect.die("unused"),
196 reconcileOrigin: () => Effect.die("unused"),
197 prepareDeprovision: () => Effect.die("unused"),
198 deprovision: () => Effect.die("unused"),
199 release: (request) =>
200 Effect.gen(function* () {
201 releases.push(request);
202 if (input?.releaseDelayMs !== undefined) {
203 yield* TestClock.adjust(input.releaseDelayMs);
204 }
205 if (request.expectedTunnelId === input?.skipTunnelId) {
206 return false;
207 }
208 if (request.expectedTunnelId !== undefined) {
209 // Surface Cloudflare failures the same way the real release does.
210 yield* tunnelClient.delete(request.expectedTunnelId).pipe(
211 Effect.mapError(
212 (cause) =>
213 new ManagedEndpointProvider.ManagedEndpointDeprovisioningFailed({
214 stage: "delete-tunnel",
215 userId: request.userId,
216 environmentId: request.environmentId,
217 tunnelId: request.expectedTunnelId!,
218 cause,
219 }),
220 ),
221 );
222 }
223 return true;
224 }),
225 });
226 const config = RelayConfiguration.RelayConfiguration.of({
227 relayIssuer: "https://relay.example.test",
228 apns: {
229 environment: "sandbox",
230 teamId: "team-id",
231 keyId: "key-id",
232 privateKey: Redacted.make("private-key"),
233 bundleId: "com.t3tools.t3code.dev",
234 },
235 apnsDeliveryJobSigningSecret: Redacted.make("job-secret"),
236 clerkSecretKey: Redacted.make("clerk-secret"),
237 clerkPublishableKey: "pk_test_test",
238 clerkJwtAudience: "t3-code-relay",
239 cloudMintPrivateKey: Redacted.make("cloud-private-key"),
240 cloudMintPublicKey: "cloud-public-key",
241 managedEndpointBaseDomain: "example.test",
242 managedEndpointNamespace: input?.namespace ?? "prod",
243 managedEndpointCleanupMode: input?.cleanupMode ?? "enabled",
244 legacyManagedEndpointCleanupMode: input?.legacyCleanupMode ?? "off",
245 ...(input?.legacyTunnelGraceMinutes === undefined
246 ? {}
247 : { legacyTunnelGraceMinutes: input.legacyTunnelGraceMinutes }),
248 });
249
250 return {
251 listRequests,
252 deleted,
253 releases,
254 layer: ManagedEndpointReaper.layer.pipe(
255 Layer.provide(
256 Layer.mergeAll(
257 RelayConfiguration.layer(config),
258 ManagedEndpointProvider.layerTunnelClient(tunnelClient),
259 Layer.succeed(ManagedEndpointProvider.ManagedEndpointProvider, provider),
260 Layer.succeed(ManagedEndpointAllocations.ManagedEndpointAllocations, allocationService),
261 ),
262 ),
263 ),
264 };
265}
266
267describe("ManagedEndpointReaper", () => {
268 it.effect("removes expired down and inactive tunnels from recoverable environments", () => {
269 const state = harness({
270 tunnels: [
271 tunnel({
272 id: "down-1",
273 suffix: "aaaaaaaaaaaaaaaa",
274 status: "down",
275 timestamp: "2026-08-25T11:55:00.000Z",
276 }),
277 tunnel({
278 id: "inactive-1",
279 suffix: "bbbbbbbbbbbbbbbb",
280 status: "inactive",
281 timestamp: "2026-08-25T10:59:00.000Z",
282 }),
283 ],
284 allocations: [
285 allocation({ tunnelId: "down-1", recoveryEnabled: true }),
286 allocation({ tunnelId: "inactive-1", recoveryEnabled: true }),
287 ],
288 });
289
290 return Effect.gen(function* () {
291 yield* TestClock.setTime(NOW_MILLIS);
292 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
293 expect(yield* reaper.sweep).toMatchObject({
294 scanned: 2,
295 deleted: 2,
296 skippedLegacy: 0,
297 failed: 0,
298 });
299 expect(state.deleted).toEqual(["down-1", "inactive-1"]);
300 // A host that registered recovery gets a new tunnel on its own.
301 expect(state.releases.some((release) => release.markReleased === true)).toBe(false);
302 expect(state.releases.map((request) => request.expectedTunnelId)).toEqual([
303 "down-1",
304 "inactive-1",
305 ]);
306 expect(state.listRequests).toEqual([
307 {
308 isDeleted: false,
309 includePrefix: PREFIX,
310 status: "down",
311 existedAt: "2026-08-25T11:55:00.000Z",
312 wasInactiveAt: "2026-08-25T11:55:00.000Z",
313 page: 1,
314 perPage: 100,
315 },
316 {
317 isDeleted: false,
318 includePrefix: PREFIX,
319 status: "inactive",
320 existedAt: "2026-08-25T11:00:00.000Z",
321 page: 1,
322 perPage: 100,
323 },
324 ]);
325 }).pipe(Effect.provide(state.layer));
326 });
327
328 it.effect("keeps a tunnel that never connected until it is an hour old", () => {
329 const state = harness({
330 tunnels: [
331 tunnel({
332 id: "pairing",
333 suffix: "cccccccccccccccc",
334 status: "inactive",
335 timestamp: "2026-08-25T11:30:00.000Z",
336 }),
337 ],
338 allocations: [allocation({ tunnelId: "pairing", recoveryEnabled: true })],
339 });
340
341 return Effect.gen(function* () {
342 yield* TestClock.setTime(NOW_MILLIS);
343 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
344 expect((yield* reaper.sweep).deleted).toBe(0);
345 expect(state.deleted).toEqual([]);
346 }).pipe(Effect.provide(state.layer));
347 });
348
349 it.effect("keeps recent tunnels, other stages, and tunnels without valid timestamps", () => {
350 const state = harness({
351 tunnels: [
352 tunnel({
353 id: "recent",
354 suffix: "aaaaaaaaaaaaaaaa",
355 status: "down",
356 timestamp: "2026-08-25T11:55:01.000Z",
357 }),
358 tunnel({
359 id: "other-stage",
360 prefix: `${PREFIX}julius-`,
361 suffix: "bbbbbbbbbbbbbbbb",
362 status: "down",
363 timestamp: "2026-08-25T11:00:00.000Z",
364 }),
365 tunnel({
366 id: "missing-time",
367 suffix: "cccccccccccccccc",
368 status: "inactive",
369 timestamp: null,
370 }),
371 ],
372 });
373
374 return Effect.gen(function* () {
375 yield* TestClock.setTime(NOW_MILLIS);
376 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
377 expect(yield* reaper.sweep).toMatchObject({
378 scanned: 0,
379 deleted: 0,
380 skippedLegacy: 0,
381 failed: 0,
382 });
383 expect(state.deleted).toEqual([]);
384 }).pipe(Effect.provide(state.layer));
385 });
386
387 it.effect("keeps tunnels owned by environments that cannot recover yet", () => {
388 const state = harness({
389 tunnels: [
390 tunnel({
391 id: "legacy",
392 suffix: "aaaaaaaaaaaaaaaa",
393 status: "down",
394 timestamp: "2026-08-25T11:00:00.000Z",
395 }),
396 ],
397 allocations: [allocation({ tunnelId: "legacy", recoveryEnabled: false })],
398 });
399
400 return Effect.gen(function* () {
401 yield* TestClock.setTime(NOW_MILLIS);
402 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
403 expect(yield* reaper.sweep).toMatchObject({
404 scanned: 1,
405 deleted: 0,
406 skippedLegacy: 1,
407 failed: 0,
408 });
409 expect(state.deleted).toEqual([]);
410 }).pipe(Effect.provide(state.layer));
411 });
412
413 it.effect("counts an expired tunnel with no allocation instead of deleting it", () => {
414 const state = harness({
415 tunnels: [
416 tunnel({
417 id: "orphan",
418 suffix: "cccccccccccccccc",
419 status: "down",
420 timestamp: "2026-08-25T11:00:00.000Z",
421 }),
422 ],
423 });
424
425 return Effect.gen(function* () {
426 yield* TestClock.setTime(NOW_MILLIS);
427 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
428 expect(yield* reaper.sweep).toMatchObject({
429 scanned: 1,
430 wouldDelete: 0,
431 deleted: 0,
432 skippedOrphan: 1,
433 });
434 expect(state.deleted).toEqual([]);
435 }).pipe(Effect.provide(state.layer));
436 });
437 it.effect("keeps an expired tunnel while its allocation is incomplete", () => {
438 const state = harness({
439 tunnels: [
440 tunnel({
441 id: "unrecorded",
442 suffix: "aaaaaaaaaaaaaaaa",
443 status: "inactive",
444 timestamp: "2026-08-25T11:00:00.000Z",
445 }),
446 ],
447 allocations: [allocation({ tunnelId: null, recoveryEnabled: false })],
448 });
449
450 return Effect.gen(function* () {
451 yield* TestClock.setTime(NOW_MILLIS);
452 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
453 expect((yield* reaper.sweep).deleted).toBe(0);
454 expect(state.deleted).toEqual([]);
455 }).pipe(Effect.provide(state.layer));
456 });
457
458 it.effect("measures legacy age, uncounted skips, and Cloudflare totals", () => {
459 const down = (id: string, suffix: string, timestamp: string) =>
460 tunnel({ id, suffix, status: "down", timestamp });
461 const tunnels = [
462 // Legacy owners, down for 1, 10, 40, and 100 days, plus one just past
463 // seven days, which must already count as over seven.
464 down("legacy-1d", "1111111111111111", "2026-08-24T11:00:00.000Z"),
465 down("legacy-7d1h", "7777777777777777", "2026-08-18T11:00:00.000Z"),
466 down("legacy-10d", "2222222222222222", "2026-08-15T11:00:00.000Z"),
467 down("legacy-40d", "3333333333333333", "2026-07-16T11:00:00.000Z"),
468 down("legacy-100d", "4444444444444444", "2026-05-17T11:00:00.000Z"),
469 // The allocation now records a different tunnel under this name.
470 down("stale", "5555555555555555", "2026-08-25T11:00:00.000Z"),
471 // The allocation has not recorded a tunnel yet.
472 down("pending", "6666666666666666", "2026-08-25T11:00:00.000Z"),
473 ];
474 const state = harness({
475 tunnels,
476 allocations: [
477 allocation({ tunnelId: "legacy-1d", recoveryEnabled: false }),
478 allocation({ tunnelId: "legacy-7d1h", recoveryEnabled: false }),
479 allocation({ tunnelId: "legacy-10d", recoveryEnabled: false }),
480 allocation({ tunnelId: "legacy-40d", recoveryEnabled: false }),
481 allocation({ tunnelId: "legacy-100d", recoveryEnabled: false }),
482 {
483 ...allocation({ tunnelId: "current", recoveryEnabled: true }),
484 tunnelName: `${PREFIX}5555555555555555`,
485 },
486 {
487 ...allocation({ tunnelId: null, recoveryEnabled: false }),
488 tunnelName: `${PREFIX}6666666666666666`,
489 },
490 ],
491 cleanupMode: "dry-run",
492 });
493
494 return Effect.gen(function* () {
495 yield* TestClock.setTime(NOW_MILLIS);
496 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
497 expect(yield* reaper.sweep).toMatchObject({
498 scanned: 7,
499 skippedLegacy: 5,
500 legacyOver7Days: 4,
501 legacyOver30Days: 2,
502 legacyOver90Days: 1,
503 skippedReplaced: 1,
504 skippedUnrecorded: 1,
505 totalDown: 7,
506 totalInactive: 0,
507 wouldDelete: 0,
508 });
509 }).pipe(Effect.provide(state.layer));
510 });
511
512 describe("legacy cleanup", () => {
513 const legacyTunnels = [
514 // Down for 5 days: inside the legacy grace period.
515 tunnel({
516 id: "legacy-recent",
517 suffix: "1111111111111111",
518 status: "down",
519 timestamp: "2026-08-20T12:00:00.000Z",
520 }),
521 // Down for 40 days: past it.
522 tunnel({
523 id: "legacy-old",
524 suffix: "2222222222222222",
525 status: "down",
526 timestamp: "2026-07-16T12:00:00.000Z",
527 }),
528 // Never connected, created 40 days ago.
529 tunnel({
530 id: "legacy-never",
531 suffix: "3333333333333333",
532 status: "inactive",
533 timestamp: "2026-07-16T12:00:00.000Z",
534 }),
535 ];
536 const legacyOwners = legacyTunnels.map((entry) =>
537 allocation({ tunnelId: entry.id!, recoveryEnabled: false }),
538 );
539
540 it.effect("leaves legacy tunnels alone while legacy cleanup is off", () => {
541 const state = harness({ tunnels: legacyTunnels, allocations: legacyOwners });
542 return Effect.gen(function* () {
543 yield* TestClock.setTime(NOW_MILLIS);
544 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
545 expect(yield* reaper.sweep).toMatchObject({
546 skippedLegacy: 3,
547 wouldDeleteLegacy: 0,
548 deletedLegacy: 0,
549 });
550 expect(state.deleted).toEqual([]);
551 }).pipe(Effect.provide(state.layer));
552 });
553
554 it.effect("counts legacy tunnels past the grace period in dry-run", () => {
555 const state = harness({
556 tunnels: legacyTunnels,
557 allocations: legacyOwners,
558 legacyCleanupMode: "dry-run",
559 });
560 return Effect.gen(function* () {
561 yield* TestClock.setTime(NOW_MILLIS);
562 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
563 expect(yield* reaper.sweep).toMatchObject({
564 legacyMode: "dry-run",
565 wouldDeleteLegacy: 2,
566 deletedLegacy: 0,
567 attempted: 0,
568 });
569 expect(state.deleted).toEqual([]);
570 }).pipe(Effect.provide(state.layer));
571 });
572
573 it.effect("deletes only legacy tunnels past the grace period", () => {
574 const state = harness({
575 tunnels: legacyTunnels,
576 allocations: legacyOwners,
577 legacyCleanupMode: "enabled",
578 });
579 return Effect.gen(function* () {
580 yield* TestClock.setTime(NOW_MILLIS);
581 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
582 expect(yield* reaper.sweep).toMatchObject({
583 wouldDeleteLegacy: 2,
584 deletedLegacy: 2,
585 deleted: 2,
586 });
587 expect([...state.deleted].sort()).toEqual(["legacy-never", "legacy-old"]);
588 // A legacy host needs an update to recover, so its user is told why.
589 expect(state.releases.every((release) => release.markReleased === true)).toBe(true);
590 // The release re-checks each tunnel against the 7-day cutoff, not
591 // the 5-minute cutoff used for recoverable tunnels.
592 for (const release of state.releases) {
593 expect(release.expectedInactiveBefore).toBe("2026-08-18T12:00:00.000Z");
594 }
595 }).pipe(Effect.provide(state.layer));
596 });
597
598 it.effect.each([
599 { stage: "canary", expected: 3 },
600 { stage: "prod", expected: 2 },
601 ] as const)(
602 "applies the grace-period override only off prod ($stage)",
603 ({ stage, expected }) => {
604 const stageLegacyTunnels = legacyTunnels.map((entry) => ({
605 ...entry,
606 name: entry.name!.replace(PREFIX, `t3coderelay-managedendpoint-${stage}-`),
607 }));
608 const state = harness({
609 namespace: stage,
610 tunnels: stageLegacyTunnels,
611 allocations: legacyOwners,
612 legacyCleanupMode: "dry-run",
613 // Ten minutes: the 10-day-old tunnel qualifies only if this applies.
614 legacyTunnelGraceMinutes: 10,
615 });
616 return Effect.gen(function* () {
617 yield* TestClock.setTime(NOW_MILLIS);
618 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
619 expect((yield* reaper.sweep).wouldDeleteLegacy).toBe(expected);
620 }).pipe(Effect.provide(state.layer));
621 },
622 );
623
624 it.effect("runs legacy cleanup while recoverable cleanup is off", () => {
625 const recoverable = tunnel({
626 id: "recoverable",
627 suffix: "4444444444444444",
628 status: "down",
629 timestamp: "2026-08-25T11:00:00.000Z",
630 });
631 const state = harness({
632 tunnels: [...legacyTunnels, recoverable],
633 allocations: [
634 ...legacyOwners,
635 allocation({ tunnelId: "recoverable", recoveryEnabled: true }),
636 ],
637 cleanupMode: "off",
638 legacyCleanupMode: "enabled",
639 });
640 return Effect.gen(function* () {
641 yield* TestClock.setTime(NOW_MILLIS);
642 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
643 expect(yield* reaper.sweep).toMatchObject({ wouldDelete: 0, deletedLegacy: 2 });
644 expect(state.deleted).not.toContain("recoverable");
645 }).pipe(Effect.provide(state.layer));
646 });
647 });
648
649 it.effect("does not count a tunnel that was replaced before its release", () => {
650 const state = harness({
651 tunnels: [
652 tunnel({
653 id: "replaced",
654 suffix: "aaaaaaaaaaaaaaaa",
655 status: "down",
656 timestamp: "2026-08-25T11:00:00.000Z",
657 }),
658 ],
659 allocations: [allocation({ tunnelId: "replaced", recoveryEnabled: true })],
660 skipTunnelId: "replaced",
661 });
662
663 return Effect.gen(function* () {
664 yield* TestClock.setTime(NOW_MILLIS);
665 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
666 expect((yield* reaper.sweep).deleted).toBe(0);
667 expect(state.deleted).toEqual([]);
668 }).pipe(Effect.provide(state.layer));
669 });
670
671 it.effect("stops the sweep after a structured Cloudflare rate limit error", () => {
672 const state = harness({
673 tunnels: [
674 tunnel({
675 id: "limited",
676 suffix: "aaaaaaaaaaaaaaaa",
677 status: "down",
678 timestamp: "2026-08-25T11:00:00.000Z",
679 }),
680 tunnel({
681 id: "next",
682 suffix: "bbbbbbbbbbbbbbbb",
683 status: "down",
684 timestamp: "2026-08-25T11:00:00.000Z",
685 }),
686 ],
687 allocations: [
688 allocation({ tunnelId: "limited", recoveryEnabled: true }),
689 allocation({ tunnelId: "next", recoveryEnabled: true }),
690 ],
691 rateLimitedTunnelId: "limited",
692 });
693
694 return Effect.gen(function* () {
695 yield* TestClock.setTime(NOW_MILLIS);
696 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
697 expect(yield* reaper.sweep).toMatchObject({
698 attempted: 1,
699 deleted: 0,
700 failed: 1,
701 truncated: true,
702 });
703 expect(state.deleted).toEqual([]);
704 }).pipe(Effect.provide(state.layer));
705 });
706
707 it.effect("starts no new deletion after a rate limit, even with deletions in flight", () => {
708 // Enough candidates to fill every concurrent slot several times over.
709 const entries = Array.from({ length: 12 }, (_, index) =>
710 tunnel({
711 id: index === 0 ? "limited" : `ok-${index}`,
712 suffix: index.toString(16).padStart(16, "0"),
713 status: "down",
714 timestamp: "2026-08-25T11:00:00.000Z",
715 }),
716 );
717 const state = harness({
718 tunnels: entries,
719 allocations: recoverableOwners(entries),
720 rateLimitedTunnelId: "limited",
721 });
722
723 return Effect.gen(function* () {
724 yield* TestClock.setTime(NOW_MILLIS);
725 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
726 const result = yield* reaper.sweep;
727 expect(result.truncated).toBe(true);
728 expect(result.failed).toBe(1);
729 // Releases already running may finish, but none start afterwards.
730 expect(result.attempted).toBeLessThanOrEqual(
731 ManagedEndpointReaper.MANAGED_ENDPOINT_SWEEP_DELETE_CONCURRENCY,
732 );
733 expect(state.releases.length).toBe(result.attempted);
734 }).pipe(Effect.provide(state.layer));
735 });
736
737 it.effect("stops starting deletions when the sweep's time budget runs out", () => {
738 const entries = Array.from({ length: 40 }, (_, index) =>
739 tunnel({
740 id: `slow-${index}`,
741 suffix: index.toString(16).padStart(16, "0"),
742 status: "down",
743 timestamp: "2026-08-25T11:00:00.000Z",
744 }),
745 );
746 const state = harness({
747 tunnels: entries,
748 allocations: recoverableOwners(entries),
749 // Ten seconds each: the 90-second budget allows about 9 rounds.
750 releaseDelayMs: 10_000,
751 });
752
753 return Effect.gen(function* () {
754 yield* TestClock.setTime(NOW_MILLIS);
755 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
756 const result = yield* reaper.sweep;
757 expect(result.truncated).toBe(true);
758 expect(result.attempted).toBeGreaterThan(0);
759 expect(result.attempted).toBeLessThan(entries.length);
760 expect(result.deleted).toBe(result.attempted);
761 }).pipe(Effect.provide(state.layer));
762 });
763
764 it.effect("counts listing time against the deletion budget", () => {
765 const entries = Array.from({ length: 40 }, (_, index) =>
766 tunnel({
767 id: `slow-${index}`,
768 suffix: index.toString(16).padStart(16, "0"),
769 status: "down",
770 timestamp: "2026-08-25T11:00:00.000Z",
771 }),
772 );
773 const sweepWith = (listDelayMs: number) =>
774 Effect.gen(function* () {
775 yield* TestClock.setTime(NOW_MILLIS);
776 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
777 return (yield* reaper.sweep).attempted;
778 }).pipe(
779 Effect.provide(
780 harness({
781 tunnels: entries,
782 allocations: recoverableOwners(entries),
783 releaseDelayMs: 10_000,
784 listDelayMs,
785 }).layer,
786 ),
787 );
788
789 return Effect.gen(function* () {
790 const fastListing = yield* sweepWith(0);
791 // Two list requests of 20 seconds each use 40 of the 90-second budget.
792 const slowListing = yield* sweepWith(20_000);
793 expect(slowListing).toBeLessThan(fastListing);
794 });
795 });
796
797 it.effect("continues past a page of older hosts to find recoverable tunnels", () => {
798 const entries = Array.from({ length: 101 }, (_, index) =>
799 tunnel({
800 id: `tunnel-${index}`,
801 suffix: index.toString(16).padStart(16, "0"),
802 status: "down",
803 timestamp: "2026-08-25T11:00:00.000Z",
804 }),
805 );
806 const state = harness({
807 tunnels: entries,
808 allocations: entries.map((entry, index) =>
809 allocation({ tunnelId: entry.id!, recoveryEnabled: index === 100 }),
810 ),
811 });
812
813 return Effect.gen(function* () {
814 yield* TestClock.setTime(NOW_MILLIS);
815 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
816 expect(yield* reaper.sweep).toMatchObject({
817 scanned: 101,
818 deleted: 1,
819 skippedLegacy: 100,
820 failed: 0,
821 });
822 expect(state.deleted).toEqual(["tunnel-100"]);
823 expect(
824 state.listRequests
825 .filter((request) => request.status === "down")
826 .map((request) => request.page),
827 ).toEqual([1, 2]);
828 }).pipe(Effect.provide(state.layer));
829 });
830
831 it.effect("collects every page before deletions shift Cloudflare pagination", () => {
832 const entries = Array.from({ length: 120 }, (_, index) =>
833 tunnel({
834 id: `tunnel-${index}`,
835 suffix: index.toString(16).padStart(16, "0"),
836 status: "down",
837 timestamp: "2026-08-25T11:00:00.000Z",
838 }),
839 );
840 const state = harness({
841 tunnels: entries,
842 allocations: entries.map((entry, index) =>
843 allocation({ tunnelId: entry.id!, recoveryEnabled: index < 50 || index >= 100 }),
844 ),
845 });
846
847 return Effect.gen(function* () {
848 yield* TestClock.setTime(NOW_MILLIS);
849 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
850 expect(yield* reaper.sweep).toMatchObject({
851 scanned: 120,
852 deleted: 70,
853 skippedLegacy: 50,
854 failed: 0,
855 });
856 expect(state.deleted).toContain("tunnel-119");
857 }).pipe(Effect.provide(state.layer));
858 });
859
860 it.effect("limits each cleanup run to 100 tunnel deletions", () => {
861 const entries = Array.from({ length: 105 }, (_, index) =>
862 tunnel({
863 id: `tunnel-${index}`,
864 suffix: index.toString(16).padStart(16, "0"),
865 status: "down",
866 timestamp: "2026-08-25T11:00:00.000Z",
867 }),
868 );
869 const state = harness({ tunnels: entries, allocations: recoverableOwners(entries) });
870
871 return Effect.gen(function* () {
872 yield* TestClock.setTime(NOW_MILLIS);
873 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
874 expect((yield* reaper.sweep).deleted).toBe(100);
875 expect(state.deleted).toHaveLength(100);
876 }).pipe(Effect.provide(state.layer));
877 });
878
879 it.effect("records dry-run counters on the sweep span", () => {
880 const spans: Array<Tracer.NativeSpan> = [];
881 const tracer = Tracer.make({
882 span: (options) => {
883 const span = new Tracer.NativeSpan(options);
884 spans.push(span);
885 return span;
886 },
887 });
888 const expired = [
889 tunnel({
890 id: "recoverable",
891 suffix: "aaaaaaaaaaaaaaaa",
892 status: "down",
893 timestamp: "2026-08-25T11:00:00.000Z",
894 }),
895 tunnel({
896 id: "legacy",
897 suffix: "bbbbbbbbbbbbbbbb",
898 status: "down",
899 timestamp: "2026-08-25T11:00:00.000Z",
900 }),
901 ];
902 const state = harness({
903 cleanupMode: "dry-run",
904 tunnels: expired,
905 allocations: [
906 allocation({ tunnelId: "recoverable", recoveryEnabled: true }),
907 allocation({ tunnelId: "legacy", recoveryEnabled: false }),
908 ],
909 });
910
911 return Effect.gen(function* () {
912 yield* TestClock.setTime(NOW_MILLIS);
913 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
914 yield* reaper.sweep;
915 const sweepSpan = spans.find((span) => span.name === "relay.managed_endpoint_reaper.sweep");
916 expect(Object.fromEntries(sweepSpan?.attributes ?? [])).toMatchObject({
917 "relay.managed_endpoint_reaper.mode": "dry-run",
918 "relay.managed_endpoint_reaper.scanned": 2,
919 "relay.managed_endpoint_reaper.wouldDelete": 1,
920 "relay.managed_endpoint_reaper.skippedLegacy": 1,
921 "relay.managed_endpoint_reaper.deleted": 0,
922 });
923 expect(state.deleted).toEqual([]);
924 }).pipe(Effect.provide(state.layer), Effect.withTracer(tracer));
925 });
926
927 it.effect("does no Cloudflare work while cleanup is off", () => {
928 const state = harness({
929 cleanupMode: "off",
930 tunnels: [
931 tunnel({
932 id: "expired",
933 suffix: "aaaaaaaaaaaaaaaa",
934 status: "down",
935 timestamp: "2026-08-25T11:00:00.000Z",
936 }),
937 ],
938 allocations: [allocation({ tunnelId: "expired", recoveryEnabled: true })],
939 });
940
941 return Effect.gen(function* () {
942 yield* TestClock.setTime(NOW_MILLIS);
943 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
944 expect(yield* reaper.sweep).toEqual({
945 mode: "off",
946 legacyMode: "off",
947 listRequests: 0,
948 scanned: 0,
949 attempted: 0,
950 deleted: 0,
951 wouldDelete: 0,
952 wouldDeleteLegacy: 0,
953 deletedLegacy: 0,
954 skippedLegacy: 0,
955 legacyOver7Days: 0,
956 legacyOver30Days: 0,
957 legacyOver90Days: 0,
958 skippedOrphan: 0,
959 skippedReplaced: 0,
960 skippedUnrecorded: 0,
961 totalDown: null,
962 totalInactive: null,
963 failed: 0,
964 truncated: false,
965 });
966 expect(state.listRequests).toEqual([]);
967 expect(state.deleted).toEqual([]);
968 }).pipe(Effect.provide(state.layer));
969 });
970
971 it.effect("reports candidates without mutating them in dry-run mode", () => {
972 const state = harness({
973 cleanupMode: "dry-run",
974 tunnels: [
975 tunnel({
976 id: "expired",
977 suffix: "aaaaaaaaaaaaaaaa",
978 status: "down",
979 timestamp: "2026-08-25T11:00:00.000Z",
980 }),
981 ],
982 allocations: [allocation({ tunnelId: "expired", recoveryEnabled: true })],
983 });
984
985 return Effect.gen(function* () {
986 yield* TestClock.setTime(NOW_MILLIS);
987 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
988 expect(yield* reaper.sweep).toMatchObject({
989 mode: "dry-run",
990 scanned: 1,
991 attempted: 0,
992 deleted: 0,
993 wouldDelete: 1,
994 });
995 expect(state.deleted).toEqual([]);
996 expect(state.releases).toEqual([]);
997 }).pipe(Effect.provide(state.layer));
998 });
999
1000 it.effect("caps failed deletion attempts and Cloudflare list pages", () => {
1001 const entries = Array.from({ length: 250 }, (_, index) =>
1002 tunnel({
1003 id: `failed-${index}`,
1004 suffix: index.toString(16).padStart(16, "0"),
1005 status: "down",
1006 timestamp: "2026-08-25T11:00:00.000Z",
1007 }),
1008 );
1009 const state = harness({
1010 tunnels: entries,
1011 allocations: recoverableOwners(entries),
1012 failAllDeletes: true,
1013 });
1014
1015 return Effect.gen(function* () {
1016 yield* TestClock.setTime(NOW_MILLIS);
1017 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
1018 expect(yield* reaper.sweep).toMatchObject({
1019 attempted: 100,
1020 deleted: 0,
1021 failed: 100,
1022 truncated: true,
1023 });
1024 expect(state.listRequests.length).toBeLessThanOrEqual(
1025 ManagedEndpointReaper.MANAGED_ENDPOINT_SWEEP_LIST_REQUEST_LIMIT,
1026 );
1027 }).pipe(Effect.provide(state.layer));
1028 });
1029
1030 it.effect("rotates the deletion order so persistent failures do not starve later tunnels", () => {
1031 const entries = Array.from({ length: 200 }, (_, index) =>
1032 tunnel({
1033 id: `tunnel-${index}`,
1034 suffix: index.toString(16).padStart(16, "0"),
1035 status: "down",
1036 timestamp: "2026-08-25T11:00:00.000Z",
1037 }),
1038 );
1039 // The first 100 candidates always fail to delete.
1040 const state = harness({
1041 tunnels: entries,
1042 allocations: recoverableOwners(entries),
1043 failDeleteWhen: (id) => Number(id.split("-")[1]) < 100,
1044 });
1045
1046 return Effect.gen(function* () {
1047 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
1048 yield* TestClock.setTime(NOW_MILLIS);
1049 yield* reaper.sweep;
1050 yield* TestClock.setTime(NOW_MILLIS + 5 * 60 * 1_000);
1051 yield* reaper.sweep;
1052 const later = state.deleted.filter((id) => Number(id.split("-")[1]) >= 100);
1053 expect(later.length).toBeGreaterThan(0);
1054 }).pipe(Effect.provide(state.layer));
1055 });
1056
1057 it.effect("rotates bounded pages across a large legacy prefix", () => {
1058 const entries = Array.from({ length: 1_000 }, (_, index) =>
1059 tunnel({
1060 id: `legacy-${index}`,
1061 suffix: index.toString(16).padStart(16, "0"),
1062 status: "down",
1063 timestamp: "2026-08-25T11:00:00.000Z",
1064 }),
1065 );
1066 const state = harness({
1067 cleanupMode: "dry-run",
1068 tunnels: entries,
1069 allocations: entries.map((entry) =>
1070 allocation({ tunnelId: entry.id!, recoveryEnabled: false }),
1071 ),
1072 });
1073
1074 return Effect.gen(function* () {
1075 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
1076 yield* TestClock.setTime(NOW_MILLIS);
1077 yield* reaper.sweep;
1078 const firstPages = state.listRequests
1079 .filter((request) => request.status === "down")
1080 .map((request) => request.page);
1081 state.listRequests.length = 0;
1082 yield* TestClock.setTime(NOW_MILLIS + 5 * 60 * 1_000);
1083 yield* reaper.sweep;
1084 const secondPages = state.listRequests
1085 .filter((request) => request.status === "down")
1086 .map((request) => request.page);
1087 expect(firstPages).not.toEqual(secondPages);
1088 expect(firstPages).toHaveLength(5);
1089 expect(secondPages).toHaveLength(5);
1090 }).pipe(Effect.provide(state.layer));
1091 });
1092
1093 it.effect("rotates the attempt budget across both statuses over consecutive sweeps", () => {
1094 const entries = (["down", "inactive"] as const).flatMap((status) =>
1095 Array.from({ length: 100 }, (_, index) =>
1096 tunnel({
1097 id: `${status}-${index}`,
1098 suffix: `${status === "down" ? "a" : "b"}${index.toString(16).padStart(15, "0")}`,
1099 status,
1100 timestamp: "2026-08-25T10:00:00.000Z",
1101 }),
1102 ),
1103 );
1104 const state = harness({ tunnels: entries, allocations: recoverableOwners(entries) });
1105
1106 return Effect.gen(function* () {
1107 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
1108 yield* TestClock.setTime(NOW_MILLIS);
1109 yield* reaper.sweep;
1110 const firstStatus = state.deleted[0]!.split("-")[0];
1111 expect(state.deleted).toHaveLength(100);
1112 expect(state.deleted.every((id) => id.startsWith(`${firstStatus}-`))).toBe(true);
1113
1114 state.deleted.length = 0;
1115 yield* TestClock.setTime(NOW_MILLIS + 5 * 60 * 1_000);
1116 yield* reaper.sweep;
1117 expect(state.deleted).toHaveLength(100);
1118 expect(state.deleted.every((id) => !id.startsWith(`${firstStatus}-`))).toBe(true);
1119 }).pipe(Effect.provide(state.layer));
1120 });
1121});