diff --git a/.github/workflows/deploy-relay.yml b/.github/workflows/deploy-relay.yml index c902cfd3a3d2228629fcefa858623fbc9e346c61..400f7152c617e606c6ebebc1d536bc2ec3cbfec1 100644 --- a/.github/workflows/deploy-relay.yml +++ b/.github/workflows/deploy-relay.yml @@ -40,6 +40,7 @@ jobs: RELAY_API_ZONE_NAME: ${{ vars.RELAY_API_ZONE_NAME }} RELAY_TUNNEL_ZONE_NAME: ${{ vars.RELAY_TUNNEL_ZONE_NAME }} RELAY_TUNNEL_CLEANUP_MODE: ${{ vars.RELAY_TUNNEL_CLEANUP_MODE }} + RELAY_LEGACY_TUNNEL_CLEANUP_MODE: ${{ vars.RELAY_LEGACY_TUNNEL_CLEANUP_MODE }} CLERK_PUBLISHABLE_KEY: ${{ vars.CLERK_PUBLISHABLE_KEY }} CLERK_JWT_AUDIENCE: ${{ vars.CLERK_JWT_AUDIENCE }} APNS_ENVIRONMENT: ${{ vars.APNS_ENVIRONMENT }} diff --git a/docs/operations/release.md b/docs/operations/release.md index 0ce945f84030b0f4d4329eaa33e1715f0297b030..c1875251996071dffb1ba26c13cdc98f9e6346b5 100644 --- a/docs/operations/release.md +++ b/docs/operations/release.md @@ -134,6 +134,8 @@ Optional `production` environment variables: - `RELAY_DOMAIN` when overriding the derived `relay.` domain - `RELAY_TUNNEL_CLEANUP_MODE` with `off`, `dry-run`, or `enabled`. Missing and blank values use `off`. +- `RELAY_LEGACY_TUNNEL_CLEANUP_MODE` with the same values, for tunnels whose host never registered + recovery. Missing and blank values use `off`. Required `production` environment secrets: @@ -168,7 +170,7 @@ because those builds register recovery and replace a deleted tunnel after wake. 1. Deploy the relay and migration with cleanup `off`. 2. Release the server build and confirm current hosts register recovery. Older hosts stay marked - legacy and are never candidates. + legacy and are only candidates under the legacy switch below. 3. Set `dry-run`, run a relay deploy, and read the sweep counters (`scanned`, `wouldDelete`, `skippedLegacy`, `skippedOrphan`, `failed`, `truncated`) across several sweeps. Each sweep records them, and the active `mode`, as `relay.managed_endpoint_reaper.*` attributes on its @@ -185,6 +187,32 @@ deploy without force. Confirm the new `mode` on the next sweep span. To roll back, set cleanup to `off` and run a relay deploy before downgrading any host. Keep the recovery endpoints deployed while current server builds are in use. The nullable columns can stay. +### Legacy tunnel cleanup + +A legacy tunnel belongs to a host that never registered recovery, usually one that went offline +before the recovery build shipped. `RELAY_LEGACY_TUNNEL_CLEANUP_MODE` deletes these once Cloudflare +reports them down, or never connected, for more than 7 days. It is independent of +`RELAY_TUNNEL_CLEANUP_MODE`, and every other check still applies. + +A deleted legacy tunnel keeps its allocation, so its hostname is kept. When the host comes back: + +- On a build with recovery, the connector is rejected and the host requests a replacement tunnel at + the same hostname. +- On an older build with a CLI link, startup provisions a new tunnel. +- On an older build linked from web or mobile, the host stays offline until T3 Code on that computer + is updated. + +1. Run `vp run --filter t3code-relay tunnels:census` with a read-only Cloudflare token. It counts + tunnels in every relay stage. The reaper only sees its own stage's tunnels, so clean up the rest + by hand. +2. Set the legacy mode to `dry-run`, deploy, and read `wouldDeleteLegacy`, `legacyOver30Days`, + `totalDown`, and `totalInactive` on the sweep spans for a day. +3. Run the legacy steps of the disposable-host canary below. +4. Before enabling, confirm the web and mobile builds that show the "update T3 Code on that computer" + message are live. Without them, a user whose older host lost its tunnel only sees it as offline. +5. Set the legacy mode to `enabled`. One sweep deletes at most 100 tunnels, so a backlog of + 20,000 takes about 17 hours. Watch `deletedLegacy`, `failed`, and `truncated`. + ### Disposable-host canary This test has not been run against a real Cloudflare account. Run it against a disposable relay @@ -210,6 +238,19 @@ stage, test Cloudflare account, disposable host, and disposable T3 home. Keep pr 8. Resume the legacy child with `kill -CONT ` and confirm its tunnel reconnects. 9. Repeat with a physical sleep and wake cycle on a disposable laptop before broad rollout. +Legacy cleanup, on the same disposable stage: + +10. Set `RELAY_LEGACY_TUNNEL_GRACE_MINUTES=10` and the legacy mode to `dry-run`, then deploy. The + override shortens the 7-day grace period and is ignored on `prod`. Pause the legacy child again + and wait until Cloudflare reports it down for over ten minutes. +11. Confirm the sweep counts it in `wouldDeleteLegacy`, then set the legacy mode to `enabled` and + deploy. Confirm the legacy tunnel is deleted and its allocation row remains. +12. With the legacy host still on its old build, resume the child. A CLI-linked host provisions a + new tunnel on its next restart; a web- or mobile-linked host stays offline. +13. Update that host to the current build and start it. Confirm it requests recovery and is + reachable at the same hostname. +14. Remove `RELAY_LEGACY_TUNNEL_GRACE_MINUTES` from the disposable stage. + ## Marketing site deployment On nightly releases, the release workflow builds the same commit as a staged diff --git a/infra/relay/.env.example b/infra/relay/.env.example index 9270ec5ff3b7fe2e0f4803928db08268fef4db3c..3e7b6b1d109dda76b7b61559db9d4ee9a85b7c4c 100644 --- a/infra/relay/.env.example +++ b/infra/relay/.env.example @@ -8,6 +8,12 @@ RELAY_TUNNEL_ZONE_NAME=tunnels.example.com # Optional: inactive tunnel cleanup. Start with dry-run, verify the cleanup # logs, then set enabled. Unset and off both disable cleanup. # RELAY_TUNNEL_CLEANUP_MODE=off +# Optional: cleanup for tunnels whose host never registered recovery, once they +# have been down for 30 days. Same values, independent of the mode above. +# RELAY_LEGACY_TUNNEL_CLEANUP_MODE=off +# Canary stages only: shorten the 30-day legacy grace period, in minutes. +# Ignored on prod. +# RELAY_LEGACY_TUNNEL_GRACE_MINUTES=10 # Optional: Relay domain override # Set this only when the derived relay hostname should not be used. diff --git a/infra/relay/src/Config.test.ts b/infra/relay/src/Config.test.ts index e37d605d06448f948281a230c55edb089469a949..e3dbf75791d2c715f167f494fb878d00ff92d26e 100644 --- a/infra/relay/src/Config.test.ts +++ b/infra/relay/src/Config.test.ts @@ -1,8 +1,13 @@ import { expect, it } from "@effect/vitest"; import * as ConfigProvider from "effect/ConfigProvider"; import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; -import { managedEndpointCleanupModeConfig } from "./Config.ts"; +import { + legacyManagedEndpointCleanupModeConfig, + legacyTunnelGraceMinutesConfig, + managedEndpointCleanupModeConfig, +} from "./Config.ts"; it.effect.each([ { name: "missing", env: {}, expected: "off" }, @@ -33,3 +38,41 @@ it.effect("rejects an invalid cleanup mode", () => expect(error.message).toContain('Expected "off" | "dry-run" | "enabled"'); }), ); + +it.effect("reads the legacy cleanup mode independently of the main one", () => + Effect.gen(function* () { + const provider = ConfigProvider.fromEnv({ + env: { RELAY_TUNNEL_CLEANUP_MODE: "enabled", RELAY_LEGACY_TUNNEL_CLEANUP_MODE: "dry-run" }, + }); + expect(yield* managedEndpointCleanupModeConfig.parse(provider)).toBe("enabled"); + expect(yield* legacyManagedEndpointCleanupModeConfig.parse(provider)).toBe("dry-run"); + expect( + yield* legacyManagedEndpointCleanupModeConfig.parse(ConfigProvider.fromEnv({ env: {} })), + ).toBe("off"); + }), +); + +it.effect.each([ + { name: "missing", env: {}, expected: Option.none() }, + { + name: "positive", + env: { RELAY_LEGACY_TUNNEL_GRACE_MINUTES: "10" }, + expected: Option.some(10), + }, +] as const)("loads a $name legacy grace override", ({ env, expected }) => + Effect.gen(function* () { + const minutes = yield* legacyTunnelGraceMinutesConfig.parse(ConfigProvider.fromEnv({ env })); + expect(minutes).toEqual(expected); + }), +); + +it.effect.each(["0", "-10"])("rejects a grace override of %s minutes", (value) => + Effect.gen(function* () { + const error = yield* Effect.flip( + legacyTunnelGraceMinutesConfig.parse( + ConfigProvider.fromEnv({ env: { RELAY_LEGACY_TUNNEL_GRACE_MINUTES: value } }), + ), + ); + expect(error._tag).toBe("ConfigError"); + }), +); diff --git a/infra/relay/src/Config.ts b/infra/relay/src/Config.ts index e2822b53caef4593693e89f4233055695e1544f0..de4f8273395c4c13292a4dd203aaad136b9b748b 100644 --- a/infra/relay/src/Config.ts +++ b/infra/relay/src/Config.ts @@ -12,13 +12,38 @@ export const ManagedEndpointCleanupMode = Schema.Literals(["off", "dry-run", "en export type ManagedEndpointCleanupMode = typeof ManagedEndpointCleanupMode.Type; const decodeManagedEndpointCleanupMode = Schema.decodeUnknownEffect(ManagedEndpointCleanupMode); -export const managedEndpointCleanupModeConfig = Config.String("RELAY_TUNNEL_CLEANUP_MODE").pipe( - Config.withDefault("off"), - Config.map((value) => value.trim() || "off"), - Config.mapEffect((value) => - decodeManagedEndpointCleanupMode(value).pipe( - Effect.mapError((error) => new Config.ConfigError(error)), +const RELAY_TUNNEL_CLEANUP_MODE = "RELAY_TUNNEL_CLEANUP_MODE"; +/** Separate switch for tunnels whose host never registered recovery. */ +const RELAY_LEGACY_TUNNEL_CLEANUP_MODE = "RELAY_LEGACY_TUNNEL_CLEANUP_MODE"; + +const cleanupModeConfig = (name: string) => + Config.String(name).pipe( + Config.withDefault("off"), + Config.map((value) => value.trim() || "off"), + Config.mapEffect((value) => + decodeManagedEndpointCleanupMode(value).pipe( + Effect.mapError((error) => new Config.ConfigError(error)), + ), ), + ); + +export const managedEndpointCleanupModeConfig = cleanupModeConfig(RELAY_TUNNEL_CLEANUP_MODE); +export const legacyManagedEndpointCleanupModeConfig = cleanupModeConfig( + RELAY_LEGACY_TUNNEL_CLEANUP_MODE, +); + +/** + * Overrides the 7-day legacy grace period, in minutes, so the disposable + * canary stage can exercise legacy cleanup. Ignored on the prod stage. + */ +const RELAY_LEGACY_TUNNEL_GRACE_MINUTES = "RELAY_LEGACY_TUNNEL_GRACE_MINUTES"; + +// A zero or negative override would be ignored at runtime, silently leaving +// the canary on the 7-day grace period, so reject it when the deploy reads it. +export const legacyTunnelGraceMinutesConfig = Config.option( + Config.schema( + Schema.NumberFromString.pipe(Schema.check(Schema.isInt(), Schema.isGreaterThan(0))), + RELAY_LEGACY_TUNNEL_GRACE_MINUTES, ), ); @@ -45,6 +70,9 @@ export class RelayConfiguration extends Context.Service< readonly managedEndpointBaseDomain: string | undefined; readonly managedEndpointNamespace: string | undefined; readonly managedEndpointCleanupMode?: ManagedEndpointCleanupMode; + readonly legacyManagedEndpointCleanupMode?: ManagedEndpointCleanupMode; + /** Canary-only override of the legacy grace period; ignored on prod. */ + readonly legacyTunnelGraceMinutes?: number; } >()("t3code-relay/Config/RelayConfiguration") {} diff --git a/infra/relay/src/environments/ManagedEndpointReaper.test.ts b/infra/relay/src/environments/ManagedEndpointReaper.test.ts index 06d38fa9395e433c010435cb0997f681d567a8f5..60481191da9e70eb14b840337fe1039c5141338b 100644 --- a/infra/relay/src/environments/ManagedEndpointReaper.test.ts +++ b/infra/relay/src/environments/ManagedEndpointReaper.test.ts @@ -73,6 +73,8 @@ function harness(input?: { readonly refreshedTunnels?: ReadonlyMap; readonly skipTunnelId?: string; readonly cleanupMode?: RelayConfiguration.ManagedEndpointCleanupMode; + readonly legacyCleanupMode?: RelayConfiguration.ManagedEndpointCleanupMode; + readonly legacyTunnelGraceMinutes?: number; }) { const listRequests: ManagedEndpointProvider.ManagedEndpointTunnelListRequest[] = []; const deleted: string[] = []; @@ -228,6 +230,10 @@ function harness(input?: { managedEndpointBaseDomain: "example.test", managedEndpointNamespace: input?.namespace ?? "prod", managedEndpointCleanupMode: input?.cleanupMode ?? "enabled", + legacyManagedEndpointCleanupMode: input?.legacyCleanupMode ?? "off", + ...(input?.legacyTunnelGraceMinutes === undefined + ? {} + : { legacyTunnelGraceMinutes: input.legacyTunnelGraceMinutes }), }); return { @@ -490,6 +496,141 @@ describe("ManagedEndpointReaper", () => { }).pipe(Effect.provide(state.layer)); }); + describe("legacy cleanup", () => { + const legacyTunnels = [ + // Down for 5 days: inside the legacy grace period. + tunnel({ + id: "legacy-recent", + suffix: "1111111111111111", + status: "down", + timestamp: "2026-08-20T12:00:00.000Z", + }), + // Down for 40 days: past it. + tunnel({ + id: "legacy-old", + suffix: "2222222222222222", + status: "down", + timestamp: "2026-07-16T12:00:00.000Z", + }), + // Never connected, created 40 days ago. + tunnel({ + id: "legacy-never", + suffix: "3333333333333333", + status: "inactive", + timestamp: "2026-07-16T12:00:00.000Z", + }), + ]; + const legacyOwners = legacyTunnels.map((entry) => + allocation({ tunnelId: entry.id!, recoveryEnabled: false }), + ); + + it.effect("leaves legacy tunnels alone while legacy cleanup is off", () => { + const state = harness({ tunnels: legacyTunnels, allocations: legacyOwners }); + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect(yield* reaper.sweep).toMatchObject({ + skippedLegacy: 3, + wouldDeleteLegacy: 0, + deletedLegacy: 0, + }); + expect(state.deleted).toEqual([]); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("counts legacy tunnels past the grace period in dry-run", () => { + const state = harness({ + tunnels: legacyTunnels, + allocations: legacyOwners, + legacyCleanupMode: "dry-run", + }); + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect(yield* reaper.sweep).toMatchObject({ + legacyMode: "dry-run", + wouldDeleteLegacy: 2, + deletedLegacy: 0, + attempted: 0, + }); + expect(state.deleted).toEqual([]); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("deletes only legacy tunnels past the grace period", () => { + const state = harness({ + tunnels: legacyTunnels, + allocations: legacyOwners, + legacyCleanupMode: "enabled", + }); + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect(yield* reaper.sweep).toMatchObject({ + wouldDeleteLegacy: 2, + deletedLegacy: 2, + deleted: 2, + }); + expect([...state.deleted].sort()).toEqual(["legacy-never", "legacy-old"]); + // The release re-checks each tunnel against the 7-day cutoff, not + // the 5-minute cutoff used for recoverable tunnels. + for (const release of state.releases) { + expect(release.expectedInactiveBefore).toBe("2026-08-18T12:00:00.000Z"); + } + }).pipe(Effect.provide(state.layer)); + }); + + it.effect.each([ + { stage: "canary", expected: 3 }, + { stage: "prod", expected: 2 }, + ] as const)( + "applies the grace-period override only off prod ($stage)", + ({ stage, expected }) => { + const stageLegacyTunnels = legacyTunnels.map((entry) => ({ + ...entry, + name: entry.name!.replace(PREFIX, `t3coderelay-managedendpoint-${stage}-`), + })); + const state = harness({ + namespace: stage, + tunnels: stageLegacyTunnels, + allocations: legacyOwners, + legacyCleanupMode: "dry-run", + // Ten minutes: the 10-day-old tunnel qualifies only if this applies. + legacyTunnelGraceMinutes: 10, + }); + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect((yield* reaper.sweep).wouldDeleteLegacy).toBe(expected); + }).pipe(Effect.provide(state.layer)); + }, + ); + + it.effect("runs legacy cleanup while recoverable cleanup is off", () => { + const recoverable = tunnel({ + id: "recoverable", + suffix: "4444444444444444", + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }); + const state = harness({ + tunnels: [...legacyTunnels, recoverable], + allocations: [ + ...legacyOwners, + allocation({ tunnelId: "recoverable", recoveryEnabled: true }), + ], + cleanupMode: "off", + legacyCleanupMode: "enabled", + }); + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect(yield* reaper.sweep).toMatchObject({ wouldDelete: 0, deletedLegacy: 2 }); + expect(state.deleted).not.toContain("recoverable"); + }).pipe(Effect.provide(state.layer)); + }); + }); + it.effect("does not count a tunnel that was replaced before its release", () => { const state = harness({ tunnels: [ @@ -697,11 +838,14 @@ describe("ManagedEndpointReaper", () => { const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; expect(yield* reaper.sweep).toEqual({ mode: "off", + legacyMode: "off", listRequests: 0, scanned: 0, attempted: 0, deleted: 0, wouldDelete: 0, + wouldDeleteLegacy: 0, + deletedLegacy: 0, skippedLegacy: 0, legacyOver7Days: 0, legacyOver30Days: 0, diff --git a/infra/relay/src/environments/ManagedEndpointReaper.ts b/infra/relay/src/environments/ManagedEndpointReaper.ts index 7590d253401aa18b7f75f7c9098415858521c63f..3d3a2f2056c0af2a96e0980033ffbedfe89ab0b5 100644 --- a/infra/relay/src/environments/ManagedEndpointReaper.ts +++ b/infra/relay/src/environments/ManagedEndpointReaper.ts @@ -5,7 +5,10 @@ import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as RelayConfiguration from "../Config.ts"; -import { managedEndpointTunnelNamePrefix } from "../deploymentConfig.ts"; +import { + MANAGED_ENDPOINT_ZONE_OWNER_STAGE, + managedEndpointTunnelNamePrefix, +} from "../deploymentConfig.ts"; import * as ManagedEndpointAllocations from "./ManagedEndpointAllocations.ts"; import * as ManagedEndpointProvider from "./ManagedEndpointProvider.ts"; @@ -19,14 +22,23 @@ export const MANAGED_ENDPOINT_SWEEP_LIST_REQUEST_LIMIT = 10; // Age buckets for legacy candidates, in days since the tunnel went down (or // was created, for one that never connected). const MANAGED_ENDPOINT_LEGACY_AGE_BUCKET_DAYS = [7, 30, 90] as const; +// A host that never registered recovery cannot replace a deleted tunnel on +// its own build. Only delete its tunnel after it has been gone this long. A +// returning host that has updated recovers the tunnel at the same hostname; +// one that has not sees the client's "update T3 Code" message instead. +const MANAGED_ENDPOINT_LEGACY_GRACE_PERIOD_DAYS = 7; export interface ManagedEndpointSweepResult { readonly mode: RelayConfiguration.ManagedEndpointCleanupMode; + readonly legacyMode: RelayConfiguration.ManagedEndpointCleanupMode; readonly listRequests: number; readonly scanned: number; readonly attempted: number; readonly deleted: number; readonly wouldDelete: number; + /** Legacy tunnels past the legacy grace period, and how many were deleted. */ + readonly wouldDeleteLegacy: number; + readonly deletedLegacy: number; readonly skippedLegacy: number; /** Legacy candidates, by days since they went down: over 7, 30, and 90. */ readonly legacyOver7Days: number; @@ -131,13 +143,17 @@ function rotatedPages(input: { const emptyResult = ( mode: RelayConfiguration.ManagedEndpointCleanupMode, + legacyMode: RelayConfiguration.ManagedEndpointCleanupMode, ): ManagedEndpointSweepResult => ({ mode, + legacyMode, listRequests: 0, scanned: 0, attempted: 0, deleted: 0, wouldDelete: 0, + wouldDeleteLegacy: 0, + deletedLegacy: 0, skippedLegacy: 0, legacyOver7Days: 0, legacyOver30Days: 0, @@ -159,8 +175,19 @@ export const make = Effect.gen(function* () { const sweep = Effect.gen(function* () { const mode = config.managedEndpointCleanupMode ?? "off"; + const legacyMode = config.legacyManagedEndpointCleanupMode ?? "off"; const namespace = config.managedEndpointNamespace; - if (mode === "off" || !namespace) return emptyResult(mode); + if ((mode === "off" && legacyMode === "off") || !namespace) { + return emptyResult(mode, legacyMode); + } + // The override exists for the disposable canary stage; prod always + // waits the full grace period. + const legacyGraceMinutes = + namespace !== MANAGED_ENDPOINT_ZONE_OWNER_STAGE && + config.legacyTunnelGraceMinutes !== undefined + ? config.legacyTunnelGraceMinutes + : MANAGED_ENDPOINT_LEGACY_GRACE_PERIOD_DAYS * 24 * 60; + const legacyCutoff = DateTime.subtract(yield* DateTime.now, { minutes: legacyGraceMinutes }); const now = yield* DateTime.now; const cutoffFor = (status: "down" | "inactive") => @@ -248,6 +275,8 @@ export const make = Effect.gen(function* () { let attempted = 0; let deleted = 0; let wouldDelete = 0; + let wouldDeleteLegacy = 0; + let deletedLegacy = 0; let skippedLegacy = 0; const legacyOverDays = new Map( MANAGED_ENDPOINT_LEGACY_AGE_BUCKET_DAYS.map((days) => [days, 0]), @@ -258,7 +287,6 @@ export const make = Effect.gen(function* () { let failed = 0; for (const { tunnel, status, cutoff } of uniqueExpired) { - const cutoffIso = DateTime.formatIso(cutoff); const allocation = recordedByTunnelName.get(tunnel.name); if ( allocation !== undefined && @@ -269,16 +297,6 @@ export const make = Effect.gen(function* () { continue; } const owner = allocation?.tunnelId === tunnel.id ? allocation : undefined; - if (owner !== undefined && !owner.recoveryEnabled) { - skippedLegacy += 1; - const inactiveDays = inactiveDaysAt(tunnel, status, now); - for (const days of MANAGED_ENDPOINT_LEGACY_AGE_BUCKET_DAYS) { - if (inactiveDays !== null && inactiveDays > days) { - legacyOverDays.set(days, (legacyOverDays.get(days) ?? 0) + 1); - } - } - continue; - } if (allocation !== undefined && owner === undefined) { skippedUnrecorded += 1; continue; @@ -290,19 +308,41 @@ export const make = Effect.gen(function* () { skippedOrphan += 1; continue; } - wouldDelete += 1; - if (mode === "dry-run") continue; + const legacy = !owner.recoveryEnabled; + if (legacy) { + skippedLegacy += 1; + const inactiveDays = inactiveDaysAt(tunnel, status, now); + for (const days of MANAGED_ENDPOINT_LEGACY_AGE_BUCKET_DAYS) { + if (inactiveDays !== null && inactiveDays > days) { + legacyOverDays.set(days, (legacyOverDays.get(days) ?? 0) + 1); + } + } + if ( + legacyMode === "off" || + !isExpiredManagedTunnel({ tunnel, status, prefix, cutoff: legacyCutoff }) + ) { + continue; + } + wouldDeleteLegacy += 1; + if (legacyMode === "dry-run") continue; + } else { + if (mode === "off") continue; + wouldDelete += 1; + if (mode === "dry-run") continue; + } if (attempted >= MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT) { truncated = true; break; } attempted += 1; + // The release re-reads the tunnel and deletes only if it is still in + // this status and inactive since before this tunnel's cutoff. const result = yield* provider .release({ userId: owner.userId, environmentId: owner.environmentId, expectedTunnelId: tunnel.id, - expectedInactiveBefore: cutoffIso, + expectedInactiveBefore: DateTime.formatIso(legacy ? legacyCutoff : cutoff), expectedStatus: status, }) .pipe(Effect.result); @@ -311,6 +351,7 @@ export const make = Effect.gen(function* () { yield* Effect.logWarning("Failed to delete an inactive managed tunnel", { tunnelId: tunnel.id, tunnelName: tunnel.name, + legacy, cause: result.failure, }); if (isRateLimited(result.failure)) { @@ -319,21 +360,26 @@ export const make = Effect.gen(function* () { } } else if (result.success) { deleted += 1; + if (legacy) deletedLegacy += 1; yield* Effect.logInfo("Deleted an inactive managed tunnel", { tunnelId: tunnel.id, tunnelName: tunnel.name, status, + legacy, }); } } return { mode, + legacyMode, listRequests, scanned: uniqueExpired.length, attempted, deleted, wouldDelete, + wouldDeleteLegacy, + deletedLegacy, skippedLegacy, legacyOver7Days: legacyOverDays.get(7) ?? 0, legacyOver30Days: legacyOverDays.get(30) ?? 0, diff --git a/infra/relay/src/worker.ts b/infra/relay/src/worker.ts index 8a5872fd190e1e693394cac290c3271297e5f116..71d0b4a0d787c6c75eb0b19dac675efe1187952b 100644 --- a/infra/relay/src/worker.ts +++ b/infra/relay/src/worker.ts @@ -184,6 +184,11 @@ export const layer = Api.make( const managedEndpointDnsBinding = yield* Cloudflare.DNS.ReadWriteDns(managedEndpointZone); const managedEndpointZoneName = yield* managedEndpointZone.name; const managedEndpointCleanupMode = yield* RelayConfiguration.managedEndpointCleanupModeConfig; + const legacyManagedEndpointCleanupMode = + yield* RelayConfiguration.legacyManagedEndpointCleanupModeConfig; + const legacyTunnelGraceMinutes = Option.getOrUndefined( + yield* RelayConfiguration.legacyTunnelGraceMinutesConfig, + ); // Keys are endpoint keys or hashes over them, which already differ per // stage, so stages sharing an account cannot collide in these namespaces. const hookRateLimit = yield* Cloudflare.RateLimit("HOOK_RATE_LIMIT", { @@ -221,6 +226,8 @@ export const layer = Api.make( managedEndpointBaseDomain: yield* managedEndpointZoneName, managedEndpointNamespace: stage, managedEndpointCleanupMode, + legacyManagedEndpointCleanupMode, + ...(legacyTunnelGraceMinutes === undefined ? {} : { legacyTunnelGraceMinutes }), }); });