1import * as ec2 from "@distilled.cloud/aws/ec2";
2import * as Effect from "effect/Effect";
3import * as Result from "effect/Result";
4
5/**
6 * Shared teardown accelerator for network interfaces that AWS releases
7 * *asynchronously* after their owning resource is deleted.
8 *
9 * The canonical case is a VPC-attached Lambda Function: `DeleteFunction`
10 * returns immediately, but the function's Hyperplane ENIs stay `in-use` in
11 * their subnets/security groups for several minutes, and Lambda's own reaper
12 * may not delete the detached ENI for up to ~20 minutes. Until the ENI is
13 * gone, `DeleteSubnet`/`DeleteSecurityGroup` fail with `DependencyViolation`
14 * — historically forcing users to run `destroy` twice, minutes apart.
15 *
16 * `retryWhileLingeringEnis` wraps a subnet/security-group delete call (or an
17 * internet-gateway detach, which `DependencyViolation`s while any ENI in the
18 * VPC still holds a mapped public address): on every `DependencyViolation`
19 * it observes the ENIs still occupying the resource, explicitly deletes any
20 * *detached* (status `available`) Lambda ENI instead of waiting for Lambda's
21 * reaper, and — while the blockers are exclusively Lambda ENIs that are
22 * provably on their way out — keeps waiting on an extended budget that
23 * covers AWS's worst-case release window. Any other lingering dependency
24 * keeps today's shorter budget.
25 */
26
27/**
28 * ENIs that their owning service tears down asynchronously and that are safe
29 * for us to delete once detached. Only Lambda Hyperplane ENIs qualify today:
30 * every other managed type (NAT gateways, VPC endpoints, ELB, EFS mount
31 * targets, ECS task ENIs) is deleted by the API call that deletes its owner
32 * — and their owners are proper alchemy resources ordered ahead of the
33 * subnet/SG by the deletion graph.
34 *
35 * Matching note (observed live): while attached, a Hyperplane ENI reports
36 * `InterfaceType: "lambda"` — but the moment Lambda detaches it, the type
37 * flips to plain `"interface"` and only the description (`AWS Lambda VPC
38 * ENI-{functionName}`) still identifies it. Match both, like Terraform does.
39 */
40const isReapableEni = (eni: ec2.NetworkInterface): boolean =>
41 eni.InterfaceType === "lambda" ||
42 (eni.Description?.startsWith("AWS Lambda VPC ENI") ?? false);
43
44export interface LingeringEniScope {
45 /** ENI filter naming the resource being deleted or detached. */
46 readonly name: "subnet-id" | "group-id" | "vpc-id";
47 readonly value: string;
48}
49
50/**
51 * Observe ENIs still occupying the resource and delete any detached
52 * reapable ones. Best-effort by design: the reaper only accelerates the
53 * outer DependencyViolation retry, so an account that cannot describe or
54 * delete ENIs degrades to plain waiting instead of failing the destroy.
55 */
56const reapLingeringEnis = Effect.fn(function* (
57 scope: LingeringEniScope,
58 session: { note: (note: string) => Effect.Effect<void> },
59) {
60 const described = yield* ec2
61 .describeNetworkInterfaces({
62 Filters: [{ Name: scope.name, Values: [scope.value] }],
63 })
64 .pipe(Effect.catch(() => Effect.succeed({ NetworkInterfaces: [] })));
65
66 const lingering = (described.NetworkInterfaces ?? []).filter(isReapableEni);
67
68 let reaped = 0;
69 let pending = 0;
70 for (const eni of lingering) {
71 if (eni.Status !== "available" || eni.NetworkInterfaceId === undefined) {
72 pending += 1;
73 continue;
74 }
75 const outcome = yield* ec2
76 .deleteNetworkInterface({
77 NetworkInterfaceId: eni.NetworkInterfaceId,
78 })
79 .pipe(
80 Effect.as("deleted" as const),
81 // Already gone counts as progress; anything else (raced back to
82 // in-use, throttle, missing IAM permission) must not fail the
83 // destroy — reaping is purely an accelerator, so degrade to the
84 // outer retry's plain waiting.
85 Effect.catchTag("InvalidNetworkInterfaceID.NotFound", () =>
86 Effect.succeed("deleted" as const),
87 ),
88 Effect.catch(() => Effect.succeed("pending" as const)),
89 );
90 if (outcome === "deleted") {
91 yield* session.note(
92 `Deleted detached Lambda ENI ${eni.NetworkInterfaceId}`,
93 );
94 reaped += 1;
95 } else {
96 pending += 1;
97 }
98 }
99
100 return {
101 /** Reapable ENIs still occupying the resource after this pass. */
102 pendingReapable: pending,
103 /** Detached reapable ENIs actually deleted this pass. */
104 reaped,
105 };
106});
107
108/**
109 * Retry `deleteCall` while it fails with a dependency violation, reaping
110 * lingering ENIs between attempts.
111 *
112 * - Base budget (~12 min) matches the historical subnet schedule: fast
113 * exponential start, capped at 30-second steps.
114 * - While the observed blockers include reapable ENIs (attached or just
115 * reaped), the budget extends to ~25 min — Lambda's documented worst-case
116 * ENI release window — because those blockers are guaranteed to clear.
117 */
118export const retryWhileLingeringEnis = <A, E extends { _tag: string }, R>(
119 deleteCall: Effect.Effect<A, E, R>,
120 options: {
121 scope: LingeringEniScope;
122 isDependencyViolation: (error: E) => boolean;
123 session: { note: (note: string) => Effect.Effect<void> };
124 },
125) =>
126 Effect.gen(function* () {
127 const BASE_BUDGET_MILLIS = 12 * 60 * 1000;
128 const EXTENDED_BUDGET_MILLIS = 25 * 60 * 1000;
129
130 let elapsedMillis = 0;
131 let sawReapable = false;
132
133 for (let attempt = 1; ; attempt++) {
134 const result = yield* Effect.result(deleteCall);
135 if (Result.isSuccess(result)) {
136 return result.success;
137 }
138 const error = result.failure;
139 if (!options.isDependencyViolation(error)) {
140 return yield* Effect.fail(error);
141 }
142
143 const { pendingReapable, reaped } = yield* reapLingeringEnis(
144 options.scope,
145 options.session,
146 );
147 if (reaped > 0) {
148 // An ENI just left — the next attempt has a real chance; skip the
149 // backoff sleep and try immediately.
150 continue;
151 }
152 sawReapable ||= pendingReapable > 0;
153
154 const budget = sawReapable ? EXTENDED_BUDGET_MILLIS : BASE_BUDGET_MILLIS;
155 if (elapsedMillis >= budget) {
156 return yield* Effect.fail(error);
157 }
158
159 yield* options.session.note(
160 pendingReapable > 0
161 ? `Waiting for AWS to release ${pendingReapable} lingering ENI(s)... (attempt ${attempt})`
162 : `Waiting for dependencies to clear... (attempt ${attempt})`,
163 );
164
165 // Fast exponential start capped at 30-second steps (the historical
166 // subnet schedule).
167 const delayMillis = Math.min(1000 * 1.5 ** (attempt - 1), 30_000);
168 yield* Effect.sleep(delayMillis);
169 elapsedMillis += delayMillis;
170 }
171 });