1import * as logs from "@distilled.cloud/aws/cloudwatch-logs";
2import * as ec2 from "@distilled.cloud/aws/ec2";
3import * as iam from "@distilled.cloud/aws/iam";
4import * as lambda from "@distilled.cloud/aws/lambda";
5import * as rds from "@distilled.cloud/aws/rds";
6import * as secretsmanager from "@distilled.cloud/aws/secrets-manager";
7import * as Data from "effect/Data";
8import * as Effect from "effect/Effect";
9import * as Schedule from "effect/Schedule";
10import * as Stream from "effect/Stream";
11
12/**
13 * Out-of-band orphan reaper for the RDSData bindings fixture.
14 *
15 * The suite's scratch stack keeps its state IN MEMORY, so a run that dies
16 * between deploy and a completed destroy (Aurora SV2 provisioning/teardown
17 * runs 10+ minutes — an easy target for a killed process) orphans the whole
18 * fixture with no state left to destroy from. This reaper finds fixture
19 * resources by their deterministic physical-name prefix (`RDSDataBindings-`,
20 * from `createPhysicalName`'s `{stack}-{id}-{stage}-` scheme) and the
21 * `alchemy::stack` ownership tag / fixed `10.61.0.0/16` CIDR for EC2, and
22 * deletes them with typed, idempotent calls.
23 *
24 * It runs as a pre-clean in `beforeAll` (before deploy) and as an
25 * `Effect.ensuring` finalizer in `afterAll` (even when destroy fails), so a
26 * SLOW run converges to zero leftovers regardless of how the previous run
27 * ended. Every delete catches its not-found tag; bounded retries cover
28 * `DependencyViolation` (Lambda ENIs release asynchronously after function
29 * deletion) and in-flight RDS state transitions.
30 */
31
32/** Stack name of the scratch stack in Bindings.test.ts. */
33const STACK = "RDSDataBindings";
34/** Physical names are `{stack}-{id}-{stage}-{suffix}`; RDS lowercases identifiers. */
35const PREFIX = `${STACK.toLowerCase()}-`;
36/** The fixture VPC's deterministic CIDR (see infra.ts). */
37const VPC_CIDR = "10.61.0.0/16";
38
39const matches = (name: string | undefined): name is string =>
40 !!name && name.toLowerCase().startsWith(PREFIX);
41
42class OrphanStillPresent extends Data.TaggedError("OrphanStillPresent")<{
43 readonly kind: string;
44}> {}
45
46/** ~10 minute ceiling for slow asynchronous releases (RDS deletes, Lambda ENIs). */
47const slowRelease = Schedule.max([
48 Schedule.spaced("15 seconds"),
49 Schedule.recurs(40),
50]);
51/** ~2 minute ceiling for ordinary dependency-violation races. */
52const dependencyRelease = Schedule.max([
53 Schedule.spaced("10 seconds"),
54 Schedule.recurs(12),
55]);
56
57const reapFunctions = Effect.gen(function* () {
58 const pages = yield* lambda.listFunctions.pages({}).pipe(Stream.runCollect);
59 const names = Array.from(pages)
60 .flatMap((page) => page.Functions ?? [])
61 .map((fn) => fn.FunctionName)
62 .filter(matches);
63 yield* Effect.forEach(
64 names,
65 (FunctionName) =>
66 Effect.logInfo(
67 `RDSData reap: deleting Lambda function ${FunctionName}`,
68 ).pipe(
69 Effect.andThen(lambda.deleteFunction({ FunctionName })),
70 Effect.retry({
71 while: (e) =>
72 e._tag === "ResourceConflictException" ||
73 e._tag === "TooManyRequestsException",
74 schedule: dependencyRelease,
75 }),
76 Effect.catchTag("ResourceNotFoundException", () => Effect.void),
77 ),
78 { discard: true },
79 );
80});
81
82const reapLogGroups = Effect.gen(function* () {
83 const pages = yield* logs.describeLogGroups
84 .pages({ logGroupNamePrefix: `/aws/lambda/${STACK}-` })
85 .pipe(Stream.runCollect);
86 const names = Array.from(pages)
87 .flatMap((page) => page.logGroups ?? [])
88 .flatMap((group) => (group.logGroupName ? [group.logGroupName] : []));
89 yield* Effect.forEach(
90 names,
91 (logGroupName) =>
92 Effect.logInfo(`RDSData reap: deleting log group ${logGroupName}`).pipe(
93 Effect.andThen(logs.deleteLogGroup({ logGroupName })),
94 Effect.catchTag("ResourceNotFoundException", () => Effect.void),
95 ),
96 { discard: true },
97 );
98});
99
100const reapRoles = Effect.gen(function* () {
101 const pages = yield* iam.listRoles.pages({}).pipe(Stream.runCollect);
102 const names = Array.from(pages)
103 .flatMap((page) => page.Roles ?? [])
104 .map((role) => role.RoleName)
105 .filter(matches);
106 yield* Effect.forEach(
107 names,
108 (RoleName) =>
109 Effect.gen(function* () {
110 yield* Effect.logInfo(`RDSData reap: deleting IAM role ${RoleName}`);
111 const attached = yield* iam.listAttachedRolePolicies
112 .pages({ RoleName })
113 .pipe(Stream.runCollect);
114 yield* Effect.forEach(
115 Array.from(attached)
116 .flatMap((page) => page.AttachedPolicies ?? [])
117 .flatMap((policy) => (policy.PolicyArn ? [policy.PolicyArn] : [])),
118 (PolicyArn) =>
119 iam
120 .detachRolePolicy({ RoleName, PolicyArn })
121 .pipe(
122 Effect.catchTag("NoSuchEntityException", () => Effect.void),
123 ),
124 { discard: true },
125 );
126 const inline = yield* iam.listRolePolicies
127 .pages({ RoleName })
128 .pipe(Stream.runCollect);
129 yield* Effect.forEach(
130 Array.from(inline).flatMap((page) => page.PolicyNames ?? []),
131 (PolicyName) =>
132 iam
133 .deleteRolePolicy({ RoleName, PolicyName })
134 .pipe(
135 Effect.catchTag("NoSuchEntityException", () => Effect.void),
136 ),
137 { discard: true },
138 );
139 yield* iam.deleteRole({ RoleName }).pipe(
140 Effect.retry({
141 while: (e) => e._tag === "DeleteConflictException",
142 schedule: dependencyRelease,
143 }),
144 Effect.catchTag("NoSuchEntityException", () => Effect.void),
145 );
146 }).pipe(
147 // The role may vanish between the list and the detach walk.
148 Effect.catchTag("NoSuchEntityException", () => Effect.void),
149 ),
150 { discard: true },
151 );
152});
153
154const countDbInstances = rds.describeDBInstances.pages({}).pipe(
155 Stream.runCollect,
156 Effect.map(
157 (pages) =>
158 Array.from(pages)
159 .flatMap((page) => page.DBInstances ?? [])
160 .filter((instance) => matches(instance.DBInstanceIdentifier)).length,
161 ),
162);
163
164const reapDbInstances = Effect.gen(function* () {
165 const pages = yield* rds.describeDBInstances
166 .pages({})
167 .pipe(Stream.runCollect);
168 const ids = Array.from(pages)
169 .flatMap((page) => page.DBInstances ?? [])
170 .map((instance) => instance.DBInstanceIdentifier)
171 .filter(matches);
172 yield* Effect.forEach(
173 ids,
174 (id) =>
175 Effect.logInfo(`RDSData reap: deleting DB instance ${id}`).pipe(
176 Effect.andThen(
177 rds.deleteDBInstance({
178 DBInstanceIdentifier: id,
179 SkipFinalSnapshot: true,
180 DeleteAutomatedBackups: true,
181 }),
182 ),
183 // InvalidDBInstanceStateFault: already deleting — the wait below
184 // confirms it actually goes away.
185 Effect.catchTag(
186 ["DBInstanceNotFoundFault", "InvalidDBInstanceStateFault"],
187 () => Effect.void,
188 ),
189 ),
190 { discard: true },
191 );
192 if (ids.length > 0) {
193 yield* countDbInstances.pipe(
194 Effect.repeat({ schedule: slowRelease, until: (n) => n === 0 }),
195 Effect.flatMap((n) =>
196 n === 0
197 ? Effect.void
198 : Effect.fail(new OrphanStillPresent({ kind: "RDS DB instances" })),
199 ),
200 );
201 }
202});
203
204const countDbClusters = rds.describeDBClusters.pages({}).pipe(
205 Stream.runCollect,
206 Effect.map(
207 (pages) =>
208 Array.from(pages)
209 .flatMap((page) => page.DBClusters ?? [])
210 .filter((cluster) => matches(cluster.DBClusterIdentifier)).length,
211 ),
212);
213
214const reapDbClusters = Effect.gen(function* () {
215 const pages = yield* rds.describeDBClusters.pages({}).pipe(Stream.runCollect);
216 const ids = Array.from(pages)
217 .flatMap((page) => page.DBClusters ?? [])
218 .map((cluster) => cluster.DBClusterIdentifier)
219 .filter(matches);
220 yield* Effect.forEach(
221 ids,
222 (id) =>
223 Effect.logInfo(`RDSData reap: deleting DB cluster ${id}`).pipe(
224 Effect.andThen(
225 rds.deleteDBCluster({
226 DBClusterIdentifier: id,
227 SkipFinalSnapshot: true,
228 }),
229 ),
230 // InvalidDBClusterStateFault: already deleting / member still
231 // detaching — the wait below confirms convergence either way.
232 Effect.catchTag(
233 ["DBClusterNotFoundFault", "InvalidDBClusterStateFault"],
234 () => Effect.void,
235 ),
236 ),
237 { discard: true },
238 );
239 if (ids.length > 0) {
240 yield* countDbClusters.pipe(
241 Effect.repeat({ schedule: slowRelease, until: (n) => n === 0 }),
242 Effect.flatMap((n) =>
243 n === 0
244 ? Effect.void
245 : Effect.fail(new OrphanStillPresent({ kind: "RDS DB clusters" })),
246 ),
247 );
248 }
249});
250
251const reapDbSubnetGroups = Effect.gen(function* () {
252 const pages = yield* rds.describeDBSubnetGroups
253 .pages({})
254 .pipe(Stream.runCollect);
255 const names = Array.from(pages)
256 .flatMap((page) => page.DBSubnetGroups ?? [])
257 .map((group) => group.DBSubnetGroupName)
258 .filter(matches);
259 yield* Effect.forEach(
260 names,
261 (name) =>
262 Effect.logInfo(`RDSData reap: deleting DB subnet group ${name}`).pipe(
263 Effect.andThen(rds.deleteDBSubnetGroup({ DBSubnetGroupName: name })),
264 Effect.retry({
265 while: (e) => e._tag === "InvalidDBSubnetGroupStateFault",
266 schedule: dependencyRelease,
267 }),
268 Effect.catchTag("DBSubnetGroupNotFoundFault", () => Effect.void),
269 ),
270 { discard: true },
271 );
272});
273
274const reapSecrets = Effect.gen(function* () {
275 const pages = yield* secretsmanager.listSecrets
276 .pages({
277 IncludePlannedDeletion: true,
278 Filters: [{ Key: "name", Values: [STACK] }],
279 })
280 .pipe(Stream.runCollect);
281 const names = Array.from(pages)
282 .flatMap((page) => page.SecretList ?? [])
283 .map((secret) => secret.Name)
284 .filter(matches);
285 yield* Effect.forEach(
286 names,
287 (name) =>
288 Effect.logInfo(`RDSData reap: force-deleting secret ${name}`).pipe(
289 Effect.andThen(
290 secretsmanager.deleteSecret({
291 SecretId: name,
292 ForceDeleteWithoutRecovery: true,
293 }),
294 ),
295 // InvalidRequestException: deletion already in progress.
296 Effect.catchTag(
297 ["ResourceNotFoundException", "InvalidRequestException"],
298 () => Effect.void,
299 ),
300 ),
301 { discard: true },
302 );
303});
304
305/**
306 * Lambda-managed ENIs release asynchronously (minutes) after their function
307 * is deleted; they block SG/subnet/VPC deletion until then. Each pass deletes
308 * the ENIs that have reached `available`, then re-counts.
309 */
310const sweepNetworkInterfaces = Effect.fn(function* (vpcId: string) {
311 const describe = ec2
312 .describeNetworkInterfaces({
313 Filters: [{ Name: "vpc-id", Values: [vpcId] }],
314 })
315 .pipe(Effect.map((response) => response.NetworkInterfaces ?? []));
316 yield* Effect.gen(function* () {
317 const enis = yield* describe;
318 yield* Effect.forEach(
319 enis.filter(
320 (eni) => eni.Status === "available" && eni.NetworkInterfaceId,
321 ),
322 (eni) =>
323 Effect.logInfo(
324 `RDSData reap: deleting ENI ${eni.NetworkInterfaceId}`,
325 ).pipe(
326 Effect.andThen(
327 ec2.deleteNetworkInterface({
328 NetworkInterfaceId: eni.NetworkInterfaceId,
329 }),
330 ),
331 Effect.catchTag(
332 [
333 "InvalidNetworkInterfaceID.NotFound",
334 "InvalidNetworkInterface.InUse",
335 ],
336 () => Effect.void,
337 ),
338 ),
339 { discard: true },
340 );
341 return (yield* describe).length;
342 }).pipe(
343 Effect.repeat({ schedule: slowRelease, until: (n) => n === 0 }),
344 Effect.flatMap((n) =>
345 n === 0
346 ? Effect.void
347 : Effect.fail(
348 new OrphanStillPresent({ kind: `network interfaces in ${vpcId}` }),
349 ),
350 ),
351 );
352});
353
354const reapVpc = Effect.fn(function* (vpcId: string) {
355 yield* Effect.logInfo(`RDSData reap: tearing down VPC ${vpcId}`);
356 yield* sweepNetworkInterfaces(vpcId);
357
358 // Security groups: break the LambdaSG <- DbSG ingress cross-reference
359 // first, then delete (bounded DependencyViolation retry).
360 const groups = yield* ec2
361 .describeSecurityGroups({
362 Filters: [{ Name: "vpc-id", Values: [vpcId] }],
363 })
364 .pipe(
365 Effect.map((response) =>
366 (response.SecurityGroups ?? []).filter(
367 (group) => group.GroupName !== "default" && group.GroupId,
368 ),
369 ),
370 );
371 yield* Effect.forEach(
372 groups,
373 (group) =>
374 (group.IpPermissions ?? []).length > 0
375 ? ec2
376 .revokeSecurityGroupIngress({
377 GroupId: group.GroupId,
378 IpPermissions: group.IpPermissions,
379 })
380 .pipe(
381 Effect.catchTag(
382 ["InvalidGroup.NotFound", "InvalidPermission.NotFound"],
383 () => Effect.void,
384 ),
385 )
386 : Effect.void,
387 { discard: true },
388 );
389 yield* Effect.forEach(
390 groups,
391 (group) =>
392 Effect.logInfo(
393 `RDSData reap: deleting security group ${group.GroupId} (${group.GroupName})`,
394 ).pipe(
395 Effect.andThen(ec2.deleteSecurityGroup({ GroupId: group.GroupId })),
396 Effect.retry({
397 while: (e) => e._tag === "DependencyViolation",
398 schedule: dependencyRelease,
399 }),
400 Effect.catchTag("InvalidGroup.NotFound", () => Effect.void),
401 ),
402 { discard: true },
403 );
404
405 const subnets = yield* ec2
406 .describeSubnets({ Filters: [{ Name: "vpc-id", Values: [vpcId] }] })
407 .pipe(
408 Effect.map((response) =>
409 (response.Subnets ?? []).flatMap((subnet) =>
410 subnet.SubnetId ? [subnet.SubnetId] : [],
411 ),
412 ),
413 );
414 yield* Effect.forEach(
415 subnets,
416 (SubnetId) =>
417 Effect.logInfo(`RDSData reap: deleting subnet ${SubnetId}`).pipe(
418 Effect.andThen(ec2.deleteSubnet({ SubnetId })),
419 Effect.retry({
420 while: (e) => e._tag === "DependencyViolation",
421 schedule: dependencyRelease,
422 }),
423 Effect.catchTag("InvalidSubnetID.NotFound", () => Effect.void),
424 ),
425 { discard: true },
426 );
427
428 yield* ec2.deleteVpc({ VpcId: vpcId }).pipe(
429 Effect.retry({
430 while: (e) => e._tag === "DependencyViolation",
431 schedule: dependencyRelease,
432 }),
433 Effect.catchTag("InvalidVpcID.NotFound", () => Effect.void),
434 );
435});
436
437const reapVpcs = Effect.gen(function* () {
438 const byTag = yield* ec2.describeVpcs({
439 Filters: [{ Name: "tag:alchemy::stack", Values: [STACK] }],
440 });
441 const byCidr = yield* ec2.describeVpcs({
442 Filters: [{ Name: "cidr", Values: [VPC_CIDR] }],
443 });
444 const vpcIds = [
445 ...new Set(
446 [...(byTag.Vpcs ?? []), ...(byCidr.Vpcs ?? [])].flatMap((vpc) =>
447 vpc.VpcId ? [vpc.VpcId] : [],
448 ),
449 ),
450 ];
451 yield* Effect.forEach(vpcIds, reapVpc, { discard: true });
452});
453
454/**
455 * Delete every leftover RDSData fixture resource, in dependency order.
456 * Idempotent: a run against a clean account is a handful of empty list calls.
457 *
458 * Requires the AWS provider environment (wrap with `Core.withProviders`).
459 */
460export const reapRDSDataOrphans = Effect.gen(function* () {
461 yield* reapFunctions;
462 yield* reapLogGroups;
463 yield* reapRoles;
464 yield* reapDbInstances;
465 yield* reapDbClusters;
466 yield* reapDbSubnetGroups;
467 yield* reapSecrets;
468 yield* reapVpcs;
469});