| 1 | /** |
| 2 | * Destroy robustness: a destroy (or the GC phase of a deploy) must delete |
| 3 | * everything it can, aggregate every failure, and never let one bad row |
| 4 | * block the rest of the teardown. |
| 5 | * |
| 6 | * - a failing provider `delete` must not interrupt sibling deletions |
| 7 | * - an upstream's delete is skipped (state retained) when a dependent fails |
| 8 | * - a "zombie" state row whose resource type has no registered provider |
| 9 | * (the type was removed/renamed without an alias) is FATAL at plan time: |
| 10 | * the plan dies with a typed `MissingProviderError` and nothing is |
| 11 | * deployed or destroyed — the program and state disagree, and without |
| 12 | * the provider the row cannot be deleted anyway |
| 13 | * - an attr-less row whose `read` recovery fails must not block siblings |
| 14 | */ |
| 15 | import { MissingProviderError } from "@/Provider"; |
| 16 | import { Stack } from "@/Stack"; |
| 17 | import { State, type ResourceState } from "@/State"; |
| 18 | import * as Test from "@/Test/Alchemy"; |
| 19 | import { describe, expect } from "alchemy-test"; |
| 20 | import * as Cause from "effect/Cause"; |
| 21 | import * as Effect from "effect/Effect"; |
| 22 | import * as Exit from "effect/Exit"; |
| 23 | import * as Layer from "effect/Layer"; |
| 24 | import { |
| 25 | TestLayers, |
| 26 | TestResource, |
| 27 | TestResourceHooks, |
| 28 | } from "./test.resources.ts"; |
| 29 | |
| 30 | const { test } = Test.make({ providers: TestLayers() }); |
| 31 | |
| 32 | const getState = Effect.fn(function* (fqn: string) { |
| 33 | const state = yield* yield* State; |
| 34 | const stk = yield* Stack; |
| 35 | return yield* state.get({ stack: stk.name, stage: stk.stage, fqn }); |
| 36 | }); |
| 37 | |
| 38 | const clearState = Effect.fn(function* (fqn: string) { |
| 39 | const state = yield* yield* State; |
| 40 | const stk = yield* Stack; |
| 41 | yield* state.delete({ stack: stk.name, stage: stk.stage, fqn }); |
| 42 | }); |
| 43 | |
| 44 | const seed = Effect.fn(function* (fqn: string, value: ResourceState) { |
| 45 | const state = yield* yield* State; |
| 46 | const stk = yield* Stack; |
| 47 | yield* state.set({ stack: stk.name, stage: stk.stage, fqn, value }); |
| 48 | }); |
| 49 | |
| 50 | const instanceId = "852f6ec2e19b66589825efe14dca2971"; |
| 51 | |
| 52 | const hooks = (impl: { |
| 53 | delete?: (id: string) => Effect.Effect<void, any>; |
| 54 | read?: (id: string) => Effect.Effect<any, any>; |
| 55 | }): Layer.Layer<TestResourceHooks> => Layer.succeed(TestResourceHooks, impl); |
| 56 | |
| 57 | const failsWith = (exit: Exit.Exit<unknown, unknown>, tag: string): boolean => |
| 58 | Exit.isFailure(exit) && |
| 59 | exit.cause.reasons.some( |
| 60 | (r) => |
| 61 | Cause.isFailReason(r) && |
| 62 | (r.error as { _tag?: string } | undefined)?._tag === tag, |
| 63 | ); |
| 64 | |
| 65 | describe("destroy aggregates failures", { tags: ["unit", "local"] }, () => { |
| 66 | test.provider( |
| 67 | "a failing delete does not interrupt sibling deletions", |
| 68 | (stack) => |
| 69 | Effect.gen(function* () { |
| 70 | yield* stack.deploy( |
| 71 | Effect.gen(function* () { |
| 72 | yield* TestResource("A", { string: "a" }); |
| 73 | yield* TestResource("B", { string: "b" }); |
| 74 | yield* TestResource("C", { string: "c" }); |
| 75 | }), |
| 76 | ); |
| 77 | |
| 78 | const exit = yield* stack.destroy().pipe( |
| 79 | Effect.provide( |
| 80 | hooks({ |
| 81 | delete: (id) => |
| 82 | id === "A" |
| 83 | ? Effect.fail(new Error("A refuses to die")) |
| 84 | : id === "B" |
| 85 | ? // Slow sibling: before aggregation, A's failure |
| 86 | // interrupted B mid-delete and left its row behind. |
| 87 | Effect.sleep("150 millis") |
| 88 | : Effect.void, |
| 89 | }), |
| 90 | ), |
| 91 | Effect.exit, |
| 92 | ); |
| 93 | |
| 94 | // The destroy still fails overall... |
| 95 | expect(Exit.isFailure(exit)).toBe(true); |
| 96 | // ...but every deletable resource was deleted. |
| 97 | expect(yield* getState("B")).toBeUndefined(); |
| 98 | expect(yield* getState("C")).toBeUndefined(); |
| 99 | // The failed row keeps its state for the next attempt. |
| 100 | expect((yield* getState("A"))?.status).toBe("deleting"); |
| 101 | }), |
| 102 | ); |
| 103 | |
| 104 | test.provider( |
| 105 | "an upstream's delete is skipped when a dependent fails", |
| 106 | (stack) => |
| 107 | Effect.gen(function* () { |
| 108 | yield* stack.deploy( |
| 109 | Effect.gen(function* () { |
| 110 | const a = yield* TestResource("A", { string: "a" }); |
| 111 | yield* TestResource("B", { string: a.string }); |
| 112 | }), |
| 113 | ); |
| 114 | |
| 115 | const deleted: string[] = []; |
| 116 | const exit = yield* stack.destroy().pipe( |
| 117 | Effect.provide( |
| 118 | hooks({ |
| 119 | delete: (id) => |
| 120 | id === "B" |
| 121 | ? Effect.fail(new Error("B refuses to die")) |
| 122 | : Effect.sync(() => void deleted.push(id)), |
| 123 | }), |
| 124 | ), |
| 125 | Effect.exit, |
| 126 | ); |
| 127 | |
| 128 | expect(Exit.isFailure(exit)).toBe(true); |
| 129 | // A still has a dependent in the cloud — its physical delete must |
| 130 | // not have been attempted and its state must be retained. |
| 131 | expect(deleted).not.toContain("A"); |
| 132 | expect(yield* getState("A")).toBeDefined(); |
| 133 | expect(yield* getState("B")).toBeDefined(); |
| 134 | }), |
| 135 | ); |
| 136 | |
| 137 | test.provider( |
| 138 | "an attr-less row with failing read recovery does not block siblings", |
| 139 | (stack) => |
| 140 | Effect.gen(function* () { |
| 141 | yield* stack.deploy( |
| 142 | Effect.gen(function* () { |
| 143 | yield* TestResource("A", { string: "a" }); |
| 144 | }), |
| 145 | ); |
| 146 | // An interrupted create: `creating` with no attr snapshot. |
| 147 | yield* seed("Zombie", { |
| 148 | instanceId, |
| 149 | providerVersion: 0, |
| 150 | logicalId: "Zombie", |
| 151 | fqn: "Zombie", |
| 152 | namespace: undefined, |
| 153 | resourceType: "Test.TestResource", |
| 154 | status: "creating", |
| 155 | props: { string: "z" }, |
| 156 | attr: undefined, |
| 157 | bindings: [], |
| 158 | downstream: [], |
| 159 | }); |
| 160 | |
| 161 | const exit = yield* stack.destroy().pipe( |
| 162 | Effect.provide( |
| 163 | hooks({ |
| 164 | read: () => Effect.fail(new Error("read exploded")), |
| 165 | // Slow sibling so a fail-fast interruption would be observable. |
| 166 | delete: () => Effect.sleep("150 millis"), |
| 167 | }), |
| 168 | ), |
| 169 | Effect.exit, |
| 170 | ); |
| 171 | |
| 172 | expect(Exit.isFailure(exit)).toBe(true); |
| 173 | expect(yield* getState("A")).toBeUndefined(); |
| 174 | // The unrecoverable row is retained, not silently dropped. |
| 175 | expect(yield* getState("Zombie")).toBeDefined(); |
| 176 | }), |
| 177 | ); |
| 178 | }); |
| 179 | |
| 180 | describe( |
| 181 | "zombie rows (no registered provider)", |
| 182 | { tags: ["unit", "local"] }, |
| 183 | () => { |
| 184 | const ghostRow = (fqn: string): ResourceState => ({ |
| 185 | instanceId, |
| 186 | providerVersion: 0, |
| 187 | logicalId: fqn, |
| 188 | fqn, |
| 189 | namespace: undefined, |
| 190 | resourceType: "Test.Vanished", |
| 191 | status: "created", |
| 192 | props: { name: "ghost" }, |
| 193 | attr: { name: "ghost" }, |
| 194 | bindings: [], |
| 195 | downstream: [], |
| 196 | }); |
| 197 | |
| 198 | const diesWithMissingProvider = ( |
| 199 | exit: Exit.Exit<unknown, unknown>, |
| 200 | ): MissingProviderError | undefined => { |
| 201 | if (!Exit.isFailure(exit)) return undefined; |
| 202 | const reason = exit.cause.reasons.find( |
| 203 | (r) => Cause.isDieReason(r) && r.defect instanceof MissingProviderError, |
| 204 | ); |
| 205 | return reason && Cause.isDieReason(reason) |
| 206 | ? (reason.defect as MissingProviderError) |
| 207 | : undefined; |
| 208 | }; |
| 209 | |
| 210 | test.provider("destroy dies at plan time and destroys nothing", (stack) => |
| 211 | Effect.gen(function* () { |
| 212 | yield* stack.deploy( |
| 213 | Effect.gen(function* () { |
| 214 | yield* TestResource("A", { string: "a" }); |
| 215 | }), |
| 216 | ); |
| 217 | yield* seed("Ghost", ghostRow("Ghost")); |
| 218 | |
| 219 | const exit = yield* stack.destroy().pipe(Effect.exit); |
| 220 | |
| 221 | const defect = diesWithMissingProvider(exit); |
| 222 | expect(defect?.resourceType).toBe("Test.Vanished"); |
| 223 | expect(defect?.fqn).toBe("Ghost"); |
| 224 | // Fatal at plan time: NOTHING was destroyed — A's state is intact. |
| 225 | expect((yield* getState("A"))?.status).toBe("created"); |
| 226 | expect(yield* getState("Ghost")).toBeDefined(); |
| 227 | |
| 228 | // Remediation path: clear the zombie row (as the error message |
| 229 | // instructs), then destroy proceeds normally. |
| 230 | yield* clearState("Ghost"); |
| 231 | yield* stack.destroy(); |
| 232 | expect(yield* getState("A")).toBeUndefined(); |
| 233 | }), |
| 234 | ); |
| 235 | |
| 236 | test.provider( |
| 237 | "a deploy with a zombie orphan dies at plan time and applies nothing", |
| 238 | (stack) => |
| 239 | Effect.gen(function* () { |
| 240 | yield* seed("Ghost", ghostRow("Ghost")); |
| 241 | |
| 242 | const exit = yield* stack |
| 243 | .deploy( |
| 244 | Effect.gen(function* () { |
| 245 | const a = yield* TestResource("A", { string: "a" }); |
| 246 | return a.string; |
| 247 | }), |
| 248 | ) |
| 249 | .pipe(Effect.exit); |
| 250 | |
| 251 | expect(diesWithMissingProvider(exit)?.resourceType).toBe( |
| 252 | "Test.Vanished", |
| 253 | ); |
| 254 | // Fatal at plan time: the program was NOT applied. |
| 255 | expect(yield* getState("A")).toBeUndefined(); |
| 256 | expect(yield* getState("Ghost")).toBeDefined(); |
| 257 | |
| 258 | yield* clearState("Ghost"); |
| 259 | }), |
| 260 | ); |
| 261 | }, |
| 262 | ); |