1import * as AWS from "@/AWS";
2import {
3 amazonLinux2023,
4 Instance,
5 Subnet,
6 Vpc,
7 type InstanceProps,
8} from "@/AWS/EC2";
9import * as Provider from "@/Provider";
10import { State } from "@/State/State";
11import * as ec2 from "@distilled.cloud/aws/ec2";
12import * as Test from "./VpcTest.ts";
13import { describe, expect } from "alchemy-test";
14import * as Effect from "effect/Effect";
15import { MinimumLogLevel } from "effect/References";
16import { assertInstanceTerminated, assertVpcGone } from "./Gone.ts";
17
18// The fixed-IP fixture keeps two VPCs to verify allocation scope. Tests run
19// sequentially so the file holds at most these two VPCs.
20const { test } = Test.make({ providers: AWS.providers() }, 2);
21
22const logLevel = Effect.provideService(
23 MinimumLogLevel,
24 process.env.DEBUG ? "Debug" : "Info",
25);
26
27describe.sequential(
28 "Instance",
29 { tags: ["provider:aws", "provider:aws:ec2", "live"] },
30 () => {
31 // `list()` enumerates every non-terminated instance in the account/region via
32 // the paginated `ec2.describeInstances` op (items nested under
33 // Reservations[].Instances[]). Deploy a real instance, resolve the provider
34 // from context with the typed `findProvider`, call `list()`, and assert the
35 // deployed instance appears in the exhaustively paginated result.
36 test.provider(
37 "list enumerates the deployed instance",
38 (stack) =>
39 Effect.gen(function* () {
40 yield* stack.destroy();
41
42 const imageId = amazonLinux2023();
43
44 // The testing account has no default VPC, so provision a VPC + subnet to
45 // launch the instance into.
46 const deployed = yield* stack.deploy(
47 Effect.gen(function* () {
48 const vpc = yield* Vpc("ListInstanceVpc", {
49 cidrBlock: "10.0.0.0/16",
50 });
51 const subnet = yield* Subnet("ListInstanceSubnet", {
52 vpcId: vpc.vpcId,
53 cidrBlock: "10.0.1.0/24",
54 });
55 const instance = yield* Instance("ListInstance", {
56 imageId,
57 instanceType: "t3.micro",
58 subnetId: subnet.subnetId,
59 });
60 return { vpc, instance };
61 }),
62 );
63
64 const provider = yield* Provider.findProvider(Instance);
65 const all = yield* provider.list();
66
67 expect(
68 all.some((x) => x.instanceId === deployed.instance.instanceId),
69 ).toBe(true);
70
71 yield* stack.destroy();
72
73 // Zero-orphan proof: the instance reached a terminal state and the VPC
74 // (which cannot delete while any ENI lingers) is gone.
75 yield* assertInstanceTerminated(deployed.instance.instanceId);
76 yield* assertVpcGone(deployed.vpc.vpcId);
77 }).pipe(logLevel),
78 { timeout: 240_000 },
79 );
80
81 // Replacement must launch a distinct physical instance. The create phase of a
82 // replacement runs under a freshly minted generation id, so the tag-based
83 // recovery lookup (branded with `alchemy::instance`) can never re-adopt the
84 // old generation's live instance — which the cleanup phase then terminates,
85 // leaving state pointing at a terminated instance (#1026).
86 test.provider.skipIf(!!process.env.FAST)(
87 "replace launches a distinct instance and terminates only the old one",
88 (stack) =>
89 Effect.gen(function* () {
90 yield* stack.destroy();
91
92 const imageId = amazonLinux2023();
93
94 const program = (userData: string, privateIpAddress?: string) =>
95 Effect.gen(function* () {
96 const vpc = yield* Vpc("ReplaceInstanceVpc", {
97 cidrBlock: "10.0.0.0/16",
98 });
99 const subnet = yield* Subnet("ReplaceInstanceSubnet", {
100 vpcId: vpc.vpcId,
101 cidrBlock: "10.0.1.0/24",
102 });
103 const instance = yield* Instance("ReplaceInstance", {
104 imageId,
105 instanceType: "t3.micro",
106 subnetId: subnet.subnetId,
107 userData,
108 privateIpAddress,
109 tags: { Name: "alchemy-replace-instance-test" },
110 });
111 return { vpc, instance };
112 });
113
114 const first = yield* stack.deploy(
115 program("#!/bin/bash\necho generation-one\n"),
116 );
117 const next = program("#!/bin/bash\necho generation-two\n");
118 const automatic = yield* stack.plan(next);
119 expect(automatic.resources.ReplaceInstance).toMatchObject({
120 action: "replace",
121 deleteFirst: false,
122 });
123 const pinned = yield* stack.plan(
124 program(
125 "#!/bin/bash\necho generation-two\n",
126 first.instance.privateIpAddress,
127 ),
128 );
129 expect(pinned.resources.ReplaceInstance).toMatchObject({
130 action: "replace",
131 deleteFirst: true,
132 });
133
134 // userData is a force-new prop, so this deploy plans a replacement.
135 const second = yield* stack.deploy(next);
136
137 // The replacement created a distinct physical instance...
138 expect(second.instance.instanceId).not.toBe(
139 first.instance.instanceId,
140 );
141
142 // ...that is alive after cleanup (out-of-band via distilled)...
143 const live = yield* ec2.describeInstances({
144 InstanceIds: [second.instance.instanceId],
145 });
146 const liveState =
147 live.Reservations?.[0]?.Instances?.[0]?.State?.Name ?? "unknown";
148 expect(["pending", "running"]).toContain(liveState);
149
150 // ...while the old generation was the one terminated.
151 yield* assertInstanceTerminated(first.instance.instanceId);
152
153 yield* stack.destroy();
154
155 // Zero-orphan proof for the replacement generation and the VPC.
156 yield* assertInstanceTerminated(second.instance.instanceId);
157 yield* assertVpcGone(second.vpc.vpcId);
158 }).pipe(logLevel),
159 { timeout: 600_000 },
160 );
161
162 const fixedIpProgram = (props: Partial<InstanceProps> = {}) =>
163 Effect.gen(function* () {
164 const vpc = yield* Vpc("FixedIpVpc", { cidrBlock: "10.0.0.0/16" });
165 const subnet = yield* Subnet("FixedIpSubnet", {
166 vpcId: vpc.vpcId,
167 cidrBlock: "10.0.1.0/24",
168 });
169 const otherVpc = yield* Vpc("OtherFixedIpVpc", {
170 cidrBlock: "10.0.0.0/16",
171 });
172 const otherSubnet = yield* Subnet("OtherFixedIpSubnet", {
173 vpcId: otherVpc.vpcId,
174 cidrBlock: "10.0.1.0/24",
175 });
176 const instance = yield* Instance("FixedIpInstance", {
177 imageId: amazonLinux2023(),
178 instanceType: "t3.micro",
179 subnetId: subnet.subnetId,
180 privateIpAddress: "10.0.1.10",
181 userData: "#!/bin/bash\necho generation-one\n",
182 ...props,
183 });
184 return { vpc, subnet, otherVpc, otherSubnet, instance };
185 });
186
187 test.provider(
188 "fixed-IP replacements delete first only when reusing the same allocation",
189 (stack) =>
190 Effect.gen(function* () {
191 yield* stack.destroy();
192 const first = yield* stack.deploy(fixedIpProgram());
193 const unchanged = yield* stack.plan(fixedIpProgram());
194 expect(unchanged.resources.FixedIpInstance?.action).toBe("noop");
195
196 const tags = { Name: "alchemy-fixed-ip-replacement" };
197 const update = yield* stack.plan(fixedIpProgram({ tags }));
198 expect(update.resources.FixedIpInstance?.action).toBe("update");
199 const updated = yield* stack.deploy(fixedIpProgram({ tags }));
200 expect(updated.instance.instanceId).toBe(first.instance.instanceId);
201
202 const differentIp = yield* stack.plan(
203 fixedIpProgram({
204 privateIpAddress: "10.0.1.11",
205 }),
206 );
207 expect(differentIp.resources.FixedIpInstance).toMatchObject({
208 action: "replace",
209 deleteFirst: false,
210 });
211 const differentSubnet = yield* stack.plan(
212 fixedIpProgram({
213 subnetId: first.otherSubnet.subnetId,
214 }),
215 );
216 expect(differentSubnet.resources.FixedIpInstance).toMatchObject({
217 action: "replace",
218 deleteFirst: false,
219 });
220
221 const next = fixedIpProgram({
222 userData: "#!/bin/bash\necho generation-two\n",
223 tags,
224 });
225 const replacement = yield* stack.plan(next);
226 expect(replacement.resources.FixedIpInstance).toMatchObject({
227 action: "replace",
228 deleteFirst: true,
229 });
230 const second = yield* stack.deploy(next);
231 expect(second.instance.instanceId).not.toBe(
232 first.instance.instanceId,
233 );
234 const live = yield* ec2.describeInstances({
235 InstanceIds: [second.instance.instanceId],
236 });
237 expect(live.Reservations?.[0]?.Instances?.[0]).toMatchObject({
238 PrivateIpAddress: "10.0.1.10",
239 SubnetId: first.subnet.subnetId,
240 State: { Name: "running" },
241 });
242 yield* assertInstanceTerminated(first.instance.instanceId);
243 const settled = yield* stack.plan(next);
244 expect(settled.resources.FixedIpInstance?.action).toBe("noop");
245
246 yield* stack.destroy();
247 yield* assertInstanceTerminated(second.instance.instanceId);
248 yield* assertVpcGone(first.vpc.vpcId);
249 yield* assertVpcGone(first.otherVpc.vpcId);
250 }).pipe(logLevel),
251 { timeout: 120_000 },
252 );
253
254 test.provider(
255 "recovers a launched fixed-IP replacement whose output was not persisted",
256 (stack) =>
257 Effect.gen(function* () {
258 yield* stack.destroy();
259 const state = yield* yield* State;
260 const key = {
261 stack: stack.name,
262 stage: stack.stage,
263 fqn: "FixedIpInstance",
264 };
265 const first = yield* stack.deploy(fixedIpProgram());
266 const firstState = yield* state.get(key);
267 if (
268 firstState?.status !== "created" &&
269 firstState?.status !== "updated"
270 ) {
271 return yield* Effect.die(
272 new Error("Expected a deployed original instance"),
273 );
274 }
275
276 const next = fixedIpProgram({ privateIpAddress: "10.0.1.11" });
277 const second = yield* stack.deploy(next);
278 const secondState = yield* state.get(key);
279 if (
280 secondState?.status !== "created" &&
281 secondState?.status !== "updated"
282 ) {
283 return yield* Effect.die(
284 new Error("Expected a deployed replacement instance"),
285 );
286 }
287 // Retain the real candidate's generation and tags, but simulate the
288 // interrupted commit between its EC2 launch and persisting its output.
289 yield* state.set({
290 ...key,
291 value: {
292 ...secondState,
293 status: "replacing",
294 attr: undefined,
295 deleteFirst: false,
296 old: firstState,
297 },
298 });
299 const recovery = yield* stack.plan(next);
300 expect(recovery.resources.FixedIpInstance).toMatchObject({
301 action: "replace",
302 deleteFirst: false,
303 });
304 expect(recovery.resources.FixedIpInstance).not.toMatchObject({
305 restart: true,
306 });
307 const recovered = yield* stack.deploy(next);
308 expect(recovered.instance.instanceId).toBe(
309 second.instance.instanceId,
310 );
311 const live = yield* ec2.describeInstances({
312 InstanceIds: [recovered.instance.instanceId],
313 });
314 expect(live.Reservations?.[0]?.Instances?.[0]).toMatchObject({
315 PrivateIpAddress: "10.0.1.11",
316 State: { Name: "running" },
317 });
318 const settled = yield* stack.plan(next);
319 expect(settled.resources.FixedIpInstance?.action).toBe("noop");
320
321 yield* stack.destroy();
322 yield* assertInstanceTerminated(first.instance.instanceId);
323 yield* assertInstanceTerminated(second.instance.instanceId);
324 yield* assertVpcGone(first.vpc.vpcId);
325 yield* assertVpcGone(first.otherVpc.vpcId);
326 }).pipe(logLevel),
327 { timeout: 120_000 },
328 );
329 },
330);