1import * as AWS from "@/AWS";
2import { SecurityGroup } from "@/AWS/EC2";
3import { Cluster } from "@/AWS/ECS/Cluster.ts";
4import { Service } from "@/AWS/ECS/Service.ts";
5import * as Cloudflare from "@/Cloudflare/index.ts";
6import * as Test from "@/Test/Alchemy";
7import * as ecs from "@distilled.cloud/aws/ecs";
8import * as elbv2 from "@distilled.cloud/aws/elastic-load-balancing-v2";
9import { expect } from "alchemy-test";
10import * as ConfigProvider from "effect/ConfigProvider";
11import * as Effect from "effect/Effect";
12import * as Layer from "effect/Layer";
13import * as Schedule from "effect/Schedule";
14import * as HttpClient from "effect/http/HttpClient";
15import * as pathe from "pathe";
16import { getDefaultVpcNetwork } from "../DefaultVpc.ts";
17import type { OtelSink } from "./fixtures/otel-collector-worker.ts";
18import TestTask from "./fixtures/task.ts";
19import {
20 E2E_CLUSTER_NAME,
21 E2E_TEST_TITLE,
22 reclaimTaskE2EOrphans,
23 scanTaskE2EOrphans,
24} from "./reclaimTaskE2EOrphans.ts";
25
26const { test } = Test.make({
27 providers: Layer.mergeAll(AWS.providers(), Cloudflare.providers()),
28});
29
30// The OTLP sink the task exports to (a workers.dev URL is reachable from
31// Fargate). Declared identically in both deploy steps so the second deploy
32// keeps it.
33const collectorMain = pathe.resolve(
34 import.meta.dirname,
35 "fixtures/otel-collector-worker.ts",
36);
37const collectorWorker = () =>
38 Cloudflare.Worker("EcsOtelCollector", {
39 main: collectorMain,
40 env: {
41 SINK: Cloudflare.DurableObject<OtelSink>("OtelSink"),
42 },
43 compatibility: { date: "2024-09-23" },
44 });
45
46// Full end-to-end: build + push the bundled Task image, run it on Fargate
47// behind an Alchemy-managed public ALB, and prove over HTTP that (a) the
48// `{ fetch }` handler is served and (b) the `ServerHost.run` background loop is
49// actually executing inside the deployed container (`/ticks` keeps climbing).
50//
51// This is the real-deploy regression for #706. Docker/ECR + Fargate placement
52// + the ALB health ramp takes about five minutes even on the standing default
53// VPC, so keep it out of the hard-240 default sweep. Run it explicitly with
54// `AWS_TEST_SLOW=1`; `FAST=1` always keeps the lifecycle disabled.
55test.provider.skipIf(!process.env.AWS_TEST_SLOW || !!process.env.FAST)(
56 // The title doubles as the scratch stack name, which prefixes every
57 // physical name — the orphan sweep prefix-matches on it.
58 E2E_TEST_TITLE,
59 (stack) =>
60 Effect.gen(function* () {
61 yield* stack.destroy();
62
63 // The scratch stack's state is in-memory, so a hard-killed prior run
64 // (vitest hard timeout / OOM) orphans everything it deployed — the
65 // engine can never see it again. Reclaim those leftovers up front, and
66 // guarantee the same sweep runs on success, failure, AND interruption.
67 yield* reclaimTaskE2EOrphans;
68 yield* Effect.addFinalizer(() =>
69 reclaimTaskE2EOrphans.pipe(Effect.orDie),
70 );
71
72 // Reuse the standing public network. Creating a dedicated VPC here is
73 // both unnecessary for the Task regression and races the account's VPC
74 // quota during c128 sweeps. An internet-facing ALB needs two AZs.
75 const { vpcId, subnetIds } = yield* getDefaultVpcNetwork;
76 if (subnetIds.length < 2) {
77 return yield* Effect.die(
78 new Error("default VPC has fewer than 2 subnets"),
79 );
80 }
81 const publicSubnetIds = subnetIds.slice(0, 2);
82
83 // Deploy the OTLP collector first: the task fixture resolves
84 // `COLLECTOR_URL` from the deployer's config at deploy time.
85 const collector = yield* stack.deploy(
86 Effect.gen(function* () {
87 return yield* collectorWorker();
88 }),
89 );
90 const collected = `${collector.url}/collected`;
91
92 const currentConfig = yield* ConfigProvider.ConfigProvider;
93 const {
94 url,
95 targetGroupArn,
96 clusterArn,
97 serviceName,
98 taskDefinitionArn,
99 } = yield* stack
100 .deploy(
101 Effect.gen(function* () {
102 yield* collectorWorker();
103 // The default VPC supplies public routing and subnets; this managed
104 // security group remains isolated to the test and is swept exactly.
105 const securityGroup = yield* SecurityGroup("EcsE2ESg", {
106 vpcId,
107 description: "alchemy ecs task e2e",
108 ingress: [
109 {
110 ipProtocol: "tcp",
111 fromPort: 80,
112 toPort: 80,
113 cidrIpv4: "0.0.0.0/0",
114 description: "ALB ingress",
115 },
116 {
117 ipProtocol: "tcp",
118 fromPort: 3000,
119 toPort: 3000,
120 cidrIpv4: "0.0.0.0/0",
121 description: "container traffic",
122 },
123 ],
124 egress: [
125 {
126 ipProtocol: "-1",
127 cidrIpv4: "0.0.0.0/0",
128 description: "all outbound",
129 },
130 ],
131 });
132
133 const cluster = yield* Cluster("EcsE2ECluster", {
134 clusterName: E2E_CLUSTER_NAME,
135 });
136
137 // The bundled long-running Task (builds + pushes the image).
138 const task = yield* TestTask;
139
140 const service = yield* Service("EcsE2EService", {
141 cluster,
142 task: {
143 taskDefinitionArn: task.taskDefinitionArn,
144 containerName: task.containerName,
145 port: task.port,
146 },
147 desiredCount: 1,
148 public: true,
149 listenerPort: 80,
150 healthCheckPath: "/health",
151 vpcId,
152 subnets: publicSubnetIds,
153 securityGroups: [securityGroup.groupId],
154 assignPublicIp: true,
155 });
156
157 return {
158 url: service.url,
159 targetGroupArn: service.targetGroupArn,
160 clusterArn: service.clusterArn,
161 serviceName: service.serviceName,
162 taskDefinitionArn: service.taskDefinitionArn,
163 };
164 }),
165 )
166 .pipe(
167 Effect.provideService(
168 ConfigProvider.ConfigProvider,
169 ConfigProvider.orElse(
170 ConfigProvider.fromUnknown({ COLLECTOR_URL: collector.url }),
171 currentConfig,
172 ),
173 ),
174 );
175
176 expect(url).toBeTruthy();
177 expect(targetGroupArn).toBeTruthy();
178
179 // Service.deploy now blocks until the exact task definition is the sole
180 // completed deployment, ECS counts have settled, and every registered
181 // target is healthy. Prove that contract immediately out-of-band.
182 const describedService = yield* ecs.describeServices({
183 cluster: clusterArn,
184 services: [serviceName],
185 });
186 const deployedService = describedService.services?.[0];
187 expect(deployedService?.taskDefinition).toBe(taskDefinitionArn);
188 expect(deployedService?.desiredCount).toBe(1);
189 expect(deployedService?.runningCount).toBe(1);
190 expect(deployedService?.pendingCount).toBe(0);
191 expect(deployedService?.deployments).toHaveLength(1);
192 expect(deployedService?.deployments?.[0]?.status).toBe("PRIMARY");
193
194 const targetHealth = yield* elbv2.describeTargetHealth({
195 TargetGroupArn: targetGroupArn!,
196 });
197 expect(
198 (targetHealth.TargetHealthDescriptions ?? []).map(
199 (target) => target.TargetHealth?.State,
200 ),
201 ).toEqual(["healthy"]);
202
203 // The ALB has been active for a while now, so its DNS resolves. Probe the
204 // public endpoint (still retry through edge/DNS propagation).
205 const health = yield* HttpClient.get(`${url}/health`).pipe(
206 Effect.flatMap((res) =>
207 res.status === 200
208 ? Effect.succeed(res)
209 : Effect.fail(new Error(`/health returned ${res.status}`)),
210 ),
211 Effect.tapError((error) => Effect.logError(error)),
212 Effect.retry({ schedule: Schedule.spaced("6 seconds"), times: 30 }),
213 );
214 expect(health.status).toBe(200);
215 expect(yield* health.json).toEqual({ ok: true });
216
217 // Prove the ServerHost.run background loop is executing in-container:
218 // the tick counter climbs between two reads.
219 const readTicks = HttpClient.get(`${url}/ticks`).pipe(
220 Effect.flatMap((res) => res.json),
221 Effect.map((body) => (body as { ticks: number }).ticks),
222 );
223 const first = yield* readTicks;
224 yield* Effect.sleep("3 seconds");
225 const second = yield* readTicks;
226 expect(second).toBeGreaterThan(first);
227
228 // Process-mode telemetry: spans from SEPARATE requests must arrive in
229 // one batched traces payload (interval batching on the process root
230 // scope) — a per-request flush would put each span in its own POST
231 // and the pair would never share a payload item. Retry attempts with
232 // fresh tags in case a pair straddles a 5s export window.
233 const fetchCollected = HttpClient.get(collected).pipe(
234 Effect.flatMap((res) => res.json),
235 Effect.map(
236 (body) =>
237 (body as { items: { signal: string; payload: unknown }[] }).items,
238 ),
239 );
240 const attemptBatch = (i: number) =>
241 Effect.gen(function* () {
242 yield* HttpClient.get(`${url}/work?tag=a${i}`);
243 yield* HttpClient.get(`${url}/work?tag=b${i}`);
244 return yield* fetchCollected.pipe(
245 Effect.map((items) =>
246 items.some((item) => {
247 if (item.signal !== "traces") return false;
248 const payload = JSON.stringify(item.payload);
249 return (
250 payload.includes(`work:a${i}`) &&
251 payload.includes(`work:b${i}`)
252 );
253 }),
254 ),
255 Effect.repeat({
256 schedule: Schedule.spaced("3 seconds"),
257 until: (found): boolean => found,
258 times: 8,
259 }),
260 Effect.catchCause(() => Effect.succeed(false)),
261 );
262 });
263 let batched = false;
264 for (let i = 0; i < 3 && !batched; i++) {
265 batched = yield* attemptBatch(i);
266 }
267 expect(batched).toBe(true);
268
269 // Logs from the same requests ship through the process-mode logger.
270 const items = yield* fetchCollected;
271 expect(
272 items.some(
273 (item) =>
274 item.signal === "logs" &&
275 JSON.stringify(item.payload).includes("ecs-work-log:"),
276 ),
277 ).toBe(true);
278 // And the process-mode resource attributes are stamped.
279 expect(JSON.stringify(items)).toContain("otel-ecs-e2e");
280
281 yield* stack.destroy();
282
283 // Sweep the stragglers `stack.destroy()` cannot reach: the INACTIVE
284 // task-definition revisions left by `deregisterTaskDefinition`, plus
285 // anything a partial destroy failure abandoned. This is what makes a
286 // PASSING run leave zero cloud resources behind.
287 yield* reclaimTaskE2EOrphans;
288
289 // Clean-slate proof: a passing run leaves ZERO cloud resources.
290 expect(yield* scanTaskE2EOrphans).toEqual([]);
291 }),
292 {
293 tags: [
294 "provider:aws",
295 "provider:aws:ec2",
296 "provider:aws:ecr",
297 "provider:aws:ecs",
298 "provider:aws:iam",
299 "provider:cloudflare",
300 "provider:cloudflare:worker",
301 "live",
302 ],
303 timeout: 1_200_000,
304 },
305);