1import * as AWS from "@/AWS";
2import { Application } from "@/AWS/EMRServerless";
3import { Role } from "@/AWS/IAM";
4import * as Provider from "@/Provider";
5import * as Test from "@/Test/Alchemy";
6import * as emr from "@distilled.cloud/aws/emr-serverless";
7import { expect } from "alchemy-test";
8import * as Duration from "effect/Duration";
9import * as Effect from "effect/Effect";
10import * as Schedule from "effect/Schedule";
11
12const { test } = Test.make({ providers: AWS.providers() });
13
14// Deterministic names — same on every run of the test case.
15const APP_NAME = "alchemy-test-emrs-app";
16const JOB_APP_NAME = "alchemy-test-emrs-job";
17const RELEASE_LABEL = "emr-7.5.0";
18
19// Ungated typed-error probe: proves the distilled error union carries the
20// not-found tag the read/observe/delete paths depend on.
21test.provider(
22 "typed error semantics on a nonexistent application",
23 () =>
24 Effect.gen(function* () {
25 const error = yield* Effect.flip(
26 emr.getApplication({ applicationId: "00abcdefabcdef01" }),
27 );
28 expect(error._tag).toBe("ResourceNotFoundException");
29 }),
30 { tags: ["provider:aws", "provider:aws:emrserverless", "live"] },
31);
32
33// Full ungated lifecycle: an application in the CREATED state is free (billing
34// only starts when the application starts workers), and creation is bounded
35// (~1 min). create → CREATED, no-op redeploy, in-place update, destroy,
36// verify gone.
37test.provider(
38 "application lifecycle: create, no-op, update, destroy, verify gone",
39 (stack) =>
40 Effect.gen(function* () {
41 // Clean slate in case a previous run died mid-flight.
42 yield* stack.destroy();
43
44 const deployApp = (idleMinutes: number) =>
45 stack.deploy(
46 Effect.gen(function* () {
47 return yield* Application("App", {
48 applicationName: APP_NAME,
49 releaseLabel: RELEASE_LABEL,
50 autoStartConfiguration: { enabled: true },
51 autoStopConfiguration: {
52 enabled: true,
53 idleTimeout: Duration.minutes(idleMinutes),
54 },
55 tags: { purpose: "alchemy-test" },
56 });
57 }),
58 );
59
60 const created = yield* deployApp(15);
61 expect(created.applicationName).toBe(APP_NAME);
62 // the API echoes the engine type in mixed case ("Spark"/"Hive")
63 expect(created.type).toBe("Spark");
64 expect(created.releaseLabel).toBe(RELEASE_LABEL);
65 expect(created.applicationArn).toContain("/applications/");
66 expect(created.state).toBe("CREATED");
67
68 // Out-of-band verification via distilled.
69 const observed = yield* emr.getApplication({
70 applicationId: created.applicationId,
71 });
72 expect(observed.application.state).toBe("CREATED");
73 expect(
74 observed.application.autoStopConfiguration?.idleTimeoutMinutes,
75 ).toBe(15);
76 expect(observed.application.tags?.purpose).toBe("alchemy-test");
77
78 // The provider's list() enumerates it.
79 const provider = yield* Provider.findProvider(Application);
80 const all = yield* provider.list();
81 expect(all.some((a) => a.applicationId === created.applicationId)).toBe(
82 true,
83 );
84
85 // No-op redeploy: same application, no replacement.
86 const noop = yield* deployApp(15);
87 expect(noop.applicationId).toBe(created.applicationId);
88
89 // In-place update: shrink the auto-stop idle timeout.
90 const updated = yield* deployApp(5);
91 expect(updated.applicationId).toBe(created.applicationId);
92 const afterUpdate = yield* emr.getApplication({
93 applicationId: created.applicationId,
94 });
95 expect(
96 afterUpdate.application.autoStopConfiguration?.idleTimeoutMinutes,
97 ).toBe(5);
98
99 // Destroy and verify deletion out-of-band (deleted applications either
100 // disappear or briefly linger as TERMINATED).
101 yield* stack.destroy();
102 yield* assertApplicationGone(created.applicationId);
103 }),
104 {
105 tags: ["provider:aws", "provider:aws:emrserverless", "live"],
106 timeout: 240_000,
107 },
108);
109
110// Live JobRun (SparkPi from the EMR image itself — no S3 assets needed).
111// Starting the application and running the job bills real worker-seconds and
112// takes minutes, so it is gated behind AWS_TEST_SLOW=1.
113test.provider.skipIf(!process.env.AWS_TEST_SLOW)(
114 "live Spark job run reaches SUCCESS (AWS_TEST_SLOW=1)",
115 (stack) =>
116 Effect.gen(function* () {
117 yield* stack.destroy();
118
119 const { app, role } = yield* stack.deploy(
120 Effect.gen(function* () {
121 const role = yield* Role("JobRole", {
122 assumeRolePolicyDocument: {
123 Version: "2012-10-17",
124 Statement: [
125 {
126 Effect: "Allow",
127 Principal: { Service: "emr-serverless.amazonaws.com" },
128 Action: ["sts:AssumeRole"],
129 },
130 ],
131 },
132 });
133 const app = yield* Application("JobApp", {
134 applicationName: JOB_APP_NAME,
135 releaseLabel: RELEASE_LABEL,
136 autoStopConfiguration: {
137 enabled: true,
138 idleTimeout: "1 minute",
139 },
140 });
141 return { app, role };
142 }),
143 );
144
145 const jobToken = yield* Effect.sync(() => crypto.randomUUID());
146 const started = yield* emr.startJobRun({
147 clientToken: jobToken,
148 applicationId: app.applicationId,
149 executionRoleArn: role.roleArn,
150 jobDriver: {
151 sparkSubmit: {
152 entryPoint: "local:///usr/lib/spark/examples/src/main/python/pi.py",
153 sparkSubmitParameters:
154 "--conf spark.executor.cores=1 --conf spark.executor.memory=2g --conf spark.executor.instances=1 --conf spark.driver.cores=1 --conf spark.driver.memory=2g",
155 },
156 },
157 executionTimeoutMinutes: 10,
158 });
159
160 // Poll the job run to a terminal state (app cold-start ~1 min + SparkPi
161 // ~1-2 min; bounded at ~8 min).
162 const finished = yield* emr
163 .getJobRun({
164 applicationId: app.applicationId,
165 jobRunId: started.jobRunId,
166 })
167 .pipe(
168 Effect.map((response) => response.jobRun),
169 Effect.repeat({
170 schedule: Schedule.spaced("10 seconds"),
171 until: (run) =>
172 run.state === "SUCCESS" ||
173 run.state === "FAILED" ||
174 run.state === "CANCELLED",
175 times: 48,
176 }),
177 );
178 expect(finished.stateDetails).toBeDefined();
179 expect(finished.state).toBe("SUCCESS");
180
181 // Destroy immediately (the provider stops the STARTED application
182 // before deleting it) and verify gone.
183 yield* stack.destroy();
184 yield* assertApplicationGone(app.applicationId);
185 }),
186 {
187 tags: [
188 "provider:aws",
189 "provider:aws:emrserverless",
190 "provider:aws:iam",
191 "live",
192 ],
193 timeout: 900_000,
194 },
195);
196
197const assertApplicationGone = (applicationId: string) =>
198 Effect.gen(function* () {
199 const state = yield* emr.getApplication({ applicationId }).pipe(
200 Effect.map((response) => response.application.state),
201 Effect.catchTag("ResourceNotFoundException", () =>
202 Effect.succeed("gone"),
203 ),
204 );
205 if (state !== "gone" && state !== "TERMINATED") {
206 return yield* Effect.fail(
207 new Error(`application ${applicationId} still exists (${state})`),
208 );
209 }
210 }).pipe(
211 Effect.retry({
212 schedule: Schedule.max([
213 Schedule.fixed("5 seconds"),
214 Schedule.recurs(12),
215 ]),
216 }),
217 );