1import * as AWS from "@/AWS";
2import { Job } from "@/AWS/Glue";
3import { Role } from "@/AWS/IAM";
4import { Bucket } from "@/AWS/S3";
5import * as Output from "@/Output";
6import * as Test from "@/Test/Alchemy";
7import * as glue from "@distilled.cloud/aws/glue";
8import * as s3 from "@distilled.cloud/aws/s3";
9import { expect } from "alchemy-test";
10import * as Effect from "effect/Effect";
11import * as Schedule from "effect/Schedule";
12
13const { test } = Test.make({ providers: AWS.providers() });
14
15const getJob = (name: string) =>
16 glue.getJob({ JobName: name }).pipe(
17 Effect.map((r) => r.Job),
18 Effect.catchTag("EntityNotFoundException", () => Effect.succeed(undefined)),
19 );
20
21const jobRole = () =>
22 Role("GlueJobRole", {
23 assumeRolePolicyDocument: {
24 Version: "2012-10-17",
25 Statement: [
26 {
27 Effect: "Allow",
28 Principal: { Service: "glue.amazonaws.com" },
29 Action: ["sts:AssumeRole"],
30 },
31 ],
32 },
33 managedPolicyArns: [
34 "arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole",
35 "arn:aws:iam::aws:policy/AmazonS3FullAccess",
36 ],
37 });
38
39test.provider(
40 "create, update, delete Glue job definition",
41 (stack) =>
42 Effect.gen(function* () {
43 yield* stack.destroy();
44
45 const created = yield* stack.deploy(
46 Effect.gen(function* () {
47 const bucket = yield* Bucket("JobBucket", { forceDestroy: true });
48 const role = yield* jobRole();
49 const job = yield* Job("Etl", {
50 role: role.roleArn,
51 command: {
52 name: "pythonshell",
53 pythonVersion: "3.9",
54 scriptLocation: Output.interpolate`s3://${bucket.bucketName}/scripts/etl.py`,
55 },
56 maxCapacity: 0.0625,
57 glueVersion: "3.0",
58 defaultArguments: { "--job-language": "python" },
59 tags: { Environment: "test" },
60 });
61 return { bucket, role, job };
62 }),
63 );
64
65 expect(created.job.jobName).toBeDefined();
66 expect(created.job.jobArn).toContain(`:job/${created.job.jobName}`);
67
68 // out-of-band verification
69 const observed = yield* getJob(created.job.jobName);
70 expect(observed?.Name).toEqual(created.job.jobName);
71 expect(observed?.Command?.Name).toEqual("pythonshell");
72 expect(observed?.Command?.ScriptLocation).toEqual(
73 `s3://${created.bucket.bucketName}/scripts/etl.py`,
74 );
75 expect(observed?.GlueVersion).toEqual("3.0");
76 expect(observed?.DefaultArguments?.["--job-language"]).toEqual("python");
77
78 // tags (jobs ARE ARN-taggable)
79 const tags = yield* glue.getTags({ ResourceArn: created.job.jobArn });
80 expect(tags.Tags?.["alchemy::id"]).toBeDefined();
81 expect(tags.Tags?.Environment).toEqual("test");
82
83 // update: change default arguments + timeout
84 yield* stack.deploy(
85 Effect.gen(function* () {
86 const bucket = yield* Bucket("JobBucket", { forceDestroy: true });
87 const role = yield* jobRole();
88 const job = yield* Job("Etl", {
89 role: role.roleArn,
90 command: {
91 name: "pythonshell",
92 pythonVersion: "3.9",
93 scriptLocation: Output.interpolate`s3://${bucket.bucketName}/scripts/etl.py`,
94 },
95 maxCapacity: 0.0625,
96 glueVersion: "3.0",
97 description: "curated ETL",
98 defaultArguments: { "--job-language": "python", "--extra": "1" },
99 timeout: "30 minutes",
100 tags: { Environment: "test" },
101 });
102 return { job };
103 }),
104 );
105
106 const reobserved = yield* getJob(created.job.jobName);
107 expect(reobserved?.Description).toEqual("curated ETL");
108 expect(reobserved?.DefaultArguments?.["--extra"]).toEqual("1");
109 expect(reobserved?.Timeout).toEqual(30);
110
111 yield* stack.destroy();
112 const gone = yield* getJob(created.job.jobName);
113 expect(gone).toBeUndefined();
114 }),
115 {
116 tags: [
117 "provider:aws",
118 "provider:aws:glue",
119 "provider:aws:iam",
120 "provider:aws:s3",
121 "live",
122 ],
123 },
124);
125
126// A live job run is billed and takes ~1-2 minutes — gated behind AWS_TEST_SLOW=1.
127test.provider.skipIf(!process.env.AWS_TEST_SLOW)(
128 "run a live Python shell job to success",
129 (stack) =>
130 Effect.gen(function* () {
131 yield* stack.destroy();
132
133 const deployed = yield* stack.deploy(
134 Effect.gen(function* () {
135 const bucket = yield* Bucket("LiveJobBucket", { forceDestroy: true });
136 const role = yield* jobRole();
137 const job = yield* Job("LiveEtl", {
138 role: role.roleArn,
139 command: {
140 name: "pythonshell",
141 pythonVersion: "3.9",
142 scriptLocation: Output.interpolate`s3://${bucket.bucketName}/scripts/noop.py`,
143 },
144 maxCapacity: 0.0625,
145 glueVersion: "3.0",
146 });
147 return { bucket, job };
148 }),
149 );
150
151 // upload a trivial script the job will execute
152 yield* s3.putObject({
153 Bucket: deployed.bucket.bucketName,
154 Key: "scripts/noop.py",
155 Body: new TextEncoder().encode("print('hello from glue')\n"),
156 });
157
158 // Start a run and poll to a terminal state. A run started seconds after
159 // the IAM role is created can FAIL with "should be given assume role
160 // permissions for Glue Service" while the trust propagates — retry the
161 // whole start+await through that window (bounded).
162 const runOnce = Effect.gen(function* () {
163 const { JobRunId } = yield* glue.startJobRun({
164 JobName: deployed.job.jobName,
165 });
166 const finalRun = yield* glue
167 .getJobRun({ JobName: deployed.job.jobName, RunId: JobRunId! })
168 .pipe(
169 Effect.map((r) => r.JobRun),
170 Effect.repeat({
171 schedule: Schedule.spaced("15 seconds"),
172 until: (run) =>
173 run?.JobRunState === "SUCCEEDED" ||
174 run?.JobRunState === "FAILED" ||
175 run?.JobRunState === "TIMEOUT" ||
176 run?.JobRunState === "ERROR",
177 times: 20,
178 }),
179 );
180 yield* Effect.log(
181 `job run state=${finalRun?.JobRunState} error=${finalRun?.ErrorMessage}`,
182 );
183 if (
184 finalRun?.JobRunState !== "SUCCEEDED" &&
185 finalRun?.ErrorMessage?.includes("assume role permissions")
186 ) {
187 return yield* Effect.fail("role-not-yet-assumable" as const);
188 }
189 return finalRun;
190 });
191
192 const finalRun = yield* runOnce.pipe(
193 Effect.retry({ schedule: Schedule.spaced("30 seconds"), times: 4 }),
194 );
195 expect(finalRun?.JobRunState).toEqual("SUCCEEDED");
196
197 yield* stack.destroy();
198 }),
199 {
200 tags: [
201 "provider:aws",
202 "provider:aws:glue",
203 "provider:aws:iam",
204 "provider:aws:s3",
205 "live",
206 ],
207 timeout: 420_000,
208 },
209);