1import * as AWS from "@/AWS";
2import { AWSEnvironment } from "@/AWS/Environment.ts";
3import { Role } from "@/AWS/IAM/Role.ts";
4import { Bucket } from "@/AWS/S3/Bucket.ts";
5import { Cluster } from "@/AWS/SageMaker";
6import * as Test from "@/Test/Alchemy";
7import * as s3 from "@distilled.cloud/aws/s3";
8import * as sagemaker from "@distilled.cloud/aws/sagemaker";
9import { expect } from "alchemy-test";
10import * as Effect from "effect/Effect";
11
12const { test } = Test.make({ providers: AWS.providers() });
13
14// Ungated typed-error probe: prove describeCluster returns the typed
15// ResourceNotFound for a nonexistent HyperPod cluster. Runs in every CI pass.
16test.provider(
17 "describeCluster on a nonexistent cluster fails with ResourceNotFound",
18 () =>
19 Effect.gen(function* () {
20 const error = yield* Effect.flip(
21 sagemaker.describeCluster({
22 ClusterName: "alchemy-nonexistent-hyperpod-cluster-probe",
23 }),
24 );
25 expect(error._tag).toBe("ResourceNotFound");
26 }),
27 { tags: ["provider:aws", "provider:aws:sagemaker", "live"] },
28);
29
30const findCluster = (name: string) =>
31 sagemaker
32 .describeCluster({ ClusterName: name })
33 .pipe(Effect.catchTag("ResourceNotFound", () => Effect.succeed(undefined)));
34
35// A live HyperPod cluster provisions real ML compute and takes 10-25 minutes
36// to reach InService (plus several minutes to delete). Gated behind:
37// AWS_TEST_SAGEMAKER_HYPERPOD=1
38test.provider.skipIf(!process.env.AWS_TEST_SAGEMAKER_HYPERPOD)(
39 "create minimal Slurm HyperPod cluster, wait InService, destroy",
40 (stack) =>
41 Effect.gen(function* () {
42 yield* stack.destroy();
43
44 const { accountId } = yield* AWSEnvironment.current;
45 // The AmazonSageMakerClusterInstanceRolePolicy managed policy only
46 // grants read access to buckets whose name starts with `sagemaker-`.
47 const bucketName = `sagemaker-alchemy-hyperpod-test-${accountId}`;
48
49 const supporting = Effect.gen(function* () {
50 const bucket = yield* Bucket("HyperPodLifecycleBucket", {
51 bucketName,
52 forceDestroy: true,
53 });
54 const role = yield* Role("HyperPodInstanceRole", {
55 assumeRolePolicyDocument: {
56 Version: "2012-10-17",
57 Statement: [
58 {
59 Effect: "Allow",
60 Principal: { Service: "sagemaker.amazonaws.com" },
61 Action: ["sts:AssumeRole"],
62 },
63 ],
64 },
65 managedPolicyArns: [
66 "arn:aws:iam::aws:policy/AmazonSageMakerClusterInstanceRolePolicy",
67 ],
68 });
69 return { bucket, role };
70 });
71
72 // Step 1: bucket + role only — the lifecycle script must exist in S3
73 // before cluster creation starts.
74 const { bucket } = yield* stack.deploy(supporting);
75 yield* s3.putObject({
76 Bucket: bucket.bucketName,
77 Key: "lifecycle/on_create.sh",
78 Body: new TextEncoder().encode(
79 '#!/bin/bash\nset -e\necho "alchemy hyperpod on_create complete"\n',
80 ),
81 ContentType: "text/x-shellscript",
82 });
83
84 // Step 2: add the cluster.
85 const { cluster } = yield* stack.deploy(
86 Effect.gen(function* () {
87 const { role } = yield* supporting;
88 const cluster = yield* Cluster("TestHyperPod", {
89 instanceGroups: {
90 controller: {
91 InstanceType: "ml.t3.medium",
92 InstanceCount: 1,
93 ExecutionRole: role.roleArn,
94 LifeCycleConfig: {
95 // bucketName is the deterministic string computed above —
96 // interpolating bucket.bucketName here would coerce an
97 // unresolved Output.
98 SourceS3Uri: `s3://${bucketName}/lifecycle`,
99 OnCreate: "on_create.sh",
100 },
101 },
102 },
103 nodeRecovery: "None",
104 tags: { purpose: "alchemy-test" },
105 });
106 return { cluster };
107 }),
108 );
109
110 expect(cluster.clusterArn).toContain(":cluster/");
111 expect(cluster.clusterStatus).toBe("InService");
112
113 // Out-of-band verification via distilled.
114 const described = yield* findCluster(cluster.clusterName);
115 expect(described?.ClusterStatus).toBe("InService");
116 expect(
117 described?.InstanceGroups?.map((g) => g.InstanceGroupName),
118 ).toEqual(["controller"]);
119
120 // Destroy and verify gone.
121 yield* stack.destroy();
122 const gone = yield* findCluster(cluster.clusterName);
123 expect(gone).toBeUndefined();
124 }),
125 {
126 tags: [
127 "provider:aws",
128 "provider:aws:iam",
129 "provider:aws:s3",
130 "provider:aws:sagemaker",
131 "live",
132 ],
133 timeout: 3_600_000,
134 },
135);