1import * as GCP from "@/GCP";
2import * as Test from "@/Test/Alchemy";
3import * as dataproc from "@distilled.cloud/gcp/dataproc_v1";
4import { expect } from "alchemy-test";
5import * as Effect from "effect/Effect";
6import { MinimumLogLevel } from "effect/References";
7import * as Schedule from "effect/Schedule";
8import { GcpEnvironment } from "@/GCP/Environment";
9
10const { test } = Test.make({ providers: GCP.providers() });
11
12const logLevel = Effect.provideService(
13 MinimumLogLevel,
14 process.env.DEBUG ? "Debug" : "Info",
15);
16
17// Cluster create + update + delete takes ~8 minutes (observed 475s).
18const runLifecycle = !!process.env.GCP_TEST_SLOW && !process.env.FAST;
19
20const waitUntilGone = (
21 projectId: string,
22 region: string,
23 clusterName: string,
24) =>
25 dataproc.getProjectsRegionsClusters({ projectId, region, clusterName }).pipe(
26 Effect.as("found" as const),
27 Effect.catchTag("NotFound", () => Effect.succeed("gone" as const)),
28 Effect.repeat({
29 schedule: Schedule.spaced("3 seconds"),
30 until: (status) => status === "gone",
31 times: 10,
32 }),
33 );
34
35test.provider(
36 "getProjectsRegionsClusters on a missing cluster fails with NotFound",
37 (stack) =>
38 Effect.gen(function* () {
39 const { project } = yield* GcpEnvironment.current;
40 yield* stack.destroy();
41
42 const error = yield* Effect.flip(
43 dataproc.getProjectsRegionsClusters({
44 projectId: project,
45 region: "us-central1",
46 clusterName: "alchemy-dataproc-missing",
47 }),
48 );
49 expect(error._tag).toEqual("NotFound");
50
51 yield* stack.destroy();
52 }).pipe(logLevel),
53 { tags: ["provider:gcp", "provider:gcp:dataproc", "live"], timeout: 90_000 },
54);
55
56test.provider.skipIf(!runLifecycle)(
57 "create, update, and delete a dataproc cluster",
58 (stack) =>
59 Effect.gen(function* () {
60 yield* stack.destroy();
61
62 const created = yield* stack.deploy(
63 Effect.gen(function* () {
64 return yield* GCP.Dataproc.Cluster("Spark", {
65 region: "us-central1",
66 clusterType: "SINGLE_NODE",
67 masterMachineType: "e2-standard-2",
68 masterBootDiskSizeGb: 30,
69 labels: { env: "test" },
70 });
71 }),
72 );
73
74 expect(created.name).toContain("/clusters/");
75 expect(created.clusterName).toEqual(expect.any(String));
76 expect(created.region).toEqual("us-central1");
77 expect(created.clusterType).toEqual("SINGLE_NODE");
78 expect(created.labels).toMatchObject({ env: "test" });
79 expect(created.state).toEqual("RUNNING");
80
81 const fetched = yield* dataproc.getProjectsRegionsClusters({
82 projectId: created.project,
83 region: created.region,
84 clusterName: created.clusterName,
85 });
86 expect(fetched.clusterName).toEqual(created.clusterName);
87 expect(fetched.labels?.env).toEqual("test");
88 expect(fetched.status?.state).toEqual("RUNNING");
89
90 const updated = yield* stack.deploy(
91 Effect.gen(function* () {
92 return yield* GCP.Dataproc.Cluster("Spark", {
93 clusterName: created.clusterName,
94 region: "us-central1",
95 clusterType: "SINGLE_NODE",
96 masterMachineType: "e2-standard-2",
97 masterBootDiskSizeGb: 30,
98 labels: { env: "prod", role: "spark" },
99 });
100 }),
101 );
102
103 expect(updated.name).toEqual(created.name);
104 expect(updated.labels).toMatchObject({ env: "prod", role: "spark" });
105
106 const refetched = yield* dataproc.getProjectsRegionsClusters({
107 projectId: created.project,
108 region: created.region,
109 clusterName: created.clusterName,
110 });
111 expect(refetched.labels?.env).toEqual("prod");
112 expect(refetched.labels?.role).toEqual("spark");
113
114 yield* stack.destroy();
115
116 const gone = yield* waitUntilGone(
117 created.project,
118 created.region,
119 created.clusterName,
120 );
121 expect(gone).toEqual("gone");
122 }).pipe(logLevel),
123 {
124 tags: ["provider:gcp", "provider:gcp:dataproc", "live"],
125 timeout: 1_800_000,
126 },
127);