1import * as compute from "@distilled.cloud/gcp/compute_v1";
2import * as container from "@distilled.cloud/gcp/container_v1";
3import * as resourcemanager from "@distilled.cloud/gcp/cloudresourcemanager_v3";
4import * as firestore from "@distilled.cloud/gcp/firestore_v1";
5import * as Alchemy from "alchemy";
6import * as GCP from "alchemy/GCP";
7import * as Kubernetes from "alchemy/Kubernetes";
8import * as Test from "alchemy/Test/Bun";
9import { describe, expect } from "bun:test";
10import * as Data from "effect/Data";
11import * as Effect from "effect/Effect";
12import * as Layer from "effect/Layer";
13import * as Schedule from "effect/Schedule";
14import * as FetchHttpClient from "effect/http/FetchHttpClient";
15import * as HttpClient from "effect/http/HttpClient";
16import * as HttpClientRequest from "effect/http/HttpClientRequest";
17import { spawnSync } from "node:child_process";
18import Stack from "../alchemy.run.ts";
19import { seedEntries } from "../src/SeedJob.ts";
20
21const { test, beforeAll, afterAll, deploy, destroy } = Test.make({
22 providers: Layer.mergeAll(GCP.providers(), Kubernetes.providers()),
23 state: Alchemy.localState(),
24});
25
26// Out-of-band calls to the Google APIs resolve the same stored credentials
27// the deploy uses, so the test runs against the configured profile.
28const GcpHttp = Layer.mergeAll(
29 GCP.GcpAuth,
30 GCP.fromAuthProvider(),
31 FetchHttpClient.layer,
32);
33
34// `Api` and `SeedJob` are built from `main`, and `Web`'s image is mirrored
35// into Artifact Registry — both need a local Docker daemon.
36const dockerAvailable =
37 spawnSync("docker", ["info"], { stdio: "ignore", timeout: 15_000 }).status ===
38 0;
39
40// The project comes from the same credential the deploy uses.
41const currentProject = GCP.GcpEnvironment.current.pipe(
42 Effect.map((env) => env.project),
43 Effect.provide(GCP.fromCredentials().pipe(Layer.provide(GcpHttp))),
44);
45
46// Deploy, tests and destroy all sit behind the same Docker guard.
47describe.skipIf(!dockerAvailable)("gcp-gke", () => {
48 // Autopilot cluster creation dominates the deploy (~5–10 min).
49 const stack = beforeAll(deploy(Stack), { timeout: 1_800_000 });
50
51 const found = Effect.as("found" as const);
52 const gone = () => Effect.succeed("gone" as const);
53
54 /** Poll until a deleted resource reads as gone. */
55 const expectGone = <E, R>(
56 what: string,
57 status: Effect.Effect<"found" | "gone", E, R>,
58 ) =>
59 status.pipe(
60 Effect.repeat({
61 schedule: Schedule.spaced("5 seconds"),
62 until: (status) => status === "gone",
63 times: 12,
64 }),
65 Effect.tap((status) =>
66 Effect.sync(() => expect(`${what}: ${status}`).toBe(`${what}: gone`)),
67 ),
68 );
69
70 const projectNumber = (project: string) =>
71 resourcemanager
72 .getProjects({ name: `projects/${project}` })
73 .pipe(Effect.map((p) => (p.name ?? "").split("/").pop() ?? ""));
74
75 /** The Workload Identity Federation principal of a Kubernetes ServiceAccount. */
76 const ksaPrincipal = (
77 project: string,
78 number: string,
79 namespace: string,
80 ksa: string,
81 ) =>
82 `principal://iam.googleapis.com/projects/${number}/locations/global/` +
83 `workloadIdentityPools/${project}.svc.id.goog/subject/ns/${namespace}/sa/${ksa}`;
84
85 /** Project IAM bindings (v3, with conditions) that include `member`. */
86 const projectBindingsOf = (member: string) =>
87 currentProject.pipe(
88 Effect.flatMap((project) =>
89 resourcemanager.getIamPolicyProjects({
90 resource: `projects/${project}`,
91 body: { options: { requestedPolicyVersion: 3 } },
92 }),
93 ),
94 Effect.map((policy) =>
95 (policy.bindings ?? []).filter((binding) =>
96 (binding.members ?? []).includes(member),
97 ),
98 ),
99 );
100
101 /** Forwarding rules GKE created for Services in the `guestbook` namespace. */
102 const guestbookForwardingRules = currentProject.pipe(
103 Effect.flatMap((project) =>
104 compute.listForwardingRules({ project, region: "us-central1" }),
105 ),
106 Effect.map((page) =>
107 (page.items ?? []).filter((rule) =>
108 (rule.description ?? "").includes(
109 '"kubernetes.io/service-name":"guestbook/',
110 ),
111 ),
112 ),
113 );
114
115 class NotReady extends Data.TaggedError("NotReady")<{ detail: string }> {}
116
117 /**
118 * Re-run `attempt` every 10s until it succeeds with a value `ready`
119 * accepts. Transport errors count as not ready: a fresh load balancer
120 * refuses connections until its backends are healthy.
121 */
122 const until = <A, E, R>(
123 attempt: Effect.Effect<A, E, R>,
124 ready: (value: A) => boolean,
125 times: number,
126 ) =>
127 attempt.pipe(
128 Effect.filterOrFail(
129 ready,
130 (value) => new NotReady({ detail: JSON.stringify(value) }),
131 ),
132 Effect.retry({ schedule: Schedule.spaced("10 seconds"), times }),
133 );
134
135 const get = (url: string) =>
136 HttpClient.execute(HttpClientRequest.get(url)).pipe(
137 Effect.flatMap((response) =>
138 response.text.pipe(
139 Effect.map((text) => ({ status: response.status, text })),
140 ),
141 ),
142 );
143
144 /** The principals of the stack's two workloads. */
145 const principalsOf = (outputs: {
146 namespace: string;
147 apiServiceAccount: string;
148 seedJobServiceAccount: string;
149 }) =>
150 Effect.gen(function* () {
151 const project = yield* currentProject;
152 const number = yield* projectNumber(project);
153 return [
154 ksaPrincipal(
155 project,
156 number,
157 outputs.namespace,
158 outputs.apiServiceAccount,
159 ),
160 ksaPrincipal(
161 project,
162 number,
163 outputs.namespace,
164 outputs.seedJobServiceAccount,
165 ),
166 ];
167 }).pipe(Effect.provide(GcpHttp));
168
169 afterAll.skipIf(!!process.env.NO_DESTROY)(
170 Effect.gen(function* () {
171 const outputs = yield* stack;
172 const principals =
173 outputs === undefined ? [] : yield* principalsOf(outputs);
174 yield* destroy(Stack);
175 if (outputs === undefined) return;
176
177 // Every grant is revoked from the workloads' principals.
178 for (const principal of principals) {
179 const bindings = yield* projectBindingsOf(principal).pipe(
180 Effect.provide(GcpHttp),
181 );
182 expect(bindings).toEqual([]);
183 }
184 // The LoadBalancer Services were drained before the cluster went, so
185 // no load balancer leaked.
186 expect(
187 yield* guestbookForwardingRules.pipe(Effect.provide(GcpHttp)),
188 ).toEqual([]);
189 yield* expectGone(
190 "cluster",
191 container
192 .getProjectsLocationsClusters({ name: outputs.clusterName })
193 .pipe(
194 found,
195 Effect.catchTag("NotFound", gone),
196 Effect.provide(GcpHttp),
197 ),
198 );
199 yield* expectGone(
200 "database",
201 firestore
202 .getProjectsDatabases({ name: outputs.databaseName })
203 .pipe(
204 found,
205 Effect.catchTag("NotFound", gone),
206 Effect.provide(GcpHttp),
207 ),
208 );
209 }),
210 { timeout: 1_800_000 },
211 );
212
213 const baseUrlOf = (url: string | undefined) => {
214 if (url === undefined) throw new Error("the service has no URL");
215 return url.replace(/\/+$/, "");
216 };
217
218 interface Entry {
219 id: string;
220 author: string;
221 message: string;
222 }
223
224 test(
225 "grants each binding to its Kubernetes ServiceAccount principal",
226 Effect.gen(function* () {
227 const outputs = yield* stack;
228 for (const principal of yield* principalsOf(outputs)) {
229 const bindings = yield* projectBindingsOf(principal).pipe(
230 Effect.provide(GcpHttp),
231 );
232 // `roles/datastore.user` on the project, conditioned to the
233 // guestbook database only.
234 expect(bindings).toHaveLength(1);
235 expect(bindings[0]!.role).toEqual("roles/datastore.user");
236 expect(bindings[0]!.condition?.expression).toContain(
237 `resource.name == "${outputs.databaseName}"`,
238 );
239 }
240 }),
241 { timeout: 120_000 },
242 );
243
244 test(
245 "the SeedJob seeded the guestbook through Workload Identity",
246 Effect.gen(function* () {
247 const { apiUrl } = yield* stack;
248 const baseUrl = baseUrlOf(apiUrl);
249 expect(baseUrl).toMatch(/^http:\/\/[^/]+:3000$/);
250
251 // Autopilot provisions nodes for the Job's pod on demand, and a fresh
252 // Firestore grant takes a few minutes to propagate; the Job retries
253 // with backoff meanwhile.
254 const listed = yield* until(
255 get(`${baseUrl}/entries`).pipe(
256 Effect.map((response) =>
257 response.status === 200
258 ? (JSON.parse(response.text) as { entries: Entry[] })
259 : { entries: [] as Entry[] },
260 ),
261 ),
262 (body) =>
263 seedEntries.every((seed) =>
264 body.entries.some((entry) => entry.id === seed.id),
265 ),
266 60,
267 );
268 for (const seed of seedEntries) {
269 expect(listed.entries).toContainEqual({
270 id: seed.id,
271 author: seed.id,
272 message: seed.message,
273 });
274 }
275
276 const ada = yield* get(`${baseUrl}/entries/ada`);
277 expect(ada.status).toBe(200);
278 expect(JSON.parse(ada.text)).toMatchObject({ id: "ada", author: "ada" });
279 }),
280 { timeout: 720_000 },
281 );
282
283 test(
284 "signs the guestbook and reads the entry back",
285 Effect.gen(function* () {
286 const { apiUrl, databaseName } = yield* stack;
287 const baseUrl = baseUrlOf(apiUrl);
288
289 // Until the Firestore grant has propagated the API answers 500.
290 const created = yield* until(
291 HttpClient.execute(
292 HttpClientRequest.post(
293 `${baseUrl}/entries?author=integ&message=hello%20from%20gke`,
294 ),
295 ).pipe(
296 Effect.flatMap((response) =>
297 response.text.pipe(
298 Effect.map((text) => ({ status: response.status, text })),
299 ),
300 ),
301 ),
302 (response) => response.status !== 500,
303 42,
304 );
305 expect(created.status).toBe(200);
306 const entry = JSON.parse(created.text) as Entry;
307 expect(entry).toMatchObject({
308 author: "integ",
309 message: "hello from gke",
310 });
311
312 const read = yield* get(`${baseUrl}/entries/${entry.id}`);
313 expect(read.status).toBe(200);
314 expect(JSON.parse(read.text)).toEqual(entry);
315
316 // The write is a real Firestore document.
317 const document = yield* firestore
318 .getProjectsDatabasesDocuments({
319 name: `${databaseName}/documents/entries/${entry.id}`,
320 })
321 .pipe(Effect.orDie, Effect.provide(GcpHttp));
322 expect(document.fields?.message?.stringValue).toEqual("hello from gke");
323
324 const missing = yield* get(`${baseUrl}/entries/nope-nope`);
325 expect(missing.status).toBe(404);
326 }),
327 { timeout: 600_000 },
328 );
329
330 test(
331 "serves the external nginx Web deployment",
332 Effect.gen(function* () {
333 const { webUrl } = yield* stack;
334 const baseUrl = baseUrlOf(webUrl);
335 expect(baseUrl).toMatch(/^http:\/\/[^/:]+$/);
336 const res = yield* until(get(`${baseUrl}/`), (r) => r.status === 200, 24);
337 expect(res.text).toContain("nginx");
338 }),
339 { timeout: 300_000 },
340 );
341});