1import * as firestore from "@distilled.cloud/gcp/firestore_v1";
2import * as run from "@distilled.cloud/gcp/run_v2";
3import * as storage from "@distilled.cloud/gcp/storage_v1";
4import * as Alchemy from "alchemy";
5import * as GCP from "alchemy/GCP";
6import * as Test from "alchemy/Test/Bun";
7import { describe, expect } from "bun:test";
8import { createHash } from "node:crypto";
9import * as Effect from "effect/Effect";
10import * as Layer from "effect/Layer";
11import * as Schedule from "effect/Schedule";
12import * as FetchHttpClient from "effect/http/FetchHttpClient";
13import * as HttpClient from "effect/http/HttpClient";
14import * as HttpClientRequest from "effect/http/HttpClientRequest";
15import { spawnSync } from "node:child_process";
16import Stack from "../alchemy.run.ts";
17
18const { test, beforeAll, afterAll, deploy, destroy } = Test.make({
19 providers: GCP.providers(),
20 state: Alchemy.localState(),
21});
22
23const { getWhenReady } = Test;
24
25// Out-of-band calls to the Google APIs resolve the same stored credentials
26// the deploy uses, so the test runs against the configured profile.
27const GcpHttp = Layer.mergeAll(
28 GCP.GcpAuth,
29 GCP.fromAuthProvider(),
30 FetchHttpClient.layer,
31);
32
33// The project comes from the same credential the deploy uses.
34const currentProject = GCP.GcpEnvironment.current.pipe(
35 Effect.map((env) => env.project),
36 Effect.provide(GCP.fromCredentials().pipe(Layer.provide(GcpHttp))),
37);
38
39// Both services are built from `main`, which needs a local image build.
40const dockerAvailable =
41 spawnSync("docker", ["info"], { stdio: "ignore", timeout: 15_000 }).status ===
42 0;
43
44// Kept for the post-destroy checks in `afterAll`.
45let deployed:
46 | {
47 url: string | undefined;
48 indexerUrl: string | undefined;
49 location: string;
50 bucketName: string;
51 databaseName: string;
52 }
53 | undefined;
54
55// Deploy, tests and destroy all sit behind the same Docker guard.
56describe.skipIf(!dockerAvailable)("gcp-storage-uploads", () => {
57 const stack = beforeAll(
58 deploy(Stack).pipe(
59 Effect.tap((outputs) =>
60 Effect.sync(() => {
61 deployed = outputs;
62 }),
63 ),
64 ),
65 { timeout: 900_000 },
66 );
67
68 const gone = <A, E extends { _tag: string }, R>(
69 get: Effect.Effect<A, E, R>,
70 ) =>
71 get.pipe(
72 Effect.as("found" as const),
73 Effect.catchIf(
74 (error) => error._tag === "NotFound",
75 () => Effect.succeed("gone" as const),
76 ),
77 );
78
79 afterAll.skipIf(!!process.env.NO_DESTROY)(
80 Effect.gen(function* () {
81 yield* destroy(Stack);
82 if (deployed === undefined) return;
83 const { bucketName, databaseName, url, indexerUrl, location } = deployed;
84
85 // Destroy leaves nothing behind: bucket (and its notification),
86 // database, and both services.
87 expect(yield* gone(storage.getBuckets({ bucket: bucketName }))).toEqual(
88 "gone",
89 );
90 // A deleted database may linger briefly with `deleteTime` set.
91 const database = yield* firestore
92 .getProjectsDatabases({ name: databaseName })
93 .pipe(
94 Effect.map((database) =>
95 database.deleteTime === undefined ? "found" : "gone",
96 ),
97 Effect.catchTag("NotFound", () => Effect.succeed("gone" as const)),
98 );
99 expect(database).toEqual("gone");
100 const project = yield* currentProject;
101 const services = yield* run.listProjectsLocationsServices({
102 parent: `projects/${project}/locations/${location}`,
103 });
104 const uris = (services.services ?? []).flatMap((service) => [
105 service.uri,
106 ...(service.urls ?? []),
107 ]);
108 expect(uris).not.toContain(url);
109 expect(uris).not.toContain(indexerUrl);
110 }).pipe(Effect.provide(GcpHttp)),
111 { timeout: 600_000 },
112 );
113
114 const baseUrlOf = (url: string | undefined) => {
115 if (url === undefined) throw new Error("the service has no URL");
116 return url.replace(/\/+$/, "");
117 };
118
119 const sha256 = (bytes: Uint8Array) =>
120 createHash("sha256").update(bytes).digest("hex");
121
122 const put = (baseUrl: string, name: string, body: Uint8Array, type: string) =>
123 HttpClient.execute(
124 HttpClientRequest.put(`${baseUrl}/files/${name}`).pipe(
125 HttpClientRequest.bodyUint8Array(body, type),
126 ),
127 );
128
129 interface FileRecord {
130 name: string;
131 object: string;
132 generation: string;
133 size: number;
134 contentType: string;
135 sha256: string;
136 indexedAt: string;
137 }
138
139 const listFiles = (baseUrl: string) =>
140 Effect.gen(function* () {
141 const res = yield* HttpClient.execute(
142 HttpClientRequest.get(`${baseUrl}/files`),
143 );
144 if (res.status !== 200) return [] as FileRecord[];
145 return ((yield* res.json) as unknown as { files: FileRecord[] }).files;
146 });
147
148 test(
149 "serves the health route and rejects bad names",
150 Effect.gen(function* () {
151 const { url } = yield* stack;
152 expect(url).toMatch(/^https:\/\//);
153 const baseUrl = baseUrlOf(url);
154 const health = yield* getWhenReady(`${baseUrl}/`);
155 expect(health.status).toBe(200);
156
157 const bad = yield* put(
158 baseUrl,
159 encodeURIComponent("../escape"),
160 new TextEncoder().encode("nope"),
161 "text/plain",
162 );
163 expect(bad.status).toBe(400);
164
165 const missing = yield* HttpClient.execute(
166 HttpClientRequest.get(`${baseUrl}/files/never-uploaded.txt`),
167 );
168 expect(missing.status).toBe(404);
169 }),
170 { timeout: 120_000 },
171 );
172
173 test(
174 "uploads are stored, indexed with their sha256, and deleted",
175 Effect.gen(function* () {
176 const { url, bucketName, databaseName } = yield* stack;
177 const baseUrl = baseUrlOf(url);
178 yield* getWhenReady(`${baseUrl}/`);
179
180 const files = [
181 {
182 name: "hello.txt",
183 body: new TextEncoder().encode("Hello, uploads!\n"),
184 type: "text/plain",
185 },
186 {
187 name: "random.bin",
188 body: crypto.getRandomValues(new Uint8Array(64 * 1024)),
189 type: "application/octet-stream",
190 },
191 ];
192
193 for (const file of files) {
194 const res = yield* put(baseUrl, file.name, file.body, file.type);
195 expect(res.status).toBe(202);
196 expect(yield* res.json).toMatchObject({
197 name: file.name,
198 object: `uploads/${file.name}`,
199 size: file.body.byteLength,
200 });
201
202 // The bytes read back through the API are the bytes uploaded.
203 const back = yield* HttpClient.execute(
204 HttpClientRequest.get(`${baseUrl}/files/${file.name}`),
205 );
206 expect(back.status).toBe(200);
207 expect(back.headers["content-type"]).toContain(file.type);
208 expect(new Uint8Array(yield* back.arrayBuffer)).toEqual(file.body);
209 }
210
211 // The indexer runs off the bucket notification. A fresh deploy's
212 // project-level Firestore grants can take minutes to propagate; until
213 // then the push fails and Pub/Sub redelivers.
214 const indexed = yield* listFiles(baseUrl).pipe(
215 Effect.repeat({
216 schedule: Schedule.spaced("10 seconds"),
217 until: (records) =>
218 files.every((file) =>
219 records.some((record) => record.name === file.name),
220 ),
221 times: 42,
222 }),
223 );
224
225 for (const file of files) {
226 const record = indexed.find((record) => record.name === file.name)!;
227 expect(record).toMatchObject({
228 object: `uploads/${file.name}`,
229 size: file.body.byteLength,
230 sha256: sha256(file.body),
231 });
232 expect(record.contentType).toContain(file.type);
233 expect(record.generation).toMatch(/^\d+$/);
234
235 // The record is a real Firestore document.
236 const document = yield* firestore
237 .getProjectsDatabasesDocuments({
238 name: `${databaseName}/documents/files/${file.name}`,
239 })
240 .pipe(Effect.provide(GcpHttp));
241 expect(document.fields?.sha256?.stringValue).toEqual(sha256(file.body));
242
243 // And the generation it indexed is the live object's.
244 const object = yield* storage
245 .getObjects({ bucket: bucketName, object: `uploads/${file.name}` })
246 .pipe(Effect.provide(GcpHttp));
247 expect(String(object.generation)).toEqual(record.generation);
248 }
249
250 const [deleted] = files;
251 const del = yield* HttpClient.execute(
252 HttpClientRequest.delete(`${baseUrl}/files/${deleted!.name}`),
253 );
254 expect(del.status).toBe(204);
255
256 const after = yield* HttpClient.execute(
257 HttpClientRequest.get(`${baseUrl}/files/${deleted!.name}`),
258 );
259 expect(after.status).toBe(404);
260 const remaining = yield* listFiles(baseUrl);
261 expect(remaining.map((record) => record.name)).toEqual(["random.bin"]);
262
263 expect(
264 yield* gone(
265 storage
266 .getObjects({
267 bucket: bucketName,
268 object: `uploads/${deleted!.name}`,
269 })
270 .pipe(Effect.provide(GcpHttp)),
271 ),
272 ).toEqual("gone");
273 }),
274 { timeout: 600_000 },
275 );
276});