1import * as storage from "@distilled.cloud/gcp/storage_v1";
2import * as Data from "effect/Data";
3import * as Effect from "effect/Effect";
4import * as Schedule from "effect/Schedule";
5import * as Stream from "effect/Stream";
6import { Unowned } from "../../AdoptPolicy.ts";
7import { isResolved } from "../../Diff.ts";
8import { createPhysicalName } from "../../PhysicalName.ts";
9import * as Provider from "../../Provider.ts";
10import { Resource } from "../../Resource.ts";
11import { tagRecord } from "../../Tags.ts";
12import { GcpEnvironment } from "../Environment.ts";
13import {
14 createInternalLabels,
15 diffLabels,
16 hasAlchemyLabels,
17 stripInternalLabels,
18 toLabels,
19} from "../Labels.ts";
20import type { Providers } from "../Providers.ts";
21
22export type BucketProps = {
23 /**
24 * Globally-unique bucket name. If omitted, a unique name is generated from
25 * the stack, stage, and logical id. Must be 3-63 characters, lowercase,
26 * and DNS compatible.
27 */
28 bucketName?: string;
29 /**
30 * Location of the bucket (`US`, `EU`, `US-CENTRAL1`, …). Immutable —
31 * changing it replaces the bucket.
32 * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`)
33 */
34 location?: string;
35 /**
36 * Default storage class for new objects.
37 * @default "STANDARD"
38 */
39 storageClass?: string;
40 /**
41 * Whether object versioning is enabled.
42 * @default false
43 */
44 versioning?: boolean;
45 /**
46 * User labels. Alchemy ownership labels are merged in automatically.
47 */
48 labels?: Record<string, string>;
49 /**
50 * Delete all objects (and versions) before destroying the bucket.
51 * @default false
52 */
53 forceDestroy?: boolean;
54 /**
55 * Enable uniform bucket-level access. Required for managed folders.
56 * Hierarchical namespace implies this is enabled. Fine-grained ACLs
57 * cannot be used when this is true.
58 * @default false
59 */
60 uniformBucketLevelAccess?: boolean;
61 /**
62 * Enable hierarchical namespace (first-class folders). Immutable —
63 * must be set at create time; changing it replaces the bucket.
64 * Implies uniform bucket-level access.
65 * @default false
66 */
67 hierarchicalNamespace?: boolean;
68 /**
69 * Static website configuration. Cloud Storage applies it when the bucket
70 * is served through a CNAME or an external HTTPS load balancer (backend
71 * bucket) — not on `storage.googleapis.com/{bucket}/…` path-style URLs.
72 * Omitting it removes the configuration.
73 */
74 website?: BucketWebsite;
75};
76
77export interface BucketWebsite {
78 /**
79 * Object suffix appended to directory-style requests, e.g. `index.html`
80 * so `/docs/` serves `docs/index.html`.
81 */
82 mainPageSuffix?: string;
83 /** Object served (with status 404) when a requested object is missing. */
84 notFoundPage?: string;
85}
86
87export type Bucket = Resource<
88 "GCP.Storage.Bucket",
89 BucketProps,
90 {
91 /** Globally unique bucket name. */
92 bucketName: string;
93 /** Location of the bucket. */
94 location: string;
95 /** Location type (`region`, `multi-region`, `dual-region`). */
96 locationType: string | undefined;
97 /** Default storage class. */
98 storageClass: string | undefined;
99 /** Whether object versioning is enabled. */
100 versioning: boolean;
101 /** User labels (Alchemy ownership labels stripped). */
102 labels: Record<string, string>;
103 /** GCS self-link. */
104 selfLink: string | undefined;
105 /** Numeric project that owns the bucket. */
106 projectNumber: string | undefined;
107 /** RFC3339 creation timestamp. */
108 timeCreated: string | undefined;
109 /** Whether uniform bucket-level access is enabled. */
110 uniformBucketLevelAccess: boolean;
111 /** Whether hierarchical namespace is enabled. */
112 hierarchicalNamespace: boolean;
113 /** Static website configuration, if set. */
114 website: BucketWebsite | undefined;
115 },
116 never,
117 Providers
118>;
119
120/**
121 * A Google Cloud Storage bucket.
122 *
123 * ### Creating a Bucket
124 * **Example:** Generated name
125 * ```typescript
126 * const bucket = yield* GCP.Storage.Bucket("assets", {
127 * forceDestroy: true,
128 * });
129 * ```
130 *
131 * **Example:** Explicit name, location, and labels
132 * ```typescript
133 * const bucket = yield* GCP.Storage.Bucket("assets", {
134 * bucketName: "my-app-assets",
135 * location: "US-CENTRAL1",
136 * storageClass: "STANDARD",
137 * versioning: true,
138 * labels: { env: "prod" },
139 * forceDestroy: true,
140 * });
141 * ```
142 *
143 * **Example:** Hierarchical namespace (folders)
144 * ```typescript
145 * const bucket = yield* GCP.Storage.Bucket("tree", {
146 * hierarchicalNamespace: true,
147 * uniformBucketLevelAccess: true,
148 * forceDestroy: true,
149 * });
150 * ```
151 *
152 * **Example:** Static website bucket with public read
153 * ```typescript
154 * const site = yield* GCP.Storage.Bucket("site", {
155 * uniformBucketLevelAccess: true,
156 * website: { mainPageSuffix: "index.html", notFoundPage: "404.html" },
157 * forceDestroy: true,
158 * });
159 * yield* GCP.IAM.Member("PublicRead", {
160 * kind: "storage.bucket",
161 * name: site.bucketName,
162 * role: "roles/storage.objectViewer",
163 * member: "allUsers",
164 * });
165 * ```
166 *
167 * ### Updating a Bucket
168 * **Example:** Change labels
169 * ```typescript
170 * const bucket = yield* GCP.Storage.Bucket("assets", {
171 * forceDestroy: true,
172 * labels: { env: "prod" },
173 * });
174 * ```
175 *
176 * ### Binding from a Function
177 * **Example:** PutObject from Cloud Run
178 * ```typescript
179 * export class Api extends GCP.Function<Api>()(
180 * "Api",
181 * { main: import.meta.url },
182 * Effect.gen(function* () {
183 * const bucket = yield* GCP.Storage.Bucket("assets", { forceDestroy: true });
184 * const putObject = yield* GCP.Storage.PutObject(bucket);
185 * return {
186 * fetch: Effect.gen(function* () {
187 * yield* putObject({ name: "hello.txt", body: "Hello!" }).pipe(
188 * Effect.orDie,
189 * );
190 * return HttpServerResponse.text("ok");
191 * }),
192 * };
193 * }).pipe(Effect.provide([GCP.Storage.PutObjectHttp])),
194 * ) {}
195 * ```
196 *
197 * ### Destroying a Bucket
198 * **Example:** Empty then delete
199 * ```typescript
200 * const bucket = yield* GCP.Storage.Bucket("assets", {
201 * forceDestroy: true,
202 * });
203 * ```
204 * `forceDestroy: true` deletes objects before the bucket. `alchemy destroy` waits until the bucket is gone.
205 *
206 * @resource
207 * @category Storage
208 */
209export const Bucket = Resource<Bucket>("GCP.Storage.Bucket");
210
211export class BucketNotResolved extends Data.TaggedError(
212 "GCP.Storage.BucketNotResolved",
213)<{
214 bucketName: string;
215}> {}
216
217const userLabels = (
218 labels: Record<string, string | undefined> | null | undefined,
219): Record<string, string> => stripInternalLabels(tagRecord(labels));
220
221const toName = (id: string, name: string | undefined, existing?: string) =>
222 Effect.gen(function* () {
223 return (
224 name ??
225 existing ??
226 (yield* createPhysicalName({
227 id,
228 maxLength: 63,
229 lowercase: true,
230 }))
231 );
232 });
233
234const toAttrs = (bucket: storage.Bucket) => ({
235 bucketName: bucket.name ?? bucket.id ?? "",
236 location: bucket.location ?? "",
237 locationType: bucket.locationType,
238 storageClass: bucket.storageClass,
239 versioning: bucket.versioning?.enabled === true,
240 labels: userLabels(bucket.labels),
241 selfLink: bucket.selfLink,
242 projectNumber: bucket.projectNumber,
243 timeCreated: bucket.timeCreated,
244 uniformBucketLevelAccess:
245 bucket.iamConfiguration?.uniformBucketLevelAccess?.enabled === true,
246 hierarchicalNamespace: bucket.hierarchicalNamespace?.enabled === true,
247 website: toWebsite(bucket.website),
248});
249
250const toWebsite = (
251 website: storage.BucketWebsite | undefined,
252): BucketWebsite | undefined =>
253 website === undefined ||
254 (website.mainPageSuffix === undefined && website.notFoundPage === undefined)
255 ? undefined
256 : {
257 mainPageSuffix: website.mainPageSuffix,
258 notFoundPage: website.notFoundPage,
259 };
260
261const getByName = (bucketName: string) =>
262 storage
263 .getBuckets({ bucket: bucketName, projection: "full" })
264 .pipe(Effect.catchTag("NotFound", () => Effect.succeed(undefined)));
265
266const emptyBucket = (bucketName: string) =>
267 storage.listObjects
268 .items({ bucket: bucketName, versions: true, maxResults: 1000 })
269 .pipe(
270 Stream.runCollect,
271 Effect.flatMap((chunk) =>
272 Effect.forEach(
273 chunk,
274 (object) =>
275 object.name
276 ? storage
277 .deleteObjects({
278 bucket: bucketName,
279 object: object.name,
280 generation: object.generation,
281 })
282 .pipe(Effect.catchTag("NotFound", () => Effect.void))
283 : Effect.void,
284 { concurrency: 8 },
285 ),
286 ),
287 Effect.catchTag("NotFound", () => Effect.void),
288 );
289
290const emptyFolders = (bucketName: string) =>
291 storage.listFolders.items({ bucket: bucketName, pageSize: 1000 }).pipe(
292 Stream.runCollect,
293 Effect.flatMap((chunk) => {
294 const folders = Array.from(chunk)
295 .map((folder) => folder.name)
296 .filter((name): name is string => !!name)
297 .sort((left, right) => right.length - left.length);
298 return Effect.forEach(
299 folders,
300 (folder) =>
301 storage
302 .deleteFolders({ bucket: bucketName, folder })
303 .pipe(Effect.catchTag("NotFound", () => Effect.void)),
304 { concurrency: 1 },
305 );
306 }),
307 Effect.catchTag("NotFound", () => Effect.void),
308 );
309
310const emptyManagedFolders = (bucketName: string) =>
311 storage.listManagedFolders.items({ bucket: bucketName, pageSize: 1000 }).pipe(
312 Stream.runCollect,
313 Effect.flatMap((chunk) => {
314 const folders = Array.from(chunk)
315 .map((folder) => folder.name)
316 .filter((name): name is string => !!name)
317 .sort((left, right) => right.length - left.length);
318 return Effect.forEach(
319 folders,
320 (managedFolder) =>
321 storage
322 .deleteManagedFolders({
323 bucket: bucketName,
324 managedFolder,
325 allowNonEmpty: true,
326 })
327 .pipe(Effect.catchTag("NotFound", () => Effect.void)),
328 { concurrency: 1 },
329 );
330 }),
331 Effect.catchTag("NotFound", () => Effect.void),
332 );
333
334export const BucketProvider = () =>
335 Provider.succeed(Bucket, {
336 stables: [
337 "bucketName",
338 "location",
339 "locationType",
340 "projectNumber",
341 "timeCreated",
342 "selfLink",
343 ],
344
345 diff: Effect.fn(function* ({ news, olds, output }) {
346 if (!isResolved(news)) return undefined;
347 // Bucket names are immutable: a new name is a new bucket, and the
348 // old one must be deleted rather than left behind.
349 const recordedName = output?.bucketName ?? olds?.bucketName;
350 if (
351 news.bucketName !== undefined &&
352 recordedName !== undefined &&
353 news.bucketName !== recordedName
354 ) {
355 return { action: "replace" as const, deleteFirst: false };
356 }
357 const previous = olds?.location ?? output?.location;
358 const next = news.location;
359 if (
360 previous !== undefined &&
361 next !== undefined &&
362 previous.toUpperCase() !== next.toUpperCase()
363 ) {
364 const previousName = olds?.bucketName ?? output?.bucketName;
365 const nextName = news.bucketName ?? previousName;
366 return {
367 action: "replace" as const,
368 deleteFirst: nextName !== undefined && nextName === previousName,
369 };
370 }
371 const previousHns =
372 olds?.hierarchicalNamespace ?? output?.hierarchicalNamespace;
373 const nextHns = news.hierarchicalNamespace === true;
374 if (previousHns !== undefined && previousHns !== nextHns) {
375 const previousName = olds?.bucketName ?? output?.bucketName;
376 const nextName = news.bucketName ?? previousName;
377 return {
378 action: "replace" as const,
379 deleteFirst: nextName !== undefined && nextName === previousName,
380 };
381 }
382 return undefined;
383 }),
384
385 read: Effect.fn(function* ({ id, olds, output }) {
386 const bucketName = yield* toName(
387 id,
388 olds?.bucketName,
389 output?.bucketName,
390 );
391 const existing = yield* getByName(bucketName);
392 if (existing === undefined) return undefined;
393 const attrs = toAttrs(existing);
394 return (yield* hasAlchemyLabels(id, tagRecord(existing.labels)))
395 ? attrs
396 : Unowned(attrs);
397 }),
398
399 list: () =>
400 Effect.gen(function* () {
401 const env = yield* GcpEnvironment.current;
402 return yield* storage.listBuckets
403 .items({ project: env.project, projection: "full" })
404 .pipe(
405 Stream.filter((bucket) =>
406 Object.keys(bucket.labels ?? {}).some((key) =>
407 key.startsWith("alchemy-"),
408 ),
409 ),
410 Stream.map(toAttrs),
411 Stream.runCollect,
412 Effect.map((chunk) => Array.from(chunk)),
413 );
414 }),
415
416 reconcile: Effect.fn(function* ({ id, news, output }) {
417 const env = yield* GcpEnvironment.current;
418 const bucketName = yield* toName(id, news.bucketName, output?.bucketName);
419 const location =
420 news.location ?? output?.location ?? env.region.toUpperCase();
421 const storageClass = news.storageClass ?? "STANDARD";
422 const versioning = news.versioning === true;
423 const hierarchicalNamespace = news.hierarchicalNamespace === true;
424 const uniformBucketLevelAccess =
425 news.uniformBucketLevelAccess === true || hierarchicalNamespace;
426 const configureIam =
427 news.uniformBucketLevelAccess !== undefined || hierarchicalNamespace;
428 const desiredLabels = {
429 ...toLabels(news.labels),
430 ...(yield* createInternalLabels(id)),
431 };
432 const desiredWebsite = toWebsite(news.website);
433
434 let current = yield* getByName(bucketName);
435
436 if (current === undefined) {
437 const created = yield* storage
438 .insertBuckets({
439 project: env.project,
440 projection: "full",
441 body: {
442 name: bucketName,
443 location,
444 storageClass,
445 versioning: { enabled: versioning },
446 labels: desiredLabels,
447 hierarchicalNamespace: hierarchicalNamespace
448 ? { enabled: true }
449 : undefined,
450 website: desiredWebsite,
451 iamConfiguration: configureIam
452 ? {
453 uniformBucketLevelAccess: {
454 enabled: uniformBucketLevelAccess,
455 },
456 }
457 : undefined,
458 },
459 })
460 .pipe(Effect.catchTag("Conflict", () => getByName(bucketName)));
461 current = created ?? undefined;
462 }
463
464 if (current === undefined) {
465 return yield* new BucketNotResolved({ bucketName });
466 }
467
468 const observedLabels = tagRecord(current.labels);
469 const { upsert, removed } = diffLabels(observedLabels, desiredLabels);
470 const labelsChanged = upsert.length > 0 || removed.length > 0;
471 const storageClassChanged =
472 (current.storageClass ?? "STANDARD") !== storageClass;
473 const versioningChanged =
474 (current.versioning?.enabled === true) !== versioning;
475 const ublChanged =
476 configureIam &&
477 (current.iamConfiguration?.uniformBucketLevelAccess?.enabled ===
478 true) !==
479 uniformBucketLevelAccess;
480 const observedWebsite = toWebsite(current.website);
481 const websiteChanged =
482 observedWebsite?.mainPageSuffix !== desiredWebsite?.mainPageSuffix ||
483 observedWebsite?.notFoundPage !== desiredWebsite?.notFoundPage;
484
485 if (
486 labelsChanged ||
487 storageClassChanged ||
488 versioningChanged ||
489 ublChanged ||
490 websiteChanged
491 ) {
492 const nextLabels: Record<string, string | null> = { ...desiredLabels };
493 for (const key of removed) {
494 nextLabels[key] = null;
495 }
496 current = yield* storage.patchBuckets({
497 bucket: bucketName,
498 projection: "full",
499 body: {
500 storageClass,
501 versioning: { enabled: versioning },
502 labels: nextLabels as unknown as Record<string, string>,
503 iamConfiguration: ublChanged
504 ? {
505 uniformBucketLevelAccess: {
506 enabled: uniformBucketLevelAccess,
507 },
508 }
509 : undefined,
510 // PATCH merges nested objects and clears a field only when it
511 // is sent as JSON null.
512 website: websiteChanged
513 ? ((desiredWebsite === undefined
514 ? null
515 : {
516 mainPageSuffix: desiredWebsite.mainPageSuffix ?? null,
517 notFoundPage: desiredWebsite.notFoundPage ?? null,
518 }) as unknown as storage.BucketWebsite)
519 : undefined,
520 },
521 });
522 }
523
524 return toAttrs(current);
525 }),
526
527 delete: Effect.fn(function* ({ olds, output, force }) {
528 const mayEmpty = olds.forceDestroy === true || force === true;
529 const empty = Effect.gen(function* () {
530 yield* emptyBucket(output.bucketName);
531 if (output.hierarchicalNamespace) {
532 yield* emptyFolders(output.bucketName);
533 }
534 if (output.uniformBucketLevelAccess) {
535 yield* emptyManagedFolders(output.bucketName);
536 }
537 });
538 // A writer that is still running (e.g. a Dataflow job staging temp
539 // files) can add objects between the empty and the delete, which fails
540 // with 409 "not empty"; with forceDestroy, empty again and retry.
541 yield* Effect.gen(function* () {
542 if (mayEmpty) yield* empty;
543 yield* storage
544 .deleteBuckets({ bucket: output.bucketName })
545 .pipe(Effect.catchTag("NotFound", () => Effect.void));
546 }).pipe(
547 Effect.retry({
548 while: (error) => mayEmpty && error._tag === "Conflict",
549 times: 6,
550 schedule: Schedule.spaced("10 seconds"),
551 }),
552 );
553 }),
554 });