1import * as biglake from "@distilled.cloud/gcp/biglake_v1";
2import * as Effect from "effect/Effect";
3import { Unowned } from "../../AdoptPolicy.ts";
4import { isResolved } from "../../Diff.ts";
5import * as Provider from "../../Provider.ts";
6import { Resource } from "../../Resource.ts";
7import { tagRecord } from "../../Tags.ts";
8import { GcpEnvironment } from "../Environment.ts";
9import type { Providers } from "../Providers.ts";
10import {
11 BiglakeNotResolved,
12 listLocations,
13 createInternalLabels,
14 expandDatabase,
15 hasAlchemyLabelMap,
16 hasAlchemyLabels,
17 ignoreGone,
18 listCatalogs,
19 listChildResources,
20 listDatabases,
21 listTables,
22 locationParent,
23 mergeParameters,
24 missingGet,
25 namedOf,
26 normalizeLocation,
27 parseResourceName,
28 replaceOnIdentity,
29 retryTransient,
30 sameJson,
31 sameText,
32 toPhysicalId,
33 updateMaskOf,
34 userLabels,
35 waitUntilGone,
36} from "./internal.ts";
37
38export type SerDeInfo = {
39 /**
40 * Fully qualified Java class name of the serialization library.
41 */
42 serializationLib?: string;
43};
44
45export type StorageDescriptor = {
46 /**
47 * Cloud Storage folder URI where the table data is stored, starting
48 * with `gs://`.
49 */
50 locationUri?: string;
51 /**
52 * Fully qualified Java class name of the input format.
53 */
54 inputFormat?: string;
55 /**
56 * Fully qualified Java class name of the output format.
57 */
58 outputFormat?: string;
59 /**
60 * Serializer and deserializer information.
61 */
62 serdeInfo?: SerDeInfo;
63};
64
65export type HiveTableOptions = {
66 /**
67 * Hive table type (`MANAGED_TABLE`, `EXTERNAL_TABLE`, …).
68 */
69 tableType?: string;
70 /**
71 * User-supplied Hive table parameters. Alchemy ownership keys
72 * (`alchemy-stack`, `alchemy-stage`, `alchemy-id`) are merged in
73 * automatically — BigLake tables have no labels field.
74 */
75 parameters?: Record<string, string>;
76 /**
77 * Physical storage information of the data.
78 */
79 storageDescriptor?: StorageDescriptor;
80};
81
82export type CatalogsDatabasesTableProps = {
83 /**
84 * Parent database. Full name
85 * `projects/{project}/locations/{location}/catalogs/{catalog}/databases/{database}`
86 * or the database id (combined with `catalog` and `location`).
87 * Immutable — changing it replaces the table.
88 */
89 database: string;
90 /**
91 * Parent catalog used when `database` is a bare id.
92 */
93 catalog?: string;
94 /**
95 * Table id (the `{table}` segment). If omitted, a unique id is
96 * generated. Immutable — changing it replaces the table.
97 */
98 tableId?: string;
99 /**
100 * Location used when `database` is a bare id.
101 * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`)
102 */
103 location?: string;
104 /**
105 * Table type.
106 * @default "HIVE"
107 */
108 type?: biglake.TableTypeEnum | (string & {});
109 /**
110 * Hive table options. Required when `type` is `HIVE`.
111 */
112 hiveOptions?: HiveTableOptions;
113};
114
115export type CatalogsDatabasesTable = Resource<
116 "GCP.BigLake.CatalogsDatabasesTable",
117 CatalogsDatabasesTableProps,
118 {
119 /** Full resource name. */
120 name: string;
121 /** Table id (last path segment). */
122 tableId: string;
123 /** Parent database resource name. */
124 database: string;
125 /** Parent catalog resource name. */
126 catalog: string;
127 /** Project id. */
128 project: string;
129 /** Location id. */
130 location: string;
131 /** Table type. */
132 type: string | undefined;
133 /** Hive table type (`MANAGED_TABLE`, `EXTERNAL_TABLE`, …). */
134 tableType: string | undefined;
135 /** User Hive parameters (Alchemy ownership keys stripped). */
136 parameters: Record<string, string>;
137 /** Physical storage descriptor. */
138 storageDescriptor: StorageDescriptor | undefined;
139 /** Server checksum used for optimistic updates. */
140 etag: string | undefined;
141 /** RFC3339 creation timestamp. */
142 createTime: string | undefined;
143 /** RFC3339 last-update timestamp. */
144 updateTime: string | undefined;
145 /** RFC3339 deletion timestamp, if soft-deleted. */
146 deleteTime: string | undefined;
147 /** RFC3339 expire timestamp after delete. */
148 expireTime: string | undefined;
149 },
150 never,
151 Providers
152>;
153
154/**
155 * A Hive table inside a BigLake Metastore database.
156 *
157 * Tables have no labels field — Alchemy stamps ownership into
158 * `hiveOptions.parameters` so `list` / nuke can find them. Database,
159 * table id, and type are immutable. Hive options update in place.
160 *
161 * ### Creating a Table
162 * **Example:** Managed Hive table
163 * ```typescript
164 * const table = yield* GCP.BigLake.CatalogsDatabasesTable("Events", {
165 * database: database.name,
166 * hiveOptions: {
167 * tableType: "MANAGED_TABLE",
168 * storageDescriptor: {
169 * locationUri: `gs://${bucket.bucketName}/events`,
170 * inputFormat: "org.apache.hadoop.mapred.SequenceFileInputFormat",
171 * outputFormat:
172 * "org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat",
173 * },
174 * parameters: { owner: "analytics" },
175 * },
176 * });
177 * ```
178 *
179 * **Example:** Named external table
180 * ```typescript
181 * const table = yield* GCP.BigLake.CatalogsDatabasesTable("Events", {
182 * database: database.name,
183 * tableId: "page_views",
184 * hiveOptions: {
185 * tableType: "EXTERNAL_TABLE",
186 * storageDescriptor: {
187 * locationUri: "gs://my-bucket/page_views",
188 * },
189 * },
190 * });
191 * ```
192 *
193 * @resource
194 * @category BigLake
195 */
196export const CatalogsDatabasesTable = Resource<CatalogsDatabasesTable>(
197 "GCP.BigLake.CatalogsDatabasesTable",
198);
199
200const databaseOf = (
201 database: string,
202 project: string,
203 location: string,
204 catalog?: string,
205) => expandDatabase(database, project, location, catalog);
206
207const resourceName = (database: string, tableId: string) =>
208 `${database}/tables/${tableId}`;
209
210const storageDescriptorOf = (
211 descriptor: biglake.StorageDescriptor | undefined,
212): StorageDescriptor | undefined => {
213 if (descriptor === undefined) return undefined;
214 return {
215 locationUri: descriptor.locationUri,
216 inputFormat: descriptor.inputFormat,
217 outputFormat: descriptor.outputFormat,
218 serdeInfo: descriptor.serdeInfo,
219 };
220};
221
222const toAttrs = (table: biglake.Table, project: string) => {
223 const name = table.name ?? "";
224 const parsed = parseResourceName(name, "tables");
225 const catalogParsed = parseResourceName(parsed.parent, "databases");
226 return {
227 name,
228 tableId: parsed.id,
229 database: parsed.parent,
230 catalog: catalogParsed.parent,
231 project,
232 location: parsed.location,
233 type: table.type,
234 tableType: table.hiveOptions?.tableType,
235 parameters: userLabels(table.hiveOptions?.parameters),
236 storageDescriptor: storageDescriptorOf(
237 table.hiveOptions?.storageDescriptor,
238 ),
239 etag: table.etag,
240 createTime: table.createTime,
241 updateTime: table.updateTime,
242 deleteTime: table.deleteTime,
243 expireTime: table.expireTime,
244 };
245};
246
247const getByName = missingGet(
248 biglake.getProjectsLocationsCatalogsDatabasesTables,
249);
250
251const desiredHiveOptions = (
252 news: CatalogsDatabasesTableProps,
253 parameters: Record<string, string>,
254): biglake.HiveTableOptions => ({
255 tableType: news.hiveOptions?.tableType,
256 parameters,
257 storageDescriptor: news.hiveOptions?.storageDescriptor,
258});
259
260export const CatalogsDatabasesTableProvider = () =>
261 Provider.succeed(CatalogsDatabasesTable, {
262 stables: [
263 "name",
264 "tableId",
265 "database",
266 "catalog",
267 "project",
268 "location",
269 "createTime",
270 ],
271
272 diff: Effect.fn(function* ({ news, olds, output }) {
273 if (!isResolved(news)) return undefined;
274 const env = yield* GcpEnvironment.current;
275 const location = normalizeLocation(
276 news.location ?? olds?.location ?? output?.location,
277 env.region,
278 );
279 const typeChanged =
280 (olds?.type ?? output?.type) !== undefined &&
281 (news.type ?? "HIVE") !== (olds?.type ?? output?.type);
282 if (typeChanged) {
283 return { action: "replace" as const, deleteFirst: true };
284 }
285 return replaceOnIdentity({
286 previousId: olds?.tableId ?? output?.tableId,
287 nextId: news.tableId,
288 previousLocation: normalizeLocation(
289 olds?.location ?? output?.location,
290 env.region,
291 ),
292 nextLocation: location,
293 previousParent: olds?.database ?? output?.database,
294 nextParent: databaseOf(
295 news.database,
296 env.project,
297 location,
298 news.catalog,
299 ),
300 });
301 }),
302
303 read: Effect.fn(function* ({ id, olds, output }) {
304 const env = yield* GcpEnvironment.current;
305 const location = normalizeLocation(
306 olds?.location ?? output?.location,
307 env.region,
308 );
309 const tableId = yield* toPhysicalId(id, olds?.tableId, output?.tableId);
310 const database =
311 olds?.database !== undefined
312 ? databaseOf(olds.database, env.project, location, olds.catalog)
313 : (output?.database ?? "");
314 const name =
315 output?.name ??
316 (database.length > 0 ? resourceName(database, tableId) : "");
317 const existing = yield* getByName(name);
318 if (existing === undefined) return undefined;
319 const attrs = toAttrs(existing, env.project);
320 return (yield* hasAlchemyLabels(
321 id,
322 tagRecord(existing.hiveOptions?.parameters),
323 ))
324 ? attrs
325 : Unowned(attrs);
326 }),
327
328 list: () =>
329 Effect.gen(function* () {
330 const env = yield* GcpEnvironment.current;
331 const catalogs = (yield* Effect.forEach(
332 listLocations(env.region),
333 (location) => listCatalogs(locationParent(env.project, location)),
334 )).flat();
335 const databases = yield* listChildResources(
336 namedOf(catalogs),
337 listDatabases,
338 );
339 const tables = yield* listChildResources(
340 namedOf(databases),
341 listTables,
342 );
343 return tables
344 .filter((item) => hasAlchemyLabelMap(item.hiveOptions?.parameters))
345 .map((item) => toAttrs(item, env.project));
346 }),
347
348 reconcile: Effect.fn(function* ({ id, news, output }) {
349 const env = yield* GcpEnvironment.current;
350 const location = normalizeLocation(
351 news.location ?? output?.location,
352 env.region,
353 );
354 const database = databaseOf(
355 news.database,
356 env.project,
357 location,
358 news.catalog,
359 );
360 const tableId = yield* toPhysicalId(id, news.tableId, output?.tableId);
361 const name = output?.name ?? resourceName(database, tableId);
362 const type = news.type ?? "HIVE";
363 const parameters = mergeParameters(
364 news.hiveOptions?.parameters,
365 yield* createInternalLabels(id),
366 );
367 const hiveOptions = desiredHiveOptions(news, parameters);
368
369 let current = yield* getByName(name);
370
371 if (current === undefined) {
372 const created = yield* retryTransient(
373 biglake.createProjectsLocationsCatalogsDatabasesTables({
374 parent: database,
375 tableId,
376 body: { type, hiveOptions },
377 }),
378 ).pipe(Effect.catchTag("Conflict", () => getByName(name)));
379 current = created ?? undefined;
380 }
381
382 if (current === undefined) {
383 return yield* new BiglakeNotResolved({ name });
384 }
385
386 const currentName = current.name ?? name;
387 const typeChanged = !sameText(current.type, type);
388 const hiveChanged = !sameJson(current.hiveOptions, hiveOptions);
389
390 if (typeChanged || hiveChanged) {
391 current = yield* retryTransient(
392 biglake.patchProjectsLocationsCatalogsDatabasesTables({
393 name: currentName,
394 updateMask: updateMaskOf(
395 typeChanged ? "type" : undefined,
396 hiveChanged ? "hiveOptions" : undefined,
397 ),
398 body: {
399 name: currentName,
400 type,
401 hiveOptions,
402 etag: current.etag,
403 },
404 }),
405 );
406 }
407
408 return toAttrs(current, env.project);
409 }),
410
411 delete: Effect.fn(function* ({ output }) {
412 if (!output.name) return;
413 yield* ignoreGone(
414 biglake.deleteProjectsLocationsCatalogsDatabasesTables({
415 name: output.name,
416 }),
417 );
418 yield* waitUntilGone(getByName(output.name));
419 }),
420 });