1import * as dataplex from "@distilled.cloud/gcp/dataplex_v1";
2import * as Data from "effect/Data";
3import * as Effect from "effect/Effect";
4import * as Stream from "effect/Stream";
5import { createPhysicalName } from "../../PhysicalName.ts";
6import { tagRecord } from "../../Tags.ts";
7import {
8 createInternalLabels,
9 hasAlchemyLabels,
10 stripInternalLabels,
11} from "../Labels.ts";
12
13export const MAX_ID_LENGTH = 63;
14export const MAX_ENTITY_ID_LENGTH = 256;
15
16export class DataplexNotResolved extends Data.TaggedError(
17 "GCP.Dataplex.NotResolved",
18)<{
19 name: string;
20}> {}
21
22export class DataplexStillExists extends Data.TaggedError(
23 "GCP.Dataplex.StillExists",
24)<{
25 name: string;
26}> {}
27
28export const lastSegment = (value: string) => {
29 const trimmed = value.replace(/\/+$/, "");
30 const parts = trimmed.split("/");
31 return parts[parts.length - 1] || trimmed;
32};
33
34export const normalizeLocation = (
35 location: string | undefined,
36 defaultLocation: string,
37) => lastSegment(location ?? defaultLocation).toLowerCase();
38
39export const locationParent = (project: string, location: string) =>
40 `projects/${project}/locations/${location}`;
41
42export const parseResourceName = (name: string, collection: string) => {
43 const parts = name.split("/").filter((part) => part.length > 0);
44 const collectionAt = parts.lastIndexOf(collection);
45 const locationsAt = parts.lastIndexOf("locations");
46 const projectsAt = parts.lastIndexOf("projects");
47 return {
48 project:
49 projectsAt >= 0 && parts[projectsAt + 1] ? parts[projectsAt + 1]! : "",
50 // API resource names always carry a `locations/{location}` segment.
51 location:
52 locationsAt >= 0 && parts[locationsAt + 1] ? parts[locationsAt + 1]! : "",
53 id:
54 collectionAt >= 0 && parts[collectionAt + 1]
55 ? parts[collectionAt + 1]!
56 : lastSegment(name),
57 parent:
58 collectionAt > 0
59 ? parts.slice(0, collectionAt).join("/")
60 : parts.slice(0, Math.max(0, parts.length - 1)).join("/"),
61 };
62};
63
64export const expandParent = (
65 value: string,
66 project: string,
67 location: string,
68 collection: string,
69) => {
70 if (value.includes("/")) return value;
71 return `projects/${project}/locations/${location}/${collection}/${value}`;
72};
73
74export const rfc1035 = (name: string, maxLength = MAX_ID_LENGTH): string => {
75 let next = name
76 .toLowerCase()
77 .replace(/[^a-z0-9-]/g, "-")
78 .replace(/-+/g, "-")
79 .replace(/^-+|-+$/g, "");
80 if (!/^[a-z]/.test(next)) next = `a${next}`;
81 next = next.slice(0, maxLength).replace(/-+$/g, "");
82 if (next.length === 0) return "dataplex";
83 if (!/[a-z0-9]$/.test(next)) next = `${next.slice(0, maxLength - 1)}0`;
84 return next.slice(0, maxLength);
85};
86
87export const snakeId = (
88 name: string,
89 maxLength = MAX_ENTITY_ID_LENGTH,
90): string => {
91 let next = name
92 .toLowerCase()
93 .replace(/[^a-z0-9_]/g, "_")
94 .replace(/_+/g, "_")
95 .replace(/^_+|_+$/g, "");
96 if (!/^[a-z]/.test(next)) next = `e${next}`;
97 next = next.slice(0, maxLength).replace(/_+$/g, "");
98 if (next.length === 0) return "entity";
99 if (!/[a-z0-9]$/.test(next)) next = `${next.slice(0, maxLength - 1)}0`;
100 return next.slice(0, maxLength);
101};
102
103export const toPhysicalRfc1035 = (
104 id: string,
105 explicit: string | undefined,
106 existing: string | undefined,
107 maxLength = MAX_ID_LENGTH,
108) =>
109 Effect.gen(function* () {
110 if (explicit !== undefined) return rfc1035(explicit, maxLength);
111 if (existing !== undefined) return existing;
112 return rfc1035(
113 yield* createPhysicalName({ id, maxLength, lowercase: true }),
114 maxLength,
115 );
116 });
117
118export const toPhysicalSnake = (
119 id: string,
120 explicit: string | undefined,
121 existing: string | undefined,
122 maxLength = MAX_ENTITY_ID_LENGTH,
123) =>
124 Effect.gen(function* () {
125 if (explicit !== undefined) return snakeId(explicit, maxLength);
126 if (existing !== undefined) return existing;
127 return snakeId(
128 yield* createPhysicalName({
129 id,
130 maxLength,
131 lowercase: true,
132 delimiter: "_",
133 }),
134 maxLength,
135 );
136 });
137
138export const userLabels = (
139 labels: Record<string, string | undefined> | null | undefined,
140): Record<string, string> => stripInternalLabels(tagRecord(labels));
141
142export const hasAlchemyLabelMap = (
143 labels: Record<string, string | undefined> | null | undefined,
144) => Object.keys(labels ?? {}).some((key) => key.startsWith("alchemy-"));
145
146export const ownedLabels = (id: string, labels: Record<string, string>) =>
147 hasAlchemyLabels(id, labels);
148
149export const createOwnership = (id: string) => createInternalLabels(id);
150
151export const canonical = (value: unknown): unknown => {
152 if (value === undefined || value === null) return undefined;
153 if (typeof value === "boolean" || typeof value === "number") return value;
154 if (typeof value === "string") {
155 return value.length === 0 ? undefined : value;
156 }
157 if (Array.isArray(value)) {
158 const items = value.map(canonical).filter((item) => item !== undefined);
159 return items.length === 0 ? undefined : items;
160 }
161 if (typeof value === "object") {
162 const entries = Object.entries(value as Record<string, unknown>)
163 .map(([key, item]) => [key, canonical(item)] as const)
164 .filter(([, item]) => item !== undefined)
165 .sort(([left], [right]) => left.localeCompare(right));
166 if (entries.length === 0) return undefined;
167 return Object.fromEntries(entries);
168 }
169 return undefined;
170};
171
172export const fingerprint = (value: unknown): string =>
173 JSON.stringify(canonical(value) ?? null);
174
175export const sameJson = (left: unknown, right: unknown) =>
176 fingerprint(left) === fingerprint(right);
177
178export const sameStringList = (
179 left: readonly string[] | undefined,
180 right: readonly string[] | undefined,
181) =>
182 JSON.stringify([...(left ?? [])].sort()) ===
183 JSON.stringify([...(right ?? [])].sort());
184
185export const isPendingState = (state: string | undefined) =>
186 state === "CREATING" ||
187 state === "DELETING" ||
188 state === "UPDATING" ||
189 state === "STATE_UNSPECIFIED" ||
190 state === undefined ||
191 state === "";
192
193export const collectPages = <Page, Item, E, R>(
194 stream: Stream.Stream<Page, E, R>,
195 pick: (page: Page) => readonly Item[] | undefined,
196) =>
197 stream.pipe(
198 Stream.flatMap((page) => Stream.fromIterable(pick(page) ?? [])),
199 Stream.runCollect,
200 Effect.map((chunk) => Array.from(chunk)),
201 );
202
203const emptyOnMissing = <A, E extends { readonly _tag: string }, R>(
204 effect: Effect.Effect<A[], E, R>,
205) =>
206 effect.pipe(
207 Effect.catchIf(
208 (error) => error._tag === "NotFound",
209 () => Effect.succeed([] as A[]),
210 ),
211 );
212
213// `us-central1` was the fixed default before `GCP.Region`; list paths keep
214// it alongside the stack region so resources created under it are found.
215const listLocations = (region: string) => [...new Set([region, "us-central1"])];
216
217const acrossLocations = <A, E, R>(
218 region: string,
219 list: (location: string) => Effect.Effect<A[], E, R>,
220) =>
221 Effect.forEach(listLocations(region), list).pipe(
222 Effect.map((groups) => groups.flat()),
223 );
224
225export const listLakes = (project: string, region: string) =>
226 acrossLocations(region, (location) =>
227 emptyOnMissing(
228 collectPages(
229 dataplex.listProjectsLocationsLakes.pages({
230 parent: locationParent(project, location),
231 pageSize: 1000,
232 }),
233 (page) => page.lakes,
234 ),
235 ),
236 );
237
238export const listGlossaries = (project: string, region: string) =>
239 acrossLocations(region, (location) =>
240 emptyOnMissing(
241 collectPages(
242 dataplex.listProjectsLocationsGlossaries.pages({
243 parent: locationParent(project, location),
244 pageSize: 1000,
245 }),
246 (page) => page.glossaries,
247 ),
248 ),
249 );
250
251export const listTerms = (glossary: string) =>
252 emptyOnMissing(
253 collectPages(
254 dataplex.listProjectsLocationsGlossariesTerms.pages({
255 parent: glossary,
256 pageSize: 1000,
257 }),
258 (page) => page.terms,
259 ),
260 );
261
262export const listTasks = (lake: string) =>
263 emptyOnMissing(
264 collectPages(
265 dataplex.listProjectsLocationsLakesTasks.pages({
266 parent: lake,
267 pageSize: 1000,
268 }),
269 (page) => page.tasks,
270 ),
271 );
272
273export const listZones = (lake: string) =>
274 emptyOnMissing(
275 collectPages(
276 dataplex.listProjectsLocationsLakesZones.pages({
277 parent: lake,
278 pageSize: 1000,
279 }),
280 (page) => page.zones,
281 ),
282 );
283
284export const listAssets = (zone: string) =>
285 emptyOnMissing(
286 collectPages(
287 dataplex.listProjectsLocationsLakesZonesAssets.pages({
288 parent: zone,
289 pageSize: 1000,
290 }),
291 (page) => page.assets,
292 ),
293 );
294
295export const listEntities = (
296 zone: string,
297 view: dataplex.ListProjectsLocationsLakesZonesEntitiesViewEnum,
298) =>
299 emptyOnMissing(
300 collectPages(
301 dataplex.listProjectsLocationsLakesZonesEntities.pages({
302 parent: zone,
303 view,
304 pageSize: 500,
305 }),
306 (page) => page.entities,
307 ),
308 );
309
310export const listPartitions = (entity: string) =>
311 emptyOnMissing(
312 collectPages(
313 dataplex.listProjectsLocationsLakesZonesEntitiesPartitions.pages({
314 parent: entity,
315 pageSize: 500,
316 }),
317 (page) => page.partitions,
318 ),
319 );
320
321export const listMetadataFeeds = (project: string, region: string) =>
322 acrossLocations(region, (location) =>
323 emptyOnMissing(
324 collectPages(
325 dataplex.listProjectsLocationsMetadataFeeds.pages({
326 parent: locationParent(project, location),
327 pageSize: 1000,
328 }),
329 (page) => page.metadataFeeds,
330 ),
331 ),
332 );
333
334export const listChildResources = <A, E, R>(
335 lakes: readonly dataplex.GoogleCloudDataplexV1Lake[],
336 list: (lakeName: string) => Effect.Effect<A[], E, R>,
337) =>
338 Effect.forEach(
339 lakes.filter((lake) => (lake.name ?? "").length > 0),
340 (lake) => list(lake.name!),
341 { concurrency: 4 },
342 ).pipe(Effect.map((groups) => groups.flat()));
343
344export const replaceIfChanged = (
345 previous: string | undefined,
346 next: string | undefined,
347) => previous !== undefined && next !== undefined && previous !== next;