1import * as cw from "@distilled.cloud/gcp/contentwarehouse_v1";
2import * as Data from "effect/Data";
3import * as Effect from "effect/Effect";
4import * as Schedule from "effect/Schedule";
5import * as Stream from "effect/Stream";
6import { createPhysicalName } from "../../PhysicalName.ts";
7import {
8 alchemyLabelKeys,
9 createInternalLabels,
10 hasAlchemyLabels,
11} from "../Labels.ts";
12import { waitForOperation as waitForGcpOperation } from "../Operation.ts";
13
14export const DEFAULT_LOCATION = "us";
15export const LIST_LOCATIONS = ["us", "eu"] as const;
16export const MAX_ID_LENGTH = 63;
17export const MAX_DISPLAY_NAME_LENGTH = 128;
18export const MAX_DESCRIPTION_LENGTH = 8000;
19export const MAX_CONTEXT_LENGTH = 64;
20
21export class ContentwarehouseOperationPending extends Data.TaggedError(
22 "GCP.ContentWarehouse.OperationPending",
23)<{
24 operation: string;
25}> {}
26
27export class ResourceNotResolved extends Data.TaggedError(
28 "GCP.ContentWarehouse.ResourceNotResolved",
29)<{
30 name: string;
31}> {}
32
33export class ResourceStillExists extends Data.TaggedError(
34 "GCP.ContentWarehouse.ResourceStillExists",
35)<{
36 name: string;
37}> {}
38
39export const lastSegment = (value: string) => {
40 const trimmed = value.replace(/\/+$/, "");
41 const parts = trimmed.split("/");
42 return parts[parts.length - 1] || trimmed;
43};
44
45export const rfc1035 = (name: string, fallback = "cwh"): string => {
46 let next = name
47 .toLowerCase()
48 .replace(/[^a-z0-9-]/g, "-")
49 .replace(/-+/g, "-")
50 .replace(/^-+|-+$/g, "");
51 if (!/^[a-z]/.test(next)) next = `c${next}`;
52 next = next.slice(0, MAX_ID_LENGTH).replace(/-+$/g, "");
53 if (next.length === 0) return fallback;
54 if (!/[a-z0-9]$/.test(next)) next = `${next.slice(0, MAX_ID_LENGTH - 1)}0`;
55 return next.slice(0, MAX_ID_LENGTH);
56};
57
58export const normalizeLocation = (location: string | undefined) =>
59 lastSegment(location ?? DEFAULT_LOCATION).toLowerCase();
60
61export const locationParent = (project: string, location: string | undefined) =>
62 `projects/${project}/locations/${normalizeLocation(location)}`;
63
64export const parseName = (name: string, collection: string) => {
65 const parts = name.split("/").filter((part) => part.length > 0);
66 const collectionAt = parts.lastIndexOf(collection);
67 const locationsAt = parts.lastIndexOf("locations");
68 const projectsAt = parts.lastIndexOf("projects");
69 return {
70 project:
71 projectsAt >= 0 && parts[projectsAt + 1] ? parts[projectsAt + 1]! : "",
72 location:
73 locationsAt >= 0 && parts[locationsAt + 1]
74 ? parts[locationsAt + 1]!
75 : DEFAULT_LOCATION,
76 id:
77 collectionAt >= 0 && parts[collectionAt + 1]
78 ? parts[collectionAt + 1]!
79 : lastSegment(name),
80 parent:
81 collectionAt > 0
82 ? parts.slice(0, collectionAt).join("/")
83 : parts.slice(0, Math.max(0, parts.length - 1)).join("/"),
84 };
85};
86
87export const toPhysicalId = (
88 id: string,
89 explicit: string | undefined,
90 existing: string | undefined,
91 fallback = "cwh",
92) =>
93 Effect.gen(function* () {
94 if (explicit !== undefined) return rfc1035(explicit, fallback);
95 if (existing !== undefined) return existing;
96 return rfc1035(
97 yield* createPhysicalName({
98 id,
99 maxLength: MAX_ID_LENGTH,
100 lowercase: true,
101 }),
102 fallback,
103 );
104 });
105
106export const sameText = (left: string | undefined, right: string | undefined) =>
107 (left ?? "") === (right ?? "");
108
109export const canonical = (value: unknown): unknown => {
110 if (value === undefined || value === null) return undefined;
111 if (typeof value === "boolean" || typeof value === "number") return value;
112 if (typeof value === "string") {
113 return value.length === 0 ? undefined : value;
114 }
115 if (Array.isArray(value)) {
116 const items = value.map(canonical).filter((item) => item !== undefined);
117 return items.length === 0 ? undefined : items;
118 }
119 if (typeof value === "object") {
120 const entries = Object.entries(value as Record<string, unknown>)
121 .map(([key, item]) => [key, canonical(item)] as const)
122 .filter(([, item]) => item !== undefined)
123 .sort(([left], [right]) => left.localeCompare(right));
124 if (entries.length === 0) return undefined;
125 return Object.fromEntries(entries);
126 }
127 return undefined;
128};
129
130export const fingerprint = (value: unknown): string =>
131 JSON.stringify(canonical(value) ?? null);
132
133export const sameJson = (left: unknown, right: unknown) =>
134 fingerprint(left) === fingerprint(right);
135
136const markerOf = (
137 labels: Record<string, string>,
138 stack: string,
139 stage: string,
140 id: string,
141) =>
142 `[alchemy ${alchemyLabelKeys.stack}=${stack} ${alchemyLabelKeys.stage}=${stage} ${alchemyLabelKeys.id}=${id}]`;
143
144const compactMarkerOf = (stack: string, stage: string, id: string) =>
145 `[alc ${stack} ${stage} ${id}]`;
146
147const shrinkMarker = (
148 labels: Record<string, string>,
149 maxLength: number,
150 build: (stack: string, stage: string, id: string) => string,
151) => {
152 let stack = labels[alchemyLabelKeys.stack] ?? "x";
153 let stage = labels[alchemyLabelKeys.stage] ?? "x";
154 let id = labels[alchemyLabelKeys.id] ?? "x";
155 let marker = build(stack, stage, id);
156 while (
157 marker.length > maxLength &&
158 (stack.length > 1 || stage.length > 1 || id.length > 1)
159 ) {
160 if (stack.length >= stage.length && stack.length >= id.length) {
161 stack = stack.slice(0, -1);
162 } else if (stage.length >= id.length) {
163 stage = stage.slice(0, -1);
164 } else {
165 id = id.slice(0, -1);
166 }
167 marker = build(stack, stage, id);
168 }
169 return marker.slice(0, maxLength);
170};
171
172export const encodeOwnership = (
173 labels: Record<string, string>,
174 text: string | undefined,
175): string => {
176 const marker = shrinkMarker(
177 labels,
178 MAX_DESCRIPTION_LENGTH,
179 (stack, stage, id) => markerOf(labels, stack, stage, id),
180 );
181 const trimmed = text?.trim();
182 return trimmed && trimmed.length > 0 ? `${marker}\n${trimmed}` : marker;
183};
184
185export const encodeOwnershipLine = (
186 labels: Record<string, string>,
187 text: string | undefined,
188 maxLength = MAX_DISPLAY_NAME_LENGTH,
189): string => {
190 const marker =
191 maxLength < 54
192 ? shrinkMarker(labels, maxLength, compactMarkerOf)
193 : shrinkMarker(labels, maxLength, (stack, stage, id) =>
194 markerOf(labels, stack, stage, id),
195 );
196 const trimmed = text?.replace(/[\r\n]+/g, " ").trim();
197 if (!trimmed) return marker;
198 return `${marker} ${trimmed}`.slice(0, maxLength);
199};
200
201export const parseOwnership = (
202 text: string | undefined,
203): {
204 labels: Record<string, string>;
205 text: string | undefined;
206} => {
207 if (text?.startsWith("[alc ")) {
208 const end = text.indexOf("]");
209 if (end < 0) return { labels: {}, text };
210 const parts = text.slice("[alc ".length, end).trim().split(/\s+/);
211 const labels: Record<string, string> = {};
212 if (parts[0]) labels[alchemyLabelKeys.stack] = parts[0]!;
213 if (parts[1]) labels[alchemyLabelKeys.stage] = parts[1]!;
214 if (parts[2]) labels[alchemyLabelKeys.id] = parts[2]!;
215 const rest = text.slice(end + 1).replace(/^[\s\n]+/, "");
216 return { labels, text: rest.length > 0 ? rest : undefined };
217 }
218 if (!text?.startsWith("[alchemy ")) {
219 return { labels: {}, text };
220 }
221 const end = text.indexOf("]");
222 if (end < 0) return { labels: {}, text };
223 const labels: Record<string, string> = {};
224 for (const part of text.slice("[alchemy ".length, end).split(/\s+/)) {
225 const eq = part.indexOf("=");
226 if (eq > 0) {
227 labels[part.slice(0, eq)] = part.slice(eq + 1);
228 }
229 }
230 const rest = text.slice(end + 1).replace(/^[\s\n]+/, "");
231 return { labels, text: rest.length > 0 ? rest : undefined };
232};
233
234export const hasOwnershipMarker = (text: string | undefined) =>
235 Object.keys(parseOwnership(text).labels).some((key) =>
236 key.startsWith("alchemy-"),
237 );
238
239const prefixMatch = (expected: string, observed: string) =>
240 expected === observed ||
241 expected.startsWith(observed) ||
242 observed.startsWith(expected);
243
244export const ownedByAlchemy = (id: string, text: string | undefined) =>
245 Effect.gen(function* () {
246 const expected = yield* createInternalLabels(id);
247 const { labels } = parseOwnership(text);
248 if (!hasOwnershipMarker(text)) return false;
249 const exact = yield* hasAlchemyLabels(id, labels);
250 if (exact) return true;
251 return (
252 prefixMatch(
253 expected[alchemyLabelKeys.stack] ?? "",
254 labels[alchemyLabelKeys.stack] ?? "",
255 ) &&
256 prefixMatch(
257 expected[alchemyLabelKeys.stage] ?? "",
258 labels[alchemyLabelKeys.stage] ?? "",
259 ) &&
260 prefixMatch(
261 expected[alchemyLabelKeys.id] ?? "",
262 labels[alchemyLabelKeys.id] ?? "",
263 )
264 );
265 });
266
267export const replaceOnIdentity = (input: {
268 previousId?: string;
269 nextId?: string;
270 previousLocation?: string;
271 nextLocation?: string;
272 extra?: boolean;
273}) => {
274 const previousLocation = normalizeLocation(input.previousLocation);
275 const nextLocation = normalizeLocation(input.nextLocation);
276 const replace =
277 (input.extra ?? false) ||
278 (input.previousId !== undefined &&
279 input.nextId !== undefined &&
280 input.nextId !== input.previousId) ||
281 (input.previousLocation !== undefined && previousLocation !== nextLocation);
282 if (!replace) return undefined;
283 const samePhysical =
284 previousLocation === nextLocation &&
285 input.previousId !== undefined &&
286 input.nextId === input.previousId;
287 return {
288 action: "replace" as const,
289 deleteFirst: samePhysical,
290 };
291};
292
293export const collectPages = <Page, A, E, R>(
294 pages: Stream.Stream<Page, E, R>,
295 items: (page: Page) => readonly A[] | undefined,
296) =>
297 pages.pipe(
298 Stream.flatMap((page) => Stream.fromIterable(items(page) ?? [])),
299 Stream.runCollect,
300 Effect.map((chunk) => Array.from(chunk)),
301 );
302
303export const ignoreList =
304 <A>(fallback: A) =>
305 <A1, E extends { readonly _tag: string }, R>(
306 self: Effect.Effect<A1, E, R>,
307 ): Effect.Effect<A1 | A, E, R> =>
308 self.pipe(
309 Effect.catchIf(
310 (error): error is Extract<E, { readonly _tag: "NotFound" }> =>
311 error._tag === "NotFound",
312 () => Effect.succeed(fallback),
313 ),
314 );
315
316export const waitUntilExists = <A, E extends { readonly _tag: string }, R>(
317 get: Effect.Effect<A | undefined, E, R>,
318 name: string,
319): Effect.Effect<NonNullable<A>, E | ResourceNotResolved, R> =>
320 get.pipe(
321 Effect.filterOrFail(
322 (value): value is NonNullable<A> => value != null,
323 () => new ResourceNotResolved({ name }),
324 ),
325 Effect.retry({
326 while: (error) =>
327 error._tag === "GCP.ContentWarehouse.ResourceNotResolved",
328 times: 8,
329 schedule: Schedule.spaced("1 second"),
330 }),
331 );
332
333export const waitUntilGone = <A, E extends { readonly _tag: string }, R>(
334 get: Effect.Effect<A | undefined, E, R>,
335 name: string,
336): Effect.Effect<void, E | ResourceStillExists, R> =>
337 get.pipe(
338 Effect.filterOrFail(
339 (value) => value === undefined,
340 () => new ResourceStillExists({ name }),
341 ),
342 Effect.retry({
343 while: (error) =>
344 error._tag === "GCP.ContentWarehouse.ResourceStillExists",
345 times: 10,
346 schedule: Schedule.spaced("2 seconds"),
347 }),
348 Effect.asVoid,
349 );
350
351/** Wait for an operation through the shared GCP waiter. */
352export const waitForOperation = (
353 operation: cw.GoogleLongrunningOperation,
354 options?: { notFoundOk?: boolean; alreadyExistsOk?: boolean },
355) =>
356 waitForGcpOperation(
357 operation,
358 (name) =>
359 cw.getProjectsLocationsOperations({ name }).pipe(
360 Effect.catchTag("NotFound", (error) =>
361 options?.notFoundOk === true
362 ? Effect.succeed<cw.GoogleLongrunningOperation>({
363 name,
364 done: true,
365 })
366 : Effect.fail(error),
367 ),
368 ),
369 { budget: "10 minutes" },
370 ).pipe(
371 // ALREADY_EXISTS (6): a concurrent create won the race. NOT_FOUND (5)
372 // is success for a delete.
373 Effect.catchIf(
374 (error) =>
375 error._tag === "GCP.OperationFailed" &&
376 ((options?.alreadyExistsOk !== false && error.code === 6) ||
377 (options?.notFoundOk === true && error.code === 5)),
378 () => Effect.void,
379 ),
380 // Re-read the finished operation for its typed response and metadata.
381 Effect.flatMap(() =>
382 operation.name === undefined || operation.name.length === 0
383 ? Effect.succeed(operation)
384 : cw
385 .getProjectsLocationsOperations({ name: operation.name })
386 .pipe(Effect.catchTag("NotFound", () => Effect.succeed(operation))),
387 ),
388 );
389
390const isReady = (state: string | undefined) =>
391 state === "PROJECT_STATE_COMPLETED";
392
393const needsProvision = (state: string | undefined) =>
394 state === undefined ||
395 state === "PROJECT_STATE_UNSPECIFIED" ||
396 state === "PROJECT_STATE_NOT_FOUND" ||
397 state === "PROJECT_STATE_DELETED";
398
399export const ensureProject = (parent: string) =>
400 Effect.gen(function* () {
401 const status = yield* cw
402 .getStatusProjectsLocations({ location: parent })
403 .pipe(
404 Effect.catchTag("NotFound", () =>
405 Effect.succeed<cw.GoogleCloudContentwarehouseV1ProjectStatus>({
406 state: "PROJECT_STATE_NOT_FOUND",
407 }),
408 ),
409 );
410 if (isReady(status.state)) return status;
411
412 if (status.state === "PROJECT_STATE_PENDING") {
413 const pending = yield* cw
414 .getStatusProjectsLocations({ location: parent })
415 .pipe(
416 Effect.filterOrFail(
417 (current) => isReady(current.state),
418 () =>
419 new ContentwarehouseOperationPending({
420 operation: parent,
421 }),
422 ),
423 Effect.retry({
424 while: (error) =>
425 error._tag === "GCP.ContentWarehouse.OperationPending",
426 times: 10,
427 schedule: Schedule.spaced("5 seconds"),
428 }),
429 );
430 return pending;
431 }
432
433 if (!needsProvision(status.state)) return status;
434
435 const operation = yield* cw
436 .initializeProjectsLocations({
437 location: parent,
438 body: {
439 accessControlMode: "ACL_MODE_UNIVERSAL_ACCESS",
440 databaseType: "DB_INFRA_SPANNER",
441 documentCreatorDefaultRole: "DOCUMENT_ADMIN",
442 },
443 })
444 .pipe(Effect.catchTag("Conflict", () => Effect.succeed(undefined)));
445 if (operation !== undefined) {
446 yield* waitForOperation(operation);
447 }
448 return yield* cw.getStatusProjectsLocations({ location: parent }).pipe(
449 Effect.filterOrFail(
450 (current) => isReady(current.state),
451 () => new ContentwarehouseOperationPending({ operation: parent }),
452 ),
453 Effect.retry({
454 while: (error) =>
455 error._tag === "GCP.ContentWarehouse.OperationPending",
456 times: 10,
457 schedule: Schedule.spaced("5 seconds"),
458 }),
459 );
460 });
461
462export const ownershipSynonymWord = (labels: Record<string, string>) =>
463 encodeOwnershipLine(labels, undefined, 80);
464
465export const withOwnershipSynonyms = (
466 labels: Record<string, string>,
467 synonyms:
468 | readonly cw.GoogleCloudContentwarehouseV1SynonymSetSynonym[]
469 | undefined,
470): cw.GoogleCloudContentwarehouseV1SynonymSetSynonym[] => {
471 const marker = ownershipSynonymWord(labels);
472 const rest = (synonyms ?? []).filter(
473 (group) =>
474 !(group.words ?? []).some(
475 (word) => word.startsWith("[alc ") || word.startsWith("[alchemy "),
476 ),
477 );
478 return [{ words: [marker] }, ...rest];
479};
480
481export const userSynonyms = (
482 synonyms:
483 | readonly cw.GoogleCloudContentwarehouseV1SynonymSetSynonym[]
484 | undefined,
485): cw.GoogleCloudContentwarehouseV1SynonymSetSynonym[] =>
486 (synonyms ?? []).filter(
487 (group) =>
488 !(group.words ?? []).some(
489 (word) => word.startsWith("[alc ") || word.startsWith("[alchemy "),
490 ),
491 );
492
493export const synonymOwnershipText = (
494 synonyms:
495 | readonly cw.GoogleCloudContentwarehouseV1SynonymSetSynonym[]
496 | undefined,
497) => {
498 for (const group of synonyms ?? []) {
499 for (const word of group.words ?? []) {
500 if (word.startsWith("[alc ") || word.startsWith("[alchemy ")) {
501 return word;
502 }
503 }
504 }
505 return undefined;
506};
507
508export const defaultTextProperty = (
509 name = "title",
510): cw.GoogleCloudContentwarehouseV1PropertyDefinition => ({
511 name,
512 displayName: name,
513 isSearchable: true,
514 isFilterable: true,
515 textTypeOptions: {},
516});
517
518export const appendProperties = (
519 observed:
520 | readonly cw.GoogleCloudContentwarehouseV1PropertyDefinition[]
521 | undefined,
522 desired:
523 | readonly cw.GoogleCloudContentwarehouseV1PropertyDefinition[]
524 | undefined,
525) => {
526 const current = [...(observed ?? [])];
527 const seen = new Set(
528 current
529 .map((item) => item.name?.toLowerCase())
530 .filter((item): item is string => item !== undefined),
531 );
532 let appended = false;
533 for (const property of desired ?? []) {
534 const key = property.name?.toLowerCase();
535 if (key === undefined || seen.has(key)) continue;
536 current.push(property);
537 seen.add(key);
538 appended = true;
539 }
540 return { properties: current, appended };
541};