1import * as cloudsearch from "@distilled.cloud/gcp/cloudsearch_v1";
2import * as Data from "effect/Data";
3import * as Effect from "effect/Effect";
4import * as Schedule from "effect/Schedule";
5import * as Stream from "effect/Stream";
6import { createPhysicalName } from "../../PhysicalName.ts";
7import {
8 alchemyLabelKeys,
9 createInternalLabels,
10 hasAlchemyLabels,
11} from "../Labels.ts";
12import { waitForOperation as waitForGcpOperation } from "../Operation.ts";
13
14export const DATASOURCE_PREFIX = "datasources/";
15export const SEARCH_APPLICATION_PREFIX = "searchapplications/";
16export const MAX_DISPLAY_NAME_LENGTH = 300;
17export const MAX_SHORT_NAME_LENGTH = 32;
18
19const RESERVED_SHORT_NAMES = new Set([
20 "mail",
21 "gmail",
22 "docs",
23 "drive",
24 "groups",
25 "sites",
26 "calendar",
27 "hangouts",
28 "gplus",
29 "keep",
30 "people",
31 "teams",
32]);
33
34export class ResourceNotResolved extends Data.TaggedError(
35 "GCP.CloudSearch.ResourceNotResolved",
36)<{
37 name: string;
38}> {}
39
40export const lastSegment = (value: string) => {
41 const trimmed = value.replace(/\/+$/, "");
42 const parts = trimmed.split("/");
43 return parts[parts.length - 1] || trimmed;
44};
45
46export const sameText = (left: string | undefined, right: string | undefined) =>
47 (left ?? "") === (right ?? "");
48
49export const sameBoolean = (
50 left: boolean | undefined,
51 right: boolean | undefined,
52) => (left ?? false) === (right ?? false);
53
54export const jsonEqual = (left: unknown, right: unknown) =>
55 JSON.stringify(left ?? null) === JSON.stringify(right ?? null);
56
57export const updateMaskOf = (...fields: Array<string | undefined>) =>
58 fields.filter((field): field is string => field !== undefined).join(",");
59
60const markerOf = (
61 _labels: Record<string, string>,
62 stack: string,
63 stage: string,
64 id: string,
65) =>
66 `[alchemy ${alchemyLabelKeys.stack}=${stack} ${alchemyLabelKeys.stage}=${stage} ${alchemyLabelKeys.id}=${id}]`;
67
68const fitMarker = (labels: Record<string, string>, maxLength: number) => {
69 let stack = labels[alchemyLabelKeys.stack] ?? "x";
70 let stage = labels[alchemyLabelKeys.stage] ?? "x";
71 let id = labels[alchemyLabelKeys.id] ?? "x";
72 let marker = markerOf(labels, stack, stage, id);
73 while (
74 marker.length > maxLength &&
75 (stack.length > 1 || stage.length > 1 || id.length > 1)
76 ) {
77 if (stack.length >= stage.length && stack.length >= id.length) {
78 stack = stack.slice(0, -1);
79 } else if (stage.length >= id.length) {
80 stage = stage.slice(0, -1);
81 } else {
82 id = id.slice(0, -1);
83 }
84 marker = markerOf(labels, stack, stage, id);
85 }
86 return marker.slice(0, maxLength);
87};
88
89export const encodeOwnershipLine = (
90 labels: Record<string, string>,
91 text: string | undefined,
92 maxLength = MAX_DISPLAY_NAME_LENGTH,
93): string => {
94 const trimmed = text?.replace(/[\r\n]+/g, " ").trim();
95 if (!trimmed) return fitMarker(labels, maxLength);
96 const minMarker = 24;
97 const reserved = Math.min(
98 trimmed.length + 1,
99 Math.max(0, maxLength - minMarker),
100 );
101 const marker = fitMarker(labels, maxLength - reserved);
102 return `${marker} ${trimmed}`.slice(0, maxLength);
103};
104
105export const parseOwnership = (
106 text: string | undefined,
107): {
108 labels: Record<string, string>;
109 text: string | undefined;
110} => {
111 if (!text?.startsWith("[alchemy ")) {
112 return { labels: {}, text };
113 }
114 const end = text.indexOf("]");
115 if (end < 0) return { labels: {}, text };
116 const labels: Record<string, string> = {};
117 for (const part of text.slice("[alchemy ".length, end).split(/\s+/)) {
118 const eq = part.indexOf("=");
119 if (eq > 0) {
120 labels[part.slice(0, eq)] = part.slice(eq + 1);
121 }
122 }
123 const rest = text.slice(end + 1).replace(/^[\s\n]+/, "");
124 return { labels, text: rest.length > 0 ? rest : undefined };
125};
126
127export const hasOwnershipMarker = (text: string | undefined) => {
128 if (
129 Object.keys(parseOwnership(text).labels).some((key) =>
130 key.startsWith("alchemy-"),
131 )
132 ) {
133 return true;
134 }
135 return (text ?? "").toLowerCase().includes("alchemy-");
136};
137
138const prefixMatch = (expected: string, observed: string) =>
139 expected === observed ||
140 expected.startsWith(observed) ||
141 observed.startsWith(expected);
142
143export const ownedByAlchemy = (id: string, text: string | undefined) =>
144 Effect.gen(function* () {
145 const expected = yield* createInternalLabels(id);
146 const { labels } = parseOwnership(text);
147 if (!hasOwnershipMarker(text)) return false;
148 const exact = yield* hasAlchemyLabels(id, labels);
149 if (exact) return true;
150 return (
151 prefixMatch(
152 expected[alchemyLabelKeys.stack] ?? "",
153 labels[alchemyLabelKeys.stack] ?? "",
154 ) &&
155 prefixMatch(
156 expected[alchemyLabelKeys.stage] ?? "",
157 labels[alchemyLabelKeys.stage] ?? "",
158 ) &&
159 prefixMatch(
160 expected[alchemyLabelKeys.id] ?? "",
161 labels[alchemyLabelKeys.id] ?? "",
162 )
163 );
164 });
165
166export const ownershipLabels = (id: string) => createInternalLabels(id);
167
168export const toGeneratedName = (
169 id: string,
170 requested: string | undefined,
171 existing: string | undefined,
172 maxLength = 40,
173) =>
174 Effect.gen(function* () {
175 if (requested !== undefined && requested.length > 0) {
176 return requested.slice(0, maxLength);
177 }
178 if (existing !== undefined && existing.length > 0) {
179 return existing.slice(0, maxLength);
180 }
181 const generated = yield* createPhysicalName({
182 id,
183 maxLength,
184 lowercase: true,
185 });
186 const next = /^[a-z]/.test(generated)
187 ? generated
188 : `c${generated}`.slice(0, maxLength);
189 return next.length >= 4 ? next : `${next}xxxx`.slice(0, maxLength);
190 });
191
192export const sanitizeShortName = (value: string): string => {
193 let next = value.replace(/[^a-zA-Z0-9]/g, "").toLowerCase();
194 if (next.startsWith("google")) next = `a${next}`;
195 if (RESERVED_SHORT_NAMES.has(next)) next = `a${next}`;
196 if (!/^[a-z]/.test(next)) next = `a${next}`;
197 next = next.slice(0, MAX_SHORT_NAME_LENGTH);
198 if (next.length < 4) next = `${next}src`.slice(0, MAX_SHORT_NAME_LENGTH);
199 if (RESERVED_SHORT_NAMES.has(next) || next.startsWith("google")) {
200 next = `a${next}`.slice(0, MAX_SHORT_NAME_LENGTH);
201 }
202 return next;
203};
204
205export const toShortName = (
206 id: string,
207 requested: string | undefined,
208 existing: string | undefined,
209) =>
210 Effect.gen(function* () {
211 if (requested !== undefined && requested.length > 0) {
212 return sanitizeShortName(requested);
213 }
214 if (existing !== undefined && existing.length > 0) {
215 return sanitizeShortName(existing);
216 }
217 const generated = yield* createPhysicalName({
218 id,
219 maxLength: MAX_SHORT_NAME_LENGTH,
220 lowercase: true,
221 });
222 return sanitizeShortName(generated);
223 });
224
225const emptyList = <A>() => Effect.succeed([] as A[]);
226
227export const catchMissing = <A, E extends { readonly _tag: string }, R>(
228 effect: Effect.Effect<A, E, R>,
229) =>
230 effect.pipe(
231 Effect.catchIf(
232 (error): error is E & { readonly _tag: "NotFound" } =>
233 error._tag === "NotFound",
234 () => Effect.succeed(undefined),
235 ),
236 );
237
238export const ignoreMissing = <E extends { readonly _tag: string }, R>(
239 effect: Effect.Effect<unknown, E, R>,
240) =>
241 effect.pipe(
242 Effect.catchIf(
243 (error): error is E & { readonly _tag: "NotFound" } =>
244 error._tag === "NotFound",
245 () => Effect.void,
246 ),
247 );
248
249export const toDatasourceName = (value: string) => {
250 if (value.length === 0) return value;
251 return value.startsWith(DATASOURCE_PREFIX)
252 ? value
253 : `${DATASOURCE_PREFIX}${value}`;
254};
255
256export const datasourceIdOf = (name: string) =>
257 name.startsWith(DATASOURCE_PREFIX)
258 ? name.slice(DATASOURCE_PREFIX.length)
259 : lastSegment(name);
260
261export const toSearchApplicationName = (value: string) => {
262 if (value.length === 0) return value;
263 if (value.startsWith(SEARCH_APPLICATION_PREFIX)) return value;
264 if (value.startsWith("applications/")) {
265 return `${SEARCH_APPLICATION_PREFIX}${value.slice("applications/".length)}`;
266 }
267 return `${SEARCH_APPLICATION_PREFIX}${value}`;
268};
269
270export const searchApplicationIdOf = (name: string) => {
271 if (name.startsWith(SEARCH_APPLICATION_PREFIX)) {
272 return name.slice(SEARCH_APPLICATION_PREFIX.length);
273 }
274 if (name.startsWith("applications/")) {
275 return name.slice("applications/".length);
276 }
277 return lastSegment(name);
278};
279
280const stringField = (value: unknown, key: string): string | undefined => {
281 if (value === null || value === undefined || typeof value !== "object") {
282 return undefined;
283 }
284 const field = (value as Record<string, unknown>)[key];
285 return typeof field === "string" && field.length > 0 ? field : undefined;
286};
287
288export const operationResourceName = (operation: cloudsearch.Operation) =>
289 stringField(operation.response, "name") ??
290 stringField(operation.metadata, "name");
291
292/** Wait for an operation through the shared GCP waiter. */
293export const waitForOperation = (
294 operation: cloudsearch.Operation,
295 options?: { notFoundOk?: boolean; alreadyExistsOk?: boolean },
296) =>
297 waitForGcpOperation(
298 operation,
299 (name) =>
300 cloudsearch
301 .getOperations({ name })
302 .pipe(
303 Effect.catchTag("NotFound", (error) =>
304 options?.notFoundOk === true
305 ? Effect.succeed<cloudsearch.Operation>({ name, done: true })
306 : Effect.fail(error),
307 ),
308 ),
309 { budget: "10 minutes" },
310 ).pipe(
311 // ALREADY_EXISTS (6): a concurrent create won the race. NOT_FOUND (5)
312 // is success for a delete.
313 Effect.catchIf(
314 (error) =>
315 error._tag === "GCP.OperationFailed" &&
316 ((options?.alreadyExistsOk !== false && error.code === 6) ||
317 (options?.notFoundOk === true && error.code === 5)),
318 () => Effect.void,
319 ),
320 // Re-read the finished operation for its typed response and metadata.
321 Effect.flatMap(() =>
322 operation.name === undefined || operation.name.length === 0
323 ? Effect.succeed(operation)
324 : cloudsearch
325 .getOperations({ name: operation.name })
326 .pipe(Effect.catchTag("NotFound", () => Effect.succeed(operation))),
327 ),
328 );
329
330export const waitUntilExists = <A, E, R>(
331 get: Effect.Effect<A | undefined, E, R>,
332 name: string,
333): Effect.Effect<NonNullable<A>, E | ResourceNotResolved, R> =>
334 get.pipe(
335 Effect.filterOrFail(
336 (value): value is NonNullable<A> => value !== undefined,
337 () => new ResourceNotResolved({ name }),
338 ),
339 Effect.retry({
340 while: (error) => error instanceof ResourceNotResolved,
341 times: 8,
342 schedule: Schedule.spaced("2 seconds"),
343 }),
344 );
345
346export const getDatasource = (name: string) =>
347 name.length === 0
348 ? Effect.succeed(undefined)
349 : catchMissing(
350 cloudsearch.getSettingsDatasources({ name: toDatasourceName(name) }),
351 );
352
353export const listDatasources = () =>
354 cloudsearch.listSettingsDatasources.pages({ pageSize: 1000 }).pipe(
355 Stream.flatMap((page) => Stream.fromIterable(page.sources ?? [])),
356 Stream.runCollect,
357 Effect.map((chunk) => Array.from(chunk)),
358 Effect.catchTag("NotFound", () => emptyList<cloudsearch.DataSource>()),
359 );
360
361export const listOwnedDatasources = () =>
362 listDatasources().pipe(
363 Effect.map((sources) =>
364 sources.filter((source) => hasOwnershipMarker(source.displayName)),
365 ),
366 );
367
368export const findOwnedDatasource = (id: string) =>
369 Effect.gen(function* () {
370 const sources = yield* listDatasources();
371 for (const source of sources) {
372 if (yield* ownedByAlchemy(id, source.displayName)) {
373 return source;
374 }
375 }
376 return undefined;
377 });
378
379export const getSearchApplication = (name: string) =>
380 name.length === 0
381 ? Effect.succeed(undefined)
382 : catchMissing(
383 cloudsearch.getSettingsSearchapplications({
384 name: toSearchApplicationName(name),
385 }),
386 );
387
388export const listSearchApplications = () =>
389 cloudsearch.listSettingsSearchapplications.pages({ pageSize: 100 }).pipe(
390 Stream.flatMap((page) =>
391 Stream.fromIterable(page.searchApplications ?? []),
392 ),
393 Stream.runCollect,
394 Effect.map((chunk) => Array.from(chunk)),
395 Effect.catchTag("NotFound", () =>
396 emptyList<cloudsearch.SearchApplication>(),
397 ),
398 );
399
400export const listOwnedSearchApplications = () =>
401 listSearchApplications().pipe(
402 Effect.map((apps) =>
403 apps.filter((app) => hasOwnershipMarker(app.displayName)),
404 ),
405 );
406
407export const findOwnedSearchApplication = (id: string) =>
408 Effect.gen(function* () {
409 const apps = yield* listSearchApplications();
410 for (const app of apps) {
411 if (yield* ownedByAlchemy(id, app.displayName)) {
412 return app;
413 }
414 }
415 return undefined;
416 });
417
418export const findDatasourceByDisplayName = (displayName: string) =>
419 listDatasources().pipe(
420 Effect.map((sources) =>
421 sources.find((source) => source.displayName === displayName),
422 ),
423 );
424
425export const findSearchApplicationByDisplayName = (displayName: string) =>
426 listSearchApplications().pipe(
427 Effect.map((apps) => apps.find((app) => app.displayName === displayName)),
428 );