1import * as ces from "@distilled.cloud/gcp/ces_v1";
2import * as Data from "effect/Data";
3import * as Effect from "effect/Effect";
4import * as Schedule from "effect/Schedule";
5import * as Stream from "effect/Stream";
6import { createPhysicalName } from "../../PhysicalName.ts";
7import { tagRecord } from "../../Tags.ts";
8import {
9 alchemyLabelKeys,
10 createInternalLabels,
11 hasAlchemyLabels,
12 stripInternalLabels,
13} from "../Labels.ts";
14import { isTransientGcpError } from "../Errors.ts";
15
16// CES resource ids must match `[a-zA-Z0-9][a-zA-Z0-9-_]{4,35}`.
17export const MAX_ID_LENGTH = 36;
18export const MAX_DISPLAY_NAME_LENGTH = 128;
19export const MAX_DESCRIPTION_LENGTH = 8000;
20
21export class CesPending extends Data.TaggedError("GCP.CES.Pending")<{
22 name: string;
23}> {}
24
25export const lastSegment = (value: string) => {
26 const trimmed = value.replace(/\/+$/, "");
27 const parts = trimmed.split("/");
28 return parts[parts.length - 1] || trimmed;
29};
30
31export const parentOf = (name: string) => {
32 const parts = name.split("/").filter((part) => part.length > 0);
33 return parts.slice(0, -2).join("/");
34};
35
36export const locationParent = (project: string, location: string) =>
37 `projects/${project}/locations/${location}`;
38
39export const normalizeLocation = (
40 location: string | undefined,
41 fallback: string,
42) => lastSegment(location ?? fallback).toLowerCase();
43
44export const parseResourceName = (
45 name: string,
46 collection: string,
47 fallbackLocation: string,
48) => {
49 const parts = name.split("/").filter((part) => part.length > 0);
50 const collectionAt = parts.lastIndexOf(collection);
51 const locationsAt = parts.lastIndexOf("locations");
52 const projectsAt = parts.lastIndexOf("projects");
53 const appsAt = parts.lastIndexOf("apps");
54 return {
55 project:
56 projectsAt >= 0 && parts[projectsAt + 1] ? parts[projectsAt + 1]! : "",
57 location:
58 locationsAt >= 0 && parts[locationsAt + 1]
59 ? parts[locationsAt + 1]!
60 : fallbackLocation,
61 appId: appsAt >= 0 && parts[appsAt + 1] ? parts[appsAt + 1]! : "",
62 app: appsAt >= 0 ? parts.slice(0, appsAt + 2).join("/") : parentOf(name),
63 id:
64 collectionAt >= 0 && parts[collectionAt + 1]
65 ? parts[collectionAt + 1]!
66 : lastSegment(name),
67 parent:
68 collectionAt > 0
69 ? parts.slice(0, collectionAt).join("/")
70 : parts.slice(0, Math.max(0, parts.length - 1)).join("/"),
71 };
72};
73
74export const expandApp = (value: string, project: string, location: string) =>
75 value.includes("/apps/")
76 ? value.replace(/\/+$/, "")
77 : `${locationParent(project, location)}/apps/${value}`;
78
79export const sameText = (left: string | undefined, right: string | undefined) =>
80 (left ?? "") === (right ?? "");
81
82export const canonical = (value: unknown): unknown => {
83 if (value === undefined || value === null) return undefined;
84 if (typeof value === "boolean" || typeof value === "number") return value;
85 if (typeof value === "string") {
86 return value.length === 0 ? undefined : value;
87 }
88 if (Array.isArray(value)) {
89 const items = value.map(canonical).filter((item) => item !== undefined);
90 return items.length === 0 ? undefined : items;
91 }
92 if (typeof value === "object") {
93 const entries = Object.entries(value as Record<string, unknown>)
94 .map(([key, item]) => [key, canonical(item)] as const)
95 .filter(([, item]) => item !== undefined)
96 .sort(([left], [right]) => left.localeCompare(right));
97 if (entries.length === 0) return undefined;
98 return Object.fromEntries(entries);
99 }
100 return undefined;
101};
102
103export const fingerprint = (value: unknown): string =>
104 JSON.stringify(canonical(value) ?? null);
105
106export const sameJson = (left: unknown, right: unknown) =>
107 fingerprint(left) === fingerprint(right);
108
109export const updateMaskOf = (...fields: Array<string | undefined>) =>
110 fields.filter((field): field is string => field !== undefined).join(",");
111
112export const toPhysicalId = (
113 id: string,
114 explicit: string | undefined,
115 existing: string | undefined,
116 maxLength = MAX_ID_LENGTH,
117) =>
118 Effect.gen(function* () {
119 if (explicit !== undefined) return explicit;
120 if (existing !== undefined) return existing;
121 const generated = yield* createPhysicalName({
122 id,
123 maxLength,
124 lowercase: true,
125 });
126 const next = /^[a-z]/.test(generated)
127 ? generated
128 : `c${generated}`.slice(0, maxLength);
129 return next.length >= 4 ? next : `${next}xxxx`.slice(0, maxLength);
130 });
131
132export const userMetadata = (
133 metadata: Record<string, string | undefined> | null | undefined,
134): Record<string, string> => stripInternalLabels(tagRecord(metadata));
135
136export const hasAlchemyLabelMap = (
137 labels: Record<string, string | undefined> | null | undefined,
138) => Object.keys(labels ?? {}).some((key) => key.startsWith("alchemy-"));
139
140const markerOf = (
141 labels: Record<string, string>,
142 stack: string,
143 stage: string,
144 id: string,
145) =>
146 `[alchemy ${alchemyLabelKeys.stack}=${stack} ${alchemyLabelKeys.stage}=${stage} ${alchemyLabelKeys.id}=${id}]`;
147
148const compactMarkerOf = (stack: string, stage: string, id: string) =>
149 `[alc ${stack} ${stage} ${id}]`;
150
151const shrinkMarker = (
152 labels: Record<string, string>,
153 maxLength: number,
154 build: (stack: string, stage: string, id: string) => string,
155) => {
156 let stack = labels[alchemyLabelKeys.stack] ?? "x";
157 let stage = labels[alchemyLabelKeys.stage] ?? "x";
158 let id = labels[alchemyLabelKeys.id] ?? "x";
159 let marker = build(stack, stage, id);
160 while (
161 marker.length > maxLength &&
162 (stack.length > 1 || stage.length > 1 || id.length > 1)
163 ) {
164 if (stack.length >= stage.length && stack.length >= id.length) {
165 stack = stack.slice(0, -1);
166 } else if (stage.length >= id.length) {
167 stage = stage.slice(0, -1);
168 } else {
169 id = id.slice(0, -1);
170 }
171 marker = build(stack, stage, id);
172 }
173 return marker.slice(0, maxLength);
174};
175
176export const encodeOwnership = (
177 labels: Record<string, string>,
178 text: string | undefined,
179): string => {
180 const marker = shrinkMarker(
181 labels,
182 MAX_DESCRIPTION_LENGTH,
183 (stack, stage, id) => markerOf(labels, stack, stage, id),
184 );
185 const trimmed = text?.trim();
186 return trimmed && trimmed.length > 0 ? `${marker}\n${trimmed}` : marker;
187};
188
189export const encodeOwnershipLine = (
190 labels: Record<string, string>,
191 text: string | undefined,
192 maxLength = MAX_DISPLAY_NAME_LENGTH,
193): string => {
194 const trimmed = text?.replace(/[\r\n]+/g, " ").trim();
195 // Shrink the marker, not the user text, so the display name round-trips.
196 const budget = trimmed
197 ? Math.max(maxLength - trimmed.length - 1, 16)
198 : maxLength;
199 const marker =
200 budget < 54
201 ? shrinkMarker(labels, budget, compactMarkerOf)
202 : shrinkMarker(labels, budget, (stack, stage, id) =>
203 markerOf(labels, stack, stage, id),
204 );
205 if (!trimmed) return marker;
206 return `${marker} ${trimmed}`.slice(0, maxLength);
207};
208
209export const parseOwnership = (
210 text: string | undefined,
211): {
212 labels: Record<string, string>;
213 text: string | undefined;
214} => {
215 if (text?.startsWith("[alc ")) {
216 const end = text.indexOf("]");
217 if (end < 0) return { labels: {}, text };
218 const parts = text.slice("[alc ".length, end).trim().split(/\s+/);
219 const labels: Record<string, string> = {};
220 if (parts[0]) labels[alchemyLabelKeys.stack] = parts[0]!;
221 if (parts[1]) labels[alchemyLabelKeys.stage] = parts[1]!;
222 if (parts[2]) labels[alchemyLabelKeys.id] = parts[2]!;
223 const rest = text.slice(end + 1).replace(/^[\s\n]+/, "");
224 return { labels, text: rest.length > 0 ? rest : undefined };
225 }
226 if (!text?.startsWith("[alchemy ")) {
227 return { labels: {}, text };
228 }
229 const end = text.indexOf("]");
230 if (end < 0) return { labels: {}, text };
231 const labels: Record<string, string> = {};
232 for (const part of text.slice("[alchemy ".length, end).split(/\s+/)) {
233 const eq = part.indexOf("=");
234 if (eq > 0) {
235 labels[part.slice(0, eq)] = part.slice(eq + 1);
236 }
237 }
238 const rest = text.slice(end + 1).replace(/^[\s\n]+/, "");
239 return { labels, text: rest.length > 0 ? rest : undefined };
240};
241
242export const hasOwnershipMarker = (text: string | undefined) =>
243 Object.keys(parseOwnership(text).labels).some((key) =>
244 key.startsWith("alchemy-"),
245 );
246
247const prefixMatch = (expected: string, observed: string) =>
248 expected === observed ||
249 expected.startsWith(observed) ||
250 observed.startsWith(expected);
251
252export const ownedByAlchemy = (id: string, text: string | undefined) =>
253 Effect.gen(function* () {
254 const expected = yield* createInternalLabels(id);
255 const { labels } = parseOwnership(text);
256 if (!hasOwnershipMarker(text)) return false;
257 const exact = yield* hasAlchemyLabels(id, labels);
258 if (exact) return true;
259 return (
260 prefixMatch(
261 expected[alchemyLabelKeys.stack] ?? "",
262 labels[alchemyLabelKeys.stack] ?? "",
263 ) &&
264 prefixMatch(
265 expected[alchemyLabelKeys.stage] ?? "",
266 labels[alchemyLabelKeys.stage] ?? "",
267 ) &&
268 prefixMatch(
269 expected[alchemyLabelKeys.id] ?? "",
270 labels[alchemyLabelKeys.id] ?? "",
271 )
272 );
273 });
274
275export const ownedByAlchemyLabels = (
276 id: string,
277 labels: Record<string, string | undefined> | null | undefined,
278) =>
279 Effect.gen(function* () {
280 const expected = yield* createInternalLabels(id);
281 const observed = tagRecord(labels);
282 if (!hasAlchemyLabelMap(observed)) return false;
283 const exact = yield* hasAlchemyLabels(id, observed);
284 if (exact) return true;
285 return (
286 prefixMatch(
287 expected[alchemyLabelKeys.stack] ?? "",
288 observed[alchemyLabelKeys.stack] ?? "",
289 ) &&
290 prefixMatch(
291 expected[alchemyLabelKeys.stage] ?? "",
292 observed[alchemyLabelKeys.stage] ?? "",
293 ) &&
294 prefixMatch(
295 expected[alchemyLabelKeys.id] ?? "",
296 observed[alchemyLabelKeys.id] ?? "",
297 )
298 );
299 });
300
301export const replaceOnIdentity = (input: {
302 previousId?: string;
303 nextId?: string;
304 previousParent?: string;
305 nextParent?: string;
306 extra?: boolean;
307}) => {
308 if (input.extra === true) {
309 return { action: "replace" as const, deleteFirst: false };
310 }
311 if (
312 input.previousId !== undefined &&
313 input.nextId !== undefined &&
314 input.previousId !== input.nextId
315 ) {
316 return { action: "replace" as const, deleteFirst: false };
317 }
318 if (
319 input.previousParent !== undefined &&
320 input.nextParent !== undefined &&
321 input.previousParent !== input.nextParent
322 ) {
323 return { action: "replace" as const, deleteFirst: false };
324 }
325 return undefined;
326};
327
328export const retryTransient = <A, E extends { readonly _tag: string }, R>(
329 effect: Effect.Effect<A, E, R>,
330) =>
331 effect.pipe(
332 Effect.retry({
333 while: isTransientGcpError,
334 times: 8,
335 schedule: Schedule.exponential("250 millis"),
336 }),
337 );
338
339export class DeleteNotConfirmed extends Data.TaggedError(
340 "GCP.CES.DeleteNotConfirmed",
341)<{}> {}
342
343/** Poll until the resource is gone; fails if it is still readable after ~60s. */
344export const waitUntilGone = <A, E, R>(
345 get: Effect.Effect<A | undefined, E, R>,
346) =>
347 get.pipe(
348 Effect.repeat({
349 schedule: Schedule.spaced("2 seconds"),
350 until: (value) => value === undefined,
351 times: 30,
352 }),
353 Effect.flatMap((value): Effect.Effect<void, DeleteNotConfirmed> =>
354 value === undefined ? Effect.void : Effect.fail(new DeleteNotConfirmed()),
355 ),
356 );
357
358export const waitForVisible = <A, E, R>(
359 get: Effect.Effect<A | undefined, E, R>,
360) =>
361 get.pipe(
362 Effect.filterOrFail(
363 (value): value is A => value !== undefined,
364 () => new CesPending({ name: "" }),
365 ),
366 Effect.retry({
367 // New apps can take a couple of minutes to become readable.
368 while: (error) => error instanceof CesPending,
369 times: 40,
370 schedule: Schedule.spaced("3 seconds"),
371 }),
372 Effect.catchIf(
373 (error): error is CesPending => error instanceof CesPending,
374 () => Effect.succeed(undefined),
375 ),
376 );
377
378const emptyList = <A>() => Effect.succeed<A[]>([]);
379
380type ListedError = ces.NotFound | ces.Forbidden | ces.GcpOpError;
381
382export const collectPages = <Page, Item, R>(
383 pages: Stream.Stream<Page, ListedError, R>,
384 items: (page: Page) => readonly Item[] | null | undefined,
385) =>
386 pages.pipe(
387 Stream.flatMap((page) => Stream.fromIterable(items(page) ?? [])),
388 Stream.runCollect,
389 Effect.map((chunk) => Array.from(chunk)),
390 Effect.catchTag("NotFound", () => emptyList<Item>()),
391 );
392
393export const listApps = (parent: string) =>
394 parent.length === 0
395 ? emptyList<ces.App>()
396 : collectPages(
397 ces.listProjectsLocationsApps.pages({ parent, pageSize: 100 }),
398 (page) => page.apps,
399 );
400
401export const namedApps = (project: string, location: string) =>
402 listApps(locationParent(project, location)).pipe(
403 Effect.map((apps) =>
404 apps.filter(
405 (app): app is ces.App & { name: string } =>
406 typeof app.name === "string" && app.name.length > 0,
407 ),
408 ),
409 );
410
411export const forEachApp = <A, E, R>(
412 project: string,
413 list: (parent: string) => Effect.Effect<A[], E, R>,
414 location: string,
415) =>
416 Effect.gen(function* () {
417 // Children of Alchemy-owned apps (ownership lives in app metadata).
418 const apps = (yield* namedApps(project, location)).filter((app) =>
419 hasAlchemyLabelMap(app.metadata),
420 );
421 const groups = yield* Effect.forEach(apps, (app) => list(app.name), {
422 concurrency: 4,
423 });
424 return groups.flat();
425 });
426
427export const DEFAULT_OPENAPI_SCHEMA = JSON.stringify({
428 openapi: "3.0.0",
429 info: { title: "Alchemy", version: "1.0.0" },
430 paths: {
431 "/ping": {
432 get: {
433 operationId: "ping",
434 responses: {
435 "200": { description: "ok" },
436 },
437 },
438 },
439 },
440});
441
442export const DEFAULT_CHANNEL_PROFILE: ces.ChannelProfile = {
443 channelType: "API",
444 profileId: "api",
445};
446
447export const toolKind = (tool: {
448 fileSearchTool?: unknown;
449 pythonFunction?: unknown;
450 googleSearchTool?: unknown;
451 mcpTool?: unknown;
452 remoteAgentTool?: unknown;
453 widgetTool?: unknown;
454 systemTool?: unknown;
455 openApiTool?: unknown;
456 connectorTool?: unknown;
457 dataStoreTool?: unknown;
458 agentTool?: unknown;
459 clientFunction?: unknown;
460}) => {
461 if (tool.fileSearchTool) return "fileSearchTool";
462 if (tool.pythonFunction) return "pythonFunction";
463 if (tool.googleSearchTool) return "googleSearchTool";
464 if (tool.mcpTool) return "mcpTool";
465 if (tool.remoteAgentTool) return "remoteAgentTool";
466 if (tool.widgetTool) return "widgetTool";
467 if (tool.systemTool) return "systemTool";
468 if (tool.openApiTool) return "openApiTool";
469 if (tool.connectorTool) return "connectorTool";
470 if (tool.dataStoreTool) return "dataStoreTool";
471 if (tool.agentTool) return "agentTool";
472 return "clientFunction";
473};
474
475export const toolsetKind = (toolset: {
476 openApiToolset?: unknown;
477 mcpToolset?: unknown;
478 connectorToolset?: unknown;
479}) => {
480 if (toolset.mcpToolset) return "mcpToolset";
481 if (toolset.connectorToolset) return "connectorToolset";
482 return "openApiToolset";
483};
484
485export const toolOwnershipText = (tool: ces.Tool) =>
486 tool.clientFunction?.description ??
487 tool.pythonFunction?.description ??
488 tool.googleSearchTool?.description ??
489 tool.fileSearchTool?.description ??
490 tool.remoteAgentTool?.description ??
491 tool.widgetTool?.description ??
492 tool.openApiTool?.description ??
493 tool.connectorTool?.description ??
494 tool.dataStoreTool?.description ??
495 tool.agentTool?.description ??
496 tool.systemTool?.description ??
497 tool.mcpTool?.description;
498
499export const stampToolDescription = (
500 tool: ces.Tool,
501 ownership: Record<string, string>,
502): ces.Tool => {
503 const stamp = (text: string | undefined) => encodeOwnership(ownership, text);
504 if (tool.pythonFunction) {
505 return {
506 ...tool,
507 pythonFunction: {
508 ...tool.pythonFunction,
509 description: stamp(tool.pythonFunction.description),
510 },
511 };
512 }
513 if (tool.googleSearchTool) {
514 return {
515 ...tool,
516 googleSearchTool: {
517 ...tool.googleSearchTool,
518 description: stamp(tool.googleSearchTool.description),
519 },
520 };
521 }
522 if (tool.fileSearchTool) {
523 return {
524 ...tool,
525 fileSearchTool: {
526 ...tool.fileSearchTool,
527 description: stamp(tool.fileSearchTool.description),
528 },
529 };
530 }
531 if (tool.remoteAgentTool) {
532 return {
533 ...tool,
534 remoteAgentTool: {
535 ...tool.remoteAgentTool,
536 description: stamp(tool.remoteAgentTool.description),
537 },
538 };
539 }
540 if (tool.widgetTool) {
541 return {
542 ...tool,
543 widgetTool: {
544 ...tool.widgetTool,
545 description: stamp(tool.widgetTool.description),
546 },
547 };
548 }
549 if (tool.openApiTool) {
550 return {
551 ...tool,
552 openApiTool: {
553 ...tool.openApiTool,
554 description: stamp(tool.openApiTool.description),
555 },
556 };
557 }
558 if (tool.connectorTool) {
559 return {
560 ...tool,
561 connectorTool: {
562 ...tool.connectorTool,
563 description: stamp(tool.connectorTool.description),
564 },
565 };
566 }
567 if (tool.dataStoreTool) {
568 return {
569 ...tool,
570 dataStoreTool: {
571 ...tool.dataStoreTool,
572 description: stamp(tool.dataStoreTool.description),
573 },
574 };
575 }
576 if (tool.agentTool) {
577 return {
578 ...tool,
579 agentTool: {
580 ...tool.agentTool,
581 description: stamp(tool.agentTool.description),
582 },
583 };
584 }
585 if (tool.systemTool) {
586 return {
587 ...tool,
588 systemTool: {
589 ...tool.systemTool,
590 description: stamp(tool.systemTool.description),
591 },
592 };
593 }
594 if (tool.mcpTool) {
595 return {
596 ...tool,
597 mcpTool: {
598 ...tool.mcpTool,
599 description: stamp(tool.mcpTool.description),
600 },
601 };
602 }
603 return {
604 ...tool,
605 clientFunction: {
606 name: tool.clientFunction?.name ?? "alchemy_fn",
607 description: stamp(tool.clientFunction?.description),
608 parameters: tool.clientFunction?.parameters,
609 response: tool.clientFunction?.response,
610 },
611 };
612};
613
614export const unstampToolDescription = (tool: ces.Tool): ces.Tool => {
615 const unstamp = (text: string | undefined) => parseOwnership(text).text;
616 if (tool.pythonFunction) {
617 return {
618 ...tool,
619 pythonFunction: {
620 ...tool.pythonFunction,
621 description: unstamp(tool.pythonFunction.description),
622 },
623 };
624 }
625 if (tool.googleSearchTool) {
626 return {
627 ...tool,
628 googleSearchTool: {
629 ...tool.googleSearchTool,
630 description: unstamp(tool.googleSearchTool.description),
631 },
632 };
633 }
634 if (tool.fileSearchTool) {
635 return {
636 ...tool,
637 fileSearchTool: {
638 ...tool.fileSearchTool,
639 description: unstamp(tool.fileSearchTool.description),
640 },
641 };
642 }
643 if (tool.remoteAgentTool) {
644 return {
645 ...tool,
646 remoteAgentTool: {
647 ...tool.remoteAgentTool,
648 description: unstamp(tool.remoteAgentTool.description),
649 },
650 };
651 }
652 if (tool.widgetTool) {
653 return {
654 ...tool,
655 widgetTool: {
656 ...tool.widgetTool,
657 description: unstamp(tool.widgetTool.description),
658 },
659 };
660 }
661 if (tool.openApiTool) {
662 return {
663 ...tool,
664 openApiTool: {
665 ...tool.openApiTool,
666 description: unstamp(tool.openApiTool.description),
667 },
668 };
669 }
670 if (tool.connectorTool) {
671 return {
672 ...tool,
673 connectorTool: {
674 ...tool.connectorTool,
675 description: unstamp(tool.connectorTool.description),
676 },
677 };
678 }
679 if (tool.dataStoreTool) {
680 return {
681 ...tool,
682 dataStoreTool: {
683 ...tool.dataStoreTool,
684 description: unstamp(tool.dataStoreTool.description),
685 },
686 };
687 }
688 if (tool.agentTool) {
689 return {
690 ...tool,
691 agentTool: {
692 ...tool.agentTool,
693 description: unstamp(tool.agentTool.description),
694 },
695 };
696 }
697 if (tool.systemTool) {
698 return {
699 ...tool,
700 systemTool: {
701 ...tool.systemTool,
702 description: unstamp(tool.systemTool.description),
703 },
704 };
705 }
706 if (tool.mcpTool) {
707 return {
708 ...tool,
709 mcpTool: {
710 ...tool.mcpTool,
711 description: unstamp(tool.mcpTool.description),
712 },
713 };
714 }
715 if (tool.clientFunction) {
716 return {
717 ...tool,
718 clientFunction: {
719 ...tool.clientFunction,
720 description: unstamp(tool.clientFunction.description),
721 },
722 };
723 }
724 return tool;
725};