1import * as aiplatform from "@distilled.cloud/gcp/aiplatform_v1";
2import * as Data from "effect/Data";
3import * as Effect from "effect/Effect";
4import * as Schedule from "effect/Schedule";
5import * as Stream from "effect/Stream";
6import { Unowned } from "../../AdoptPolicy.ts";
7import { isResolved } from "../../Diff.ts";
8import * as Provider from "../../Provider.ts";
9import { Resource } from "../../Resource.ts";
10import { GcpEnvironment } from "../Environment.ts";
11import { createInternalLabels, hasAlchemyLabels } from "../Labels.ts";
12import type { Providers } from "../Providers.ts";
13import { waitForOperation } from "./operations.ts";
14import {
15 fingerprint,
16 normalizeLocation,
17 parseResourceName,
18 specifiedEquals,
19 toPhysicalRfc1035,
20} from "./helpers.ts";
21
22const MAX_NAME_LENGTH = 63;
23
24export type EvaluationSetProps = {
25 /**
26 * Evaluation set id (the `{evaluation_set}` segment). Assigned by
27 * Vertex on create. Provide to target an existing set.
28 */
29 evaluationSetId?: string;
30 /**
31 * Region. Immutable — changing it replaces the set.
32 * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`)
33 */
34 location?: string;
35 /**
36 * Display name. Defaults to a generated id when omitted.
37 */
38 displayName?: string;
39 /**
40 * Evaluation item resource names that belong to this set.
41 */
42 evaluationItems: string[];
43 /**
44 * Static agent configs keyed by agent id.
45 */
46 agentConfigs?: Record<string, aiplatform.GoogleCloudAiplatformV1AgentConfig>;
47 /**
48 * Caller metadata. Evaluation Sets have no labels field, so Alchemy
49 * ownership keys (`alchemy-stack`, `alchemy-stage`, `alchemy-id`) are
50 * merged into this object for `list` / nuke.
51 */
52 metadata?: unknown;
53};
54
55export type EvaluationSet = Resource<
56 "GCP.AIPlatform.EvaluationSet",
57 EvaluationSetProps,
58 {
59 /** Full resource name. */
60 name: string;
61 /** Evaluation set id (last path segment). */
62 evaluationSetId: string;
63 /** Project id. */
64 project: string;
65 /** Location id. */
66 location: string;
67 /** Display name. */
68 displayName: string | undefined;
69 /** Member evaluation item names. */
70 evaluationItems: string[];
71 /** Agent configs. */
72 agentConfigs: aiplatform.GoogleCloudAiplatformV1AgentConfigMap | undefined;
73 /** Caller metadata with Alchemy ownership keys stripped. */
74 metadata: unknown;
75 /** RFC3339 creation timestamp. */
76 createTime: string | undefined;
77 /** RFC3339 last-update timestamp. */
78 updateTime: string | undefined;
79 },
80 never,
81 Providers
82>;
83
84/**
85 * A Vertex AI Evaluation Set — a collection of Evaluation Items evaluated
86 * together.
87 *
88 * Evaluation Sets have no labels field, so Alchemy stamps ownership into
89 * `metadata`. Member items and agent configs update in place; display
90 * name, metadata, and location changes replace the set.
91 *
92 * ### Creating an Evaluation Set
93 * **Example:** Group request items
94 * ```typescript
95 * const set = yield* GCP.AIPlatform.EvaluationSet("Prompts", {
96 * evaluationItems: [item.name],
97 * });
98 * ```
99 *
100 * ### Updating an Evaluation Set
101 * **Example:** Add items
102 * ```typescript
103 * // Same logical id, changed props: the engine updates it in place.
104 * const set = yield* GCP.AIPlatform.EvaluationSet("Prompts", {
105 * evaluationItems: [item.name, extra.name],
106 * });
107 * ```
108 *
109 * @resource
110 * @category AIPlatform
111 */
112export const EvaluationSet = Resource<EvaluationSet>(
113 "GCP.AIPlatform.EvaluationSet",
114);
115
116export class EvaluationSetNotResolved extends Data.TaggedError(
117 "GCP.AIPlatform.EvaluationSetNotResolved",
118)<{
119 name: string;
120}> {}
121
122export class EvaluationSetStillExists extends Data.TaggedError(
123 "GCP.AIPlatform.EvaluationSetStillExists",
124)<{
125 name: string;
126}> {}
127
128const resourceName = (project: string, location: string, setId: string) =>
129 `projects/${project}/locations/${location}/evaluationSets/${setId}`;
130
131const metadataObject = (value: unknown): Record<string, unknown> => {
132 if (value !== null && typeof value === "object" && !Array.isArray(value)) {
133 return { ...(value as Record<string, unknown>) };
134 }
135 if (value === undefined) return {};
136 return { value };
137};
138
139const encodeMetadata = (
140 ownership: Record<string, string>,
141 metadata: unknown,
142): Record<string, unknown> => ({
143 ...metadataObject(metadata),
144 ...ownership,
145});
146
147const stripOwnershipMeta = (metadata: unknown): unknown => {
148 const rest = Object.fromEntries(
149 Object.entries(metadataObject(metadata)).filter(
150 ([key]) => !key.startsWith("alchemy-"),
151 ),
152 );
153 return Object.keys(rest).length === 0 ? undefined : rest;
154};
155
156const metadataLabels = (metadata: unknown): Record<string, string> => {
157 const out: Record<string, string> = {};
158 for (const [key, value] of Object.entries(metadataObject(metadata))) {
159 if (key.startsWith("alchemy-") && typeof value === "string") {
160 out[key] = value;
161 }
162 }
163 return out;
164};
165
166const hasOwnershipMarker = (metadata: unknown) =>
167 Object.keys(metadataLabels(metadata)).length > 0;
168
169const sortedItems = (items: readonly string[] | undefined) =>
170 [...(items ?? [])].sort((left, right) => left.localeCompare(right));
171
172const toAttrs = (
173 set: aiplatform.GoogleCloudAiplatformV1EvaluationSet,
174 project: string,
175) => {
176 const name = set.name ?? "";
177 const parsed = parseResourceName(name, "evaluationSets");
178 return {
179 name,
180 evaluationSetId: parsed.id,
181 project: parsed.project || project,
182 location: parsed.location,
183 displayName: set.displayName,
184 evaluationItems: sortedItems(set.evaluationItems),
185 agentConfigs: set.agentConfigs,
186 metadata: stripOwnershipMeta(set.metadata),
187 createTime: set.createTime,
188 updateTime: set.updateTime,
189 };
190};
191
192const getByName = (name: string) =>
193 aiplatform
194 .getProjectsLocationsEvaluationSets({ name })
195 .pipe(Effect.catchTag("NotFound", () => Effect.succeed(undefined)));
196
197const listOwned = (project: string, location = "-") =>
198 aiplatform.listProjectsLocationsEvaluationSets
199 .pages({
200 parent: `projects/${project}/locations/${location}`,
201 pageSize: 100,
202 })
203 .pipe(
204 Stream.flatMap((page) => Stream.fromIterable(page.evaluationSets ?? [])),
205 Stream.filter((set) => hasOwnershipMarker(set.metadata)),
206 Stream.runCollect,
207 Effect.map((chunk) => Array.from(chunk)),
208 Effect.catchTag("NotFound", () => Effect.succeed([])),
209 );
210
211const findOwned = (id: string, project: string, location?: string) =>
212 Effect.gen(function* () {
213 const sets = yield* listOwned(project, location);
214 for (const set of sets) {
215 if (yield* hasAlchemyLabels(id, metadataLabels(set.metadata))) {
216 return set;
217 }
218 }
219 return undefined;
220 });
221
222const waitUntilGone = (name: string) =>
223 getByName(name).pipe(
224 Effect.flatMap((set) =>
225 set === undefined
226 ? Effect.void
227 : Effect.fail(new EvaluationSetStillExists({ name })),
228 ),
229 Effect.retry({
230 while: (error) =>
231 error._tag === "GCP.AIPlatform.EvaluationSetStillExists",
232 times: 10,
233 schedule: Schedule.spaced("2 seconds"),
234 }),
235 );
236
237export const EvaluationSetProvider = () =>
238 Provider.succeed(EvaluationSet, {
239 stables: ["name", "evaluationSetId", "project", "location", "createTime"],
240
241 diff: Effect.fn(function* ({ news, olds, output }) {
242 const env = yield* GcpEnvironment.current;
243 if (!isResolved(news)) return undefined;
244 const previousId = olds?.evaluationSetId ?? output?.evaluationSetId;
245 const nextId = news.evaluationSetId ?? previousId;
246 const previousLocation = normalizeLocation(
247 olds?.location ?? output?.location,
248 env.region,
249 );
250 const nextLocation = normalizeLocation(
251 news.location ?? output?.location,
252 env.region,
253 );
254 const displayNameChanged =
255 olds !== undefined && news.displayName !== olds.displayName;
256 const metadataChanged =
257 olds !== undefined &&
258 fingerprint(news.metadata ?? null) !==
259 fingerprint(olds.metadata ?? null);
260 const replace =
261 (previousId !== undefined &&
262 nextId !== undefined &&
263 nextId !== previousId) ||
264 previousLocation !== nextLocation ||
265 displayNameChanged ||
266 metadataChanged;
267 if (!replace) return undefined;
268 return { action: "replace" as const, deleteFirst: false };
269 }),
270
271 read: Effect.fn(function* ({ id, olds, output }) {
272 const env = yield* GcpEnvironment.current;
273 const location = normalizeLocation(
274 olds?.location ?? output?.location,
275 env.region,
276 );
277 const setId = olds?.evaluationSetId ?? output?.evaluationSetId;
278 const name =
279 output?.name ??
280 (setId ? resourceName(env.project, location, setId) : undefined);
281 const existing = name
282 ? yield* getByName(name)
283 : yield* findOwned(id, env.project, location);
284 if (existing === undefined) return undefined;
285 const attrs = toAttrs(existing, env.project);
286 return (yield* hasAlchemyLabels(id, metadataLabels(existing.metadata)))
287 ? attrs
288 : Unowned(attrs);
289 }),
290
291 list: () =>
292 Effect.gen(function* () {
293 const env = yield* GcpEnvironment.current;
294 const sets = yield* listOwned(env.project);
295 return sets.map((set) => toAttrs(set, env.project));
296 }),
297
298 reconcile: Effect.fn(function* ({ id, news, output }) {
299 const env = yield* GcpEnvironment.current;
300 const location = normalizeLocation(
301 news.location ?? output?.location,
302 env.region,
303 );
304 const setId = news.evaluationSetId ?? output?.evaluationSetId;
305 const name =
306 output?.name ??
307 (setId ? resourceName(env.project, location, setId) : undefined);
308 const ownership = yield* createInternalLabels(id);
309 const desiredMetadata = encodeMetadata(ownership, news.metadata);
310 const displayName =
311 news.displayName ??
312 (yield* toPhysicalRfc1035(id, undefined, undefined, MAX_NAME_LENGTH));
313 const evaluationItems = sortedItems(news.evaluationItems);
314
315 let current = name
316 ? yield* getByName(name)
317 : yield* findOwned(id, env.project, location);
318
319 if (current === undefined) {
320 current = yield* aiplatform
321 .createProjectsLocationsEvaluationSets({
322 parent: `projects/${env.project}/locations/${location}`,
323 body: {
324 displayName,
325 evaluationItems,
326 agentConfigs: news.agentConfigs,
327 metadata: desiredMetadata,
328 },
329 })
330 .pipe(
331 Effect.catchTag("Conflict", () =>
332 findOwned(id, env.project, location),
333 ),
334 );
335 }
336
337 if (current === undefined) {
338 return yield* new EvaluationSetNotResolved({
339 name: name ?? displayName,
340 });
341 }
342
343 const resolvedName = current.name ?? name ?? "";
344 // Only `evaluation_items` and `agent_configs` are mutable; display
345 // name and metadata changes replace the set (see `diff`).
346 const itemsChanged =
347 fingerprint(sortedItems(current.evaluationItems)) !==
348 fingerprint(evaluationItems);
349 const agentChanged =
350 news.agentConfigs !== undefined &&
351 !specifiedEquals(news.agentConfigs, current.agentConfigs);
352
353 if (itemsChanged || agentChanged) {
354 const updateMask = [
355 itemsChanged ? "evaluation_items" : undefined,
356 agentChanged ? "agent_configs" : undefined,
357 ].filter((field): field is string => field !== undefined);
358
359 current = yield* aiplatform
360 .patchProjectsLocationsEvaluationSets({
361 name: resolvedName,
362 updateMask: updateMask.join(","),
363 body: {
364 name: resolvedName,
365 evaluationItems,
366 agentConfigs: news.agentConfigs,
367 },
368 })
369 .pipe(
370 Effect.retry({
371 while: (error) => error._tag === "Conflict",
372 times: 8,
373 schedule: Schedule.spaced("5 seconds"),
374 }),
375 );
376 }
377
378 return toAttrs(current, env.project);
379 }),
380
381 delete: Effect.fn(function* ({ output }) {
382 const operation = yield* aiplatform
383 .deleteProjectsLocationsEvaluationSets({ name: output.name })
384 .pipe(
385 Effect.catchTag("NotFound", () => Effect.succeed(undefined)),
386 Effect.retry({
387 while: (error) => error._tag === "Conflict",
388 times: 8,
389 schedule: Schedule.spaced("5 seconds"),
390 }),
391 );
392 if (operation !== undefined) {
393 yield* waitForOperation(operation, { notFoundOk: true });
394 }
395 yield* waitUntilGone(output.name);
396 }),
397 });