1import * as aiplatform from "@distilled.cloud/gcp/aiplatform_v1";
2import * as Data from "effect/Data";
3import * as Effect from "effect/Effect";
4import * as Schedule from "effect/Schedule";
5import { Unowned } from "../../AdoptPolicy.ts";
6import { isResolved } from "../../Diff.ts";
7import * as Provider from "../../Provider.ts";
8import { Resource } from "../../Resource.ts";
9import { GcpEnvironment } from "../Environment.ts";
10import { createInternalLabels, hasAlchemyLabels } from "../Labels.ts";
11import type { Providers } from "../Providers.ts";
12import { listLocations } from "./names.ts";
13import { resourceNameFromOperation, waitForOperation } from "./operations.ts";
14import {
15 AiPlatformNotResolved,
16 AiPlatformStillExists,
17 collectPages,
18 encodeDisplayName,
19 hasDisplayNameOwnership,
20 jsonEqual,
21 locationParent,
22 normalizeLocation,
23 ownedById,
24 parseDisplayName,
25 parseResourceName,
26} from "./shared.ts";
27
28const COLLECTION = "onlineEvaluators";
29
30export type OnlineEvaluatorRandomSampling = {
31 /** Percentage of traces to sample (`1`–`100`). */
32 percentage?: number;
33};
34
35export type OnlineEvaluatorConfig = {
36 /** Random sampling method. */
37 randomSampling?: OnlineEvaluatorRandomSampling;
38 /** Max evaluations per 10-minute run (`"0"` is unbounded). */
39 maxEvaluatedSamplesPerRun?: string;
40};
41
42export type MetricSource = {
43 /** Registered metric resource name. */
44 metricResourceName?: string;
45};
46
47export type CloudObservability = {
48 /** OpenTelemetry semantic convention version (e.g. `"1.39.0"`). */
49 openTelemetry?: { semconvVersion?: string };
50 /** Log view used to query logs. */
51 logView?: string;
52 /** Trace view used to query traces. */
53 traceView?: string;
54};
55
56export type OnlineEvaluatorProps = {
57 /**
58 * Vertex AI location. Immutable — changing it replaces the evaluator.
59 * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`)
60 */
61 location?: string;
62 /**
63 * Human-readable name (max 63 characters). OnlineEvaluator has no
64 * labels field, so Alchemy ownership is stored in a compact `[alc …]`
65 * displayName prefix for `list` / nuke.
66 */
67 displayName?: string;
68 /**
69 * Agent resource the evaluator scores. Immutable.
70 */
71 agentResource: string;
72 /**
73 * Metric sources (max 25). At least one is required.
74 */
75 metricSources: MetricSource[];
76 /**
77 * Sampling configuration. Defaults to 10% random sampling.
78 */
79 config?: OnlineEvaluatorConfig;
80 /**
81 * Cloud Observability (Trace / Logging) data source.
82 */
83 cloudObservability?: CloudObservability;
84};
85
86export type OnlineEvaluator = Resource<
87 "GCP.AIPlatform.OnlineEvaluator",
88 OnlineEvaluatorProps,
89 {
90 /** Full resource name. */
91 name: string;
92 /** Evaluator id (last path segment). */
93 onlineEvaluatorId: string;
94 /** Project id. */
95 project: string;
96 /** Location id. */
97 location: string;
98 /** User display name with the Alchemy ownership prefix stripped. */
99 displayName: string | undefined;
100 /** Agent resource being evaluated. */
101 agentResource: string | undefined;
102 /** Server-reported state (`ACTIVE`, `SUSPENDED`, `FAILED`, …). */
103 state: string | undefined;
104 /** RFC3339 creation timestamp. */
105 createTime: string | undefined;
106 /** RFC3339 last-update timestamp. */
107 updateTime: string | undefined;
108 },
109 never,
110 Providers
111>;
112
113/**
114 * A Vertex AI Online Evaluator — periodically scores agent traces against
115 * registered metrics.
116 *
117 * OnlineEvaluator has no labels, so Alchemy stamps ownership into the
118 * display name. Changing `location` or `agentResource` replaces the
119 * evaluator. Display name, config, and metric sources update in place.
120 *
121 * ### Creating an Online Evaluator
122 * **Example:** Sample 10% of traces
123 * ```typescript
124 * const evaluator = yield* GCP.AIPlatform.OnlineEvaluator("Quality", {
125 * agentResource: engine.name,
126 * metricSources: [{ metricResourceName: metric.name }],
127 * config: { randomSampling: { percentage: 10 } },
128 * });
129 * ```
130 *
131 * @resource
132 * @category AIPlatform
133 */
134export const OnlineEvaluator = Resource<OnlineEvaluator>(
135 "GCP.AIPlatform.OnlineEvaluator",
136);
137
138export class OnlineEvaluatorNotResolved extends Data.TaggedError(
139 "GCP.AIPlatform.OnlineEvaluatorNotResolved",
140)<{
141 name: string;
142}> {}
143
144const defaultConfig = (
145 config: OnlineEvaluatorConfig | undefined,
146): aiplatform.GoogleCloudAiplatformV1OnlineEvaluatorConfig => ({
147 randomSampling: {
148 percentage: config?.randomSampling?.percentage ?? 10,
149 },
150 maxEvaluatedSamplesPerRun: config?.maxEvaluatedSamplesPerRun,
151});
152
153const toAttrs = (
154 evaluator: aiplatform.GoogleCloudAiplatformV1OnlineEvaluator,
155 project: string,
156) => {
157 const name = evaluator.name ?? "";
158 const parsed = parseResourceName(name, COLLECTION);
159 const display = parseDisplayName(evaluator.displayName);
160 return {
161 name,
162 onlineEvaluatorId: parsed.id,
163 project: parsed.project || project,
164 location: parsed.location,
165 displayName: display.displayName,
166 agentResource: evaluator.agentResource,
167 state: evaluator.state,
168 createTime: evaluator.createTime,
169 updateTime: evaluator.updateTime,
170 };
171};
172
173const getByName = (name: string) =>
174 aiplatform
175 .getProjectsLocationsOnlineEvaluators({ name })
176 .pipe(Effect.catchTag("NotFound", () => Effect.succeed(undefined)));
177
178const waitUntilExists = (name: string) =>
179 getByName(name).pipe(
180 Effect.filterOrFail(
181 (
182 evaluator,
183 ): evaluator is aiplatform.GoogleCloudAiplatformV1OnlineEvaluator =>
184 evaluator !== undefined,
185 () => new AiPlatformNotResolved({ name }),
186 ),
187 Effect.retry({
188 while: (error) => error._tag === "GCP.AIPlatform.NotResolved",
189 times: 8,
190 schedule: Schedule.spaced("2 seconds"),
191 }),
192 );
193
194const waitUntilGone = (name: string) =>
195 getByName(name).pipe(
196 Effect.filterOrFail(
197 (evaluator) => evaluator === undefined,
198 () => new AiPlatformStillExists({ name }),
199 ),
200 Effect.asVoid,
201 Effect.retry({
202 while: (error) => error._tag === "GCP.AIPlatform.StillExists",
203 times: 10,
204 schedule: Schedule.spaced("2 seconds"),
205 }),
206 );
207
208const findOwned = (parent: string, id: string) =>
209 Effect.gen(function* () {
210 const pages = yield* collectPages(
211 aiplatform.listProjectsLocationsOnlineEvaluators.pages({
212 parent,
213 pageSize: 100,
214 }),
215 ).pipe(Effect.catchTag("NotFound", () => Effect.succeed([])));
216 const evaluators = pages.flatMap((page) => page.onlineEvaluators ?? []);
217 for (const evaluator of evaluators) {
218 const parsed = parseDisplayName(evaluator.displayName);
219 if (yield* ownedById(id, parsed.labels)) {
220 return evaluator;
221 }
222 }
223 return undefined;
224 });
225
226export const OnlineEvaluatorProvider = () =>
227 Provider.succeed(OnlineEvaluator, {
228 stables: [
229 "name",
230 "onlineEvaluatorId",
231 "project",
232 "location",
233 "agentResource",
234 "createTime",
235 ],
236
237 diff: Effect.fn(function* ({ news, olds, output }) {
238 const env = yield* GcpEnvironment.current;
239 if (!isResolved(news)) return undefined;
240 const previousLocation = normalizeLocation(
241 olds?.location ?? output?.location,
242 env.region,
243 );
244 const nextLocation = normalizeLocation(
245 news.location ?? output?.location,
246 env.region,
247 );
248 const agentChanged =
249 news.agentResource !==
250 (olds?.agentResource ?? output?.agentResource ?? news.agentResource);
251 const replace =
252 previousLocation !== nextLocation ||
253 (olds !== undefined && agentChanged);
254 if (!replace) return undefined;
255 return {
256 action: "replace" as const,
257 deleteFirst: previousLocation === nextLocation,
258 };
259 }),
260
261 read: Effect.fn(function* ({ id, olds, output }) {
262 const env = yield* GcpEnvironment.current;
263 const location = normalizeLocation(
264 olds?.location ?? output?.location,
265 env.region,
266 );
267 const parent = locationParent(env.project, location);
268 const existing =
269 output?.name !== undefined
270 ? yield* getByName(output.name)
271 : yield* findOwned(parent, id);
272 if (existing === undefined) return undefined;
273 const attrs = toAttrs(existing, env.project);
274 const parsed = parseDisplayName(existing.displayName);
275 return (yield* hasAlchemyLabels(id, parsed.labels)) ||
276 (yield* ownedById(id, parsed.labels))
277 ? attrs
278 : Unowned(attrs);
279 }),
280
281 list: () =>
282 Effect.gen(function* () {
283 const env = yield* GcpEnvironment.current;
284 const pages = (yield* Effect.forEach(
285 listLocations(env.region),
286 (location) =>
287 collectPages(
288 aiplatform.listProjectsLocationsOnlineEvaluators.pages({
289 parent: locationParent(env.project, location),
290 pageSize: 100,
291 }),
292 ).pipe(Effect.catchTag("NotFound", () => Effect.succeed([]))),
293 )).flat();
294 return pages.flatMap((page) =>
295 (page.onlineEvaluators ?? [])
296 .filter((evaluator) =>
297 hasDisplayNameOwnership(evaluator.displayName),
298 )
299 .map((evaluator) => toAttrs(evaluator, env.project)),
300 );
301 }),
302
303 reconcile: Effect.fn(function* ({ id, news, output }) {
304 const env = yield* GcpEnvironment.current;
305 const location = normalizeLocation(
306 news.location ?? output?.location,
307 env.region,
308 );
309 const parent = locationParent(env.project, location);
310 const internal = yield* createInternalLabels(id);
311 const displayName = encodeDisplayName(internal, news.displayName);
312 const config = defaultConfig(news.config);
313
314 let current =
315 output?.name !== undefined
316 ? yield* getByName(output.name)
317 : yield* findOwned(parent, id);
318
319 if (current === undefined) {
320 const created =
321 yield* aiplatform.createProjectsLocationsOnlineEvaluators({
322 parent,
323 body: {
324 displayName,
325 agentResource: news.agentResource,
326 metricSources: news.metricSources,
327 config,
328 cloudObservability: news.cloudObservability,
329 },
330 });
331 yield* waitForOperation(created, { alreadyExistsOk: true });
332 const createdName = resourceNameFromOperation(created);
333 current =
334 createdName !== undefined
335 ? yield* waitUntilExists(createdName)
336 : yield* findOwned(parent, id);
337 }
338
339 if (current === undefined || current.name === undefined) {
340 return yield* new OnlineEvaluatorNotResolved({
341 name: output?.name ?? parent,
342 });
343 }
344
345 const observedName = current.name;
346 const observedDisplay = parseDisplayName(current.displayName);
347 const displayChanged =
348 (observedDisplay.displayName ?? "") !== (news.displayName ?? "");
349 const configChanged = !jsonEqual(current.config, config);
350 const metricsChanged = !jsonEqual(
351 current.metricSources,
352 news.metricSources,
353 );
354
355 if (displayChanged || configChanged || metricsChanged) {
356 const updateMask = [
357 displayChanged ? "display_name" : undefined,
358 configChanged ? "config" : undefined,
359 metricsChanged ? "metric_sources" : undefined,
360 ].filter((field): field is string => field !== undefined);
361 const patched =
362 yield* aiplatform.patchProjectsLocationsOnlineEvaluators({
363 name: observedName,
364 updateMask: updateMask.join(","),
365 body: {
366 name: observedName,
367 displayName,
368 config,
369 metricSources: news.metricSources,
370 },
371 });
372 yield* waitForOperation(patched);
373 current = yield* getByName(observedName);
374 }
375
376 if (current === undefined) {
377 return yield* new OnlineEvaluatorNotResolved({ name: observedName });
378 }
379 return toAttrs(current, env.project);
380 }),
381
382 delete: Effect.fn(function* ({ output }) {
383 const operation = yield* aiplatform
384 .deleteProjectsLocationsOnlineEvaluators({ name: output.name })
385 .pipe(Effect.catchTag("NotFound", () => Effect.succeed(undefined)));
386 if (operation !== undefined) {
387 yield* waitForOperation(operation, { notFoundOk: true });
388 }
389 yield* waitUntilGone(output.name);
390 }),
391 });