1import * as datalabeling from "@distilled.cloud/gcp/datalabeling_v1beta1";
2import * as Data from "effect/Data";
3import * as Effect from "effect/Effect";
4import { Unowned } from "../../AdoptPolicy.ts";
5import { isResolved } from "../../Diff.ts";
6import * as Provider from "../../Provider.ts";
7import { Resource } from "../../Resource.ts";
8import { GcpEnvironment } from "../Environment.ts";
9import { createInternalLabels } from "../Labels.ts";
10import type { Providers } from "../Providers.ts";
11import {
12 encodeOwnership,
13 findOwned,
14 hasOwnershipMarker,
15 ignoreGone,
16 listEvaluationJobs,
17 MAX_EVALUATION_DESCRIPTION_LENGTH,
18 noRetryLayer,
19 ownedByAlchemy,
20 parseOwnership,
21 parseResourceName,
22 projectParent,
23 replaceOnIdentity,
24 retryDelete,
25 retryTransient,
26 sameJson,
27 sameText,
28 updateMaskOf,
29 waitUntilGone,
30} from "./internal.ts";
31
32export type EvaluationJobConfig =
33 datalabeling.GoogleCloudDatalabelingV1beta1EvaluationJobConfig;
34
35export type EvaluationJobProps = {
36 /**
37 * Evaluation job id (the last segment of
38 * `projects/{project}/evaluationJobs/{evaluation_job}`).
39 * Server-assigned on create. Immutable — changing it replaces the
40 * job.
41 */
42 evaluationJobId?: string;
43 /**
44 * Description of the job. Maximum 25,000 characters. Evaluation jobs
45 * have no labels field, so Alchemy stamps ownership into this field.
46 * Immutable — changing it replaces the job.
47 */
48 description?: string;
49 /**
50 * Annotation spec set resource name describing the labels the model
51 * outputs. Immutable — changing it replaces the job.
52 */
53 annotationSpecSet: string;
54 /**
55 * AI Platform Prediction model version sampled for evaluation, as
56 * `projects/{project}/models/{model}/versions/{version}`. Immutable —
57 * changing it replaces the job.
58 */
59 modelVersion: string;
60 /**
61 * Run interval, in crontab or English-like format. Must be at least
62 * one day. Immutable — changing it replaces the job.
63 */
64 schedule: string;
65 /**
66 * When true, Data Labeling assigns human labelers for missing ground
67 * truth. When false, ground truth is read from the job's BigQuery
68 * table.
69 * @default false
70 */
71 labelMissingGroundTruth?: boolean;
72 /**
73 * Configuration for sampling, import keys, human annotation, and
74 * metrics. Only `humanAnnotationConfig.instruction`, `exampleCount`,
75 * and `exampleSamplePercentage` update in place; any other change
76 * replaces the job.
77 */
78 evaluationJobConfig: EvaluationJobConfig;
79 /**
80 * When true, pause the job after ensure. Pausing a paused job is a
81 * no-op; resuming a running or scheduled job is a no-op.
82 * @default false
83 */
84 paused?: boolean;
85};
86
87export type EvaluationJob = Resource<
88 "GCP.DataLabeling.EvaluationJob",
89 EvaluationJobProps,
90 {
91 /** Full resource name `projects/{project}/evaluationJobs/{job}`. */
92 name: string;
93 /** Evaluation job id (last path segment). */
94 evaluationJobId: string;
95 /** Project id. */
96 project: string;
97 /** User description with the Alchemy ownership prefix stripped. */
98 description: string | undefined;
99 /** Annotation spec set resource name. */
100 annotationSpecSet: string | undefined;
101 /** Model version sampled for evaluation. */
102 modelVersion: string | undefined;
103 /** Run interval. */
104 schedule: string | undefined;
105 /** Whether the service provides ground-truth labels. */
106 labelMissingGroundTruth: boolean;
107 /** Job configuration. */
108 evaluationJobConfig: EvaluationJobConfig | undefined;
109 /** Server-reported job state. */
110 state: string | undefined;
111 /** Whether the job is paused. */
112 paused: boolean;
113 /** Failed attempts, if any. */
114 attempts:
115 | datalabeling.GoogleCloudDatalabelingV1beta1AttemptList
116 | undefined;
117 /** RFC3339 creation timestamp. */
118 createTime: string | undefined;
119 },
120 never,
121 Providers
122>;
123
124/**
125 * A Data Labeling evaluation job that periodically samples predictions
126 * from an AI Platform model version and scores them.
127 *
128 * Patch can only update `evaluationJobConfig.humanAnnotationConfig.instruction`,
129 * `evaluationJobConfig.exampleCount`, and
130 * `evaluationJobConfig.exampleSamplePercentage`. Pause and resume are
131 * separate RPCs. Every other field is identity — changing it replaces
132 * the job. There is no labels API, so Alchemy stamps ownership into
133 * `description`.
134 *
135 * ### Creating an Evaluation Job
136 * **Example:** Daily image classification eval
137 * ```typescript
138 * const evaluationJobConfig = {
139 * exampleCount: 100,
140 * exampleSamplePercentage: 0.1,
141 * evaluationConfig: {},
142 * inputConfig: {
143 * dataType: "IMAGE",
144 * annotationType: "IMAGE_CLASSIFICATION_ANNOTATION",
145 * classificationMetadata: { isMultiLabel: false },
146 * bigquerySource: {
147 * inputUri: "bq://my-project.eval.predictions",
148 * },
149 * },
150 * bigqueryImportKeys: {
151 * data_json_key: "data",
152 * label_json_key: "label",
153 * label_score_json_key: "score",
154 * },
155 * imageClassificationConfig: {
156 * annotationSpecSet: specs.name,
157 * },
158 * };
159 * const job = yield* GCP.DataLabeling.EvaluationJob("DailyEval", {
160 * annotationSpecSet: specs.name,
161 * modelVersion:
162 * "projects/my-project/models/classifier/versions/v1",
163 * schedule: "every 24 hours",
164 * evaluationJobConfig,
165 * });
166 * ```
167 *
168 * ### Pausing an Evaluation Job
169 * **Example:** Pause sampling
170 * ```typescript
171 * // Same logical id, changed props: the engine updates it in place.
172 * const job = yield* GCP.DataLabeling.EvaluationJob("DailyEval", {
173 * annotationSpecSet: specs.name,
174 * modelVersion:
175 * "projects/my-project/models/classifier/versions/v1",
176 * schedule: "every 24 hours",
177 * evaluationJobConfig,
178 * paused: true,
179 * });
180 * ```
181 *
182 * @resource
183 * @category DataLabeling
184 */
185export const EvaluationJob = Resource<EvaluationJob>(
186 "GCP.DataLabeling.EvaluationJob",
187);
188
189export class EvaluationJobNotResolved extends Data.TaggedError(
190 "GCP.DataLabeling.EvaluationJobNotResolved",
191)<{
192 name: string;
193}> {}
194
195const resourceName = (project: string, evaluationJobId: string) =>
196 `${projectParent(project)}/evaluationJobs/${evaluationJobId}`;
197
198const isPaused = (state: string | undefined) => state === "PAUSED";
199
200const configIdentity = (config: EvaluationJobConfig | undefined) => {
201 if (config === undefined) return undefined;
202 const {
203 exampleCount: _exampleCount,
204 exampleSamplePercentage: _exampleSamplePercentage,
205 humanAnnotationConfig,
206 ...rest
207 } = config;
208 const { instruction: _instruction, ...humanRest } =
209 humanAnnotationConfig ?? {};
210 return {
211 ...rest,
212 humanAnnotationConfig:
213 Object.keys(humanRest).length > 0 ? humanRest : undefined,
214 };
215};
216
217const toAttrs = (
218 job: datalabeling.GoogleCloudDatalabelingV1beta1EvaluationJob,
219 project: string,
220) => {
221 const name = job.name ?? "";
222 const parsed = parseResourceName(name, "evaluationJobs");
223 return {
224 name,
225 evaluationJobId: parsed.id,
226 project: parsed.project || project,
227 description: parseOwnership(job.description).text,
228 annotationSpecSet: job.annotationSpecSet,
229 modelVersion: job.modelVersion,
230 schedule: job.schedule,
231 labelMissingGroundTruth: job.labelMissingGroundTruth === true,
232 evaluationJobConfig: job.evaluationJobConfig,
233 state: job.state,
234 paused: isPaused(job.state),
235 attempts: job.attempts,
236 createTime: job.createTime,
237 };
238};
239
240const getByName = (name: string) =>
241 name.length === 0
242 ? Effect.succeed(undefined)
243 : datalabeling.getProjectsEvaluationJobs({ name }).pipe(
244 Effect.provide(noRetryLayer),
245 Effect.catchTag("NotFound", () => Effect.succeed(undefined)),
246 Effect.catchTag("BadGateway", () => Effect.succeed(undefined)),
247 );
248
249const findByOwnership = (id: string, project: string) =>
250 Effect.gen(function* () {
251 const rows = yield* listEvaluationJobs(projectParent(project));
252 return yield* findOwned(id, rows, (row) => row.description);
253 });
254
255export const EvaluationJobProvider = () =>
256 Provider.succeed(EvaluationJob, {
257 stables: ["name", "evaluationJobId", "project", "createTime"],
258
259 diff: Effect.fn(function* ({ news, olds, output }) {
260 if (!isResolved(news)) return undefined;
261 const extra =
262 (output?.annotationSpecSet !== undefined &&
263 !sameText(news.annotationSpecSet, output.annotationSpecSet)) ||
264 (output?.modelVersion !== undefined &&
265 !sameText(news.modelVersion, output.modelVersion)) ||
266 (output?.schedule !== undefined &&
267 !sameText(news.schedule, output.schedule)) ||
268 (output !== undefined &&
269 (news.labelMissingGroundTruth === true) !==
270 output.labelMissingGroundTruth) ||
271 (olds !== undefined &&
272 !sameText(news.description, output?.description)) ||
273 (output !== undefined &&
274 !sameJson(
275 configIdentity(news.evaluationJobConfig),
276 configIdentity(output.evaluationJobConfig),
277 ));
278 return replaceOnIdentity({
279 previousId: olds?.evaluationJobId ?? output?.evaluationJobId,
280 nextId: news.evaluationJobId,
281 extra,
282 });
283 }),
284
285 read: Effect.fn(function* ({ id, olds, output }) {
286 const env = yield* GcpEnvironment.current;
287 const evaluationJobId =
288 olds?.evaluationJobId ??
289 output?.evaluationJobId ??
290 (output?.name
291 ? parseResourceName(output.name, "evaluationJobs").id
292 : "");
293 const name =
294 output?.name ??
295 (evaluationJobId.length > 0
296 ? resourceName(env.project, evaluationJobId)
297 : "");
298 const existing =
299 (yield* getByName(name)) ?? (yield* findByOwnership(id, env.project));
300 if (existing === undefined) return undefined;
301 const attrs = toAttrs(existing, env.project);
302 return (yield* ownedByAlchemy(id, existing.description))
303 ? attrs
304 : Unowned(attrs);
305 }),
306
307 list: () =>
308 Effect.gen(function* () {
309 const env = yield* GcpEnvironment.current;
310 const rows = yield* listEvaluationJobs(projectParent(env.project));
311 return rows
312 .filter((row) => hasOwnershipMarker(row.description))
313 .map((row) => toAttrs(row, env.project));
314 }),
315
316 reconcile: Effect.fn(function* ({ id, news, output }) {
317 const env = yield* GcpEnvironment.current;
318 const evaluationJobId = news.evaluationJobId ?? output?.evaluationJobId;
319 const name =
320 output?.name ??
321 (evaluationJobId !== undefined
322 ? resourceName(env.project, evaluationJobId)
323 : "");
324 const ownership = yield* createInternalLabels(id);
325 const description = encodeOwnership(
326 ownership,
327 news.description,
328 MAX_EVALUATION_DESCRIPTION_LENGTH,
329 );
330 const labelMissingGroundTruth = news.labelMissingGroundTruth === true;
331 const desiredPaused = news.paused === true;
332
333 let current =
334 (yield* getByName(name)) ?? (yield* findByOwnership(id, env.project));
335
336 if (current === undefined) {
337 const created = yield* retryTransient(
338 datalabeling.createProjectsEvaluationJobs({
339 parent: projectParent(env.project),
340 body: {
341 job: {
342 description,
343 annotationSpecSet: news.annotationSpecSet,
344 modelVersion: news.modelVersion,
345 schedule: news.schedule,
346 labelMissingGroundTruth,
347 evaluationJobConfig: news.evaluationJobConfig,
348 },
349 },
350 }),
351 ).pipe(
352 Effect.catchTag("Conflict", () => findByOwnership(id, env.project)),
353 );
354 current = created ?? undefined;
355 }
356
357 if (current === undefined) {
358 return yield* new EvaluationJobNotResolved({
359 name: name || projectParent(env.project),
360 });
361 }
362
363 const currentName = current.name ?? name;
364 const observedConfig = current.evaluationJobConfig;
365 const desiredConfig = news.evaluationJobConfig;
366 const instructionChanged = !sameText(
367 observedConfig?.humanAnnotationConfig?.instruction,
368 desiredConfig.humanAnnotationConfig?.instruction,
369 );
370 const exampleCountChanged =
371 (observedConfig?.exampleCount ?? undefined) !==
372 (desiredConfig.exampleCount ?? undefined);
373 const sampleChanged =
374 (observedConfig?.exampleSamplePercentage ?? undefined) !==
375 (desiredConfig.exampleSamplePercentage ?? undefined);
376
377 if (instructionChanged || exampleCountChanged || sampleChanged) {
378 current = yield* retryTransient(
379 datalabeling.patchProjectsEvaluationJobs({
380 name: currentName,
381 updateMask: updateMaskOf(
382 instructionChanged
383 ? "evaluationJobConfig.humanAnnotationConfig.instruction"
384 : undefined,
385 exampleCountChanged
386 ? "evaluationJobConfig.exampleCount"
387 : undefined,
388 sampleChanged
389 ? "evaluationJobConfig.exampleSamplePercentage"
390 : undefined,
391 ),
392 body: {
393 evaluationJobConfig: {
394 humanAnnotationConfig: desiredConfig.humanAnnotationConfig,
395 exampleCount: desiredConfig.exampleCount,
396 exampleSamplePercentage: desiredConfig.exampleSamplePercentage,
397 },
398 },
399 }),
400 );
401 }
402
403 if (desiredPaused && !isPaused(current.state)) {
404 yield* retryTransient(
405 datalabeling.pauseProjectsEvaluationJobs({
406 name: currentName,
407 body: {},
408 }),
409 );
410 current = (yield* getByName(currentName)) ?? current;
411 } else if (!desiredPaused && isPaused(current.state)) {
412 yield* retryTransient(
413 datalabeling.resumeProjectsEvaluationJobs({
414 name: currentName,
415 body: {},
416 }),
417 );
418 current = (yield* getByName(currentName)) ?? current;
419 }
420
421 return toAttrs(current, env.project);
422 }),
423
424 delete: Effect.fn(function* ({ output }) {
425 if (!output.name) return;
426 yield* ignoreGone(
427 retryDelete(
428 datalabeling.deleteProjectsEvaluationJobs({ name: output.name }),
429 ),
430 );
431 yield* waitUntilGone(getByName(output.name));
432 }),
433 });