1import * as GCP from "@/GCP";
2import * as Test from "@/Test/Alchemy";
3import * as datalabeling from "@distilled.cloud/gcp/datalabeling_v1beta1";
4import { expect } from "alchemy-test";
5import * as Effect from "effect/Effect";
6import {
7 logLevel,
8 probe,
9 currentProject,
10 runLifecycle,
11 waitUntilGone,
12} from "./common.ts";
13
14const { test } = Test.make({ providers: GCP.providers() });
15
16test.provider(
17 "getProjectsEvaluationJobs on a missing job fails with a typed tag",
18 (stack) =>
19 Effect.gen(function* () {
20 const project = yield* currentProject;
21 yield* stack.destroy();
22
23 const error = yield* Effect.flip(
24 probe(
25 datalabeling.getProjectsEvaluationJobs({
26 name: `projects/${project}/evaluationJobs/alchemy-missing-job`,
27 }),
28 ),
29 );
30 expect(error._tag).toEqual("NotFound");
31
32 yield* stack.destroy();
33 }).pipe(logLevel),
34 {
35 tags: ["provider:gcp", "provider:gcp:datalabeling", "live"],
36 timeout: 90_000,
37 },
38);
39
40test.provider.skipIf(!runLifecycle)(
41 "create, update, and delete an evaluation job",
42 (stack) =>
43 Effect.gen(function* () {
44 const project = yield* currentProject;
45 yield* stack.destroy();
46
47 const created = yield* stack.deploy(
48 Effect.gen(function* () {
49 const specs = yield* GCP.DataLabeling.AnnotationSpecSet("Classes", {
50 displayName: "pets",
51 annotationSpecs: [{ displayName: "dog" }, { displayName: "cat" }],
52 });
53 const job = yield* GCP.DataLabeling.EvaluationJob("DailyEval", {
54 annotationSpecSet: specs.name,
55 modelVersion: `projects/${project}/models/classifier/versions/v1`,
56 schedule: "every 24 hours",
57 description: "daily sample",
58 evaluationJobConfig: {
59 exampleCount: 50,
60 exampleSamplePercentage: 0.1,
61 evaluationConfig: {},
62 inputConfig: {
63 dataType: "IMAGE",
64 annotationType: "IMAGE_CLASSIFICATION_ANNOTATION",
65 classificationMetadata: { isMultiLabel: false },
66 bigquerySource: {
67 inputUri: `bq://${project}.eval.predictions`,
68 },
69 },
70 bigqueryImportKeys: {
71 data_json_key: "data",
72 label_json_key: "label",
73 label_score_json_key: "score",
74 },
75 imageClassificationConfig: {
76 annotationSpecSet: specs.name,
77 },
78 },
79 });
80 return { specs, job };
81 }),
82 );
83
84 expect(created.job.name).toContain("/evaluationJobs/");
85 expect(created.job.annotationSpecSet).toEqual(created.specs.name);
86 expect(created.job.description).toEqual("daily sample");
87 expect(created.job.evaluationJobConfig?.exampleCount).toEqual(50);
88
89 const fetched = yield* datalabeling.getProjectsEvaluationJobs({
90 name: created.job.name,
91 });
92 expect(fetched.name).toEqual(created.job.name);
93 expect(fetched.description).toContain("alchemy-id=");
94
95 const updated = yield* stack.deploy(
96 Effect.gen(function* () {
97 const specs = yield* GCP.DataLabeling.AnnotationSpecSet("Classes", {
98 annotationSpecSetId: created.specs.annotationSpecSetId,
99 displayName: "pets",
100 annotationSpecs: [{ displayName: "dog" }, { displayName: "cat" }],
101 });
102 const job = yield* GCP.DataLabeling.EvaluationJob("DailyEval", {
103 evaluationJobId: created.job.evaluationJobId,
104 annotationSpecSet: specs.name,
105 modelVersion: created.job.modelVersion ?? "",
106 schedule: created.job.schedule ?? "every 24 hours",
107 description: "daily sample",
108 evaluationJobConfig: {
109 ...created.job.evaluationJobConfig,
110 exampleCount: 80,
111 exampleSamplePercentage: 0.2,
112 imageClassificationConfig: {
113 annotationSpecSet: specs.name,
114 },
115 },
116 });
117 return { specs, job };
118 }),
119 );
120
121 expect(updated.job.name).toEqual(created.job.name);
122 expect(updated.job.evaluationJobConfig?.exampleCount).toEqual(80);
123
124 yield* stack.destroy();
125 const gone = yield* waitUntilGone(
126 datalabeling.getProjectsEvaluationJobs({ name: created.job.name }),
127 );
128 expect(gone).toEqual("gone");
129 }).pipe(logLevel),
130 {
131 tags: ["provider:gcp", "provider:gcp:datalabeling", "live"],
132 timeout: 90_000,
133 },
134);