1import * as GCP from "@/GCP";
2import { GcpEnvironment } from "@/GCP/Environment";
3import * as Test from "@/Test/Alchemy";
4import * as aiplatform from "@distilled.cloud/gcp/aiplatform_v1";
5import { expect } from "alchemy-test";
6import * as Effect from "effect/Effect";
7import { MinimumLogLevel } from "effect/References";
8import * as Schedule from "effect/Schedule";
9
10const { test } = Test.make({ providers: GCP.providers() });
11
12const logLevel = Effect.provideService(
13 MinimumLogLevel,
14 process.env.DEBUG ? "Debug" : "Info",
15);
16
17// Evaluators need an existing evaluation metric; a missing one fails with
18// BadRequest "Request contains an invalid argument." Set
19// GCP_TEST_AIPLATFORM_EVALUATION_METRIC to a metric resource name to run
20// the lifecycle.
21const evaluationMetric = process.env.GCP_TEST_AIPLATFORM_EVALUATION_METRIC;
22const runLifecycle = !process.env.FAST && !!evaluationMetric;
23
24const waitUntilGone = (name: string) =>
25 aiplatform.getProjectsLocationsOnlineEvaluators({ name }).pipe(
26 Effect.as("found" as const),
27 Effect.catchTag("NotFound", () => Effect.succeed("gone" as const)),
28 Effect.repeat({
29 schedule: Schedule.spaced("2 seconds"),
30 until: (status) => status === "gone",
31 times: 10,
32 }),
33 );
34
35test.provider(
36 "getProjectsLocationsOnlineEvaluators on a missing evaluator fails with a typed tag",
37 (stack) =>
38 Effect.gen(function* () {
39 const { project } = yield* GcpEnvironment.current;
40 const parent = `projects/${project}/locations/us-central1`;
41 yield* stack.destroy();
42
43 const error = yield* Effect.flip(
44 aiplatform.getProjectsLocationsOnlineEvaluators({
45 name: `${parent}/onlineEvaluators/1234567890123456789`,
46 }),
47 );
48 expect(error._tag).toEqual("NotFound");
49 const page = yield* aiplatform.listProjectsLocationsOnlineEvaluators({
50 parent,
51 pageSize: 10,
52 });
53 expect(
54 (page.onlineEvaluators ?? []).map((item) => item.name),
55 ).not.toContain(`${parent}/onlineEvaluators/1234567890123456789`);
56
57 yield* stack.destroy();
58 }).pipe(logLevel),
59 {
60 tags: ["provider:gcp", "provider:gcp:aiplatform", "live"],
61 timeout: 90_000,
62 },
63);
64
65test.provider.skipIf(!runLifecycle)(
66 "create, update, and delete an online evaluator",
67 (stack) =>
68 Effect.gen(function* () {
69 yield* stack.destroy();
70
71 const created = yield* stack.deploy(
72 Effect.gen(function* () {
73 const engine = yield* GCP.AIPlatform.ReasoningEngine("Agent", {
74 location: "us-central1",
75 displayName: "alchemy-eval-engine",
76 description: "test",
77 labels: { env: "test" },
78 spec: { agentFramework: "custom" },
79 });
80 const evaluator = yield* GCP.AIPlatform.OnlineEvaluator("Quality", {
81 location: "us-central1",
82 displayName: "alchemy-online-eval",
83 agentResource: engine.name,
84 metricSources: [
85 {
86 metricResourceName: evaluationMetric,
87 },
88 ],
89 config: { randomSampling: { percentage: 10 } },
90 });
91 return { engine, evaluator };
92 }),
93 );
94
95 expect(created.evaluator.name).toContain("/onlineEvaluators/");
96 expect(created.evaluator.location).toEqual("us-central1");
97 expect(created.evaluator.displayName).toEqual("alchemy-online-eval");
98
99 const fetched = yield* aiplatform.getProjectsLocationsOnlineEvaluators({
100 name: created.evaluator.name,
101 });
102 expect(fetched.name).toEqual(created.evaluator.name);
103
104 const updated = yield* stack.deploy(
105 Effect.gen(function* () {
106 const engine = yield* GCP.AIPlatform.ReasoningEngine("Agent", {
107 reasoningEngineId: created.engine.reasoningEngineId,
108 location: "us-central1",
109 displayName: "alchemy-eval-engine",
110 description: "test",
111 labels: { env: "test" },
112 spec: { agentFramework: "custom" },
113 });
114 const evaluator = yield* GCP.AIPlatform.OnlineEvaluator("Quality", {
115 location: "us-central1",
116 displayName: "alchemy-online-eval-v2",
117 agentResource: engine.name,
118 metricSources: [
119 {
120 metricResourceName: evaluationMetric,
121 },
122 ],
123 config: { randomSampling: { percentage: 20 } },
124 });
125 return { engine, evaluator };
126 }),
127 );
128
129 expect(updated.evaluator.name).toEqual(created.evaluator.name);
130 expect(updated.evaluator.displayName).toEqual("alchemy-online-eval-v2");
131
132 yield* stack.destroy();
133
134 const gone = yield* waitUntilGone(created.evaluator.name);
135 expect(gone).toEqual("gone");
136 }).pipe(logLevel),
137 {
138 tags: ["provider:gcp", "provider:gcp:aiplatform", "live"],
139 timeout: 180_000,
140 },
141);