1import * as aiGateway from "@distilled.cloud/cloudflare/ai-gateway";
2import * as Effect from "effect/Effect";
3import * as Predicate from "effect/Predicate";
4import * as Stream from "effect/Stream";
5import { deepEqual, isResolved } from "../../Diff.ts";
6import { createPhysicalName } from "../../PhysicalName.ts";
7import * as Provider from "../../Provider.ts";
8import { Resource } from "../../Resource.ts";
9import { CloudflareEnvironment } from "../CloudflareEnvironment.ts";
10import type { Providers } from "../Providers.ts";
11
12const TypeId = "Cloudflare.AI.Evaluation" as const;
13type TypeId = typeof TypeId;
14
15export type EvaluationProps = {
16 /**
17 * The AI Gateway the evaluation runs against. Changing the gateway
18 * triggers a replacement.
19 */
20 gatewayId: string;
21 /**
22 * Human readable evaluation name. If omitted, a unique name is generated
23 * from the app, stage, and logical ID. Evaluations are immutable —
24 * changing the name triggers a replacement.
25 * @default ${app}-${stage}-${id}
26 */
27 name?: string;
28 /**
29 * The datasets (saved log filters) the evaluation processes. Usually the
30 * `datasetId` attributes of `Dataset` resources on the same
31 * gateway. Changing the datasets triggers a replacement.
32 */
33 datasetIds: string[];
34 /**
35 * The evaluation types to run (e.g. speed, cost). Discover ids with
36 * `listEvaluationTypes` — the mandatory `speed` and `cost` types are
37 * account-global constants. Changing the types triggers a replacement.
38 */
39 evaluationTypeIds: string[];
40};
41
42export type EvaluationAttributes = {
43 /**
44 * Server-generated evaluation identifier. Stable for the lifetime of the
45 * evaluation.
46 */
47 evaluationId: string;
48 /**
49 * The Cloudflare account the evaluation belongs to.
50 */
51 accountId: string;
52 /**
53 * The AI Gateway the evaluation runs against.
54 */
55 gatewayId: string;
56 /**
57 * Human readable evaluation name.
58 */
59 name: string;
60 /**
61 * The dataset ids the evaluation processes.
62 */
63 datasetIds: string[];
64 /**
65 * The evaluation type ids the evaluation runs.
66 */
67 evaluationTypeIds: string[];
68 /**
69 * Whether Cloudflare has finished processing the evaluation. Evaluations
70 * are asynchronous jobs over logged traffic — this starts `false`.
71 */
72 processed: boolean;
73 /**
74 * Number of logs the evaluation covers.
75 */
76 totalLogs: number;
77 /**
78 * When the evaluation was created.
79 */
80 createdAt: string;
81 /**
82 * When the evaluation was last modified.
83 */
84 modifiedAt: string;
85};
86
87export type Evaluation = Resource<
88 TypeId,
89 EvaluationProps,
90 EvaluationAttributes,
91 never,
92 Providers
93>;
94
95/**
96 * An evaluation job on a Cloudflare.AI. Gateway.
97 *
98 * Evaluations measure performance (speed, cost, feedback) of the logged
99 * traffic captured by one or more datasets on a gateway. They are
100 * create-only on Cloudflare's side: any prop change replaces the
101 * evaluation with a fresh job.
102 * ### Creating an Evaluation
103 * **Example:** Evaluate a dataset for speed and cost
104 * ```typescript
105 * const gateway = yield* Cloudflare.AI.Gateway("Gateway");
106 *
107 * const dataset = yield* Cloudflare.AI.Dataset("SuccessLogs", {
108 * gatewayId: gateway.gatewayId,
109 * filters: [{ key: "success", operator: "eq", value: [true] }],
110 * });
111 *
112 * const types = yield* listEvaluationTypes(gateway.accountId);
113 * const evaluation = yield* Cloudflare.AI.Evaluation("Baseline", {
114 * gatewayId: gateway.gatewayId,
115 * datasetIds: [dataset.datasetId],
116 * evaluationTypeIds: types
117 * .filter((t) => t.mandatory)
118 * .map((t) => t.id),
119 * });
120 * ```
121 *
122 * @see https://developers.cloudflare.com/ai-gateway/evaluations/
123 *
124 * @resource
125 * @product AI Gateway
126 * @category AI
127 */
128export const Evaluation = Resource<Evaluation>(TypeId, {
129 aliases: ["Cloudflare.AiGateway.Evaluation"],
130});
131
132/**
133 * Returns true if the given value is an Evaluation resource.
134 */
135export const isEvaluation = (value: unknown): value is Evaluation =>
136 Predicate.hasProperty(value, "Type") && value.Type === TypeId;
137
138/**
139 * List the evaluation types available on the account (speed, cost,
140 * feedback, ...). The `mandatory` types must be included in every
141 * evaluation.
142 */
143export const listEvaluationTypes = (accountId: string) =>
144 aiGateway
145 .listEvaluationTypes({ accountId, perPage: 50 })
146 .pipe(Effect.map((page) => page.result));
147
148export const EvaluationProvider = () =>
149 Provider.succeed(Evaluation, {
150 stables: ["evaluationId", "accountId", "gatewayId", "createdAt"],
151 diff: Effect.fn(function* ({ id, olds, news, output }) {
152 if (!isResolved(news)) return undefined;
153 const { accountId } = yield* yield* CloudflareEnvironment;
154 if ((output?.accountId ?? accountId) !== accountId) {
155 return { action: "replace" } as const;
156 }
157 if (output === undefined) return undefined;
158 // Evaluations have no update API — any change is a replacement.
159 // Auto-generated names are engine-owned: the deployed name stays
160 // authoritative even if the generator would name this id differently
161 // today. Only an explicit user-provided name can force a replace.
162 const newName = news.name ?? output.name;
163 if (
164 output.gatewayId !== news.gatewayId ||
165 output.name !== newName ||
166 !deepEqual(output.datasetIds, news.datasetIds) ||
167 !deepEqual(output.evaluationTypeIds, news.evaluationTypeIds)
168 ) {
169 return { action: "replace" } as const;
170 }
171 return undefined;
172 }),
173 read: Effect.fn(function* ({ id, olds, output }) {
174 const { accountId } = yield* yield* CloudflareEnvironment;
175 const acct = output?.accountId ?? accountId;
176 const gatewayId =
177 output?.gatewayId ?? (olds?.gatewayId as string | undefined);
178 if (gatewayId === undefined) return undefined;
179 const known = {
180 datasetIds:
181 output?.datasetIds ??
182 (olds?.datasetIds as string[] | undefined) ??
183 [],
184 evaluationTypeIds:
185 output?.evaluationTypeIds ??
186 (olds?.evaluationTypeIds as string[] | undefined) ??
187 [],
188 };
189
190 if (output?.evaluationId) {
191 const observed = yield* getEvaluation(
192 acct,
193 gatewayId,
194 output.evaluationId,
195 );
196 return observed ? toAttributes(observed, acct, known) : undefined;
197 }
198 // Cold read — recover from lost state by matching the deterministic
199 // physical name. Names are not unique on Cloudflare's side; an exact
200 // match on our generated/explicit name is the best identity we have.
201 const name = yield* createEvaluationName(id, olds?.name);
202 const match = yield* findByName(acct, gatewayId, name);
203 return match ? toAttributes(match, acct, known) : undefined;
204 }),
205 reconcile: Effect.fn(function* ({ id, news, output }) {
206 const { accountId } = yield* yield* CloudflareEnvironment;
207 const gatewayId = news.gatewayId as string;
208 // Prefer the deployed name: regenerating would target a different
209 // resource if the generator's output for this id ever drifts.
210 const name = output?.name ?? (yield* createEvaluationName(id, news.name));
211 const datasetIds = news.datasetIds as string[];
212 const evaluationTypeIds = news.evaluationTypeIds;
213
214 // Observe — the evaluationId cached on `output` is a hint, not a
215 // guarantee: a missing evaluation falls through and we recreate.
216 const observed = output?.evaluationId
217 ? yield* getEvaluation(
218 output.accountId ?? accountId,
219 gatewayId,
220 output.evaluationId,
221 )
222 : undefined;
223
224 if (observed) {
225 // Evaluations are immutable; diff routes every prop change to a
226 // replacement, so an observed evaluation is already converged.
227 return toAttributes(observed, accountId, {
228 datasetIds: output?.datasetIds ?? datasetIds,
229 evaluationTypeIds: output?.evaluationTypeIds ?? evaluationTypeIds,
230 });
231 }
232
233 // Ensure — greenfield (or out-of-band delete). Cloudflare enforces
234 // name uniqueness, so a conflict (e.g. a leftover from an interrupted
235 // run or a create race) is converged by adopting the existing
236 // evaluation by name instead of failing — evaluations are immutable,
237 // and diff routes any prop change to a replacement, so a name match
238 // is already the desired evaluation.
239 const created = yield* aiGateway
240 .createEvaluation({
241 accountId,
242 gatewayId,
243 name,
244 datasetIds,
245 evaluationTypeIds,
246 })
247 .pipe(
248 Effect.catchTag("EvaluationNameAlreadyExists", (originalError) =>
249 findByName(accountId, gatewayId, name).pipe(
250 Effect.flatMap((match) =>
251 match ? Effect.succeed(match) : Effect.fail(originalError),
252 ),
253 ),
254 ),
255 );
256 return toAttributes(created, accountId, {
257 datasetIds,
258 evaluationTypeIds,
259 });
260 }),
261 delete: Effect.fn(function* ({ output }) {
262 yield* aiGateway
263 .deleteEvaluation({
264 accountId: output.accountId,
265 gatewayId: output.gatewayId,
266 id: output.evaluationId,
267 })
268 // Cloudflare reports both a missing evaluation and a missing parent
269 // gateway with code 7002 — either way it's already gone.
270 .pipe(Effect.catchTag("EvaluationNotFound", () => Effect.void));
271 }),
272 // Evaluations are scoped under an AI Gateway, which itself is
273 // account-scoped. Fan out: exhaustively paginate every gateway in the
274 // account, then exhaustively paginate the evaluations under each gateway
275 // (bounded concurrency) and hydrate each into the same Attributes shape
276 // `read` produces (via `toAttributes`). A gateway deleted mid-enumeration
277 // returns an empty evaluation list, so no per-item not-found handling is
278 // required. We pass `[]` for known type ids: the listed evaluation only
279 // echoes type ids back once results exist, and `toAttributes` falls back
280 // to the result-derived ids otherwise.
281 list: Effect.fn(function* () {
282 const { accountId } = yield* yield* CloudflareEnvironment;
283 const gatewayIds = yield* aiGateway.listAiGateways
284 .pages({ accountId })
285 .pipe(
286 Stream.runCollect,
287 Effect.map((chunk) =>
288 Array.from(chunk).flatMap((page) =>
289 (page.result ?? []).map((gateway) => gateway.id),
290 ),
291 ),
292 );
293 const rows = yield* Effect.forEach(
294 gatewayIds,
295 (gatewayId) =>
296 aiGateway.listEvaluations.pages({ accountId, gatewayId }).pipe(
297 Stream.runCollect,
298 Effect.map((chunk) =>
299 Array.from(chunk).flatMap((page) =>
300 (page.result ?? []).map((evaluation) =>
301 toAttributes(evaluation, accountId, {
302 datasetIds: [],
303 evaluationTypeIds: [],
304 }),
305 ),
306 ),
307 ),
308 ),
309 { concurrency: 10 },
310 );
311 return rows.flat();
312 }),
313 });
314
315/**
316 * Read an evaluation by id, mapping "gone" (`EvaluationNotFound`,
317 * Cloudflare error code 7002 — which also covers a deleted parent gateway)
318 * to `undefined`.
319 */
320const getEvaluation = (accountId: string, gatewayId: string, id: string) =>
321 aiGateway
322 .getEvaluation({ accountId, gatewayId, id })
323 .pipe(
324 Effect.catchTag("EvaluationNotFound", () => Effect.succeed(undefined)),
325 );
326
327/**
328 * Find an evaluation by exact name. Cloudflare's `name` query is a fuzzy
329 * match, so re-check exactly client-side. If several evaluations carry the
330 * same name, pick the oldest for determinism. A missing parent gateway
331 * returns an empty list on this endpoint.
332 */
333const findByName = (accountId: string, gatewayId: string, name: string) =>
334 aiGateway.listEvaluations
335 .items({ accountId, gatewayId, name, perPage: 50 })
336 .pipe(
337 Stream.filter((e) => e.name === name),
338 Stream.runCollect,
339 Effect.map((chunk) =>
340 Array.from(chunk)
341 .sort((a, b) => a.createdAt.localeCompare(b.createdAt))
342 .at(0),
343 ),
344 );
345
346const createEvaluationName = (id: string, name: string | undefined) =>
347 Effect.gen(function* () {
348 return name ?? (yield* createPhysicalName({ id, lowercase: true }));
349 });
350
351const toAttributes = (
352 evaluation:
353 | aiGateway.GetEvaluationResponse
354 | aiGateway.CreateEvaluationResponse
355 | aiGateway.ListEvaluationsResponse["result"][number],
356 accountId: string,
357 known: { datasetIds: string[]; evaluationTypeIds: string[] },
358): EvaluationAttributes => ({
359 evaluationId: evaluation.id,
360 accountId,
361 gatewayId: evaluation.gatewayId,
362 name: evaluation.name,
363 // The API does not always echo the datasets back (the create response
364 // returns none), so fall back to what we asked for.
365 datasetIds:
366 evaluation.datasets.length > 0
367 ? evaluation.datasets.map((d) => d.id)
368 : known.datasetIds,
369 // The API only echoes evaluation type ids back once results exist, so
370 // prefer the observed result types and fall back to what we asked for.
371 evaluationTypeIds:
372 evaluation.results.length > 0
373 ? [...new Set(evaluation.results.map((r) => r.evaluationTypeId))]
374 : known.evaluationTypeIds,
375 processed: evaluation.processed,
376 totalLogs: evaluation.totalLogs,
377 createdAt: evaluation.createdAt,
378 modifiedAt: evaluation.modifiedAt,
379});