1import * as CloudflareClient from "@effect/ai-cloudflare/CloudflareClient"
2import * as CloudflareDecisionModel from "@effect/ai-cloudflare/CloudflareDecisionModel"
3import { assert, describe, it } from "@effect/vitest"
4import { Effect, Layer, Redacted, Schema } from "effect"
5import { Decision, DecisionModel, Model } from "effect/ai"
6import * as HttpClient from "effect/http/HttpClient"
7import type * as HttpClientError from "effect/http/HttpClientError"
8import type * as HttpClientRequest from "effect/http/HttpClientRequest"
9import * as HttpClientResponse from "effect/http/HttpClientResponse"
10
11const Triage = Decision.make({
12 input: Schema.Struct({ message: Schema.String }),
13 decisions: {
14 team: Decision.classify({
15 instructions: "Which team should handle this request?",
16 criteria: { billing: "Payments", technical: "Outages" }
17 }),
18 severity: Decision.rate({
19 instructions: "How severe is the impact?",
20 criteria: ["Minor", "Major", "Critical"]
21 }),
22 urgent: Decision.probability({
23 instructions: "Does this need immediate attention?",
24 criteria: { false: "Can wait", true: "Act now" }
25 })
26 }
27})
28
29const input = { message: "Checkout is down" }
30
31const answers = {
32 team: {
33 type: "choice",
34 choice: "technical",
35 confidence: 0.9,
36 probabilities: { billing: 0.1, technical: 0.9 }
37 },
38 severity: {
39 type: "score",
40 score: 1.75,
41 confidence: 0.8,
42 probabilities: { "0": 0.05, "1": 0.15, "2": 0.8 },
43 legend: { "0": "Minor", "1": "Major", "2": "Critical" }
44 },
45 urgent: { type: "noul", noul: 0.99 }
46}
47
48const clientLayer = (
49 answers: unknown,
50 onRequest: (request: HttpClientRequest.HttpClientRequest) => void = () => {},
51 model = "clef"
52) =>
53 CloudflareClient.layer({ accountId: "test-account", apiKey: Redacted.make("test-token") }).pipe(
54 Layer.provide(Layer.succeed(
55 HttpClient.HttpClient,
56 HttpClient.makeWith(
57 Effect.map((request) => {
58 onRequest(request)
59 return HttpClientResponse.fromWeb(
60 request,
61 Response.json({
62 success: true,
63 result: { model, answers, usage: { input_tokens: 120, output_tokens: 0 } }
64 })
65 )
66 }),
67 Effect.succeed as HttpClient.HttpClient.Preprocess<HttpClientError.HttpClientError, never>
68 )
69 ))
70 )
71
72const layer = (answers: unknown) =>
73 CloudflareDecisionModel.layer({ model: "clef" }).pipe(Layer.provide(clientLayer(answers)))
74
75describe("CloudflareDecisionModel", () => {
76 it.effect("sends every decision as a question and maps the answers", () =>
77 Effect.gen(function*() {
78 const result = yield* DecisionModel.decide(Triage, { input })
79 assert.strictEqual(yield* Model.ProviderName, "cloudflare")
80 assert.strictEqual(yield* Model.ModelName, "clef")
81 assert.strictEqual(result.answers.team.label, "technical")
82 assert.deepStrictEqual({ ...result.answers.team.probabilities }, { billing: 0.1, technical: 0.9 })
83 assert.strictEqual(result.answers.team.confidence, 0.9)
84 assert.strictEqual(result.answers.severity.label, "Critical")
85 assert.strictEqual(result.answers.severity.rating, 1.75)
86 assert.deepStrictEqual({ ...result.answers.severity.probabilities }, { Minor: 0.05, Major: 0.15, Critical: 0.8 })
87 assert.strictEqual(result.answers.urgent.probability, 0.99)
88 assert.strictEqual(result.usage.inputTokens, 120)
89 assert.strictEqual(result.usage.outputTokens, 0)
90 }).pipe(
91 Effect.provide(CloudflareDecisionModel.model("clef")),
92 Effect.provide(clientLayer(answers, (request) => {
93 assert.strictEqual(request.body._tag, "Uint8Array")
94 if (request.body._tag === "Uint8Array") {
95 assert.deepStrictEqual(JSON.parse(new TextDecoder().decode(request.body.body)), {
96 model: "clef",
97 state: input,
98 questions: {
99 team: {
100 type: "choice",
101 instructions: Triage.decisions.team.instructions,
102 criteria: Triage.decisions.team.criteria
103 },
104 severity: {
105 type: "score",
106 instructions: Triage.decisions.severity.instructions,
107 criteria: Triage.decisions.severity.criteria
108 },
109 urgent: {
110 type: "noul",
111 instructions: Triage.decisions.urgent.instructions,
112 criteria: Triage.decisions.urgent.criteria
113 }
114 }
115 })
116 }
117 }))
118 ))
119
120 it.effect("sends an arbitrary model identifier through the decision request", () => {
121 const model = "clef-next"
122 return Effect.gen(function*() {
123 const result = yield* DecisionModel.decide(Triage, { input })
124 assert.strictEqual(yield* Model.ModelName, model)
125 assert.strictEqual(result.answers.urgent.probability, 0.99)
126 }).pipe(
127 Effect.provide(CloudflareDecisionModel.model(model)),
128 Effect.provide(clientLayer(answers, (request) => {
129 assert.strictEqual(
130 request.url,
131 `https://api.cloudflare.com/client/v4/accounts/test-account/ai/run/@cf/cloudflare/${model}`
132 )
133 assert.strictEqual(request.body._tag, "Uint8Array")
134 if (request.body._tag === "Uint8Array") {
135 assert.strictEqual(JSON.parse(new TextDecoder().decode(request.body.body)).model, model)
136 }
137 }, model))
138 )
139 })
140
141 it.effect("accepts probabilities with four-decimal rounding drift", () =>
142 Effect.gen(function*() {
143 const result = yield* DecisionModel.decide(Triage, { input })
144 assert.strictEqual(result.answers.severity.label, "Minor")
145 assert.closeTo(result.answers.severity.probabilities.Minor, 1 / 3, 1e-10)
146 }).pipe(Effect.provide(layer({
147 ...answers,
148 severity: {
149 ...answers.severity,
150 score: 1,
151 confidence: 0.3333,
152 probabilities: { "0": 0.3333, "1": 0.3333, "2": 0.3333 }
153 }
154 }))))
155
156 for (
157 const [name, invalid] of [
158 ["a missing answer", { team: answers.team, severity: answers.severity }],
159 ["a skipped rating level", {
160 ...answers,
161 severity: { ...answers.severity, probabilities: { "0": 0.2, "2": 0.8 } }
162 }],
163 ["drift beyond four decimals", {
164 ...answers,
165 team: { ...answers.team, probabilities: { billing: 0.1, technical: 0.89 } }
166 }]
167 ] as const
168 ) {
169 it.effect(`fails with InvalidOutputError for ${name}`, () =>
170 Effect.gen(function*() {
171 const error = yield* Effect.flip(DecisionModel.decide(Triage, { input }))
172 assert.strictEqual(error.reason._tag, "InvalidOutputError")
173 }).pipe(Effect.provide(layer(invalid))))
174 }
175})