1import { describe, it } from "@effect/vitest"
2import { assertFalse, assertTrue, deepStrictEqual, strictEqual } from "@effect/vitest/utils"
3import { Context, DateTime, Effect, Fiber, identity, Latch, Schema, SchemaGetter, Stream } from "effect"
4import { AiError, LanguageModel, Response, Tool, Toolkit } from "effect/ai"
5import { TestClock } from "effect/testing"
6import * as TestUtils from "./utils.ts"
7
8describe("Tool", () => {
9 describe("User Defined", () => {
10 it.effect("decodes parameters with services provided to the outer handle Effect", () =>
11 Effect.gen(function*() {
12 class Decoder extends Context.Service<Decoder, { readonly value: number }>()("test/ToolkitDecoder") {}
13 const parameter = Schema.String.pipe(Schema.decodeTo(Schema.Number, {
14 decode: SchemaGetter.transformEffect(() => Effect.map(Decoder, (decoder) => decoder.value)),
15 encode: SchemaGetter.transform(String)
16 }))
17 const toolkit = Toolkit.make(Tool.make("Decode", {
18 parameters: Schema.Struct({ n: parameter }),
19 success: Schema.Number
20 }))
21 const handlers = yield* toolkit.pipe(Effect.provide(toolkit.toLayer({
22 Decode: ({ n }) => Effect.succeed(n)
23 })))
24
25 // Different service values in the two phases show which one decodes
26 // the parameters. The Stream's declared requirement is still provided.
27 const stream = yield* handlers.handle("Decode", { n: "1" }).pipe(
28 Effect.provideService(Decoder, { value: 42 })
29 )
30 const results = yield* Stream.runCollect(stream).pipe(
31 Effect.provideService(Decoder, { value: 1 })
32 )
33 strictEqual(results[0]?.result, 42)
34 }))
35
36 it.effect("should return tool call handler successes", () =>
37 Effect.gen(function*() {
38 const toolkit = Toolkit.make(FailureModeReturn)
39
40 const toolResult = { testSuccess: "failure-mode-return-tool" }
41 const handlers = toolkit.toLayer({
42 FailureModeReturn: () => Effect.succeed(toolResult)
43 })
44
45 const toolCallId = "tool-123"
46 const toolName = "FailureModeReturn"
47
48 const response = yield* LanguageModel.generateText({
49 prompt: "Test",
50 toolkit
51 }).pipe(
52 TestUtils.withLanguageModel({
53 generateText: [{
54 type: "tool-call",
55 id: toolCallId,
56 name: toolName,
57 params: { testParam: "test-param" }
58 }]
59 }),
60 Effect.provide(handlers)
61 )
62
63 deepStrictEqual(response.toolResults, [
64 Response.makePart("tool-result", {
65 id: toolCallId,
66 isFailure: false,
67 name: toolName,
68 result: toolResult,
69 encodedResult: toolResult,
70 providerExecuted: false,
71 preliminary: false
72 })
73 ])
74 }))
75
76 it.effect("should return tool call handler failures when failure mode is return", () =>
77 Effect.gen(function*() {
78 const toolkit = Toolkit.make(FailureModeReturn)
79
80 const toolResult = { testFailure: "failure-mode-return-tool" }
81 const handlers = toolkit.toLayer({
82 FailureModeReturn: () => Effect.fail(toolResult)
83 })
84
85 const toolCallId = "tool-123"
86 const toolName = "FailureModeReturn"
87
88 const response = yield* LanguageModel.generateText({
89 prompt: "Test",
90 toolkit
91 }).pipe(
92 TestUtils.withLanguageModel({
93 generateText: [{
94 type: "tool-call",
95 id: toolCallId,
96 name: toolName,
97 params: { testParam: "test-param" }
98 }]
99 }),
100 Effect.provide(handlers)
101 )
102
103 deepStrictEqual(response.toolResults, [
104 Response.makePart("tool-result", {
105 id: toolCallId,
106 name: toolName,
107 isFailure: true,
108 result: toolResult,
109 encodedResult: toolResult,
110 providerExecuted: false,
111 preliminary: false
112 })
113 ])
114 }))
115
116 it.effect("should encode returned failures with the failure schema", () =>
117 Effect.gen(function*() {
118 const toolkit = Toolkit.make(Tool.make("FailureEncoding", {
119 success: Schema.Number,
120 failure: Schema.NumberFromString,
121 failureMode: "return"
122 }))
123 const handlers = yield* toolkit.pipe(Effect.provide(toolkit.toLayer({
124 FailureEncoding: () => Effect.fail(404)
125 })))
126 const results = yield* handlers.handle("FailureEncoding", {}).pipe(Effect.flatMap(Stream.runCollect))
127
128 strictEqual(results[0].encodedResult, "404")
129 }))
130
131 it.effect("should fail the extracted return-mode stream on invalid result encoding", () =>
132 Effect.gen(function*() {
133 const toolkit = Toolkit.make(Tool.make("Positive", {
134 failureMode: "return",
135 success: Schema.Number.check(Schema.isGreaterThan(0))
136 }))
137 const handlers = yield* toolkit.pipe(Effect.provide(toolkit.toLayer({
138 Positive: () => Effect.succeed(-1)
139 })))
140 const inner = yield* handlers.handle("Positive", {})
141 const error = yield* Stream.runCollect(inner).pipe(Effect.flip)
142
143 strictEqual(error._tag, "AiError")
144 strictEqual(error.module, "Toolkit")
145 strictEqual(error.method, "Positive.handle")
146 strictEqual(error.reason._tag, "ToolResultEncodingError")
147 }))
148
149 it.effect("should return tool call handler failures with failure mode return using OpenAI transformer", () =>
150 Effect.gen(function*() {
151 const toolkit = Toolkit.make(FailureModeReturn)
152
153 const toolResult = { testFailure: "failure-mode-return-tool" }
154 const handlers = toolkit.toLayer({
155 FailureModeReturn: () => Effect.fail(toolResult)
156 })
157
158 const toolCallId = "tool-123"
159 const toolName = "FailureModeReturn"
160
161 const response = yield* LanguageModel.generateText({
162 prompt: "Test",
163 toolkit
164 }).pipe(
165 TestUtils.withLanguageModel({
166 generateText: [{
167 type: "tool-call",
168 id: toolCallId,
169 name: toolName,
170 params: { testParam: "test-param" }
171 }]
172 }),
173 Effect.provide(handlers)
174 )
175
176 deepStrictEqual(response.toolResults, [
177 Response.makePart("tool-result", {
178 id: toolCallId,
179 name: toolName,
180 isFailure: true,
181 result: toolResult,
182 encodedResult: toolResult,
183 providerExecuted: false,
184 preliminary: false
185 })
186 ])
187 }))
188
189 it.effect("should raise an error on tool call handler failures when failure mode is error", () =>
190 Effect.gen(function*() {
191 const toolkit = Toolkit.make(FailureModeError)
192
193 const toolResult = { testFailure: "failure-mode-error-tool" }
194 const handlers = toolkit.toLayer({
195 FailureModeError: () => Effect.fail(toolResult)
196 })
197
198 const toolCallId = "tool-123"
199 const toolName = "FailureModeError"
200
201 const response = yield* LanguageModel.generateText({
202 prompt: "Test",
203 toolkit
204 }).pipe(
205 TestUtils.withLanguageModel({
206 generateText: [{
207 type: "tool-call",
208 id: toolCallId,
209 name: toolName,
210 params: { testParam: "test-param" }
211 }]
212 }),
213 Effect.provide(handlers),
214 Effect.flip
215 )
216
217 deepStrictEqual(response, toolResult)
218 }))
219
220 it.effect("should return a failed tool result when tool call parameters are invalid and failure mode is return", () =>
221 Effect.gen(function*() {
222 const toolkit = Toolkit.make(FailureModeReturn)
223
224 const toolResult = { testSuccess: "failure-mode-return-tool" }
225 const handlers = toolkit.toLayer({
226 FailureModeReturn: () => Effect.succeed(toolResult)
227 })
228
229 const toolCallId = "tool-123"
230 const toolName = "FailureModeReturn"
231
232 const response = yield* LanguageModel.generateText({
233 prompt: "Test",
234 toolkit
235 }).pipe(
236 TestUtils.withLanguageModel({
237 generateText: [{
238 type: "tool-call",
239 id: toolCallId,
240 name: toolName,
241 params: {}
242 }]
243 }),
244 Effect.provide(handlers)
245 )
246
247 const description = `Missing key\n at ["testParam"]`
248 deepStrictEqual(response.toolResults, [
249 Response.toolResultPart({
250 id: toolCallId,
251 name: toolName,
252 isFailure: true,
253 providerExecuted: false,
254 preliminary: false,
255 result: AiError.make({
256 module: "Toolkit",
257 method: "FailureModeReturn.handle",
258 reason: new AiError.ToolParameterValidationError({
259 toolName,
260 description
261 })
262 }),
263 encodedResult: {
264 _tag: "AiError",
265 module: "Toolkit",
266 method: "FailureModeReturn.handle",
267 reason: {
268 _tag: "ToolParameterValidationError",
269 toolName,
270 description
271 }
272 }
273 })
274 ])
275 }))
276
277 it.effect("should return a failed tool result when tool call parameters contain NaN and failure mode is return", () =>
278 Effect.gen(function*() {
279 const toolkit = Toolkit.make(FailureModeReturn)
280
281 const handlers = toolkit.toLayer({
282 FailureModeReturn: () => Effect.succeed({ testSuccess: "unused" })
283 })
284
285 const toolCallId = "tool-123"
286 const toolName = "FailureModeReturn"
287
288 const response = yield* LanguageModel.generateText({
289 prompt: "Test",
290 toolkit
291 }).pipe(
292 TestUtils.withLanguageModel({
293 generateText: [{
294 type: "tool-call",
295 id: toolCallId,
296 name: toolName,
297 params: { testParam: NaN }
298 }]
299 }),
300 Effect.provide(handlers)
301 )
302
303 strictEqual(response.toolResults.length, 1)
304 const toolResult = response.toolResults[0]
305 strictEqual(toolResult.isFailure, true)
306 const result = toolResult.result as AiError.AiError
307 strictEqual(result._tag, "AiError")
308 strictEqual(result.reason._tag, "ToolParameterValidationError")
309 }))
310
311 it.effect("should raise an error when tool call parameters are invalid and failure mode is error", () =>
312 Effect.gen(function*() {
313 const toolkit = Toolkit.make(FailureModeError)
314
315 const handlers = toolkit.toLayer({
316 FailureModeError: () => Effect.succeed({ testSuccess: "unused" })
317 })
318
319 const toolCallId = "tool-123"
320 const toolName = "FailureModeError"
321
322 const response = yield* LanguageModel.generateText({
323 prompt: "Test",
324 toolkit
325 }).pipe(
326 TestUtils.withLanguageModel({
327 generateText: [{
328 type: "tool-call",
329 id: toolCallId,
330 name: toolName,
331 params: { testParam: NaN }
332 }]
333 }),
334 Effect.provide(handlers),
335 Effect.flip
336 )
337
338 deepStrictEqual(
339 response,
340 AiError.make({
341 module: "Toolkit",
342 method: "FailureModeError.handle",
343 reason: new AiError.ToolParameterValidationError({
344 toolName: "FailureModeError",
345 description: `Expected string\n at ["testParam"]`
346 })
347 })
348 )
349 }))
350
351 it.effect("should return AiError when user returns an AiErrorReason when failure mode is return", () =>
352 Effect.gen(function*() {
353 const toolkit = Toolkit.make(FailureModeReturn)
354
355 const reason = new AiError.RateLimitError({})
356 const handlers = toolkit.toLayer({
357 FailureModeReturn: () => Effect.fail(reason)
358 })
359
360 const toolCallId = "tool-123"
361 const toolName = "FailureModeReturn"
362
363 const response = yield* LanguageModel.generateText({
364 prompt: "Test",
365 toolkit
366 }).pipe(
367 TestUtils.withLanguageModel({
368 generateText: [{
369 type: "tool-call",
370 id: toolCallId,
371 name: toolName,
372 params: { testParam: "test-param" }
373 }]
374 }),
375 Effect.provide(handlers)
376 )
377
378 deepStrictEqual(response.toolResults, [
379 Response.toolResultPart({
380 id: toolCallId,
381 name: toolName,
382 isFailure: true,
383 providerExecuted: false,
384 preliminary: false,
385 result: AiError.make({
386 module: "Toolkit",
387 method: "FailureModeReturn.handle",
388 reason
389 }),
390 encodedResult: {
391 _tag: "AiError",
392 module: "Toolkit",
393 method: "FailureModeReturn.handle",
394 reason: { _tag: "RateLimitError", metadata: {} }
395 }
396 })
397 ])
398 }))
399
400 it.effect("should raise an AiError when user returns an AiErrorReason when failure mode is error", () =>
401 Effect.gen(function*() {
402 const toolkit = Toolkit.make(FailureModeError)
403
404 const reason = new AiError.RateLimitError({})
405 const handlers = toolkit.toLayer({
406 FailureModeError: () => Effect.fail(reason)
407 })
408
409 const toolCallId = "tool-123"
410 const toolName = "FailureModeError"
411
412 const response = yield* LanguageModel.generateText({
413 prompt: "Test",
414 toolkit
415 }).pipe(
416 TestUtils.withLanguageModel({
417 generateText: [{
418 type: "tool-call",
419 id: toolCallId,
420 name: toolName,
421 params: { testParam: "test-param" }
422 }]
423 }),
424 Effect.provide(handlers),
425 Effect.flip
426 )
427
428 deepStrictEqual(
429 response,
430 AiError.make({
431 module: "Toolkit",
432 method: "FailureModeError.handle",
433 reason
434 })
435 )
436 }))
437
438 describe("Preliminary Results", () => {
439 it.effect("should not have preliminary results when generateText is used", () =>
440 Effect.gen(function*() {
441 const toolkit = Toolkit.make(IncrementalTool)
442
443 const toolCallId = "tool-123"
444 const toolName = "IncrementalTool"
445 const preliminaryResult = { status: "loading", progress: 50 } as const
446 const terminalResult = { status: "complete" } as const
447
448 const handlers = toolkit.toLayer({
449 IncrementalTool: Effect.fnUntraced(function*(_, ctx) {
450 yield* ctx.preliminary(preliminaryResult)
451 return terminalResult
452 })
453 })
454
455 const response = yield* LanguageModel.generateText({
456 prompt: "Test",
457 toolkit
458 }).pipe(
459 TestUtils.withLanguageModel({
460 generateText: [{
461 type: "tool-call",
462 id: toolCallId,
463 name: toolName,
464 params: { input: "test" }
465 }]
466 }),
467 Effect.provide(handlers)
468 )
469
470 const hasPreliminaryResults = response.content.some((part) => {
471 return part.type === "tool-result" && part.preliminary === true
472 })
473
474 assertFalse(hasPreliminaryResults)
475 deepStrictEqual(
476 response.content[1],
477 Response.toolResultPart({
478 id: toolCallId,
479 name: toolName,
480 isFailure: false,
481 result: terminalResult,
482 encodedResult: terminalResult,
483 providerExecuted: false,
484 preliminary: false
485 })
486 )
487 }))
488
489 it.effect("should have preliminary results when streamText is used", () =>
490 Effect.gen(function*() {
491 const toolkit = Toolkit.make(IncrementalTool)
492
493 const toolCallId = "tool-123"
494 const toolName = "IncrementalTool"
495 const preliminaryResult = { status: "loading", progress: 50 } as const
496 const terminalResult = { status: "complete" } as const
497
498 const handlers = toolkit.toLayer({
499 IncrementalTool: Effect.fnUntraced(function*(_, ctx) {
500 yield* ctx.preliminary(preliminaryResult)
501 return terminalResult
502 })
503 })
504
505 const response = yield* LanguageModel.streamText({
506 prompt: "Test",
507 toolkit
508 }).pipe(
509 Stream.runCollect,
510 TestUtils.withLanguageModel({
511 streamText: [{
512 type: "tool-call",
513 id: toolCallId,
514 name: toolName,
515 params: { input: "test" }
516 }]
517 }),
518 Effect.provide(handlers)
519 )
520
521 deepStrictEqual(
522 response[1],
523 Response.toolResultPart({
524 id: toolCallId,
525 name: toolName,
526 isFailure: false,
527 result: preliminaryResult,
528 encodedResult: preliminaryResult,
529 providerExecuted: false,
530 preliminary: true
531 })
532 )
533
534 deepStrictEqual(
535 response[2],
536 Response.toolResultPart({
537 id: toolCallId,
538 name: toolName,
539 isFailure: false,
540 result: terminalResult,
541 encodedResult: terminalResult,
542 providerExecuted: false,
543 preliminary: false
544 })
545 )
546 }))
547
548 it.effect("should handle multiple preliminary results in sequence", () =>
549 Effect.gen(function*() {
550 const toolkit = Toolkit.make(IncrementalTool)
551
552 const toolCallId = "tool-123"
553 const toolName = "IncrementalTool"
554
555 const handlers = toolkit.toLayer({
556 IncrementalTool: Effect.fnUntraced(function*(_, ctx) {
557 yield* ctx.preliminary({ status: "loading", progress: 0 })
558 yield* ctx.preliminary({ status: "processing", progress: 50 })
559 yield* ctx.preliminary({ status: "finalizing", progress: 90 })
560 return { status: "complete" }
561 })
562 })
563
564 const response = yield* LanguageModel.streamText({
565 prompt: "Test",
566 toolkit
567 }).pipe(
568 Stream.runCollect,
569 TestUtils.withLanguageModel({
570 streamText: [{
571 type: "tool-call",
572 id: toolCallId,
573 name: toolName,
574 params: { input: "test" }
575 }]
576 }),
577 Effect.provide(handlers)
578 )
579
580 const toolResults = response.filter((part) => part.type === "tool-result")
581
582 // Should have 3 preliminary + 1 final = 4 results
583 strictEqual(toolResults.length, 4)
584
585 // Verify ordering and preliminary flags
586 strictEqual(toolResults[0].preliminary, true)
587 deepStrictEqual(toolResults[0].result, { status: "loading", progress: 0 })
588
589 strictEqual(toolResults[1].preliminary, true)
590 deepStrictEqual(toolResults[1].result, { status: "processing", progress: 50 })
591
592 strictEqual(toolResults[2].preliminary, true)
593 deepStrictEqual(toolResults[2].result, { status: "finalizing", progress: 90 })
594
595 strictEqual(toolResults[3].preliminary, false)
596 deepStrictEqual(toolResults[3].result, { status: "complete" })
597 }))
598
599 it.effect("should handle preliminary results for long-running tools", () =>
600 Effect.gen(function*() {
601 const toolkit = Toolkit.make(IncrementalTool)
602
603 const toolCallId = "tool-123"
604 const toolName = "IncrementalTool"
605
606 const handlers = toolkit.toLayer({
607 IncrementalTool: Effect.fnUntraced(function*(_, ctx) {
608 yield* ctx.preliminary({ status: "loading", progress: 0 })
609 yield* Effect.sleep("5 seconds")
610 return { status: "complete" }
611 })
612 })
613
614 const terminalLatch = yield* Latch.make()
615 const assertions = { preliminary: false, terminal: false }
616
617 const fiber = yield* LanguageModel.streamText({
618 prompt: "Test",
619 toolkit
620 }).pipe(
621 Stream.tap(Effect.fnUntraced(function*(part) {
622 if (part.type === "tool-result") {
623 if (!assertions.preliminary) {
624 yield* terminalLatch.open
625 strictEqual(part.preliminary, true)
626 deepStrictEqual(part.result, { status: "loading", progress: 0 })
627 assertions.preliminary = true
628 } else {
629 strictEqual(part.preliminary, false)
630 deepStrictEqual(part.result, { status: "complete" })
631 assertions.terminal = true
632 }
633 }
634 })),
635 Stream.runCollect,
636 TestUtils.withLanguageModel({
637 streamText: [{
638 type: "tool-call",
639 id: toolCallId,
640 name: toolName,
641 params: { input: "test" }
642 }]
643 }),
644 Effect.provide(handlers),
645 Effect.forkScoped
646 )
647
648 yield* terminalLatch.await
649 yield* TestClock.adjust("5 seconds")
650 const response = yield* Fiber.join(fiber)
651
652 assertTrue(Object.values(assertions).every(identity))
653
654 // Should have 1 preliminary + 1 final = 2 results
655 const toolResults = response.filter((part) => part.type === "tool-result")
656 strictEqual(toolResults.length, 2)
657 }))
658
659 it.effect("should emit preliminary results before failure when handler fails", () =>
660 Effect.gen(function*() {
661 const toolkit = Toolkit.make(IncrementalToolWithFailure)
662
663 const toolCallId = "tool-123"
664 const toolName = "IncrementalToolWithFailure"
665
666 const handlers = toolkit.toLayer({
667 IncrementalToolWithFailure: Effect.fnUntraced(function*(_, ctx) {
668 yield* ctx.preliminary({ status: "loading" })
669 yield* ctx.preliminary({ status: "processing" })
670 return yield* Effect.fail({ error: "Something went wrong" })
671 })
672 })
673
674 const response = yield* LanguageModel.streamText({
675 prompt: "Test",
676 toolkit
677 }).pipe(
678 Stream.runCollect,
679 TestUtils.withLanguageModel({
680 streamText: [{
681 type: "tool-call",
682 id: toolCallId,
683 name: toolName,
684 params: { input: "test" }
685 }]
686 }),
687 Effect.provide(handlers)
688 )
689
690 const toolResults = response.filter((part) => part.type === "tool-result")
691
692 // Should have 2 preliminary + 1 final (failure) = 3 results
693 strictEqual(toolResults.length, 3)
694
695 // Preliminary results emitted before failure
696 strictEqual(toolResults[0].preliminary, true)
697 strictEqual(toolResults[0].isFailure, false)
698 deepStrictEqual(toolResults[0].result, { status: "loading" })
699
700 strictEqual(toolResults[1].preliminary, true)
701 strictEqual(toolResults[1].isFailure, false)
702 deepStrictEqual(toolResults[1].result, { status: "processing" })
703
704 // Final result is the failure
705 strictEqual(toolResults[2].preliminary, false)
706 strictEqual(toolResults[2].isFailure, true)
707 deepStrictEqual(toolResults[2].result, { error: "Something went wrong" })
708 }))
709
710 it.effect("should interleave results from concurrent tools with preliminary results", () =>
711 Effect.gen(function*() {
712 const toolkit = Toolkit.make(ConcurrentIncrementalTool)
713
714 const handlers = toolkit.toLayer({
715 ConcurrentIncrementalTool: Effect.fnUntraced(function*({ name, delay }, ctx) {
716 yield* ctx.preliminary({ status: `loading`, job: name, progress: 50 })
717 yield* Effect.sleep(delay)
718 return { status: `complete`, job: name }
719 })
720 })
721
722 const latch = yield* Latch.make()
723 const toolResults: Array<Response.ToolResultParts<Toolkit.Tools<typeof toolkit>>> = []
724
725 const fiber = yield* LanguageModel.streamText({
726 prompt: "Test",
727 toolkit
728 }).pipe(
729 Stream.tap(() => latch.open),
730 Stream.tap((part) => {
731 if (part.type === "tool-result") {
732 return Effect.sync(() => toolResults.push(part))
733 }
734 return Effect.void
735 }),
736 Stream.runCollect,
737 TestUtils.withLanguageModel({
738 streamText: [
739 {
740 type: "tool-call",
741 id: "tool-fast",
742 name: "ConcurrentIncrementalTool",
743 params: { name: "fast", delay: 1000 }
744 },
745 {
746 type: "tool-call",
747 id: "tool-slow",
748 name: "ConcurrentIncrementalTool",
749 params: { name: "slow", delay: 2000 }
750 }
751 ]
752 }),
753 Effect.provide(handlers),
754 Effect.forkScoped
755 )
756
757 // Wait for stream to start (first item emitted)
758 yield* latch.await
759
760 // Give forked tool handlers time to emit preliminary results
761 yield* Effect.yieldNow
762 yield* Effect.yieldNow
763 yield* Effect.yieldNow
764
765 // Initially both preliminary results should be emitted immediately
766 strictEqual(toolResults.length, 2)
767 strictEqual(toolResults[0].preliminary, true)
768 deepStrictEqual(toolResults[0].result, { status: "loading", job: "fast", progress: 50 })
769 strictEqual(toolResults[1].preliminary, true)
770 deepStrictEqual(toolResults[1].result, { status: "loading", job: "slow", progress: 50 })
771
772 // After 1 second, fast tool completes
773 yield* TestClock.adjust("1 second")
774 yield* Effect.yieldNow
775 strictEqual(toolResults.length, 3)
776 strictEqual(toolResults[2].preliminary, false)
777 deepStrictEqual(toolResults[2].result, { status: "complete", job: "fast" })
778
779 // After 2 more seconds (3 total), slow tool completes
780 yield* TestClock.adjust("2 seconds")
781 yield* Effect.yieldNow
782 yield* Fiber.join(fiber)
783
784 strictEqual(toolResults.length, 4)
785 strictEqual(toolResults[3].preliminary, false)
786 deepStrictEqual(toolResults[3].result, { status: "complete", job: "slow" })
787 }))
788 })
789 })
790
791 describe("Provider Defined", () => {
792 it.effect("should return tool call successes from the model", () =>
793 Effect.gen(function*() {
794 const tool = NoHandlerRequired({
795 testArg: "test-arg"
796 })
797 const toolkit = Toolkit.make(tool)
798
799 const toolCallId = "tool-123"
800 const toolResult = { testSuccess: "provider-defined-tool" }
801
802 const response = yield* LanguageModel.generateText({
803 prompt: "Test",
804 toolkit
805 }).pipe(
806 TestUtils.withLanguageModel({
807 generateText: [
808 {
809 type: "tool-call",
810 id: toolCallId,
811 name: tool.name,
812 providerExecuted: true,
813 params: { testParam: "test-param" }
814 },
815 {
816 type: "tool-result",
817 id: toolCallId,
818 name: tool.name,
819 isFailure: false,
820 result: toolResult,
821 providerExecuted: true
822 }
823 ]
824 })
825 )
826
827 deepStrictEqual(response.toolResults, [
828 Response.makePart("tool-result", {
829 id: toolCallId,
830 name: tool.name,
831 isFailure: false,
832 result: toolResult,
833 encodedResult: toolResult,
834 providerExecuted: true,
835 preliminary: false
836 })
837 ])
838 }))
839
840 it.effect("should return tool call failures from the model", () =>
841 Effect.gen(function*() {
842 const tool = NoHandlerRequired({
843 testArg: "test-arg"
844 })
845 const toolkit = Toolkit.make(tool)
846
847 const toolCallId = "tool-123"
848 const toolResult = { testFailure: "provider-defined-tool" }
849
850 const response = yield* LanguageModel.generateText({
851 prompt: "Test",
852 toolkit
853 }).pipe(
854 TestUtils.withLanguageModel({
855 generateText: [
856 {
857 type: "tool-call",
858 id: toolCallId,
859 name: tool.name,
860 providerExecuted: true,
861 params: { testParam: "test-param" }
862 },
863 {
864 type: "tool-result",
865 id: toolCallId,
866 isFailure: true,
867 name: tool.name,
868 result: toolResult,
869 providerExecuted: true
870 }
871 ]
872 })
873 )
874
875 deepStrictEqual(response.toolResults, [
876 Response.makePart("tool-result", {
877 id: toolCallId,
878 name: tool.name,
879 isFailure: true,
880 result: toolResult,
881 encodedResult: toolResult,
882 providerExecuted: true,
883 preliminary: false
884 })
885 ])
886 }))
887
888 it.effect("should return tool call handler successes", () =>
889 Effect.gen(function*() {
890 const tool = HandlerRequired({
891 testArg: "test-arg"
892 })
893
894 const toolCallId = "tool-123"
895 const toolResult = { testSuccess: "provider-defined-tool" }
896
897 const toolkit = Toolkit.make(tool)
898 const handlers = toolkit.toLayer({
899 HandlerRequired: () => Effect.succeed(toolResult)
900 })
901
902 const response = yield* LanguageModel.generateText({
903 prompt: "Test",
904 toolkit
905 }).pipe(
906 TestUtils.withLanguageModel({
907 generateText: [
908 {
909 type: "tool-call",
910 id: toolCallId,
911 name: tool.name,
912 // Given this provider-defined tool requires a user-space
913 // handler, it is not considered `providerExecuted`
914 providerExecuted: false,
915 params: { testParam: "test-param" }
916 }
917 ]
918 }),
919 Effect.provide(handlers)
920 )
921
922 deepStrictEqual(response.toolResults, [
923 Response.makePart("tool-result", {
924 id: toolCallId,
925 name: tool.name,
926 isFailure: false,
927 result: toolResult,
928 encodedResult: toolResult,
929 providerExecuted: false,
930 preliminary: false
931 })
932 ])
933 }))
934
935 it.effect("should return tool call handler failures when failure mode is return", () =>
936 Effect.gen(function*() {
937 const tool = HandlerRequired({
938 failureMode: "return",
939 testArg: "test-arg"
940 })
941
942 const toolCallId = "tool-123"
943 const toolResult = { testFailure: "provider-defined-tool" }
944
945 const toolkit = Toolkit.make(tool)
946 const handlers = toolkit.toLayer({
947 HandlerRequired: () => Effect.fail(toolResult)
948 })
949
950 const response = yield* LanguageModel.generateText({
951 prompt: "Test",
952 toolkit
953 }).pipe(
954 TestUtils.withLanguageModel({
955 generateText: [
956 {
957 type: "tool-call",
958 id: toolCallId,
959 name: tool.name,
960 // Given this provider-defined tool requires a user-space
961 // handler, it is not considered `providerExecuted`
962 providerExecuted: false,
963 params: { testParam: "test-param" }
964 }
965 ]
966 }),
967 Effect.provide(handlers)
968 )
969
970 deepStrictEqual(response.toolResults, [
971 Response.makePart("tool-result", {
972 id: toolCallId,
973 name: tool.name,
974 isFailure: true,
975 result: toolResult,
976 encodedResult: toolResult,
977 providerExecuted: false,
978 preliminary: false
979 })
980 ])
981 }))
982
983 it.effect("should raise an error on tool call handler failures when failure mode is error", () =>
984 Effect.gen(function*() {
985 const tool = HandlerRequired({
986 testArg: "test-arg"
987 })
988
989 const toolCallId = "tool-123"
990 const toolResult = { testFailure: "provider-defined-tool" }
991
992 const toolkit = Toolkit.make(tool)
993 const handlers = toolkit.toLayer({
994 HandlerRequired: () => Effect.fail(toolResult)
995 })
996
997 const response = yield* LanguageModel.generateText({
998 prompt: "Test",
999 toolkit
1000 }).pipe(
1001 TestUtils.withLanguageModel({
1002 generateText: [
1003 {
1004 type: "tool-call",
1005 id: toolCallId,
1006 name: tool.name,
1007 // Given this provider-defined tool requires a user-space
1008 // handler, it is not considered `providerExecuted`
1009 providerExecuted: false,
1010 params: { testParam: "test-param" }
1011 }
1012 ]
1013 }),
1014 Effect.provide(handlers),
1015 Effect.flip
1016 )
1017
1018 deepStrictEqual(response, toolResult)
1019 }))
1020
1021 it.effect("should return a failed tool result when tool call parameters are invalid and failure mode is return", () =>
1022 Effect.gen(function*() {
1023 const tool = HandlerRequired({
1024 failureMode: "return",
1025 testArg: "test-arg"
1026 })
1027
1028 const toolCallId = "tool-123"
1029 const toolResult = { testSuccess: "provider-defined-tool" }
1030
1031 const toolkit = Toolkit.make(tool)
1032 const handlers = toolkit.toLayer({
1033 HandlerRequired: () => Effect.succeed(toolResult)
1034 })
1035
1036 const response = yield* LanguageModel.generateText({
1037 prompt: "Test",
1038 toolkit
1039 }).pipe(
1040 TestUtils.withLanguageModel({
1041 generateText: [
1042 {
1043 type: "tool-call",
1044 id: toolCallId,
1045 name: tool.name,
1046 // Given this provider-defined tool requires a user-space
1047 // handler, it is not considered `providerExecuted`
1048 providerExecuted: false,
1049 params: {}
1050 }
1051 ]
1052 }),
1053 Effect.provide(handlers)
1054 )
1055
1056 const description = `Missing key\n at ["testParam"]`
1057 deepStrictEqual(response.toolResults, [
1058 Response.toolResultPart({
1059 id: toolCallId,
1060 name: tool.name,
1061 isFailure: true,
1062 providerExecuted: false,
1063 preliminary: false,
1064 result: AiError.make({
1065 module: "Toolkit",
1066 method: "HandlerRequired.handle",
1067 reason: new AiError.ToolParameterValidationError({
1068 toolName: "HandlerRequired",
1069 description
1070 })
1071 }),
1072 encodedResult: {
1073 _tag: "AiError",
1074 module: "Toolkit",
1075 method: "HandlerRequired.handle",
1076 reason: {
1077 _tag: "ToolParameterValidationError",
1078 toolName: "HandlerRequired",
1079 description
1080 }
1081 }
1082 })
1083 ])
1084 }))
1085
1086 describe("addDependency", () => {
1087 it("preserves the provider-defined tool identity", () => {
1088 const tool = HandlerRequired({ testArg: "test-arg" })
1089 const withDep = tool.addDependency(TestDependency)
1090 assertTrue(Tool.isProviderDefined(withDep))
1091 assertFalse(Tool.isUserDefined(withDep))
1092 assertFalse(Tool.isDynamic(withDep))
1093 })
1094
1095 it("preserves the provider-defined tool id", () => {
1096 const tool = HandlerRequired({ testArg: "test-arg" })
1097 const withDep = tool.addDependency(TestDependency)
1098 strictEqual(withDep.id, "test.handler_required")
1099 })
1100 })
1101
1102 it("getStrictMode returns undefined without throwing", () => {
1103 const tool = NoHandlerRequired({ testArg: "test-arg" })
1104 strictEqual(Tool.getStrictMode(tool), undefined)
1105 })
1106
1107 it("annotate preserves the provider-defined tool identity", () => {
1108 const tool = NoHandlerRequired({ testArg: "test-arg" }).annotate(Tool.Strict, false)
1109 assertTrue(Tool.isProviderDefined(tool))
1110 strictEqual(Tool.getStrictMode(tool), false)
1111 })
1112 })
1113})
1114
1115class TestDependency extends Context.Service<TestDependency, { readonly value: number }>()("TestDependency") {}
1116
1117const FailureModeError = Tool.make("FailureModeError", {
1118 description: "A test tool",
1119 parameters: Schema.Struct({
1120 testParam: Schema.String
1121 }),
1122 success: Schema.Struct({
1123 testSuccess: Schema.String
1124 }),
1125 failure: Schema.Struct({
1126 testFailure: Schema.String
1127 })
1128})
1129
1130const FailureModeReturn = Tool.make("FailureModeReturn", {
1131 description: "A test tool",
1132 failureMode: "return",
1133 parameters: Schema.Struct({
1134 testParam: Schema.String
1135 }),
1136 success: Schema.Struct({
1137 testSuccess: Schema.String
1138 }),
1139 failure: Schema.Struct({
1140 testFailure: Schema.String
1141 })
1142})
1143
1144const IncrementalTool = Tool.make("IncrementalTool", {
1145 description: "A test tool",
1146 parameters: Schema.Struct({ input: Schema.String }),
1147 success: Schema.Union([
1148 Schema.Struct({ status: Schema.Literal("loading"), progress: Schema.Number }),
1149 Schema.Struct({ status: Schema.Literal("processing"), progress: Schema.Number }),
1150 Schema.Struct({ status: Schema.Literal("finalizing"), progress: Schema.Number }),
1151 Schema.Struct({ status: Schema.Literal("complete") })
1152 ])
1153})
1154
1155const ConcurrentIncrementalTool = Tool.make("ConcurrentIncrementalTool", {
1156 description: "A test tool",
1157 parameters: Schema.Struct({ name: Schema.String, delay: Schema.DurationFromMillis }),
1158 success: Schema.Union([
1159 Schema.Struct({ status: Schema.Literal("loading"), job: Schema.String, progress: Schema.Number }),
1160 Schema.Struct({ status: Schema.Literal("complete"), job: Schema.String })
1161 ])
1162})
1163
1164const IncrementalToolWithFailure = Tool.make("IncrementalToolWithFailure", {
1165 description: "A test tool",
1166 failureMode: "return",
1167 parameters: Schema.Struct({ input: Schema.String }),
1168 success: Schema.Struct({
1169 status: Schema.String,
1170 progress: Schema.optional(Schema.Number)
1171 }),
1172 failure: Schema.Struct({
1173 error: Schema.String
1174 })
1175})
1176
1177const NoHandlerRequired = Tool.providerDefined({
1178 id: "test.no_handler_required",
1179 customName: "NoHandlerRequired",
1180 providerName: "no_handler_required",
1181 args: Schema.Struct({
1182 testArg: Schema.String
1183 }),
1184 parameters: Schema.Struct({
1185 testParam: Schema.String
1186 }),
1187 success: Schema.Struct({
1188 testSuccess: Schema.String
1189 }),
1190 failure: Schema.Struct({
1191 testFailure: Schema.String
1192 })
1193})
1194
1195const HandlerRequired = Tool.providerDefined({
1196 id: "test.handler_required",
1197 customName: "HandlerRequired",
1198 providerName: "handler_required",
1199 requiresHandler: true,
1200 args: Schema.Struct({
1201 testArg: Schema.String
1202 }),
1203 parameters: Schema.Struct({
1204 testParam: Schema.String
1205 }),
1206 success: Schema.Struct({
1207 testSuccess: Schema.String
1208 }),
1209 failure: Schema.Struct({
1210 testFailure: Schema.String
1211 })
1212})
1213
1214describe("setNeedsApproval", () => {
1215 it("sets a static approval requirement without mutating the original tool", () => {
1216 const tool = Tool.make("TestTool", { needsApproval: true })
1217 const updated = tool.setNeedsApproval(false)
1218
1219 strictEqual(tool.needsApproval, true)
1220 strictEqual(updated.needsApproval, false)
1221 })
1222
1223 it("sets a dynamic approval requirement", () => {
1224 const needsApproval = (params: { readonly dangerous: boolean }) => params.dangerous
1225 const tool = Tool.make("TestTool", {
1226 parameters: Schema.Struct({ dangerous: Schema.Boolean })
1227 }).setNeedsApproval(needsApproval)
1228
1229 strictEqual(tool.needsApproval, needsApproval)
1230 })
1231
1232 it("preserves dynamic tool identity", () => {
1233 const tool = Tool.dynamic("TestTool", {
1234 parameters: { type: "object", properties: {} }
1235 }).setNeedsApproval(true)
1236
1237 assertTrue(Tool.isDynamic(tool))
1238 strictEqual(tool.needsApproval, true)
1239 })
1240})
1241
1242describe("Dynamic", () => {
1243 it("setParameters replaces a raw JSON Schema", () => {
1244 const tool = Tool.dynamic("TestTool", {
1245 parameters: {
1246 type: "object",
1247 properties: { query: { type: "string" } },
1248 required: ["query"]
1249 }
1250 }).setParameters(Schema.Struct({ count: Schema.Number }))
1251
1252 deepStrictEqual(Tool.getJsonSchema(tool).required, ["count"], "expected the replacement schema")
1253 })
1254
1255 describe("isDynamic", () => {
1256 it.effect("returns true for dynamic tools with Effect Schema", () =>
1257 Effect.gen(function*() {
1258 const tool = Tool.dynamic("TestTool", {
1259 parameters: Schema.Struct({ query: Schema.String })
1260 })
1261 assertTrue(Tool.isDynamic(tool))
1262 }))
1263
1264 it.effect("returns true for dynamic tools with JSON Schema", () =>
1265 Effect.gen(function*() {
1266 const tool = Tool.dynamic("TestTool", {
1267 parameters: { type: "object", properties: {} }
1268 })
1269 assertTrue(Tool.isDynamic(tool))
1270 }))
1271
1272 it.effect("returns false for user-defined tools", () =>
1273 Effect.gen(function*() {
1274 const tool = Tool.make("TestTool", {
1275 parameters: Schema.Struct({ query: Schema.String })
1276 })
1277 assertFalse(Tool.isDynamic(tool))
1278 }))
1279
1280 it.effect("returns false for provider-defined tools", () =>
1281 Effect.gen(function*() {
1282 const tool = Tool.providerDefined({
1283 id: "test.provider_tool",
1284 customName: "ProviderTool",
1285 providerName: "provider_tool"
1286 })()
1287 assertFalse(Tool.isDynamic(tool))
1288 }))
1289
1290 it.effect("returns false for non-tool values", () =>
1291 Effect.gen(function*() {
1292 assertFalse(Tool.isDynamic(null))
1293 assertFalse(Tool.isDynamic(undefined))
1294 assertFalse(Tool.isDynamic({}))
1295 assertFalse(Tool.isDynamic({ name: "fake" }))
1296 }))
1297 })
1298
1299 describe("getJsonSchema", () => {
1300 it.effect("Tool.make EmptyParams produces same output as omitting parameters", () =>
1301 Effect.gen(function*() {
1302 const withEmpty = Tool.make("A", { parameters: Tool.EmptyParams })
1303 const withoutParams = Tool.make("B")
1304 const withEmptyJsonSchema = Tool.getJsonSchema(withEmpty)
1305 deepStrictEqual(withEmptyJsonSchema, Tool.getJsonSchema(withoutParams))
1306 deepStrictEqual(withEmptyJsonSchema, {
1307 "type": "object",
1308 "additionalProperties": false
1309 })
1310 }))
1311
1312 it.effect("derives JSON Schema from Effect Schema parameters", () =>
1313 Effect.gen(function*() {
1314 const tool = Tool.dynamic("TestTool", {
1315 parameters: Schema.Struct({
1316 query: Schema.String,
1317 limit: Schema.optional(Schema.Number)
1318 })
1319 })
1320
1321 const jsonSchema = Tool.getJsonSchema(tool)
1322
1323 strictEqual(jsonSchema.type, "object")
1324 deepStrictEqual(jsonSchema.required, ["query"])
1325 assertTrue("properties" in jsonSchema && "query" in (jsonSchema.properties as object))
1326 assertTrue("properties" in jsonSchema && "limit" in (jsonSchema.properties as object))
1327 }))
1328
1329 it.effect("returns JSON Schema directly when provided as parameters", () =>
1330 Effect.gen(function*() {
1331 const inputSchema = {
1332 type: "object",
1333 properties: {
1334 query: { type: "string", description: "Search query" },
1335 limit: { type: "number", minimum: 1, maximum: 100 }
1336 },
1337 required: ["query"],
1338 additionalProperties: false
1339 } as const
1340
1341 const tool = Tool.dynamic("TestTool", { parameters: inputSchema })
1342 const jsonSchema = Tool.getJsonSchema(tool)
1343
1344 deepStrictEqual(jsonSchema, inputSchema)
1345 }))
1346
1347 it.effect("preserves complex JSON Schema features like $ref and oneOf", () =>
1348 Effect.gen(function*() {
1349 const inputSchema = {
1350 type: "object",
1351 properties: {
1352 value: { oneOf: [{ type: "string" }, { type: "number" }] },
1353 item: { $ref: "#/$defs/Item" }
1354 },
1355 $defs: {
1356 Item: { type: "object", properties: { name: { type: "string" } } }
1357 }
1358 } as const
1359
1360 const tool = Tool.dynamic("TestTool", { parameters: inputSchema })
1361 const jsonSchema = Tool.getJsonSchema(tool)
1362
1363 deepStrictEqual(jsonSchema, inputSchema)
1364 }))
1365 })
1366
1367 describe("handler execution", () => {
1368 it.effect("decodes parameters with Effect Schema before calling handler", () =>
1369 Effect.gen(function*() {
1370 const SearchTool = Tool.dynamic("SearchTool", {
1371 parameters: Schema.Struct({
1372 query: Schema.String,
1373 limit: Schema.Number
1374 }),
1375 success: Schema.Array(Schema.String)
1376 })
1377
1378 const toolkit = Toolkit.make(SearchTool)
1379
1380 const layer = toolkit.toLayer({
1381 SearchTool: ({ query, limit }) => Effect.succeed(Array.from({ length: limit }, (_, i) => `${query}-${i}`))
1382 })
1383
1384 const response = yield* LanguageModel.generateText({
1385 prompt: "Test",
1386 toolkit
1387 }).pipe(
1388 TestUtils.withLanguageModel({
1389 generateText: [{
1390 type: "tool-call",
1391 id: "tool-123",
1392 name: "SearchTool",
1393 params: { query: "test", limit: 3 }
1394 }]
1395 }),
1396 Effect.provide(layer)
1397 )
1398
1399 deepStrictEqual(response.toolResults[0].result, ["test-0", "test-1", "test-2"])
1400 }))
1401
1402 it.effect("passes parameters through as unknown with JSON Schema", () =>
1403 Effect.gen(function*() {
1404 const SearchTool = Tool.dynamic("SearchTool", {
1405 parameters: {
1406 type: "object",
1407 properties: { query: { type: "string" }, limit: { type: "number" } }
1408 },
1409 success: Schema.Array(Schema.String)
1410 })
1411
1412 const toolkit = Toolkit.make(SearchTool)
1413
1414 const layer = toolkit.toLayer({
1415 SearchTool: (params: unknown) =>
1416 Effect.gen(function*() {
1417 const { query, limit } = params as { query: string; limit: number }
1418 return Array.from({ length: limit }, (_, i) => `${query}-${i}`)
1419 })
1420 })
1421
1422 const response = yield* LanguageModel.generateText({
1423 prompt: "Test",
1424 toolkit
1425 }).pipe(
1426 TestUtils.withLanguageModel({
1427 generateText: [{
1428 type: "tool-call",
1429 id: "tool-123",
1430 name: "SearchTool",
1431 params: { query: "test", limit: 3 }
1432 }]
1433 }),
1434 Effect.provide(layer)
1435 )
1436
1437 deepStrictEqual(response.toolResults[0].result, ["test-0", "test-1", "test-2"])
1438 }))
1439
1440 it.effect("encodes success result using success schema", () =>
1441 Effect.gen(function*() {
1442 const tool = Tool.dynamic("TestTool", {
1443 parameters: { type: "object", properties: {} },
1444 success: Schema.Struct({ timestamp: Schema.DateTimeUtcFromMillis })
1445 })
1446
1447 const toolkit = Toolkit.make(tool)
1448
1449 const layer = toolkit.toLayer({
1450 TestTool: () => Effect.succeed({ timestamp: DateTime.fromDateUnsafe(new Date(1000)) })
1451 })
1452
1453 const response = yield* LanguageModel.generateText({
1454 prompt: "Test",
1455 toolkit
1456 }).pipe(
1457 TestUtils.withLanguageModel({
1458 generateText: [{
1459 type: "tool-call",
1460 id: "tool-123",
1461 name: "TestTool",
1462 params: {}
1463 }]
1464 }),
1465 Effect.provide(layer)
1466 )
1467
1468 const toolResult = response.toolResults[0]
1469 assertTrue(!toolResult.isFailure, "expected a successful tool result")
1470 deepStrictEqual(toolResult.result.timestamp, DateTime.makeUnsafe(new Date(1000)))
1471 }))
1472 })
1473
1474 describe("Strict", () => {
1475 it("defaults to undefined", () => {
1476 const tool = Tool.make("TestTool")
1477 strictEqual(Tool.getStrictMode(tool), undefined)
1478 })
1479
1480 it("can be set to true", () => {
1481 const tool = Tool.make("TestTool").annotate(Tool.Strict, true)
1482 strictEqual(Tool.getStrictMode(tool), true)
1483 })
1484
1485 it("can be set to false", () => {
1486 const tool = Tool.make("TestTool").annotate(Tool.Strict, false)
1487 strictEqual(Tool.getStrictMode(tool), false)
1488 })
1489
1490 it("works with dynamic tools", () => {
1491 const tool = Tool.dynamic("TestTool", {
1492 parameters: { type: "object", properties: {} }
1493 }).annotate(Tool.Strict, false)
1494 strictEqual(Tool.getStrictMode(tool), false)
1495 })
1496
1497 it("works with provider-defined tools", () => {
1498 const tool = Tool.providerDefined({
1499 id: "test.provider_tool",
1500 customName: "ProviderTool",
1501 providerName: "provider_tool"
1502 })().annotate(Tool.Strict, false)
1503 strictEqual(Tool.getStrictMode(tool), false)
1504 })
1505 })
1506})