| 1 | import { assert, describe, it } from "@effect/vitest" |
| 2 | import { Effect } from "effect" |
| 3 | import { Prompt, Tokenizer } from "effect/ai" |
| 4 | |
| 5 | describe("Tokenizer", () => { |
| 6 | it.effect("truncate accounts for costs between messages", () => |
| 7 | Effect.gen(function*() { |
| 8 | const tokenizer = Tokenizer.make({ |
| 9 | tokenize: (prompt) => |
| 10 | Effect.succeed( |
| 11 | Array.from({ length: Math.max(0, prompt.content.length * 2 - 1) }, (_, index) => index) |
| 12 | ) |
| 13 | }) |
| 14 | const input = Prompt.make([ |
| 15 | { role: "user", content: "a" }, |
| 16 | { role: "assistant", content: "b" } |
| 17 | ]) |
| 18 | |
| 19 | const output = yield* tokenizer.truncate(input, 2) |
| 20 | |
| 21 | assert.deepStrictEqual(output.content, [input.content[1]]) |
| 22 | assert.isAtMost((yield* tokenizer.tokenize(output)).length, 2) |
| 23 | })) |
| 24 | }) |