authorgravatar for 275381468+derektrimm@users.noreply.github.comDerek Trimm <275381468+derektrimm@users.noreply.github.com> 2026-10-07 02:57:58-05:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-10-07 00:57:58-07:00
logb77108bc41506ad7ba28839d9a8e04ba79fe340b
tree2d08bd1b2ceccb9f5d7b739aa00ff4280f92f5c0
parent453e596da7e98d973e3853a5c2728681dedf363b
signature Signed by PGP key B5690EEEBB952194

fix(server): stop storing tool image bytes no client reads (#16652)

6 files changed, 488 insertions(+), 15 deletions(-)

apps/server/src/assets/AssetAccess.ts+1-5
...@@ -1,7 +1,6 @@...@@ -1,7 +1,6 @@
1import type { AssetResource } from "@t3tools/contracts";1import type { AssetResource } from "@t3tools/contracts";
2import {2import {
3 AssetAttachmentNotFoundError,3 AssetAttachmentNotFoundError,
4 PROVIDER_SEND_TURN_MAX_IMAGE_BYTES,
5 AssetGitHubMediaUrlValidationError,4 AssetGitHubMediaUrlValidationError,
6 AssetPreviewTypeValidationError,5 AssetPreviewTypeValidationError,
7 AssetProjectFaviconInspectionError,6 AssetProjectFaviconInspectionError,
...@@ -33,7 +32,7 @@ import {...@@ -33,7 +32,7 @@ import {
33} from "@t3tools/shared/imageDimensions";32} from "@t3tools/shared/imageDimensions";
34import { githubMediaFetchUrl, githubMediaFileName } from "@t3tools/shared/githubMedia";33import { githubMediaFetchUrl, githubMediaFileName } from "@t3tools/shared/githubMedia";
35import { PROJECT_FAVICON_FALLBACK_MARKER } from "@t3tools/shared/projectFavicon";34import { PROJECT_FAVICON_FALLBACK_MARKER } from "@t3tools/shared/projectFavicon";
36import { toolOutputImages } from "@t3tools/shared/toolOutput";35import { MAX_TOOL_OUTPUT_IMAGE_BASE64_LENGTH, toolOutputImages } from "@t3tools/shared/toolOutput";
37import * as Clock from "effect/Clock";36import * as Clock from "effect/Clock";
38import * as Crypto from "effect/Crypto";37import * as Crypto from "effect/Crypto";
39import * as Effect from "effect/Effect";38import * as Effect from "effect/Effect";
...@@ -216,9 +215,6 @@ const optionOnNotFound = <A, R>(...@@ -216,9 +215,6 @@ const optionOnNotFound = <A, R>(
216 }),215 }),
217 );216 );
218217
219// The largest image a provider turn accepts, as base64 (4 characters per 3 bytes).
220const MAX_TOOL_OUTPUT_IMAGE_BASE64_LENGTH = Math.ceil(PROVIDER_SEND_TURN_MAX_IMAGE_BYTES / 3) * 4;
221
222/**218/**
223 * Decodes one image a tool returned inline; null when the stored item has no219 * Decodes one image a tool returned inline; null when the stored item has no
224 * such image, or it is larger than a provider turn accepts.220 * such image, or it is larger than a provider turn accepts.
apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts+92
...@@ -27,6 +27,7 @@ import * as Layer from "effect/Layer";...@@ -27,6 +27,7 @@ import * as Layer from "effect/Layer";
27import * as Stream from "effect/Stream";27import * as Stream from "effect/Stream";
28import * as TestClock from "effect/testing/TestClock";28import * as TestClock from "effect/testing/TestClock";
2929
30import { toolOutputImages } from "@t3tools/shared/toolOutput";
30import * as SqlitePersistence from "../persistence/Sqlite.ts";31import * as SqlitePersistence from "../persistence/Sqlite.ts";
31import * as EventSink from "./EventSink.ts";32import * as EventSink from "./EventSink.ts";
32import * as EventStore from "./EventStore.ts";33import * as EventStore from "./EventStore.ts";
...@@ -1169,6 +1170,97 @@ layer("ProviderEventIngestorV2", (it) => {...@@ -1169,6 +1170,97 @@ layer("ProviderEventIngestorV2", (it) => {
1169 }),1170 }),
1170 );1171 );
11711172
1173 it.effect("stores tool image bytes only where a tool-output-image asset serves them", () =>
1174 Effect.gen(function* () {
1175 const now = yield* DateTime.now;
1176 const eventSink = yield* EventSink.EventSinkV2;
1177 const eventStore = yield* EventStore.EventStoreV2;
1178 const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
1179 const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;
1180 const idAllocator = yield* IdAllocator.IdAllocatorV2;
1181 const threadEvent = yield* threadCreatedEvent(now);
1182 const providerSessionId = yield* idAllocator.allocate.providerSession({
1183 providerInstanceId: modelSelection.instanceId,
1184 threadId: threadEvent.threadId,
1185 });
1186 const readBase64 = Buffer.alloc(30_000, 7).toString("base64");
1187 const screenshotBase64 = Buffer.alloc(20_000, 9).toString("base64");
1188 const toolItem = (
1189 id: string,
1190 ordinal: number,
1191 toolName: string,
1192 output: unknown,
1193 ): OrchestrationV2TurnItem => ({
1194 id: TurnItemId.make(id),
1195 threadId: threadEvent.threadId,
1196 runId: null,
1197 nodeId: null,
1198 providerThreadId: null,
1199 providerTurnId: null,
1200 nativeItemRef: null,
1201 parentItemId: null,
1202 ordinal,
1203 status: "completed",
1204 title: toolName,
1205 startedAt: now,
1206 completedAt: now,
1207 updatedAt: now,
1208 type: "dynamic_tool",
1209 toolName,
1210 input: {},
1211 output,
1212 });
1213 const read = toolItem("turn-item:read-image", 1, "Read", {
1214 type: "image",
1215 file: { base64: readBase64, type: "image/png", originalSize: 30_000 },
1216 });
1217 const screenshot = toolItem("turn-item:screenshot", 2, "mcp__t3-code__device_screenshot", {
1218 content: [
1219 {
1220 type: "image",
1221 source: { type: "base64", media_type: "image/png", data: screenshotBase64 },
1222 },
1223 ],
1224 });
1225
1226 yield* eventSink.write({ events: [threadEvent] });
1227 for (const turnItem of [read, screenshot]) {
1228 yield* ingestor.ingestNormalized({
1229 providerSessionId,
1230 providerInstanceId: modelSelection.instanceId,
1231 threadId: threadEvent.threadId,
1232 event: { type: "turn_item.updated", driver: CODEX_DRIVER, turnItem },
1233 });
1234 }
1235
1236 const storedEvents = yield* eventStore
1237 .read({ threadId: threadEvent.threadId, eventType: "turn-item.updated" })
1238 .pipe(Stream.runCollect);
1239 const storedJson = JSON.stringify(Array.from(storedEvents, (stored) => stored.event));
1240 const projectedRead = yield* projectionStore.getTurnItem({
1241 threadId: threadEvent.threadId,
1242 itemId: read.id,
1243 });
1244 const projectedScreenshot = yield* projectionStore.getTurnItem({
1245 threadId: threadEvent.threadId,
1246 itemId: screenshot.id,
1247 });
1248
1249 assert.equal(storedJson.includes(readBase64), false);
1250 assert.equal(storedJson.includes(screenshotBase64), true);
1251 assert.deepEqual(projectedRead?.type === "dynamic_tool" ? projectedRead.output : null, {
1252 type: "image",
1253 file: { type: "image/png", originalSize: 30_000, sizeBytes: 30_000 },
1254 });
1255 assert.deepEqual(
1256 toolOutputImages(
1257 projectedScreenshot?.type === "dynamic_tool" ? projectedScreenshot.output : null,
1258 ),
1259 [{ mimeType: "image/png", data: screenshotBase64 }],
1260 );
1261 }),
1262 );
1263
1172 it.effect("routes provider-owned child artifacts to their child app thread", () =>1264 it.effect("routes provider-owned child artifacts to their child app thread", () =>
1173 Effect.gen(function* () {1265 Effect.gen(function* () {
1174 const now = yield* DateTime.now;1266 const now = yield* DateTime.now;
apps/server/src/orchestration-v2/ProviderEventIngestor.ts+2-1
...@@ -33,6 +33,7 @@ import * as IdAllocator from "./IdAllocator.ts";...@@ -33,6 +33,7 @@ import * as IdAllocator from "./IdAllocator.ts";
33import { ProviderAdapterV2Event } from "./ProviderAdapter.ts";33import { ProviderAdapterV2Event } from "./ProviderAdapter.ts";
34import { makeProviderFailureTurnItem } from "./ProviderFailure.ts";34import { makeProviderFailureTurnItem } from "./ProviderFailure.ts";
35import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts";35import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts";
36import { stripUnservedToolOutputImageBytes } from "./toolOutputImageBytes.ts";
3637
37export class ProviderEventNormalizeError extends Schema.TaggedError<ProviderEventNormalizeError>()(38export class ProviderEventNormalizeError extends Schema.TaggedError<ProviderEventNormalizeError>()(
38 "ProviderEventNormalizeError",39 "ProviderEventNormalizeError",
...@@ -465,7 +466,7 @@ export const layer: Layer.Layer<...@@ -465,7 +466,7 @@ export const layer: Layer.Layer<
465 yield* makeDomainEvent(input, {466 yield* makeDomainEvent(input, {
466 type: "turn-item.updated",467 type: "turn-item.updated",
467 threadId: input.event.turnItem.threadId,468 threadId: input.event.turnItem.threadId,
468 payload: input.event.turnItem,469 payload: stripUnservedToolOutputImageBytes(input.event.turnItem),
469 runId: input.event.turnItem.runId,470 runId: input.event.turnItem.runId,
470 nodeId: input.event.turnItem.nodeId,471 nodeId: input.event.turnItem.nodeId,
471 }),472 }),
apps/server/src/orchestration-v2/toolOutputImageBytes.test.ts created+234
...@@ -0,0 +1,234 @@
1import { NodeId, ThreadId, TurnItemId, type OrchestrationV2TurnItem } from "@t3tools/contracts";
2import {
3 MAX_TOOL_OUTPUT_IMAGE_BASE64_LENGTH,
4 MAX_TOOL_OUTPUT_IMAGES,
5 toolOutputImages,
6} from "@t3tools/shared/toolOutput";
7import * as DateTime from "effect/DateTime";
8import { describe, expect, it } from "vite-plus/test";
9
10import { stripUnservedToolOutputImageBytes } from "./toolOutputImageBytes.ts";
11
12const PNG_BYTES = 3_000;
13const PNG_BASE64 = Buffer.alloc(PNG_BYTES, 7).toString("base64");
14// 3,001 bytes encode with `=` padding, which must not count as data.
15const PADDED_BASE64 = Buffer.alloc(PNG_BYTES + 1, 9).toString("base64");
16
17const now = DateTime.makeUnsafe("2026-10-06T00:00:00.000Z");
18
19const itemBase = {
20 id: TurnItemId.make("turn-item:tool"),
21 threadId: ThreadId.make("thread:tool"),
22 runId: null,
23 nodeId: NodeId.make("node:tool"),
24 providerThreadId: null,
25 providerTurnId: null,
26 nativeItemRef: null,
27 parentItemId: null,
28 ordinal: 1,
29 status: "completed",
30 title: null,
31 startedAt: now,
32 completedAt: now,
33 updatedAt: now,
34} as const;
35
36function toolItem(output: unknown, toolName = "Read"): OrchestrationV2TurnItem {
37 return {
38 ...itemBase,
39 type: "dynamic_tool",
40 toolName,
41 input: { file_path: "/tmp/shot.png" },
42 output,
43 };
44}
45
46function outputOf(item: OrchestrationV2TurnItem): unknown {
47 return item.type === "dynamic_tool" ? item.output : undefined;
48}
49
50describe("stripUnservedToolOutputImageBytes", () => {
51 it("replaces the base64 of Claude's structured Read result with its measured size", () => {
52 const item = toolItem({
53 type: "image",
54 file: {
55 base64: PNG_BASE64,
56 type: "image/png",
57 originalSize: PNG_BYTES,
58 dimensions: { originalWidth: 180, originalHeight: 180 },
59 },
60 });
61
62 expect(outputOf(stripUnservedToolOutputImageBytes(item))).toEqual({
63 type: "image",
64 file: {
65 type: "image/png",
66 originalSize: PNG_BYTES,
67 dimensions: { originalWidth: 180, originalHeight: 180 },
68 sizeBytes: PNG_BYTES,
69 },
70 });
71 });
72
73 it("strips ACP image shapes: Grok's native ImageContent and an MCP result behind an error", () => {
74 const item = toolItem({
75 rawOutput: { type: "ReadFile", ImageContent: { data: PNG_BASE64, mime_type: "image/png" } },
76 failed: {
77 error: "partial result",
78 result: [{ type: "image", data: PADDED_BASE64, mimeType: "image/png" }],
79 },
80 });
81
82 expect(outputOf(stripUnservedToolOutputImageBytes(item))).toEqual({
83 rawOutput: {
84 type: "ReadFile",
85 ImageContent: { mime_type: "image/png", sizeBytes: PNG_BYTES },
86 },
87 failed: {
88 error: "partial result",
89 result: [{ type: "image", mimeType: "image/png", sizeBytes: PNG_BYTES + 1 }],
90 },
91 });
92 });
93
94 it("strips a Cursor image part that names no mime type", () => {
95 const item = toolItem({
96 content: [{ text: { text: "captured" } }, { image: { data: PNG_BASE64 } }],
97 });
98
99 expect(outputOf(stripUnservedToolOutputImageBytes(item))).toEqual({
100 content: [{ text: { text: "captured" } }, { image: { sizeBytes: PNG_BYTES } }],
101 });
102 });
103
104 it("keeps the screenshots a tool-output-image asset serves", () => {
105 const claudeImage = {
106 type: "image",
107 source: { type: "base64", media_type: "image/png", data: PNG_BASE64 },
108 };
109 const mcpImage = { type: "image", data: PADDED_BASE64, mimeType: "image/jpeg" };
110 const item = toolItem(
111 { content: [{ type: "text", text: "captured" }, claudeImage, mcpImage] },
112 "mcp__t3-code__device_screenshot",
113 );
114
115 const stripped = stripUnservedToolOutputImageBytes(item);
116
117 expect(stripped).toBe(item);
118 expect(toolOutputImages(outputOf(stripped))).toEqual([
119 { mimeType: "image/png", data: PNG_BASE64 },
120 { mimeType: "image/jpeg", data: PADDED_BASE64 },
121 ]);
122 });
123
124 it("strips only the images past the last one an asset can serve", () => {
125 const images = Array.from({ length: MAX_TOOL_OUTPUT_IMAGES + 2 }, () => ({
126 type: "image",
127 data: PNG_BASE64,
128 mimeType: "image/png",
129 }));
130
131 const output = outputOf(stripUnservedToolOutputImageBytes(toolItem(images)));
132
133 expect(Array.isArray(output) ? output.slice(0, MAX_TOOL_OUTPUT_IMAGES) : null).toEqual(
134 images.slice(0, MAX_TOOL_OUTPUT_IMAGES),
135 );
136 expect(Array.isArray(output) ? output.slice(MAX_TOOL_OUTPUT_IMAGES) : null).toEqual([
137 { type: "image", mimeType: "image/png", sizeBytes: PNG_BYTES },
138 { type: "image", mimeType: "image/png", sizeBytes: PNG_BYTES },
139 ]);
140 });
141
142 it("strips a served-position image larger than an asset may serve", () => {
143 const oversized = "A".repeat(MAX_TOOL_OUTPUT_IMAGE_BASE64_LENGTH + 4);
144 const servable = { type: "image", data: PNG_BASE64, mimeType: "image/png" };
145 const item = toolItem({
146 content: [
147 { type: "image", source: { type: "base64", media_type: "image/png", data: oversized } },
148 servable,
149 ],
150 });
151
152 const output = outputOf(stripUnservedToolOutputImageBytes(item)) as {
153 content: ReadonlyArray<unknown>;
154 };
155
156 expect(output.content).toEqual([
157 {
158 type: "image",
159 source: {
160 type: "base64",
161 media_type: "image/png",
162 sizeBytes: ((MAX_TOOL_OUTPUT_IMAGE_BASE64_LENGTH + 4) / 4) * 3,
163 },
164 },
165 servable,
166 ]);
167 expect(output.content[1]).toBe(servable);
168 });
169
170 it("keeps image fields whose value is not a base64 body", () => {
171 const item = toolItem({
172 nested: {
173 byId: { type: "image", data: "asset-id-42" },
174 svg: {
175 mimeType: "image/svg+xml",
176 data: `<svg xmlns='http://www.w3.org/2000/svg'>${"<circle r='4'/>".repeat(8)}</svg>`,
177 },
178 dataUrl: { type: "image", data: `data:image/png;base64,${PNG_BASE64}` },
179 remote: { type: "image", source: { type: "url", url: "https://example.test/shot.png" } },
180 tiny: { type: "image", data: "AA==", mimeType: "image/png" },
181 notImage: { mimeType: "application/pdf", data: PNG_BASE64 },
182 },
183 });
184
185 expect(stripUnservedToolOutputImageBytes(item)).toBe(item);
186 });
187
188 it("measures the size instead of trusting one the provider supplied", () => {
189 const item = toolItem({
190 type: "image",
191 file: { base64: PNG_BASE64, type: "image/png", sizeBytes: 0 },
192 });
193
194 expect(outputOf(stripUnservedToolOutputImageBytes(item))).toEqual({
195 type: "image",
196 file: { type: "image/png", sizeBytes: PNG_BYTES },
197 });
198 });
199
200 it("returns items without image bytes by reference", () => {
201 const nested = { files: [{ path: "src/app.ts" }] };
202 const item = toolItem({ stdout: "data: 42", data: PNG_BASE64, nested });
203 const command: OrchestrationV2TurnItem = {
204 ...itemBase,
205 type: "command_execution",
206 input: "cat shot.png | base64",
207 output: PNG_BASE64,
208 };
209
210 const stripped = stripUnservedToolOutputImageBytes(item);
211
212 expect(stripped).toBe(item);
213 expect(outputOf(stripped)).toBe(outputOf(item));
214 expect(stripUnservedToolOutputImageBytes(command)).toBe(command);
215 expect(stripUnservedToolOutputImageBytes(toolItem("plain text output"))).toEqual(
216 toolItem("plain text output"),
217 );
218 });
219
220 it("does not mutate the provider's item and shares untouched siblings", () => {
221 const untouched = { type: "text", text: "captured" };
222 const output = {
223 error: "partial result",
224 result: [untouched, { type: "image", data: PNG_BASE64, mimeType: "image/png" }],
225 };
226 const item = toolItem(output);
227 const snapshot = structuredClone(output);
228
229 const stripped = outputOf(stripUnservedToolOutputImageBytes(item)) as typeof output;
230
231 expect(outputOf(item)).toEqual(snapshot);
232 expect(stripped.result[0]).toBe(untouched);
233 });
234});
apps/server/src/orchestration-v2/toolOutputImageBytes.ts created+135
...@@ -0,0 +1,135 @@
1import type { OrchestrationV2TurnItem } from "@t3tools/contracts";
2import {
3 MAX_TOOL_OUTPUT_IMAGE_BASE64_LENGTH,
4 readToolOutputImage,
5 toolOutputImageBlocks,
6} from "@t3tools/shared/toolOutput";
7
8const IMAGE_MIME_KEYS = ["mimeType", "mime_type", "media_type", "type"] as const;
9const IMAGE_BODY_KEYS = new Set(["data", "blob", "base64"]);
10const MAX_DEPTH = 32;
11// Shorter strings are identifiers or tiny fixtures, not pixels worth removing.
12const MIN_BODY_LENGTH = 64;
13const BASE64_BODY = /^[A-Za-z0-9+/_-]+={0,2}$/;
14
15type UnknownRecord = Record<string, unknown>;
16
17function hasImageMime(record: UnknownRecord): boolean {
18 return IMAGE_MIME_KEYS.some((key) => {
19 const mime = record[key];
20 return typeof mime === "string" && mime.toLowerCase().startsWith("image/");
21 });
22}
23
24/**
25 * The record that carries an image's bytes: an MCP or ACP `type: "image"`
26 * block, Cursor's `image` part (its mime type is optional), or a record with
27 * an `image/*` mime field such as Claude's `source`, the `file` of Claude's
28 * structured Read result, and Grok's `ImageContent`.
29 */
30function isImageBody(record: UnknownRecord, key: string | undefined): boolean {
31 if (record.type === "image" || key === "image") return true;
32 if (!hasImageMime(record)) return false;
33 return (
34 record.type === undefined ||
35 record.type === "base64" ||
36 (typeof record.type === "string" && record.type.toLowerCase().startsWith("image/"))
37 );
38}
39
40function isBase64Body(value: unknown): value is string {
41 return typeof value === "string" && value.length >= MIN_BODY_LENGTH && BASE64_BODY.test(value);
42}
43
44function base64DecodedLength(encoded: string): number {
45 let length = encoded.length;
46 while (length > 0 && encoded.charCodeAt(length - 1) === 0x3d /* = */) {
47 length -= 1;
48 }
49 return Math.floor((length * 3) / 4);
50}
51
52interface StripContext {
53 readonly output: unknown;
54 served: ReadonlyArray<unknown> | undefined;
55}
56
57/** A block the asset route serves: in a served position and within its size limit. */
58function isServed(context: StripContext, record: UnknownRecord): boolean {
59 context.served ??= toolOutputImageBlocks(context.output);
60 if (!context.served.includes(record)) return false;
61 const data = readToolOutputImage(record)?.data;
62 return data === undefined || data.length <= MAX_TOOL_OUTPUT_IMAGE_BASE64_LENGTH;
63}
64
65function strip(
66 value: unknown,
67 key: string | undefined,
68 context: StripContext,
69 depth: number,
70): unknown {
71 if (depth > MAX_DEPTH || typeof value !== "object" || value === null) {
72 return value;
73 }
74 if (Array.isArray(value)) {
75 let next: Array<unknown> | undefined;
76 for (let index = 0; index < value.length; index += 1) {
77 const entry = value[index];
78 const stripped = strip(entry, undefined, context, depth + 1);
79 if (stripped !== entry) {
80 next ??= value.slice();
81 next[index] = stripped;
82 }
83 }
84 return next ?? value;
85 }
86 const record = value as UnknownRecord;
87 // Every served block is a `type: "image"` block; keep it whole.
88 if (record.type === "image" && isServed(context, record)) return record;
89 const imageBody = isImageBody(record, key);
90 let next: UnknownRecord | undefined;
91 let removedBody: string | undefined;
92 for (const entryKey in record) {
93 if (!Object.hasOwn(record, entryKey)) continue;
94 const entry = record[entryKey];
95 if (imageBody && IMAGE_BODY_KEYS.has(entryKey) && isBase64Body(entry)) {
96 next ??= { ...record };
97 delete next[entryKey];
98 removedBody = entry;
99 continue;
100 }
101 const stripped = strip(entry, entryKey, context, depth + 1);
102 if (stripped !== entry) {
103 next ??= { ...record };
104 next[entryKey] = stripped;
105 }
106 }
107 if (next !== undefined && removedBody !== undefined) {
108 // Measured from the removed body, never taken from a provider's size field.
109 next.sizeBytes = base64DecodedLength(removedBody);
110 }
111 return next ?? record;
112}
113
114/**
115 * Replaces the base64 image bodies in a dynamic tool's output with their
116 * decoded byte size, except the blocks a `tool-output-image` asset serves.
117 *
118 * Tools that read or capture an image return the whole file as base64, and a
119 * turn item is stored twice: in the event log and in the projection. Only the
120 * blocks `toolOutputImageBlocks` finds, within the asset size limit, are ever
121 * read back; previews of a read file load from `viewedImagePath`, and context
122 * handoff skips tool items.
123 * Strings that are not a base64 body are kept, so identifiers and inline SVG
124 * under the same keys survive. Containers are copied only when a descendant
125 * changes; an item without such bytes is returned by reference.
126 */
127export function stripUnservedToolOutputImageBytes(
128 item: OrchestrationV2TurnItem,
129): OrchestrationV2TurnItem {
130 if (item.type !== "dynamic_tool" || typeof item.output !== "object" || item.output === null) {
131 return item;
132 }
133 const output = strip(item.output, undefined, { output: item.output, served: undefined }, 0);
134 return output === item.output ? item : { ...item, output };
135}
packages/shared/src/toolOutput.ts+24-9
...@@ -1,4 +1,7 @@...@@ -1,4 +1,7 @@
1import { isProviderSendTurnSupportedImageMimeType } from "@t3tools/contracts";1import {
2 isProviderSendTurnSupportedImageMimeType,
3 PROVIDER_SEND_TURN_MAX_IMAGE_BYTES,
4} from "@t3tools/contracts";
2import * as Predicate from "effect/Predicate";5import * as Predicate from "effect/Predicate";
36
4import {7import {
...@@ -223,19 +226,31 @@ function outputBlocks(value: unknown): ReadonlyArray<unknown> {...@@ -223,19 +226,31 @@ function outputBlocks(value: unknown): ReadonlyArray<unknown> {
223/** A tool returns one screenshot or a few frames; more would only flood the timeline. */226/** A tool returns one screenshot or a few frames; more would only flood the timeline. */
224export const MAX_TOOL_OUTPUT_IMAGES = 8;227export const MAX_TOOL_OUTPUT_IMAGES = 8;
225228
229/** The largest image a `tool-output-image` asset serves: a provider turn's limit, as base64. */
230export const MAX_TOOL_OUTPUT_IMAGE_BASE64_LENGTH =
231 Math.ceil(PROVIDER_SEND_TURN_MAX_IMAGE_BYTES / 3) * 4;
232
233/**
234 * The blocks `toolOutputImages` reads, by reference. These hold the only image
235 * bytes in a tool output that a `tool-output-image` asset can serve, up to
236 * `MAX_TOOL_OUTPUT_IMAGE_BASE64_LENGTH`.
237 */
238export function toolOutputImageBlocks(value: unknown): ReadonlyArray<unknown> {
239 const blocks: unknown[] = [];
240 for (const block of outputBlocks(value)) {
241 if (readToolOutputImage(block) === null) continue;
242 blocks.push(block);
243 if (blocks.length === MAX_TOOL_OUTPUT_IMAGES) break;
244 }
245 return blocks;
246}
247
226/**248/**
227 * The first images in a tool output, in order. The order is the249 * The first images in a tool output, in order. The order is the
228 * `tool-output-image` asset index, so servers and clients agree on it.250 * `tool-output-image` asset index, so servers and clients agree on it.
229 */251 */
230export function toolOutputImages(value: unknown): ReadonlyArray<ToolOutputImage> {252export function toolOutputImages(value: unknown): ReadonlyArray<ToolOutputImage> {
231 const images: ToolOutputImage[] = [];253 return toolOutputImageBlocks(value).flatMap((block) => readToolOutputImage(block) ?? []);
232 for (const block of outputBlocks(value)) {
233 const image = readToolOutputImage(block);
234 if (image === null) continue;
235 images.push(image);
236 if (images.length === MAX_TOOL_OUTPUT_IMAGES) break;
237 }
238 return images;
239}254}
240255
241/**256/**