authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-02-12 15:19:46-08:00
committergravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-02-12 15:22:03-08:00
log3cf58f80d10bb7208b4e9cbefbf26d69ca315608
treed565ff2b5bb4c1b869ff007afee02cedb82aa5fb
parent44f3b3cb7c82c6a2cfd33dad92bb76260f9e32ff

Add image paste drag drop dialog

11 files changed, 725 insertions(+), 42 deletions(-)

apps/server/src/codexAppServerManager.test.ts+115-1
......@@ -1,11 +1,42 @@
1import { describe, expect, it } from "vitest";
1import { describe, expect, it, vi } from "vitest";
22
33import {
4 CodexAppServerManager,
45 classifyCodexStderrLine,
56 isRecoverableThreadResumeError,
67 normalizeCodexModelSlug,
78} from "./codexAppServerManager";
89
10function createSendTurnHarness() {
11 const manager = new CodexAppServerManager();
12 const context = {
13 session: {
14 sessionId: "sess_1",
15 provider: "codex",
16 status: "ready",
17 threadId: "thread_1",
18 createdAt: "2026-02-10T00:00:00.000Z",
19 updatedAt: "2026-02-10T00:00:00.000Z",
20 },
21 };
22
23 const requireSession = vi
24 .spyOn(manager as unknown as { requireSession: (sessionId: string) => unknown }, "requireSession")
25 .mockReturnValue(context);
26 const sendRequest = vi
27 .spyOn(manager as unknown as { sendRequest: (...args: unknown[]) => Promise<unknown> }, "sendRequest")
28 .mockResolvedValue({
29 turn: {
30 id: "turn_1",
31 },
32 });
33 const updateSession = vi
34 .spyOn(manager as unknown as { updateSession: (...args: unknown[]) => void }, "updateSession")
35 .mockImplementation(() => {});
36
37 return { manager, context, requireSession, sendRequest, updateSession };
38}
39
940describe("classifyCodexStderrLine", () => {
1041 it("ignores empty lines", () => {
1142 expect(classifyCodexStderrLine(" ")).toBeNull();
......@@ -75,3 +106,86 @@ describe("isRecoverableThreadResumeError", () => {
75106 ).toBe(false);
76107 });
77108});
109
110describe("sendTurn", () => {
111 it("sends text and image user input items to turn/start", async () => {
112 const { manager, context, requireSession, sendRequest, updateSession } = createSendTurnHarness();
113
114 const result = await manager.sendTurn({
115 sessionId: "sess_1",
116 input: "Inspect this image",
117 images: [
118 {
119 name: "error.png",
120 mimeType: "image/png",
121 sizeBytes: 1_024,
122 dataUrl: "data:image/png;base64,AAAA",
123 },
124 ],
125 model: "gpt-5.3",
126 effort: "high",
127 });
128
129 expect(result).toEqual({
130 threadId: "thread_1",
131 turnId: "turn_1",
132 });
133 expect(requireSession).toHaveBeenCalledWith("sess_1");
134 expect(sendRequest).toHaveBeenCalledWith(context, "turn/start", {
135 threadId: "thread_1",
136 input: [
137 {
138 type: "text",
139 text: "Inspect this image",
140 text_elements: [],
141 },
142 {
143 type: "image",
144 url: "data:image/png;base64,AAAA",
145 },
146 ],
147 model: "gpt-5.3-codex",
148 effort: "high",
149 });
150 expect(updateSession).toHaveBeenCalledWith(context, {
151 status: "running",
152 activeTurnId: "turn_1",
153 });
154 });
155
156 it("supports image-only turns", async () => {
157 const { manager, context, sendRequest } = createSendTurnHarness();
158
159 await manager.sendTurn({
160 sessionId: "sess_1",
161 images: [
162 {
163 name: "diagram.png",
164 mimeType: "image/png",
165 sizeBytes: 256,
166 dataUrl: "data:image/png;base64,BBBB",
167 },
168 ],
169 });
170
171 expect(sendRequest).toHaveBeenCalledWith(context, "turn/start", {
172 threadId: "thread_1",
173 input: [
174 {
175 type: "image",
176 url: "data:image/png;base64,BBBB",
177 },
178 ],
179 });
180 });
181
182 it("rejects empty turn input", async () => {
183 const { manager } = createSendTurnHarness();
184
185 await expect(
186 manager.sendTurn({
187 sessionId: "sess_1",
188 }),
189 ).rejects.toThrow("Turn input must include text or image attachments.");
190 });
191});
apps/server/src/codexAppServerManager.ts+26-8
......@@ -257,20 +257,38 @@ export class CodexAppServerManager extends EventEmitter<CodexAppServerManagerEve
257257 throw new Error("Session is missing a thread id.");
258258 }
259259
260 const turnInput: Array<
261 | { type: "text"; text: string; text_elements: [] }
262 | { type: "image"; url: string }
263 > = [];
264 if (input.input) {
265 turnInput.push({
266 type: "text",
267 text: input.input,
268 text_elements: [],
269 });
270 }
271 for (const image of input.images ?? []) {
272 turnInput.push({
273 type: "image",
274 url: image.dataUrl,
275 });
276 }
277 if (turnInput.length === 0) {
278 throw new Error("Turn input must include text or image attachments.");
279 }
280
260281 const turnStartParams: {
261282 threadId: string;
262 input: Array<{ type: "text"; text: string; text_elements: [] }>;
283 input: Array<
284 | { type: "text"; text: string; text_elements: [] }
285 | { type: "image"; url: string }
286 >;
263287 model?: string;
264288 effort?: string;
265289 } = {
266290 threadId: context.session.threadId,
267 input: [
268 {
269 type: "text",
270 text: input.input,
271 text_elements: [],
272 },
273 ],
291 input: turnInput,
274292 };
275293 const normalizedModel = normalizeCodexModelSlug(input.model);
276294 if (normalizedModel) {
apps/web/src/components/ChatView.tsx+352-22
......@@ -1,9 +1,14 @@
11import {
22 PROVIDER_SEND_TURN_MAX_INPUT_CHARS,
3 PROVIDER_SEND_TURN_MAX_IMAGE_BYTES,
4 PROVIDER_SEND_TURN_MAX_IMAGES,
5 type ProviderSendTurnImageInput,
36 type ProviderApprovalDecision,
47 type ProviderEvent,
58} from "@t3tools/contracts";
69import {
10 type ClipboardEvent,
11 type DragEvent,
712 type FormEvent,
813 Fragment,
914 type KeyboardEvent,
......@@ -33,6 +38,7 @@ import {
3338 formatTimestamp,
3439} from "../session-logic";
3540import { useStore } from "../store";
41import type { ChatImageAttachment } from "../types";
3642import BranchToolbar from "./BranchToolbar";
3743import GitActionsControl from "./GitActionsControl";
3844import { isTerminalToggleShortcut } from "../terminal-shortcuts";
......@@ -57,6 +63,9 @@ function editorLabel(editor: (typeof EDITORS)[number]): string {
5763
5864const LAST_EDITOR_KEY = "t3code:last-editor";
5965const MAX_VISIBLE_WORK_LOG_ENTRIES = 6;
66const IMAGE_SIZE_LIMIT_LABEL = `${Math.round(PROVIDER_SEND_TURN_MAX_IMAGE_BYTES / (1024 * 1024))}MB`;
67const IMAGE_ONLY_BOOTSTRAP_PROMPT =
68 "[User attached one or more images without additional text. Respond using the conversation context and the attached image(s).]";
6069
6170function workToneClass(tone: "thinking" | "tool" | "info" | "error"): string {
6271 if (tone === "error") return "text-rose-300/50 dark:text-rose-300/50";
......@@ -80,6 +89,16 @@ interface EnsuredSessionInfo {
8089 continuityState: SessionContinuityState;
8190}
8291
92interface ComposerImageAttachment extends Omit<ChatImageAttachment, "previewUrl"> {
93 previewUrl: string;
94 file: File;
95}
96
97interface ExpandedImagePreview {
98 src: string;
99 name: string;
100}
101
83102function asRecord(value: unknown): Record<string, unknown> | undefined {
84103 if (!value || typeof value !== "object") return undefined;
85104 return value as Record<string, unknown>;
......@@ -131,10 +150,30 @@ function derivePendingApprovals(events: ProviderEvent[]): PendingApprovalCard[]
131150 return Array.from(pending.values());
132151}
133152
153function readFileAsDataUrl(file: File): Promise<string> {
154 return new Promise((resolve, reject) => {
155 const reader = new FileReader();
156 reader.addEventListener("load", () => {
157 if (typeof reader.result === "string") {
158 resolve(reader.result);
159 return;
160 }
161 reject(new Error("Could not read image data."));
162 });
163 reader.addEventListener("error", () => {
164 reject(reader.error ?? new Error("Failed to read image."));
165 });
166 reader.readAsDataURL(file);
167 });
168}
169
134170export default function ChatView() {
135171 const { state, dispatch } = useStore();
136172 const api = useNativeApi();
137173 const [prompt, setPrompt] = useState("");
174 const [composerImages, setComposerImages] = useState<ComposerImageAttachment[]>([]);
175 const [isDragOverComposer, setIsDragOverComposer] = useState(false);
176 const [expandedImage, setExpandedImage] = useState<ExpandedImagePreview | null>(null);
138177 const [isSending, setIsSending] = useState(false);
139178 const [isConnecting, setIsConnecting] = useState(false);
140179 const [isModelMenuOpen, setIsModelMenuOpen] = useState(false);
......@@ -152,6 +191,8 @@ export default function ChatView() {
152191 const [terminalFocusRequestId, setTerminalFocusRequestId] = useState(0);
153192 const messagesEndRef = useRef<HTMLDivElement>(null);
154193 const textareaRef = useRef<HTMLTextAreaElement>(null);
194 const composerImagesRef = useRef<ComposerImageAttachment[]>([]);
195 const dragDepthRef = useRef(0);
155196 const modelMenuRef = useRef<HTMLDivElement>(null);
156197 const editorMenuRef = useRef<HTMLDivElement>(null);
157198 const terminalOpenByThreadRef = useRef<Record<string, boolean>>({});
......@@ -264,7 +305,15 @@ export default function ChatView() {
264305 (activeThread.messages.length > 0 ||
265306 (activeThread.session !== null && activeThread.session.status !== "closed")),
266307 );
267 const terminalShortcutHint = navigator.platform.includes("Mac") ? "\u2318J" : "Ctrl+J";
308 const terminalShortcutHint = navigator.platform.includes("Mac")
309 ? "\u2318J"
310 : "Ctrl+J";
311 const revokePreviewUrls = useCallback((images: Array<{ previewUrl?: string }>) => {
312 for (const image of images) {
313 if (!image.previewUrl) continue;
314 URL.revokeObjectURL(image.previewUrl);
315 }
316 }, []);
268317 const focusComposer = useCallback(() => {
269318 const textarea = textareaRef.current;
270319 if (!textarea) return;
......@@ -319,6 +368,44 @@ export default function ChatView() {
319368 setExpandedWorkGroups({});
320369 }, [activeThread?.id]);
321370
371 useEffect(() => {
372 composerImagesRef.current = composerImages;
373 }, [composerImages]);
374
375 useEffect(() => {
376 setComposerImages((existing) => {
377 revokePreviewUrls(existing);
378 return [];
379 });
380 dragDepthRef.current = 0;
381 setIsDragOverComposer(false);
382 setExpandedImage(null);
383 }, [activeThread?.id, revokePreviewUrls]);
384
385 useEffect(() => {
386 return () => {
387 revokePreviewUrls(composerImagesRef.current);
388 };
389 }, [revokePreviewUrls]);
390
391 useEffect(() => {
392 if (!expandedImage) {
393 return;
394 }
395
396 const onKeyDown = (event: globalThis.KeyboardEvent) => {
397 if (event.key !== "Escape") {
398 return;
399 }
400 setExpandedImage(null);
401 };
402
403 window.addEventListener("keydown", onKeyDown);
404 return () => {
405 window.removeEventListener("keydown", onKeyDown);
406 };
407 }, [expandedImage]);
408
322409 const activeWorktreePath = activeThread?.worktreePath;
323410
324411 useEffect(() => {
......@@ -431,6 +518,124 @@ export default function ChatView() {
431518 setIsEditorMenuOpen(false);
432519 };
433520
521 const setThreadError = (threadId: string | null, error: string | null) => {
522 if (!threadId) return;
523 dispatch({
524 type: "SET_ERROR",
525 threadId,
526 error,
527 });
528 };
529
530 const addComposerImages = (files: File[]) => {
531 if (!activeThreadId || files.length === 0) return;
532
533 const nextImages = [...composerImagesRef.current];
534 let error: string | null = null;
535 for (const file of files) {
536 if (!file.type.startsWith("image/")) {
537 error = `Unsupported file type for '${file.name}'. Please attach image files only.`;
538 continue;
539 }
540 if (file.size > PROVIDER_SEND_TURN_MAX_IMAGE_BYTES) {
541 error = `'${file.name}' exceeds the ${IMAGE_SIZE_LIMIT_LABEL} attachment limit.`;
542 continue;
543 }
544 if (nextImages.length >= PROVIDER_SEND_TURN_MAX_IMAGES) {
545 error = `You can attach up to ${PROVIDER_SEND_TURN_MAX_IMAGES} images per message.`;
546 break;
547 }
548
549 const previewUrl = URL.createObjectURL(file);
550 nextImages.push({
551 id: crypto.randomUUID(),
552 name: file.name || "image",
553 mimeType: file.type,
554 sizeBytes: file.size,
555 previewUrl,
556 file,
557 });
558 }
559
560 setComposerImages(nextImages);
561 setThreadError(activeThreadId, error);
562 };
563
564 const removeComposerImage = (imageId: string) => {
565 setComposerImages((existing) => {
566 const match = existing.find((image) => image.id === imageId);
567 if (match?.previewUrl) {
568 URL.revokeObjectURL(match.previewUrl);
569 }
570 return existing.filter((image) => image.id !== imageId);
571 });
572 };
573
574 const openImageDialog = (src: string | undefined, name: string) => {
575 if (!src) {
576 return;
577 }
578 setExpandedImage({ src, name });
579 };
580
581 const onComposerPaste = (event: ClipboardEvent<HTMLTextAreaElement>) => {
582 const files = Array.from(event.clipboardData.files);
583 if (files.length === 0) {
584 return;
585 }
586 const imageFiles = files.filter((file) => file.type.startsWith("image/"));
587 if (imageFiles.length === 0) {
588 return;
589 }
590 event.preventDefault();
591 addComposerImages(imageFiles);
592 };
593
594 const onComposerDragEnter = (event: DragEvent<HTMLDivElement>) => {
595 if (!event.dataTransfer.types.includes("Files")) {
596 return;
597 }
598 event.preventDefault();
599 dragDepthRef.current += 1;
600 setIsDragOverComposer(true);
601 };
602
603 const onComposerDragOver = (event: DragEvent<HTMLDivElement>) => {
604 if (!event.dataTransfer.types.includes("Files")) {
605 return;
606 }
607 event.preventDefault();
608 event.dataTransfer.dropEffect = "copy";
609 setIsDragOverComposer(true);
610 };
611
612 const onComposerDragLeave = (event: DragEvent<HTMLDivElement>) => {
613 if (!event.dataTransfer.types.includes("Files")) {
614 return;
615 }
616 event.preventDefault();
617 const nextTarget = event.relatedTarget;
618 if (nextTarget instanceof Node && event.currentTarget.contains(nextTarget)) {
619 return;
620 }
621 dragDepthRef.current = Math.max(0, dragDepthRef.current - 1);
622 if (dragDepthRef.current === 0) {
623 setIsDragOverComposer(false);
624 }
625 };
626
627 const onComposerDrop = (event: DragEvent<HTMLDivElement>) => {
628 if (!event.dataTransfer.types.includes("Files")) {
629 return;
630 }
631 event.preventDefault();
632 dragDepthRef.current = 0;
633 setIsDragOverComposer(false);
634 const files = Array.from(event.dataTransfer.files);
635 addComposerImages(files);
636 focusComposer();
637 };
638
434639 const ensureSession = async (cwdOverride?: string): Promise<EnsuredSessionInfo | null> => {
435640 if (!api || !activeThread || !activeProject) return null;
436641 if (activeThread.session && activeThread.session.status !== "closed") {
......@@ -492,8 +697,9 @@ export default function ChatView() {
492697 e.preventDefault();
493698 if (!api || !activeThread || isSending || isConnecting) return;
494699 const trimmed = prompt.trim();
495 if (!trimmed) return;
700 if (!trimmed && composerImages.length === 0) return;
496701 if (!activeProject) return;
702 const composerImagesSnapshot = [...composerImages];
497703
498704 // On first message: lock in branch + create worktree if needed.
499705 let sessionCwd: string | undefined;
......@@ -529,7 +735,12 @@ export default function ChatView() {
529735
530736 // Auto-title from first message
531737 if (activeThread.messages.length === 0) {
532 const title = trimmed.length > 50 ? `${trimmed.slice(0, 50)}...` : trimmed;
738 const titleSeed =
739 trimmed ||
740 (composerImagesSnapshot.length > 0
741 ? `Image: ${composerImagesSnapshot[0]?.name ?? "attachment"}`
742 : "New thread");
743 const title = titleSeed.length > 50 ? `${titleSeed.slice(0, 50)}...` : titleSeed;
533744 dispatch({
534745 type: "SET_THREAD_TITLE",
535746 threadId: activeThread.id,
......@@ -537,43 +748,61 @@ export default function ChatView() {
537748 });
538749 }
539750
540 dispatch({
541 type: "SET_ERROR",
542 threadId: activeThread.id,
543 error: null,
544 });
751 setThreadError(activeThread.id, null);
752 const messageImages: ChatImageAttachment[] = composerImagesSnapshot.map((image) => ({
753 id: image.id,
754 name: image.name,
755 mimeType: image.mimeType,
756 sizeBytes: image.sizeBytes,
757 previewUrl: image.previewUrl,
758 }));
545759 dispatch({
546760 type: "PUSH_USER_MESSAGE",
547761 threadId: activeThread.id,
548762 id: crypto.randomUUID(),
549763 text: trimmed,
764 ...(messageImages.length > 0 ? { images: messageImages } : {}),
550765 });
551766 const previousMessages = activeThread.messages;
552767 setPrompt("");
768 setComposerImages([]);
553769
554770 const sessionInfo = await ensureSession(sessionCwd);
555771 if (!sessionInfo) return;
556772
557773 setIsSending(true);
558774 try {
775 const turnImages = await Promise.all(
776 composerImagesSnapshot.map(async (image): Promise<ProviderSendTurnImageInput> => ({
777 name: image.name,
778 mimeType: image.mimeType,
779 sizeBytes: image.sizeBytes,
780 dataUrl: await readFileAsDataUrl(image.file),
781 })),
782 );
559783 const shouldBootstrap =
560784 previousMessages.length > 0 &&
561785 (sessionInfo.continuityState === "new" || sessionInfo.continuityState === "fallback_new");
786 const latestPromptForBootstrap = trimmed || IMAGE_ONLY_BOOTSTRAP_PROMPT;
562787 const input = shouldBootstrap
563 ? buildBootstrapInput(previousMessages, trimmed, PROVIDER_SEND_TURN_MAX_INPUT_CHARS).text
564 : trimmed;
788 ? buildBootstrapInput(
789 previousMessages,
790 latestPromptForBootstrap,
791 PROVIDER_SEND_TURN_MAX_INPUT_CHARS,
792 ).text
793 : (trimmed || undefined);
565794 await api.providers.sendTurn({
566795 sessionId: sessionInfo.sessionId,
567 input,
796 ...(input ? { input } : {}),
797 ...(turnImages.length > 0 ? { images: turnImages } : {}),
568798 model: selectedModel || undefined,
569799 effort: selectedEffort || undefined,
570800 });
571801 } catch (err) {
572 dispatch({
573 type: "SET_ERROR",
574 threadId: activeThread.id,
575 error: err instanceof Error ? err.message : "Failed to send message.",
576 });
802 setThreadError(
803 activeThread.id,
804 err instanceof Error ? err.message : "Failed to send message.",
805 );
577806 } finally {
578807 setIsSending(false);
579808 }
......@@ -876,13 +1105,39 @@ export default function ChatView() {
8761105 }
8771106
8781107 if (timelineEntry.message.role === "user") {
1108 const userImages = timelineEntry.message.images ?? [];
8791109 return (
8801110 <Fragment key={timelineEntry.id}>
8811111 <div className="flex justify-end">
8821112 <div className="max-w-[80%] rounded-2xl rounded-br-sm border border-border bg-secondary px-4 py-3">
883 <pre className="whitespace-pre-wrap wrap-break-word font-mono text-sm leading-relaxed text-foreground">
884 {timelineEntry.message.text}
885 </pre>
1113 {userImages.length > 0 && (
1114 <div className="mb-2 grid max-w-[420px] grid-cols-2 gap-2">
1115 {userImages.map((image) => (
1116 <div
1117 key={image.id}
1118 className="overflow-hidden rounded-lg border border-border/80 bg-background/70"
1119 >
1120 {image.previewUrl ? (
1121 <img
1122 src={image.previewUrl}
1123 alt={image.name}
1124 className="h-full max-h-[220px] w-full cursor-zoom-in object-cover"
1125 onClick={() => openImageDialog(image.previewUrl, image.name)}
1126 />
1127 ) : (
1128 <div className="flex min-h-[72px] items-center justify-center px-2 py-3 text-center text-[11px] text-muted-foreground/70">
1129 {image.name}
1130 </div>
1131 )}
1132 </div>
1133 ))}
1134 </div>
1135 )}
1136 {timelineEntry.message.text && (
1137 <pre className="whitespace-pre-wrap break-words font-mono text-sm leading-relaxed text-foreground">
1138 {timelineEntry.message.text}
1139 </pre>
1140 )}
8861141 <p className="mt-1.5 text-right text-[10px] text-muted-foreground/30">
8871142 {formatTimestamp(timelineEntry.message.createdAt)}
8881143 </p>
......@@ -945,9 +1200,48 @@ export default function ChatView() {
9451200 {/* Input bar */}
9461201 <div className="px-5 pb-1 pt-2">
9471202 <form onSubmit={onSend} className="mx-auto max-w-3xl">
948 <div className="group rounded-[20px] border border-border bg-card transition-colors duration-200 focus-within:border-ring">
1203 <div
1204 className={`group rounded-[20px] border bg-card transition-colors duration-200 focus-within:border-ring ${
1205 isDragOverComposer ? "border-primary/70 bg-accent/30" : "border-border"
1206 }`}
1207 onDragEnter={onComposerDragEnter}
1208 onDragOver={onComposerDragOver}
1209 onDragLeave={onComposerDragLeave}
1210 onDrop={onComposerDrop}
1211 >
9491212 {/* Textarea area */}
9501213 <div className="px-4 pt-4 pb-2">
1214 {composerImages.length > 0 && (
1215 <div className="mb-3 flex flex-wrap gap-2">
1216 {composerImages.map((image) => (
1217 <div
1218 key={image.id}
1219 className="relative h-16 w-16 overflow-hidden rounded-lg border border-border/80 bg-background"
1220 >
1221 {image.previewUrl ? (
1222 <img
1223 src={image.previewUrl}
1224 alt={image.name}
1225 className="h-full w-full cursor-zoom-in object-cover"
1226 onClick={() => openImageDialog(image.previewUrl, image.name)}
1227 />
1228 ) : (
1229 <div className="flex h-full w-full items-center justify-center px-1 text-center text-[10px] text-muted-foreground/70">
1230 {image.name}
1231 </div>
1232 )}
1233 <button
1234 type="button"
1235 className="absolute right-1 top-1 flex h-4 w-4 items-center justify-center rounded-full bg-background/80 text-[11px] leading-none text-foreground transition-colors duration-150 hover:bg-background"
1236 onClick={() => removeComposerImage(image.id)}
1237 aria-label={`Remove ${image.name}`}
1238 >
1239 ×
1240 </button>
1241 </div>
1242 ))}
1243 </div>
1244 )}
9511245 <textarea
9521246 ref={textareaRef}
9531247 className="w-full resize-none bg-transparent text-[14px] leading-relaxed text-foreground placeholder:text-muted-foreground/35 focus:outline-none"
......@@ -955,8 +1249,11 @@ export default function ChatView() {
9551249 value={prompt}
9561250 onChange={(e) => setPrompt(e.target.value)}
9571251 onKeyDown={onKeyDown}
1252 onPaste={onComposerPaste}
9581253 placeholder={
959 phase === "disconnected" ? "Ask for follow-up changes" : "Ask anything..."
1254 phase === "disconnected"
1255 ? "Ask for follow-up changes or attach images"
1256 : "Ask anything, or attach images..."
9601257 }
9611258 disabled={isSending || isConnecting}
9621259 />
......@@ -1146,7 +1443,7 @@ export default function ChatView() {
11461443 <button
11471444 type="submit"
11481445 className="flex h-8 w-8 items-center justify-center rounded-full bg-primary/90 text-primary-foreground transition-all duration-150 hover:bg-primary hover:scale-105 disabled:opacity-30 disabled:hover:scale-100"
1149 disabled={isSending || isConnecting || !prompt.trim()}
1446 disabled={isSending || isConnecting || (!prompt.trim() && composerImages.length === 0)}
11501447 aria-label={
11511448 isConnecting ? "Connecting" : isSending ? "Sending" : "Send message"
11521449 }
......@@ -1221,6 +1518,39 @@ export default function ChatView() {
12211518 }
12221519 />
12231520 )}
1521
1522 {expandedImage && (
1523 <div
1524 className="fixed inset-0 z-50 flex items-center justify-center bg-black/75 px-4 py-6 [-webkit-app-region:no-drag]"
1525 role="dialog"
1526 aria-modal="true"
1527 aria-label="Expanded image preview"
1528 onClick={() => setExpandedImage(null)}
1529 >
1530 <div
1531 className="relative isolate max-h-[92vh] max-w-[92vw]"
1532 onClick={(event) => event.stopPropagation()}
1533 >
1534 <button
1535 type="button"
1536 className="absolute right-2 top-2 z-30 flex h-8 w-8 cursor-pointer items-center justify-center rounded-full bg-black/60 text-xl leading-none text-white transition-colors duration-150 hover:bg-black/80 focus:outline-none focus-visible:ring-2 focus-visible:ring-white/80 [-webkit-app-region:no-drag]"
1537 onClick={() => setExpandedImage(null)}
1538 aria-label="Close image preview"
1539 >
1540 <span className="pointer-events-none">×</span>
1541 </button>
1542 <img
1543 src={expandedImage.src}
1544 alt={expandedImage.name}
1545 className="max-h-[86vh] max-w-[92vw] select-none rounded-lg border border-border/70 bg-background object-contain shadow-2xl"
1546 draggable={false}
1547 />
1548 <p className="mt-2 max-w-[92vw] truncate text-center text-xs text-muted-foreground/80">
1549 {expandedImage.name}
1550 </p>
1551 </div>
1552 </div>
1553 )}
12241554 </div>
12251555 );
12261556}
apps/web/src/historyBootstrap.test.ts+27
......@@ -91,4 +91,31 @@ describe("buildBootstrapInput", () => {
9191 expect(result.omittedCount).toBe(1);
9292 expect(result.truncated).toBe(true);
9393 });
94
95 it("captures user image attachment context in transcript blocks", () => {
96 const result = buildBootstrapInput(
97 [
98 {
99 id: "u-image",
100 role: "user",
101 text: "",
102 images: [
103 {
104 id: "img-1",
105 name: "screenshot.png",
106 mimeType: "image/png",
107 sizeBytes: 2_048,
108 },
109 ],
110 createdAt: "2026-02-09T00:00:00.000Z",
111 streaming: false,
112 },
113 ],
114 "What does this error mean?",
115 1_500,
116 );
117
118 expect(result.text).toContain("Attached image");
119 expect(result.text).toContain("screenshot.png");
120 });
94121});
apps/web/src/historyBootstrap.ts+28-3
......@@ -18,8 +18,33 @@ function messageRoleLabel(message: ChatMessage): "USER" | "ASSISTANT" {
1818 return message.role === "assistant" ? "ASSISTANT" : "USER";
1919}
2020
21function imageSummary(message: ChatMessage): string | null {
22 const count = message.images?.length ?? 0;
23 if (count === 0) {
24 return null;
25 }
26
27 const names = message.images?.slice(0, 3).map((image) => image.name) ?? [];
28 const namesSummary = names.join(", ");
29 const extraCount = count - names.length;
30 const extraSummary = extraCount > 0 ? ` (+${extraCount} more)` : "";
31 return `[Attached image${count === 1 ? "" : "s"}: ${namesSummary}${extraSummary}]`;
32}
33
2134function buildMessageBlock(message: ChatMessage): string {
22 return `${messageRoleLabel(message)}:\n${message.text}`;
35 const text = message.text;
36 const images = imageSummary(message);
37
38 if (text && images) {
39 return `${messageRoleLabel(message)}:\n${text}\n${images}`;
40 }
41 if (text) {
42 return `${messageRoleLabel(message)}:\n${text}`;
43 }
44 if (images) {
45 return `${messageRoleLabel(message)}:\n${images}`;
46 }
47 return `${messageRoleLabel(message)}:\n(empty message)`;
2348}
2449
2550function finalizeWithPrompt(
......@@ -68,7 +93,7 @@ export function buildBootstrapInput(
6893 let includedNewestFirst: string[] = [];
6994 for (const block of newestFirstBlocks) {
7095 const nextNewestFirst = [...includedNewestFirst, block];
71 const nextChronological = [...nextNewestFirst].reverse();
96 const nextChronological = nextNewestFirst.toReversed();
7297 const omittedCount = newestFirstBlocks.length - nextChronological.length;
7398 const transcriptBody =
7499 omittedCount > 0
......@@ -80,7 +105,7 @@ export function buildBootstrapInput(
80105 includedNewestFirst = nextNewestFirst;
81106 }
82107
83 let includedChronological = [...includedNewestFirst].reverse();
108 let includedChronological = includedNewestFirst.toReversed();
84109 while (true) {
85110 const omittedCount = newestFirstBlocks.length - includedChronological.length;
86111 const transcriptBody =
apps/web/src/persistenceSchema.test.ts+26-1
......@@ -200,6 +200,15 @@ describe("toPersistedState", () => {
200200 id: "m-1",
201201 role: "user",
202202 text: "Hi",
203 images: [
204 {
205 id: "img-1",
206 name: "diagram.png",
207 mimeType: "image/png",
208 sizeBytes: 4_096,
209 previewUrl: "blob:preview-1",
210 },
211 ],
203212 createdAt: "2026-02-08T10:00:00.000Z",
204213 streaming: false,
205214 },
......@@ -236,7 +245,23 @@ describe("toPersistedState", () => {
236245 model: "gpt-5.3-codex",
237246 terminalOpen: true,
238247 terminalHeight: 320,
239 messages: thread.messages,
248 messages: [
249 {
250 id: "m-1",
251 role: "user",
252 text: "Hi",
253 images: [
254 {
255 id: "img-1",
256 name: "diagram.png",
257 mimeType: "image/png",
258 sizeBytes: 4_096,
259 },
260 ],
261 createdAt: "2026-02-08T10:00:00.000Z",
262 streaming: false,
263 },
264 ],
240265 createdAt: thread.createdAt,
241266 branch: null,
242267 worktreePath: null,
apps/web/src/persistenceSchema.ts+40-5
......@@ -23,6 +23,16 @@ const persistedMessageSchema = z.object({
2323 id: z.string().min(1),
2424 role: z.enum(["user", "assistant"]),
2525 text: z.string(),
26 images: z
27 .array(
28 z.object({
29 id: z.string().min(1),
30 name: z.string().min(1),
31 mimeType: z.string().min(1),
32 sizeBytes: z.number().int().min(1),
33 }),
34 )
35 .optional(),
2636 createdAt: z.string().min(1),
2737 streaming: z.boolean(),
2838});
......@@ -126,10 +136,19 @@ function hydrateThread(
126136 terminalOpen: thread.terminalOpen ?? false,
127137 terminalHeight: thread.terminalHeight ?? DEFAULT_THREAD_TERMINAL_HEIGHT,
128138 session: null,
129 messages: thread.messages.map((message) => ({
130 ...message,
131 streaming: false,
132 })),
139 messages: thread.messages.map((message) => {
140 const hydratedImages = message.images?.map((image) => ({
141 ...image,
142 }));
143 return {
144 id: message.id,
145 role: message.role,
146 text: message.text,
147 ...(hydratedImages && hydratedImages.length > 0 ? { images: hydratedImages } : {}),
148 createdAt: message.createdAt,
149 streaming: false,
150 };
151 }),
133152 events: [],
134153 error: null,
135154 createdAt: thread.createdAt,
......@@ -189,7 +208,23 @@ export function toPersistedState(
189208 model: thread.model,
190209 terminalOpen: thread.terminalOpen,
191210 terminalHeight: thread.terminalHeight,
192 messages: thread.messages,
211 messages: thread.messages.map((message) => ({
212 id: message.id,
213 role: message.role,
214 text: message.text,
215 ...(message.images && message.images.length > 0
216 ? {
217 images: message.images.map((image) => ({
218 id: image.id,
219 name: image.name,
220 mimeType: image.mimeType,
221 sizeBytes: image.sizeBytes,
222 })),
223 }
224 : {}),
225 createdAt: message.createdAt,
226 streaming: message.streaming,
227 })),
193228 createdAt: thread.createdAt,
194229 branch: thread.branch,
195230 worktreePath: thread.worktreePath,
apps/web/src/store.ts+9-1
......@@ -13,6 +13,7 @@ import { resolveModelSlug } from "./model-logic";
1313import { hydratePersistedState, toPersistedState } from "./persistenceSchema";
1414import { applyEventToMessages, asObject, asString, evolveSession } from "./session-logic";
1515import {
16 type ChatImageAttachment,
1617 DEFAULT_RUNTIME_MODE,
1718 type Project,
1819 type RuntimeMode,
......@@ -37,7 +38,13 @@ type Action =
3738 activeAssistantItemRef: { current: string | null };
3839 }
3940 | { type: "UPDATE_SESSION"; threadId: string; session: ProviderSession }
40 | { type: "PUSH_USER_MESSAGE"; threadId: string; id: string; text: string }
41 | {
42 type: "PUSH_USER_MESSAGE";
43 threadId: string;
44 id: string;
45 text: string;
46 images?: ChatImageAttachment[];
47 }
4148 | { type: "SET_ERROR"; threadId: string; error: string | null }
4249 | { type: "SET_THREAD_TITLE"; threadId: string; title: string }
4350 | { type: "SET_THREAD_MODEL"; threadId: string; model: string }
......@@ -380,6 +387,7 @@ export function reducer(state: AppState, action: Action): AppState {
380387 id: action.id,
381388 role: "user" as const,
382389 text: action.text,
390 ...(action.images && action.images.length > 0 ? { images: action.images } : {}),
383391 createdAt: new Date().toISOString(),
384392 streaming: false,
385393 },
apps/web/src/types.ts+9
......@@ -5,10 +5,19 @@ export type RuntimeMode = "approval-required" | "full-access";
55export const DEFAULT_RUNTIME_MODE: RuntimeMode = "full-access";
66export const DEFAULT_THREAD_TERMINAL_HEIGHT = 280;
77
8export interface ChatImageAttachment {
9 id: string;
10 name: string;
11 mimeType: string;
12 sizeBytes: number;
13 previewUrl?: string;
14}
15
816export interface ChatMessage {
917 id: string;
1018 role: "user" | "assistant";
1119 text: string;
20 images?: ChatImageAttachment[];
1221 createdAt: string;
1322 streaming: boolean;
1423}
packages/contracts/src/provider.test.ts+56
......@@ -1,6 +1,7 @@
11import { describe, expect, it } from "vitest";
22
33import {
4 PROVIDER_SEND_TURN_MAX_IMAGES,
45 providerEventSchema,
56 providerRespondToRequestInputSchema,
67 providerSendTurnInputSchema,
......@@ -40,9 +41,64 @@ describe("providerSendTurnInputSchema", () => {
4041 effort: " high ",
4142 });
4243 expect(parsed.input).toBe("summarize this repo");
44 expect(parsed.images).toEqual([]);
4345 expect(parsed.model).toBe("gpt-5.2-codex");
4446 expect(parsed.effort).toBe("high");
4547 });
48
49 it("accepts image-only turns", () => {
50 const parsed = providerSendTurnInputSchema.parse({
51 sessionId: "sess_1",
52 images: [
53 {
54 name: "diagram.png",
55 mimeType: "image/png",
56 sizeBytes: 1_024,
57 dataUrl: "data:image/png;base64,AAAA",
58 },
59 ],
60 });
61 expect(parsed.input).toBeUndefined();
62 expect(parsed.images).toHaveLength(1);
63 });
64
65 it("rejects turns with neither text nor images", () => {
66 expect(() =>
67 providerSendTurnInputSchema.parse({
68 sessionId: "sess_1",
69 }),
70 ).toThrow();
71 });
72
73 it("rejects non-image data urls", () => {
74 expect(() =>
75 providerSendTurnInputSchema.parse({
76 sessionId: "sess_1",
77 images: [
78 {
79 name: "not-image.txt",
80 mimeType: "text/plain",
81 sizeBytes: 25,
82 dataUrl: "data:text/plain;base64,QQ==",
83 },
84 ],
85 }),
86 ).toThrow();
87 });
88
89 it("rejects more than the max image count", () => {
90 expect(() =>
91 providerSendTurnInputSchema.parse({
92 sessionId: "sess_1",
93 images: Array.from({ length: PROVIDER_SEND_TURN_MAX_IMAGES + 1 }, (_, index) => ({
94 name: `image-${index}.png`,
95 mimeType: "image/png",
96 sizeBytes: 1_024,
97 dataUrl: "data:image/png;base64,AAAA",
98 })),
99 }),
100 ).toThrow();
101 });
46102});
47103
48104describe("providerEventSchema", () => {
packages/contracts/src/provider.ts+37-1
......@@ -55,12 +55,46 @@ export const providerSessionStartInputSchema = z.object({
5555});
5656
5757export const PROVIDER_SEND_TURN_MAX_INPUT_CHARS = 120_000;
58export const PROVIDER_SEND_TURN_MAX_IMAGES = 8;
59export const PROVIDER_SEND_TURN_MAX_IMAGE_BYTES = 10 * 1024 * 1024;
60export const PROVIDER_SEND_TURN_MAX_IMAGE_DATA_URL_CHARS = 14_000_000;
61
62export const providerSendTurnImageInputSchema = z
63 .object({
64 name: z.string().trim().min(1).max(255),
65 mimeType: z
66 .string()
67 .trim()
68 .min(1)
69 .max(100)
70 .regex(/^image\//i, "mimeType must be an image/* MIME type"),
71 sizeBytes: z.number().int().min(1).max(PROVIDER_SEND_TURN_MAX_IMAGE_BYTES),
72 dataUrl: z.string().trim().min(1).max(PROVIDER_SEND_TURN_MAX_IMAGE_DATA_URL_CHARS),
73 })
74 .superRefine((value, ctx) => {
75 if (!value.dataUrl.startsWith("data:image/")) {
76 ctx.addIssue({
77 code: z.ZodIssueCode.custom,
78 message: "Image dataUrl must start with data:image/",
79 path: ["dataUrl"],
80 });
81 }
82 });
5883
5984export const providerSendTurnInputSchema = z.object({
6085 sessionId: z.string().min(1),
61 input: z.string().trim().min(1).max(PROVIDER_SEND_TURN_MAX_INPUT_CHARS),
86 input: z.string().trim().min(1).max(PROVIDER_SEND_TURN_MAX_INPUT_CHARS).optional(),
87 images: z.array(providerSendTurnImageInputSchema).max(PROVIDER_SEND_TURN_MAX_IMAGES).default([]),
6288 model: z.string().trim().min(1).optional(),
6389 effort: z.string().trim().min(1).optional(),
90}).superRefine((value, ctx) => {
91 if (!value.input && value.images.length === 0) {
92 ctx.addIssue({
93 code: z.ZodIssueCode.custom,
94 message: "Either input text or at least one image is required",
95 path: ["input"],
96 });
97 }
6498});
6599
66100export const providerTurnStartResultSchema = z.object({
......@@ -110,6 +144,8 @@ export type ProviderApprovalDecision = z.infer<typeof providerApprovalDecisionSc
110144export type ProviderSessionStatus = z.infer<typeof providerSessionStatusSchema>;
111145export type ProviderSession = z.infer<typeof providerSessionSchema>;
112146export type ProviderSessionStartInput = z.input<typeof providerSessionStartInputSchema>;
147export type ProviderSendTurnImage = z.infer<typeof providerSendTurnImageInputSchema>;
148export type ProviderSendTurnImageInput = z.input<typeof providerSendTurnImageInputSchema>;
113149export type ProviderSendTurnInput = z.input<typeof providerSendTurnInputSchema>;
114150export type ProviderTurnStartResult = z.infer<typeof providerTurnStartResultSchema>;
115151export type ProviderInterruptTurnInput = z.input<typeof providerInterruptTurnInputSchema>;