authorgravatar for me@t3.ggTheo Browne <me@t3.gg> 2026-09-04 12:13:24-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-09-04 12:13:24-07:00
log15eda897d37aad232fca5089f5eb6ef7b726c31c
tree434de06d2fc2e87cb85ecc65bbf7d37c17d449d4
parent6365919f2e5bcfb4fa4020b95e19af26ae40979f
signature Signed by PGP key B5690EEEBB952194

perf(web): defer image URL requests for thread history (#9760)

4 files changed, 196 insertions(+), 30 deletions(-)

apps/web/src/components/ChatView.tsx+6-20
......@@ -104,6 +104,7 @@ import {
104104 deriveWorkLogEntries,
105105 hasActionableProposedPlan,
106106 isLatestTurnSettled,
107 selectHandoffImageResources,
107108 type TimelineEntriesProjection,
108109} from "../session-logic";
109110import { type LegendListRef } from "@legendapp/list/react";
......@@ -2763,35 +2764,20 @@ export default function ChatView(props: ChatViewProps) {
27632764 },
27642765 [activeThreadRef, downloadFileAttachment],
27652766 );
2766 const serverAttachmentIds = useMemo(() => {
2767 const attachmentIds = new Set<string>();
2768 for (const message of serverMessages ?? []) {
2769 for (const attachment of message.attachments ?? []) {
2770 if (isImageAttachment(attachment)) {
2771 attachmentIds.add(attachment.id);
2772 }
2773 }
2774 }
2775 return [...attachmentIds];
2776 }, [serverMessages]);
27772767 const serverAttachmentResources = useMemo(
2778 () =>
2779 serverAttachmentIds.map((attachmentId) => ({
2780 _tag: "attachment" as const,
2781 attachmentId,
2782 })),
2783 [serverAttachmentIds],
2768 () => selectHandoffImageResources(serverMessages, attachmentPreviewHandoffByMessageId),
2769 [serverMessages, attachmentPreviewHandoffByMessageId],
27842770 );
27852771 const serverAttachmentUrls = useAssetUrls(environmentId, serverAttachmentResources);
27862772 const serverAttachmentUrlById = useMemo(
27872773 () =>
27882774 new Map(
2789 serverAttachmentIds.flatMap((attachmentId, index) => {
2775 serverAttachmentResources.flatMap((resource, index) => {
27902776 const url = serverAttachmentUrls[index];
2791 return url ? [[attachmentId, url] as const] : [];
2777 return url ? [[resource.attachmentId, url] as const] : [];
27922778 }),
27932779 ),
2794 [serverAttachmentIds, serverAttachmentUrls],
2780 [serverAttachmentResources, serverAttachmentUrls],
27952781 );
27962782 const displayServerMessages = useMemo<ReadonlyArray<ChatMessage>>(() => {
27972783 if (!serverMessages) return [];
apps/web/src/components/chat/MessagesTimeline.tsx+19-2
......@@ -52,7 +52,9 @@ import {
5252import { FileDiff } from "@pierre/diffs/react";
5353import { DiffWorkerPoolProvider } from "../DiffWorkerPoolProvider";
5454import {
55 createMessageAttachmentPreviewProjector,
5556 deriveTimelineEntries,
57 selectMessageImageResources,
5658 workEntryDisplayIndicatesToolFailure,
5759 workEntrySignalsSevereFailure,
5860 workLogEntryIsToolLike,
......@@ -99,7 +101,7 @@ import {
99101 ZapIcon,
100102} from "lucide-react";
101103import { Button } from "../ui/button";
102import { useAssetUrlRefresh, useAssetUrlState } from "../../assets/assetUrls";
104import { useAssetUrlRefresh, useAssetUrls, useAssetUrlState } from "../../assets/assetUrls";
103105import { MediaVideoPlayer } from "../media/MediaVideoPlayer";
104106import { getVirtualizedScrollFadeClassName } from "../ui/scroll-area";
105107import {
......@@ -1227,9 +1229,24 @@ function UserVideoAttachment({ file }: { readonly file: ChatFileAttachment }) {
12271229
12281230function UserTimelineRow({ row }: { row: Extract<TimelineRow, { kind: "message" }> }) {
12291231 const ctx = use(TimelineRowCtx);
1232 const resources = useMemo(
1233 () => selectMessageImageResources(row.message.attachments),
1234 [row.message.attachments],
1235 );
1236 const previewUrls = useAssetUrls(ctx.activeThreadEnvironmentId, resources);
1237 const [projectPreviews] = useState(createMessageAttachmentPreviewProjector);
1238 const messageWithPreviews = useMemo(() => {
1239 const urlsById = new Map(
1240 resources.flatMap((resource, index) => {
1241 const url = previewUrls[index];
1242 return url ? [[resource.attachmentId, url] as const] : [];
1243 }),
1244 );
1245 return projectPreviews(row.message, (attachment) => urlsById.get(attachment.id));
1246 }, [previewUrls, projectPreviews, resources, row.message]);
12301247 // The attachment union has an open member, so guards (not literal type
12311248 // comparisons) split it. Unknown types render as inert rows below the files.
1232 const userImages = (row.message.attachments ?? []).filter(isImageAttachment);
1249 const userImages = (messageWithPreviews.attachments ?? []).filter(isImageAttachment);
12331250 const userFiles = (row.message.attachments ?? []).filter(isFileAttachment);
12341251 const userVideos = userFiles.filter(isVideoAttachment);
12351252 const otherUserFiles = userFiles.filter((file) => !isVideoAttachment(file));
apps/web/src/session-logic.test.ts+124
......@@ -21,6 +21,8 @@ import {
2121 findLatestProposedPlan,
2222 hasActionableProposedPlan,
2323 isLatestTurnSettled,
24 selectHandoffImageResources,
25 selectMessageImageResources,
2426 workEntryIndicatesToolFailure,
2527 workEntryIndicatesToolNeutralStatus,
2628 workEntryIndicatesToolSuccess,
......@@ -2075,6 +2077,128 @@ describe("deriveWorkLogEntries", () => {
20752077 });
20762078});
20772079
2080describe("image asset requests", () => {
2081 const image = {
2082 type: "image" as const,
2083 id: "image",
2084 name: "image.png",
2085 mimeType: "image/png",
2086 sizeBytes: 42,
2087 };
2088 const message = {
2089 id: MessageId.make("image-message"),
2090 role: "user" as const,
2091 text: "Inspect these images",
2092 turnId: null,
2093 createdAt: "2026-09-04T00:00:00.000Z",
2094 updatedAt: "2026-09-04T00:00:00.000Z",
2095 streaming: false,
2096 attachments: [image],
2097 };
2098
2099 it("requests the whole row's gallery and crops without signing local preview IDs", () => {
2100 const attachments = Object.freeze([
2101 image,
2102 { ...image, id: "second" },
2103 { ...image, id: "crop", name: "preview-annotation-1.png" },
2104 { ...image, id: "local", previewUrl: "blob:local" },
2105 { ...image, id: "inline", previewUrl: "data:image/png;base64,AA==" },
2106 { ...image, id: "provided", previewUrl: "https://preview.test/image" },
2107 { ...image, type: "file" as const, id: "file", mimeType: "application/pdf" },
2108 { ...image, type: "future", id: "unknown" },
2109 image,
2110 ]);
2111
2112 expect(selectMessageImageResources(attachments)).toEqual([
2113 { _tag: "attachment", attachmentId: "image" },
2114 { _tag: "attachment", attachmentId: "second" },
2115 { _tag: "attachment", attachmentId: "crop" },
2116 { _tag: "attachment", attachmentId: "provided" },
2117 ]);
2118 });
2119
2120 it("requests offscreen handoffs without signing the rest of the loaded history", () => {
2121 const history = {
2122 ...message,
2123 id: MessageId.make("history"),
2124 attachments: [{ ...image, id: "history-image" }],
2125 };
2126 const offscreen = {
2127 ...message,
2128 id: MessageId.make("offscreen"),
2129 attachments: [image, { ...image, id: "crop", name: "preview-annotation-1.png" }],
2130 };
2131 const empty = {
2132 ...message,
2133 id: MessageId.make("empty"),
2134 attachments: [{ ...image, id: "empty" }],
2135 };
2136 const assistant = {
2137 ...message,
2138 id: MessageId.make("assistant"),
2139 role: "assistant" as const,
2140 attachments: [{ ...image, id: "assistant-image" }],
2141 };
2142 expect(
2143 selectHandoffImageResources([history, message, offscreen, empty, assistant], {
2144 [message.id]: ["blob:message"],
2145 [offscreen.id]: ["blob:offscreen", "blob:crop"],
2146 [empty.id]: [],
2147 [assistant.id]: ["blob:unused"],
2148 }),
2149 ).toEqual([
2150 { _tag: "attachment", attachmentId: "image" },
2151 { _tag: "attachment", attachmentId: "crop" },
2152 ]);
2153 });
2154
2155 it("does not scan history when no handoff is pending", () => {
2156 let reads = 0;
2157 const messages = new Proxy([message], {
2158 get(target, property, receiver) {
2159 if (property === "0") reads += 1;
2160 return Reflect.get(target, property, receiver);
2161 },
2162 });
2163 const empty = selectHandoffImageResources(messages, {});
2164 expect(reads).toBe(0);
2165 expect(empty).toHaveLength(0);
2166 expect(selectHandoffImageResources(undefined, { missing: ["blob:missing"] })).toBe(empty);
2167 expect(selectMessageImageResources(undefined)).toBe(empty);
2168 });
2169
2170 it("hands signed URLs to a mounted row only after the local preview is released", () => {
2171 const server = createMessageAttachmentPreviewProjector();
2172 const handoff = createMessageAttachmentPreviewProjector();
2173 const row = createMessageAttachmentPreviewProjector();
2174 const pending = handoff(
2175 server(message, () => undefined),
2176 () => "blob:pending",
2177 );
2178 expect(selectMessageImageResources(pending.attachments)).toEqual([]);
2179 expect(selectHandoffImageResources([message], { [message.id]: ["blob:pending"] })).toEqual([
2180 { _tag: "attachment", attachmentId: image.id },
2181 ]);
2182
2183 const ready = server(message, () => "https://server.test/image");
2184 expect(selectMessageImageResources(handoff(ready, () => "blob:pending").attachments)).toEqual(
2185 [],
2186 );
2187 const released = server(message, () => undefined);
2188 expect(selectMessageImageResources(released.attachments)).toEqual([
2189 { _tag: "attachment", attachmentId: image.id },
2190 ]);
2191 const displayed = row(released, () => "https://server.test/image");
2192 expect(displayed).toEqual(ready);
2193 expect(pending.attachments?.[0]).toMatchObject({ previewUrl: "blob:pending" });
2194 expect(row(released, () => "https://server.test/renewed").attachments?.[0]).toMatchObject({
2195 previewUrl: "https://server.test/renewed",
2196 });
2197 expect(displayed.attachments?.[0]).toMatchObject({ previewUrl: "https://server.test/image" });
2198 expect(row(released, () => undefined)).toBe(message);
2199 });
2200});
2201
20782202describe("deriveTimelineEntries", () => {
20792203 const streamingMessage = {
20802204 id: MessageId.make("streaming-message"),
apps/web/src/session-logic.ts+47-8
......@@ -12,6 +12,7 @@ import { extractToolActivityPresentation } from "@t3tools/client-runtime/work-lo
1212import {
1313 ApprovalRequestId,
1414 isToolLifecycleItemType,
15 type AssetResource,
1516 type OrchestrationLatestTurn,
1617 type OrchestrationThreadActivity,
1718 type OrchestrationProposedPlanId,
......@@ -24,14 +25,15 @@ import {
2425 type TurnId,
2526} from "@t3tools/contracts";
2627
27import type {
28 ChatAttachment,
29 ChatMessage,
30 ProposedPlan,
31 SessionPhase,
32 Thread,
33 ThreadSession,
34 TurnDiffSummary,
28import {
29 isImageAttachment,
30 type ChatAttachment,
31 type ChatMessage,
32 type ProposedPlan,
33 type SessionPhase,
34 type Thread,
35 type ThreadSession,
36 type TurnDiffSummary,
3537} from "./types";
3638
3739export type ProviderPickerKind = ProviderDriverKind;
......@@ -1883,6 +1885,43 @@ function mergeTimelineEntrySuffix(
18831885 return merged;
18841886}
18851887
1888type AttachmentResource = Extract<AssetResource, { readonly _tag: "attachment" }>;
1889const EMPTY_IMAGE_RESOURCES = Object.freeze<ReadonlyArray<AttachmentResource>>([]);
1890
1891/** A mounted row requests its stored images. Local previews keep their existing URLs. */
1892export function selectMessageImageResources(
1893 attachments: ChatMessage["attachments"],
1894): ReadonlyArray<AttachmentResource> {
1895 const attachmentIds = new Set<string>();
1896 for (const attachment of attachments ?? []) {
1897 if (!isImageAttachment(attachment)) continue;
1898 const previewUrl = attachment.previewUrl;
1899 if (previewUrl?.startsWith("blob:") || previewUrl?.startsWith("data:")) continue;
1900 attachmentIds.add(attachment.id);
1901 }
1902 return attachmentIds.size === 0
1903 ? EMPTY_IMAGE_RESOURCES
1904 : Array.from(attachmentIds, (attachmentId) => ({ _tag: "attachment", attachmentId }));
1905}
1906
1907/** Handoffs need server URLs even while their message rows are unmounted. */
1908export function selectHandoffImageResources(
1909 messages: ReadonlyArray<ChatMessage> | undefined,
1910 handoffs: Readonly<Record<string, ReadonlyArray<string>>>,
1911): ReadonlyArray<AttachmentResource> {
1912 if (Object.keys(handoffs).length === 0) return EMPTY_IMAGE_RESOURCES;
1913 const attachmentIds = new Set<string>();
1914 for (const message of messages ?? []) {
1915 if (message.role !== "user" || !handoffs[message.id]?.length) continue;
1916 for (const attachment of message.attachments ?? []) {
1917 if (isImageAttachment(attachment)) attachmentIds.add(attachment.id);
1918 }
1919 }
1920 return attachmentIds.size === 0
1921 ? EMPTY_IMAGE_RESOURCES
1922 : Array.from(attachmentIds, (attachmentId) => ({ _tag: "attachment", attachmentId }));
1923}
1924
18861925/** Own one mapper per preview stage. Immutable messages retain unchanged preview objects. */
18871926export function createMessageAttachmentPreviewProjector() {
18881927 const attachmentsBySource = new WeakMap<