authorgravatar for 159104116+whoisaldo@users.noreply.github.comAli Younes <159104116+whoisaldo@users.noreply.github.com> 2026-10-11 03:40:36-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-10-11 02:40:36-05:00
log8d199b4f7b918079161d5f43e0146c099301be08
tree35fc1bf71253d38407d478bc551265d25693ea1b
parent20254307084f1101ced00dfadeddc7804a7de45d
signature Signed by PGP key B5690EEEBB952194

fix(mobile): image-only messages ask the agent to respond, like desktop (#16480)

Co-authored-by: Yash Singh <saiansh2525@gmail.com>

7 files changed, 59 insertions(+), 12 deletions(-)

apps/mobile/src/lib/composerContext.test.ts+15
......@@ -8,6 +8,7 @@ import {
88} from "@t3tools/contracts";
99import { collectComposerInlineTokens } from "@t3tools/shared/composerInlineTokens";
1010import {
11 ATTACHMENT_ONLY_BOOTSTRAP_PROMPT,
1112 collectComposerContextReferences,
1213 formatComposerContextReference,
1314 projectComposerContextForProvider,
......@@ -207,6 +208,20 @@ describe("mobile composer context", () => {
207208});
208209
209210describe("host context compatibility", () => {
211 it("sends an image-only prompt as the attachment-only prompt and keeps the image", () => {
212 const context: OrchestrationMessageContext = { version: 1, records: [image] };
213 const reference = formatComposerContextReference(image);
214 expect(serializeComposerMessageForServer(reference, context, true)).toEqual({
215 text: ATTACHMENT_ONLY_BOOTSTRAP_PROMPT,
216 context,
217 });
218 const withProse = `${reference} what is this?`;
219 expect(serializeComposerMessageForServer(withProse, context, true)).toEqual({
220 text: withProse,
221 context,
222 });
223 });
224
210225 it.each(["existing-thread", "new-task"])("serializes %s sends for an older host", (path) => {
211226 const pr = pullRequestComposerContext(
212227 {
apps/mobile/src/lib/composerContext.ts+11-2
......@@ -13,8 +13,10 @@ import {
1313} from "@t3tools/contracts";
1414import * as Schema from "effect/Schema";
1515import {
16 ATTACHMENT_ONLY_BOOTSTRAP_PROMPT,
1617 collectComposerContextReferences,
1718 formatComposerContextReference,
19 isImageReferencesOnly,
1820 replaceComposerContextReferences,
1921 sanitizeComposerContextLabel,
2022} from "@t3tools/shared/composerContextReferences";
......@@ -229,12 +231,19 @@ export function reidentifyComposerContext(
229231 };
230232}
231233
232/** Keep queued records canonical; choose the wire format against the host at dispatch time. */
234/**
235 * Keep queued records canonical; choose the wire format against the host at dispatch time.
236 * An image-only prompt goes out as the attachment-only prompt web sends, so the agent is still
237 * asked to respond. The images stay in the message's attachments and context records.
238 */
233239export function serializeComposerMessageForServer(
234 text: string,
240 composerText: string,
235241 context: OrchestrationMessageContext | undefined,
236242 supportsInlineMessageContext: boolean,
237243): { text: string; context?: OrchestrationMessageContext } {
244 const text = isImageReferencesOnly(composerText)
245 ? ATTACHMENT_ONLY_BOOTSTRAP_PROMPT
246 : composerText;
238247 return supportsInlineMessageContext
239248 ? { text, ...(context ? { context } : {}) }
240249 : { text: serializeLegacyContextMessage({ text, records: context?.records ?? [] }) };
apps/web/src/components/ChatView.tsx+2-4
......@@ -607,10 +607,8 @@ import {
607607 supportsServerUpdateThreadContinuation,
608608} from "../versionSkew";
609609import { useAssetUrls } from "../assets/assetUrls";
610import {
611 ATTACHMENT_ONLY_BOOTSTRAP_PROMPT,
612 recallableComposerPrompt,
613} from "./chat/composerPromptHistory";
610import { ATTACHMENT_ONLY_BOOTSTRAP_PROMPT } from "@t3tools/shared/composerContextReferences";
611import { recallableComposerPrompt } from "./chat/composerPromptHistory";
614612import { observeResize } from "~/lib/observeResize";
615613
616614const EMPTY_PROVIDERS: ServerProvider[] = [];
apps/web/src/components/chat/composerPromptHistory.test.ts+1-1
......@@ -1,8 +1,8 @@
11import { describe, expect, it } from "vite-plus/test";
22
3import { ATTACHMENT_ONLY_BOOTSTRAP_PROMPT } from "@t3tools/shared/composerContextReferences";
34import { buildPlanImplementationPrompt } from "../../proposedPlan";
45import {
5 ATTACHMENT_ONLY_BOOTSTRAP_PROMPT,
66 buildComposerPromptHistoryEntries,
77 recallableComposerPrompt,
88 stepComposerPromptHistory,
apps/web/src/components/chat/composerPromptHistory.ts+4-5
......@@ -1,4 +1,7 @@
1import { collectComposerContextReferences } from "@t3tools/shared/composerContextReferences";
1import {
2 ATTACHMENT_ONLY_BOOTSTRAP_PROMPT,
3 collectComposerContextReferences,
4} from "@t3tools/shared/composerContextReferences";
25import { PLAN_IMPLEMENTATION_PROMPT_PREFIX } from "../../proposedPlan";
36
47/**
......@@ -15,10 +18,6 @@ const REVIEW_COMMENT_BLOCK_PATTERN = /<review_comment\b[^>]*>[\s\S]*?<\/review_c
1518const TRAILING_LEGACY_CONTEXT =
1619 /\n*<(terminal_context|element_context|preview_annotation)>\n([\s\S]*?)\n<\/\1>\s*$/;
1720
18/** Text sent in place of an empty prompt when a message is attachments only. */
19export const ATTACHMENT_ONLY_BOOTSTRAP_PROMPT =
20 "[User attached one or more files without additional text. Respond using the conversation context and the attached files.]";
21
2221export interface ComposerPromptHistoryMessage {
2322 readonly id: string;
2423 readonly role: string;
packages/shared/src/composerContextReferences.test.ts+12
......@@ -6,6 +6,7 @@ import {
66 formatComposerContextHref,
77 formatComposerContextProviderMarker,
88 formatComposerContextReference,
9 isImageReferencesOnly,
910 parseComposerContextHref,
1011 projectComposerContextForProvider,
1112 replaceComposerContextReferences,
......@@ -86,6 +87,17 @@ describe("labels and reference links", () => {
8687 expect(collectComposerContextReferences("[x](https://example.com)")).toEqual([]);
8788 });
8889
90 it("recognizes a prompt made only of image references", () => {
91 const image = "![a.png](t3-context://v1/image/ctx_1)";
92 const file = "[notes.md](t3-context://v1/file/ctx_3)";
93 expect(isImageReferencesOnly(image)).toBe(true);
94 expect(isImageReferencesOnly(` ${image}\n![b.png](t3-context://v1/image/ctx_2) `)).toBe(true);
95 expect(isImageReferencesOnly(`${image} what is this?`)).toBe(false);
96 expect(isImageReferencesOnly(file)).toBe(false);
97 expect(isImageReferencesOnly(`${image} ${file}`)).toBe(false);
98 expect(isImageReferencesOnly(" ")).toBe(false);
99 });
100
89101 it("replaces occurrences in place", () => {
90102 const text = "a [x](t3-context://v1/skill/ctx_1) b [y](t3-context://v1/file/ctx_2) c";
91103 expect(replaceComposerContextReferences(text, (o) => `<${o.contextId}>`)).toBe(
packages/shared/src/composerContextReferences.ts+14
......@@ -92,6 +92,20 @@ export function collectComposerContextReferences(
9292 return occurrences;
9393}
9494
95/** Text sent in place of an empty prompt when a message is attachments only. */
96export const ATTACHMENT_ONLY_BOOTSTRAP_PROMPT =
97 "[User attached one or more files without additional text. Respond using the conversation context and the attached files.]";
98
99/** Whether `text` is image references and nothing else, which is how an image-only prompt reads. */
100export function isImageReferencesOnly(text: string): boolean {
101 const references = collectComposerContextReferences(text);
102 return (
103 references.length > 0 &&
104 references.every((reference) => reference.kind === "image") &&
105 replaceComposerContextReferences(text, () => "").trim() === ""
106 );
107}
108
95109export function replaceComposerContextReferences(
96110 text: string,
97111 replace: (occurrence: ComposerContextReferenceOccurrence) => string,