diff --git a/apps/mobile/src/lib/composerContext.test.ts b/apps/mobile/src/lib/composerContext.test.ts index d28b58d055b0b81aed4543c4cc1f1185c2bd3adc..3b00f7c5e7c4d19f7148156376c70540377209d4 100644 --- a/apps/mobile/src/lib/composerContext.test.ts +++ b/apps/mobile/src/lib/composerContext.test.ts @@ -8,6 +8,7 @@ import { } from "@t3tools/contracts"; import { collectComposerInlineTokens } from "@t3tools/shared/composerInlineTokens"; import { + ATTACHMENT_ONLY_BOOTSTRAP_PROMPT, collectComposerContextReferences, formatComposerContextReference, projectComposerContextForProvider, @@ -207,6 +208,20 @@ describe("mobile composer context", () => { }); describe("host context compatibility", () => { + it("sends an image-only prompt as the attachment-only prompt and keeps the image", () => { + const context: OrchestrationMessageContext = { version: 1, records: [image] }; + const reference = formatComposerContextReference(image); + expect(serializeComposerMessageForServer(reference, context, true)).toEqual({ + text: ATTACHMENT_ONLY_BOOTSTRAP_PROMPT, + context, + }); + const withProse = `${reference} what is this?`; + expect(serializeComposerMessageForServer(withProse, context, true)).toEqual({ + text: withProse, + context, + }); + }); + it.each(["existing-thread", "new-task"])("serializes %s sends for an older host", (path) => { const pr = pullRequestComposerContext( { diff --git a/apps/mobile/src/lib/composerContext.ts b/apps/mobile/src/lib/composerContext.ts index c5422405c96606ce3016580e283d57905b909643..f15d9cc300ea677f09042df7692a72758c33ce23 100644 --- a/apps/mobile/src/lib/composerContext.ts +++ b/apps/mobile/src/lib/composerContext.ts @@ -13,8 +13,10 @@ import { } from "@t3tools/contracts"; import * as Schema from "effect/Schema"; import { + ATTACHMENT_ONLY_BOOTSTRAP_PROMPT, collectComposerContextReferences, formatComposerContextReference, + isImageReferencesOnly, replaceComposerContextReferences, sanitizeComposerContextLabel, } from "@t3tools/shared/composerContextReferences"; @@ -229,12 +231,19 @@ export function reidentifyComposerContext( }; } -/** Keep queued records canonical; choose the wire format against the host at dispatch time. */ +/** + * Keep queued records canonical; choose the wire format against the host at dispatch time. + * An image-only prompt goes out as the attachment-only prompt web sends, so the agent is still + * asked to respond. The images stay in the message's attachments and context records. + */ export function serializeComposerMessageForServer( - text: string, + composerText: string, context: OrchestrationMessageContext | undefined, supportsInlineMessageContext: boolean, ): { text: string; context?: OrchestrationMessageContext } { + const text = isImageReferencesOnly(composerText) + ? ATTACHMENT_ONLY_BOOTSTRAP_PROMPT + : composerText; return supportsInlineMessageContext ? { text, ...(context ? { context } : {}) } : { text: serializeLegacyContextMessage({ text, records: context?.records ?? [] }) }; diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 20c4d08b5bcfc4928d4ddeab6202b2666a20852a..84f5b476a13352af9f31e21fdbc47f2820a1a9bc 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -607,10 +607,8 @@ import { supportsServerUpdateThreadContinuation, } from "../versionSkew"; import { useAssetUrls } from "../assets/assetUrls"; -import { - ATTACHMENT_ONLY_BOOTSTRAP_PROMPT, - recallableComposerPrompt, -} from "./chat/composerPromptHistory"; +import { ATTACHMENT_ONLY_BOOTSTRAP_PROMPT } from "@t3tools/shared/composerContextReferences"; +import { recallableComposerPrompt } from "./chat/composerPromptHistory"; import { observeResize } from "~/lib/observeResize"; const EMPTY_PROVIDERS: ServerProvider[] = []; diff --git a/apps/web/src/components/chat/composerPromptHistory.test.ts b/apps/web/src/components/chat/composerPromptHistory.test.ts index b3752cf7dbeb303cc70299865d705c8dd4c8d4a7..deddc4679412b9fb5601f85eeefb327508bd1da5 100644 --- a/apps/web/src/components/chat/composerPromptHistory.test.ts +++ b/apps/web/src/components/chat/composerPromptHistory.test.ts @@ -1,8 +1,8 @@ import { describe, expect, it } from "vite-plus/test"; +import { ATTACHMENT_ONLY_BOOTSTRAP_PROMPT } from "@t3tools/shared/composerContextReferences"; import { buildPlanImplementationPrompt } from "../../proposedPlan"; import { - ATTACHMENT_ONLY_BOOTSTRAP_PROMPT, buildComposerPromptHistoryEntries, recallableComposerPrompt, stepComposerPromptHistory, diff --git a/apps/web/src/components/chat/composerPromptHistory.ts b/apps/web/src/components/chat/composerPromptHistory.ts index f50280f6d242a7dbfee15a0ca620a2b8909bbb4a..0977637cdf16c8e25194c8948cfa4bf5487b8950 100644 --- a/apps/web/src/components/chat/composerPromptHistory.ts +++ b/apps/web/src/components/chat/composerPromptHistory.ts @@ -1,4 +1,7 @@ -import { collectComposerContextReferences } from "@t3tools/shared/composerContextReferences"; +import { + ATTACHMENT_ONLY_BOOTSTRAP_PROMPT, + collectComposerContextReferences, +} from "@t3tools/shared/composerContextReferences"; import { PLAN_IMPLEMENTATION_PROMPT_PREFIX } from "../../proposedPlan"; /** @@ -15,10 +18,6 @@ const REVIEW_COMMENT_BLOCK_PATTERN = /]*>[\s\S]*?<\/review_c const TRAILING_LEGACY_CONTEXT = /\n*<(terminal_context|element_context|preview_annotation)>\n([\s\S]*?)\n<\/\1>\s*$/; -/** Text sent in place of an empty prompt when a message is attachments only. */ -export const ATTACHMENT_ONLY_BOOTSTRAP_PROMPT = - "[User attached one or more files without additional text. Respond using the conversation context and the attached files.]"; - export interface ComposerPromptHistoryMessage { readonly id: string; readonly role: string; diff --git a/packages/shared/src/composerContextReferences.test.ts b/packages/shared/src/composerContextReferences.test.ts index bd078c8f1d44e6514cb908a41191779d565b6bef..8b59a6c261b9eff9a14f29b7f02af8a6d76a92da 100644 --- a/packages/shared/src/composerContextReferences.test.ts +++ b/packages/shared/src/composerContextReferences.test.ts @@ -6,6 +6,7 @@ import { formatComposerContextHref, formatComposerContextProviderMarker, formatComposerContextReference, + isImageReferencesOnly, parseComposerContextHref, projectComposerContextForProvider, replaceComposerContextReferences, @@ -86,6 +87,17 @@ describe("labels and reference links", () => { expect(collectComposerContextReferences("[x](https://example.com)")).toEqual([]); }); + it("recognizes a prompt made only of image references", () => { + const image = "![a.png](t3-context://v1/image/ctx_1)"; + const file = "[notes.md](t3-context://v1/file/ctx_3)"; + expect(isImageReferencesOnly(image)).toBe(true); + expect(isImageReferencesOnly(` ${image}\n![b.png](t3-context://v1/image/ctx_2) `)).toBe(true); + expect(isImageReferencesOnly(`${image} what is this?`)).toBe(false); + expect(isImageReferencesOnly(file)).toBe(false); + expect(isImageReferencesOnly(`${image} ${file}`)).toBe(false); + expect(isImageReferencesOnly(" ")).toBe(false); + }); + it("replaces occurrences in place", () => { const text = "a [x](t3-context://v1/skill/ctx_1) b [y](t3-context://v1/file/ctx_2) c"; expect(replaceComposerContextReferences(text, (o) => `<${o.contextId}>`)).toBe( diff --git a/packages/shared/src/composerContextReferences.ts b/packages/shared/src/composerContextReferences.ts index f7877fe937a69ecaaa982b8723a2eb0972eaac7d..ae58648990809fa6015c43a58c41a6f22fb3e75b 100644 --- a/packages/shared/src/composerContextReferences.ts +++ b/packages/shared/src/composerContextReferences.ts @@ -92,6 +92,20 @@ export function collectComposerContextReferences( return occurrences; } +/** Text sent in place of an empty prompt when a message is attachments only. */ +export const ATTACHMENT_ONLY_BOOTSTRAP_PROMPT = + "[User attached one or more files without additional text. Respond using the conversation context and the attached files.]"; + +/** Whether `text` is image references and nothing else, which is how an image-only prompt reads. */ +export function isImageReferencesOnly(text: string): boolean { + const references = collectComposerContextReferences(text); + return ( + references.length > 0 && + references.every((reference) => reference.kind === "image") && + replaceComposerContextReferences(text, () => "").trim() === "" + ); +} + export function replaceComposerContextReferences( text: string, replace: (occurrence: ComposerContextReferenceOccurrence) => string,