authorgravatar for me@t3.ggTheo Browne <me@t3.gg> 2026-08-30 18:08:36-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-08-30 18:08:36-07:00
log352710d497cc640553e3e18e23fb5a5f3f890466
tree4d04b1e46f932375f59b8b430c15bea31d7f8ee6
parent8b817cbcaad71a53e2ef73f3881067f8aa8094bc
signature Signed by PGP key B5690EEEBB952194

feat(mobile): add offline iPhone voice input (#8614)

Co-authored-by: Julius Marminge <julius@mac.lan>

36 files changed, 3001 insertions(+), 225 deletions(-)

apps/mobile/app.config.ts+9
...@@ -295,6 +295,15 @@ const config: ExpoConfig = {...@@ -295,6 +295,15 @@ const config: ExpoConfig = {
295 },295 },
296 },296 },
297 ],297 ],
298 [
299 "expo-audio",
300 {
301 microphonePermission: "Allow T3 Code to use your microphone for voice input.",
302 recordAudioAndroid: false,
303 enableBackgroundPlayback: false,
304 enableBackgroundRecording: false,
305 },
306 ],
298 [307 [
299 "expo-camera",308 "expo-camera",
300 {309 {
apps/mobile/modules/t3-composer-editor/ios/T3ComposerEditorModule.swift+3
...@@ -29,6 +29,9 @@ public class T3ComposerEditorModule: Module {...@@ -29,6 +29,9 @@ public class T3ComposerEditorModule: Module {
29 Prop("editable") { (view: T3ComposerEditorView, editable: Bool) in29 Prop("editable") { (view: T3ComposerEditorView, editable: Bool) in
30 view.setEditable(editable)30 view.setEditable(editable)
31 }31 }
32 Prop("readOnly") { (view: T3ComposerEditorView, readOnly: Bool) in
33 view.setReadOnly(readOnly)
34 }
32 Prop("scrollEnabled") { (view: T3ComposerEditorView, scrollEnabled: Bool) in35 Prop("scrollEnabled") { (view: T3ComposerEditorView, scrollEnabled: Bool) in
33 view.setScrollEnabled(scrollEnabled)36 view.setScrollEnabled(scrollEnabled)
34 }37 }
apps/mobile/modules/t3-composer-editor/ios/T3ComposerEditorView.swift+40-2
...@@ -60,10 +60,21 @@ private final class ComposerTextAttachment: NSTextAttachment {...@@ -60,10 +60,21 @@ private final class ComposerTextAttachment: NSTextAttachment {
60private final class ComposerTextView: UITextView {60private final class ComposerTextView: UITextView {
61 private static let pastedImageDirectoryName = "t3-composer-paste"61 private static let pastedImageDirectoryName = "t3-composer-paste"
62 private static let stalePastedImageAge: TimeInterval = 60 * 6062 private static let stalePastedImageAge: TimeInterval = 60 * 60
63 private static let readOnlyActions = Set([
64 "cut:",
65 "delete:",
66 "paste:",
67 "redo:",
68 "toggleBoldface:",
69 "toggleItalics:",
70 "toggleUnderline:",
71 "undo:",
72 ])
6373
64 var onPasteImages: (([String]) -> Void)?74 var onPasteImages: (([String]) -> Void)?
65 var onAttributedMutation: (() -> Void)?75 var onAttributedMutation: (() -> Void)?
66 var onSubmit: (() -> Void)?76 var onSubmit: (() -> Void)?
77 var isReadOnly = false
6778
68 override var keyCommands: [UIKeyCommand]? {79 override var keyCommands: [UIKeyCommand]? {
69 var commands = super.keyCommands ?? []80 var commands = super.keyCommands ?? []
...@@ -83,6 +94,9 @@ private final class ComposerTextView: UITextView {...@@ -83,6 +94,9 @@ private final class ComposerTextView: UITextView {
83 }94 }
8495
85 override func canPerformAction(_ action: Selector, withSender sender: Any?) -> Bool {96 override func canPerformAction(_ action: Selector, withSender sender: Any?) -> Bool {
97 if isReadOnly && Self.readOnlyActions.contains(NSStringFromSelector(action)) {
98 return false
99 }
86 if action == #selector(paste(_:)) {100 if action == #selector(paste(_:)) {
87 let pasteboard = UIPasteboard.general101 let pasteboard = UIPasteboard.general
88 if pasteboard.hasImages ||102 if pasteboard.hasImages ||
...@@ -96,6 +110,9 @@ private final class ComposerTextView: UITextView {...@@ -96,6 +110,9 @@ private final class ComposerTextView: UITextView {
96 }110 }
97111
98 override func paste(_ sender: Any?) {112 override func paste(_ sender: Any?) {
113 guard !isReadOnly else {
114 return
115 }
99 let pasteboard = UIPasteboard.general116 let pasteboard = UIPasteboard.general
100 let imageProviders = pasteboard.itemProviders.filter {117 let imageProviders = pasteboard.itemProviders.filter {
101 $0.canLoadObject(ofClass: UIImage.self)118 $0.canLoadObject(ofClass: UIImage.self)
...@@ -117,6 +134,9 @@ private final class ComposerTextView: UITextView {...@@ -117,6 +134,9 @@ private final class ComposerTextView: UITextView {
117 }134 }
118135
119 override func deleteBackward() {136 override func deleteBackward() {
137 guard !isReadOnly else {
138 return
139 }
120 guard selectedRange.length == 0, selectedRange.location > 0 else {140 guard selectedRange.length == 0, selectedRange.location > 0 else {
121 super.deleteBackward()141 super.deleteBackward()
122 return142 return
...@@ -160,9 +180,12 @@ private final class ComposerTextView: UITextView {...@@ -160,9 +180,12 @@ private final class ComposerTextView: UITextView {
160 }180 }
161181
162 group.notify(queue: .main) { [weak self] in182 group.notify(queue: .main) { [weak self] in
183 guard let self, !self.isReadOnly else {
184 return
185 }
163 let urls = images.compactMap { $0 }.compactMap(Self.writeTemporaryImage)186 let urls = images.compactMap { $0 }.compactMap(Self.writeTemporaryImage)
164 if !urls.isEmpty {187 if !urls.isEmpty {
165 self?.onPasteImages?(urls)188 self.onPasteImages?(urls)
166 }189 }
167 }190 }
168 }191 }
...@@ -175,6 +198,9 @@ private final class ComposerTextView: UITextView {...@@ -175,6 +198,9 @@ private final class ComposerTextView: UITextView {
175 }198 }
176199
177 override func cut(_ sender: Any?) {200 override func cut(_ sender: Any?) {
201 guard !isReadOnly else {
202 return
203 }
178 guard isEditable, selectedRange.length > 0 else {204 guard isEditable, selectedRange.length > 0 else {
179 return super.cut(sender)205 return super.cut(sender)
180 }206 }
...@@ -306,6 +332,7 @@ public final class T3ComposerEditorView: ExpoView, UITextViewDelegate, UITextDro...@@ -306,6 +332,7 @@ public final class T3ComposerEditorView: ExpoView, UITextViewDelegate, UITextDro
306 private var contentInsetVertical: CGFloat = 0332 private var contentInsetVertical: CGFloat = 0
307 private var shouldAutoFocus = false333 private var shouldAutoFocus = false
308 private var didAutoFocus = false334 private var didAutoFocus = false
335 private var isReadOnly = false
309 private var isApplyingControlledValue = false336 private var isApplyingControlledValue = false
310 private var nativeEventCount = 0337 private var nativeEventCount = 0
311 private var lastContentSize = CGSize.zero338 private var lastContentSize = CGSize.zero
...@@ -451,6 +478,11 @@ public final class T3ComposerEditorView: ExpoView, UITextViewDelegate, UITextDro...@@ -451,6 +478,11 @@ public final class T3ComposerEditorView: ExpoView, UITextViewDelegate, UITextDro
451 textView.isEditable = editable478 textView.isEditable = editable
452 }479 }
453480
481 func setReadOnly(_ readOnly: Bool) {
482 isReadOnly = readOnly
483 textView.isReadOnly = readOnly
484 }
485
454 func setScrollEnabled(_ scrollEnabled: Bool) {486 func setScrollEnabled(_ scrollEnabled: Bool) {
455 textView.isScrollEnabled = scrollEnabled487 textView.isScrollEnabled = scrollEnabled
456 }488 }
...@@ -504,13 +536,16 @@ public final class T3ComposerEditorView: ExpoView, UITextViewDelegate, UITextDro...@@ -504,13 +536,16 @@ public final class T3ComposerEditorView: ExpoView, UITextViewDelegate, UITextDro
504 replacementText text: String536 replacementText text: String
505 ) -> Bool {537 ) -> Bool {
506 restoreBaseTypingAttributes()538 restoreBaseTypingAttributes()
507 return true539 return !isReadOnly
508 }540 }
509541
510 public func textDroppableView(542 public func textDroppableView(
511 _ textDroppableView: UIView & UITextDroppable,543 _ textDroppableView: UIView & UITextDroppable,
512 proposalForDrop drop: UITextDropRequest544 proposalForDrop drop: UITextDropRequest
513 ) -> UITextDropProposal {545 ) -> UITextDropProposal {
546 guard !isReadOnly else {
547 return UITextDropProposal(operation: .cancel)
548 }
514 guard droppedImageProviders(in: drop) != nil else {549 guard droppedImageProviders(in: drop) != nil else {
515 return drop.suggestedProposal550 return drop.suggestedProposal
516 }551 }
...@@ -527,6 +562,9 @@ public final class T3ComposerEditorView: ExpoView, UITextViewDelegate, UITextDro...@@ -527,6 +562,9 @@ public final class T3ComposerEditorView: ExpoView, UITextViewDelegate, UITextDro
527 _ textDroppableView: UIView & UITextDroppable,562 _ textDroppableView: UIView & UITextDroppable,
528 willPerformDrop drop: UITextDropRequest563 willPerformDrop drop: UITextDropRequest
529 ) {564 ) {
565 guard !isReadOnly else {
566 return
567 }
530 guard let imageProviders = droppedImageProviders(in: drop) else {568 guard let imageProviders = droppedImageProviders(in: drop) else {
531 return569 return
532 }570 }
apps/mobile/package.json+4-1
...@@ -55,6 +55,7 @@...@@ -55,6 +55,7 @@
55 "@noble/curves": "catalog:",55 "@noble/curves": "catalog:",
56 "@noble/hashes": "catalog:",56 "@noble/hashes": "catalog:",
57 "@pierre/diffs": "catalog:",57 "@pierre/diffs": "catalog:",
58 "@react-native-ai/apple": "0.12.0",
58 "@react-native-menu/menu": "^2.0.0",59 "@react-native-menu/menu": "^2.0.0",
59 "@react-navigation/elements": "2.9.26",60 "@react-navigation/elements": "2.9.26",
60 "@react-navigation/native": "7.3.4",61 "@react-navigation/native": "7.3.4",
...@@ -75,6 +76,7 @@...@@ -75,6 +76,7 @@
75 "effect": "catalog:",76 "effect": "catalog:",
76 "expo": "~57.0.18",77 "expo": "~57.0.18",
77 "expo-asset": "~57.0.15",78 "expo-asset": "~57.0.15",
79 "expo-audio": "~57.0.4",
78 "expo-auth-session": "~57.0.10",80 "expo-auth-session": "~57.0.10",
79 "expo-blur": "~57.0.2",81 "expo-blur": "~57.0.2",
80 "expo-build-properties": "~57.0.15",82 "expo-build-properties": "~57.0.15",
...@@ -143,7 +145,8 @@...@@ -143,7 +145,8 @@
143 "autolinking": {145 "autolinking": {
144 "buildFromSource": [146 "buildFromSource": [
145 "react-native-screens",147 "react-native-screens",
146 "@react-native-menu/menu"148 "@react-native-menu/menu",
149 "expo-audio"
147 ]150 ]
148 }151 }
149 },152 },
apps/mobile/src/components/GlassSurface.tsx+5-1
...@@ -1,5 +1,5 @@...@@ -1,5 +1,5 @@
1import { GlassView, isGlassEffectAPIAvailable } from "expo-glass-effect";1import { GlassView, isGlassEffectAPIAvailable } from "expo-glass-effect";
2import type { ReactNode } from "react";2import type { ReactNode, Ref } from "react";
3import {3import {
4 Platform,4 Platform,
5 useColorScheme,5 useColorScheme,
...@@ -20,6 +20,7 @@ const ThemedGlassView = withUniwind(GlassView, {...@@ -20,6 +20,7 @@ const ThemedGlassView = withUniwind(GlassView, {
20});20});
2121
22interface GlassSurfaceProps extends ViewProps {22interface GlassSurfaceProps extends ViewProps {
23 readonly ref?: Ref<View>;
23 readonly children: ReactNode;24 readonly children: ReactNode;
24 readonly glassEffectStyle?: "clear" | "regular" | "none";25 readonly glassEffectStyle?: "clear" | "regular" | "none";
25 readonly tintColor?: ColorValue;26 readonly tintColor?: ColorValue;
...@@ -32,6 +33,7 @@ interface GlassSurfaceProps extends ViewProps {...@@ -32,6 +33,7 @@ interface GlassSurfaceProps extends ViewProps {
32}33}
3334
34export function GlassSurface({35export function GlassSurface({
36 ref,
35 children,37 children,
36 glassEffectStyle = "regular",38 glassEffectStyle = "regular",
37 chrome = "default",39 chrome = "default",
...@@ -68,6 +70,7 @@ export function GlassSurface({...@@ -68,6 +70,7 @@ export function GlassSurface({
68 return (70 return (
69 <ThemedGlassView71 <ThemedGlassView
70 {...props}72 {...props}
73 ref={ref}
71 className={cn(74 className={cn(
72 chrome === "none"75 chrome === "none"
73 ? "border-0 border-transparent bg-transparent"76 ? "border-0 border-transparent bg-transparent"
...@@ -90,6 +93,7 @@ export function GlassSurface({...@@ -90,6 +93,7 @@ export function GlassSurface({
90 return (93 return (
91 <View94 <View
92 {...props}95 {...props}
96 ref={ref}
93 className={cn(97 className={cn(
94 chrome === "none"98 chrome === "none"
95 ? "border-0 border-transparent bg-transparent"99 ? "border-0 border-transparent bg-transparent"
apps/mobile/src/features/threads/NewTaskDraftScreen.tsx+143-69
...@@ -12,6 +12,7 @@ import {...@@ -12,6 +12,7 @@ import {
12 KeyboardStickyView,12 KeyboardStickyView,
13 useKeyboardState,13 useKeyboardState,
14} from "react-native-keyboard-controller";14} from "react-native-keyboard-controller";
15import Animated from "react-native-reanimated";
15import { useSafeAreaInsets } from "react-native-safe-area-context";16import { useSafeAreaInsets } from "react-native-safe-area-context";
16import { useUniwindTheme } from "../../lib/useUniwindTheme";17import { useUniwindTheme } from "../../lib/useUniwindTheme";
17import { useFontFamily } from "../../lib/useFontFamily";18import { useFontFamily } from "../../lib/useFontFamily";
...@@ -34,9 +35,17 @@ import { ComposerAttachmentStrip } from "../../components/ComposerAttachmentStri...@@ -34,9 +35,17 @@ import { ComposerAttachmentStrip } from "../../components/ComposerAttachmentStri
34import { ProviderIcon } from "../../components/ProviderIcon";35import { ProviderIcon } from "../../components/ProviderIcon";
35import { SymbolView } from "../../components/AppSymbol";36import { SymbolView } from "../../components/AppSymbol";
36import { AppText as Text } from "../../components/AppText";37import { AppText as Text } from "../../components/AppText";
37import { ComposerSurface } from "./ThreadComposer";38import { COMPOSER_LAYOUT_TRANSITION, ComposerSurface } from "./ThreadComposer";
38import { ComposerCommandPopover } from "./ComposerCommandPopover";39import { ComposerCommandPopover } from "./ComposerCommandPopover";
39import { useComposerCommandMenu } from "./use-composer-command-menu";40import { useComposerCommandMenu } from "./use-composer-command-menu";
41import {
42 ComposerDictationCancelAction,
43 ComposerDictationPrimaryAction,
44 ComposerDictationStatus,
45 ComposerDictationToolbar,
46} from "../voice-input/ComposerDictationControl";
47import { useVoiceInputController } from "../voice-input/useVoiceInputController";
48import { resolveVoiceComposerPresentation } from "../voice-input/voiceInputPresentation";
40import {49import {
41 useThreadSettingsSheetPresentation,50 useThreadSettingsSheetPresentation,
42 type NavigationWithFinishTransitioning,51 type NavigationWithFinishTransitioning,
...@@ -240,6 +249,7 @@ export function NewTaskDraftScreen(props: {...@@ -240,6 +249,7 @@ export function NewTaskDraftScreen(props: {
240 // Cancel would otherwise abandon the screen while the task still starts.249 // Cancel would otherwise abandon the screen while the task still starts.
241 const composerMenu = useComposerCommandMenu({250 const composerMenu = useComposerCommandMenu({
242 draftMessage: flow.prompt,251 draftMessage: flow.prompt,
252 ownerKey: flow.draftKey,
243 environmentId: selectedProject?.environmentId ?? null,253 environmentId: selectedProject?.environmentId ?? null,
244 projectCwd:254 projectCwd:
245 (flow.workspaceMode === "worktree"255 (flow.workspaceMode === "worktree"
...@@ -251,6 +261,19 @@ export function NewTaskDraftScreen(props: {...@@ -251,6 +261,19 @@ export function NewTaskDraftScreen(props: {
251 onChangeDraftMessage: flow.setPrompt,261 onChangeDraftMessage: flow.setPrompt,
252 onUpdateInteractionMode: flow.planModeEnabled ? flow.setInteractionMode : undefined,262 onUpdateInteractionMode: flow.planModeEnabled ? flow.setInteractionMode : undefined,
253 });263 });
264 const voiceInput = useVoiceInputController({
265 ownerKey: flow.draftKey,
266 draftMessage: flow.prompt,
267 selection: composerMenu.selection,
268 disabled: isIncomingShareTransferPending || isImportingShare || flow.submitting,
269 onChangeDraftMessage: flow.setPrompt,
270 onChangeSelection: composerMenu.onSelectionChange,
271 });
272 const voicePresentation = resolveVoiceComposerPresentation(
273 voiceInput.state,
274 voiceInput.elapsedSeconds,
275 );
276 const isVoiceInputPresented = voicePresentation.statusLabel !== null;
254 usePreventRemove(277 usePreventRemove(
255 (isIncomingShareTransferPending && !isProjectPickerReturnActive) ||278 (isIncomingShareTransferPending && !isProjectPickerReturnActive) ||
256 isCancellingShareImport ||279 isCancellingShareImport ||
...@@ -693,7 +716,7 @@ export function NewTaskDraftScreen(props: {...@@ -693,7 +716,7 @@ export function NewTaskDraftScreen(props: {
693 const showBranchLoading = flow.branchesLoading && flow.availableBranches.length === 0;716 const showBranchLoading = flow.branchesLoading && flow.availableBranches.length === 0;
694717
695 async function handlePickImages(): Promise<void> {718 async function handlePickImages(): Promise<void> {
696 if (isComposerInteractionLocked) {719 if (isComposerInteractionLocked || voiceInput.isBusy) {
697 return;720 return;
698 }721 }
699 const result = await pickComposerImages({ existingCount: flow.attachments.length });722 const result = await pickComposerImages({ existingCount: flow.attachments.length });
...@@ -710,7 +733,7 @@ export function NewTaskDraftScreen(props: {...@@ -710,7 +733,7 @@ export function NewTaskDraftScreen(props: {
710 }733 }
711734
712 async function handlePickFiles(): Promise<void> {735 async function handlePickFiles(): Promise<void> {
713 if (isComposerInteractionLocked) {736 if (isComposerInteractionLocked || voiceInput.isBusy) {
714 return;737 return;
715 }738 }
716 const maxBytes =739 const maxBytes =
...@@ -755,6 +778,7 @@ export function NewTaskDraftScreen(props: {...@@ -755,6 +778,7 @@ export function NewTaskDraftScreen(props: {
755 );778 );
756779
757 async function handleStart(): Promise<void> {780 async function handleStart(): Promise<void> {
781 if (voiceInput.blocksSubmission) return;
758 const selectedProject = flow.selectedProject;782 const selectedProject = flow.selectedProject;
759 const draftKey = flow.draftKey;783 const draftKey = flow.draftKey;
760 if (!selectedProject || !draftKey) {784 if (!selectedProject || !draftKey) {
...@@ -941,6 +965,7 @@ export function NewTaskDraftScreen(props: {...@@ -941,6 +965,7 @@ export function NewTaskDraftScreen(props: {
941 isIncomingShareReady &&965 isIncomingShareReady &&
942 !isImportingShare &&966 !isImportingShare &&
943 !flow.submitting &&967 !flow.submitting &&
968 !voiceInput.blocksSubmission &&
944 !(flow.workspaceMode === "worktree" && !flow.selectedBranchName);969 !(flow.workspaceMode === "worktree" && !flow.selectedBranchName);
945 const promptEditor = (970 const promptEditor = (
946 <ComposerEditor971 <ComposerEditor
...@@ -949,6 +974,7 @@ export function NewTaskDraftScreen(props: {...@@ -949,6 +974,7 @@ export function NewTaskDraftScreen(props: {
949 // Focusing is a user action, so presenting the form sheet has one motion.974 // Focusing is a user action, so presenting the form sheet has one motion.
950 autoFocus={false}975 autoFocus={false}
951 editable={!isComposerInteractionLocked}976 editable={!isComposerInteractionLocked}
977 readOnly={voiceInput.freezesEditor}
952 multiline978 multiline
953 scrollEnabled979 scrollEnabled
954 value={flow.prompt}980 value={flow.prompt}
...@@ -1028,7 +1054,7 @@ export function NewTaskDraftScreen(props: {...@@ -1028,7 +1054,7 @@ export function NewTaskDraftScreen(props: {
1028 <ComposerInlineControl1054 <ComposerInlineControl
1029 accessibilityLabel={`Environment: ${selectedEnvironmentLabel}`}1055 accessibilityLabel={`Environment: ${selectedEnvironmentLabel}`}
1030 chevronDirection="right"1056 chevronDirection="right"
1031 disabled={isComposerInteractionLocked}1057 disabled={isComposerInteractionLocked || voiceInput.isBusy}
1032 icon="desktopcomputer"1058 icon="desktopcomputer"
1033 label={`on ${selectedEnvironmentLabel}`}1059 label={`on ${selectedEnvironmentLabel}`}
1034 maxWidth={260}1060 maxWidth={260}
...@@ -1063,7 +1089,7 @@ export function NewTaskDraftScreen(props: {...@@ -1063,7 +1089,7 @@ export function NewTaskDraftScreen(props: {
1063 <ComposerInlineControl1089 <ComposerInlineControl
1064 accessibilityHint={`Switches to ${flow.workspaceMode === "local" ? "a new worktree" : "the current checkout"}`}1090 accessibilityHint={`Switches to ${flow.workspaceMode === "local" ? "a new worktree" : "the current checkout"}`}
1065 accessibilityLabel={workspaceLabel}1091 accessibilityLabel={workspaceLabel}
1066 disabled={isComposerInteractionLocked}1092 disabled={isComposerInteractionLocked || voiceInput.isBusy}
1067 iconNode={1093 iconNode={
1068 <NewTaskWorkspaceIcon1094 <NewTaskWorkspaceIcon
1069 workspaceMode={flow.workspaceMode}1095 workspaceMode={flow.workspaceMode}
...@@ -1090,7 +1116,7 @@ export function NewTaskDraftScreen(props: {...@@ -1090,7 +1116,7 @@ export function NewTaskDraftScreen(props: {
10901116
1091 const composerDock = (1117 const composerDock = (
1092 <View className="bg-sheet px-4 pt-1" style={{ paddingBottom: controlsBottomPadding }}>1118 <View className="bg-sheet px-4 pt-1" style={{ paddingBottom: controlsBottomPadding }}>
1093 {composerMenu.trigger && composerMenu.items.length > 0 ? (1119 {!voiceInput.isBusy && composerMenu.trigger && composerMenu.items.length > 0 ? (
1094 <View className="mb-2">1120 <View className="mb-2">
1095 <ComposerCommandPopover1121 <ComposerCommandPopover
1096 items={composerMenu.items}1122 items={composerMenu.items}
...@@ -1103,7 +1129,6 @@ export function NewTaskDraftScreen(props: {...@@ -1103,7 +1129,6 @@ export function NewTaskDraftScreen(props: {
1103 <View className="pb-1">{workspaceControls}</View>1129 <View className="pb-1">{workspaceControls}</View>
11041130
1105 <ComposerSurface1131 <ComposerSurface
1106 animateLayout={false}
1107 style={{1132 style={{
1108 borderRadius: 26,1133 borderRadius: 26,
1109 minHeight: 140,1134 minHeight: 140,
...@@ -1119,73 +1144,115 @@ export function NewTaskDraftScreen(props: {...@@ -1119,73 +1144,115 @@ export function NewTaskDraftScreen(props: {
1119 attachments={flow.attachments}1144 attachments={flow.attachments}
1120 imageBorderRadius={16}1145 imageBorderRadius={16}
1121 imageSize={72}1146 imageSize={72}
1122 onRemove={isComposerInteractionLocked ? () => undefined : flow.removeAttachment}1147 onRemove={
1148 isComposerInteractionLocked || voiceInput.isBusy
1149 ? () => undefined
1150 : flow.removeAttachment
1151 }
1123 />1152 />
1124 </View>1153 </View>
1125 ) : null}1154 ) : null}
11261155
1127 {promptEditor}1156 {promptEditor}
1157 <View className="h-1" />
11281158
1129 <ComposerToolbarRow paddingBottom={0} paddingHorizontal={0} paddingTop={4}>1159 <Animated.View layout={COMPOSER_LAYOUT_TRANSITION} collapsable={false}>
1130 <ComposerToolbarScroller contentPaddingRight={8} fadeSurface="sheet">1160 <ComposerDictationToolbar showsDictation={isVoiceInputPresented}>
1131 <ComposerToolbarButton1161 <ComposerToolbarRow paddingBottom={0} paddingHorizontal={0} paddingTop={0}>
1132 accessibilityLabel="Add attachment"1162 <ComposerDictationCancelAction
1133 disabled={isComposerInteractionLocked}1163 presentation={voicePresentation}
1134 icon="plus"1164 onCancel={voiceInput.cancel}
1135 onPress={() => {
1136 if (selectedEnvironmentServerConfig?.environment.capabilities.fileAttachments) {
1137 Alert.alert("Add attachment", undefined, [
1138 { text: "Photos", onPress: () => void handlePickImages() },
1139 { text: "Files", onPress: () => void handlePickFiles() },
1140 { text: "Cancel", style: "cancel" },
1141 ]);
1142 return;
1143 }
1144 void handlePickImages();
1145 }}
1146 showChevron={false}
1147 />
1148 <ComposerInlineControl
1149 accessibilityLabel="Model and reasoning settings"
1150 disabled={isComposerInteractionLocked}
1151 emphasized
1152 iconNode={
1153 <ProviderIcon provider={flow.selectedModelOption?.providerDriver} size={16} />
1154 }
1155 label={flow.selectedModelOption?.label ?? "Choose model"}
1156 maxWidth={152}
1157 onPress={settingsSheetPresentation.open}
1158 />
1159 {flow.planModeEnabled ? (
1160 <ComposerInlineControl
1161 accessibilityHint={`Switches to ${flow.interactionMode === "plan" ? "Build" : "Plan"} mode`}
1162 accessibilityLabel={`Interaction mode: ${flow.interactionMode === "plan" ? "Plan" : "Build"}`}
1163 disabled={isComposerInteractionLocked}
1164 emphasized
1165 icon={
1166 flow.interactionMode === "plan"
1167 ? { ios: "list.bullet.clipboard", android: "auto_awesome" }
1168 : { ios: "hammer", android: "construction" }
1169 }
1170 label={flow.interactionMode === "plan" ? "Plan" : "Build"}
1171 onPress={() =>
1172 flow.setInteractionMode(flow.interactionMode === "plan" ? "default" : "plan")
1173 }
1174 showChevron={false}
1175 />1165 />
1176 ) : null}1166 {isVoiceInputPresented ? (
1177 </ComposerToolbarScroller>1167 <ComposerDictationStatus
1178 <ComposerToolbarButton1168 audioLevels={voiceInput.audioLevels}
1179 accessibilityLabel={1169 elapsedSeconds={voiceInput.elapsedSeconds}
1180 flow.submitting ? "Starting task" : environmentConnected ? "Start task" : "Queue task"1170 phase={voiceInput.state.phase}
1181 }1171 presentation={voicePresentation}
1182 disabled={!canStart}1172 onDismissError={voiceInput.cancel}
1183 icon={environmentConnected ? "arrow.up" : "tray.and.arrow.up"}1173 />
1184 onPress={() => void handleStart()}1174 ) : (
1185 showChevron={false}1175 <ComposerToolbarScroller contentPaddingRight={8} fadeSurface="sheet">
1186 variant="primary"1176 <ComposerToolbarButton
1187 />1177 accessibilityLabel="Add attachment"
1188 </ComposerToolbarRow>1178 disabled={isComposerInteractionLocked}
1179 icon="plus"
1180 onPress={() => {
1181 if (
1182 selectedEnvironmentServerConfig?.environment.capabilities.fileAttachments
1183 ) {
1184 Alert.alert("Add attachment", undefined, [
1185 { text: "Photos", onPress: () => void handlePickImages() },
1186 { text: "Files", onPress: () => void handlePickFiles() },
1187 { text: "Cancel", style: "cancel" },
1188 ]);
1189 return;
1190 }
1191 void handlePickImages();
1192 }}
1193 showChevron={false}
1194 />
1195 <ComposerInlineControl
1196 accessibilityLabel="Model and reasoning settings"
1197 disabled={isComposerInteractionLocked}
1198 emphasized
1199 iconNode={
1200 <ProviderIcon provider={flow.selectedModelOption?.providerDriver} size={16} />
1201 }
1202 label={flow.selectedModelOption?.label ?? "Choose model"}
1203 maxWidth={152}
1204 onPress={settingsSheetPresentation.open}
1205 />
1206 {flow.planModeEnabled ? (
1207 <ComposerInlineControl
1208 accessibilityHint={`Switches to ${flow.interactionMode === "plan" ? "Build" : "Plan"} mode`}
1209 accessibilityLabel={`Interaction mode: ${flow.interactionMode === "plan" ? "Plan" : "Build"}`}
1210 disabled={isComposerInteractionLocked}
1211 emphasized
1212 icon={
1213 flow.interactionMode === "plan"
1214 ? { ios: "list.bullet.clipboard", android: "auto_awesome" }
1215 : { ios: "hammer", android: "construction" }
1216 }
1217 label={flow.interactionMode === "plan" ? "Plan" : "Build"}
1218 onPress={() =>
1219 flow.setInteractionMode(
1220 flow.interactionMode === "plan" ? "default" : "plan",
1221 )
1222 }
1223 showChevron={false}
1224 />
1225 ) : null}
1226 </ComposerToolbarScroller>
1227 )}
1228 <ComposerDictationPrimaryAction
1229 state={voiceInput.state}
1230 presentation={voicePresentation}
1231 isAvailable={voiceInput.isAvailable}
1232 disabled={isIncomingShareTransferPending || isImportingShare || flow.submitting}
1233 onStart={voiceInput.start}
1234 onConfirm={voiceInput.stop}
1235 onCancel={voiceInput.cancel}
1236 />
1237 {voicePresentation.showsSend ? (
1238 <ComposerToolbarButton
1239 accessibilityLabel={
1240 flow.submitting
1241 ? "Starting task"
1242 : environmentConnected
1243 ? "Start task"
1244 : "Queue task"
1245 }
1246 disabled={!canStart}
1247 icon={environmentConnected ? "arrow.up" : "tray.and.arrow.up"}
1248 onPress={() => void handleStart()}
1249 showChevron={false}
1250 variant="primary"
1251 />
1252 ) : null}
1253 </ComposerToolbarRow>
1254 </ComposerDictationToolbar>
1255 </Animated.View>
1189 </ComposerSurface>1256 </ComposerSurface>
1190 </View>1257 </View>
1191 );1258 );
...@@ -1226,10 +1293,17 @@ export function NewTaskDraftScreen(props: {...@@ -1226,10 +1293,17 @@ export function NewTaskDraftScreen(props: {
12261293
1227 {heroViewport}1294 {heroViewport}
1228 <KeyboardStickyView1295 <KeyboardStickyView
1229 style={{ position: "absolute", bottom: 0, left: 0, right: 0 }}1296 pointerEvents="box-none"
1297 style={{ position: "absolute", top: 0, bottom: 0, left: 0, right: 0 }}
1230 offset={{ closed: 0, opened: keyboardOpenedOffset }}1298 offset={{ closed: 0, opened: keyboardOpenedOffset }}
1231 >1299 >
1232 {composerDock}1300 <Animated.View
1301 layout={COMPOSER_LAYOUT_TRANSITION}
1302 pointerEvents="box-none"
1303 style={{ position: "absolute", bottom: 0, left: 0, right: 0 }}
1304 >
1305 {composerDock}
1306 </Animated.View>
1233 </KeyboardStickyView>1307 </KeyboardStickyView>
1234 </View>1308 </View>
1235 );1309 );
apps/mobile/src/features/threads/ThreadComposer.tsx+210-127
...@@ -35,6 +35,7 @@ import Animated, {...@@ -35,6 +35,7 @@ import Animated, {
35 FadeOut,35 FadeOut,
36 FadeOutDown,36 FadeOutDown,
37 LinearTransition,37 LinearTransition,
38 ReduceMotion,
38 useAnimatedStyle,39 useAnimatedStyle,
39 useSharedValue,40 useSharedValue,
40 withTiming,41 withTiming,
...@@ -62,6 +63,15 @@ import type { RemoteClientConnectionState } from "../../lib/connection";...@@ -62,6 +63,15 @@ import type { RemoteClientConnectionState } from "../../lib/connection";
62import { resolveProviderOptionDescriptors } from "../../lib/providerOptions";63import { resolveProviderOptionDescriptors } from "../../lib/providerOptions";
63import { ComposerCommandPopover } from "./ComposerCommandPopover";64import { ComposerCommandPopover } from "./ComposerCommandPopover";
64import { useComposerCommandMenu } from "./use-composer-command-menu";65import { useComposerCommandMenu } from "./use-composer-command-menu";
66import {
67 ComposerDictationCancelAction,
68 ComposerDictationDraftContent,
69 ComposerDictationPrimaryAction,
70 ComposerDictationStatus,
71 ComposerDictationToolbar,
72} from "../voice-input/ComposerDictationControl";
73import { useVoiceInputController } from "../voice-input/useVoiceInputController";
74import { resolveVoiceComposerPresentation } from "../voice-input/voiceInputPresentation";
65import {75import {
66 type ExistingThreadSettingsRouteSession,76 type ExistingThreadSettingsRouteSession,
67 useExistingThreadSettingsRoutePresentation,77 useExistingThreadSettingsRoutePresentation,
...@@ -135,12 +145,14 @@ export const COMPOSER_TRANSITION_DURATION_MS = 220;...@@ -135,12 +145,14 @@ export const COMPOSER_TRANSITION_DURATION_MS = 220;
135export const COMPOSER_LAYOUT_TRANSITION =145export const COMPOSER_LAYOUT_TRANSITION =
136 Platform.OS === "android"146 Platform.OS === "android"
137 ? undefined147 ? undefined
138 : LinearTransition.duration(COMPOSER_TRANSITION_DURATION_MS);148 : LinearTransition.duration(COMPOSER_TRANSITION_DURATION_MS).reduceMotion(ReduceMotion.System);
149
150const AnimatedGlassSurface = Animated.createAnimatedComponent(GlassSurface);
139151
140export function ComposerSurface(props: {152export function ComposerSurface(props: {
141 readonly children: ReactNode;153 readonly children: ReactNode;
142 readonly style: ViewStyle;154 readonly style: ViewStyle;
143 /** Existing thread composers morph between pill and card layouts. */155 /** Morphs between the compact and expanded composer layouts. */
144 readonly animateLayout?: boolean;156 readonly animateLayout?: boolean;
145}) {157}) {
146 const targetBorderRadius =158 const targetBorderRadius =
...@@ -151,19 +163,21 @@ export function ComposerSurface(props: {...@@ -151,19 +163,21 @@ export function ComposerSurface(props: {
151 animatedBorderRadius.value = shouldAnimate163 animatedBorderRadius.value = shouldAnimate
152 ? withTiming(targetBorderRadius, {164 ? withTiming(targetBorderRadius, {
153 duration: COMPOSER_TRANSITION_DURATION_MS,165 duration: COMPOSER_TRANSITION_DURATION_MS,
166 reduceMotion: ReduceMotion.System,
154 })167 })
155 : targetBorderRadius;168 : targetBorderRadius;
156 }, [animatedBorderRadius, shouldAnimate, targetBorderRadius]);169 }, [animatedBorderRadius, shouldAnimate, targetBorderRadius]);
157 const animatedShapeStyle = useAnimatedStyle(() => ({170 const animatedShapeStyle = useAnimatedStyle(() => ({
158 borderRadius: animatedBorderRadius.value,171 borderRadius: animatedBorderRadius.value,
159 }));172 }));
173 const layoutTransition = shouldAnimate ? COMPOSER_LAYOUT_TRANSITION : undefined;
160174
161 // Clipping here keeps the expanded toolbar inside the glass while the175 // Each native frame follows the same transition. Animating only the outer
162 // bottom-anchored host owns the geometry transition.176 // clip leaves the glass and content at their final height on the first frame.
163 return (177 return (
164 <Animated.View178 <Animated.View
165 className="shadow-[0_6px_28px] shadow-adaptive-black-a15-a35"179 className="shadow-[0_6px_28px] shadow-adaptive-black-a15-a35"
166 layout={props.animateLayout === false ? undefined : COMPOSER_LAYOUT_TRANSITION}180 layout={layoutTransition}
167 style={[181 style={[
168 animatedShapeStyle,182 animatedShapeStyle,
169 {183 {
...@@ -173,7 +187,7 @@ export function ComposerSurface(props: {...@@ -173,7 +187,7 @@ export function ComposerSurface(props: {
173 },187 },
174 ]}188 ]}
175 >189 >
176 <GlassSurface190 <AnimatedGlassSurface
177 chrome="none"191 chrome="none"
178 fallbackClassName="border border-border bg-card-translucent"192 fallbackClassName="border border-border bg-card-translucent"
179 glassEffectStyle="regular"193 glassEffectStyle="regular"
...@@ -181,15 +195,18 @@ export function ComposerSurface(props: {...@@ -181,15 +195,18 @@ export function ComposerSurface(props: {
181 // Keep native glass out of the interactive content's layout path.195 // Keep native glass out of the interactive content's layout path.
182 pointerEvents="none"196 pointerEvents="none"
183 tintColor="transparent"197 tintColor="transparent"
184 style={{198 layout={layoutTransition}
185 position: "absolute",199 style={[{ position: "absolute", inset: 0 }, animatedShapeStyle]}
186 inset: 0,
187 borderRadius: targetBorderRadius,
188 }}
189 >200 >
190 {null}201 {null}
191 </GlassSurface>202 </AnimatedGlassSurface>
192 <View style={props.style}>{props.children}</View>203 <Animated.View
204 collapsable={false}
205 layout={layoutTransition}
206 style={[props.style, animatedShapeStyle]}
207 >
208 {props.children}
209 </Animated.View>
193 </Animated.View>210 </Animated.View>
194 );211 );
195}212}
...@@ -297,13 +314,59 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer...@@ -297,13 +314,59 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer
297314
298 const [previewImageUri, setPreviewImageUri] = useState<string | null>(null);315 const [previewImageUri, setPreviewImageUri] = useState<string | null>(null);
299 const hasContent = props.draftMessage.trim().length > 0 || props.draftAttachments.length > 0;316 const hasContent = props.draftMessage.trim().length > 0 || props.draftAttachments.length > 0;
300 // Opening and presentation count as active so the composer stays expanded317 const showStopAction =
301 // while focus moves between its native editor and the settings picker.318 props.selectedThread.session?.status === "running" ||
319 props.selectedThread.session?.status === "starting";
320
321 const sendLabel =
322 props.connectionState !== "connected" || props.queueCount > 0 ? "Queue" : "Send";
323 const currentModelSelection = props.selectedThread.modelSelection;
324 const currentRuntimeMode = props.selectedThread.runtimeMode;
325 const connectionStatus = composerConnectionStatus({
326 connectionError: props.connectionError,
327 connectionState: props.connectionState,
328 environmentLabel: props.environmentLabel,
329 threadSyncPhase: props.threadSyncPhase,
330 });
331 const selectedProviderStatus = useMemo(() => {
332 if (!props.serverConfig) return null;
333 return (
334 props.serverConfig.providers.find(
335 (p) => p.instanceId === props.selectedThread.modelSelection.instanceId,
336 ) ?? null
337 );
338 }, [props.serverConfig, props.selectedThread.modelSelection.instanceId]);
339 const composerOwnerKey = scopedThreadKey(props.environmentId, props.selectedThread.id);
340
341 const composerMenu = useComposerCommandMenu({
342 draftMessage: props.draftMessage,
343 ownerKey: composerOwnerKey,
344 environmentId: props.environmentId,
345 projectCwd: props.projectCwd,
346 selectedProviderStatus,
347 hasThread: true,
348 onChangeDraftMessage: props.onChangeDraftMessage,
349 onUpdateInteractionMode: props.onUpdateInteractionMode,
350 });
351 const voiceInput = useVoiceInputController({
352 ownerKey: composerOwnerKey,
353 draftMessage: props.draftMessage,
354 selection: composerMenu.selection,
355 onChangeDraftMessage: props.onChangeDraftMessage,
356 onChangeSelection: composerMenu.onSelectionChange,
357 });
358 const voicePresentation = resolveVoiceComposerPresentation(
359 voiceInput.state,
360 voiceInput.elapsedSeconds,
361 );
362 const isVoiceInputPresented = voicePresentation.statusLabel !== null;
363 // An open draft stays visible; only a collapsed composer becomes a voice strip.
302 const isExpanded = isFocused || settingsSheetPresentation.isActive;364 const isExpanded = isFocused || settingsSheetPresentation.isActive;
303 const canSend = hasContent;365 const showsCompactDictation = isVoiceInputPresented && !isExpanded;
366 const isToolbarVisible = isExpanded || isVoiceInputPresented;
367 const canSend = hasContent && !voiceInput.blocksSubmission;
304368
305 // Keep the parent's feed inset synchronized while the settings sheet keeps369 // Keep the feed inset aligned with the card or compact dictation strip.
306 // the composer expanded between editor focus events.
307 useEffect(() => {370 useEffect(() => {
308 onExpandedChange?.(isExpanded);371 onExpandedChange?.(isExpanded);
309 }, [isExpanded, onExpandedChange]);372 }, [isExpanded, onExpandedChange]);
...@@ -337,41 +400,10 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer...@@ -337,41 +400,10 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer
337 }400 }
338 onEditorFocusChange?.(false);401 onEditorFocusChange?.(false);
339 }, [onEditorFocusChange, onExpandedChange, settingsSheetPresentation.isActive]);402 }, [onEditorFocusChange, onExpandedChange, settingsSheetPresentation.isActive]);
340 const showStopAction =
341 props.selectedThread.session?.status === "running" ||
342 props.selectedThread.session?.status === "starting";
343
344 const sendLabel =
345 props.connectionState !== "connected" || props.queueCount > 0 ? "Queue" : "Send";
346 const currentModelSelection = props.selectedThread.modelSelection;
347 const currentRuntimeMode = props.selectedThread.runtimeMode;
348 const connectionStatus = composerConnectionStatus({
349 connectionError: props.connectionError,
350 connectionState: props.connectionState,
351 environmentLabel: props.environmentLabel,
352 threadSyncPhase: props.threadSyncPhase,
353 });
354 const selectedProviderStatus = useMemo(() => {
355 if (!props.serverConfig) return null;
356 return (
357 props.serverConfig.providers.find(
358 (p) => p.instanceId === props.selectedThread.modelSelection.instanceId,
359 ) ?? null
360 );
361 }, [props.serverConfig, props.selectedThread.modelSelection.instanceId]);
362
363 const composerMenu = useComposerCommandMenu({
364 draftMessage: props.draftMessage,
365 environmentId: props.environmentId,
366 projectCwd: props.projectCwd,
367 selectedProviderStatus,
368 hasThread: true,
369 onChangeDraftMessage: props.onChangeDraftMessage,
370 onUpdateInteractionMode: props.onUpdateInteractionMode,
371 });
372 const { onSendMessage } = props;403 const { onSendMessage } = props;
373404
374 const handleSend = useCallback(async () => {405 const handleSend = useCallback(async () => {
406 if (voiceInput.blocksSubmission) return;
375 const threadKey = scopedThreadKey(props.environmentId, props.selectedThread.id);407 const threadKey = scopedThreadKey(props.environmentId, props.selectedThread.id);
376 if (inFlightThreadIdsRef.current.has(threadKey)) return;408 if (inFlightThreadIdsRef.current.has(threadKey)) return;
377 inFlightThreadIdsRef.current.add(threadKey);409 inFlightThreadIdsRef.current.add(threadKey);
...@@ -398,6 +430,7 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer...@@ -398,6 +430,7 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer
398 props.environmentLabel,430 props.environmentLabel,
399 props.selectedThread.id,431 props.selectedThread.id,
400 props.selectedThread.title,432 props.selectedThread.title,
433 voiceInput.blocksSubmission,
401 ]);434 ]);
402435
403 // ── Model menu ───────────────────────────────────────────436 // ── Model menu ───────────────────────────────────────────
...@@ -426,7 +459,7 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer...@@ -426,7 +459,7 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer
426 }),459 }),
427 [currentModelOption?.capabilities, currentModelSelection.options],460 [currentModelOption?.capabilities, currentModelSelection.options],
428 );461 );
429 const settingsOwnerId = scopedThreadKey(props.environmentId, props.selectedThread.id);462 const settingsOwnerId = composerOwnerKey;
430 const settingsRouteSession = useMemo<ExistingThreadSettingsRouteSession>(463 const settingsRouteSession = useMemo<ExistingThreadSettingsRouteSession>(
431 () => ({464 () => ({
432 ownerId: settingsOwnerId,465 ownerId: settingsOwnerId,
...@@ -512,7 +545,7 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer...@@ -512,7 +545,7 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer
512 className="relative w-full self-center"545 className="relative w-full self-center"
513 style={{ maxWidth: props.contentMaxWidth }}546 style={{ maxWidth: props.contentMaxWidth }}
514 >547 >
515 {composerMenu.trigger && composerMenu.items.length > 0 ? (548 {!voiceInput.isBusy && composerMenu.trigger && composerMenu.items.length > 0 ? (
516 <View className="absolute inset-x-0 bottom-full z-10 mb-2">549 <View className="absolute inset-x-0 bottom-full z-10 mb-2">
517 <ComposerCommandPopover550 <ComposerCommandPopover
518 items={composerMenu.items}551 items={composerMenu.items}
...@@ -547,31 +580,36 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer...@@ -547,31 +580,36 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer
547 borderRadius: 27,580 borderRadius: 27,
548 overflow: "hidden" as const,581 overflow: "hidden" as const,
549 paddingHorizontal: 14,582 paddingHorizontal: 14,
550 paddingVertical: 5,583 paddingVertical: showsCompactDictation ? 2 : 5,
551 }584 }
552 }585 }
553 >586 >
554 {/* Attachment strip — inside the card, above the text input */}587 <ComposerDictationDraftContent
555 {isExpanded ? (588 className={isExpanded ? undefined : "flex-row items-center"}
589 collapsed={showsCompactDictation}
590 >
591 {isExpanded ? (
592 <Animated.View
593 className={props.draftAttachments.length > 0 ? "pb-2.5" : undefined}
594 entering={FadeIn.duration(160)}
595 exiting={FadeOut.duration(120)}
596 >
597 <ComposerAttachmentStrip
598 attachments={props.draftAttachments}
599 onRemove={voiceInput.isBusy ? () => undefined : props.onRemoveDraftImage}
600 onPressImage={voiceInput.isBusy ? undefined : onPressImage}
601 />
602 </Animated.View>
603 ) : null}
556 <Animated.View604 <Animated.View
557 className={props.draftAttachments.length > 0 ? "pb-2.5" : undefined}605 className={isExpanded ? undefined : "min-w-0 flex-1"}
558 entering={FadeIn.duration(160)}606 layout={COMPOSER_LAYOUT_TRANSITION}
559 exiting={FadeOut.duration(120)}
560 >607 >
561 <ComposerAttachmentStrip
562 attachments={props.draftAttachments}
563 onRemove={props.onRemoveDraftImage}
564 onPressImage={onPressImage}
565 />
566 </Animated.View>
567 ) : null}
568
569 <View className={isExpanded ? undefined : "flex-row items-center"}>
570 <View className={isExpanded ? undefined : "min-w-0 flex-1"}>
571 <ComposerEditor608 <ComposerEditor
572 ref={inputRef}609 ref={inputRef}
573 multiline610 multiline
574 value={props.draftMessage}611 value={props.draftMessage}
612 readOnly={voiceInput.freezesEditor}
575 skills={selectedProviderStatus?.skills ?? []}613 skills={selectedProviderStatus?.skills ?? []}
576 selection={composerMenu.selection}614 selection={composerMenu.selection}
577 onChangeText={props.onChangeDraftMessage}615 onChangeText={props.onChangeDraftMessage}
...@@ -604,8 +642,8 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer...@@ -604,8 +642,8 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer
604 color: foregroundColor,642 color: foregroundColor,
605 }}643 }}
606 />644 />
607 </View>645 </Animated.View>
608 {!isExpanded && props.draftAttachments.length > 0 ? (646 {!isExpanded && !voiceInput.isBusy && props.draftAttachments.length > 0 ? (
609 <View className="flex-row gap-1 pl-1">647 <View className="flex-row gap-1 pl-1">
610 {props.draftAttachments.slice(0, 3).map((attachment) =>648 {props.draftAttachments.slice(0, 3).map((attachment) =>
611 attachment.type === "image" ? (649 attachment.type === "image" ? (
...@@ -637,8 +675,22 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer...@@ -637,8 +675,22 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer
637 ) : null}675 ) : null}
638 </View>676 </View>
639 ) : null}677 ) : null}
640 {!isExpanded ? (678 {!isExpanded && !voiceInput.isBusy ? (
641 <Animated.View entering={FadeIn.duration(180)} exiting={FadeOut.duration(100)}>679 <Animated.View
680 className="flex-row items-center gap-1.5"
681 entering={FadeIn.duration(180)}
682 exiting={FadeOut.duration(100)}
683 >
684 {voiceInput.isAvailable ? (
685 <ComposerDictationPrimaryAction
686 state={voiceInput.state}
687 presentation={voicePresentation}
688 isAvailable={voiceInput.isAvailable}
689 onStart={voiceInput.start}
690 onConfirm={voiceInput.stop}
691 onCancel={voiceInput.cancel}
692 />
693 ) : null}
642 {showStopAction ? (694 {showStopAction ? (
643 <ControlPill icon="stop.fill" variant="danger" onPress={props.onStopThread} />695 <ControlPill icon="stop.fill" variant="danger" onPress={props.onStopThread} />
644 ) : (696 ) : (
...@@ -651,14 +703,16 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer...@@ -651,14 +703,16 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer
651 )}703 )}
652 </Animated.View>704 </Animated.View>
653 ) : null}705 ) : null}
654 </View>706 {isExpanded ? <View className="h-1" /> : null}
655 <View707 </ComposerDictationDraftContent>
656 accessibilityElementsHidden={!isExpanded}708 <Animated.View
709 accessibilityElementsHidden={!isToolbarVisible}
657 collapsable={false}710 collapsable={false}
658 importantForAccessibility={isExpanded ? "auto" : "no-hide-descendants"}711 importantForAccessibility={isToolbarVisible ? "auto" : "no-hide-descendants"}
659 pointerEvents={isExpanded ? "auto" : "none"}712 layout={COMPOSER_LAYOUT_TRANSITION}
713 pointerEvents={isToolbarVisible ? "auto" : "none"}
660 style={714 style={
661 isExpanded715 isToolbarVisible
662 ? undefined716 ? undefined
663 : {717 : {
664 height: 0,718 height: 0,
...@@ -667,58 +721,87 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer...@@ -667,58 +721,87 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer
667 }721 }
668 }722 }
669 >723 >
670 <ComposerToolbarRow paddingBottom={0} paddingHorizontal={0} paddingTop={4}>724 <ComposerDictationToolbar
671 <View className="min-w-0 flex-1 flex-row items-center gap-2">725 showsDictation={isVoiceInputPresented}
672 <ComposerToolbarButton726 visible={isToolbarVisible}
673 accessibilityLabel="Add attachment"727 >
674 icon="plus"728 <ComposerToolbarRow paddingBottom={0} paddingHorizontal={0} paddingTop={0}>
675 onPress={() => {729 <ComposerDictationCancelAction
676 if (props.serverConfig?.environment.capabilities.fileAttachments) {730 presentation={voicePresentation}
677 Alert.alert("Add attachment", undefined, [731 onCancel={voiceInput.cancel}
678 { text: "Photos", onPress: () => void props.onPickDraftImages() },
679 { text: "Files", onPress: () => void props.onPickDraftFiles() },
680 { text: "Cancel", style: "cancel" },
681 ]);
682 return;
683 }
684 void props.onPickDraftImages();
685 }}
686 showChevron={false}
687 />732 />
688 <View className="min-w-0 flex-1" style={{ maxWidth: 152 }}>733 {isVoiceInputPresented ? (
689 <ComposerInlineControl734 <ComposerDictationStatus
690 accessibilityLabel="Model and reasoning settings"735 audioLevels={voiceInput.audioLevels}
691 emphasized736 elapsedSeconds={voiceInput.elapsedSeconds}
692 iconNode={737 phase={voiceInput.state.phase}
693 <ProviderIcon provider={currentModelOption?.providerDriver} size={16} />738 presentation={voicePresentation}
694 }739 onDismissError={voiceInput.cancel}
695 label={currentModelOption?.label ?? currentModelSelection.model}
696 maxWidth={152}
697 onPress={openSettings}
698 />740 />
699 </View>741 ) : (
700 </View>742 <View className="min-w-0 flex-1 flex-row items-center gap-2">
701 <View className="shrink-0 flex-row items-center gap-2">743 <ComposerToolbarButton
702 <ComposerToolbarButton744 accessibilityLabel="Add attachment"
703 accessibilityLabel={sendLabel}745 icon="plus"
704 icon="arrow.up"746 onPress={() => {
705 variant="primary"747 if (props.serverConfig?.environment.capabilities.fileAttachments) {
706 disabled={!canSend}748 Alert.alert("Add attachment", undefined, [
707 onPress={handleSend}749 { text: "Photos", onPress: () => void props.onPickDraftImages() },
708 showChevron={false}750 { text: "Files", onPress: () => void props.onPickDraftFiles() },
709 />751 { text: "Cancel", style: "cancel" },
710 {showStopAction ? (752 ]);
711 <ComposerToolbarButton753 return;
712 accessibilityLabel="Stop"754 }
713 icon="stop.fill"755 void props.onPickDraftImages();
714 variant="danger"756 }}
715 onPress={props.onStopThread}757 showChevron={false}
716 showChevron={false}758 />
759 <View className="min-w-0 flex-1" style={{ maxWidth: 152 }}>
760 <ComposerInlineControl
761 accessibilityLabel="Model and reasoning settings"
762 emphasized
763 iconNode={
764 <ProviderIcon provider={currentModelOption?.providerDriver} size={16} />
765 }
766 label={currentModelOption?.label ?? currentModelSelection.model}
767 maxWidth={152}
768 onPress={openSettings}
769 />
770 </View>
771 </View>
772 )}
773 <View className="shrink-0 flex-row items-center gap-2">
774 <ComposerDictationPrimaryAction
775 state={voiceInput.state}
776 presentation={voicePresentation}
777 isAvailable={voiceInput.isAvailable}
778 onStart={voiceInput.start}
779 onConfirm={voiceInput.stop}
780 onCancel={voiceInput.cancel}
717 />781 />
718 ) : null}782 {voicePresentation.showsSend ? (
719 </View>783 <ComposerToolbarButton
720 </ComposerToolbarRow>784 accessibilityLabel={sendLabel}
721 </View>785 icon="arrow.up"
786 variant="primary"
787 disabled={!canSend}
788 onPress={handleSend}
789 showChevron={false}
790 />
791 ) : null}
792 {showStopAction ? (
793 <ComposerToolbarButton
794 accessibilityLabel="Stop agent"
795 icon="stop.fill"
796 variant="danger"
797 onPress={props.onStopThread}
798 showChevron={false}
799 />
800 ) : null}
801 </View>
802 </ComposerToolbarRow>
803 </ComposerDictationToolbar>
804 </Animated.View>
722 </ComposerSurface>805 </ComposerSurface>
723806
724 {/* Queue count */}807 {/* Queue count */}
apps/mobile/src/features/threads/ThreadDetailScreen.tsx+16-3
...@@ -272,6 +272,14 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread...@@ -272,6 +272,14 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread
272 const selectedThreadKeyRef = useRef(selectedThreadKey);272 const selectedThreadKeyRef = useRef(selectedThreadKey);
273 const lastScrolledSubmittedMessageIdRef = useRef<MessageId | null>(null);273 const lastScrolledSubmittedMessageIdRef = useRef<MessageId | null>(null);
274 const [composerExpanded, setComposerExpanded] = useState(false);274 const [composerExpanded, setComposerExpanded] = useState(false);
275 const [composerFocused, setComposerFocused] = useState(false);
276 const handleComposerFocusChange = useCallback(
277 (focused: boolean) => {
278 setComposerFocused(focused);
279 handleOwnedInputFocusChange(focused);
280 },
281 [handleOwnedInputFocusChange],
282 );
275 const [anchorMessageId, setAnchorMessageId] = useState<MessageId | null>(null);283 const [anchorMessageId, setAnchorMessageId] = useState<MessageId | null>(null);
276 const [submittedMessageId, setSubmittedMessageId] = useState<MessageId | null>(null);284 const [submittedMessageId, setSubmittedMessageId] = useState<MessageId | null>(null);
277 const [endFollowEnabled, setEndFollowEnabled] = useState(true);285 const [endFollowEnabled, setEndFollowEnabled] = useState(true);
...@@ -282,7 +290,10 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread...@@ -282,7 +290,10 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread
282 // animation, so the composer would ride down flush to the screen edge and290 // animation, so the composer would ride down flush to the screen edge and
283 // then snap up into the inset. On iOS blur precedes the hide, so the291 // then snap up into the inset. On iOS blur precedes the hide, so the
284 // focus-keyed inset is already in place while the composer rides down.292 // focus-keyed inset is already in place while the composer rides down.
285 const composerBottomInset = (Platform.OS === "android" ? isKeyboardVisible : composerExpanded)293 // Dictation keeps that focus while the composer switches to its compact pill.
294 const composerBottomInset = (
295 Platform.OS === "android" ? isKeyboardVisible : composerExpanded || composerFocused
296 )
286 ? 0297 ? 0
287 : Math.max(insets.bottom, 12);298 : Math.max(insets.bottom, 12);
288 const contentPresentationKind = props.contentPresentation.kind;299 const contentPresentationKind = props.contentPresentation.kind;
...@@ -508,7 +519,9 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread...@@ -508,7 +519,9 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread
508519
509 useLayoutEffect(() => {520 useLayoutEffect(() => {
510 selectedThreadKeyRef.current = selectedThreadKey;521 selectedThreadKeyRef.current = selectedThreadKey;
511 }, [selectedThreadKey]);522 // A replaced or unmounted native editor may not emit a blur event.
523 setComposerFocused(false);
524 }, [selectedThreadKey, showContent]);
512525
513 useEffect(() => {526 useEffect(() => {
514 setAnchorMessageId(null);527 setAnchorMessageId(null);
...@@ -810,7 +823,7 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread...@@ -810,7 +823,7 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread
810 onUpdateRuntimeMode={props.onUpdateThreadRuntimeMode}823 onUpdateRuntimeMode={props.onUpdateThreadRuntimeMode}
811 onUpdateInteractionMode={props.onUpdateThreadInteractionMode}824 onUpdateInteractionMode={props.onUpdateThreadInteractionMode}
812 onExpandedChange={setComposerExpanded}825 onExpandedChange={setComposerExpanded}
813 onEditorFocusChange={handleOwnedInputFocusChange}826 onEditorFocusChange={handleComposerFocusChange}
814 />827 />
815 </View>828 </View>
816 </View>829 </View>
apps/mobile/src/features/threads/use-composer-command-menu.test.ts created+13
...@@ -0,0 +1,13 @@
1import { describe, expect, it, vi } from "vite-plus/test";
2
3vi.mock("../../state/use-composer-path-search", () => ({
4 useComposerPathSearch: () => ({ entries: [], isPending: false }),
5}));
6
7import { composerSelectionAtEnd } from "./use-composer-command-menu";
8
9describe("composerSelectionAtEnd", () => {
10 it("resets a changed draft owner to the new draft end", () => {
11 expect(composerSelectionAtEnd("queued task 🧪")).toEqual({ start: 14, end: 14 });
12 });
13});
apps/mobile/src/features/threads/use-composer-command-menu.ts+14-5
...@@ -9,16 +9,21 @@ import {...@@ -9,16 +9,21 @@ import {
9 normalizeSearchQuery,9 normalizeSearchQuery,
10 scoreQueryMatch,10 scoreQueryMatch,
11} from "@t3tools/shared/searchRanking";11} from "@t3tools/shared/searchRanking";
12import { useCallback, useEffect, useMemo, useState } from "react";12import { useCallback, useEffect, useMemo, useRef, useState } from "react";
1313
14import type { ComposerEditorSelection } from "../../components/ComposerEditor";14import type { ComposerEditorSelection } from "../../components/ComposerEditor";
15import { useComposerPathSearch } from "../../state/use-composer-path-search";15import { useComposerPathSearch } from "../../state/use-composer-path-search";
16import type { ComposerCommandItem } from "./ComposerCommandPopover";16import type { ComposerCommandItem } from "./ComposerCommandPopover";
17import { matchesSlashSkillQuery } from "./composerSlashSkillSearch";17import { matchesSlashSkillQuery } from "./composerSlashSkillSearch";
1818
19export function composerSelectionAtEnd(draftMessage: string): ComposerEditorSelection {
20 return { start: draftMessage.length, end: draftMessage.length };
21}
22
19/** Shared autocomplete for thread composers and unsent new-task drafts. */23/** Shared autocomplete for thread composers and unsent new-task drafts. */
20export function useComposerCommandMenu({24export function useComposerCommandMenu({
21 draftMessage,25 draftMessage,
26 ownerKey,
22 environmentId,27 environmentId,
23 projectCwd,28 projectCwd,
24 selectedProviderStatus,29 selectedProviderStatus,
...@@ -28,6 +33,7 @@ export function useComposerCommandMenu({...@@ -28,6 +33,7 @@ export function useComposerCommandMenu({
28 onUpdateInteractionMode,33 onUpdateInteractionMode,
29}: {34}: {
30 readonly draftMessage: string;35 readonly draftMessage: string;
36 readonly ownerKey: string | null;
31 readonly environmentId: EnvironmentId | null;37 readonly environmentId: EnvironmentId | null;
32 readonly projectCwd: string | null;38 readonly projectCwd: string | null;
33 readonly selectedProviderStatus: ServerProvider | null;39 readonly selectedProviderStatus: ServerProvider | null;
...@@ -36,10 +42,8 @@ export function useComposerCommandMenu({...@@ -36,10 +42,8 @@ export function useComposerCommandMenu({
36 readonly onChangeDraftMessage: (value: string) => void;42 readonly onChangeDraftMessage: (value: string) => void;
37 readonly onUpdateInteractionMode?: (mode: ProviderInteractionMode) => void;43 readonly onUpdateInteractionMode?: (mode: ProviderInteractionMode) => void;
38}) {44}) {
39 const [selection, setSelection] = useState(() => ({45 const [selection, setSelection] = useState(() => composerSelectionAtEnd(draftMessage));
40 start: draftMessage.length,46 const previousOwnerKeyRef = useRef(ownerKey);
41 end: draftMessage.length,
42 }));
43 const onSelectionChange = useCallback((nextSelection: ComposerEditorSelection) => {47 const onSelectionChange = useCallback((nextSelection: ComposerEditorSelection) => {
44 setSelection(nextSelection);48 setSelection(nextSelection);
45 }, []);49 }, []);
...@@ -54,6 +58,11 @@ export function useComposerCommandMenu({...@@ -54,6 +58,11 @@ export function useComposerCommandMenu({
54 return { start, end: selectionEnd };58 return { start, end: selectionEnd };
55 });59 });
56 }, [draftMessage.length]);60 }, [draftMessage.length]);
61 useEffect(() => {
62 if (previousOwnerKeyRef.current === ownerKey) return;
63 previousOwnerKeyRef.current = ownerKey;
64 setSelection(composerSelectionAtEnd(draftMessage));
65 }, [draftMessage, ownerKey]);
5766
58 const trigger = useMemo(() => {67 const trigger = useMemo(() => {
59 if (!enabled || selection.start !== selection.end) {68 if (!enabled || selection.start !== selection.end) {
apps/mobile/src/features/voice-input/ComposerDictationControl.tsx created+360
...@@ -0,0 +1,360 @@
1import type { VoiceInputPhase, VoiceInputState } from "@t3tools/client-runtime/voice-input";
2import { memo, useCallback, useLayoutEffect, useState, type ReactNode } from "react";
3import {
4 ActivityIndicator,
5 Linking,
6 Platform,
7 Pressable,
8 View,
9 type LayoutChangeEvent,
10} from "react-native";
11import Animated, {
12 Easing,
13 FadeIn,
14 FadeOut,
15 LinearTransition,
16 ReduceMotion,
17 useAnimatedStyle,
18 useSharedValue,
19 withTiming,
20 type SharedValue,
21} from "react-native-reanimated";
22
23import { AppText as Text } from "../../components/AppText";
24import { SymbolView, type AppSymbolName } from "../../components/AppSymbol";
25import { cn } from "../../lib/cn";
26import type { VoiceComposerPresentation } from "./voiceInputPresentation";
27import { VOICE_WAVEFORM_SAMPLE_COUNT } from "./voiceInputMetering";
28
29const DICTATION_TIMING = {
30 duration: 220,
31 easing: Easing.out(Easing.cubic),
32 reduceMotion: ReduceMotion.System,
33} as const;
34const DICTATION_LAYOUT =
35 Platform.OS === "android"
36 ? undefined
37 : LinearTransition.duration(DICTATION_TIMING.duration).reduceMotion(ReduceMotion.System);
38const DICTATION_ENTERING = FadeIn.duration(180).reduceMotion(ReduceMotion.System);
39const DICTATION_EXITING = FadeOut.duration(120).reduceMotion(ReduceMotion.System);
40const WAVEFORM_BAR_HEIGHT = 32;
41const WAVEFORM_MIN_BAR_HEIGHT = 2;
42const WAVEFORM_BAR_SPACING = 5;
43const WAVEFORM_TIMING = {
44 duration: 100,
45 easing: Easing.out(Easing.quad),
46 reduceMotion: ReduceMotion.System,
47} as const;
48
49/** Keeps the native editor mounted when compact dictation replaces the draft area. */
50export function ComposerDictationDraftContent(props: {
51 readonly children: ReactNode;
52 readonly className?: string;
53 readonly collapsed: boolean;
54}) {
55 const visibility = useSharedValue(props.collapsed ? 0 : 1);
56 useLayoutEffect(() => {
57 visibility.value = withTiming(props.collapsed ? 0 : 1, DICTATION_TIMING);
58 }, [props.collapsed, visibility]);
59 const animatedStyle = useAnimatedStyle(() => ({
60 opacity: visibility.value,
61 transform: [{ translateY: -4 * (1 - visibility.value) }],
62 }));
63
64 return (
65 <Animated.View
66 accessibilityElementsHidden={props.collapsed}
67 importantForAccessibility={props.collapsed ? "no-hide-descendants" : "auto"}
68 collapsable={false}
69 className={props.className}
70 layout={DICTATION_LAYOUT}
71 style={[animatedStyle, { overflow: "hidden" }, props.collapsed ? { height: 0 } : undefined]}
72 >
73 {props.children}
74 </Animated.View>
75 );
76}
77
78/** Crossfades controls within one toolbar row while the draft keeps its position. */
79export function ComposerDictationToolbar(props: {
80 readonly children: ReactNode;
81 readonly showsDictation: boolean;
82 readonly visible?: boolean;
83}) {
84 return (
85 <View className="relative h-11">
86 <Animated.View
87 key={props.showsDictation ? "dictation" : "draft"}
88 className="absolute inset-0"
89 entering={DICTATION_ENTERING}
90 exiting={props.visible === false ? undefined : DICTATION_EXITING}
91 >
92 {props.children}
93 </Animated.View>
94 </View>
95 );
96}
97
98const WaveformBar = memo(function WaveformBar(props: {
99 readonly audioLevels: SharedValue<number[]>;
100 readonly sampleIndex: number;
101}) {
102 const { audioLevels, sampleIndex } = props;
103 const animatedStyle = useAnimatedStyle(() => {
104 const level = audioLevels.value[sampleIndex] ?? 0;
105 return {
106 opacity: withTiming(0.22 + level * 0.78, WAVEFORM_TIMING),
107 transform: [
108 {
109 scaleY: withTiming(
110 (WAVEFORM_MIN_BAR_HEIGHT + level * (WAVEFORM_BAR_HEIGHT - WAVEFORM_MIN_BAR_HEIGHT)) /
111 WAVEFORM_BAR_HEIGHT,
112 WAVEFORM_TIMING,
113 ),
114 },
115 ],
116 };
117 });
118
119 return (
120 <Animated.View
121 className="w-0.5 rounded-full bg-foreground"
122 style={[{ height: WAVEFORM_BAR_HEIGHT }, animatedStyle]}
123 />
124 );
125});
126
127const VoiceWaveform = memo(function VoiceWaveform(props: {
128 readonly audioLevels: SharedValue<number[]>;
129}) {
130 const [barCount, setBarCount] = useState(0);
131 const handleLayout = useCallback((event: LayoutChangeEvent) => {
132 setBarCount(
133 Math.max(
134 1,
135 Math.min(
136 VOICE_WAVEFORM_SAMPLE_COUNT,
137 Math.floor(event.nativeEvent.layout.width / WAVEFORM_BAR_SPACING),
138 ),
139 ),
140 );
141 }, []);
142
143 return (
144 <View
145 accessibilityElementsHidden
146 importantForAccessibility="no-hide-descendants"
147 className="min-w-0 flex-1 flex-row items-center justify-between overflow-hidden"
148 style={{ height: WAVEFORM_BAR_HEIGHT }}
149 onLayout={handleLayout}
150 >
151 {Array.from({ length: barCount }, (_, index) => (
152 <WaveformBar
153 key={index}
154 audioLevels={props.audioLevels}
155 sampleIndex={VOICE_WAVEFORM_SAMPLE_COUNT - barCount + index}
156 />
157 ))}
158 </View>
159 );
160});
161
162function VoiceActionButton(props: {
163 readonly accessibilityLabel: string;
164 readonly disabled?: boolean;
165 readonly icon: AppSymbolName;
166 readonly loading?: boolean;
167 readonly onPress: () => void;
168 readonly variant?: "plain" | "primary";
169}) {
170 const variant = props.variant ?? "plain";
171 const loadingVisibility = useSharedValue(props.loading ? 1 : 0);
172 useLayoutEffect(() => {
173 loadingVisibility.value = withTiming(props.loading ? 1 : 0, DICTATION_TIMING);
174 }, [loadingVisibility, props.loading]);
175 const primaryStyle = useAnimatedStyle(() => ({ opacity: 1 - loadingVisibility.value }));
176
177 return (
178 <Pressable
179 accessibilityLabel={props.accessibilityLabel}
180 accessibilityRole="button"
181 accessibilityState={{ busy: props.loading, disabled: props.disabled }}
182 className={cn(
183 "items-center justify-center active:opacity-70",
184 variant === "primary" ? "size-11" : "size-9",
185 )}
186 disabled={props.disabled}
187 hitSlop={variant === "plain" ? 4 : undefined}
188 onPress={props.onPress}
189 style={{ opacity: props.disabled && !props.loading ? 0.4 : 1 }}
190 >
191 <View
192 className={cn(
193 "items-center justify-center",
194 variant === "primary" ? "size-11 rounded-full bg-subtle" : "size-9",
195 )}
196 >
197 {variant === "primary" ? (
198 <Animated.View
199 className="absolute inset-0 rounded-full bg-primary"
200 style={primaryStyle}
201 />
202 ) : null}
203 <Animated.View
204 key={props.loading ? "loading" : "icon"}
205 className="absolute inset-0 items-center justify-center"
206 entering={DICTATION_ENTERING}
207 exiting={DICTATION_EXITING}
208 >
209 {props.loading ? (
210 <ActivityIndicator size="small" colorClassName="accent-icon-muted" />
211 ) : (
212 <SymbolView
213 name={props.icon}
214 size={17}
215 tintColorClassName={
216 variant === "primary" ? "accent-primary-foreground" : "accent-icon"
217 }
218 type="monochrome"
219 />
220 )}
221 </Animated.View>
222 </View>
223 </Pressable>
224 );
225}
226
227export function ComposerDictationStatus(props: {
228 readonly audioLevels: SharedValue<number[]>;
229 readonly elapsedSeconds: number;
230 readonly phase: VoiceInputPhase;
231 readonly presentation: VoiceComposerPresentation;
232 readonly onDismissError: () => void;
233}) {
234 const recordingVisibility = useSharedValue(props.phase === "recording" ? 1 : 0);
235 useLayoutEffect(() => {
236 recordingVisibility.value = withTiming(props.phase === "recording" ? 1 : 0, DICTATION_TIMING);
237 }, [props.phase, recordingVisibility]);
238 const waveformStyle = useAnimatedStyle(() => ({
239 opacity: recordingVisibility.value,
240 transform: [{ translateY: -4 * (1 - recordingVisibility.value) }],
241 }));
242 const labelStyle = useAnimatedStyle(() => ({
243 opacity: 1 - recordingVisibility.value,
244 transform: [{ translateY: 4 * recordingVisibility.value }],
245 }));
246
247 if (!props.presentation.statusLabel) return null;
248 const isError = props.presentation.statusKind === "error";
249 const elapsedLabel = `${Math.floor(props.elapsedSeconds / 60)}:${String(props.elapsedSeconds % 60).padStart(2, "0")}`;
250 return (
251 <View className="relative h-11 min-w-0 flex-1 justify-center">
252 {isError ? (
253 <View className="min-w-0 flex-row items-center gap-1.5 px-2">
254 <Text className="min-w-0 flex-1 text-sm text-red-400" numberOfLines={2}>
255 {props.presentation.statusLabel}
256 </Text>
257 <Pressable
258 accessibilityLabel="Dismiss voice input error"
259 accessibilityRole="button"
260 className="size-7 items-center justify-center active:opacity-70"
261 hitSlop={8}
262 onPress={props.onDismissError}
263 >
264 <SymbolView
265 name="xmark"
266 size={12}
267 tintColorClassName="accent-icon-muted"
268 type="monochrome"
269 />
270 </Pressable>
271 </View>
272 ) : (
273 <View
274 accessible
275 accessibilityLabel={props.presentation.statusLabel}
276 accessibilityLiveRegion={props.phase === "recording" ? "none" : "polite"}
277 className="h-11"
278 >
279 <Animated.View
280 className="absolute inset-0 min-w-0 flex-row items-center gap-2 px-1"
281 style={waveformStyle}
282 >
283 <VoiceWaveform audioLevels={props.audioLevels} />
284 <Text
285 className="text-xs text-foreground-muted"
286 numberOfLines={1}
287 style={{ fontVariant: ["tabular-nums"] }}
288 >
289 {elapsedLabel}
290 </Text>
291 </Animated.View>
292 <Animated.View className="absolute inset-0 justify-center px-2" style={labelStyle}>
293 <Text className="text-center text-sm text-foreground-muted" numberOfLines={1}>
294 {props.presentation.statusLabel}
295 </Text>
296 </Animated.View>
297 </View>
298 )}
299 </View>
300 );
301}
302
303export function ComposerDictationCancelAction(props: {
304 readonly presentation: VoiceComposerPresentation;
305 readonly onCancel: () => void;
306}) {
307 if (props.presentation.leadingAction !== "cancel") return null;
308 return (
309 <VoiceActionButton
310 accessibilityLabel="Cancel dictation"
311 icon="xmark"
312 onPress={props.onCancel}
313 />
314 );
315}
316
317export function ComposerDictationPrimaryAction(props: {
318 readonly state: VoiceInputState;
319 readonly presentation: VoiceComposerPresentation;
320 readonly isAvailable: boolean;
321 readonly disabled?: boolean;
322 readonly onStart: () => void;
323 readonly onConfirm: () => void;
324 readonly onCancel: () => void;
325}) {
326 if (props.presentation.trailingAction === "confirm") {
327 return (
328 <VoiceActionButton
329 accessibilityLabel={
330 props.presentation.confirmationEnabled
331 ? "Finish dictation"
332 : (props.presentation.statusLabel ?? "Preparing voice input")
333 }
334 disabled={!props.presentation.confirmationEnabled}
335 icon="checkmark"
336 loading={!props.presentation.confirmationEnabled}
337 onPress={props.onConfirm}
338 variant="primary"
339 />
340 );
341 }
342
343 if (!props.isAvailable) return null;
344 const openSettings = props.state.phase === "error" && props.state.errorAction === "settings";
345 return (
346 <VoiceActionButton
347 accessibilityLabel={openSettings ? "Open microphone settings" : "Start dictation"}
348 disabled={props.disabled}
349 icon="mic"
350 onPress={
351 openSettings
352 ? () => {
353 props.onCancel();
354 void Linking.openSettings();
355 }
356 : props.onStart
357 }
358 />
359 );
360}
apps/mobile/src/features/voice-input/useVoiceInputController.ts created+217
...@@ -0,0 +1,217 @@
1import {
2 RecordingPresets,
3 requestRecordingPermissionsAsync,
4 setAudioModeAsync,
5 setIsAudioActiveAsync,
6 useAudioRecorder,
7 type RecordingStatus,
8} from "expo-audio";
9import { File } from "expo-file-system";
10import { useFocusEffect } from "@react-navigation/native";
11import { useCallback, useEffect, useRef, useState } from "react";
12import { AppState } from "react-native";
13import { useSharedValue } from "react-native-reanimated";
14
15import type { ComposerEditorSelection } from "../../components/ComposerEditor";
16import { getLocalVoiceTranscriber } from "../../native/voiceTranscription";
17import {
18 VoiceInputController,
19 VOICE_RECORDING_LIMIT_SECONDS,
20 voiceInputBlocksSubmission,
21 voiceInputFreezesEditor,
22 type VoiceDraftSnapshot,
23 type VoiceInputState,
24} from "@t3tools/client-runtime/voice-input";
25import { normalizeVoiceInputDecibels, VOICE_WAVEFORM_SAMPLE_COUNT } from "./voiceInputMetering";
26
27const INITIAL_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: null };
28const VOICE_METERING_INTERVAL_MS = 80;
29const VOICE_RECORDING_OPTIONS = {
30 ...RecordingPresets.HIGH_QUALITY,
31 isMeteringEnabled: true,
32};
33
34async function releaseVoiceRecordingAudio(): Promise<void> {
35 try {
36 await setAudioModeAsync({ allowsRecording: false });
37 } finally {
38 // Expo does not deactivate AVAudioSession when recording stops or its
39 // category changes. Explicit deactivation resumes interrupted app audio.
40 await setIsAudioActiveAsync(false);
41 }
42}
43
44async function configureVoiceRecordingAudio(): Promise<void> {
45 try {
46 await setAudioModeAsync({
47 allowsRecording: true,
48 interruptionMode: "doNotMix",
49 playsInSilentMode: true,
50 shouldPlayInBackground: false,
51 });
52 await setIsAudioActiveAsync(true);
53 } catch (error) {
54 try {
55 await releaseVoiceRecordingAudio();
56 } catch {
57 // Keep the setup error. The controller has not started a recorder yet.
58 }
59 throw error;
60 }
61}
62
63export function useVoiceInputController(input: {
64 readonly ownerKey: string | null;
65 readonly draftMessage: string;
66 readonly selection: ComposerEditorSelection;
67 readonly disabled?: boolean;
68 readonly onChangeDraftMessage: (value: string) => void;
69 readonly onChangeSelection: (selection: ComposerEditorSelection) => void;
70}) {
71 const [state, setState] = useState<VoiceInputState>(INITIAL_STATE);
72 const [elapsedSeconds, setElapsedSeconds] = useState(0);
73 const elapsedSecondsRef = useRef(0);
74 const audioLevelsRef = useRef(Array<number>(VOICE_WAVEFORM_SAMPLE_COUNT).fill(0));
75 const audioLevels = useSharedValue(audioLevelsRef.current);
76 const controllerRef = useRef<VoiceInputController | null>(null);
77 const previousDraftRef = useRef({ ownerKey: input.ownerKey, text: input.draftMessage });
78 const revisionRef = useRef(0);
79 if (
80 previousDraftRef.current.ownerKey !== input.ownerKey ||
81 previousDraftRef.current.text !== input.draftMessage
82 ) {
83 previousDraftRef.current = { ownerKey: input.ownerKey, text: input.draftMessage };
84 revisionRef.current += 1;
85 }
86 const latestInputRef = useRef(input);
87 latestInputRef.current = input;
88
89 const handleRecorderStatus = useCallback((status: RecordingStatus) => {
90 controllerRef.current?.handleRecorderStatus({
91 isFinished: status.isFinished,
92 hasError: status.hasError || status.mediaServicesDidReset === true,
93 error: status.error,
94 url: status.url,
95 });
96 }, []);
97 const recorder = useAudioRecorder(VOICE_RECORDING_OPTIONS, handleRecorderStatus);
98
99 if (!controllerRef.current) {
100 controllerRef.current = new VoiceInputController({
101 recorder,
102 getTranscriber: getLocalVoiceTranscriber,
103 requestPermission: async () => {
104 const permission = await requestRecordingPermissionsAsync();
105 return { granted: permission.granted, canAskAgain: permission.canAskAgain };
106 },
107 configureRecording: configureVoiceRecordingAudio,
108 releaseRecording: releaseVoiceRecordingAudio,
109 deleteRecording: (uri) => new File(uri).delete(),
110 readDraft: (): VoiceDraftSnapshot | null => {
111 const current = latestInputRef.current;
112 if (!current.ownerKey) return null;
113 return {
114 ownerKey: current.ownerKey,
115 text: current.draftMessage,
116 selection: current.selection,
117 revision: revisionRef.current,
118 };
119 },
120 commitDraft: (text, selection) => {
121 const current = latestInputRef.current;
122 current.onChangeSelection(selection);
123 current.onChangeDraftMessage(text);
124 },
125 onStateChange: setState,
126 });
127 }
128
129 const controller = controllerRef.current;
130 const previousOwnerRef = useRef(input.ownerKey);
131 useEffect(() => {
132 if (previousOwnerRef.current === input.ownerKey) return;
133 previousOwnerRef.current = input.ownerKey;
134 controller.ownerChanged();
135 }, [controller, input.ownerKey]);
136
137 useFocusEffect(
138 useCallback(
139 () => () => {
140 controller.dispose();
141 },
142 [controller],
143 ),
144 );
145
146 useEffect(() => {
147 const subscription = AppState.addEventListener("change", (nextState) => {
148 // iOS reports `inactive` while its permission dialog is open. Only the
149 // real background state cancels preparation; recorder status handles
150 // calls and route interruptions during capture.
151 if (nextState === "background") controller.appMovedToBackground();
152 });
153 return () => subscription.remove();
154 }, [controller]);
155
156 useEffect(() => () => controller.dispose(), [controller]);
157
158 useEffect(() => {
159 if (state.phase !== "preparing" && state.phase !== "recording") return;
160
161 if (audioLevelsRef.current.some((level) => level !== 0)) {
162 audioLevelsRef.current = Array<number>(VOICE_WAVEFORM_SAMPLE_COUNT).fill(0);
163 audioLevels.value = audioLevelsRef.current;
164 }
165 if (elapsedSecondsRef.current !== 0) {
166 elapsedSecondsRef.current = 0;
167 setElapsedSeconds(0);
168 }
169 if (state.phase !== "recording") return;
170
171 const sampleRecording = () => {
172 if (controller.currentState.phase !== "recording") return;
173 const status = recorder.getStatus();
174 if (!status.isRecording) return;
175
176 const level = normalizeVoiceInputDecibels(status.metering);
177 const history = audioLevelsRef.current;
178 if (level !== 0 || history.some((sample) => sample !== 0)) {
179 const nextLevels = [...history.slice(1), level];
180 audioLevelsRef.current = nextLevels;
181 audioLevels.value = nextLevels;
182 }
183
184 const nextElapsedSeconds = Math.min(
185 VOICE_RECORDING_LIMIT_SECONDS,
186 Math.max(0, Math.floor(status.durationMillis / 1_000)),
187 );
188 if (nextElapsedSeconds !== elapsedSecondsRef.current) {
189 elapsedSecondsRef.current = nextElapsedSeconds;
190 setElapsedSeconds(nextElapsedSeconds);
191 }
192 };
193
194 sampleRecording();
195 const intervalId = setInterval(sampleRecording, VOICE_METERING_INTERVAL_MS);
196 return () => clearInterval(intervalId);
197 }, [audioLevels, controller, recorder, state.phase]);
198
199 const start = useCallback(() => {
200 if (!latestInputRef.current.disabled) void controller.start();
201 }, [controller]);
202 const stop = useCallback(() => controller.stop(), [controller]);
203 const cancel = useCallback(() => controller.cancel(), [controller]);
204
205 return {
206 isAvailable: getLocalVoiceTranscriber() !== null,
207 state,
208 audioLevels,
209 elapsedSeconds,
210 isBusy: voiceInputBlocksSubmission(state),
211 freezesEditor: voiceInputFreezesEditor(state),
212 blocksSubmission: voiceInputBlocksSubmission(state),
213 start,
214 stop,
215 cancel,
216 };
217}
apps/mobile/src/features/voice-input/voiceInputMetering.test.ts created+56
...@@ -0,0 +1,56 @@
1import { describe, expect, it } from "vite-plus/test";
2
3import { normalizeVoiceInputDecibels } from "./voiceInputMetering";
4
5describe("normalizeVoiceInputDecibels", () => {
6 it.each([undefined, Number.NaN, Number.POSITIVE_INFINITY, Number.NEGATIVE_INFINITY])(
7 "treats a missing or invalid reading %s as silence",
8 (decibels) => {
9 expect(normalizeVoiceInputDecibels(decibels)).toBe(0);
10 },
11 );
12
13 it.each([-160, -90, -60])("keeps a reading at or below the noise floor %s silent", (decibels) => {
14 expect(normalizeVoiceInputDecibels(decibels)).toBe(0);
15 });
16
17 it("keeps quiet background readings close to the baseline", () => {
18 const quiet = normalizeVoiceInputDecibels(-50);
19 expect(quiet).toBeGreaterThan(0);
20 expect(quiet).toBeLessThan(0.05);
21 });
22
23 it("keeps loud negative speech readings distinct below full height", () => {
24 const levels = [-20, -18, -12, -6, -3].map(normalizeVoiceInputDecibels);
25
26 for (const level of levels) {
27 expect(level).toBeGreaterThan(0);
28 expect(level).toBeLessThan(1);
29 }
30 expect(levels.every((level, index) => index === 0 || level > levels[index - 1]!)).toBe(true);
31 });
32
33 it("makes near-speech changes visible without an early ceiling", () => {
34 expect(normalizeVoiceInputDecibels(-6) - normalizeVoiceInputDecibels(-12)).toBeGreaterThan(
35 0.18,
36 );
37 expect(normalizeVoiceInputDecibels(-3) - normalizeVoiceInputDecibels(-12)).toBeGreaterThan(0.3);
38 });
39
40 it("increases throughout the usable microphone range", () => {
41 const levels = [-60, -55, -50, -40, -30, -20, -12, -6, -3, -0.001, 0].map(
42 normalizeVoiceInputDecibels,
43 );
44 expect(levels.every((level, index) => index === 0 || level > levels[index - 1]!)).toBe(true);
45 });
46
47 it("approaches the noise floor and full scale without a jump", () => {
48 expect(normalizeVoiceInputDecibels(-59.999)).toBeLessThan(0.001);
49 expect(normalizeVoiceInputDecibels(-0.001)).toBeGreaterThan(0.999);
50 expect(normalizeVoiceInputDecibels(-0.001)).toBeLessThan(1);
51 });
52
53 it.each([0, 6, 160])("caps only full-scale or higher readings %s at one", (decibels) => {
54 expect(normalizeVoiceInputDecibels(decibels)).toBe(1);
55 });
56});
apps/mobile/src/features/voice-input/voiceInputMetering.ts created+14
...@@ -0,0 +1,14 @@
1export const VOICE_WAVEFORM_SAMPLE_COUNT = 64;
2
3const VOICE_NOISE_FLOOR_DECIBELS = -60;
4const VOICE_NOISE_FLOOR_AMPLITUDE = 10 ** (VOICE_NOISE_FLOOR_DECIBELS / 20);
5
6/** Converts measured decibels to compressed amplitude, reserving full height for 0 dB. */
7export function normalizeVoiceInputDecibels(decibels: number | undefined) {
8 if (decibels === undefined || !Number.isFinite(decibels)) return 0;
9 if (decibels <= VOICE_NOISE_FLOOR_DECIBELS) return 0;
10 if (decibels >= 0) return 1;
11
12 const amplitude = 10 ** (decibels / 20);
13 return Math.sqrt((amplitude - VOICE_NOISE_FLOOR_AMPLITUDE) / (1 - VOICE_NOISE_FLOOR_AMPLITUDE));
14}
apps/mobile/src/features/voice-input/voiceInputPresentation.test.ts created+69
...@@ -0,0 +1,69 @@
1import { describe, expect, it } from "vite-plus/test";
2import { voiceInputFreezesEditor } from "@t3tools/client-runtime/voice-input";
3
4import { resolveVoiceComposerPresentation } from "./voiceInputPresentation";
5
6describe("resolveVoiceComposerPresentation", () => {
7 it("maps voice states to stable composer actions and editor read-only state", () => {
8 expect(
9 resolveVoiceComposerPresentation({ phase: "idle", error: null, errorAction: null }, 0),
10 ).toEqual({
11 leadingAction: null,
12 trailingAction: "mic",
13 showsSend: true,
14 statusKind: null,
15 statusLabel: null,
16 confirmationEnabled: false,
17 });
18 expect(
19 resolveVoiceComposerPresentation({ phase: "preparing", error: null, errorAction: null }, 0),
20 ).toMatchObject({
21 leadingAction: "cancel",
22 trailingAction: "confirm",
23 showsSend: false,
24 statusLabel: "Preparing",
25 confirmationEnabled: false,
26 });
27 expect(
28 resolveVoiceComposerPresentation({ phase: "recording", error: null, errorAction: null }, 64),
29 ).toMatchObject({
30 leadingAction: "cancel",
31 trailingAction: "confirm",
32 showsSend: false,
33 statusLabel: "Recording 1:04",
34 confirmationEnabled: true,
35 });
36 expect(
37 resolveVoiceComposerPresentation(
38 { phase: "transcribing", error: null, errorAction: null },
39 0,
40 ),
41 ).toMatchObject({
42 statusLabel: "Transcribing",
43 confirmationEnabled: false,
44 });
45 expect(
46 resolveVoiceComposerPresentation(
47 { phase: "error", error: "Microphone unavailable", errorAction: "retry" },
48 0,
49 ),
50 ).toMatchObject({
51 leadingAction: null,
52 trailingAction: "mic",
53 showsSend: true,
54 statusKind: "error",
55 statusLabel: "Microphone unavailable",
56 });
57
58 expect(voiceInputFreezesEditor({ phase: "preparing", error: null, errorAction: null })).toBe(
59 true,
60 );
61 expect(voiceInputFreezesEditor({ phase: "recording", error: null, errorAction: null })).toBe(
62 true,
63 );
64 expect(voiceInputFreezesEditor({ phase: "transcribing", error: null, errorAction: null })).toBe(
65 true,
66 );
67 expect(voiceInputFreezesEditor({ phase: "idle", error: null, errorAction: null })).toBe(false);
68 });
69});
apps/mobile/src/features/voice-input/voiceInputPresentation.ts created+65
...@@ -0,0 +1,65 @@
1import type { VoiceInputState } from "@t3tools/client-runtime/voice-input";
2
3export type VoiceComposerPresentation = {
4 readonly leadingAction: "cancel" | null;
5 readonly trailingAction: "mic" | "confirm";
6 readonly showsSend: boolean;
7 readonly statusKind: "active" | "error" | null;
8 readonly statusLabel: string | null;
9 readonly confirmationEnabled: boolean;
10};
11
12export function resolveVoiceComposerPresentation(
13 state: VoiceInputState,
14 elapsedSeconds: number,
15): VoiceComposerPresentation {
16 switch (state.phase) {
17 case "idle":
18 return {
19 leadingAction: null,
20 trailingAction: "mic",
21 showsSend: true,
22 statusKind: null,
23 statusLabel: null,
24 confirmationEnabled: false,
25 };
26 case "error":
27 return {
28 leadingAction: null,
29 trailingAction: "mic",
30 showsSend: true,
31 statusKind: "error",
32 statusLabel: state.error,
33 confirmationEnabled: false,
34 };
35 case "preparing":
36 return {
37 leadingAction: "cancel",
38 trailingAction: "confirm",
39 showsSend: false,
40 statusKind: "active",
41 statusLabel: "Preparing",
42 confirmationEnabled: false,
43 };
44 case "recording": {
45 const seconds = Math.max(0, Math.floor(elapsedSeconds));
46 return {
47 leadingAction: "cancel",
48 trailingAction: "confirm",
49 showsSend: false,
50 statusKind: "active",
51 statusLabel: `Recording ${Math.floor(seconds / 60)}:${String(seconds % 60).padStart(2, "0")}`,
52 confirmationEnabled: true,
53 };
54 }
55 case "transcribing":
56 return {
57 leadingAction: "cancel",
58 trailingAction: "confirm",
59 showsSend: false,
60 statusKind: "active",
61 statusLabel: "Transcribing",
62 confirmationEnabled: false,
63 };
64 }
65}
apps/mobile/src/native/T3ComposerEditor.ios.tsx+2
...@@ -62,6 +62,7 @@ interface NativeComposerEditorProps extends ViewProps {...@@ -62,6 +62,7 @@ interface NativeComposerEditorProps extends ViewProps {
62 readonly lineHeight: number;62 readonly lineHeight: number;
63 readonly contentInsetVertical: number;63 readonly contentInsetVertical: number;
64 readonly editable: boolean;64 readonly editable: boolean;
65 readonly readOnly: boolean;
65 readonly scrollEnabled: boolean;66 readonly scrollEnabled: boolean;
66 readonly autoFocus: boolean;67 readonly autoFocus: boolean;
67 readonly autoCorrect: boolean;68 readonly autoCorrect: boolean;
...@@ -243,6 +244,7 @@ export function ComposerEditor({...@@ -243,6 +244,7 @@ export function ComposerEditor({
243 }244 }
244 contentInsetVertical={contentInsetVertical}245 contentInsetVertical={contentInsetVertical}
245 editable={props.editable ?? true}246 editable={props.editable ?? true}
247 readOnly={props.readOnly ?? false}
246 scrollEnabled={props.scrollEnabled ?? true}248 scrollEnabled={props.scrollEnabled ?? true}
247 autoFocus={props.autoFocus ?? false}249 autoFocus={props.autoFocus ?? false}
248 autoCorrect={props.autoCorrect ?? true}250 autoCorrect={props.autoCorrect ?? true}
apps/mobile/src/native/T3ComposerEditor.native.tsx+1-1
...@@ -248,7 +248,7 @@ export function ComposerEditor({...@@ -248,7 +248,7 @@ export function ComposerEditor({
248 }248 }
249 contentInsetVertical={contentInsetVertical}249 contentInsetVertical={contentInsetVertical}
250 singleLineCentered={props.singleLineCentered ?? false}250 singleLineCentered={props.singleLineCentered ?? false}
251 editable={props.editable ?? true}251 editable={(props.editable ?? true) && !(props.readOnly ?? false)}
252 scrollEnabled={props.scrollEnabled ?? true}252 scrollEnabled={props.scrollEnabled ?? true}
253 autoFocus={props.autoFocus ?? false}253 autoFocus={props.autoFocus ?? false}
254 autoCorrect={props.autoCorrect ?? true}254 autoCorrect={props.autoCorrect ?? true}
apps/mobile/src/native/T3ComposerEditor.tsx+2
...@@ -16,6 +16,7 @@ export function ComposerEditor({...@@ -16,6 +16,7 @@ export function ComposerEditor({
16 textStyle,16 textStyle,
17 contentInsetVertical = 0,17 contentInsetVertical = 0,
18 singleLineCentered: _singleLineCentered,18 singleLineCentered: _singleLineCentered,
19 readOnly = false,
19 ...props20 ...props
20}: ComposerEditorProps) {21}: ComposerEditorProps) {
21 const inputRef = useRef<RNTextInput>(null);22 const inputRef = useRef<RNTextInput>(null);
...@@ -39,6 +40,7 @@ export function ComposerEditor({...@@ -39,6 +40,7 @@ export function ComposerEditor({
39 <TextInput40 <TextInput
40 ref={inputRef}41 ref={inputRef}
41 {...props}42 {...props}
43 editable={(props.editable ?? true) && !readOnly}
42 selection={selection}44 selection={selection}
43 onSelectionChange={(event) => props.onSelectionChange?.(event.nativeEvent.selection)}45 onSelectionChange={(event) => props.onSelectionChange?.(event.nativeEvent.selection)}
44 multiline={props.multiline ?? true}46 multiline={props.multiline ?? true}
apps/mobile/src/native/T3ComposerEditor.types.ts+2
...@@ -23,6 +23,8 @@ export interface ComposerEditorProps {...@@ -23,6 +23,8 @@ export interface ComposerEditorProps {
23 readonly placeholder?: string;23 readonly placeholder?: string;
24 readonly autoFocus?: boolean;24 readonly autoFocus?: boolean;
25 readonly editable?: boolean;25 readonly editable?: boolean;
26 /** Blocks user edits while preserving focus, selection, and the software keyboard on iOS. */
27 readonly readOnly?: boolean;
26 readonly scrollEnabled?: boolean;28 readonly scrollEnabled?: boolean;
27 readonly autoCorrect?: boolean;29 readonly autoCorrect?: boolean;
28 readonly spellCheck?: boolean;30 readonly spellCheck?: boolean;
apps/mobile/src/native/voiceTranscription.ios.test.ts created+140
...@@ -0,0 +1,140 @@
1import type { TranscriptionResult } from "@react-native-ai/apple/src/NativeAppleTranscription";
2import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test";
3
4import { VoiceTranscriptionError } from "@t3tools/client-runtime/voice-input";
5
6const mocks = vi.hoisted(() => ({
7 isAvailable: vi.fn<(locale: string) => boolean>(),
8 prepare: vi.fn<(locale: string) => Promise<string>>(),
9 transcribe: vi.fn<(audio: ArrayBufferLike, locale: string) => Promise<TranscriptionResult>>(),
10 readAudio: vi.fn<() => Promise<ArrayBuffer>>(),
11}));
12
13vi.mock("@react-native-ai/apple/src/NativeAppleTranscription", () => ({
14 default: {
15 isAvailable: mocks.isAvailable,
16 prepare: mocks.prepare,
17 transcribe: mocks.transcribe,
18 },
19}));
20
21vi.mock("expo-file-system", () => ({
22 File: class {
23 arrayBuffer = mocks.readAudio;
24 },
25}));
26
27import { getLocalVoiceTranscriber } from "./voiceTranscription.ios";
28
29const audio = new ArrayBuffer(4);
30const nativeTranscript: TranscriptionResult = {
31 duration: 2,
32 segments: [
33 { text: " Hej", startSecond: 0, endSecond: 1 },
34 { text: "världen. ", startSecond: 1, endSecond: 2 },
35 ],
36};
37
38function deferred<T>() {
39 let resolve!: (value: T) => void;
40 const promise = new Promise<T>((next) => {
41 resolve = next;
42 });
43 return { promise, resolve };
44}
45
46beforeEach(() => {
47 vi.resetAllMocks();
48 mocks.isAvailable.mockReturnValue(true);
49 mocks.prepare.mockResolvedValue("sv-SE");
50 mocks.readAudio.mockResolvedValue(audio);
51 mocks.transcribe.mockResolvedValue(nativeTranscript);
52});
53
54afterEach(() => {
55 vi.restoreAllMocks();
56});
57
58describe("getLocalVoiceTranscriber", () => {
59 it("keeps the selected language and Apple's resolved locale when the device language changes", async () => {
60 const resolvedOptions = Intl.DateTimeFormat().resolvedOptions();
61 const deviceLocale = vi
62 .spyOn(Intl.DateTimeFormat.prototype, "resolvedOptions")
63 .mockReturnValue({ ...resolvedOptions, locale: "sv-FI" });
64 const transcriber = getLocalVoiceTranscriber()!;
65 const options = { signal: new AbortController().signal };
66
67 deviceLocale.mockReturnValue({ ...resolvedOptions, locale: "de-DE" });
68 const prepared = await transcriber.prepare(options);
69 deviceLocale.mockReturnValue({ ...resolvedOptions, locale: "en-US" });
70
71 await expect(prepared.transcribe("file:///voice.m4a", options)).resolves.toBe("Hej världen.");
72 expect(mocks.prepare).toHaveBeenCalledWith("sv-FI");
73 expect(prepared.locale).toBe("sv-SE");
74 expect(mocks.transcribe).toHaveBeenCalledWith(audio, "sv-SE");
75 });
76
77 it("does not start native transcription after cancellation during a file read", async () => {
78 const enteredRead = deferred<void>();
79 const readResult = deferred<ArrayBuffer>();
80 mocks.readAudio.mockImplementation(() => {
81 enteredRead.resolve();
82 return readResult.promise;
83 });
84 const controller = new AbortController();
85 const options = { signal: controller.signal };
86 const prepared = await getLocalVoiceTranscriber()!.prepare(options);
87 const result = prepared
88 .transcribe("file:///voice.m4a", options)
89 .catch((error: unknown) => error);
90
91 await enteredRead.promise;
92 controller.abort();
93 readResult.resolve(audio);
94
95 const error = await result;
96 expect(error).toBeInstanceOf(VoiceTranscriptionError);
97 expect(error).toMatchObject({ code: "cancelled" });
98 expect(mocks.transcribe).not.toHaveBeenCalled();
99 });
100
101 it.each(["prepare", "transcribe"] as const)(
102 "waits for native %s to finish before settling cancellation",
103 async (phase) => {
104 const enteredNative = deferred<void>();
105 const finishNative = deferred<void>();
106 if (phase === "prepare") {
107 mocks.prepare.mockImplementation(async () => {
108 enteredNative.resolve();
109 await finishNative.promise;
110 return "sv-SE";
111 });
112 } else {
113 mocks.transcribe.mockImplementation(async () => {
114 enteredNative.resolve();
115 await finishNative.promise;
116 return nativeTranscript;
117 });
118 }
119 const controller = new AbortController();
120 const options = { signal: controller.signal };
121 const transcriber = getLocalVoiceTranscriber()!;
122 const operation =
123 phase === "prepare"
124 ? transcriber.prepare(options)
125 : (await transcriber.prepare(options)).transcribe("file:///voice.m4a", options);
126 const settled = vi.fn((value: unknown) => value);
127 const result = operation.then(settled, settled);
128
129 await enteredNative.promise;
130 controller.abort();
131 await new Promise<void>((resolve) => setImmediate(resolve));
132 expect(settled).not.toHaveBeenCalled();
133 finishNative.resolve();
134
135 const error = await result;
136 expect(error).toBeInstanceOf(VoiceTranscriptionError);
137 expect(error).toMatchObject({ code: "cancelled" });
138 },
139 );
140});
apps/mobile/src/native/voiceTranscription.ios.ts created+98
...@@ -0,0 +1,98 @@
1import AppleTranscription from "@react-native-ai/apple/src/NativeAppleTranscription";
2import { File } from "expo-file-system";
3
4import {
5 VoiceTranscriptionError,
6 throwIfVoiceTranscriptionAborted,
7 type PreparedVoiceTranscription,
8 type VoiceTranscriber,
9 type VoiceTranscriptionOptions,
10} from "@t3tools/client-runtime/voice-input";
11
12function getDeviceLocale(): string {
13 return Intl.DateTimeFormat().resolvedOptions().locale;
14}
15
16function wrapError(
17 code: "preparation-failed" | "transcription-failed",
18 message: string,
19 cause: unknown,
20): VoiceTranscriptionError {
21 if (cause instanceof VoiceTranscriptionError) {
22 return cause;
23 }
24
25 return new VoiceTranscriptionError(code, message, { cause });
26}
27
28function getNativeErrorCode(error: unknown): string | undefined {
29 if (typeof error !== "object" || error === null || !("code" in error)) {
30 return undefined;
31 }
32
33 return typeof error.code === "string" ? error.code : undefined;
34}
35
36export function getLocalVoiceTranscriber(): VoiceTranscriber | null {
37 const locale = getDeviceLocale();
38 if (!AppleTranscription.isAvailable(locale)) return null;
39 return { prepare: (options) => prepareVoiceTranscription(locale, options) };
40}
41
42async function prepareVoiceTranscription(
43 locale: string,
44 { signal }: VoiceTranscriptionOptions,
45): Promise<PreparedVoiceTranscription> {
46 throwIfVoiceTranscriptionAborted(signal);
47 if (!AppleTranscription.isAvailable(locale)) {
48 throw new VoiceTranscriptionError(
49 "unavailable",
50 "Voice transcription requires a supported device with iOS 26 or later.",
51 );
52 }
53
54 try {
55 const supportedLocale = await AppleTranscription.prepare(locale);
56 throwIfVoiceTranscriptionAborted(signal);
57 return {
58 locale: supportedLocale,
59 transcribe: (uri, options) => transcribeVoiceRecording(uri, supportedLocale, options),
60 };
61 } catch (error) {
62 throwIfVoiceTranscriptionAborted(signal);
63 if (getNativeErrorCode(error) === "AppleTranscriptionUnsupportedLocale") {
64 throw new VoiceTranscriptionError(
65 "unsupported-locale",
66 "Voice transcription does not support this device language.",
67 { cause: error },
68 );
69 }
70
71 throw wrapError(
72 "preparation-failed",
73 "Voice transcription could not prepare this language.",
74 error,
75 );
76 }
77}
78
79async function transcribeVoiceRecording(
80 uri: string,
81 locale: string,
82 { signal }: VoiceTranscriptionOptions,
83): Promise<string> {
84 try {
85 throwIfVoiceTranscriptionAborted(signal);
86 const audio = await new File(uri).arrayBuffer();
87 throwIfVoiceTranscriptionAborted(signal);
88 const result = await AppleTranscription.transcribe(audio, locale);
89 throwIfVoiceTranscriptionAborted(signal);
90 return result.segments
91 .map((segment) => segment.text)
92 .join(" ")
93 .trim();
94 } catch (error) {
95 throwIfVoiceTranscriptionAborted(signal);
96 throw wrapError("transcription-failed", "Voice transcription failed.", error);
97 }
98}
apps/mobile/src/native/voiceTranscription.ts created+5
...@@ -0,0 +1,5 @@
1import type { VoiceTranscriber } from "@t3tools/client-runtime/voice-input";
2
3export function getLocalVoiceTranscriber(): VoiceTranscriber | null {
4 return null;
5}
docs/README.md+1
...@@ -30,6 +30,7 @@ policy in [CONTRIBUTING.md](../CONTRIBUTING.md); agent rules in [AGENTS.md](../A...@@ -30,6 +30,7 @@ policy in [CONTRIBUTING.md](../CONTRIBUTING.md); agent rules in [AGENTS.md](../A
30- [Glossary](./internals/glossary.md)30- [Glossary](./internals/glossary.md)
31- [Scripts](./internals/scripts.md)31- [Scripts](./internals/scripts.md)
32- [Connection runtime](./internals/connection-runtime.md)32- [Connection runtime](./internals/connection-runtime.md)
33- [Voice input](./internals/voice-input.md)
33- [Providers](./internals/providers.md)34- [Providers](./internals/providers.md)
34- [Remote environments](./internals/remote.md)35- [Remote environments](./internals/remote.md)
35- [Server updates](./internals/server-updates.md)36- [Server updates](./internals/server-updates.md)
docs/internals/voice-input.md created+102
...@@ -0,0 +1,102 @@
1# Voice input
2
3> For maintainers. Using T3 Code? See [voice input on iPhone](../user/composer.md#voice-input-on-iphone).
4
5Voice input produces editable composer text. The current implementation records on the client and
6transcribes locally with Apple's `SpeechAnalyzer` and `SpeechTranscriber` on supported iOS 26+
7devices. Environment-provided transcription and transcription on web and desktop are not implemented.
8
9## Current boundaries
10
11The shared [`VoiceInputController`][controller] in `packages/client-runtime` owns preparation,
12recording, transcription, cancellation, temporary-file cleanup, and insertion into the captured
13draft selection. Applications import it through the [voice-input entry point][voice-input] as
14`@t3tools/client-runtime/voice-input`. Its dependencies separate capture from transcription; the
15controller imports neither React Native nor an Apple transcription API.
16
17The shared [transcription contract][transcription] defines `VoiceTranscriber`,
18`PreparedVoiceTranscription`, and transcription errors. The controller calls `getTranscriber()` once
19at the start of an operation, before asking for microphone permission. Preparation returns a resolved
20locale and a bound `transcribe` function. The controller retains that result for the recording, so a
21selection change cannot prepare with one implementation and transcribe with another.
22
23[`useVoiceInputController`][hook] supplies Expo audio capture, microphone permissions, audio-session
24management, waveform samples, and app and navigation lifecycle handling. It normalizes Expo's
25`mediaServicesDidReset` into a generic recorder error. [`voiceTranscription.ios.ts`][ios] adapts
26`@react-native-ai/apple` through `getLocalVoiceTranscriber()`, capturing the requested device locale
27and binding the prepared transcriber to Apple's resolved locale. The other-platform binding returns
28no local transcriber. That result describes the local implementation, not whether a client could use
29an environment's transcription service.
30
31Mobile's [`voiceInputPresentation.ts`][presentation] maps shared state to toolbar labels and actions.
32Waveform and toolbar rendering stay in mobile. The composer edits draft text without selecting a
33speech vendor.
34Recording captures the draft owner, revision, text, and selection. A late transcript cannot overwrite
35a different or edited draft. Only normal message submission sends the resulting text to an agent.
36
37Each operation passes one `AbortSignal` through preparation and transcription. Cancellation
38invalidates the operation and aborts that signal immediately. Implementations settle their promises
39only after their underlying work stops. The Apple binding checks cancellation between asynchronous
40steps but cannot interrupt an in-flight native request. The controller retains its session until
41that work settles, ignores its result, and cleans up the recording.
42
43## Ownership decisions
44
45The extension boundary distinguishes transcription on the client device from transcription through
46the composer's environment. These constraints apply when adding selectable transcription services:
47
48- Local means the client device, regardless of which machine hosts the environment. A device's lack
49 of local recognition does not prevent it from recording audio for an environment service.
50- Remote service configuration and API keys belong to the environment. The environment calls the
51 external service. Clients receive service identifiers, labels, and availability information, never
52 credential values. Transcription services are independent of coding-agent `providerInstances`;
53 selecting OpenAI for transcription does not select Codex for the thread.
54- The client owns its transcription preference, scoped by stable `environmentId`. Its choices are
55 supported local recognition and the services exposed by the composer's environment. A service ID
56 is meaningful only within that environment. Different clients can make different choices.
57- Resolve and capture the environment, selected service, and locale when an operation starts.
58 Preparation and transcription use the same selection; preference changes affect the next
59 recording. Capture environment identity explicitly rather than recovering it from a draft key.
60 Keep the existing draft-owner and revision checks before inserting text.
61- If the selected option is unavailable, report that state and let the user choose another option.
62 A local failure must not silently upload audio, and a disconnected environment must not redirect
63 a recording to another environment or service.
64- Transcription audio is temporary input, separate from durable chat attachments and messages.
65 Remote adapters need cancellation of upload and transcription where supported, cleanup after
66 success, failure, or cancellation, and the same protection against late results as local transcription.
67
68## Existing integration points
69
70[`ServerSettingsService`][settings] and [`ServerSecretStore`][secrets] provide environment-owned
71configuration and secret persistence. Existing settings redaction handles coding-provider environment
72variables only. Any transcription credential fields need their own explicit separation and redaction
73before settings responses or subscriptions reach client caches.
74
75[`ExecutionEnvironmentCapabilities`][capabilities] handles version skew. Remote transcription must be
76opt-in: a missing transcription capability means unsupported. The authenticated server-config
77subscription and [shared environment state][server-state] already distribute configuration per
78environment. A transcription service catalog belongs behind that capability and authenticated
79boundary. Older servers expose no remote transcription choices.
80
81The [attachment upload contracts][uploads] and [shared upload operations][attachment-state] provide a
82pattern for authorized binary uploads through an environment, including remote connections. Their
83existing chat-attachment retention is not a transcription cleanup policy.
84
85Future service selection and environment requests belong alongside the controller in
86`packages/client-runtime`, with wire contracts in `packages/contracts`. Capture and native local
87recognition remain client-specific. An environment-backed transcriber implements the same shared
88contract, with its environment and service bound when selected. The controller does not own service
89credentials, provider SDKs, or transport selection.
90
91[controller]: ../../packages/client-runtime/src/voice-input/controller.ts
92[voice-input]: ../../packages/client-runtime/src/voice-input/index.ts
93[transcription]: ../../packages/client-runtime/src/voice-input/transcription.ts
94[hook]: ../../apps/mobile/src/features/voice-input/useVoiceInputController.ts
95[presentation]: ../../apps/mobile/src/features/voice-input/voiceInputPresentation.ts
96[ios]: ../../apps/mobile/src/native/voiceTranscription.ios.ts
97[settings]: ../../apps/server/src/serverSettings.ts
98[secrets]: ../../apps/server/src/auth/ServerSecretStore.ts
99[capabilities]: ../../packages/contracts/src/environment.ts
100[server-state]: ../../packages/client-runtime/src/state/server.ts
101[uploads]: ../../packages/contracts/src/assets.ts
102[attachment-state]: ../../packages/client-runtime/src/state/attachments.ts
docs/user/composer.md+16
...@@ -38,6 +38,22 @@ uploaded. Stashed files stay uploaded on the server for 24 hours. If you restore...@@ -38,6 +38,22 @@ uploaded. Stashed files stay uploaded on the server for 24 hours. If you restore
38that, the file comes back with **Attach again** next to it. Attach the file again or remove it, then38that, the file comes back with **Attach again** next to it. Attach the file again or remove it, then
39send.39send.
4040
41## Voice input on iPhone
42
43On supported iPhones with iOS 26 or later, tap the microphone in the composer to record a message.
44An expanded composer keeps your draft visible and replaces its toolbar with waves that respond to
45your voice. A collapsed composer shows a compact recording strip instead. Tap the checkmark to
46finish and transcribe on your device. The waves fade into a transcription status, then the usual
47controls return with the text inserted at the selection where recording started. If the keyboard
48is open when you start, it stays open during voice input. You can review and edit the text before
49you send it.
50
51The first use can download Apple's speech model and needs a network connection. Later transcription
52works offline for that language. A recording can be up to five minutes long. Canceling voice input,
53leaving the screen, or an audio interruption discards the new recording and keeps the existing draft
54and attachments. T3 Code deletes the local audio file after transcription or cancellation. It sends
55only the normal message text when you submit the draft.
56
41## Commands and skills57## Commands and skills
4258
43Type `/` to open the command menu. Type `$` to find and add a skill. Skill rows show their source,59Type `/` to open the command menu. Type `$` to find and add a skill. Skill rows show their source,
packages/client-runtime/README.md+18-12
...@@ -5,18 +5,19 @@ subpath. The package intentionally has no root export....@@ -5,18 +5,19 @@ subpath. The package intentionally has no root export.
55
6## Public subpaths6## Public subpaths
77
8| Subpath | Responsibility |8| Subpath | Responsibility |
9| --------------------- | ---------------------------------------------------------------- |9| --------------------- | ----------------------------------------------------------------- |
10| `authorization` | Bearer and DPoP authorization plus token persistence contracts |10| `authorization` | Bearer and DPoP authorization plus token persistence contracts |
11| `connection` | Targets, catalog, supervision, retries, registry, and onboarding |11| `connection` | Targets, catalog, supervision, retries, registry, and onboarding |
12| `environment` | Environment identity, descriptors, endpoints, and scoped keys |12| `environment` | Environment identity, descriptors, endpoints, and scoped keys |
13| `errors` | Shared client error inspection |13| `errors` | Shared client error inspection |
14| `operations` | Multi-step application workflows |14| `operations` | Multi-step application workflows |
15| `operations/projects` | Multi-step project creation workflows |15| `operations/projects` | Multi-step project creation workflows |
16| `platform` | Platform capability and persistence service contracts |16| `platform` | Platform capability and persistence service contracts |
17| `relay` | Managed relay API and environment discovery |17| `relay` | Managed relay API and environment discovery |
18| `rpc` | HTTP/RPC clients, protocol, sessions, and subscriptions |18| `rpc` | HTTP/RPC clients, protocol, sessions, and subscriptions |
19| `state/<domain>` | Focused shared state, retention, reducers, and Atom constructors |19| `state/<domain>` | Focused shared state, retention, reducers, and Atom constructors |
20| `voice-input` | Recording lifecycle, transcription contracts, and draft insertion |
2021
21## Dependency direction22## Dependency direction
2223
...@@ -25,6 +26,11 @@ capabilities with `authorization`, `relay`, and `rpc` to supervise environment...@@ -25,6 +26,11 @@ capabilities with `authorization`, `relay`, and `rpc` to supervise environment
25sessions. Independent `state` modules consume the connection registry and expose26sessions. Independent `state` modules consume the connection registry and expose
26focused state or Atom constructors to application-owned runtimes.27focused state or Atom constructors to application-owned runtimes.
2728
29The `voice-input` controller accepts capture callbacks and a selected `VoiceTranscriber`.
30Preparation binds transcription to its implementation and resolved locale; one cancellation
31signal covers both operations. Applications provide recorder events, permissions, native
32transcription implementations, and presentation.
33
28Applications should import the narrowest relevant subpath. There is no broad34Applications should import the narrowest relevant subpath. There is no broad
29`state` export: use domain paths such as `state/shell`, `state/threads`,35`state` export: use domain paths such as `state/shell`, `state/threads`,
30`state/terminal`, or `state/vcs`. Subpath indices and explicitly exported domain36`state/terminal`, or `state/vcs`. Subpath indices and explicitly exported domain
packages/client-runtime/package.json+4
...@@ -55,6 +55,10 @@...@@ -55,6 +55,10 @@
55 "types": "./src/providerSkills.ts",55 "types": "./src/providerSkills.ts",
56 "default": "./src/providerSkills.ts"56 "default": "./src/providerSkills.ts"
57 },57 },
58 "./voice-input": {
59 "types": "./src/voice-input/index.ts",
60 "default": "./src/voice-input/index.ts"
61 },
58 "./relay": {62 "./relay": {
59 "types": "./src/relay/index.ts",63 "types": "./src/relay/index.ts",
60 "default": "./src/relay/index.ts"64 "default": "./src/relay/index.ts"
packages/client-runtime/src/voice-input/controller.test.ts created+535
...@@ -0,0 +1,535 @@
1import { beforeEach, describe, expect, it, vi } from "vite-plus/test";
2
3import {
4 resetVoiceInputGlobalsForTests,
5 resolveTranscriptCommit,
6 VoiceInputController,
7 VOICE_RECORDING_LIMIT_SECONDS,
8 voiceInputBlocksSubmission,
9 type VoiceDraftSnapshot,
10 type VoiceInputControllerDependencies,
11 type VoiceRecorder,
12} from "./controller.ts";
13import type { PreparedVoiceTranscription, VoiceTranscriber } from "./transcription.ts";
14
15function deferred<T>() {
16 let resolve!: (value: T) => void;
17 let reject!: (error: unknown) => void;
18 const promise = new Promise<T>((resolvePromise, rejectPromise) => {
19 resolve = resolvePromise;
20 reject = rejectPromise;
21 });
22 return { promise, resolve, reject };
23}
24
25class TestRecorder implements VoiceRecorder {
26 uri: string | null = "file:///voice.m4a";
27 readonly prepareToRecordAsync = vi.fn(async () => undefined);
28 readonly record = vi.fn();
29 readonly stop = vi.fn(async () => undefined);
30}
31
32function preparedTranscription(
33 transcribe: PreparedVoiceTranscription["transcribe"] = async () => "new text",
34): PreparedVoiceTranscription {
35 return { locale: "en-US", transcribe };
36}
37
38function draft(overrides: Partial<VoiceDraftSnapshot> = {}): VoiceDraftSnapshot {
39 return {
40 ownerKey: "environment:thread",
41 text: "hello world",
42 selection: { start: 6, end: 11 },
43 revision: 1,
44 ...overrides,
45 };
46}
47
48function createHarness(
49 overrides: Partial<VoiceInputControllerDependencies> = {},
50 initialDraft = draft(),
51) {
52 const recorder = new TestRecorder();
53 let currentDraft: VoiceDraftSnapshot | null = initialDraft;
54 const commits: Array<{ text: string; selection: { start: number; end: number } }> = [];
55 const deleted: string[] = [];
56 const dependencies: VoiceInputControllerDependencies = {
57 recorder,
58 getTranscriber: () => ({ prepare: async () => preparedTranscription() }),
59 requestPermission: async () => ({ granted: true, canAskAgain: true }),
60 configureRecording: async () => undefined,
61 releaseRecording: async () => undefined,
62 deleteRecording: (uri) => deleted.push(uri),
63 readDraft: () => currentDraft,
64 commitDraft: (text, selection) => commits.push({ text, selection }),
65 onStateChange: vi.fn(),
66 ...overrides,
67 };
68 return {
69 controller: new VoiceInputController(dependencies),
70 recorder,
71 commits,
72 deleted,
73 setDraft: (next: VoiceDraftSnapshot | null) => {
74 currentDraft = next;
75 },
76 };
77}
78
79describe("resolveTranscriptCommit", () => {
80 it("replaces the recorded UTF-16 selection around emoji and composer tokens", () => {
81 const text = "Fix 🧪 then $review please";
82 const tokenStart = text.indexOf("$review");
83 const captured = draft({
84 text,
85 selection: { start: tokenStart, end: tokenStart + "$review".length },
86 });
87
88 expect(resolveTranscriptCommit(captured, captured, "use the mobile skill", "en-US")).toEqual({
89 kind: "commit",
90 text: "Fix 🧪 then use the mobile skill please",
91 selection: { start: tokenStart + "use the mobile skill".length, end: tokenStart + 20 },
92 });
93 });
94
95 it("does not replace text after the owner, text, or revision changes", () => {
96 const captured = draft();
97 expect(
98 resolveTranscriptCommit(captured, draft({ ownerKey: "other" }), "text", "en-US"),
99 ).toEqual({
100 kind: "stale",
101 });
102 expect(resolveTranscriptCommit(captured, draft({ text: "newer" }), "text", "en-US")).toEqual({
103 kind: "stale",
104 });
105 expect(resolveTranscriptCommit(captured, draft({ revision: 2 }), "text", "en-US")).toEqual({
106 kind: "stale",
107 });
108 });
109
110 it("adds English spacing at empty start, middle, and end caret boundaries", () => {
111 const atEnd = draft({
112 text: "Fix cache.",
113 selection: { start: "Fix cache.".length, end: "Fix cache.".length },
114 });
115 expect(resolveTranscriptCommit(atEnd, atEnd, "Also fix tests.", "en-US")).toMatchObject({
116 kind: "commit",
117 text: "Fix cache. Also fix tests.",
118 });
119 expect(resolveTranscriptCommit(atEnd, atEnd, "Also fix tests.", "en_US")).toMatchObject({
120 kind: "commit",
121 text: "Fix cache. Also fix tests.",
122 });
123
124 const atStart = draft({ text: "Fix cache.", selection: { start: 0, end: 0 } });
125 expect(resolveTranscriptCommit(atStart, atStart, "First", "en-US")).toMatchObject({
126 kind: "commit",
127 text: "First Fix cache.",
128 });
129
130 const inMiddle = draft({ text: "Fix cache.", selection: { start: 4, end: 4 } });
131 expect(resolveTranscriptCommit(inMiddle, inMiddle, "also", "en-US")).toMatchObject({
132 kind: "commit",
133 text: "Fix also cache.",
134 });
135 });
136
137 it("does not add English boundary spaces to CJK or selected inline text", () => {
138 const cjk = draft({ text: "修正キャッシュ", selection: { start: 8, end: 8 } });
139 expect(resolveTranscriptCommit(cjk, cjk, "テストも", "ja-JP")).toMatchObject({
140 kind: "commit",
141 text: "修正キャッシュテストも",
142 });
143
144 const selected = draft({ text: "one $skill two", selection: { start: 4, end: 10 } });
145 expect(resolveTranscriptCommit(selected, selected, "new", "en-US")).toMatchObject({
146 kind: "commit",
147 text: "one new two",
148 });
149 });
150});
151
152describe("VoiceInputController", () => {
153 beforeEach(() => resetVoiceInputGlobalsForTests());
154
155 it("checks support and permission before recording", async () => {
156 const unsupported = createHarness({ getTranscriber: () => null });
157 await unsupported.controller.start();
158 expect(unsupported.controller.currentState.error).toContain("not available");
159 expect(unsupported.recorder.record).not.toHaveBeenCalled();
160
161 const denied = createHarness({
162 requestPermission: async () => ({ granted: false, canAskAgain: false }),
163 });
164 await denied.controller.start();
165 expect(denied.controller.currentState.errorAction).toBe("settings");
166 expect(denied.recorder.record).not.toHaveBeenCalled();
167 });
168
169 it.each(["permission", "transcription"] as const)(
170 "clears %s errors when switching to another draft",
171 async (failure) => {
172 const harness = createHarness(
173 failure === "permission"
174 ? { requestPermission: async () => ({ granted: false, canAskAgain: false }) }
175 : {
176 getTranscriber: () => ({
177 prepare: async () =>
178 preparedTranscription(async () => {
179 throw new Error("Transcription failed");
180 }),
181 }),
182 },
183 );
184 await harness.controller.start();
185 await harness.controller.stop();
186 expect(harness.controller.currentState).toMatchObject({
187 phase: "error",
188 error: expect.any(String),
189 errorAction: failure === "permission" ? "settings" : "retry",
190 });
191
192 harness.setDraft(draft({ ownerKey: "environment:other-thread" }));
193 harness.controller.ownerChanged();
194
195 expect(harness.controller.currentState).toEqual({
196 phase: "idle",
197 error: null,
198 errorAction: null,
199 });
200 expect(harness.commits).toEqual([]);
201 },
202 );
203
204 it.each(["permission", "preparation", "recording"] as const)(
205 "keeps the selected transcriber when preferences change during %s",
206 async (changeDuring) => {
207 const permission = deferred<{ granted: boolean; canAskAgain: boolean }>();
208 const permissionEntered = deferred<void>();
209 const preparation = deferred<void>();
210 const preparationEntered = deferred<void>();
211 const preparationSignals: AbortSignal[] = [];
212 const transcriptionSignals: AbortSignal[] = [];
213 const transcriber = (text: string): VoiceTranscriber => ({
214 prepare: async ({ signal }) => {
215 preparationSignals.push(signal);
216 preparationEntered.resolve(undefined);
217 await preparation.promise;
218 return preparedTranscription(async (_uri, { signal }) => {
219 transcriptionSignals.push(signal);
220 return text;
221 });
222 },
223 });
224 const first = transcriber("first choice");
225 const second = transcriber("second choice");
226 let selected = first;
227 const harness = createHarness({
228 getTranscriber: () => selected,
229 requestPermission: () => {
230 permissionEntered.resolve(undefined);
231 return permission.promise;
232 },
233 });
234
235 const starting = harness.controller.start();
236 await permissionEntered.promise;
237 if (changeDuring === "permission") selected = second;
238 permission.resolve({ granted: true, canAskAgain: true });
239 await preparationEntered.promise;
240 if (changeDuring === "preparation") selected = second;
241 preparation.resolve(undefined);
242 await starting;
243 if (changeDuring === "recording") selected = second;
244 await harness.controller.stop();
245
246 expect(harness.commits.map((commit) => commit.text)).toEqual(["hello first choice"]);
247
248 await harness.controller.start();
249 await harness.controller.stop();
250
251 expect(harness.commits.map((commit) => commit.text)).toEqual([
252 "hello first choice",
253 "hello second choice",
254 ]);
255 expect(preparationSignals).toHaveLength(2);
256 expect(transcriptionSignals).toHaveLength(2);
257 expect(transcriptionSignals[0]).toBe(preparationSignals[0]);
258 expect(transcriptionSignals[1]).toBe(preparationSignals[1]);
259 expect(preparationSignals[1]).not.toBe(preparationSignals[0]);
260 },
261 );
262
263 it("blocks submit while voice input can still change the draft", () => {
264 expect(voiceInputBlocksSubmission({ phase: "preparing", error: null, errorAction: null })).toBe(
265 true,
266 );
267 expect(voiceInputBlocksSubmission({ phase: "recording", error: null, errorAction: null })).toBe(
268 true,
269 );
270 expect(
271 voiceInputBlocksSubmission({ phase: "transcribing", error: null, errorAction: null }),
272 ).toBe(true);
273 expect(voiceInputBlocksSubmission({ phase: "idle", error: null, errorAction: null })).toBe(
274 false,
275 );
276 });
277
278 it("uses the native five-minute cap and commits one final transcript", async () => {
279 const harness = createHarness();
280 await harness.controller.start();
281 expect(harness.recorder.record).toHaveBeenCalledWith({
282 forDuration: VOICE_RECORDING_LIMIT_SECONDS,
283 });
284
285 const stopping = harness.controller.stop();
286 harness.controller.handleRecorderStatus({
287 isFinished: true,
288 hasError: false,
289 error: null,
290 url: "file:///voice.m4a",
291 });
292 await stopping;
293
294 expect(harness.commits).toEqual([
295 { text: "hello new text", selection: { start: 14, end: 14 } },
296 ]);
297 expect(harness.deleted).toEqual(["file:///voice.m4a"]);
298 });
299
300 it.each(["cancel", "dispose", "ownerChanged"] as const)(
301 "holds the session after %s until non-abortable transcription settles",
302 async (action) => {
303 const transcription = deferred<string>();
304 const transcriptionEntered = deferred<AbortSignal>();
305 const harness = createHarness({
306 getTranscriber: () => ({
307 prepare: async () =>
308 preparedTranscription((_uri, { signal }) => {
309 transcriptionEntered.resolve(signal);
310 return transcription.promise;
311 }),
312 }),
313 });
314 await harness.controller.start();
315 const stopping = harness.controller.stop();
316 const signal = await transcriptionEntered.promise;
317 expect(signal.aborted).toBe(false);
318 if (action === "ownerChanged") {
319 harness.setDraft(draft({ ownerKey: "environment:other-thread" }));
320 }
321 harness.controller[action]();
322 expect(signal.aborted).toBe(true);
323
324 const next = createHarness();
325 await next.controller.start();
326 expect(next.controller.currentState.error).toContain("already active");
327 expect(next.recorder.record).not.toHaveBeenCalled();
328
329 transcription.resolve("late text");
330 await stopping;
331
332 expect(harness.commits).toEqual([]);
333 expect(harness.deleted).toEqual(["file:///voice.m4a"]);
334 expect(harness.controller.currentState.phase).toBe("idle");
335
336 await next.controller.start();
337 expect(next.controller.currentState.phase).toBe("recording");
338 await next.controller.interruptRecording();
339 },
340 );
341
342 it("cancels an in-flight transcriber that rejects when its signal aborts", async () => {
343 const transcription = deferred<string>();
344 const transcriptionEntered = deferred<AbortSignal>();
345 const harness = createHarness({
346 getTranscriber: () => ({
347 prepare: async () =>
348 preparedTranscription((_uri, { signal }) => {
349 signal.addEventListener("abort", () => transcription.reject(new Error("aborted")), {
350 once: true,
351 });
352 transcriptionEntered.resolve(signal);
353 return transcription.promise;
354 }),
355 }),
356 });
357 await harness.controller.start();
358 const stopping = harness.controller.stop();
359 const signal = await transcriptionEntered.promise;
360 harness.controller.cancel();
361 await stopping;
362
363 expect(signal.aborted).toBe(true);
364 expect(harness.commits).toEqual([]);
365 expect(harness.deleted).toEqual(["file:///voice.m4a"]);
366 expect(harness.controller.currentState.phase).toBe("idle");
367 });
368
369 it("releases the microphone before transcription starts", async () => {
370 const events: string[] = [];
371 const harness = createHarness({
372 releaseRecording: async () => {
373 events.push("released");
374 },
375 getTranscriber: () => ({
376 prepare: async () =>
377 preparedTranscription(async () => {
378 events.push("transcribed");
379 return "done";
380 }),
381 }),
382 });
383 await harness.controller.start();
384 await harness.controller.stop();
385
386 expect(events).toEqual(["released", "transcribed"]);
387 });
388
389 it("retries audio-session release during final cleanup", async () => {
390 const releaseRecording = vi
391 .fn<() => Promise<void>>()
392 .mockRejectedValueOnce(new Error("busy"))
393 .mockResolvedValueOnce(undefined);
394 const harness = createHarness({ releaseRecording });
395 await harness.controller.start();
396 await harness.controller.stop();
397
398 expect(releaseRecording).toHaveBeenCalledTimes(2);
399 });
400
401 it("leaves transcription with an error when recorder finalization fails", async () => {
402 const harness = createHarness();
403 harness.recorder.stop.mockRejectedValueOnce(new Error("stop failed"));
404 await harness.controller.start();
405 await harness.controller.stop();
406
407 expect(harness.controller.currentState.phase).toBe("error");
408 expect(harness.controller.currentState.error).toContain("finish voice recording");
409 });
410
411 it("ignores a late transcript after the draft owner changes", async () => {
412 const transcription = deferred<string>();
413 const transcriptionEntered = deferred<void>();
414 const harness = createHarness({
415 getTranscriber: () => ({
416 prepare: async () =>
417 preparedTranscription(() => {
418 transcriptionEntered.resolve(undefined);
419 return transcription.promise;
420 }),
421 }),
422 });
423 await harness.controller.start();
424 const stopping = harness.controller.stop();
425 await transcriptionEntered.promise;
426 harness.setDraft(draft({ ownerKey: "environment:other-thread" }));
427 transcription.resolve("late text");
428 await stopping;
429
430 expect(harness.commits).toEqual([]);
431 expect(harness.controller.currentState.error).toContain("draft changed");
432 });
433
434 it("keeps the app-wide session locked until canceled preparation settles", async () => {
435 const preparation = deferred<PreparedVoiceTranscription>();
436 const preparationEntered = deferred<AbortSignal>();
437 const first = createHarness({
438 getTranscriber: () => ({
439 prepare: ({ signal }) => {
440 preparationEntered.resolve(signal);
441 return preparation.promise;
442 },
443 }),
444 });
445 const firstStart = first.controller.start();
446 const signal = await preparationEntered.promise;
447 first.controller.cancel();
448 expect(signal.aborted).toBe(true);
449
450 const blocked = createHarness();
451 await blocked.controller.start();
452 expect(blocked.controller.currentState.error).toContain("already active");
453
454 preparation.resolve(preparedTranscription());
455 await firstStart;
456 expect(first.recorder.record).not.toHaveBeenCalled();
457 blocked.controller.cancel();
458
459 const next = createHarness();
460 await next.controller.start();
461 expect(next.controller.currentState.phase).toBe("recording");
462 await next.controller.interruptRecording();
463 });
464
465 it("does not start the microphone for an owner that changed during preparation", async () => {
466 const preparation = deferred<PreparedVoiceTranscription>();
467 const preparationEntered = deferred<void>();
468 const harness = createHarness({
469 getTranscriber: () => ({
470 prepare: () => {
471 preparationEntered.resolve(undefined);
472 return preparation.promise;
473 },
474 }),
475 });
476 const starting = harness.controller.start();
477 await preparationEntered.promise;
478 harness.setDraft(draft({ ownerKey: "environment:other-thread", text: "other draft" }));
479 preparation.resolve(preparedTranscription());
480 await starting;
481
482 expect(harness.recorder.record).not.toHaveBeenCalled();
483 expect(harness.controller.currentState.error).toContain("no longer available");
484 });
485
486 it("discards recorder errors and audio interruptions without transcribing", async () => {
487 const transcribe = vi.fn(async () => "ignored");
488 const preparationEntered = deferred<AbortSignal>();
489 const harness = createHarness({
490 getTranscriber: () => ({
491 prepare: async ({ signal }) => {
492 preparationEntered.resolve(signal);
493 return preparedTranscription(transcribe);
494 },
495 }),
496 });
497 await harness.controller.start();
498 const signal = await preparationEntered.promise;
499 harness.recorder.uri = "file:///reset-empty.m4a";
500 await harness.controller.handleRecorderStatus({
501 isFinished: true,
502 hasError: true,
503 error: "Audio route changed",
504 url: "file:///voice.m4a",
505 });
506
507 expect(harness.commits).toEqual([]);
508 expect(transcribe).not.toHaveBeenCalled();
509 expect(signal.aborted).toBe(true);
510 expect(harness.controller.currentState.error).toBe("Audio route changed");
511 expect(harness.deleted).toEqual(["file:///voice.m4a", "file:///reset-empty.m4a"]);
512 });
513
514 it("cancels preparation when the app reaches the background", async () => {
515 const preparation = deferred<PreparedVoiceTranscription>();
516 const preparationEntered = deferred<AbortSignal>();
517 const harness = createHarness({
518 getTranscriber: () => ({
519 prepare: ({ signal }) => {
520 preparationEntered.resolve(signal);
521 return preparation.promise;
522 },
523 }),
524 });
525 const starting = harness.controller.start();
526 const signal = await preparationEntered.promise;
527 harness.controller.appMovedToBackground();
528 expect(signal.aborted).toBe(true);
529 preparation.resolve(preparedTranscription());
530 await starting;
531
532 expect(harness.recorder.record).not.toHaveBeenCalled();
533 expect(harness.controller.currentState.error).toContain("background");
534 });
535});
packages/client-runtime/src/voice-input/controller.ts created+493
...@@ -0,0 +1,493 @@
1import { replaceTextRange } from "@t3tools/shared/composerTrigger";
2
3import type { PreparedVoiceTranscription, VoiceTranscriber } from "./transcription.ts";
4
5export const VOICE_RECORDING_LIMIT_SECONDS = 5 * 60;
6
7export type VoiceInputPhase = "idle" | "preparing" | "recording" | "transcribing" | "error";
8
9export type VoiceInputState = {
10 readonly phase: VoiceInputPhase;
11 readonly error: string | null;
12 readonly errorAction: "retry" | "settings" | null;
13};
14
15export function voiceInputBlocksSubmission(state: VoiceInputState): boolean {
16 return (
17 state.phase === "preparing" || state.phase === "recording" || state.phase === "transcribing"
18 );
19}
20
21export function voiceInputFreezesEditor(state: VoiceInputState): boolean {
22 return voiceInputBlocksSubmission(state);
23}
24
25export type VoiceDraftSnapshot = {
26 readonly ownerKey: string;
27 readonly text: string;
28 readonly selection: { readonly start: number; readonly end: number };
29 readonly revision: number;
30};
31
32export type VoiceRecorderStatus = {
33 readonly isFinished: boolean;
34 readonly hasError: boolean;
35 readonly error: string | null;
36 readonly url: string | null;
37};
38
39export interface VoiceRecorder {
40 readonly uri: string | null;
41 prepareToRecordAsync(): Promise<void>;
42 record(options: { readonly forDuration: number }): void;
43 stop(): Promise<void>;
44}
45
46export type VoiceInputControllerDependencies = {
47 readonly recorder: VoiceRecorder;
48 readonly getTranscriber: () => VoiceTranscriber | null;
49 readonly requestPermission: () => Promise<{
50 readonly granted: boolean;
51 readonly canAskAgain: boolean;
52 }>;
53 readonly configureRecording: () => Promise<void>;
54 readonly releaseRecording: () => Promise<void>;
55 readonly deleteRecording: (uri: string) => void;
56 readonly readDraft: () => VoiceDraftSnapshot | null;
57 readonly commitDraft: (
58 text: string,
59 selection: { readonly start: number; readonly end: number },
60 ) => void;
61 readonly onStateChange: (state: VoiceInputState) => void;
62};
63
64type TranscriptCommitResult =
65 | {
66 readonly kind: "commit";
67 readonly text: string;
68 readonly selection: { readonly start: number; readonly end: number };
69 }
70 | { readonly kind: "stale" }
71 | { readonly kind: "empty" };
72
73export function resolveTranscriptCommit(
74 captured: VoiceDraftSnapshot,
75 current: VoiceDraftSnapshot | null,
76 transcript: string,
77 locale: string,
78): TranscriptCommitResult {
79 if (
80 !current ||
81 current.ownerKey !== captured.ownerKey ||
82 current.text !== captured.text ||
83 current.revision !== captured.revision
84 ) {
85 return { kind: "stale" };
86 }
87
88 const replacement = transcript.trim();
89 if (replacement.length === 0) {
90 return { kind: "empty" };
91 }
92
93 const isEmptySelection = captured.selection.start === captured.selection.end;
94 const normalizedLocale = locale.replaceAll("_", "-").toLowerCase();
95 const usesEnglishSpacing = normalizedLocale === "en" || normalizedLocale.startsWith("en-");
96 let insertion = replacement;
97 if (isEmptySelection && usesEnglishSpacing) {
98 const left = captured.text[captured.selection.start - 1];
99 const right = captured.text[captured.selection.start];
100 const leftNeedsBoundary =
101 left !== undefined &&
102 /[A-Za-z0-9.!?,:;)\]}'"]/.test(left) &&
103 (right === undefined || /\s/.test(right));
104 const rightNeedsBoundary =
105 right !== undefined &&
106 /[A-Za-z0-9([{'"]/.test(right) &&
107 (left === undefined || /\s/.test(left));
108 insertion = `${leftNeedsBoundary ? " " : ""}${replacement}${rightNeedsBoundary ? " " : ""}`;
109 }
110
111 const result = replaceTextRange(
112 captured.text,
113 captured.selection.start,
114 captured.selection.end,
115 insertion,
116 );
117 return {
118 kind: "commit",
119 text: result.text,
120 selection: { start: result.cursor, end: result.cursor },
121 };
122}
123
124let activeSession: symbol | null = null;
125let activeTranscriptionOperation: Promise<unknown> | null = null;
126
127function acquireSession(): symbol | null {
128 if (activeSession) return null;
129 const token = Symbol("voice-input-session");
130 activeSession = token;
131 return token;
132}
133
134function releaseSession(token: symbol | null): void {
135 if (token && activeSession === token) activeSession = null;
136}
137
138async function runTranscriptionOperation<T>(operation: () => Promise<T>): Promise<T> {
139 if (activeTranscriptionOperation) {
140 throw new Error("voice-operation-busy");
141 }
142
143 const promise = operation();
144 activeTranscriptionOperation = promise;
145 try {
146 return await promise;
147 } finally {
148 if (activeTranscriptionOperation === promise) activeTranscriptionOperation = null;
149 }
150}
151
152function errorCode(error: unknown): string | null {
153 if (typeof error !== "object" || error === null || !("code" in error)) return null;
154 return typeof error.code === "string" ? error.code : null;
155}
156
157function preparationErrorMessage(error: unknown): string {
158 if (error instanceof Error && error.message === "voice-operation-busy") {
159 return "Voice transcription is still finishing. Try again shortly.";
160 }
161 if (errorCode(error) === "unsupported-locale") {
162 return "Voice transcription is not available for this language.";
163 }
164 return "Could not prepare voice transcription.";
165}
166
167function transcriptionErrorMessage(error: unknown): string {
168 if (error instanceof Error && error.message === "voice-operation-busy") {
169 return "Voice transcription is still finishing. Try again shortly.";
170 }
171 return "Could not transcribe this recording.";
172}
173
174const IDLE_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: null };
175
176export class VoiceInputController {
177 private readonly dependencies: VoiceInputControllerDependencies;
178 private state: VoiceInputState = IDLE_STATE;
179 private operationToken = 0;
180 private sessionToken: symbol | null = null;
181 private transcription: PreparedVoiceTranscription | null = null;
182 private transcriptionAbortController: AbortController | null = null;
183 private capturedDraft: VoiceDraftSnapshot | null = null;
184 private recordingUri: string | null = null;
185 private readonly ownedRecordingUris = new Set<string>();
186 private recordingConfigured = false;
187 private finishing = false;
188
189 constructor(dependencies: VoiceInputControllerDependencies) {
190 this.dependencies = dependencies;
191 }
192
193 get currentState(): VoiceInputState {
194 return this.state;
195 }
196
197 async start(): Promise<void> {
198 if (this.state.phase !== "idle" && this.state.phase !== "error") return;
199 const initiatingDraft = this.dependencies.readDraft();
200 if (!initiatingDraft) {
201 this.setError("This draft is no longer available.", "retry");
202 return;
203 }
204 const sessionToken = acquireSession();
205 if (!sessionToken) {
206 this.setError("Another voice recording is already active.", "retry");
207 return;
208 }
209
210 this.sessionToken = sessionToken;
211 const operationToken = ++this.operationToken;
212 const abortController = new AbortController();
213 this.transcriptionAbortController = abortController;
214 this.setState({ phase: "preparing", error: null, errorAction: null });
215
216 try {
217 const transcriber = this.dependencies.getTranscriber();
218 if (!transcriber) {
219 this.setError("Voice transcription is not available.", null);
220 return;
221 }
222
223 const permission = await this.dependencies.requestPermission();
224 if (!this.isCurrent(operationToken)) return;
225 if (!permission.granted) {
226 this.setError(
227 "Microphone access is required for voice input.",
228 permission.canAskAgain ? "retry" : "settings",
229 );
230 return;
231 }
232
233 try {
234 this.transcription = await runTranscriptionOperation(() =>
235 transcriber.prepare({ signal: abortController.signal }),
236 );
237 } catch (error) {
238 if (this.isCurrent(operationToken)) this.setError(preparationErrorMessage(error), "retry");
239 return;
240 }
241 if (!this.isCurrent(operationToken)) return;
242
243 await this.dependencies.configureRecording();
244 this.recordingConfigured = true;
245 if (!this.isCurrent(operationToken)) return;
246 await this.dependencies.recorder.prepareToRecordAsync();
247 if (!this.isCurrent(operationToken)) return;
248 this.recordingUri = this.dependencies.recorder.uri;
249 this.rememberRecordingUri(this.recordingUri);
250
251 const capturedDraft = this.dependencies.readDraft();
252 if (!capturedDraft || capturedDraft.ownerKey !== initiatingDraft.ownerKey) {
253 this.setError("This draft is no longer available.", "retry");
254 return;
255 }
256 this.capturedDraft = capturedDraft;
257 this.dependencies.recorder.record({ forDuration: VOICE_RECORDING_LIMIT_SECONDS });
258 this.setState({ phase: "recording", error: null, errorAction: null });
259 } catch {
260 if (this.isCurrent(operationToken))
261 this.setError("Could not start voice recording.", "retry");
262 } finally {
263 if (this.isCurrent(operationToken) && this.state.phase === "error") {
264 await this.releaseResources();
265 } else if (!this.isCurrent(operationToken) && !this.finishing) {
266 await this.releaseResources();
267 }
268 }
269 }
270
271 stop(): Promise<void> {
272 if (this.state.phase !== "recording") return Promise.resolve();
273 return this.finishRecording(false, null);
274 }
275
276 cancel(): void {
277 switch (this.state.phase) {
278 case "idle":
279 return;
280 case "error":
281 this.setState(IDLE_STATE);
282 return;
283 case "preparing":
284 this.invalidateOperation();
285 this.setState(IDLE_STATE);
286 return;
287 case "recording":
288 this.discardRecording(null);
289 return;
290 case "transcribing":
291 this.invalidateOperation();
292 this.setState(IDLE_STATE);
293 return;
294 }
295 }
296
297 interruptRecording(
298 message = "Voice recording was interrupted.",
299 completedUri: string | null = null,
300 ): Promise<void> | void {
301 if (this.state.phase !== "recording") return;
302 this.rememberRecordingUri(completedUri);
303 this.recordingUri = completedUri ?? this.recordingUri;
304 return this.discardRecording(message);
305 }
306
307 appMovedToBackground(): Promise<void> | void {
308 if (this.state.phase === "preparing") {
309 this.invalidateOperation();
310 this.setError("Voice input stopped when the app moved to the background.", "retry");
311 return;
312 }
313 return this.interruptRecording();
314 }
315
316 handleRecorderStatus(status: VoiceRecorderStatus): Promise<void> | void {
317 if (this.state.phase !== "recording") return;
318 if (status.hasError) {
319 return this.interruptRecording(
320 status.error ?? "Voice recording was interrupted.",
321 status.url,
322 );
323 }
324 if (status.isFinished) {
325 if (!status.url) {
326 return this.interruptRecording();
327 }
328 return this.finishRecording(true, status.url);
329 }
330 }
331
332 ownerChanged(): void {
333 if (this.state.phase === "idle") return;
334 this.cancel();
335 }
336
337 dispose(): void {
338 if (this.state.phase === "recording") {
339 this.discardRecording(null);
340 return;
341 }
342 if (this.state.phase === "preparing" || this.state.phase === "transcribing") {
343 this.invalidateOperation();
344 this.setState(IDLE_STATE);
345 }
346 }
347
348 private async finishRecording(
349 alreadyStopped: boolean,
350 completedUri: string | null,
351 ): Promise<void> {
352 if (this.finishing || this.state.phase !== "recording") return;
353 this.finishing = true;
354 const operationToken = this.operationToken;
355 this.setState({ phase: "transcribing", error: null, errorAction: null });
356
357 try {
358 if (!alreadyStopped) await this.dependencies.recorder.stop();
359 await this.releaseAudioSession();
360 this.recordingUri = completedUri ?? this.dependencies.recorder.uri ?? this.recordingUri;
361 this.rememberRecordingUri(this.recordingUri);
362 if (!this.isCurrent(operationToken)) return;
363 if (
364 !this.recordingUri ||
365 !this.transcription ||
366 !this.transcriptionAbortController ||
367 !this.capturedDraft
368 ) {
369 this.setError("Could not finish voice recording.", "retry");
370 return;
371 }
372
373 const recordingUri = this.recordingUri;
374 const transcription = this.transcription;
375 const signal = this.transcriptionAbortController.signal;
376 const capturedDraft = this.capturedDraft;
377 let transcript: string;
378 try {
379 transcript = await runTranscriptionOperation(() =>
380 transcription.transcribe(recordingUri, { signal }),
381 );
382 } catch (error) {
383 if (this.isCurrent(operationToken)) {
384 this.setError(transcriptionErrorMessage(error), "retry");
385 }
386 return;
387 }
388 if (!this.isCurrent(operationToken)) return;
389
390 const result = resolveTranscriptCommit(
391 capturedDraft,
392 this.dependencies.readDraft(),
393 transcript,
394 transcription.locale,
395 );
396 if (result.kind === "stale") {
397 this.setError(
398 "The draft changed while voice input was running. The transcript was not added.",
399 "retry",
400 );
401 return;
402 }
403 if (result.kind === "empty") {
404 this.setError("No speech was detected.", "retry");
405 return;
406 }
407
408 this.dependencies.commitDraft(result.text, result.selection);
409 this.setState(IDLE_STATE);
410 } catch {
411 if (this.isCurrent(operationToken)) {
412 this.setError("Could not finish voice recording.", "retry");
413 }
414 } finally {
415 this.finishing = false;
416 await this.releaseResources();
417 }
418 }
419
420 private async discardRecording(error: string | null): Promise<void> {
421 this.invalidateOperation();
422 this.setState(
423 error
424 ? { phase: "error", error, errorAction: "retry" }
425 : { phase: "idle", error: null, errorAction: null },
426 );
427 try {
428 await this.dependencies.recorder.stop();
429 this.rememberRecordingUri(this.dependencies.recorder.uri);
430 } catch {
431 this.rememberRecordingUri(this.dependencies.recorder.uri);
432 } finally {
433 await this.releaseResources();
434 }
435 }
436
437 private async releaseResources(): Promise<void> {
438 this.rememberRecordingUri(this.recordingUri);
439 this.rememberRecordingUri(this.dependencies.recorder.uri);
440 this.recordingUri = null;
441 for (const uri of this.ownedRecordingUris) {
442 try {
443 this.dependencies.deleteRecording(uri);
444 } catch {
445 // The cache may already have removed a failed or interrupted recording.
446 }
447 }
448 this.ownedRecordingUris.clear();
449 await this.releaseAudioSession();
450 releaseSession(this.sessionToken);
451 this.sessionToken = null;
452 this.capturedDraft = null;
453 this.transcription = null;
454 this.transcriptionAbortController = null;
455 }
456
457 private rememberRecordingUri(uri: string | null): void {
458 if (uri) this.ownedRecordingUris.add(uri);
459 }
460
461 private async releaseAudioSession(): Promise<void> {
462 if (!this.recordingConfigured) return;
463 try {
464 await this.dependencies.releaseRecording();
465 this.recordingConfigured = false;
466 } catch {
467 // Final cleanup retries if the prompt release before transcription fails.
468 }
469 }
470
471 private invalidateOperation(): void {
472 this.operationToken += 1;
473 this.transcriptionAbortController?.abort();
474 }
475
476 private isCurrent(operationToken: number): boolean {
477 return operationToken === this.operationToken;
478 }
479
480 private setError(error: string, errorAction: VoiceInputState["errorAction"]): void {
481 this.setState({ phase: "error", error, errorAction });
482 }
483
484 private setState(state: VoiceInputState): void {
485 this.state = state;
486 this.dependencies.onStateChange(state);
487 }
488}
489
490export function resetVoiceInputGlobalsForTests(): void {
491 activeSession = null;
492 activeTranscriptionOperation = null;
493}
packages/client-runtime/src/voice-input/index.ts created+21
...@@ -0,0 +1,21 @@
1export {
2 VoiceInputController,
3 VOICE_RECORDING_LIMIT_SECONDS,
4 resolveTranscriptCommit,
5 voiceInputBlocksSubmission,
6 voiceInputFreezesEditor,
7 type VoiceDraftSnapshot,
8 type VoiceInputControllerDependencies,
9 type VoiceInputPhase,
10 type VoiceInputState,
11 type VoiceRecorder,
12 type VoiceRecorderStatus,
13} from "./controller.ts";
14export {
15 VoiceTranscriptionError,
16 throwIfVoiceTranscriptionAborted,
17 type PreparedVoiceTranscription,
18 type VoiceTranscriber,
19 type VoiceTranscriptionErrorCode,
20 type VoiceTranscriptionOptions,
21} from "./transcription.ts";
packages/client-runtime/src/voice-input/transcription.ts created+37
...@@ -0,0 +1,37 @@
1/** Cancellation is cooperative: settle only after the underlying work has stopped. */
2export type VoiceTranscriptionOptions = {
3 readonly signal: AbortSignal;
4};
5
6/** Binds a recording to its selected implementation and resolved locale. */
7export type PreparedVoiceTranscription = {
8 readonly locale: string;
9 readonly transcribe: (uri: string, options: VoiceTranscriptionOptions) => Promise<string>;
10};
11
12export type VoiceTranscriber = {
13 readonly prepare: (options: VoiceTranscriptionOptions) => Promise<PreparedVoiceTranscription>;
14};
15
16export type VoiceTranscriptionErrorCode =
17 | "unavailable"
18 | "unsupported-locale"
19 | "preparation-failed"
20 | "transcription-failed"
21 | "cancelled";
22
23export class VoiceTranscriptionError extends Error {
24 readonly code: VoiceTranscriptionErrorCode;
25
26 constructor(code: VoiceTranscriptionErrorCode, message: string, options?: ErrorOptions) {
27 super(message, options);
28 this.name = "VoiceTranscriptionError";
29 this.code = code;
30 }
31}
32
33export function throwIfVoiceTranscriptionAborted(signal: AbortSignal): void {
34 if (signal.aborted) {
35 throw new VoiceTranscriptionError("cancelled", "Voice transcription was cancelled.");
36 }
37}
patches/@react-native-ai__apple@0.12.0.patch created+194
...@@ -0,0 +1,194 @@
1diff --git a/ios/transcription/AppleTranscriptionImpl.swift b/ios/transcription/AppleTranscriptionImpl.swift
2index 188371a5f55fadae19187108c4a688569d0223e3..3e1cdd77b0bb8ae6c97a40d29c1854200d931eee 100644
3--- a/ios/transcription/AppleTranscriptionImpl.swift
4+++ b/ios/transcription/AppleTranscriptionImpl.swift
5@@ -12,6 +12,11 @@ import UniformTypeIdentifiers
6
7 @objc
8 public class AppleTranscriptionImpl: NSObject {
9+ private struct CollectedSegment: Sendable {
10+ let text: String
11+ let startSecond: Double
12+ let endSecond: Double
13+ }
14
15 @available(iOS 26, *)
16 private func createTranscriber(for locale: Locale) -> SpeechTranscriber {
17@@ -41,7 +46,7 @@ public class AppleTranscriptionImpl: NSObject {
18 let locale = Locale(identifier: language)
19
20 guard let supportedLocale = await SpeechTranscriber.supportedLocale(equivalentTo: locale) else {
21- reject("AppleTranscription", "Locale not supported: \(language)", nil)
22+ reject("AppleTranscriptionUnsupportedLocale", "Locale not supported: \(language)", nil)
23 return
24 }
25
26@@ -51,16 +56,14 @@ public class AppleTranscriptionImpl: NSObject {
27
28 switch status {
29 case .installed:
30- resolve(nil)
31+ resolve(supportedLocale.identifier)
32 case .supported, .downloading:
33- if let request = try? await AssetInventory.assetInstallationRequest(supporting: [transcriber]) {
34+ if let request = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) {
35 try await request.downloadAndInstall()
36- resolve(nil)
37- } else {
38- resolve(nil)
39 }
40+ resolve(supportedLocale.identifier)
41 case .unsupported:
42- reject("AppleTranscription", "Assets not supported for locale: \(supportedLocale.identifier)", nil)
43+ reject("AppleTranscriptionUnsupportedLocale", "Assets not supported for locale: \(supportedLocale.identifier)", nil)
44 @unknown default:
45 reject ("AppleTranscription", "Unknown asset inventory status", nil)
46 }
47@@ -83,57 +86,77 @@ public class AppleTranscriptionImpl: NSObject {
48 do {
49 try audioData.write(to: fileURL)
50
51- guard let audioFile = try? AVAudioFile(forReading: fileURL) else {
52- reject("AppleTranscription", "Invalid audio data", nil)
53+ let audioFile: AVAudioFile
54+ do {
55+ audioFile = try AVAudioFile(forReading: fileURL)
56+ } catch {
57+ try? FileManager.default.removeItem(at: fileURL)
58+ reject("AppleTranscription", "Invalid audio data", error)
59 return
60 }
61
62 Task {
63+ defer {
64+ try? FileManager.default.removeItem(at: fileURL)
65+ }
66+
67 do {
68 let transcriber = createTranscriber(for: Locale(identifier: language))
69
70 let analyzer = SpeechAnalyzer(modules: [transcriber])
71-
72- defer {
73- try? FileManager.default.removeItem(at: fileURL)
74- }
75-
76- var segments: [[String: Any]] = []
77-
78- Task {
79+
80+ let collectorTask = Task { () throws -> [CollectedSegment] in
81+ var segments: [CollectedSegment] = []
82+
83 for try await result in transcriber.results {
84 if result.isFinal {
85- let segment: [String: Any] = [
86- "text": String(result.text.characters),
87- "startSecond": CMTimeGetSeconds(result.range.start),
88- "endSecond": CMTimeGetSeconds(CMTimeRangeGetEnd(result.range))
89- ]
90- segments.append(segment)
91+ segments.append(
92+ CollectedSegment(
93+ text: String(result.text.characters),
94+ startSecond: CMTimeGetSeconds(result.range.start),
95+ endSecond: CMTimeGetSeconds(CMTimeRangeGetEnd(result.range))
96+ )
97+ )
98 }
99 }
100+
101+ return segments
102 }
103-
104- let lastSampleTime = try await analyzer.analyzeSequence(from: audioFile)
105-
106- if let lastSampleTime {
107- try await analyzer.finalizeAndFinish(through: lastSampleTime)
108- } else {
109+
110+ do {
111+ let lastSampleTime = try await analyzer.analyzeSequence(from: audioFile)
112+
113+ if let lastSampleTime {
114+ try await analyzer.finalizeAndFinish(through: lastSampleTime)
115+ } else {
116+ await analyzer.cancelAndFinishNow()
117+ }
118+
119+ let segments: [[String: Any]] = try await collectorTask.value.map { segment in
120+ [
121+ "text": segment.text,
122+ "startSecond": segment.startSecond,
123+ "endSecond": segment.endSecond
124+ ]
125+ }
126+ let totalDuration = if let lastSampleTime { CMTimeGetSeconds(lastSampleTime) } else { 0.0 }
127+
128+ resolve([
129+ "segments": segments,
130+ "duration": totalDuration
131+ ])
132+ } catch {
133+ collectorTask.cancel()
134 await analyzer.cancelAndFinishNow()
135+ _ = try? await collectorTask.value
136+ throw error
137 }
138-
139- let totalDuration = if let lastSampleTime { CMTimeGetSeconds(lastSampleTime) } else { 0.0 }
140-
141- let result: [String: Any] = [
142- "segments": segments,
143- "duration": totalDuration
144- ]
145-
146- resolve(result)
147 } catch {
148 reject("AppleTranscription", "Transcription failed: \(error.localizedDescription)", error)
149 }
150 }
151 } catch {
152+ try? FileManager.default.removeItem(at: fileURL)
153 reject("AppleTranscription", "Failed to write audio data: \(error.localizedDescription)", error)
154 }
155 } else {
156@@ -141,4 +164,3 @@ public class AppleTranscriptionImpl: NSObject {
157 }
158 }
159 }
160-
161diff --git a/lib/typescript/NativeAppleTranscription.d.ts b/lib/typescript/NativeAppleTranscription.d.ts
162index 985b6b3593a884c41d689346be9d73d86c55eae1..86e7936f23a8673d099f7fe18af9a31abae5a13a 100644
163--- a/lib/typescript/NativeAppleTranscription.d.ts
164+++ b/lib/typescript/NativeAppleTranscription.d.ts
165@@ -10,14 +10,14 @@ export interface TranscriptionResult {
166 }
167 export interface Spec extends TurboModule {
168 isAvailable(language: string): boolean;
169- prepare(language: string): Promise<void>;
170+ prepare(language: string): Promise<string>;
171 }
172 declare global {
173 function __apple__llm__transcribe__(data: ArrayBufferLike, language: string): Promise<TranscriptionResult>;
174 }
175 declare const _default: {
176 transcribe: (data: ArrayBufferLike, language: string) => Promise<TranscriptionResult>;
177- prepare: (language: string) => Promise<void>;
178+ prepare: (language: string) => Promise<string>;
179 isAvailable: (language: string) => boolean;
180 };
181 export default _default;
182diff --git a/src/NativeAppleTranscription.ts b/src/NativeAppleTranscription.ts
183index 13332a0176000b60b6f6f043de681db55a7b4bef..5389fb4b3a53de48721c307d2145b6770624c852 100644
184--- a/src/NativeAppleTranscription.ts
185+++ b/src/NativeAppleTranscription.ts
186@@ -14,7 +14,7 @@ export interface TranscriptionResult {
187
188 export interface Spec extends TurboModule {
189 isAvailable(language: string): boolean
190- prepare(language: string): Promise<void>
191+ prepare(language: string): Promise<string>
192 }
193
194 declare global {
patches/expo-audio@57.0.4.patch created+24
...@@ -0,0 +1,24 @@
1diff --git a/ios/AudioRecorder.swift b/ios/AudioRecorder.swift
2index 20020bff9f8ba7bb6e7f61c99b4de5d29eee00db..284c22f12e30c8c2ed16feb141876cbfd8897c90 100644
3--- a/ios/AudioRecorder.swift
4+++ b/ios/AudioRecorder.swift
5@@ -216,15 +216,15 @@ class AudioRecorder: SharedRef<AVAudioRecorder>, RecordingResultHandler {
6 }
7
8 func didFinish(_ recorder: AVAudioRecorder, successfully flag: Bool) {
9- // Update internal state when recording finishes automatically (e.g., from recordForDuration)
10- currentState = .stopped
11+ // Update internal state when AVAudioRecorder finishes or fails.
12+ currentState = flag ? .stopped : .error
13 resetDurationTracking()
14
15 emit(event: recordingStatus, payload: [
16 "id": id,
17 "isFinished": true,
18- "hasError": false,
19- "error": nil,
20+ "hasError": !flag,
21+ "error": flag ? nil : "Recording failed",
22 "url": recorder.url.absoluteString
23 ])
24 }
pnpm-lock.yaml+66-4
...@@ -88,9 +88,11 @@ patchedDependencies:...@@ -88,9 +88,11 @@ patchedDependencies:
88 '@ff-labs/fff-node@0.9.4': ab9ff544009e1891cfe3930105862d3699007f38922a79f3c98d90018deca36888 '@ff-labs/fff-node@0.9.4': ab9ff544009e1891cfe3930105862d3699007f38922a79f3c98d90018deca368
89 '@legendapp/list@3.3.5': f786e0ada19a32703f71a7230a819697ebff2527873dfb9997c8455947e0060b89 '@legendapp/list@3.3.5': f786e0ada19a32703f71a7230a819697ebff2527873dfb9997c8455947e0060b
90 '@pierre/diffs@1.3.0-beta.10': 7ef7cb0cbabb17c15cdb137554068b36f15f6f5265e73fb51452aa3380db91aa90 '@pierre/diffs@1.3.0-beta.10': 7ef7cb0cbabb17c15cdb137554068b36f15f6f5265e73fb51452aa3380db91aa
91 '@react-native-ai/apple@0.12.0': 2d09870c2848d185cb05b53ed823a46e12dba519324d8dd8e584e28731990f9d
91 '@react-native-menu/menu@2.0.0': f63d256bf6a97a873b5e628eb595bd6ef0075ddd5bdd890fc920f7a6024290dd92 '@react-native-menu/menu@2.0.0': f63d256bf6a97a873b5e628eb595bd6ef0075ddd5bdd890fc920f7a6024290dd
92 '@react-navigation/native-stack@7.17.6': e667c3cef8c78bb9ff4882ee5bd23a432247b843060a9499eb5f07e9e229555293 '@react-navigation/native-stack@7.17.6': e667c3cef8c78bb9ff4882ee5bd23a432247b843060a9499eb5f07e9e2295552
93 effect@4.0.0-beta.103: af36b7948b6f9c56623074662b51dade5699880c1a7c71245de73e13c3185fb694 effect@4.0.0-beta.103: af36b7948b6f9c56623074662b51dade5699880c1a7c71245de73e13c3185fb6
95 expo-audio@57.0.4: fa9a3e0442ed395d4071bb406e08c3a471c9a84700bdfa0b9ad7ff144c96041a
94 expo-sharing@57.0.16: 8d2e3b10eb3f52036a9a086800180ec6cebf3b75bccc5b1775117a7244d4ac4596 expo-sharing@57.0.16: 8d2e3b10eb3f52036a9a086800180ec6cebf3b75bccc5b1775117a7244d4ac45
95 react-native-gesture-handler@2.32.0: 808eb26f9e57cf4945efd3985af4d9c764da6f91f4c9764433cc868602bbf4d397 react-native-gesture-handler@2.32.0: 808eb26f9e57cf4945efd3985af4d9c764da6f91f4c9764433cc868602bbf4d3
96 react-native-keyboard-controller@1.21.13: 6e4339347bc5bb3c9ea67d85ff5c814058b211c5750f247aba59d07869a2e78798 react-native-keyboard-controller@1.21.13: 6e4339347bc5bb3c9ea67d85ff5c814058b211c5750f247aba59d07869a2e787
...@@ -239,6 +241,9 @@ importers:...@@ -239,6 +241,9 @@ importers:
239 '@pierre/diffs':241 '@pierre/diffs':
240 specifier: 'catalog:'242 specifier: 'catalog:'
241 version: 1.3.0-beta.10(patch_hash=7ef7cb0cbabb17c15cdb137554068b36f15f6f5265e73fb51452aa3380db91aa)(@shikijs/themes@4.2.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3)243 version: 1.3.0-beta.10(patch_hash=7ef7cb0cbabb17c15cdb137554068b36f15f6f5265e73fb51452aa3380db91aa)(@shikijs/themes@4.2.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3)
244 '@react-native-ai/apple':
245 specifier: 0.12.0
246 version: 0.12.0(patch_hash=2d09870c2848d185cb05b53ed823a46e12dba519324d8dd8e584e28731990f9d)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))
242 '@react-native-menu/menu':247 '@react-native-menu/menu':
243 specifier: ^2.0.0248 specifier: ^2.0.0
244 version: 2.0.0(patch_hash=f63d256bf6a97a873b5e628eb595bd6ef0075ddd5bdd890fc920f7a6024290dd)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3)249 version: 2.0.0(patch_hash=f63d256bf6a97a873b5e628eb595bd6ef0075ddd5bdd890fc920f7a6024290dd)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3)
...@@ -299,6 +304,9 @@ importers:...@@ -299,6 +304,9 @@ importers:
299 expo-asset:304 expo-asset:
300 specifier: ~57.0.15305 specifier: ~57.0.15
301 version: 57.0.15(expo@57.0.18)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3)(typescript@6.0.3)306 version: 57.0.15(expo@57.0.18)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3)(typescript@6.0.3)
307 expo-audio:
308 specifier: ~57.0.4
309 version: 57.0.4(patch_hash=fa9a3e0442ed395d4071bb406e08c3a471c9a84700bdfa0b9ad7ff144c96041a)(expo-asset@57.0.15(expo@57.0.18)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3)(typescript@6.0.3))(expo@57.0.18)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3)
302 expo-auth-session:310 expo-auth-session:
303 specifier: ~57.0.10311 specifier: ~57.0.10
304 version: 57.0.10(expo@57.0.18)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3)312 version: 57.0.10(expo@57.0.18)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3)
...@@ -990,6 +998,16 @@ importers:...@@ -990,6 +998,16 @@ importers:
990998
991packages:999packages:
9921000
1001 '@ai-sdk/provider-utils@4.0.49':
1002 resolution: {integrity: sha512-8e7pd+82bobqrFOaD5dG/PiEuvLYr5olaE3I56ch0jipR0H7sGD6ohwTUynv6k8O8QidWiyIbEsZCtr/2dyXIA==}
1003 engines: {node: '>=18.17'}
1004 peerDependencies:
1005 zod: ^3.25.76 || ^4.1.8
1006
1007 '@ai-sdk/provider@3.0.15':
1008 resolution: {integrity: sha512-XeZW1CcDF2GMbH4wejW6xBRI2QCOgnkVYUnxoeDadB1mf85riL2bMUeDoh+6gJ/r4mjNfzUPW8OjLjvwTP0u1Q==}
1009 engines: {node: '>=18'}
1010
993 '@alcalzone/ansi-tokenize@0.2.5':1011 '@alcalzone/ansi-tokenize@0.2.5':
994 resolution: {integrity: sha512-3NX/MpTdroi0aKz134A6RC2Gb2iXVECN4QaAXnvCIxxIm3C3AVB1mkUe8NaaiyvOpDfsrqWhYtj+Q6a62RrTsw==}1012 resolution: {integrity: sha512-3NX/MpTdroi0aKz134A6RC2Gb2iXVECN4QaAXnvCIxxIm3C3AVB1mkUe8NaaiyvOpDfsrqWhYtj+Q6a62RrTsw==}
995 engines: {node: '>=18'}1013 engines: {node: '>=18'}
...@@ -3808,6 +3826,11 @@ packages:...@@ -3808,6 +3826,11 @@ packages:
3808 resolution: {integrity: sha512-gMDYY2rw6OWajCcDlXSIgs2LC432YJXSb3Lm5yM187uhRgBYddoEVULi36h+IolX3r7jSb3ew7vn9FfI8NSo0A==}3826 resolution: {integrity: sha512-gMDYY2rw6OWajCcDlXSIgs2LC432YJXSb3Lm5yM187uhRgBYddoEVULi36h+IolX3r7jSb3ew7vn9FfI8NSo0A==}
3809 hasBin: true3827 hasBin: true
38103828
3829 '@react-native-ai/apple@0.12.0':
3830 resolution: {integrity: sha512-BC/kEDbCZprv1xcQCWsgapXm/WEsj5lieBDvU1vJXStrN+BG+hWM01zoqDGTD/j71zvt9DEfHcMvEqbrGY2uAA==}
3831 peerDependencies:
3832 react-native: '>=0.76.0'
3833
3811 '@react-native-masked-view/masked-view@0.3.2':3834 '@react-native-masked-view/masked-view@0.3.2':
3812 resolution: {integrity: sha512-XwuQoW7/GEgWRMovOQtX3A4PrXhyaZm0lVUiY8qJDvdngjLms9Cpdck6SmGAUNqQwcj2EadHC1HwL0bEyoa/SQ==}3835 resolution: {integrity: sha512-XwuQoW7/GEgWRMovOQtX3A4PrXhyaZm0lVUiY8qJDvdngjLms9Cpdck6SmGAUNqQwcj2EadHC1HwL0bEyoa/SQ==}
3813 peerDependencies:3836 peerDependencies:
...@@ -6464,6 +6487,14 @@ packages:...@@ -6464,6 +6487,14 @@ packages:
6464 react: '*'6487 react: '*'
6465 react-native: '*'6488 react-native: '*'
64666489
6490 expo-audio@57.0.4:
6491 resolution: {integrity: sha512-TLP8rt1UvUDzgxGnyQ0TR9hV6tNP/UJQdDu7mSK+dgEydMFoptq55D3hcUoB1gF39f3/3AUuWfOtpGM+4N4X1A==}
6492 peerDependencies:
6493 expo: '*'
6494 expo-asset: '*'
6495 react: '*'
6496 react-native: '*'
6497
6467 expo-auth-session@57.0.10:6498 expo-auth-session@57.0.10:
6468 resolution: {integrity: sha512-i1RY93LouEgal2e/Ly1r2Ytwed7/y6i4AUShVj4Gg49h/Okf8QpZO7XylwL43IHfKRrKkqapRDEN8mWe7z/HgA==}6499 resolution: {integrity: sha512-i1RY93LouEgal2e/Ly1r2Ytwed7/y6i4AUShVj4Gg49h/Okf8QpZO7XylwL43IHfKRrKkqapRDEN8mWe7z/HgA==}
6469 peerDependencies:6500 peerDependencies:
...@@ -7428,6 +7459,9 @@ packages:...@@ -7428,6 +7459,9 @@ packages:
7428 json-schema-typed@8.0.2:7459 json-schema-typed@8.0.2:
7429 resolution: {integrity: sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA==}7460 resolution: {integrity: sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA==}
74307461
7462 json-schema@0.4.0:
7463 resolution: {integrity: sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==}
7464
7431 json-stringify-safe@5.0.1:7465 json-stringify-safe@5.0.1:
7432 resolution: {integrity: sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==}7466 resolution: {integrity: sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==}
74337467
...@@ -9762,8 +9796,8 @@ packages:...@@ -9762,8 +9796,8 @@ packages:
9762 undici-types@7.16.0:9796 undici-types@7.16.0:
9763 resolution: {integrity: sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==}9797 resolution: {integrity: sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==}
97649798
9765 undici@6.26.0:9799 undici@6.28.0:
9766 resolution: {integrity: sha512-4yqz8a3n5HmGTlsbADNtr/dJlhkh/55Rq798G6ibiULcXbDtaLpTl1pvdqcbFfeoj3iSi52lePFM7h9H21cw/A==}9800 resolution: {integrity: sha512-LIY910g9TI13YS95lrMFrs8Rm/u/irgHeTWoKCoteeJ04CUJ92eEfj0rVn+7VKMPBpUPiUoBKfhNyLI23EE/KA==}
9767 engines: {node: '>=18.17'}9801 engines: {node: '>=18.17'}
97689802
9769 undici@7.27.1:9803 undici@7.27.1:
...@@ -10419,6 +10453,18 @@ packages:...@@ -10419,6 +10453,18 @@ packages:
1041910453
10420snapshots:10454snapshots:
1042110455
10456 '@ai-sdk/provider-utils@4.0.49(zod@4.4.3)':
10457 dependencies:
10458 '@ai-sdk/provider': 3.0.15
10459 '@standard-schema/spec': 1.1.0
10460 eventsource-parser: 3.1.0
10461 undici: 6.28.0
10462 zod: 4.4.3
10463
10464 '@ai-sdk/provider@3.0.15':
10465 dependencies:
10466 json-schema: 0.4.0
10467
10422 '@alcalzone/ansi-tokenize@0.2.5':10468 '@alcalzone/ansi-tokenize@0.2.5':
10423 dependencies:10469 dependencies:
10424 ansi-styles: 6.2.310470 ansi-styles: 6.2.3
...@@ -13662,6 +13708,13 @@ snapshots:...@@ -13662,6 +13708,13 @@ snapshots:
13662 prompts: 2.4.213708 prompts: 2.4.2
13663 tinyexec: 1.2.413709 tinyexec: 1.2.4
1366413710
13711 '@react-native-ai/apple@0.12.0(patch_hash=2d09870c2848d185cb05b53ed823a46e12dba519324d8dd8e584e28731990f9d)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))':
13712 dependencies:
13713 '@ai-sdk/provider': 3.0.15
13714 '@ai-sdk/provider-utils': 4.0.49(zod@4.4.3)
13715 react-native: 0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6)
13716 zod: 4.4.3
13717
13665 '@react-native-masked-view/masked-view@0.3.2(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3)':13718 '@react-native-masked-view/masked-view@0.3.2(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3)':
13666 dependencies:13719 dependencies:
13667 react: 19.2.313720 react: 19.2.3
...@@ -16307,6 +16360,13 @@ snapshots:...@@ -16307,6 +16360,13 @@ snapshots:
16307 - typescript16360 - typescript
16308 optional: true16361 optional: true
1630916362
16363 expo-audio@57.0.4(patch_hash=fa9a3e0442ed395d4071bb406e08c3a471c9a84700bdfa0b9ad7ff144c96041a)(expo-asset@57.0.15(expo@57.0.18)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3)(typescript@6.0.3))(expo@57.0.18)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3):
16364 dependencies:
16365 expo: 57.0.18(f9c992a5d7c53d81398568d3950992dc)
16366 expo-asset: 57.0.15(expo@57.0.18)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3)(typescript@6.0.3)
16367 react: 19.2.3
16368 react-native: 0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6)
16369
16310 expo-auth-session@57.0.10(expo@57.0.18)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3):16370 expo-auth-session@57.0.10(expo@57.0.18)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3):
16311 dependencies:16371 dependencies:
16312 expo-application: 57.0.2(expo@57.0.18)16372 expo-application: 57.0.2(expo@57.0.18)
...@@ -17565,6 +17625,8 @@ snapshots:...@@ -17565,6 +17625,8 @@ snapshots:
1756517625
17566 json-schema-typed@8.0.2: {}17626 json-schema-typed@8.0.2: {}
1756717627
17628 json-schema@0.4.0: {}
17629
17568 json-stringify-safe@5.0.1:17630 json-stringify-safe@5.0.1:
17569 optional: true17631 optional: true
1757017632
...@@ -18768,7 +18830,7 @@ snapshots:...@@ -18768,7 +18830,7 @@ snapshots:
18768 semver: 7.8.518830 semver: 7.8.5
18769 tar: 7.5.1618831 tar: 7.5.16
18770 tinyglobby: 0.2.1718832 tinyglobby: 0.2.17
18771 undici: 6.26.018833 undici: 6.28.0
18772 which: 6.0.118834 which: 6.0.1
1877318835
18774 node-int64@0.4.0: {}18836 node-int64@0.4.0: {}
...@@ -20559,7 +20621,7 @@ snapshots:...@@ -20559,7 +20621,7 @@ snapshots:
2055920621
20560 undici-types@7.16.0: {}20622 undici-types@7.16.0: {}
2056120623
20562 undici@6.26.0: {}20624 undici@6.28.0: {}
2056320625
20564 undici@7.27.1: {}20626 undici@7.27.1: {}
2056520627
pnpm-workspace.yaml+2
...@@ -150,9 +150,11 @@ patchedDependencies:...@@ -150,9 +150,11 @@ patchedDependencies:
150 "@ff-labs/fff-node@0.9.4": patches/@ff-labs__fff-node@0.9.4.patch150 "@ff-labs/fff-node@0.9.4": patches/@ff-labs__fff-node@0.9.4.patch
151 "@legendapp/list@3.3.5": patches/@legendapp__list@3.3.5.patch151 "@legendapp/list@3.3.5": patches/@legendapp__list@3.3.5.patch
152 "@pierre/diffs@1.3.0-beta.10": patches/@pierre%2Fdiffs@1.3.0-beta.10.patch152 "@pierre/diffs@1.3.0-beta.10": patches/@pierre%2Fdiffs@1.3.0-beta.10.patch
153 "@react-native-ai/apple@0.12.0": patches/@react-native-ai__apple@0.12.0.patch
153 "@react-native-menu/menu@2.0.0": patches/@react-native-menu__menu@2.0.0.patch154 "@react-native-menu/menu@2.0.0": patches/@react-native-menu__menu@2.0.0.patch
154 "@react-navigation/native-stack@7.17.6": patches/@react-navigation%2Fnative-stack@7.17.6.patch155 "@react-navigation/native-stack@7.17.6": patches/@react-navigation%2Fnative-stack@7.17.6.patch
155 effect@4.0.0-beta.103: patches/effect@4.0.0-beta.103.patch156 effect@4.0.0-beta.103: patches/effect@4.0.0-beta.103.patch
157 expo-audio@57.0.4: patches/expo-audio@57.0.4.patch
156 expo-sharing@57.0.16: patches/expo-sharing@57.0.16.patch158 expo-sharing@57.0.16: patches/expo-sharing@57.0.16.patch
157 react-native-gesture-handler@2.32.0: patches/react-native-gesture-handler@2.32.0.patch159 react-native-gesture-handler@2.32.0: patches/react-native-gesture-handler@2.32.0.patch
158 react-native-keyboard-controller@1.21.13: patches/react-native-keyboard-controller@1.21.13.patch160 react-native-keyboard-controller@1.21.13: patches/react-native-keyboard-controller@1.21.13.patch