From 05ac65655002bee16d48cce55d737dd4157657d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20=C5=A0=C4=87eki=C4=87?= Date: Tue, 14 Jul 2026 14:00:59 +0200 Subject: [PATCH 01/14] chore(mobile): add native speech recognition --- apps/mobile/app.config.ts | 8 ++++++++ apps/mobile/package.json | 2 ++ pnpm-lock.yaml | 41 +++++++++++++++++++++++++++++++++++++-- 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/apps/mobile/app.config.ts b/apps/mobile/app.config.ts index 299d594bec..db268396b8 100644 --- a/apps/mobile/app.config.ts +++ b/apps/mobile/app.config.ts @@ -85,6 +85,14 @@ const config: ExpoConfig = { }, }, ], + [ + 'expo-speech-recognition', + { + microphonePermission: 'Allow Kilo to use your microphone to turn speech into text.', + speechRecognitionPermission: + 'Allow Kilo to use speech recognition to turn your voice into text.', + }, + ], 'expo-router', 'expo-image', 'expo-font', diff --git a/apps/mobile/package.json b/apps/mobile/package.json index 0619565e76..fda2202936 100644 --- a/apps/mobile/package.json +++ b/apps/mobile/package.json @@ -59,12 +59,14 @@ "expo-image-picker": "~55.0.21", "expo-insights": "55.0.18", "expo-linking": "55.0.16", + "expo-localization": "~55.0.16", "expo-location": "55.1.11", "expo-notifications": "55.0.24", "expo-router": "55.0.16", "expo-screen-corner-radius": "^1.0.1", "expo-secure-store": "55.0.15", "expo-sharing": "~55.0.21", + "expo-speech-recognition": "3.1.3", "expo-splash-screen": "55.0.22", "expo-status-bar": "55.0.6", "expo-store-review": "~55.0.15", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4ab8de0f67..a752776885 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -368,6 +368,9 @@ importers: expo-linking: specifier: 55.0.16 version: 55.0.16(expo@55.0.27)(react-native@0.83.6(@babel/core@7.29.0)(@types/react@19.2.14)(bufferutil@4.1.0)(react@19.2.0)(utf-8-validate@6.0.6))(react@19.2.0) + expo-localization: + specifier: ~55.0.16 + version: 55.0.16(expo@55.0.27)(react@19.2.0) expo-location: specifier: 55.1.11 version: 55.1.11(expo@55.0.27)(typescript@5.9.3) @@ -386,6 +389,9 @@ importers: expo-sharing: specifier: ~55.0.21 version: 55.0.21(expo@55.0.27)(react-native@0.83.6(@babel/core@7.29.0)(@types/react@19.2.14)(bufferutil@4.1.0)(react@19.2.0)(utf-8-validate@6.0.6))(react@19.2.0) + expo-speech-recognition: + specifier: 3.1.3 + version: 3.1.3(expo@55.0.27)(react-native@0.83.6(@babel/core@7.29.0)(@types/react@19.2.14)(bufferutil@4.1.0)(react@19.2.0)(utf-8-validate@6.0.6))(react@19.2.0) expo-splash-screen: specifier: 55.0.22 version: 55.0.22(expo@55.0.27)(typescript@5.9.3) @@ -412,7 +418,7 @@ importers: version: 5.0.0-preview.3(react-native-css@3.0.7(@expo/metro-config@55.0.24(bufferutil@4.1.0)(expo@55.0.27)(typescript@5.9.3)(utf-8-validate@6.0.6))(lightningcss@1.30.1)(react-native@0.83.6(@babel/core@7.29.0)(@types/react@19.2.14)(bufferutil@4.1.0)(react@19.2.0)(utf-8-validate@6.0.6))(react@19.2.0))(tailwindcss@4.2.4) posthog-react-native: specifier: ^4.54.3 - version: 4.54.3(@react-navigation/native@7.1.33(react-native@0.83.6(@babel/core@7.29.0)(@types/react@19.2.14)(bufferutil@4.1.0)(react@19.2.0)(utf-8-validate@6.0.6))(react@19.2.0))(expo-application@55.0.16(expo@55.0.27))(expo-file-system@55.0.23(expo@55.0.27)(react-native@0.83.6(@babel/core@7.29.0)(@types/react@19.2.14)(bufferutil@4.1.0)(react@19.2.0)(utf-8-validate@6.0.6)))(react-native-safe-area-context@5.6.2(react-native@0.83.6(@babel/core@7.29.0)(@types/react@19.2.14)(bufferutil@4.1.0)(react@19.2.0)(utf-8-validate@6.0.6))(react@19.2.0))(react-native-svg@15.15.3(react-native@0.83.6(@babel/core@7.29.0)(@types/react@19.2.14)(bufferutil@4.1.0)(react@19.2.0)(utf-8-validate@6.0.6))(react@19.2.0)) + version: 4.54.3(@react-navigation/native@7.1.33(react-native@0.83.6(@babel/core@7.29.0)(@types/react@19.2.14)(bufferutil@4.1.0)(react@19.2.0)(utf-8-validate@6.0.6))(react@19.2.0))(expo-application@55.0.16(expo@55.0.27))(expo-file-system@55.0.23(expo@55.0.27)(react-native@0.83.6(@babel/core@7.29.0)(@types/react@19.2.14)(bufferutil@4.1.0)(react@19.2.0)(utf-8-validate@6.0.6)))(expo-localization@55.0.16(expo@55.0.27)(react@19.2.0))(react-native-safe-area-context@5.6.2(react-native@0.83.6(@babel/core@7.29.0)(@types/react@19.2.14)(bufferutil@4.1.0)(react@19.2.0)(utf-8-validate@6.0.6))(react@19.2.0))(react-native-svg@15.15.3(react-native@0.83.6(@babel/core@7.29.0)(@types/react@19.2.14)(bufferutil@4.1.0)(react@19.2.0)(utf-8-validate@6.0.6))(react@19.2.0)) react: specifier: 19.2.0 version: 19.2.0 @@ -12100,6 +12106,12 @@ packages: react: '*' react-native: '*' + expo-localization@55.0.16: + resolution: {integrity: sha512-N7GIFgSTSkShpASVMIvxu6wt5yMltoaj9Hd1LuKaAQoKNVtvTGnL2Q6sGjnmD4/NVm0yDhDmM/eBZkSkMkHTuA==} + peerDependencies: + expo: '*' + react: '*' + expo-location@55.1.11: resolution: {integrity: sha512-qbyG18CW3BMPp+p88iCrSI8I3GPw6lsnPXFJg8FfY7lxyGGqfwqGNfRX6LX7k12xDhrZzVVQ6v3t1CY6M51Zhg==} peerDependencies: @@ -12192,6 +12204,13 @@ packages: react: '*' react-native: '*' + expo-speech-recognition@3.1.3: + resolution: {integrity: sha512-vluXqggfY2AJcazYITvnV6YSE2rVe5SId5TpdjMC/m6JPUgHCOODrcHJtAQF6OUYl4f2T7oZSceGw6fJCC86Xw==} + peerDependencies: + expo: '*' + react: '*' + react-native: '*' + expo-splash-screen@55.0.22: resolution: {integrity: sha512-uvGybuDlYjVa33eTTeZlwEL6hZagRv64v+OxQVRuKOKZXImatxdE6ZGwHUXz/JxZkntZKYD8p9hBIjlupNLpAQ==} peerDependencies: @@ -16135,6 +16154,9 @@ packages: resolution: {integrity: sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==} engines: {node: '>= 18'} + rtl-detect@1.1.2: + resolution: {integrity: sha512-PGMBq03+TTG/p/cRB7HCLKJ1MgDIi07+QU1faSjiYRfmY5UsAttV9Hs08jDAHVwcOwmVLcSJkpwyfXszVjWfIQ==} + run-applescript@7.1.0: resolution: {integrity: sha512-DPe5pVFaAsinSaV6QjQ6gdiedWDcRCbUuiQfQa2wmWV7+xC9bGulGI8+TdRmoFkAPaBXk8CrAbnlY2ISniJ47Q==} engines: {node: '>=18'} @@ -28765,6 +28787,12 @@ snapshots: - expo - supports-color + expo-localization@55.0.16(expo@55.0.27)(react@19.2.0): + dependencies: + expo: 55.0.27(@babel/core@7.29.0)(@expo/dom-webview@55.0.6)(@expo/metro-runtime@55.0.11)(bufferutil@4.1.0)(expo-router@55.0.16)(react-dom@19.2.6(react@19.2.0))(react-native-worklets@0.7.4(@babel/core@7.29.0)(react-native@0.83.6(@babel/core@7.29.0)(@types/react@19.2.14)(bufferutil@4.1.0)(react@19.2.0)(utf-8-validate@6.0.6))(react@19.2.0))(react-native@0.83.6(@babel/core@7.29.0)(@types/react@19.2.14)(bufferutil@4.1.0)(react@19.2.0)(utf-8-validate@6.0.6))(react@19.2.0)(typescript@5.9.3)(utf-8-validate@6.0.6) + react: 19.2.0 + rtl-detect: 1.1.2 + expo-location@55.1.11(expo@55.0.27)(typescript@5.9.3): dependencies: '@expo/image-utils': 0.8.14(typescript@5.9.3) @@ -28886,6 +28914,12 @@ snapshots: transitivePeerDependencies: - supports-color + expo-speech-recognition@3.1.3(expo@55.0.27)(react-native@0.83.6(@babel/core@7.29.0)(@types/react@19.2.14)(bufferutil@4.1.0)(react@19.2.0)(utf-8-validate@6.0.6))(react@19.2.0): + dependencies: + expo: 55.0.27(@babel/core@7.29.0)(@expo/dom-webview@55.0.6)(@expo/metro-runtime@55.0.11)(bufferutil@4.1.0)(expo-router@55.0.16)(react-dom@19.2.6(react@19.2.0))(react-native-worklets@0.7.4(@babel/core@7.29.0)(react-native@0.83.6(@babel/core@7.29.0)(@types/react@19.2.14)(bufferutil@4.1.0)(react@19.2.0)(utf-8-validate@6.0.6))(react@19.2.0))(react-native@0.83.6(@babel/core@7.29.0)(@types/react@19.2.14)(bufferutil@4.1.0)(react@19.2.0)(utf-8-validate@6.0.6))(react@19.2.0)(typescript@5.9.3)(utf-8-validate@6.0.6) + react: 19.2.0 + react-native: 0.83.6(@babel/core@7.29.0)(@types/react@19.2.14)(bufferutil@4.1.0)(react@19.2.0)(utf-8-validate@6.0.6) + expo-splash-screen@55.0.22(expo@55.0.27)(typescript@5.9.3): dependencies: '@expo/prebuild-config': 55.0.19(expo@55.0.27)(typescript@5.9.3) @@ -33221,7 +33255,7 @@ snapshots: dependencies: '@posthog/core': 1.4.0 - posthog-react-native@4.54.3(@react-navigation/native@7.1.33(react-native@0.83.6(@babel/core@7.29.0)(@types/react@19.2.14)(bufferutil@4.1.0)(react@19.2.0)(utf-8-validate@6.0.6))(react@19.2.0))(expo-application@55.0.16(expo@55.0.27))(expo-file-system@55.0.23(expo@55.0.27)(react-native@0.83.6(@babel/core@7.29.0)(@types/react@19.2.14)(bufferutil@4.1.0)(react@19.2.0)(utf-8-validate@6.0.6)))(react-native-safe-area-context@5.6.2(react-native@0.83.6(@babel/core@7.29.0)(@types/react@19.2.14)(bufferutil@4.1.0)(react@19.2.0)(utf-8-validate@6.0.6))(react@19.2.0))(react-native-svg@15.15.3(react-native@0.83.6(@babel/core@7.29.0)(@types/react@19.2.14)(bufferutil@4.1.0)(react@19.2.0)(utf-8-validate@6.0.6))(react@19.2.0)): + posthog-react-native@4.54.3(@react-navigation/native@7.1.33(react-native@0.83.6(@babel/core@7.29.0)(@types/react@19.2.14)(bufferutil@4.1.0)(react@19.2.0)(utf-8-validate@6.0.6))(react@19.2.0))(expo-application@55.0.16(expo@55.0.27))(expo-file-system@55.0.23(expo@55.0.27)(react-native@0.83.6(@babel/core@7.29.0)(@types/react@19.2.14)(bufferutil@4.1.0)(react@19.2.0)(utf-8-validate@6.0.6)))(expo-localization@55.0.16(expo@55.0.27)(react@19.2.0))(react-native-safe-area-context@5.6.2(react-native@0.83.6(@babel/core@7.29.0)(@types/react@19.2.14)(bufferutil@4.1.0)(react@19.2.0)(utf-8-validate@6.0.6))(react@19.2.0))(react-native-svg@15.15.3(react-native@0.83.6(@babel/core@7.29.0)(@types/react@19.2.14)(bufferutil@4.1.0)(react@19.2.0)(utf-8-validate@6.0.6))(react@19.2.0)): dependencies: '@posthog/core': 1.39.6 '@posthog/types': 1.392.0 @@ -33229,6 +33263,7 @@ snapshots: '@react-navigation/native': 7.1.33(react-native@0.83.6(@babel/core@7.29.0)(@types/react@19.2.14)(bufferutil@4.1.0)(react@19.2.0)(utf-8-validate@6.0.6))(react@19.2.0) expo-application: 55.0.16(expo@55.0.27) expo-file-system: 55.0.23(expo@55.0.27)(react-native@0.83.6(@babel/core@7.29.0)(@types/react@19.2.14)(bufferutil@4.1.0)(react@19.2.0)(utf-8-validate@6.0.6)) + expo-localization: 55.0.16(expo@55.0.27)(react@19.2.0) react-native-safe-area-context: 5.6.2(react-native@0.83.6(@babel/core@7.29.0)(@types/react@19.2.14)(bufferutil@4.1.0)(react@19.2.0)(utf-8-validate@6.0.6))(react@19.2.0) react-native-svg: 15.15.3(react-native@0.83.6(@babel/core@7.29.0)(@types/react@19.2.14)(bufferutil@4.1.0)(react@19.2.0)(utf-8-validate@6.0.6))(react@19.2.0) @@ -34249,6 +34284,8 @@ snapshots: transitivePeerDependencies: - supports-color + rtl-detect@1.1.2: {} + run-applescript@7.1.0: {} run-parallel@1.2.0: From d89f691c0da54914aa1d01769a0303d0b5fb029b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20=C5=A0=C4=87eki=C4=87?= Date: Tue, 14 Jul 2026 14:07:28 +0200 Subject: [PATCH 02/14] feat(mobile): define voice input state --- .../lib/voice-input/voice-input-state.test.ts | 268 ++++++++++++++++++ .../src/lib/voice-input/voice-input-state.ts | 188 ++++++++++++ apps/mobile/vitest.config.ts | 1 + 3 files changed, 457 insertions(+) create mode 100644 apps/mobile/src/lib/voice-input/voice-input-state.test.ts create mode 100644 apps/mobile/src/lib/voice-input/voice-input-state.ts diff --git a/apps/mobile/src/lib/voice-input/voice-input-state.test.ts b/apps/mobile/src/lib/voice-input/voice-input-state.test.ts new file mode 100644 index 0000000000..0a548f12d1 --- /dev/null +++ b/apps/mobile/src/lib/voice-input/voice-input-state.test.ts @@ -0,0 +1,268 @@ +import { describe, expect, it } from 'vitest'; +import { type ExpoSpeechRecognitionErrorCode } from 'expo-speech-recognition'; + +import { + appendVoiceTranscript, + applyVoiceRecognitionResult, + classifyVoiceInputError, + classifyVoiceInputPermission, + createVoiceTranscriptState, + shouldAbortVoiceInput, +} from './voice-input-state'; + +describe('createVoiceTranscriptState', () => { + it('starts with empty finals, empty interim, and empty transcript', () => { + const state = createVoiceTranscriptState(); + expect(state).toEqual({ finalSegments: [], interim: '', transcript: '' }); + }); + + it('returns a fresh state each call so consumers cannot share references', () => { + const left = createVoiceTranscriptState(); + const right = createVoiceTranscriptState(); + expect(left).not.toBe(right); + expect(left.finalSegments).not.toBe(right.finalSegments); + }); +}); + +describe('applyVoiceRecognitionResult', () => { + it('replaces the prior interim on each partial result rather than appending', () => { + const first = applyVoiceRecognitionResult(createVoiceTranscriptState(), { + isFinal: false, + transcript: 'hel', + }); + const second = applyVoiceRecognitionResult(first, { isFinal: false, transcript: 'hello' }); + const third = applyVoiceRecognitionResult(second, { + isFinal: false, + transcript: 'hello world', + }); + + expect(third.finalSegments).toEqual([]); + expect(third.interim).toBe('hello world'); + expect(third.transcript).toBe('hello world'); + }); + + it('treats the latest interim as the only interim text after several partials', () => { + let state = createVoiceTranscriptState(); + for (const text of ['h', 'he', 'hel', 'hell', 'hello']) { + state = applyVoiceRecognitionResult(state, { isFinal: false, transcript: text }); + } + + expect(state.interim).toBe('hello'); + expect(state.transcript).toBe('hello'); + }); + + it('accumulates final segments Android-style and clears the interim', () => { + let state = createVoiceTranscriptState(); + state = applyVoiceRecognitionResult(state, { isFinal: false, transcript: 'hello' }); + state = applyVoiceRecognitionResult(state, { isFinal: true, transcript: 'hello' }); + state = applyVoiceRecognitionResult(state, { isFinal: true, transcript: 'there' }); + + expect(state.finalSegments).toEqual(['hello', 'there']); + expect(state.interim).toBe(''); + expect(state.transcript).toBe('hello there'); + }); + + it('places a new interim after the accumulated finals in the rendered transcript', () => { + let state = createVoiceTranscriptState(); + state = applyVoiceRecognitionResult(state, { isFinal: true, transcript: 'hello' }); + state = applyVoiceRecognitionResult(state, { isFinal: true, transcript: 'there' }); + state = applyVoiceRecognitionResult(state, { isFinal: false, transcript: 'friend' }); + + expect(state.finalSegments).toEqual(['hello', 'there']); + expect(state.interim).toBe('friend'); + expect(state.transcript).toBe('hello there friend'); + }); + + it('renders a final that follows an interim without leaking the interim text', () => { + let state = createVoiceTranscriptState(); + state = applyVoiceRecognitionResult(state, { isFinal: false, transcript: 'drafting' }); + state = applyVoiceRecognitionResult(state, { isFinal: true, transcript: 'drafting' }); + + expect(state.finalSegments).toEqual(['drafting']); + expect(state.interim).toBe(''); + expect(state.transcript).toBe('drafting'); + }); + + it('trims surrounding whitespace from an interim result', () => { + const state = applyVoiceRecognitionResult(createVoiceTranscriptState(), { + isFinal: false, + transcript: ' hello ', + }); + + expect(state.finalSegments).toEqual([]); + expect(state.interim).toBe('hello'); + expect(state.transcript).toBe('hello'); + }); + + it('stores a final segment with surrounding whitespace trimmed', () => { + const state = applyVoiceRecognitionResult(createVoiceTranscriptState(), { + isFinal: true, + transcript: ' hello ', + }); + + expect(state.finalSegments).toEqual(['hello']); + expect(state.interim).toBe(''); + expect(state.transcript).toBe('hello'); + }); + + it('clears the interim on a whitespace-only final and does not append an empty segment or spacing', () => { + let state = createVoiceTranscriptState(); + state = applyVoiceRecognitionResult(state, { isFinal: false, transcript: 'drafting' }); + state = applyVoiceRecognitionResult(state, { isFinal: true, transcript: ' ' }); + + expect(state.finalSegments).toEqual([]); + expect(state.interim).toBe(''); + expect(state.transcript).toBe(''); + }); +}); + +describe('appendVoiceTranscript', () => { + it('returns just the trimmed transcript when the base draft is empty', () => { + expect(appendVoiceTranscript('', 'hello')).toBe('hello'); + }); + + it('inserts exactly one space when the base does not end in whitespace', () => { + expect(appendVoiceTranscript('hello', 'world')).toBe('hello world'); + }); + + it('does not add an extra separator when the base already ends in whitespace', () => { + expect(appendVoiceTranscript('hello ', 'world')).toBe('hello world'); + expect(appendVoiceTranscript('hello\n', 'world')).toBe('hello\nworld'); + }); + + it('trims only the leading whitespace of the appended transcript', () => { + expect(appendVoiceTranscript('hello', ' world')).toBe('hello world'); + expect(appendVoiceTranscript('hello', ' world again')).toBe('hello world again'); + }); + + it('preserves the base draft when the appended transcript is empty', () => { + expect(appendVoiceTranscript('hello', '')).toBe('hello'); + expect(appendVoiceTranscript('hello', ' ')).toBe('hello'); + }); +}); + +describe('classifyVoiceInputPermission', () => { + it('returns null when the permission has been granted', () => { + expect(classifyVoiceInputPermission({ granted: true, canAskAgain: true })).toBeNull(); + expect( + classifyVoiceInputPermission({ granted: true, canAskAgain: true, restricted: true }) + ).toBeNull(); + }); + + it('returns retryable feedback for a requestable denial', () => { + const feedback = classifyVoiceInputPermission({ granted: false, canAskAgain: true }); + expect(feedback).toEqual({ + action: 'none', + availability: 'available', + message: 'Microphone access is required for voice input.', + retryable: true, + }); + }); + + it('returns open-settings feedback for a permanent denial', () => { + const feedback = classifyVoiceInputPermission({ granted: false, canAskAgain: false }); + expect(feedback).toEqual({ + action: 'open-settings', + availability: 'available', + message: 'Microphone access is off. Enable it in Settings to use voice input.', + retryable: false, + }); + }); + + it('returns a restricted feedback when the device reports a content restriction', () => { + const feedback = classifyVoiceInputPermission({ + granted: false, + canAskAgain: false, + restricted: true, + }); + expect(feedback).toEqual({ + action: 'none', + availability: 'available', + message: 'Voice input is restricted on this device.', + retryable: false, + }); + }); +}); + +describe('classifyVoiceInputError', () => { + const retryableErrorCases: { + code: ExpoSpeechRecognitionErrorCode; + message: string; + }[] = [ + { code: 'no-speech', message: 'No speech detected. Tap the microphone to try again.' }, + { code: 'speech-timeout', message: 'No speech detected. Tap the microphone to try again.' }, + { code: 'network', message: 'Voice input needs a connection right now. Try again.' }, + { code: 'busy', message: 'Voice input is busy. Try again.' }, + { + code: 'audio-capture', + message: 'Voice input stopped. Tap the microphone to try again.', + }, + { + code: 'interrupted', + message: 'Voice input stopped. Tap the microphone to try again.', + }, + { code: 'client', message: 'Voice input stopped. Tap the microphone to try again.' }, + { code: 'unknown', message: 'Voice input stopped. Tap the microphone to try again.' }, + { code: 'bad-grammar', message: 'Voice input stopped. Tap the microphone to try again.' }, + { code: 'aborted', message: 'Voice input stopped. Tap the microphone to try again.' }, + ]; + + it.each(retryableErrorCases)( + 'maps $code to a retryable available feedback with a specific message', + ({ code, message }) => { + const feedback = classifyVoiceInputError(code); + expect(feedback).toEqual({ + action: 'none', + availability: 'available', + message, + retryable: true, + }); + } + ); + + it('maps not-allowed to the permanent open-settings feedback', () => { + expect(classifyVoiceInputError('not-allowed')).toEqual({ + action: 'open-settings', + availability: 'available', + message: 'Microphone access is off. Enable it in Settings to use voice input.', + retryable: false, + }); + }); + + it('maps service-not-allowed to a non-retryable unavailable feedback', () => { + expect(classifyVoiceInputError('service-not-allowed')).toEqual({ + action: 'none', + availability: 'unavailable', + message: "Voice input isn't available on this device.", + retryable: false, + }); + }); + + it('maps language-not-supported to a non-retryable unavailable feedback', () => { + expect(classifyVoiceInputError('language-not-supported')).toEqual({ + action: 'none', + availability: 'unavailable', + message: "Voice input isn't available for this device language.", + retryable: false, + }); + }); +}); + +describe('shouldAbortVoiceInput', () => { + it('returns true when the controller is disabled', () => { + expect(shouldAbortVoiceInput({ appState: 'active', disabled: true })).toBe(true); + }); + + it('returns true when the app is not in the active state', () => { + expect(shouldAbortVoiceInput({ appState: 'background', disabled: false })).toBe(true); + expect(shouldAbortVoiceInput({ appState: 'inactive', disabled: false })).toBe(true); + }); + + it('returns true when both disabled and not active', () => { + expect(shouldAbortVoiceInput({ appState: 'background', disabled: true })).toBe(true); + }); + + it('returns false when the app is active and the controller is not disabled', () => { + expect(shouldAbortVoiceInput({ appState: 'active', disabled: false })).toBe(false); + }); +}); diff --git a/apps/mobile/src/lib/voice-input/voice-input-state.ts b/apps/mobile/src/lib/voice-input/voice-input-state.ts new file mode 100644 index 0000000000..bfea889269 --- /dev/null +++ b/apps/mobile/src/lib/voice-input/voice-input-state.ts @@ -0,0 +1,188 @@ +import { type ExpoSpeechRecognitionErrorCode } from 'expo-speech-recognition'; + +export type VoiceInputStatus = 'idle' | 'starting' | 'listening' | 'stopping'; +export type VoiceInputAvailability = 'available' | 'unavailable'; + +export type VoiceInputFeedback = { + action: 'none' | 'open-settings'; + availability: VoiceInputAvailability; + message: string; + retryable: boolean; +}; + +export type VoiceTranscriptState = { + finalSegments: string[]; + interim: string; + transcript: string; +}; + +type VoiceInputPermission = { + granted: boolean; + canAskAgain: boolean; + restricted?: boolean; +}; + +type VoiceInputLifecycleInput = { + appState: 'active' | 'background' | 'inactive'; + disabled: boolean; +}; + +export function createVoiceTranscriptState(): VoiceTranscriptState { + return { finalSegments: [], interim: '', transcript: '' }; +} + +function renderTranscript(finalSegments: readonly string[], interim: string): string { + const finals = finalSegments.join(' '); + if (!interim) { + return finals; + } + return finals ? `${finals} ${interim}` : interim; +} + +function normalizeSegment(transcript: string): string { + return transcript.trim(); +} + +export function applyVoiceRecognitionResult( + state: VoiceTranscriptState, + result: { isFinal: boolean; transcript: string } +): VoiceTranscriptState { + const normalized = normalizeSegment(result.transcript); + + if (result.isFinal) { + const finalSegments = normalized.length === 0 + ? state.finalSegments + : [...state.finalSegments, normalized]; + return { + finalSegments, + interim: '', + transcript: renderTranscript(finalSegments, ''), + }; + } + + return { + finalSegments: state.finalSegments, + interim: normalized, + transcript: renderTranscript(state.finalSegments, normalized), + }; +} + +export function appendVoiceTranscript(baseDraft: string, transcript: string): string { + const trimmedTranscript = transcript.trimStart(); + if (trimmedTranscript.length === 0) { + return baseDraft; + } + if (baseDraft.length === 0) { + return trimmedTranscript; + } + const lastChar = baseDraft.at(-1); + if (lastChar === ' ' || lastChar === '\n' || lastChar === '\t' || lastChar === '\r') { + return `${baseDraft}${trimmedTranscript}`; + } + return `${baseDraft} ${trimmedTranscript}`; +} + +export function classifyVoiceInputPermission( + permission: VoiceInputPermission +): VoiceInputFeedback | null { + if (permission.granted) { + return null; + } + if (permission.restricted) { + return { + action: 'none', + availability: 'available', + message: 'Voice input is restricted on this device.', + retryable: false, + }; + } + if (permission.canAskAgain) { + return { + action: 'none', + availability: 'available', + message: 'Microphone access is required for voice input.', + retryable: true, + }; + } + return { + action: 'open-settings', + availability: 'available', + message: 'Microphone access is off. Enable it in Settings to use voice input.', + retryable: false, + }; +} + +export function classifyVoiceInputError(code: ExpoSpeechRecognitionErrorCode): VoiceInputFeedback { + switch (code) { + case 'no-speech': + case 'speech-timeout': { + return { + action: 'none', + availability: 'available', + message: 'No speech detected. Tap the microphone to try again.', + retryable: true, + }; + } + case 'network': { + return { + action: 'none', + availability: 'available', + message: 'Voice input needs a connection right now. Try again.', + retryable: true, + }; + } + case 'busy': { + return { + action: 'none', + availability: 'available', + message: 'Voice input is busy. Try again.', + retryable: true, + }; + } + case 'audio-capture': + case 'interrupted': + case 'client': + case 'unknown': + case 'bad-grammar': + case 'aborted': { + return { + action: 'none', + availability: 'available', + message: 'Voice input stopped. Tap the microphone to try again.', + retryable: true, + }; + } + case 'not-allowed': { + return { + action: 'open-settings', + availability: 'available', + message: 'Microphone access is off. Enable it in Settings to use voice input.', + retryable: false, + }; + } + case 'service-not-allowed': { + return { + action: 'none', + availability: 'unavailable', + message: "Voice input isn't available on this device.", + retryable: false, + }; + } + case 'language-not-supported': { + return { + action: 'none', + availability: 'unavailable', + message: "Voice input isn't available for this device language.", + retryable: false, + }; + } + default: { + const unhandled: never = code; + throw new Error(`Unhandled ExpoSpeechRecognitionErrorCode: ${String(unhandled)}`); + } + } +} + +export function shouldAbortVoiceInput(input: VoiceInputLifecycleInput): boolean { + return input.disabled || input.appState !== 'active'; +} diff --git a/apps/mobile/vitest.config.ts b/apps/mobile/vitest.config.ts index c48690fa15..43281a6229 100644 --- a/apps/mobile/vitest.config.ts +++ b/apps/mobile/vitest.config.ts @@ -33,6 +33,7 @@ export default defineConfig({ 'src/lib/kilo-pass/**/*.test.ts', 'src/lib/kilo-pass/**/*.test.tsx', 'src/lib/onboarding/**/*.test.ts', + 'src/lib/voice-input/**/*.test.ts', 'src/components/**/*.test.ts', ], }, From 5af00988d5fe0b7a04e1a606baacef2cfa4b6bfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20=C5=A0=C4=87eki=C4=87?= Date: Tue, 14 Jul 2026 14:35:33 +0200 Subject: [PATCH 03/14] chore(mobile): raise subagent step limits --- apps/mobile/.kilo/agent/mobile-e2e-verifier.md | 2 +- apps/mobile/.kilo/agent/mobile-implementer.md | 2 +- apps/mobile/.kilo/agent/mobile-reviewer.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/mobile/.kilo/agent/mobile-e2e-verifier.md b/apps/mobile/.kilo/agent/mobile-e2e-verifier.md index 5e2a264ffd..50f3ad03e0 100644 --- a/apps/mobile/.kilo/agent/mobile-e2e-verifier.md +++ b/apps/mobile/.kilo/agent/mobile-e2e-verifier.md @@ -2,7 +2,7 @@ description: Verifies an approved mobile-app change end to end against local services, simulator, CLI, and related repositories mode: subagent model: kilo/kilo-auto/efficient -steps: 60 +steps: 100 permission: edit: deny external_directory: allow diff --git a/apps/mobile/.kilo/agent/mobile-implementer.md b/apps/mobile/.kilo/agent/mobile-implementer.md index 3659fecb25..163cbe9d67 100644 --- a/apps/mobile/.kilo/agent/mobile-implementer.md +++ b/apps/mobile/.kilo/agent/mobile-implementer.md @@ -2,7 +2,7 @@ description: Implements an approved mobile-app plan, including required changes in cloud services, shared packages, or sibling repositories mode: subagent model: kilo/kilo-auto/efficient -steps: 50 +steps: 80 permission: edit: allow external_directory: allow diff --git a/apps/mobile/.kilo/agent/mobile-reviewer.md b/apps/mobile/.kilo/agent/mobile-reviewer.md index d6ad444b4f..7ea057bdca 100644 --- a/apps/mobile/.kilo/agent/mobile-reviewer.md +++ b/apps/mobile/.kilo/agent/mobile-reviewer.md @@ -2,7 +2,7 @@ description: Reviews an implementation produced for an approved mobile-app plan, including cross-repository changes mode: subagent model: kilo/kilo-auto/efficient -steps: 35 +steps: 50 permission: edit: deny external_directory: allow From ad36681c81d3ff2bee1140dc0596629df82bd825 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20=C5=A0=C4=87eki=C4=87?= Date: Tue, 14 Jul 2026 14:59:34 +0200 Subject: [PATCH 04/14] feat(mobile): coordinate native voice recognition --- .../src/lib/voice-input/native-voice-input.ts | 99 ++++++ .../voice-input-controller-helpers.ts | 76 +++++ .../voice-input-controller-lifecycle.test.ts | 300 ++++++++++++++++ .../voice-input-controller-results.test.ts | 237 +++++++++++++ .../voice-input-controller-start.test.ts | 323 ++++++++++++++++++ .../voice-input-controller-test-helpers.ts | 205 +++++++++++ .../lib/voice-input/voice-input-controller.ts | 310 +++++++++++++++++ .../lib/voice-input/voice-input-listeners.ts | 109 ++++++ 8 files changed, 1659 insertions(+) create mode 100644 apps/mobile/src/lib/voice-input/native-voice-input.ts create mode 100644 apps/mobile/src/lib/voice-input/voice-input-controller-helpers.ts create mode 100644 apps/mobile/src/lib/voice-input/voice-input-controller-lifecycle.test.ts create mode 100644 apps/mobile/src/lib/voice-input/voice-input-controller-results.test.ts create mode 100644 apps/mobile/src/lib/voice-input/voice-input-controller-start.test.ts create mode 100644 apps/mobile/src/lib/voice-input/voice-input-controller-test-helpers.ts create mode 100644 apps/mobile/src/lib/voice-input/voice-input-controller.ts create mode 100644 apps/mobile/src/lib/voice-input/voice-input-listeners.ts diff --git a/apps/mobile/src/lib/voice-input/native-voice-input.ts b/apps/mobile/src/lib/voice-input/native-voice-input.ts new file mode 100644 index 0000000000..009d1ff737 --- /dev/null +++ b/apps/mobile/src/lib/voice-input/native-voice-input.ts @@ -0,0 +1,99 @@ +import { Platform } from 'react-native'; +import { + type ExpoSpeechRecognitionErrorEvent, + ExpoSpeechRecognitionModule, + type ExpoSpeechRecognitionResultEvent, +} from 'expo-speech-recognition'; + +import { + createVoiceInputController, + type VoiceInputNative, + type VoiceInputNativeEvent, +} from './voice-input-controller'; + +const ANDROID_CONTINUOUS_MIN_API_LEVEL = 33; + +function isAndroidApiLevelAtLeast(level: number): boolean { + if (Platform.OS !== 'android') { + return false; + } + const version = Platform.Version; + if (typeof version === 'number') { + return version >= level; + } + const parsed = Number.parseInt(String(version), 10); + return Number.isFinite(parsed) ? parsed >= level : false; +} + +function supportsContinuousRecognition(): boolean { + if (Platform.OS === 'ios') { + return true; + } + return isAndroidApiLevelAtLeast(ANDROID_CONTINUOUS_MIN_API_LEVEL); +} + +type ExpoSpeechRecognitionModuleType = typeof ExpoSpeechRecognitionModule; + +function bindListener( + module: ExpoSpeechRecognitionModuleType, + event: K, + listener: (event: VoiceInputNativeEvent[K]) => void +): { remove(): void } { + // The native module's addListener is generic over its full event map, so + // when called with our narrower event union the listener parameter is + // widened to an intersection of every native listener type. Per-event + // dispatch below instantiates the generic at each known event name so the + // listener type is concrete and the assignment is sound: each branch + // forwards a listener whose payload type matches the native event exactly. + if (event === 'start') { + return module.addListener('start', listener as (event: null) => void); + } + if (event === 'result') { + return module.addListener( + 'result', + listener as (event: ExpoSpeechRecognitionResultEvent) => void + ); + } + if (event === 'nomatch') { + return module.addListener('nomatch', listener as (event: null) => void); + } + if (event === 'error') { + return module.addListener( + 'error', + listener as (event: ExpoSpeechRecognitionErrorEvent) => void + ); + } + return module.addListener('end', listener as (event: null) => void); +} + +const native: VoiceInputNative = { + addListener(event, listener) { + return bindListener(ExpoSpeechRecognitionModule, event, listener); + }, + getPermissions: async () => { + const result = await ExpoSpeechRecognitionModule.getPermissionsAsync(); + return result; + }, + requestPermissions: async () => { + const result = await ExpoSpeechRecognitionModule.requestPermissionsAsync(); + return result; + }, + isRecognitionAvailable: () => ExpoSpeechRecognitionModule.isRecognitionAvailable(), + supportsContinuousRecognition, + start: options => { + ExpoSpeechRecognitionModule.start({ + continuous: options.continuous, + interimResults: options.interimResults, + lang: options.lang, + maxAlternatives: options.maxAlternatives, + }); + }, + stop: () => { + ExpoSpeechRecognitionModule.stop(); + }, + abort: () => { + ExpoSpeechRecognitionModule.abort(); + }, +}; + +export const voiceInputController = createVoiceInputController(native); diff --git a/apps/mobile/src/lib/voice-input/voice-input-controller-helpers.ts b/apps/mobile/src/lib/voice-input/voice-input-controller-helpers.ts new file mode 100644 index 0000000000..9848241cda --- /dev/null +++ b/apps/mobile/src/lib/voice-input/voice-input-controller-helpers.ts @@ -0,0 +1,76 @@ +import { classifyVoiceInputPermission, type VoiceInputFeedback } from './voice-input-state'; +import { type VoiceInputNative } from './voice-input-controller'; + +type PermissionResult = + | { kind: 'granted' } + | { kind: 'feedback'; feedback: VoiceInputFeedback } + | { kind: 'client-error' }; + +export type Lifecycle = { disposed: boolean }; + +export function isDisposed(lifecycle: Lifecycle): boolean { + return lifecycle.disposed; +} + +const noopBooleanResolver = (_ok: boolean): void => undefined; + +export function createTerminal(): { + promise: Promise; + resolve: (ok: boolean) => void; +} { + let resolveTerminal: (ok: boolean) => void = noopBooleanResolver; + const promise = new Promise(resolve => { + resolveTerminal = resolve; + }); + return { promise, resolve: resolveTerminal }; +} + +export async function waitForTerminal(current: { + terminalPromise: Promise; +}): Promise { + try { + await current.terminalPromise; + } catch { + // terminalPromise never rejects + } +} + +async function getPermissionOnce( + native: VoiceInputNative +): Promise { + try { + const current = await native.getPermissions(); + if (current.granted) { + return { kind: 'granted' }; + } + if (!current.canAskAgain) { + const feedback = classifyVoiceInputPermission(current); + return feedback ? { kind: 'feedback', feedback } : { kind: 'client-error' }; + } + return { kind: 'needs-request' }; + } catch { + return { kind: 'client-error' }; + } +} + +async function requestPermissionOnce(native: VoiceInputNative): Promise { + try { + const requested = await native.requestPermissions(); + if (requested.granted) { + return { kind: 'granted' }; + } + const feedback = classifyVoiceInputPermission(requested); + return feedback ? { kind: 'feedback', feedback } : { kind: 'client-error' }; + } catch { + return { kind: 'client-error' }; + } +} + +export async function acquirePermission(native: VoiceInputNative): Promise { + const first = await getPermissionOnce(native); + if (first.kind !== 'needs-request') { + return first; + } + const result = await requestPermissionOnce(native); + return result; +} diff --git a/apps/mobile/src/lib/voice-input/voice-input-controller-lifecycle.test.ts b/apps/mobile/src/lib/voice-input/voice-input-controller-lifecycle.test.ts new file mode 100644 index 0000000000..0fb3b487ae --- /dev/null +++ b/apps/mobile/src/lib/voice-input/voice-input-controller-lifecycle.test.ts @@ -0,0 +1,300 @@ +import { describe, expect, it } from 'vitest'; + +import { createVoiceInputController } from './voice-input-controller'; +import { + createVoiceInputNativeHarness, + makeStartOptions, + type VoiceInputNativeHarness, +} from './voice-input-controller-test-helpers'; + +async function isPending(promise: Promise): Promise { + const sentinel = Symbol('pending'); + const result = await Promise.race([promise, Promise.resolve(sentinel)]); + return result === sentinel; +} + +describe('createVoiceInputController - lifecycle', () => { + const build = (overrides: Partial = {}) => { + const harness = createVoiceInputNativeHarness(overrides); + const controller = createVoiceInputController(harness.native); + return { harness, controller }; + }; + + describe('subscribe', () => { + it('notifies for starting, listening, stopping, and idle transitions', async () => { + const { harness, controller } = build(); + const observed: string[] = []; + controller.subscribe((snap): void => { + observed.push(snap.status); + }); + const started = await controller.start(makeStartOptions({ owner: 'A' })); + expect(started).toBe(true); + + harness.emit('start', null); + const stopped = controller.stop('A'); + harness.emit('end', null); + const stopResult = await stopped; + expect(stopResult).toBe(true); + + expect(observed).toEqual( + expect.arrayContaining(['starting', 'listening', 'stopping', 'idle']) + ); + }); + + it('unsubscribe stops further notifications', async () => { + const { harness, controller } = build(); + const observed: string[] = []; + const unsubscribe = controller.subscribe((snap): void => { + observed.push(snap.status); + }); + unsubscribe(); + await controller.start(makeStartOptions()); + harness.emit('start', null); + expect(observed).toEqual([]); + }); + }); + + describe('end and terminalization', () => { + it('clears the owner, sets idle, and resolves the terminal promise with success', async () => { + const { harness, controller } = build(); + await controller.start(makeStartOptions()); + harness.emit('start', null); + const stopPromise = controller.stop('owner1'); + harness.emit('end', null); + + await expect(stopPromise).resolves.toBe(true); + expect(controller.getSnapshot().owner).toBeNull(); + expect(controller.getSnapshot().status).toBe('idle'); + }); + + it('resolves the terminal promise with false when a failure was reported before end', async () => { + const { harness, controller } = build(); + await controller.start(makeStartOptions()); + harness.emit('start', null); + const stopPromise = controller.stop('owner1'); + harness.emit('error', { error: 'no-speech', message: 'no speech' }); + harness.emit('end', null); + + await expect(stopPromise).resolves.toBe(false); + expect(controller.getSnapshot().status).toBe('idle'); + }); + + it('ignores duplicate or late events after terminalization', async () => { + const { harness, controller } = build(); + const drafts: string[] = []; + const feedback: { + action: 'none' | 'open-settings'; + availability: 'available' | 'unavailable'; + message: string; + retryable: boolean; + }[] = []; + await controller.start( + makeStartOptions({ + owner: 'A', + onDraftChange: (d): void => { + drafts.push(d); + }, + onFeedback: (f): void => { + feedback.push(f); + }, + }) + ); + harness.emit('start', null); + harness.emitResult('hello', false); + harness.emit('end', null); + await controller.stop('A'); + + drafts.length = 0; + feedback.length = 0; + + harness.emitResult('late', false); + harness.emit('error', { error: 'no-speech', message: 'late' }); + harness.emit('end', null); + + expect(drafts).toEqual([]); + expect(feedback).toEqual([]); + expect(controller.getSnapshot().status).toBe('idle'); + }); + }); + + describe('stop', () => { + it('returns true when there is no active session', async () => { + const { harness, controller } = build(); + await expect(controller.stop('A')).resolves.toBe(true); + expect(harness.mocks.stop).not.toHaveBeenCalled(); + }); + + it('returns true for a non-owner call and leaves the active session alone', async () => { + const { harness, controller } = build(); + await controller.start(makeStartOptions({ owner: 'A' })); + harness.emit('start', null); + await expect(controller.stop('B')).resolves.toBe(true); + expect(harness.mocks.stop).not.toHaveBeenCalled(); + expect(controller.getSnapshot().owner).toBe('A'); + expect(controller.getSnapshot().status).toBe('listening'); + }); + + it('owner stop calls native.stop once, sets stopping, and resolves only after end', async () => { + const { harness, controller } = build(); + await controller.start(makeStartOptions({ owner: 'A' })); + harness.emit('start', null); + + const stopPromise = controller.stop('A'); + expect(harness.mocks.stop).toHaveBeenCalledTimes(1); + expect(controller.getSnapshot().status).toBe('stopping'); + + expect(await isPending(stopPromise)).toBe(true); + + harness.emit('end', null); + await expect(stopPromise).resolves.toBe(true); + }); + + it('continues to process result events before end so the draft is updated', async () => { + const { harness, controller } = build(); + const drafts: string[] = []; + await controller.start( + makeStartOptions({ + onDraftChange: (d): void => { + drafts.push(d); + }, + }) + ); + harness.emit('start', null); + + const stopPromise = controller.stop('owner1'); + harness.emitResult('final word', true); + expect(drafts.at(-1)).toBe('final word'); + + harness.emit('end', null); + await expect(stopPromise).resolves.toBe(true); + }); + + it('reports client, terminalizes safely, and resolves false when native.stop throws', async () => { + const { harness, controller } = build(); + harness.mocks.stop = ((): void => { + throw new Error('boom'); + }) as typeof harness.mocks.stop; + const feedback: { + action: 'none' | 'open-settings'; + availability: 'available' | 'unavailable'; + message: string; + retryable: boolean; + }[] = []; + await controller.start( + makeStartOptions({ + onFeedback: (f): void => { + feedback.push(f); + }, + }) + ); + harness.emit('start', null); + + await expect(controller.stop('owner1')).resolves.toBe(false); + + expect(feedback).toEqual([ + { + action: 'none', + availability: 'available', + message: 'Voice input stopped. Tap the microphone to try again.', + retryable: true, + }, + ]); + expect(controller.getSnapshot().owner).toBeNull(); + expect(controller.getSnapshot().status).toBe('idle'); + }); + }); + + describe('abort', () => { + it('returns true when there is no active session', async () => { + const { harness, controller } = build(); + await expect(controller.abort()).resolves.toBe(true); + expect(harness.mocks.abort).not.toHaveBeenCalled(); + }); + + it('returns true for a specified non-owner call', async () => { + const { harness, controller } = build(); + await controller.start(makeStartOptions({ owner: 'A' })); + harness.emit('start', null); + await expect(controller.abort('B')).resolves.toBe(true); + expect(harness.mocks.abort).not.toHaveBeenCalled(); + expect(controller.getSnapshot().owner).toBe('A'); + }); + + it('active abort sets expectedAbort and stopping, calls native.abort once, and resolves after end', async () => { + const { harness, controller } = build(); + await controller.start(makeStartOptions({ owner: 'A' })); + harness.emit('start', null); + + const abortPromise = controller.abort('A'); + expect(harness.mocks.abort).toHaveBeenCalledTimes(1); + expect(controller.getSnapshot().status).toBe('stopping'); + + expect(await isPending(abortPromise)).toBe(true); + + harness.emit('error', { error: 'aborted', message: 'aborted' }); + harness.emit('end', null); + await expect(abortPromise).resolves.toBe(true); + }); + + it('terminalizes safely and resolves false when native.abort throws', async () => { + const { harness, controller } = build(); + harness.mocks.abort = ((): void => { + throw new Error('boom'); + }) as typeof harness.mocks.abort; + await controller.start(makeStartOptions({ owner: 'A' })); + harness.emit('start', null); + + await expect(controller.abort('A')).resolves.toBe(false); + expect(controller.getSnapshot().owner).toBeNull(); + expect(controller.getSnapshot().status).toBe('idle'); + }); + }); + + describe('dispose', () => { + it('removes every permanent native listener and clears subscribers', () => { + const { harness, controller } = build(); + controller.subscribe((): void => undefined); + controller.dispose(); + + expect(harness.listenerCount('start')).toBe(0); + expect(harness.listenerCount('result')).toBe(0); + expect(harness.listenerCount('nomatch')).toBe(0); + expect(harness.listenerCount('error')).toBe(0); + expect(harness.listenerCount('end')).toBe(0); + }); + + it('initiates an expected abort when active, terminalizes safely, and resolves', async () => { + const { harness, controller } = build(); + const feedback: { + action: 'none' | 'open-settings'; + availability: 'available' | 'unavailable'; + message: string; + retryable: boolean; + }[] = []; + await controller.start( + makeStartOptions({ + owner: 'A', + onFeedback: (f): void => { + feedback.push(f); + }, + }) + ); + harness.emit('start', null); + + controller.dispose(); + + expect(harness.mocks.abort).toHaveBeenCalledTimes(1); + expect(feedback).toEqual([]); + expect(controller.getSnapshot().owner).toBeNull(); + expect(controller.getSnapshot().status).toBe('idle'); + }); + + it('resolves even when the native module never emits end after abort', async () => { + const { harness, controller } = build(); + await controller.start(makeStartOptions({ owner: 'A' })); + harness.emit('start', null); + controller.dispose(); + expect(controller.getSnapshot().status).toBe('idle'); + }); + }); +}); diff --git a/apps/mobile/src/lib/voice-input/voice-input-controller-results.test.ts b/apps/mobile/src/lib/voice-input/voice-input-controller-results.test.ts new file mode 100644 index 0000000000..30cfe9843d --- /dev/null +++ b/apps/mobile/src/lib/voice-input/voice-input-controller-results.test.ts @@ -0,0 +1,237 @@ +import { describe, expect, it, vi } from 'vitest'; + +import { createVoiceInputController } from './voice-input-controller'; +import { + createVoiceInputNativeHarness, + deferredPermission, + makeStartOptions, + recordFeedback, + type VoiceInputNativeHarness, +} from './voice-input-controller-test-helpers'; + +describe('createVoiceInputController - events and serialization', () => { + const build = (overrides: Partial = {}) => { + const harness = createVoiceInputNativeHarness(overrides); + const controller = createVoiceInputController(harness.native); + return { harness, controller }; + }; + + describe('lifecycle events', () => { + it('moves to listening when the native start event fires for the current session', async () => { + const { harness, controller } = build(); + const result = await controller.start(makeStartOptions()); + expect(result).toBe(true); + harness.emit('start', null); + expect(controller.getSnapshot().status).toBe('listening'); + }); + + it('accumulates result transcripts and emits the appended base draft', async () => { + const { harness, controller } = build(); + const drafts: string[] = []; + const started = await controller.start( + makeStartOptions({ + baseDraft: 'hi ', + onDraftChange: (d): void => { + drafts.push(d); + }, + }) + ); + expect(started).toBe(true); + harness.emit('start', null); + + harness.emitResult('hel', false); + expect(drafts.at(-1)).toBe('hi hel'); + harness.emitResult('hello', true); + expect(drafts.at(-1)).toBe('hi hello'); + harness.emitResult('there', false); + expect(drafts.at(-1)).toBe('hi hello there'); + }); + + it('ignores empty results and empty transcripts', async () => { + const { harness, controller } = build(); + const drafts: string[] = []; + await controller.start( + makeStartOptions({ + onDraftChange: (d): void => { + drafts.push(d); + }, + }) + ); + harness.emit('start', null); + + harness.emit('result', { isFinal: false, results: [] }); + harness.emit('result', { + isFinal: false, + results: [{ transcript: '', confidence: 0, segments: [] }], + }); + expect(drafts).toEqual([]); + }); + + it('reports the no-speech feedback on nomatch, and ignores subsequent nomatch events', async () => { + const { harness, controller } = build(); + const fb = recordFeedback(); + await controller.start(makeStartOptions({ onFeedback: fb.onFeedback })); + harness.emit('start', null); + + harness.emit('nomatch', null); + harness.emit('nomatch', null); + + expect(fb.feedback).toEqual([ + { + action: 'none', + availability: 'available', + message: 'No speech detected. Tap the microphone to try again.', + retryable: true, + }, + ]); + }); + + it('reports the mapped feedback on error, and reports only the first error', async () => { + const { harness, controller } = build(); + const fb = recordFeedback(); + await controller.start(makeStartOptions({ onFeedback: fb.onFeedback })); + harness.emit('start', null); + + harness.emit('error', { error: 'network', message: 'offline' }); + harness.emit('error', { error: 'busy', message: 'busy' }); + + expect(fb.feedback).toEqual([ + { + action: 'none', + availability: 'available', + message: 'Voice input needs a connection right now. Try again.', + retryable: true, + }, + ]); + }); + + it('is silent on an expected abort error', async () => { + const { harness, controller } = build(); + const fb = recordFeedback(); + await controller.start(makeStartOptions({ onFeedback: fb.onFeedback })); + harness.emit('start', null); + const abortPromise = controller.abort('owner1'); + harness.emit('error', { error: 'aborted', message: 'aborted' }); + harness.emit('end', null); + await abortPromise; + expect(fb.feedback).toEqual([]); + }); + + it('maps a non-expected aborted error through the classifier', async () => { + const { harness, controller } = build(); + const fb = recordFeedback(); + await controller.start(makeStartOptions({ onFeedback: fb.onFeedback })); + harness.emit('start', null); + + harness.emit('error', { error: 'aborted', message: 'aborted' }); + + expect(fb.feedback).toEqual([ + { + action: 'none', + availability: 'available', + message: 'Voice input stopped. Tap the microphone to try again.', + retryable: true, + }, + ]); + }); + + it('permanently flips availability to unavailable when a service-not-allowed feedback is reported', async () => { + const { harness, controller } = build(); + const fb = recordFeedback(); + await controller.start(makeStartOptions({ onFeedback: fb.onFeedback })); + harness.emit('start', null); + expect(controller.getSnapshot().availability).toBe('available'); + + harness.emit('error', { error: 'service-not-allowed', message: 'no service' }); + expect(controller.getSnapshot().availability).toBe('unavailable'); + expect(fb.feedback).toEqual([ + { + action: 'none', + availability: 'unavailable', + message: "Voice input isn't available on this device.", + retryable: false, + }, + ]); + }); + }); + + describe('start serialization', () => { + it('waits for the previous session to end before checking permission for the new owner', async () => { + const { harness, controller } = build(); + const started = await controller.start(makeStartOptions({ owner: 'A' })); + expect(started).toBe(true); + harness.emit('start', null); + expect(harness.mocks.start).toHaveBeenCalledTimes(1); + + const bDeferred = deferredPermission(); + harness.mocks.getPermissions = vi.fn(async () => { + const permission = await bDeferred.promise; + return permission; + }); + harness.mocks.requestPermissions = vi.fn(async () => { + const permission = await bDeferred.promise; + return permission; + }); + + const second = controller.start(makeStartOptions({ owner: 'B' })); + // Let the first microtask flush: start should have entered and initiated abort on A. + await Promise.resolve(); + expect(harness.mocks.abort).toHaveBeenCalledTimes(1); + expect(controller.getSnapshot().owner).toBe('A'); + expect(controller.getSnapshot().status).toBe('stopping'); + + // B's permission must not be requested before A ends. + expect(harness.mocks.getPermissions).not.toHaveBeenCalled(); + + // A ends -> B can now request permission and complete startup. + harness.emit('end', null); + bDeferred.resolve({ granted: true, canAskAgain: true, restricted: false }); + await expect(second).resolves.toBe(true); + expect(harness.mocks.getPermissions).toHaveBeenCalledTimes(1); + expect(harness.mocks.start).toHaveBeenCalledTimes(2); + expect(controller.getSnapshot().owner).toBe('B'); + expect(controller.getSnapshot().status).toBe('starting'); + }); + + it('does not call native.start if dispose() races with a pending getPermissions', async () => { + const { harness, controller } = build(); + const deferred = deferredPermission(); + harness.mocks.getPermissions = vi.fn(async () => { + const permission = await deferred.promise; + return permission; + }); + harness.mocks.requestPermissions = vi.fn(async () => { + const permission = await deferred.promise; + return permission; + }); + + const started = controller.start(makeStartOptions({ owner: 'A' })); + await Promise.resolve(); + // getPermissions is now in flight, waiting on the deferred. + // The controller has no session yet; dispose should still resolve and the + // pending permission should not trigger native.start. + controller.dispose(); + deferred.resolve({ granted: true, canAskAgain: true, restricted: false }); + await expect(started).resolves.toBe(false); + expect(harness.mocks.start).not.toHaveBeenCalled(); + expect(controller.getSnapshot().status).toBe('idle'); + expect(controller.getSnapshot().owner).toBeNull(); + }); + + it('dispose resolves any active terminal promise and removes all listeners', async () => { + const { harness, controller } = build(); + const started = await controller.start(makeStartOptions({ owner: 'A' })); + expect(started).toBe(true); + harness.emit('start', null); + + controller.dispose(); + expect(harness.listenerCount('start')).toBe(0); + expect(harness.listenerCount('result')).toBe(0); + expect(harness.listenerCount('nomatch')).toBe(0); + expect(harness.listenerCount('error')).toBe(0); + expect(harness.listenerCount('end')).toBe(0); + expect(controller.getSnapshot().status).toBe('idle'); + expect(controller.getSnapshot().owner).toBeNull(); + }); + }); +}); diff --git a/apps/mobile/src/lib/voice-input/voice-input-controller-start.test.ts b/apps/mobile/src/lib/voice-input/voice-input-controller-start.test.ts new file mode 100644 index 0000000000..6ccfd3e8a6 --- /dev/null +++ b/apps/mobile/src/lib/voice-input/voice-input-controller-start.test.ts @@ -0,0 +1,323 @@ +import { describe, expect, it, vi } from 'vitest'; + +import { + createVoiceInputController, + type VoiceInputControllerSnapshot, +} from './voice-input-controller'; +import { + createVoiceInputNativeHarness, + makeStartOptions, + recordFeedback, + type VoiceInputNativeHarness, +} from './voice-input-controller-test-helpers'; + +describe('createVoiceInputController - start and permission', () => { + const build = (overrides: Partial = {}) => { + const harness = createVoiceInputNativeHarness(overrides); + const controller = createVoiceInputController(harness.native); + return { harness, controller }; + }; + + it('reports availability from isRecognitionAvailable()', () => { + const { controller } = build(); + expect(controller.getSnapshot().availability).toBe('available'); + + const unavailableHarness = createVoiceInputNativeHarness({ isAvailable: false }); + const unavailableController = createVoiceInputController(unavailableHarness.native); + expect(unavailableController.getSnapshot().availability).toBe('unavailable'); + }); + + it('starts idle with no owner', () => { + const { controller } = build(); + expect(controller.getSnapshot().owner).toBeNull(); + expect(controller.getSnapshot().status).toBe('idle'); + }); + + it('registers one permanent native listener for each lifecycle event', () => { + const { harness } = build(); + expect(harness.listenerCount('start')).toBe(1); + expect(harness.listenerCount('result')).toBe(1); + expect(harness.listenerCount('nomatch')).toBe(1); + expect(harness.listenerCount('error')).toBe(1); + expect(harness.listenerCount('end')).toBe(1); + }); + + describe('availability gating', () => { + it('returns false, reports unavailable feedback, and skips permission/start when unavailable', async () => { + const { harness, controller } = build({ isAvailable: false }); + const fb = recordFeedback(); + + const result = await controller.start({ + baseDraft: '', + languageTag: 'en-US', + onDraftChange: (): void => undefined, + onFeedback: fb.onFeedback, + owner: 'A', + }); + + expect(result).toBe(false); + expect(fb.feedback).toEqual([ + { + action: 'none', + availability: 'unavailable', + message: "Voice input isn't available on this device.", + retryable: false, + }, + ]); + expect(harness.mocks.getPermissions).not.toHaveBeenCalled(); + expect(harness.mocks.requestPermissions).not.toHaveBeenCalled(); + expect(harness.mocks.start).not.toHaveBeenCalled(); + expect(controller.getSnapshot().status).toBe('idle'); + }); + }); + + describe('permission flow', () => { + it('requests permission only when not granted and canAskAgain', async () => { + const { harness, controller } = build({ + permissionGranted: false, + permissionCanAskAgain: true, + }); + await controller.start(makeStartOptions()); + expect(harness.mocks.getPermissions).toHaveBeenCalledTimes(1); + expect(harness.mocks.requestPermissions).toHaveBeenCalledTimes(1); + }); + + it('does not request permission when already granted', async () => { + const { harness, controller } = build({ permissionGranted: true }); + await controller.start(makeStartOptions()); + expect(harness.mocks.getPermissions).toHaveBeenCalledTimes(1); + expect(harness.mocks.requestPermissions).not.toHaveBeenCalled(); + }); + + it('does not request permission when canAskAgain is false and reports open-settings', async () => { + const { harness, controller } = build({ + permissionGranted: false, + permissionCanAskAgain: false, + }); + const fb = recordFeedback(); + + const result = await controller.start({ + baseDraft: '', + languageTag: 'en-US', + onDraftChange: (): void => undefined, + onFeedback: fb.onFeedback, + owner: 'A', + }); + + expect(result).toBe(false); + expect(harness.mocks.requestPermissions).not.toHaveBeenCalled(); + expect(fb.feedback).toEqual([ + { + action: 'open-settings', + availability: 'available', + message: 'Microphone access is off. Enable it in Settings to use voice input.', + retryable: false, + }, + ]); + }); + + it('reports retryable feedback when request returns a requestable denial', async () => { + const { harness, controller } = build(); + const permission = { + granted: false, + canAskAgain: true, + restricted: false, + }; + harness.mocks.getPermissions = vi.fn(async () => { + const result = await Promise.resolve(permission); + return result; + }); + harness.mocks.requestPermissions = vi.fn(async () => { + const result = await Promise.resolve(permission); + return result; + }); + const fb = recordFeedback(); + + const result = await controller.start({ + baseDraft: '', + languageTag: 'en-US', + onDraftChange: (): void => undefined, + onFeedback: fb.onFeedback, + owner: 'A', + }); + + expect(result).toBe(false); + expect(fb.feedback).toEqual([ + { + action: 'none', + availability: 'available', + message: 'Microphone access is required for voice input.', + retryable: true, + }, + ]); + }); + + it('reports restricted feedback when the device reports a content restriction', async () => { + const { controller } = build({ + permissionGranted: false, + permissionCanAskAgain: false, + permissionRestricted: true, + }); + const fb = recordFeedback(); + + const result = await controller.start({ + baseDraft: '', + languageTag: 'en-US', + onDraftChange: (): void => undefined, + onFeedback: fb.onFeedback, + owner: 'A', + }); + + expect(result).toBe(false); + expect(fb.feedback).toEqual([ + { + action: 'none', + availability: 'available', + message: 'Voice input is restricted on this device.', + retryable: false, + }, + ]); + }); + + it('reports client feedback and returns false when getPermissions rejects', async () => { + const { harness, controller } = build(); + harness.mocks.getPermissions = vi.fn(async () => { + await Promise.resolve(); + throw new Error('boom'); + }); + const fb = recordFeedback(); + + const result = await controller.start({ + baseDraft: '', + languageTag: 'en-US', + onDraftChange: (): void => undefined, + onFeedback: fb.onFeedback, + owner: 'A', + }); + + expect(result).toBe(false); + expect(fb.feedback).toEqual([ + { + action: 'none', + availability: 'available', + message: 'Voice input stopped. Tap the microphone to try again.', + retryable: true, + }, + ]); + expect(harness.mocks.start).not.toHaveBeenCalled(); + }); + + it('reports client feedback and returns false when requestPermissions rejects', async () => { + const { harness, controller } = build({ + permissionGranted: false, + permissionCanAskAgain: true, + }); + harness.mocks.getPermissions = vi.fn(async () => { + const result = await Promise.resolve({ + granted: false, + canAskAgain: true, + restricted: false, + }); + return result; + }); + harness.mocks.requestPermissions = vi.fn(async () => { + await Promise.resolve(); + throw new Error('boom'); + }); + const fb = recordFeedback(); + + const result = await controller.start({ + baseDraft: '', + languageTag: 'en-US', + onDraftChange: (): void => undefined, + onFeedback: fb.onFeedback, + owner: 'A', + }); + + expect(result).toBe(false); + expect(fb.feedback).toEqual([ + { + action: 'none', + availability: 'available', + message: 'Voice input stopped. Tap the microphone to try again.', + retryable: true, + }, + ]); + expect(harness.mocks.start).not.toHaveBeenCalled(); + }); + }); + + describe('native.start invocation', () => { + it('sets owner and starting, notifies subscribers, then calls native.start with the documented options', async () => { + const { harness, controller } = build(); + const observed: VoiceInputControllerSnapshot[] = []; + controller.subscribe((snap): void => { + observed.push(snap); + }); + const fb = recordFeedback(); + + const started = await controller.start({ + baseDraft: 'hello ', + languageTag: 'fr-FR', + onDraftChange: (): void => undefined, + onFeedback: fb.onFeedback, + owner: 'A', + }); + + expect(started).toBe(true); + expect(harness.mocks.start).toHaveBeenCalledTimes(1); + expect(harness.mocks.start).toHaveBeenCalledWith({ + continuous: true, + interimResults: true, + lang: 'fr-FR', + maxAlternatives: 1, + }); + + const startingSnap = observed.find(snap => snap.status === 'starting'); + expect(startingSnap).toBeDefined(); + if (startingSnap) { + expect(startingSnap.owner).toBe('A'); + expect(startingSnap.availability).toBe('available'); + } + }); + + it('passes continuous:false when supportsContinuousRecognition returns false', async () => { + const { harness, controller } = build({ supportsContinuous: false }); + await controller.start(makeStartOptions()); + expect(harness.mocks.start).toHaveBeenCalledWith({ + continuous: false, + interimResults: true, + lang: 'en-US', + maxAlternatives: 1, + }); + }); + + it('catches a synchronous native.start throw, reports client once, terminalizes, and returns false', async () => { + const { harness, controller } = build(); + harness.mocks.start = vi.fn((): void => { + throw new Error('boom'); + }); + const fb = recordFeedback(); + + const result = await controller.start({ + baseDraft: '', + languageTag: 'en-US', + onDraftChange: (): void => undefined, + onFeedback: fb.onFeedback, + owner: 'A', + }); + + expect(result).toBe(false); + expect(fb.feedback).toEqual([ + { + action: 'none', + availability: 'available', + message: 'Voice input stopped. Tap the microphone to try again.', + retryable: true, + }, + ]); + expect(controller.getSnapshot().owner).toBeNull(); + expect(controller.getSnapshot().status).toBe('idle'); + }); + }); +}); diff --git a/apps/mobile/src/lib/voice-input/voice-input-controller-test-helpers.ts b/apps/mobile/src/lib/voice-input/voice-input-controller-test-helpers.ts new file mode 100644 index 0000000000..16fbaa5e6f --- /dev/null +++ b/apps/mobile/src/lib/voice-input/voice-input-controller-test-helpers.ts @@ -0,0 +1,205 @@ +import { type ExpoSpeechRecognitionResult } from 'expo-speech-recognition'; +import { type Mock, vi } from 'vitest'; + +import { + type VoiceInputNative, + type VoiceInputNativeEvent, + type VoiceInputNativePermission, + type VoiceInputNativeStartOptions, + type VoiceInputStartOptions, +} from './voice-input-controller'; + +type AnyListener = (event: VoiceInputNativeEvent[keyof VoiceInputNativeEvent]) => void; + +export type VoiceInputNativeControls = { + isAvailable: boolean; + supportsContinuous: boolean; + permissionGranted: boolean; + permissionCanAskAgain: boolean; + permissionRestricted: boolean; +}; + +export type VoiceInputNativeMocks = { + abort: Mock<() => void>; + getPermissions: Mock<() => Promise>; + requestPermissions: Mock<() => Promise>; + start: Mock<(options: VoiceInputNativeStartOptions) => void>; + stop: Mock<() => void>; +}; + +export type VoiceInputNativeHarness = { + controls: VoiceInputNativeControls; + mocks: VoiceInputNativeMocks; + emit( + event: keyof VoiceInputNativeEvent, + payload: VoiceInputNativeEvent[keyof VoiceInputNativeEvent] + ): void; + emitResult(transcript: string, isFinal: boolean): void; + listenerCount(event: keyof VoiceInputNativeEvent): number; + native: VoiceInputNative; +}; + +function emptyResult(transcript: string): ExpoSpeechRecognitionResult { + return { + transcript, + confidence: 0, + segments: [], + }; +} + +function buildNative( + controls: VoiceInputNativeControls, + mocks: VoiceInputNativeMocks, + listeners: Map> +): VoiceInputNative { + const addListener: VoiceInputNative['addListener'] = (event, listener) => { + const boxed = listener as AnyListener; + let set = listeners.get(event); + if (!set) { + set = new Set(); + listeners.set(event, set); + } + set.add(boxed); + return { + remove: (): void => { + const current = listeners.get(event); + if (current) { + current.delete(boxed); + } + }, + }; + }; + + return { + addListener, + getPermissions: async () => { + const result = await mocks.getPermissions(); + return result; + }, + requestPermissions: async () => { + const result = await mocks.requestPermissions(); + return result; + }, + isRecognitionAvailable: () => controls.isAvailable, + supportsContinuousRecognition: () => controls.supportsContinuous, + start: (options): void => { + mocks.start(options); + }, + stop: (): void => { + mocks.stop(); + }, + abort: (): void => { + mocks.abort(); + }, + }; +} + +export function createVoiceInputNativeHarness( + overrides: Partial = {} +): VoiceInputNativeHarness { + const controls: VoiceInputNativeControls = { + isAvailable: true, + supportsContinuous: true, + permissionGranted: true, + permissionCanAskAgain: true, + permissionRestricted: false, + ...overrides, + }; + + const permissionResponse = async (): Promise => { + const result = await Promise.resolve({ + granted: controls.permissionGranted, + canAskAgain: controls.permissionCanAskAgain, + restricted: controls.permissionRestricted, + }); + return result; + }; + + const mocks: VoiceInputNativeMocks = { + abort: vi.fn((): void => undefined), + getPermissions: vi.fn(async () => { + const result = await permissionResponse(); + return result; + }), + requestPermissions: vi.fn(async () => { + const result = await permissionResponse(); + return result; + }), + start: vi.fn((_options: VoiceInputNativeStartOptions): void => undefined), + stop: vi.fn((): void => undefined), + }; + + const listeners = new Map>(); + const native = buildNative(controls, mocks, listeners); + + const harness: VoiceInputNativeHarness = { + controls, + mocks, + emit(event, payload): void { + const set = listeners.get(event); + if (!set) { + return; + } + for (const listener of set) { + listener(payload); + } + }, + emitResult(transcript, isFinal): void { + harness.emit('result', { isFinal, results: [emptyResult(transcript)] }); + }, + listenerCount(event): number { + return listeners.get(event)?.size ?? 0; + }, + native, + }; + return harness; +} + +export function makeStartOptions( + overrides: Partial = {} +): VoiceInputStartOptions { + return { + baseDraft: '', + languageTag: 'en-US', + onDraftChange: (): void => undefined, + onFeedback: (): void => undefined, + owner: 'owner1', + ...overrides, + }; +} + +export function recordFeedback(): { + feedback: { + action: 'none' | 'open-settings'; + availability: 'available' | 'unavailable'; + message: string; + retryable: boolean; + }[]; + onFeedback: (fb: { + action: 'none' | 'open-settings'; + availability: 'available' | 'unavailable'; + message: string; + retryable: boolean; + }) => void; +} { + const feedback: { + action: 'none' | 'open-settings'; + availability: 'available' | 'unavailable'; + message: string; + retryable: boolean; + }[] = []; + return { + feedback, + onFeedback: (fb): void => { + feedback.push(fb); + }, + }; +} + +export function deferredPermission(): { + promise: Promise; + resolve: (value: VoiceInputNativePermission) => void; +} { + const { promise, resolve } = Promise.withResolvers(); + return { promise, resolve }; +} diff --git a/apps/mobile/src/lib/voice-input/voice-input-controller.ts b/apps/mobile/src/lib/voice-input/voice-input-controller.ts new file mode 100644 index 0000000000..6319d46f5c --- /dev/null +++ b/apps/mobile/src/lib/voice-input/voice-input-controller.ts @@ -0,0 +1,310 @@ +import { + type ExpoSpeechRecognitionErrorEvent, + type ExpoSpeechRecognitionResultEvent, +} from 'expo-speech-recognition'; + +import { + classifyVoiceInputError, + createVoiceTranscriptState, + type VoiceInputAvailability, + type VoiceInputFeedback, + type VoiceInputStatus, +} from './voice-input-state'; +import { + installVoiceInputListeners, + type VoiceInputSession, + type VoiceInputSessionController, +} from './voice-input-listeners'; +import { + acquirePermission, + createTerminal, + isDisposed, + type Lifecycle, + waitForTerminal, +} from './voice-input-controller-helpers'; + +export type VoiceInputNativeEvent = { + start: null; + result: ExpoSpeechRecognitionResultEvent; + nomatch: null; + error: ExpoSpeechRecognitionErrorEvent; + end: null; +}; + +export type VoiceInputNativePermission = { + granted: boolean; + canAskAgain: boolean; + restricted?: boolean; +}; + +export type VoiceInputNativeStartOptions = { + continuous: boolean; + interimResults: true; + lang: string; + maxAlternatives: 1; +}; + +export type VoiceInputNative = { + addListener( + event: K, + listener: (event: VoiceInputNativeEvent[K]) => void + ): { remove(): void }; + getPermissions(): Promise; + requestPermissions(): Promise; + isRecognitionAvailable(): boolean; + supportsContinuousRecognition(): boolean; + start(options: VoiceInputNativeStartOptions): void; + stop(): void; + abort(): void; +}; + +export type VoiceInputControllerSnapshot = { + availability: VoiceInputAvailability; + owner: string | null; + status: VoiceInputStatus; +}; + +export type VoiceInputStartOptions = { + baseDraft: string; + languageTag: string; + onDraftChange: (nextDraft: string) => void; + onFeedback: (feedback: VoiceInputFeedback) => void; + owner: string; +}; + +type VoiceInputController = { + abort(owner?: string): Promise; + dispose(): void; + getSnapshot(): VoiceInputControllerSnapshot; + start(options: VoiceInputStartOptions): Promise; + stop(owner: string): Promise; + subscribe(listener: (snapshot: VoiceInputControllerSnapshot) => void): () => void; +}; + +type Subscription = { remove(): void }; + +export function createVoiceInputController(native: VoiceInputNative): VoiceInputController { + let availability: VoiceInputAvailability = native.isRecognitionAvailable() + ? 'available' + : 'unavailable'; + let owner: string | null = null; + let status: VoiceInputStatus = 'idle'; + const subscribers = new Set<(snapshot: VoiceInputControllerSnapshot) => void>(); + let session: VoiceInputSession | null = null; + const lifecycle: Lifecycle = { disposed: false }; + + const notify = (): void => { + const snapshot: VoiceInputControllerSnapshot = { availability, owner, status }; + for (const subscriber of subscribers) { + subscriber(snapshot); + } + }; + + const reportFeedback = ( + feedback: VoiceInputFeedback, + onFeedback: (feedback: VoiceInputFeedback) => void + ): void => { + if (feedback.availability === 'unavailable' && availability !== 'unavailable') { + availability = 'unavailable'; + notify(); + } + onFeedback(feedback); + }; + + const reportClientAndTerminalize = (current: VoiceInputSession): void => { + current.failed = true; + reportFeedback(classifyVoiceInputError('client'), current.onFeedback); + terminalize(true); + }; + + const terminalize = (failed: boolean): void => { + const current = session; + if (!current || current.terminalized) { + return; + } + current.terminalized = true; + const ok = !failed; + current.terminalResolve(ok); + session = null; + owner = null; + status = 'idle'; + notify(); + }; + + const sessionController: VoiceInputSessionController = { + getSession: () => session, + notify, + reportFeedback, + setStatus: next => { + status = next; + }, + terminalize, + }; + + const subscriptions: Subscription[] = installVoiceInputListeners(native, sessionController); + + const abortActive = (current: VoiceInputSession): void => { + current.expectedAbort = true; + status = 'stopping'; + notify(); + try { + native.abort(); + } catch { + reportClientAndTerminalize(current); + } + }; + + const serializeOwnership = async (): Promise => { + const previous = session; + if (!previous) { + return true; + } + abortActive(previous); + await waitForTerminal(previous); + return !isDisposed(lifecycle); + }; + + const start = async (options: VoiceInputStartOptions): Promise => { + if (isDisposed(lifecycle)) { + return false; + } + if (!native.isRecognitionAvailable()) { + reportFeedback(classifyVoiceInputError('service-not-allowed'), options.onFeedback); + return false; + } + if (!(await serializeOwnership())) { + return false; + } + const permission = await acquirePermission(native); + if (isDisposed(lifecycle)) { + return false; + } + if (permission.kind === 'client-error') { + reportFeedback(classifyVoiceInputError('client'), options.onFeedback); + return false; + } + if (permission.kind === 'feedback') { + reportFeedback(permission.feedback, options.onFeedback); + return false; + } + if (isDisposed(lifecycle)) { + return false; + } + const terminal = createTerminal(); + const next: VoiceInputSession = { + baseDraft: options.baseDraft, + expectedAbort: false, + failed: false, + onDraftChange: options.onDraftChange, + onFeedback: options.onFeedback, + owner: options.owner, + terminalPromise: terminal.promise, + terminalResolve: terminal.resolve, + terminalized: false, + transcriptState: createVoiceTranscriptState(), + }; + session = next; + owner = options.owner; + status = 'starting'; + notify(); + + try { + native.start({ + continuous: native.supportsContinuousRecognition(), + interimResults: true, + lang: options.languageTag, + maxAlternatives: 1, + }); + } catch { + reportClientAndTerminalize(next); + return false; + } + + return true; + }; + + const stop = async (ownerArg: string): Promise => { + const current = session; + if (!current || current.owner !== ownerArg) { + return true; + } + status = 'stopping'; + notify(); + try { + native.stop(); + } catch { + current.failed = true; + current.expectedAbort = true; + reportFeedback(classifyVoiceInputError('client'), current.onFeedback); + try { + native.abort(); + } catch { + // abort may also throw; terminalize below + } + terminalize(true); + return false; + } + const result = await current.terminalPromise; + return result; + }; + + const abort = async (ownerArg?: string): Promise => { + const current = session; + if (!current) { + return true; + } + if (ownerArg !== undefined && current.owner !== ownerArg) { + return true; + } + current.expectedAbort = true; + status = 'stopping'; + notify(); + try { + native.abort(); + } catch { + terminalize(true); + return false; + } + const result = await current.terminalPromise; + return result; + }; + + const dispose = (): void => { + if (isDisposed(lifecycle)) { + return; + } + lifecycle.disposed = true; + const current = session; + if (current) { + current.expectedAbort = true; + try { + native.abort(); + } catch { + // abort may throw; force terminalize below + } + } + if (current && !current.terminalized) { + terminalize(true); + } + for (const subscription of subscriptions) { + subscription.remove(); + } + subscribers.clear(); + }; + + const subscribe = (listener: (snapshot: VoiceInputControllerSnapshot) => void): (() => void) => { + subscribers.add(listener); + return () => { + subscribers.delete(listener); + }; + }; + + return { + abort, + dispose, + getSnapshot: () => ({ availability, owner, status }), + start, + stop, + subscribe, + }; +} diff --git a/apps/mobile/src/lib/voice-input/voice-input-listeners.ts b/apps/mobile/src/lib/voice-input/voice-input-listeners.ts new file mode 100644 index 0000000000..5c09c04c62 --- /dev/null +++ b/apps/mobile/src/lib/voice-input/voice-input-listeners.ts @@ -0,0 +1,109 @@ +import { + appendVoiceTranscript, + applyVoiceRecognitionResult, + classifyVoiceInputError, + type VoiceInputFeedback, + type VoiceInputStatus, + type VoiceTranscriptState, +} from './voice-input-state'; +import { type VoiceInputNative, type VoiceInputNativeEvent } from './voice-input-controller'; +export type VoiceInputSession = { + baseDraft: string; + expectedAbort: boolean; + failed: boolean; + onDraftChange: (nextDraft: string) => void; + onFeedback: (feedback: VoiceInputFeedback) => void; + owner: string; + terminalPromise: Promise; + terminalResolve: (ok: boolean) => void; + terminalized: boolean; + transcriptState: VoiceTranscriptState; +}; + +export type VoiceInputSessionController = { + getSession(): VoiceInputSession | null; + notify(): void; + reportFeedback( + feedback: VoiceInputFeedback, + onFeedback: (feedback: VoiceInputFeedback) => void + ): void; + setStatus(status: VoiceInputStatus): void; + terminalize(failed: boolean): void; +}; + +type Subscription = { remove(): void }; + +export function installVoiceInputListeners( + native: VoiceInputNative, + controller: VoiceInputSessionController +): Subscription[] { + const onStart: (event: VoiceInputNativeEvent['start']) => void = () => { + const current = controller.getSession(); + if (!current || current.terminalized || current.expectedAbort) { + return; + } + controller.setStatus('listening'); + controller.notify(); + }; + + const onResult: (event: VoiceInputNativeEvent['result']) => void = event => { + const current = controller.getSession(); + if (!current || current.terminalized || current.expectedAbort) { + return; + } + const first = event.results[0]; + if (!first) { + return; + } + const transcript = first.transcript; + if (transcript.length === 0) { + return; + } + current.transcriptState = applyVoiceRecognitionResult(current.transcriptState, { + isFinal: event.isFinal, + transcript, + }); + const nextDraft = appendVoiceTranscript(current.baseDraft, current.transcriptState.transcript); + current.onDraftChange(nextDraft); + }; + + const onNomatch: (event: VoiceInputNativeEvent['nomatch']) => void = () => { + const current = controller.getSession(); + if (!current || current.terminalized || current.expectedAbort || current.failed) { + return; + } + current.failed = true; + controller.reportFeedback(classifyVoiceInputError('no-speech'), current.onFeedback); + }; + + const onError: (event: VoiceInputNativeEvent['error']) => void = event => { + const current = controller.getSession(); + if (!current || current.terminalized) { + return; + } + if (current.expectedAbort && event.error === 'aborted') { + return; + } + if (current.failed) { + return; + } + current.failed = true; + controller.reportFeedback(classifyVoiceInputError(event.error), current.onFeedback); + }; + + const onEnd: (event: VoiceInputNativeEvent['end']) => void = () => { + const current = controller.getSession(); + if (!current || current.terminalized) { + return; + } + controller.terminalize(current.failed); + }; + + return [ + native.addListener('start', onStart), + native.addListener('result', onResult), + native.addListener('nomatch', onNomatch), + native.addListener('error', onError), + native.addListener('end', onEnd), + ]; +} From a88c289a3b32ce5600508b23ef0352eaeb60a46a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20=C5=A0=C4=87eki=C4=87?= Date: Tue, 14 Jul 2026 15:12:46 +0200 Subject: [PATCH 05/14] feat(mobile): add reusable voice input controls --- .../agents/markdown-palette.test.ts | 1 + .../src/components/voice-input-control.tsx | 87 +++++++++++++ apps/mobile/src/lib/hooks/use-theme-colors.ts | 2 + .../lib/voice-input/voice-input-draft.test.ts | 120 +++++++++++++++++ .../src/lib/voice-input/voice-input-draft.ts | 39 ++++++ .../voice-input-view-state.test.ts | 122 ++++++++++++++++++ .../lib/voice-input/voice-input-view-state.ts | 101 +++++++++++++++ 7 files changed, 472 insertions(+) create mode 100644 apps/mobile/src/components/voice-input-control.tsx create mode 100644 apps/mobile/src/lib/voice-input/voice-input-draft.test.ts create mode 100644 apps/mobile/src/lib/voice-input/voice-input-draft.ts create mode 100644 apps/mobile/src/lib/voice-input/voice-input-view-state.test.ts create mode 100644 apps/mobile/src/lib/voice-input/voice-input-view-state.ts diff --git a/apps/mobile/src/components/agents/markdown-palette.test.ts b/apps/mobile/src/components/agents/markdown-palette.test.ts index ae9d116bee..bbd24fb1f8 100644 --- a/apps/mobile/src/components/agents/markdown-palette.test.ts +++ b/apps/mobile/src/components/agents/markdown-palette.test.ts @@ -13,6 +13,7 @@ const colors = { muted: '#F0EEE6', mutedForeground: '#7A756B', destructive: '#C25647', + destructiveForeground: '#FFFFFF', border: 'rgba(20, 15, 10, 0.09)', card: '#FFFFFF', ink2: '#3C382F', diff --git a/apps/mobile/src/components/voice-input-control.tsx b/apps/mobile/src/components/voice-input-control.tsx new file mode 100644 index 0000000000..0f95068fe7 --- /dev/null +++ b/apps/mobile/src/components/voice-input-control.tsx @@ -0,0 +1,87 @@ +import { Mic, Square } from 'lucide-react-native'; +import { ActivityIndicator, Pressable, Text } from 'react-native'; + +import { useThemeColors } from '@/lib/hooks/use-theme-colors'; +import { cn } from '@/lib/utils'; +import { type VoiceInputStatus } from '@/lib/voice-input/voice-input-state'; +import { resolveVoiceInputControlState } from '@/lib/voice-input/voice-input-view-state'; + +export type VoiceInputButtonProps = { + disabled: boolean; + onPress(): void; + status: VoiceInputStatus; +}; + +const ICON_SIZE = 18; +const HIT_SLOP = 2; +const LISTENING_BG = 'bg-red-600 dark:bg-red-500'; +const RESTING_BG = 'bg-secondary'; + +/** + * Compact voice-input toggle for composer toolbars. Renders a 10x10 visual + * square that expands to a 44pt touch target via `hitSlop`. While the + * controller is in `starting` or `stopping`, an ActivityIndicator replaces + * the icon and the press is ignored. While `listening`, a destructive + * treatment signals the active session. + */ +export function VoiceInputButton({ + disabled, + onPress, + status, +}: Readonly): React.ReactElement { + const colors = useThemeColors(); + const control = resolveVoiceInputControlState(status, disabled); + const isListeningOrStopping = status === 'listening' || status === 'stopping'; + const showSpinner = control.busy; + const iconColor = isListeningOrStopping ? colors.destructiveForeground : colors.foreground; + const restingBg = isListeningOrStopping ? LISTENING_BG : RESTING_BG; + const containerClass = cn( + 'h-10 w-10 items-center justify-center rounded-md active:opacity-70', + restingBg + ); + const renderIcon = (): React.ReactElement => { + if (showSpinner) { + return ; + } + if (control.icon === 'microphone') { + return ; + } + return ; + }; + + return ( + + {renderIcon()} + + ); +} + +export type VoiceInputStatusProps = { + status: VoiceInputStatus; +}; + +/** + * Visible "Listening..." caption rendered under the composer while the + * voice session is live. Returns `null` in every other status to keep the + * composer's footprint unchanged outside of an active session. + */ +export function VoiceInputStatus({ + status, +}: Readonly): React.ReactElement | null { + if (status !== 'listening') { + return null; + } + return ( + + Listening... + + ); +} diff --git a/apps/mobile/src/lib/hooks/use-theme-colors.ts b/apps/mobile/src/lib/hooks/use-theme-colors.ts index 5d9ae9d8f6..4c5c4a0900 100644 --- a/apps/mobile/src/lib/hooks/use-theme-colors.ts +++ b/apps/mobile/src/lib/hooks/use-theme-colors.ts @@ -13,6 +13,7 @@ const lightColors = { muted: '#F0EEE6', mutedForeground: '#6F6A61', destructive: '#BE4E3F', + destructiveForeground: '#FFFFFF', border: 'rgba(20, 15, 10, 0.09)', card: '#FFFFFF', @@ -45,6 +46,7 @@ const darkColors = { muted: '#1F1F24', mutedForeground: '#8A8680', destructive: '#F28B7A', + destructiveForeground: '#1A1A10', border: 'rgba(255, 255, 255, 0.07)', card: '#17171A', diff --git a/apps/mobile/src/lib/voice-input/voice-input-draft.test.ts b/apps/mobile/src/lib/voice-input/voice-input-draft.test.ts new file mode 100644 index 0000000000..46f3a22c04 --- /dev/null +++ b/apps/mobile/src/lib/voice-input/voice-input-draft.test.ts @@ -0,0 +1,120 @@ +import { describe, expect, it, vi } from 'vitest'; + +import { applyVoiceDraftToInput } from './voice-input-draft'; + +type NativeCall = { text: string }; +type ChangeCall = string; + +function makeInput() { + const calls: NativeCall[] = []; + const input = { + setNativeProps(props: { text: string }): void { + calls.push({ text: props.text }); + }, + }; + return { input, calls }; +} + +function makeChange() { + const calls: ChangeCall[] = []; + const onChangeText = (draft: string): void => { + calls.push(draft); + }; + return { onChangeText, calls }; +} + +describe('applyVoiceDraftToInput', () => { + it('calls input.setNativeProps before onChangeText with the same draft', () => { + const { input, calls: nativeCalls } = makeInput(); + const { onChangeText, calls: changeCalls } = makeChange(); + const order: string[] = []; + const wrappedInput = { + setNativeProps(props: { text: string }): void { + order.push('native'); + input.setNativeProps(props); + }, + }; + const wrappedChange = (draft: string): void => { + order.push('change'); + onChangeText(draft); + }; + + applyVoiceDraftToInput({ + draft: 'hello world', + input: wrappedInput, + onChangeText: wrappedChange, + }); + + expect(order).toEqual(['native', 'change']); + expect(nativeCalls).toEqual([{ text: 'hello world' }]); + expect(changeCalls).toEqual(['hello world']); + }); + + it('still invokes the change path when the input ref is null', () => { + const { onChangeText, calls } = makeChange(); + + applyVoiceDraftToInput({ draft: 'hello', input: null, onChangeText }); + + expect(calls).toEqual(['hello']); + }); + + it('does not throw and does not call setNativeProps when the input ref is null', () => { + const { onChangeText } = makeChange(); + + expect(() => { + applyVoiceDraftToInput({ draft: 'hello', input: null, onChangeText }); + }).not.toThrow(); + }); + + it('truncates both the native prop and the change callback to the same capped value when maxLength is provided', () => { + const { input, calls: nativeCalls } = makeInput(); + const { onChangeText, calls: changeCalls } = makeChange(); + + applyVoiceDraftToInput({ draft: 'abcdefghij', input, maxLength: 4, onChangeText }); + + expect(nativeCalls).toEqual([{ text: 'abcd' }]); + expect(changeCalls).toEqual(['abcd']); + }); + + it('does not truncate when maxLength is omitted, even for a long draft', () => { + const { input, calls: nativeCalls } = makeInput(); + const { onChangeText, calls: changeCalls } = makeChange(); + const longDraft = 'a'.repeat(200); + + applyVoiceDraftToInput({ draft: longDraft, input, onChangeText }); + + expect(nativeCalls).toEqual([{ text: longDraft }]); + expect(changeCalls).toEqual([longDraft]); + }); + + it('does not truncate when maxLength is larger than the draft', () => { + const { input, calls: nativeCalls } = makeInput(); + const { onChangeText, calls: changeCalls } = makeChange(); + + applyVoiceDraftToInput({ draft: 'hi', input, maxLength: 50, onChangeText }); + + expect(nativeCalls).toEqual([{ text: 'hi' }]); + expect(changeCalls).toEqual(['hi']); + }); + + it('normalizes a negative maxLength to an empty value for both native prop and callback', () => { + const { input, calls: nativeCalls } = makeInput(); + const { onChangeText, calls: changeCalls } = makeChange(); + + applyVoiceDraftToInput({ draft: 'hello', input, maxLength: -3, onChangeText }); + + expect(nativeCalls).toEqual([{ text: '' }]); + expect(changeCalls).toEqual(['']); + }); + + it('invokes the change callback exactly once per call', () => { + const onChangeText = vi.fn<(draft: string) => void>(); + const input = { + setNativeProps: vi.fn(), + }; + + applyVoiceDraftToInput({ draft: 'hello', input, onChangeText }); + + expect(onChangeText).toHaveBeenCalledTimes(1); + }); +}); diff --git a/apps/mobile/src/lib/voice-input/voice-input-draft.ts b/apps/mobile/src/lib/voice-input/voice-input-draft.ts new file mode 100644 index 0000000000..062f1fa6ee --- /dev/null +++ b/apps/mobile/src/lib/voice-input/voice-input-draft.ts @@ -0,0 +1,39 @@ +type VoiceInputTextInput = { + setNativeProps(props: { text: string }): void; +}; + +type ApplyVoiceDraftOptions = { + draft: string; + input: VoiceInputTextInput | null; + maxLength?: number; + onChangeText: (draft: string) => void; +}; + +function resolveDraftCapping(draft: string, maxLength: number | undefined): string { + if (maxLength === undefined) { + return draft; + } + const cap = Math.max(0, Math.floor(maxLength)); + return draft.slice(0, cap); +} + +/** + * Bridges an external voice-draft string into a controlled text input and a + * sibling onChangeText callback. The native prop is updated before the change + * callback fires so that the input is in sync by the time listeners observe + * the new draft. When `input` is null, only the change path is taken (the + * composer still learns the draft). When `maxLength` is provided the draft + * is truncated to that many characters; negative values are normalized to + * zero so neither the native prop nor the callback receives a negative + * slice that would strip the tail of the draft. + */ +export function applyVoiceDraftToInput({ + draft, + input, + maxLength, + onChangeText, +}: ApplyVoiceDraftOptions): void { + const next = resolveDraftCapping(draft, maxLength); + input?.setNativeProps({ text: next }); + onChangeText(next); +} diff --git a/apps/mobile/src/lib/voice-input/voice-input-view-state.test.ts b/apps/mobile/src/lib/voice-input/voice-input-view-state.test.ts new file mode 100644 index 0000000000..4be5736272 --- /dev/null +++ b/apps/mobile/src/lib/voice-input/voice-input-view-state.test.ts @@ -0,0 +1,122 @@ +import { describe, expect, it } from 'vitest'; + +import { + resolveOwnerVoiceInputView, + resolveVoiceInputControlState, + type VoiceInputControlState, +} from './voice-input-view-state'; +import { type VoiceInputControllerSnapshot } from './voice-input-controller'; + +const AVAILABLE: VoiceInputControllerSnapshot = { + availability: 'available', + owner: 'composer-A', + status: 'idle', +}; + +const UNAVAILABLE: VoiceInputControllerSnapshot = { + availability: 'unavailable', + owner: null, + status: 'idle', +}; + +describe('resolveOwnerVoiceInputView', () => { + it('marks the owner active and preserves the starting status', () => { + const view = resolveOwnerVoiceInputView({ ...AVAILABLE, status: 'starting' }, 'composer-A'); + expect(view).toEqual({ available: true, isActive: true, status: 'starting' }); + }); + + it('marks the owner active and preserves the listening status', () => { + const view = resolveOwnerVoiceInputView({ ...AVAILABLE, status: 'listening' }, 'composer-A'); + expect(view).toEqual({ available: true, isActive: true, status: 'listening' }); + }); + + it('marks the owner active and preserves the stopping status', () => { + const view = resolveOwnerVoiceInputView({ ...AVAILABLE, status: 'stopping' }, 'composer-A'); + expect(view).toEqual({ available: true, isActive: true, status: 'stopping' }); + }); + + it('returns inactive idle for an owner that does not match', () => { + const view = resolveOwnerVoiceInputView({ ...AVAILABLE, status: 'listening' }, 'composer-B'); + expect(view).toEqual({ available: true, isActive: false, status: 'idle' }); + }); + + it('returns inactive idle when the snapshot owner is null', () => { + const view = resolveOwnerVoiceInputView( + { ...AVAILABLE, owner: null, status: 'listening' }, + 'composer-A' + ); + expect(view).toEqual({ available: true, isActive: false, status: 'idle' }); + }); + + it('returns inactive idle when the snapshot owner is null and the controller is unavailable', () => { + const view = resolveOwnerVoiceInputView(UNAVAILABLE, 'composer-A'); + expect(view).toEqual({ available: false, isActive: false, status: 'idle' }); + }); + + it('reports availability unavailable while still respecting ownership rules', () => { + const view = resolveOwnerVoiceInputView({ ...UNAVAILABLE, status: 'starting' }, 'composer-A'); + expect(view).toEqual({ available: false, isActive: false, status: 'idle' }); + }); +}); + +describe('resolveVoiceInputControlState', () => { + it('idle + not disabled => Start label, microphone icon, not busy, enabled', () => { + expect(resolveVoiceInputControlState('idle', false)).toEqual({ + accessibilityLabel: 'Start voice input', + busy: false, + disabled: false, + icon: 'microphone', + showListeningStatus: false, + }); + }); + + it('idle + external disabled => Start label, microphone icon, not busy, disabled', () => { + expect(resolveVoiceInputControlState('idle', true)).toEqual({ + accessibilityLabel: 'Start voice input', + busy: false, + disabled: true, + icon: 'microphone', + showListeningStatus: false, + }); + }); + + it('starting => busy + disabled, Start label, microphone icon, no listening status', () => { + expect(resolveVoiceInputControlState('starting', false)).toEqual({ + accessibilityLabel: 'Start voice input', + busy: true, + disabled: true, + icon: 'microphone', + showListeningStatus: false, + }); + }); + + it('stopping => busy + disabled, Stop label, stop icon, no listening status', () => { + expect(resolveVoiceInputControlState('stopping', false)).toEqual({ + accessibilityLabel: 'Stop voice input', + busy: true, + disabled: true, + icon: 'stop', + showListeningStatus: false, + }); + }); + + it('listening + not disabled => Stop label, stop icon, not busy, enabled, listening status shown', () => { + expect(resolveVoiceInputControlState('listening', false)).toEqual({ + accessibilityLabel: 'Stop voice input', + busy: false, + disabled: false, + icon: 'stop', + showListeningStatus: true, + }); + }); + + it('listening + external disabled => Stop label, stop icon, not busy, disabled, listening status shown', () => { + expect(resolveVoiceInputControlState('listening', true)).toEqual({ + accessibilityLabel: 'Stop voice input', + busy: false, + disabled: true, + icon: 'stop', + showListeningStatus: true, + }); + }); +}); diff --git a/apps/mobile/src/lib/voice-input/voice-input-view-state.ts b/apps/mobile/src/lib/voice-input/voice-input-view-state.ts new file mode 100644 index 0000000000..d6bf0cb158 --- /dev/null +++ b/apps/mobile/src/lib/voice-input/voice-input-view-state.ts @@ -0,0 +1,101 @@ +import { type VoiceInputControllerSnapshot } from './voice-input-controller'; +import { type VoiceInputStatus } from './voice-input-state'; + +/** + * The per-owner projection of the shared voice-input controller snapshot. + * Owners (e.g. specific composer instances) use this to decide whether the + * voice session currently belongs to them and which status to render. + */ +type OwnerVoiceInputView = { + available: boolean; + isActive: boolean; + status: VoiceInputStatus; +}; + +/** + * Pure projection of a shared `VoiceInputControllerSnapshot` for a single + * owner. The owner is active only when the snapshot's `owner` matches the + * caller's `owner`; otherwise the view collapses to an idle, inactive state + * while still reporting the controller-wide availability. This keeps each + * composer's voice button honest about whether *its* session is in progress, + * even if a different owner has taken over the controller. + */ +export function resolveOwnerVoiceInputView( + snapshot: VoiceInputControllerSnapshot, + owner: string +): OwnerVoiceInputView { + const isActive = snapshot.owner === owner; + return { + available: snapshot.availability === 'available', + isActive, + status: isActive ? snapshot.status : 'idle', + }; +} + +export type VoiceInputControlIcon = 'microphone' | 'stop'; +export type VoiceInputAccessibilityLabel = 'Start voice input' | 'Stop voice input'; + +export type VoiceInputControlState = { + accessibilityLabel: VoiceInputAccessibilityLabel; + busy: boolean; + disabled: boolean; + icon: VoiceInputControlIcon; + showListeningStatus: boolean; +}; + +/** + * Pure derivation of the voice-input control's presentation from the + * controller's `status` and an external `disabled` flag (e.g. composer is + * read-only). The `busy` flag follows the starting/stopping transitions so + * a press during teardown is ignored; the `disabled` flag is the union of + * that busy state and the external disabled flag. While listening, the + * button shows a stop affordance and surfaces the "Listening..." status + * string for screen readers and the visible row beneath the composer. + */ +export function resolveVoiceInputControlState( + status: VoiceInputStatus, + disabled: boolean +): VoiceInputControlState { + switch (status) { + case 'starting': { + return { + accessibilityLabel: 'Start voice input', + busy: true, + disabled: true, + icon: 'microphone', + showListeningStatus: false, + }; + } + case 'listening': { + return { + accessibilityLabel: 'Stop voice input', + busy: false, + disabled, + icon: 'stop', + showListeningStatus: true, + }; + } + case 'stopping': { + return { + accessibilityLabel: 'Stop voice input', + busy: true, + disabled: true, + icon: 'stop', + showListeningStatus: false, + }; + } + case 'idle': { + return { + accessibilityLabel: 'Start voice input', + busy: false, + disabled, + icon: 'microphone', + showListeningStatus: false, + }; + } + default: { + const unhandled: never = status; + throw new Error(`Unhandled VoiceInputStatus: ${String(unhandled)}`); + } + } +} From 64cc49c18539ef98e4bdaa62cd1859cd9bea5744 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20=C5=A0=C4=87eki=C4=87?= Date: Tue, 14 Jul 2026 15:35:34 +0200 Subject: [PATCH 06/14] feat(mobile): add voice input lifecycle hook --- .../voice-input/use-voice-input-actions.ts | 136 +++++++++ .../use-voice-input-feedback.test.ts | 72 +++++ .../lib/voice-input/use-voice-input.test.ts | 272 ++++++++++++++++++ .../src/lib/voice-input/use-voice-input.ts | 147 ++++++++++ .../voice-input/voice-input-feedback.test.ts | 100 +++++++ .../lib/voice-input/voice-input-feedback.ts | 42 +++ .../voice-input/voice-input-language.test.ts | 27 ++ .../lib/voice-input/voice-input-language.ts | 21 ++ .../src/lib/voice-input/voice-input-state.ts | 7 +- 9 files changed, 820 insertions(+), 4 deletions(-) create mode 100644 apps/mobile/src/lib/voice-input/use-voice-input-actions.ts create mode 100644 apps/mobile/src/lib/voice-input/use-voice-input-feedback.test.ts create mode 100644 apps/mobile/src/lib/voice-input/use-voice-input.test.ts create mode 100644 apps/mobile/src/lib/voice-input/use-voice-input.ts create mode 100644 apps/mobile/src/lib/voice-input/voice-input-feedback.test.ts create mode 100644 apps/mobile/src/lib/voice-input/voice-input-feedback.ts create mode 100644 apps/mobile/src/lib/voice-input/voice-input-language.test.ts create mode 100644 apps/mobile/src/lib/voice-input/voice-input-language.ts diff --git a/apps/mobile/src/lib/voice-input/use-voice-input-actions.ts b/apps/mobile/src/lib/voice-input/use-voice-input-actions.ts new file mode 100644 index 0000000000..0f23fd4c2d --- /dev/null +++ b/apps/mobile/src/lib/voice-input/use-voice-input-actions.ts @@ -0,0 +1,136 @@ +import { AccessibilityInfo, Alert, Linking } from 'react-native'; +import * as Haptics from 'expo-haptics'; +import { getLocales } from 'expo-localization'; +import { toast } from 'sonner-native'; + +import { + type VoiceInputControllerSnapshot, + type VoiceInputStartOptions, +} from './voice-input-controller'; +import { + resolveVoiceInputFeedbackPresentation, + shouldAnnounceListeningTransition, +} from './voice-input-feedback'; +import { resolveVoiceInputLanguageTag } from './voice-input-language'; +import { + shouldAbortVoiceInput, + type VoiceInputFeedback, + type VoiceInputLifecycleInput, + type VoiceInputStatus, +} from './voice-input-state'; +import { resolveOwnerVoiceInputView } from './voice-input-view-state'; + +type VoiceInputControllerLike = { + abort: (owner?: string) => Promise; + getSnapshot: () => VoiceInputControllerSnapshot; + start: (options: VoiceInputStartOptions) => Promise; + stop: (owner: string) => Promise; + subscribe: (listener: (snapshot: VoiceInputControllerSnapshot) => void) => () => void; +}; + +export type VoiceInputActions = { + abort: () => Promise; + settleBeforeSubmit: () => Promise; + toggle: () => Promise; +}; + +type VoiceInputActionsConfig = { + controller: VoiceInputControllerLike; + getDisabled: () => boolean; + getDraft: () => string; + getOnDraftChange: () => (draft: string) => void; + getOwner: () => string; +}; + +async function fireHaptic(style: Haptics.ImpactFeedbackStyle): Promise { + try { + await Haptics.impactAsync(style); + } catch { + // Haptic feedback is best-effort; never surface failures to the user. + } +} + +function announceVoiceInputListening(): void { + void fireHaptic(Haptics.ImpactFeedbackStyle.Light); + AccessibilityInfo.announceForAccessibility('Listening'); +} + +export function runVoiceInputListeningFeedback( + previousOwnStatus: VoiceInputStatus | null, + nextOwnStatus: VoiceInputStatus +): void { + if (shouldAnnounceListeningTransition(previousOwnStatus, nextOwnStatus)) { + announceVoiceInputListening(); + } +} + +export function showFeedback(feedback: VoiceInputFeedback): void { + const presentation = resolveVoiceInputFeedbackPresentation(feedback); + if (presentation.kind === 'alert') { + Alert.alert(presentation.title, presentation.message, [ + { text: 'Cancel', style: 'cancel' }, + { text: 'Open Settings', onPress: () => void Linking.openSettings() }, + ]); + return; + } + toast.error(presentation.message); +} + +export function shouldAbortVoiceInputForOwner( + snapshot: VoiceInputControllerSnapshot, + owner: string, + input: VoiceInputLifecycleInput +): boolean { + const view = resolveOwnerVoiceInputView(snapshot, owner); + return view.isActive && shouldAbortVoiceInput(input); +} + +export function createVoiceInputActions(config: VoiceInputActionsConfig): VoiceInputActions { + const { controller, getDisabled, getDraft, getOnDraftChange, getOwner } = config; + + const abort = async (): Promise => { + const result = await controller.abort(getOwner()); + return result; + }; + + const settleBeforeSubmit = async (): Promise => { + const owner = getOwner(); + const snapshot = controller.getSnapshot(); + const view = resolveOwnerVoiceInputView(snapshot, owner); + if (!view.isActive) { + return true; + } + const result = await controller.stop(owner); + return result; + }; + + const toggle = async (): Promise => { + if (getDisabled()) { + return; + } + const owner = getOwner(); + const snapshot = controller.getSnapshot(); + const view = resolveOwnerVoiceInputView(snapshot, owner); + + if (view.isActive && snapshot.status === 'listening') { + void fireHaptic(Haptics.ImpactFeedbackStyle.Medium); + await controller.stop(owner); + return; + } + + if (view.isActive) { + return; + } + + const startOptions: VoiceInputStartOptions = { + baseDraft: getDraft(), + languageTag: resolveVoiceInputLanguageTag(getLocales()), + onDraftChange: getOnDraftChange(), + onFeedback: showFeedback, + owner, + }; + await controller.start(startOptions); + }; + + return { abort, settleBeforeSubmit, toggle }; +} diff --git a/apps/mobile/src/lib/voice-input/use-voice-input-feedback.test.ts b/apps/mobile/src/lib/voice-input/use-voice-input-feedback.test.ts new file mode 100644 index 0000000000..cea87e80be --- /dev/null +++ b/apps/mobile/src/lib/voice-input/use-voice-input-feedback.test.ts @@ -0,0 +1,72 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { runVoiceInputListeningFeedback, showFeedback } from './use-voice-input-actions'; + +const hapticsMock = vi.hoisted(() => ({ impactAsync: vi.fn().mockResolvedValue(undefined) })); +const accessibilityMock = vi.hoisted(() => ({ announceForAccessibility: vi.fn() })); +const alertMock = vi.hoisted(() => ({ alert: vi.fn() })); +const linkingMock = vi.hoisted(() => ({ openSettings: vi.fn() })); +const toastMock = vi.hoisted(() => ({ error: vi.fn() })); + +vi.mock('expo-haptics', () => ({ + ImpactFeedbackStyle: { Light: 'light', Medium: 'medium' }, + impactAsync: hapticsMock.impactAsync, +})); +vi.mock('expo-localization', () => ({ getLocales: () => [{ languageTag: 'en-US' }] })); +vi.mock('sonner-native', () => ({ toast: toastMock })); +vi.mock('react-native', () => ({ + AccessibilityInfo: accessibilityMock, + Alert: alertMock, + Linking: linkingMock, +})); + +describe('voice input feedback side effects', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('presents settings feedback as an alert with a working settings action', () => { + showFeedback({ + action: 'open-settings', + availability: 'available', + message: 'Microphone access is off. Enable it in Settings.', + retryable: false, + }); + + const buttons = alertMock.alert.mock.calls[0]?.[2] as + | { text: string; onPress?: () => void }[] + | undefined; + expect(alertMock.alert).toHaveBeenCalledWith( + 'Microphone access is off', + 'Microphone access is off. Enable it in Settings.', + [{ text: 'Cancel', style: 'cancel' }, { text: 'Open Settings', onPress: expect.any(Function) }] + ); + buttons?.find(button => button.text === 'Open Settings')?.onPress?.(); + expect(linkingMock.openSettings).toHaveBeenCalled(); + }); + + it('presents feedback without an action as a toast', () => { + showFeedback({ + action: 'none', + availability: 'available', + message: 'No speech detected. Tap the microphone to try again.', + retryable: true, + }); + + expect(alertMock.alert).not.toHaveBeenCalled(); + expect(toastMock.error).toHaveBeenCalledWith( + 'No speech detected. Tap the microphone to try again.' + ); + }); + + it('announces and haptics only when entering listening', () => { + runVoiceInputListeningFeedback('idle', 'listening'); + runVoiceInputListeningFeedback('listening', 'listening'); + runVoiceInputListeningFeedback('listening', 'idle'); + + expect(hapticsMock.impactAsync).toHaveBeenCalledTimes(1); + expect(hapticsMock.impactAsync).toHaveBeenCalledWith('light'); + expect(accessibilityMock.announceForAccessibility).toHaveBeenCalledOnce(); + expect(accessibilityMock.announceForAccessibility).toHaveBeenCalledWith('Listening'); + }); +}); diff --git a/apps/mobile/src/lib/voice-input/use-voice-input.test.ts b/apps/mobile/src/lib/voice-input/use-voice-input.test.ts new file mode 100644 index 0000000000..b5599916b5 --- /dev/null +++ b/apps/mobile/src/lib/voice-input/use-voice-input.test.ts @@ -0,0 +1,272 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { + type VoiceInputControllerSnapshot, + type VoiceInputStartOptions, +} from './voice-input-controller'; +import { + createVoiceInputActions, + shouldAbortVoiceInputForOwner, + showFeedback, +} from './use-voice-input-actions'; + +const hapticsMock = vi.hoisted(() => ({ + impactAsync: vi.fn<() => Promise>().mockResolvedValue(undefined), +})); + +const accessibilityMock = vi.hoisted(() => ({ + announceForAccessibility: vi.fn(), +})); + +const alertMock = vi.hoisted(() => ({ + alert: vi.fn(), +})); + +const linkingMock = vi.hoisted(() => ({ + openSettings: vi.fn(), +})); + +const toastMock = vi.hoisted(() => ({ + error: vi.fn(), +})); + +const localizationMock = vi.hoisted(() => ({ + getLocales: vi.fn<() => { languageTag: string }[]>(() => [{ languageTag: 'en-US' }]), +})); + +vi.mock('expo-haptics', () => ({ + ImpactFeedbackStyle: { Light: 'light', Medium: 'medium' }, + impactAsync: hapticsMock.impactAsync, +})); + +vi.mock('expo-localization', () => ({ + getLocales: localizationMock.getLocales, +})); + +vi.mock('sonner-native', () => ({ + toast: toastMock, +})); + +vi.mock('react-native', () => ({ + AccessibilityInfo: accessibilityMock, + Alert: alertMock, + AppState: { addEventListener: vi.fn() }, + Linking: linkingMock, + Platform: { OS: 'ios' }, +})); + +const mockController = vi.hoisted(() => { + type Subscriber = (snapshot: VoiceInputControllerSnapshot) => void; + const subscribers = new Set(); + let snapshot: VoiceInputControllerSnapshot = { + availability: 'available', + owner: null, + status: 'idle', + }; + + return { + abort: vi.fn<(owner?: string) => Promise>().mockResolvedValue(true), + getSnapshot: vi.fn<() => VoiceInputControllerSnapshot>(() => snapshot), + setSnapshot(next: VoiceInputControllerSnapshot): void { + snapshot = next; + for (const subscriber of subscribers) { + subscriber(next); + } + }, + start: vi.fn<(options: VoiceInputStartOptions) => Promise>().mockResolvedValue(true), + stop: vi.fn<(owner: string) => Promise>().mockResolvedValue(true), + subscribe: vi.fn<(listener: Subscriber) => () => void>(listener => { + subscribers.add(listener); + return () => { + subscribers.delete(listener); + }; + }), + }; +}); + +vi.mock('./native-voice-input', () => ({ + voiceInputController: mockController, +})); + +type ActionHarness = { + actions: { + abort: () => Promise; + settleBeforeSubmit: () => Promise; + toggle: () => Promise; + }; + disabled: ReturnType; + draft: ReturnType; + onDraftChange: ReturnType; + owner: string; +}; + +function buildActions( + overrides: { disabled?: boolean; draft?: string; owner?: string } = {} +): ActionHarness { + const owner = overrides.owner ?? 'owner-A'; + const draft = vi.fn<() => string>(() => overrides.draft ?? 'draft text'); + const onDraftChange = vi.fn<(nextDraft: string) => void>(); + const disabled = vi.fn<() => boolean>(() => overrides.disabled ?? false); + + const actions = createVoiceInputActions({ + controller: mockController, + getDisabled: disabled, + getDraft: draft, + getOnDraftChange: () => onDraftChange, + getOwner: () => owner, + }); + + return { actions, disabled, draft, onDraftChange, owner }; +} + +function idleSnapshot(): VoiceInputControllerSnapshot { + return { availability: 'available', owner: null, status: 'idle' }; +} + +function activeSnapshot( + owner: string, + status: VoiceInputControllerSnapshot['status'] +): VoiceInputControllerSnapshot { + return { availability: 'available', owner, status }; +} + +describe('useVoiceInput integration', () => { + beforeEach(() => { + vi.clearAllMocks(); + mockController.setSnapshot(idleSnapshot()); + localizationMock.getLocales.mockReturnValue([{ languageTag: 'en-US' }]); + }); + + afterEach(() => { + vi.clearAllMocks(); + }); + + describe('createVoiceInputActions', () => { + describe('settleBeforeSubmit', () => { + it('returns true without calling stop when this owner is not active', async () => { + const { actions } = buildActions(); + mockController.setSnapshot(idleSnapshot()); + + const result = await actions.settleBeforeSubmit(); + + expect(result).toBe(true); + expect(mockController.stop).not.toHaveBeenCalled(); + }); + + it('delegates stop(owner) and returns the controllers result when this owner is active', async () => { + const { actions, owner } = buildActions(); + mockController.setSnapshot(activeSnapshot(owner, 'listening')); + mockController.stop.mockResolvedValueOnce(true); + + const result = await actions.settleBeforeSubmit(); + + expect(mockController.stop).toHaveBeenCalledWith(owner); + expect(result).toBe(true); + }); + }); + + describe('toggle', () => { + it('starts when idle, using the latest draft and resolved language and passing draft/feedback callbacks', async () => { + const { actions, onDraftChange, owner } = buildActions({ draft: 'hello' }); + mockController.setSnapshot(idleSnapshot()); + localizationMock.getLocales.mockReturnValue([{ languageTag: 'nl-NL' }]); + + await actions.toggle(); + + expect(mockController.start).toHaveBeenCalledTimes(1); + const startOptions = mockController.start.mock.calls[0]?.[0]; + if (!startOptions) { + throw new Error('controller.start was not called'); + } + expect(startOptions.baseDraft).toBe('hello'); + expect(startOptions.languageTag).toBe('nl-NL'); + expect(startOptions.owner).toBe(owner); + expect(startOptions.onDraftChange).toBe(onDraftChange); + expect(startOptions.onFeedback).toBe(showFeedback); + }); + + it('emits a medium haptic and stops when already listening', async () => { + const { actions, owner } = buildActions(); + mockController.setSnapshot(activeSnapshot(owner, 'listening')); + + await actions.toggle(); + + expect(hapticsMock.impactAsync).toHaveBeenCalledWith('medium'); + expect(mockController.stop).toHaveBeenCalledWith(owner); + }); + + it('does nothing while starting or stopping', async () => { + const { actions, owner } = buildActions(); + mockController.setSnapshot(activeSnapshot(owner, 'starting')); + + await actions.toggle(); + + expect(hapticsMock.impactAsync).not.toHaveBeenCalled(); + expect(mockController.stop).not.toHaveBeenCalled(); + expect(mockController.start).not.toHaveBeenCalled(); + }); + + it('does nothing when disabled', async () => { + const { actions } = buildActions({ disabled: true }); + mockController.setSnapshot(idleSnapshot()); + + await actions.toggle(); + + expect(mockController.start).not.toHaveBeenCalled(); + }); + }); + + describe('abort', () => { + it('delegates abort(owner) to the controller', async () => { + const { actions, owner } = buildActions(); + + await actions.abort(); + + expect(mockController.abort).toHaveBeenCalledWith(owner); + }); + }); + }); + + describe('shouldAbortVoiceInputForOwner', () => { + it('returns true only when this owner is active and shouldAbortVoiceInput says true', () => { + const owner = 'owner-A'; + const otherOwner = 'owner-B'; + + expect( + shouldAbortVoiceInputForOwner(activeSnapshot(owner, 'listening'), owner, { + appState: 'active', + disabled: false, + }) + ).toBe(false); + + expect( + shouldAbortVoiceInputForOwner(activeSnapshot(owner, 'listening'), owner, { + appState: 'active', + disabled: true, + }) + ).toBe(true); + + expect( + shouldAbortVoiceInputForOwner(activeSnapshot(owner, 'listening'), owner, { + appState: 'background', + disabled: false, + }) + ).toBe(true); + + expect( + shouldAbortVoiceInputForOwner(activeSnapshot(otherOwner, 'listening'), owner, { + appState: 'background', + disabled: false, + }) + ).toBe(false); + + expect( + shouldAbortVoiceInputForOwner(idleSnapshot(), owner, { + appState: 'active', + disabled: false, + }) + ).toBe(false); + }); + }); + +}); diff --git a/apps/mobile/src/lib/voice-input/use-voice-input.ts b/apps/mobile/src/lib/voice-input/use-voice-input.ts new file mode 100644 index 0000000000..68fbbd1d48 --- /dev/null +++ b/apps/mobile/src/lib/voice-input/use-voice-input.ts @@ -0,0 +1,147 @@ +import { AppState, type AppStateStatus } from 'react-native'; +import { useCallback, useEffect, useRef, useSyncExternalStore } from 'react'; + +import { type VoiceInputControllerSnapshot } from './voice-input-controller'; +import { voiceInputController } from './native-voice-input'; +import { type VoiceInputStatus } from './voice-input-state'; +import { resolveOwnerVoiceInputView } from './voice-input-view-state'; +import { + createVoiceInputActions, + runVoiceInputListeningFeedback, + shouldAbortVoiceInputForOwner, + type VoiceInputActions, +} from './use-voice-input-actions'; + +type UseVoiceInputOptions = { + disabled: boolean; + getDraft: () => string; + onDraftChange: (draft: string) => void; +}; + +type UseVoiceInputResult = { + abort: () => Promise; + available: boolean; + isActive: boolean; + settleBeforeSubmit: () => Promise; + status: VoiceInputStatus; + toggle: () => Promise; +}; + +let ownerCounter = 0; + +function nextOwnerId(): string { + ownerCounter += 1; + return `voice-input-owner-${ownerCounter}`; +} + +function toLifecycleAppState(status: AppStateStatus): 'active' | 'background' | 'inactive' { + if (status === 'active') { + return 'active'; + } + if (status === 'inactive') { + return 'inactive'; + } + return 'background'; +} + +const subscribe = (listener: () => void): (() => void) => + voiceInputController.subscribe(() => { + listener(); + }); + +const getSnapshot = (): VoiceInputControllerSnapshot => voiceInputController.getSnapshot(); + +const getServerSnapshot = getSnapshot; + +export function useVoiceInput(options: UseVoiceInputOptions): UseVoiceInputResult { + const { disabled, getDraft, onDraftChange } = options; + + const ownerRef = useRef(null); + ownerRef.current ??= nextOwnerId(); + const owner = ownerRef.current; + + const getDraftRef = useRef(getDraft); + getDraftRef.current = getDraft; + + const getOnDraftChangeRef = useRef(onDraftChange); + getOnDraftChangeRef.current = onDraftChange; + + const getDisabledRef = useRef(disabled); + getDisabledRef.current = disabled; + + const snapshot = useSyncExternalStore(subscribe, getSnapshot, getServerSnapshot); + + const actionsRef = useRef(null); + actionsRef.current ??= createVoiceInputActions({ + controller: voiceInputController, + getDisabled: () => getDisabledRef.current, + getDraft: () => getDraftRef.current(), + getOnDraftChange: () => getOnDraftChangeRef.current, + getOwner: () => owner, + }); + const actions = actionsRef.current; + + const view = resolveOwnerVoiceInputView(snapshot, owner); + + const previousOwnStatusRef = useRef(null); + useEffect(() => { + const previousOwnStatus = previousOwnStatusRef.current; + previousOwnStatusRef.current = view.status; + runVoiceInputListeningFeedback(previousOwnStatus, view.status); + }, [view.status]); + + useEffect(() => { + const subscription = AppState.addEventListener('change', (nextAppState: AppStateStatus) => { + const lifecycleInput = { + appState: toLifecycleAppState(nextAppState), + disabled: getDisabledRef.current, + }; + const currentSnapshot = voiceInputController.getSnapshot(); + if (!shouldAbortVoiceInputForOwner(currentSnapshot, owner, lifecycleInput)) { + return; + } + void actions.abort(); + }); + return () => { + subscription.remove(); + }; + }, [owner, actions]); + + useEffect( + () => () => { + void actions.abort(); + }, + [actions] + ); + + useEffect(() => { + const currentSnapshot = voiceInputController.getSnapshot(); + if (!shouldAbortVoiceInputForOwner(currentSnapshot, owner, { appState: 'active', disabled })) { + return; + } + void actions.abort(); + }, [disabled, owner, actions]); + + const abort = useCallback(async () => { + const result = await actions.abort(); + return result; + }, [actions]); + + const settleBeforeSubmit = useCallback(async () => { + const result = await actions.settleBeforeSubmit(); + return result; + }, [actions]); + + const toggle = useCallback(async () => { + await actions.toggle(); + }, [actions]); + + return { + abort, + available: view.available, + isActive: view.isActive, + settleBeforeSubmit, + status: view.status, + toggle, + }; +} diff --git a/apps/mobile/src/lib/voice-input/voice-input-feedback.test.ts b/apps/mobile/src/lib/voice-input/voice-input-feedback.test.ts new file mode 100644 index 0000000000..a7b86af5c7 --- /dev/null +++ b/apps/mobile/src/lib/voice-input/voice-input-feedback.test.ts @@ -0,0 +1,100 @@ +import { describe, expect, it } from 'vitest'; + +import { + resolveVoiceInputFeedbackPresentation, + shouldAnnounceListeningTransition, + type VoiceInputFeedbackPresentation, +} from './voice-input-feedback'; +import { type VoiceInputFeedback, type VoiceInputStatus } from './voice-input-state'; + +function feedback(overrides: Partial = {}): VoiceInputFeedback { + return { + action: 'none', + availability: 'available', + message: 'Something went wrong.', + retryable: true, + ...overrides, + }; +} + +describe('resolveVoiceInputFeedbackPresentation', () => { + it('returns an alert with a fixed microphone-off title when the action is open-settings', () => { + const presentation: VoiceInputFeedbackPresentation = resolveVoiceInputFeedbackPresentation( + feedback({ + action: 'open-settings', + message: 'Microphone access is off. Enable it in Settings to use voice input.', + retryable: false, + }) + ); + + expect(presentation).toEqual({ + kind: 'alert', + title: 'Microphone access is off', + message: 'Microphone access is off. Enable it in Settings to use voice input.', + }); + }); + + it('returns a toast for a retryable feedback that has no follow-up action', () => { + const presentation: VoiceInputFeedbackPresentation = resolveVoiceInputFeedbackPresentation( + feedback({ + action: 'none', + message: 'No speech detected. Tap the microphone to try again.', + retryable: true, + }) + ); + + expect(presentation).toEqual({ + kind: 'toast', + message: 'No speech detected. Tap the microphone to try again.', + }); + }); + + it('returns a toast for a non-retryable feedback that has no follow-up action', () => { + const presentation: VoiceInputFeedbackPresentation = resolveVoiceInputFeedbackPresentation( + feedback({ + action: 'none', + availability: 'unavailable', + message: "Voice input isn't available on this device.", + retryable: false, + }) + ); + + expect(presentation).toEqual({ + kind: 'toast', + message: "Voice input isn't available on this device.", + }); + }); +}); + +describe('shouldAnnounceListeningTransition', () => { + const allStatuses: VoiceInputStatus[] = ['idle', 'starting', 'listening', 'stopping']; + + it('fires exactly once for the first transition into listening (null → listening)', () => { + expect(shouldAnnounceListeningTransition(null, 'listening')).toBe(true); + }); + + it('fires on every other non-listening → listening transition (idle, starting, stopping)', () => { + const nonListening: VoiceInputStatus[] = allStatuses.filter( + (status): boolean => status !== 'listening' + ); + for (const previous of nonListening) { + expect(shouldAnnounceListeningTransition(previous, 'listening')).toBe(true); + } + }); + + it('does not fire on a re-render that keeps the previous own status as listening', () => { + expect(shouldAnnounceListeningTransition('listening', 'listening')).toBe(false); + }); + + it('does not fire on transitions away from listening', () => { + expect(shouldAnnounceListeningTransition('listening', 'idle')).toBe(false); + expect(shouldAnnounceListeningTransition('listening', 'stopping')).toBe(false); + expect(shouldAnnounceListeningTransition('listening', 'starting')).toBe(false); + }); + + it('does not fire on transitions that never reach listening', () => { + expect(shouldAnnounceListeningTransition('idle', 'starting')).toBe(false); + expect(shouldAnnounceListeningTransition('starting', 'stopping')).toBe(false); + expect(shouldAnnounceListeningTransition('stopping', 'idle')).toBe(false); + }); +}); diff --git a/apps/mobile/src/lib/voice-input/voice-input-feedback.ts b/apps/mobile/src/lib/voice-input/voice-input-feedback.ts new file mode 100644 index 0000000000..679bdb4429 --- /dev/null +++ b/apps/mobile/src/lib/voice-input/voice-input-feedback.ts @@ -0,0 +1,42 @@ +import { type VoiceInputFeedback, type VoiceInputStatus } from './voice-input-state'; + +export type VoiceInputFeedbackPresentation = + | { kind: 'alert'; title: 'Microphone access is off'; message: string } + | { kind: 'toast'; message: string }; + +const ALERT_TITLE = 'Microphone access is off' as const; + +/** + * Pure projection of a `VoiceInputFeedback` into the surface that should + * display it. Open-settings feedback (permanent microphone denial) gets a + * native alert with a Cancel/Open Settings affordance so the user can + * recover without re-trying the gesture. Every other case is a transient + * toast — retryable failures invite the user to try again, non-retryable + * ones are informational only. Keeping this decision in a pure function + * makes the policy unit-testable and isolates React Native's `Alert` and + * `toast` from the rule. + */ +export function resolveVoiceInputFeedbackPresentation( + feedback: VoiceInputFeedback +): VoiceInputFeedbackPresentation { + if (feedback.action === 'open-settings') { + return { kind: 'alert', title: ALERT_TITLE, message: feedback.message }; + } + return { kind: 'toast', message: feedback.message }; +} + +/** + * Pure decision for whether the listening-announce + light haptic should + * fire. The only signal we care about is the owner-relative status flipping + * from any non-listening state into `listening` — every other transition + * (including a re-render that keeps the same `listening` state, a status + * flip that doesn't belong to us, or a transition to `starting` / + * `stopping` / `idle`) returns `false`. The hook calls this each render and + * only fires the side effects on a `true` result. + */ +export function shouldAnnounceListeningTransition( + previousOwnStatus: VoiceInputStatus | null, + nextOwnStatus: VoiceInputStatus +): boolean { + return previousOwnStatus !== 'listening' && nextOwnStatus === 'listening'; +} diff --git a/apps/mobile/src/lib/voice-input/voice-input-language.test.ts b/apps/mobile/src/lib/voice-input/voice-input-language.test.ts new file mode 100644 index 0000000000..56d0bf6148 --- /dev/null +++ b/apps/mobile/src/lib/voice-input/voice-input-language.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from 'vitest'; + +import { resolveVoiceInputLanguageTag } from './voice-input-language'; + +describe('resolveVoiceInputLanguageTag', () => { + it('falls back to en-US when the locales array is empty', () => { + expect(resolveVoiceInputLanguageTag([])).toBe('en-US'); + }); + + it('falls back to en-US when the first locale has no languageTag', () => { + expect(resolveVoiceInputLanguageTag([{}])).toBe('en-US'); + }); + + it('falls back to en-US when the first locale languageTag is empty', () => { + expect(resolveVoiceInputLanguageTag([{ languageTag: '' }])).toBe('en-US'); + }); + + it('returns the first locale languageTag when it is populated', () => { + expect(resolveVoiceInputLanguageTag([{ languageTag: 'nl-NL' }])).toBe('nl-NL'); + }); + + it('ignores later locales', () => { + expect(resolveVoiceInputLanguageTag([{ languageTag: 'fr-FR' }, { languageTag: 'de-DE' }])).toBe( + 'fr-FR' + ); + }); +}); diff --git a/apps/mobile/src/lib/voice-input/voice-input-language.ts b/apps/mobile/src/lib/voice-input/voice-input-language.ts new file mode 100644 index 0000000000..453c7dabd5 --- /dev/null +++ b/apps/mobile/src/lib/voice-input/voice-input-language.ts @@ -0,0 +1,21 @@ +/** + * Resolve the BCP-47 language tag for voice recognition from the device's + * locale list. The helper is defensive because runtime behavior can diverge + * from the package's static typing: `getLocales()` may return an empty + * array, or the first locale may be missing a tag, so every path falls back + * to `en-US`. The parameter shape is intentionally structural so the caller + * can pass `expo-localization`'s `Locale[]` without a type assertion. + */ +export function resolveVoiceInputLanguageTag(locales: readonly { languageTag?: string }[]): string { + const first = locales[0]; + if (!first) { + return 'en-US'; + } + + const tag = first.languageTag; + if (!tag || tag.length === 0) { + return 'en-US'; + } + + return tag; +} diff --git a/apps/mobile/src/lib/voice-input/voice-input-state.ts b/apps/mobile/src/lib/voice-input/voice-input-state.ts index bfea889269..bd2d07ebe8 100644 --- a/apps/mobile/src/lib/voice-input/voice-input-state.ts +++ b/apps/mobile/src/lib/voice-input/voice-input-state.ts @@ -22,7 +22,7 @@ type VoiceInputPermission = { restricted?: boolean; }; -type VoiceInputLifecycleInput = { +export type VoiceInputLifecycleInput = { appState: 'active' | 'background' | 'inactive'; disabled: boolean; }; @@ -50,9 +50,8 @@ export function applyVoiceRecognitionResult( const normalized = normalizeSegment(result.transcript); if (result.isFinal) { - const finalSegments = normalized.length === 0 - ? state.finalSegments - : [...state.finalSegments, normalized]; + const finalSegments = + normalized.length === 0 ? state.finalSegments : [...state.finalSegments, normalized]; return { finalSegments, interim: '', From af455fe9a0c579549a3a891eb80a84b05d3e315b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20=C5=A0=C4=87eki=C4=87?= Date: Tue, 14 Jul 2026 15:43:07 +0200 Subject: [PATCH 07/14] feat(mobile): add voice input to Kilo Chat --- .../kilo-chat/message-input-content-state.ts | 26 +++++ .../kilo-chat/message-input-content.tsx | 107 ++++++++++-------- .../kilo-chat/message-input-state.ts | 23 ++++ .../kilo-chat/message-input-view.tsx | 33 +++++- .../message-input-voice-submit.test.ts | 51 +++++++++ .../src/components/voice-input-control.tsx | 4 +- 6 files changed, 186 insertions(+), 58 deletions(-) create mode 100644 apps/mobile/src/components/kilo-chat/message-input-content-state.ts create mode 100644 apps/mobile/src/components/kilo-chat/message-input-voice-submit.test.ts diff --git a/apps/mobile/src/components/kilo-chat/message-input-content-state.ts b/apps/mobile/src/components/kilo-chat/message-input-content-state.ts new file mode 100644 index 0000000000..59cf18de49 --- /dev/null +++ b/apps/mobile/src/components/kilo-chat/message-input-content-state.ts @@ -0,0 +1,26 @@ +import { type AttachmentBlock } from '@kilocode/kilo-chat'; +import { type QueuedAttachment } from '@kilocode/kilo-chat-hooks'; + +export function resolveMessageInputSendDisabled({ + canSend, + disabled, + overLimit, +}: { + canSend: boolean; + disabled?: boolean; + overLimit: boolean; +}): boolean { + return !canSend || disabled === true || overLimit; +} + +export function editableAttachmentToPreviewRow(attachment: AttachmentBlock): QueuedAttachment { + return { + tempId: attachment.attachmentId, + attachmentId: attachment.attachmentId, + filename: attachment.filename, + mimeType: attachment.mimeType, + size: attachment.size, + status: 'ready', + progress: 1, + }; +} diff --git a/apps/mobile/src/components/kilo-chat/message-input-content.tsx b/apps/mobile/src/components/kilo-chat/message-input-content.tsx index 7a5274ac8a..3198816bb0 100644 --- a/apps/mobile/src/components/kilo-chat/message-input-content.tsx +++ b/apps/mobile/src/components/kilo-chat/message-input-content.tsx @@ -1,11 +1,16 @@ import { useEffect, useMemo, useRef, useState } from 'react'; import { AppState, type LayoutChangeEvent, Platform, type TextInput, View } from 'react-native'; -import { type AttachmentBlock } from '@kilocode/kilo-chat'; -import { type QueuedAttachment } from '@kilocode/kilo-chat-hooks'; +import { type AttachmentBlock, MESSAGE_TEXT_MAX_CHARS } from '@kilocode/kilo-chat'; import { useSafeAreaInsets } from 'react-native-safe-area-context'; import { useTextHeight } from '@/components/agents/use-text-height'; +import { applyVoiceDraftToInput } from '@/lib/voice-input/voice-input-draft'; +import { useVoiceInput } from '@/lib/voice-input/use-voice-input'; import { resolveMessageInputAppStateTransition } from './message-input-app-state'; +import { + editableAttachmentToPreviewRow, + resolveMessageInputSendDisabled, +} from './message-input-content-state'; import { MESSAGE_INPUT_FONT_SIZE, MESSAGE_INPUT_HORIZONTAL_PADDING, @@ -27,6 +32,7 @@ import { canSubmitMessageInputContent, clearSubmittedMessageInputDraft, isMessageInputOverLimit, + settleVoiceInputBeforeSubmit, shouldShowMessageInputCounter, submitMessageInputDraft, } from './message-input-state'; @@ -35,24 +41,6 @@ import { MessageInputView } from './message-input-view'; const MESSAGE_INPUT_FOCUS_RESTORE_DELAY_MS = 100; const EMPTY_READY_ATTACHMENT_BLOCKS: readonly AttachmentBlock[] = []; -function resolveSendDisabled({ - canSend, - disabled, - overLimit, -}: { - canSend: boolean; - disabled?: boolean; - overLimit: boolean; -}): boolean { - if (!canSend) { - return true; - } - if (disabled === true) { - return true; - } - return overLimit; -} - export function MessageInputContent({ onSendText, onSendContentBlocks, @@ -107,10 +95,12 @@ export function MessageInputContent({ const overLimit = isMessageInputOverLimit(valueRef.current); const showCounter = shouldShowMessageInputCounter(valueRef.current); const sendDisabled = - submitDisabled === true || resolveSendDisabled({ canSend, disabled, overLimit }); + submitDisabled === true || resolveMessageInputSendDisabled({ canSend, disabled, overLimit }); const controlsDisabled = disabled === true || submitDisabled === true; const showAttachmentButton = attachmentQueue !== null && hasAttachmentsCapability && isEditing !== true && disabled !== true; + const voiceDisabled = + disabled === true || hasUploadingAttachment || isEditing === true; const inputMeasure = useTextHeight({ minHeight: MESSAGE_INPUT_MIN_HEIGHT, maxHeight: MESSAGE_INPUT_MAX_HEIGHT, @@ -121,6 +111,33 @@ export function MessageInputContent({ initialText, }); + const handleChangeText = (text: string) => { + setDraftLength(text.length); + inputMeasure.setText(text); + applyMessageInputTextChange({ + text, + valueRef, + setCanSend, + onTyping, + readyAttachmentBlocks, + hasUploadingAttachment, + hasFailedAttachment, + }); + }; + + const voiceInput = useVoiceInput({ + disabled: voiceDisabled, + getDraft: () => valueRef.current, + onDraftChange: draft => { + applyVoiceDraftToInput({ + input: inputRef.current, + draft, + maxLength: MESSAGE_TEXT_MAX_CHARS, + onChangeText: handleChangeText, + }); + }, + }); + useEffect(() => { const clearRestoreFocusTimeout = () => { if (restoreFocusTimeoutRef.current !== null) { @@ -168,7 +185,7 @@ export function MessageInputContent({ ); }, [hasFailedAttachment, hasUploadingAttachment, readyAttachmentBlocks]); - const submit = () => { + function submitDraft() { if (disabled || submitDisabled) { return; } @@ -205,7 +222,14 @@ export function MessageInputContent({ hasUploadingAttachment, hasFailedAttachment, }); - }; + } + + async function submit() { + await settleVoiceInputBeforeSubmit({ + settleVoiceInput: voiceInput.settleBeforeSubmit, + submit: submitDraft, + }); + } function handleInputLayout(event: LayoutChangeEvent) { const nextWidth = Math.max(Math.round(event.nativeEvent.layout.width), 0); @@ -228,20 +252,6 @@ export function MessageInputContent({ onRemoveEditableAttachment?.(attachmentId); }; - const handleChangeText = (text: string) => { - setDraftLength(text.length); - inputMeasure.setText(text); - applyMessageInputTextChange({ - text, - valueRef, - setCanSend, - onTyping, - readyAttachmentBlocks, - hasUploadingAttachment, - hasFailedAttachment, - }); - }; - return ( { + void submit(); + }} + onVoiceInputToggle={() => { + void voiceInput.toggle(); + }} overLimit={overLimit} replyingTo={replyingTo} sendDisabled={sendDisabled} @@ -288,19 +303,11 @@ export function MessageInputContent({ showCounter={showCounter} shouldScroll={resolveMessageInputShouldScroll(inputMeasure.height)} typingMembers={typingMembers} + voiceInputActive={voiceInput.isActive} + voiceInputAvailable={voiceInput.available} + voiceInputDisabled={voiceDisabled} + voiceInputStatus={voiceInput.status} /> ); } - -function editableAttachmentToPreviewRow(attachment: AttachmentBlock): QueuedAttachment { - return { - tempId: attachment.attachmentId, - attachmentId: attachment.attachmentId, - filename: attachment.filename, - mimeType: attachment.mimeType, - size: attachment.size, - status: 'ready', - progress: 1, - }; -} diff --git a/apps/mobile/src/components/kilo-chat/message-input-state.ts b/apps/mobile/src/components/kilo-chat/message-input-state.ts index b85d3ca5bd..5a47390720 100644 --- a/apps/mobile/src/components/kilo-chat/message-input-state.ts +++ b/apps/mobile/src/components/kilo-chat/message-input-state.ts @@ -85,6 +85,29 @@ export function clearSubmittedMessageInputDraft({ return cleared; } +/** + * Awaits a voice-input settle callback (e.g. waiting for the final transcript of + * an in-flight listening session) and only invokes `submit` when settle resolves + * truthy. Returning `false` from settle aborts the submit so the caller can + * surface controller-reported feedback. The helper intentionally does not + * swallow rejections: a throw from settle is treated as a programmer or native + * bridge bug and propagated to the caller, again without invoking submit. + */ +export async function settleVoiceInputBeforeSubmit({ + settleVoiceInput, + submit, +}: { + settleVoiceInput: () => Promise; + submit: () => void; +}): Promise { + const settled = await settleVoiceInput(); + if (!settled) { + return false; + } + submit(); + return true; +} + export function applyMessageInputTextChange({ text, valueRef, diff --git a/apps/mobile/src/components/kilo-chat/message-input-view.tsx b/apps/mobile/src/components/kilo-chat/message-input-view.tsx index 3fc4335c8d..c654ae2140 100644 --- a/apps/mobile/src/components/kilo-chat/message-input-view.tsx +++ b/apps/mobile/src/components/kilo-chat/message-input-view.tsx @@ -6,7 +6,9 @@ import { type QueuedAttachment } from '@kilocode/kilo-chat-hooks'; import { Text } from '@/components/ui/text'; import { useThemeColors } from '@/lib/hooks/use-theme-colors'; +import { type VoiceInputStatus } from '@/lib/voice-input/voice-input-state'; import { cn } from '@/lib/utils'; +import { VoiceInputButton, VoiceInputStatus as VoiceInputStatusView } from '@/components/voice-input-control'; import { MessageAttachmentPreviewStrip } from './message-attachment-preview-strip'; import { messageInputKeyboardProps, messageInputTextStyle } from './message-input-layout'; import { type ComposerAttachmentQueue } from './message-input-types'; @@ -38,6 +40,7 @@ type Props = { onRemoveEditableAttachment: (attachmentId: string) => void; onRetryAttachment: (tempId: string) => void; onSubmit: () => void; + onVoiceInputToggle: () => void; overLimit: boolean; replyingTo?: Message | null; sendDisabled: boolean; @@ -45,6 +48,10 @@ type Props = { showCounter: boolean; shouldScroll: boolean; typingMembers: Map; + voiceInputActive: boolean; + voiceInputAvailable: boolean; + voiceInputDisabled: boolean; + voiceInputStatus: VoiceInputStatus; }; export function MessageInputView({ @@ -72,6 +79,7 @@ export function MessageInputView({ onRemoveEditableAttachment, onRetryAttachment, onSubmit, + onVoiceInputToggle, overLimit, replyingTo, sendDisabled, @@ -79,8 +87,13 @@ export function MessageInputView({ showCounter, shouldScroll, typingMembers, + voiceInputActive, + voiceInputAvailable, + voiceInputDisabled, + voiceInputStatus, }: Props) { const colors = useThemeColors(); + const inputDisabled = disabled === true || voiceInputActive; return ( <> @@ -144,14 +157,14 @@ export function MessageInputView({ {showAttachmentButton ? ( @@ -162,7 +175,7 @@ export function MessageInputView({ className={cn( 'rounded-md border bg-card px-3 text-foreground', overLimit ? 'border-destructive' : 'border-input', - disabled && 'opacity-50' + inputDisabled && 'opacity-50' )} style={[messageInputTextStyle, { height: inputHeight }]} placeholder="Message" @@ -170,8 +183,8 @@ export function MessageInputView({ defaultValue={initialText} multiline scrollEnabled={shouldScroll} - editable={!disabled} - accessibilityState={{ disabled }} + editable={!inputDisabled} + accessibilityState={{ disabled: inputDisabled }} onChangeText={onChangeText} onFocus={onInputFocus} onBlur={onInputBlur} @@ -179,6 +192,13 @@ export function MessageInputView({ {...messageInputKeyboardProps} /> + {voiceInputAvailable ? ( + + ) : null} {onCancelEdit && ( + {showCounter ? ( diff --git a/apps/mobile/src/components/kilo-chat/message-input-voice-submit.test.ts b/apps/mobile/src/components/kilo-chat/message-input-voice-submit.test.ts new file mode 100644 index 0000000000..de57775bca --- /dev/null +++ b/apps/mobile/src/components/kilo-chat/message-input-voice-submit.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it, vi } from 'vitest'; + +import { settleVoiceInputBeforeSubmit } from './message-input-state'; + +describe('settleVoiceInputBeforeSubmit', () => { + it('settles before submitting', async () => { + const order: string[] = []; + + await expect( + settleVoiceInputBeforeSubmit({ + settleVoiceInput: vi.fn().mockImplementationOnce(async () => { + await Promise.resolve(); + order.push('settle'); + return true; + }), + submit: () => { + order.push('submit'); + }, + }) + ).resolves.toBe(true); + + expect(order).toEqual(['settle', 'submit']); + }); + + it('does not submit when settlement fails', async () => { + const submit = vi.fn<() => void>(); + + await expect( + settleVoiceInputBeforeSubmit({ + settleVoiceInput: vi.fn().mockResolvedValueOnce(false), + submit, + }) + ).resolves.toBe(false); + + expect(submit).not.toHaveBeenCalled(); + }); + + it('propagates settlement errors without submitting', async () => { + const failure = new Error('native recognition crashed'); + const submit = vi.fn<() => void>(); + + await expect( + settleVoiceInputBeforeSubmit({ + settleVoiceInput: vi.fn().mockRejectedValueOnce(failure), + submit, + }) + ).rejects.toBe(failure); + + expect(submit).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/mobile/src/components/voice-input-control.tsx b/apps/mobile/src/components/voice-input-control.tsx index 0f95068fe7..2adc8c6eed 100644 --- a/apps/mobile/src/components/voice-input-control.tsx +++ b/apps/mobile/src/components/voice-input-control.tsx @@ -6,7 +6,7 @@ import { cn } from '@/lib/utils'; import { type VoiceInputStatus } from '@/lib/voice-input/voice-input-state'; import { resolveVoiceInputControlState } from '@/lib/voice-input/voice-input-view-state'; -export type VoiceInputButtonProps = { +type VoiceInputButtonProps = { disabled: boolean; onPress(): void; status: VoiceInputStatus; @@ -64,7 +64,7 @@ export function VoiceInputButton({ ); } -export type VoiceInputStatusProps = { +type VoiceInputStatusProps = { status: VoiceInputStatus; }; From 2e5025f6a337e6cb078635c7de2fd7d8d3eadfc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20=C5=A0=C4=87eki=C4=87?= Date: Tue, 14 Jul 2026 15:50:32 +0200 Subject: [PATCH 08/14] feat(mobile): add voice input to agent sessions --- .../agents/chat-composer-input-row.tsx | 178 ++++++++++++++++ .../agents/chat-composer-input-state.test.ts | 155 ++++++++++++++ .../agents/chat-composer-input-state.ts | 65 ++++++ .../src/components/agents/chat-composer.tsx | 196 ++++++++---------- .../kilo-chat/message-input-content.tsx | 5 +- .../kilo-chat/message-input-state.ts | 23 -- .../message-input-voice-submit.test.ts | 2 +- .../voice-input/voice-input-submit.test.ts | 64 ++++++ .../src/lib/voice-input/voice-input-submit.ts | 29 +++ 9 files changed, 581 insertions(+), 136 deletions(-) create mode 100644 apps/mobile/src/components/agents/chat-composer-input-row.tsx create mode 100644 apps/mobile/src/components/agents/chat-composer-input-state.test.ts create mode 100644 apps/mobile/src/components/agents/chat-composer-input-state.ts create mode 100644 apps/mobile/src/lib/voice-input/voice-input-submit.test.ts create mode 100644 apps/mobile/src/lib/voice-input/voice-input-submit.ts diff --git a/apps/mobile/src/components/agents/chat-composer-input-row.tsx b/apps/mobile/src/components/agents/chat-composer-input-row.tsx new file mode 100644 index 0000000000..f332806667 --- /dev/null +++ b/apps/mobile/src/components/agents/chat-composer-input-row.tsx @@ -0,0 +1,178 @@ +import { ArrowUp, Paperclip, Square } from 'lucide-react-native'; +import { type RefObject } from 'react'; +import { + ActivityIndicator, + type LayoutChangeEvent, + Pressable, + TextInput, + type TextStyle, + View, +} from 'react-native'; + +import { VoiceInputButton } from '@/components/voice-input-control'; +import { useThemeColors } from '@/lib/hooks/use-theme-colors'; +import { cn } from '@/lib/utils'; +import { type VoiceInputStatus } from '@/lib/voice-input/voice-input-state'; + +const PAPERCLIP_HIT_SLOP = { top: 8, bottom: 8, left: 8, right: 8 } as const; +const CONTROL_HIT_SLOP = 6; + +type ChatComposerInputRowProps = { + attachmentsEnabled: boolean; + canSend: boolean; + disabled: boolean; + inputAccessibilityDisabled: boolean; + inputEditable: boolean; + inputRef: RefObject; + isSending: boolean; + isStreaming: boolean; + maxInputHeight: number; + measureHeight: number; + onAddAttachment: () => void; + onChangeText: (text: string) => void; + onInputBlur: () => void; + onInputFocus: () => void; + onInputLayout: (event: LayoutChangeEvent) => void; + onStop: () => void; + onSubmit: () => void; + onToggleVoice: () => void; + paperclipDisabled: boolean; + placeholder: string; + textInputStyle: TextStyle; + voiceDisabled: boolean; + voiceInputAvailable: boolean; + voiceInputStatus: VoiceInputStatus; +}; + +/** + * Bottom row of the Cloud Agent `ChatComposer`: paperclip, text input, voice + * toggle, and the streaming / send control. Pure presentation — all gating + * rules come from `resolveChatComposerControlState` in + * `chat-composer-input-state.ts` and the parent owns the refs, state, and + * submit/voice orchestration. + */ +export function ChatComposerInputRow({ + attachmentsEnabled, + canSend, + disabled, + inputAccessibilityDisabled, + inputEditable, + inputRef, + isSending, + isStreaming, + maxInputHeight, + measureHeight, + onAddAttachment, + onChangeText, + onInputBlur, + onInputFocus, + onInputLayout, + onStop, + onSubmit, + onToggleVoice, + paperclipDisabled, + placeholder, + textInputStyle, + voiceDisabled, + voiceInputAvailable, + voiceInputStatus, +}: Readonly) { + const colors = useThemeColors(); + + return ( + + {attachmentsEnabled ? ( + + + + ) : null} + + + = maxInputHeight} + editable={inputEditable} + accessibilityState={{ disabled: inputAccessibilityDisabled }} + returnKeyType="default" + submitBehavior="newline" + autoCapitalize="sentences" + autoCorrect + /> + + + {!isStreaming && voiceInputAvailable ? ( + + + + ) : null} + + {isStreaming ? ( + + + + ) : ( + + {isSending ? ( + + ) : ( + + )} + + )} + + ); +} diff --git a/apps/mobile/src/components/agents/chat-composer-input-state.test.ts b/apps/mobile/src/components/agents/chat-composer-input-state.test.ts new file mode 100644 index 0000000000..0e2464d546 --- /dev/null +++ b/apps/mobile/src/components/agents/chat-composer-input-state.test.ts @@ -0,0 +1,155 @@ +import { describe, expect, it } from 'vitest'; + +import { resolveChatComposerControlState } from './chat-composer-input-state'; + +describe('resolveChatComposerControlState', () => { + it('disables nothing and allows sending when idle with text and no voice session', () => { + const state = resolveChatComposerControlState({ + attachmentsCount: 0, + attachmentMax: 5, + disabled: false, + hasText: true, + isFocused: false, + isSending: false, + isStreaming: false, + voiceInputActive: false, + }); + + expect(state).toEqual({ + canSend: true, + inputEditable: true, + inputAccessibilityDisabled: false, + paperclipDisabled: false, + showToolbar: true, + toolbarDisabled: false, + voiceDisabled: false, + }); + }); + + it('collapses send, voice, and toolbar when disabled, streaming, or sending', () => { + for (const override of [ + { disabled: true, isStreaming: false, isSending: false }, + { disabled: false, isStreaming: true, isSending: false }, + { disabled: false, isStreaming: false, isSending: true }, + ]) { + const state = resolveChatComposerControlState({ + attachmentsCount: 0, + attachmentMax: 5, + disabled: override.disabled, + hasText: true, + isFocused: false, + isSending: override.isSending, + isStreaming: override.isStreaming, + voiceInputActive: false, + }); + + expect(state.canSend).toBe(false); + expect(state.toolbarDisabled).toBe(true); + expect(state.voiceDisabled).toBe(true); + expect(state.inputEditable).toBe(false); + expect(state.inputAccessibilityDisabled).toBe(true); + } + }); + + it('does not allow send when the draft is empty even with attachments and no overrides', () => { + const state = resolveChatComposerControlState({ + attachmentsCount: 2, + attachmentMax: 5, + disabled: false, + hasText: false, + isFocused: false, + isSending: false, + isStreaming: false, + voiceInputActive: false, + }); + + expect(state.canSend).toBe(false); + expect(state.toolbarDisabled).toBe(false); + expect(state.showToolbar).toBe(true); + }); + + it('keeps the toolbar visible when focused, has text, has attachments, or voice is active', () => { + const base = { + attachmentsCount: 0, + attachmentMax: 5, + disabled: false, + hasText: false, + isFocused: false, + isSending: false, + isStreaming: false, + voiceInputActive: false, + }; + + expect(resolveChatComposerControlState({ ...base, isFocused: true }).showToolbar).toBe(true); + expect(resolveChatComposerControlState({ ...base, hasText: true }).showToolbar).toBe(true); + expect(resolveChatComposerControlState({ ...base, attachmentsCount: 1 }).showToolbar).toBe( + true + ); + expect(resolveChatComposerControlState({ ...base, voiceInputActive: true }).showToolbar).toBe( + true + ); + expect(resolveChatComposerControlState(base).showToolbar).toBe(false); + }); + + it('disables the paperclip when at or above the attachment cap', () => { + const state = resolveChatComposerControlState({ + attachmentsCount: 5, + attachmentMax: 5, + disabled: false, + hasText: true, + isFocused: false, + isSending: false, + isStreaming: false, + voiceInputActive: false, + }); + + expect(state.paperclipDisabled).toBe(true); + }); + + it('disables the paperclip while the composer is in a toolbar-disabled state', () => { + const state = resolveChatComposerControlState({ + attachmentsCount: 0, + attachmentMax: 5, + disabled: false, + hasText: true, + isFocused: false, + isSending: false, + isStreaming: true, + voiceInputActive: false, + }); + + expect(state.paperclipDisabled).toBe(true); + }); + + it('disables the paperclip and input while this owner is voice active', () => { + const state = resolveChatComposerControlState({ + attachmentsCount: 0, + attachmentMax: 5, + disabled: false, + hasText: true, + isFocused: false, + isSending: false, + isStreaming: false, + voiceInputActive: true, + }); + + expect(state.paperclipDisabled).toBe(true); + expect(state.inputEditable).toBe(false); + expect(state.inputAccessibilityDisabled).toBe(true); + }); + + it('leaves voice enabled (only toolbar gates it) when the composer is otherwise ready', () => { + const state = resolveChatComposerControlState({ + attachmentsCount: 0, + attachmentMax: 5, + disabled: false, + hasText: false, + isFocused: false, + isSending: false, + isStreaming: false, + voiceInputActive: false, + }); + + expect(state.voiceDisabled).toBe(false); + }); +}); diff --git a/apps/mobile/src/components/agents/chat-composer-input-state.ts b/apps/mobile/src/components/agents/chat-composer-input-state.ts new file mode 100644 index 0000000000..65bb27c682 --- /dev/null +++ b/apps/mobile/src/components/agents/chat-composer-input-state.ts @@ -0,0 +1,65 @@ +type ChatComposerControlInput = { + attachmentsCount: number; + attachmentMax: number; + disabled: boolean; + hasText: boolean; + isFocused: boolean; + isSending: boolean; + isStreaming: boolean; + voiceInputActive: boolean; +}; + +type ChatComposerControlState = { + /** Backend requires a non-empty prompt even with attachments. */ + canSend: boolean; + /** Mirrors `editable` on the text input. */ + inputEditable: boolean; + /** Mirrors `accessibilityState.disabled` on the text input. */ + inputAccessibilityDisabled: boolean; + /** Drives the attachment picker. */ + paperclipDisabled: boolean; + /** Toolbar (mode/variant/model row) visibility. */ + showToolbar: boolean; + /** Latches the toolbar's mode/model controls while send, stream, or disabled. */ + toolbarDisabled: boolean; + /** Mirrors `useVoiceInput`'s `disabled` — toolbar-disabled is the gate. */ + voiceDisabled: boolean; +}; + +/** + * Pure projection of the Cloud Agent `ChatComposer` control surface. Keeping + * the rules in one place lets the component stay a thin presenter and makes + * every state — happy, blocked, and listening — testable without rendering + * the composer. Voice input integrates here too: an active voice session + * makes the input read-only and locks the attachment picker while speech is + * being recognized. + */ +export function resolveChatComposerControlState( + input: ChatComposerControlInput +): ChatComposerControlState { + const { + attachmentsCount, + attachmentMax, + disabled, + hasText, + isFocused, + isSending, + isStreaming, + voiceInputActive, + } = input; + const toolbarDisabled = disabled || isStreaming || isSending; + const voiceDisabled = toolbarDisabled; + const paperclipDisabled = + toolbarDisabled || voiceInputActive || attachmentsCount >= attachmentMax; + const inputEditable = !toolbarDisabled && !voiceInputActive; + const showToolbar = isFocused || hasText || attachmentsCount > 0 || voiceInputActive; + return { + canSend: hasText && !disabled && !isStreaming && !isSending, + inputAccessibilityDisabled: !inputEditable, + inputEditable, + paperclipDisabled, + showToolbar, + toolbarDisabled, + voiceDisabled, + }; +} diff --git a/apps/mobile/src/components/agents/chat-composer.tsx b/apps/mobile/src/components/agents/chat-composer.tsx index 7df0ea92e3..ee2e429e09 100644 --- a/apps/mobile/src/components/agents/chat-composer.tsx +++ b/apps/mobile/src/components/agents/chat-composer.tsx @@ -1,13 +1,10 @@ import * as Haptics from 'expo-haptics'; import { useActionSheet } from '@expo/react-native-action-sheet'; -import { ArrowUp, Paperclip, Square } from 'lucide-react-native'; import { useCallback, useRef, useState } from 'react'; import { - ActivityIndicator, Keyboard, type LayoutChangeEvent, - Pressable, - TextInput, + type TextInput, type TextStyle, View, } from 'react-native'; @@ -19,7 +16,10 @@ import { ChatToolbar } from '@/components/agents/chat-toolbar'; import { type AgentMode } from '@/components/agents/mode-selector'; import { pickAgentAttachments } from '@/components/agents/attachment-picker'; import { useTextHeight } from '@/components/agents/use-text-height'; +import { resolveChatComposerControlState } from '@/components/agents/chat-composer-input-state'; +import { ChatComposerInputRow } from '@/components/agents/chat-composer-input-row'; import { BlurBar } from '@/components/ui/blur-bar'; +import { VoiceInputStatus } from '@/components/voice-input-control'; import { AGENT_ATTACHMENT_MAX_FILES } from '@/lib/agent-attachments/constants'; import { type AgentAttachmentWire, @@ -28,6 +28,9 @@ import { import { type ModelOption } from '@/lib/hooks/use-available-models'; import { useThemeColors } from '@/lib/hooks/use-theme-colors'; import { cn } from '@/lib/utils'; +import { useVoiceInput } from '@/lib/voice-input/use-voice-input'; +import { applyVoiceDraftToInput } from '@/lib/voice-input/voice-input-draft'; +import { settleVoiceInputBeforeSubmit } from '@/lib/voice-input/voice-input-submit'; const TEXT_INPUT_MAX_LINES = 5; const TEXT_INPUT_LINE_HEIGHT = 20; @@ -36,8 +39,7 @@ const TEXT_INPUT_HORIZONTAL_PADDING = 32; const TEXT_INPUT_MIN_HEIGHT = TEXT_INPUT_LINE_HEIGHT + TEXT_INPUT_VERTICAL_PADDING; const TEXT_INPUT_MAX_HEIGHT = TEXT_INPUT_LINE_HEIGHT * TEXT_INPUT_MAX_LINES + TEXT_INPUT_VERTICAL_PADDING; - -const PAPERCLIP_HIT_SLOP = { top: 8, bottom: 8, left: 8, right: 8 } as const; +const TEXT_INPUT_FONT_SIZE = 16; type ChatComposerProps = { onSend: (text: string, attachments?: AgentAttachmentWire) => void | Promise; @@ -86,18 +88,37 @@ export function ChatComposer({ maxHeight: TEXT_INPUT_MAX_HEIGHT, verticalPadding: TEXT_INPUT_VERTICAL_PADDING, textContentWidth: inputWidth - TEXT_INPUT_HORIZONTAL_PADDING, - fontSize: 16, + fontSize: TEXT_INPUT_FONT_SIZE, lineHeight: TEXT_INPUT_LINE_HEIGHT, }); - // The backend requires a non-empty prompt even when attachments are present. - const canSend = hasText && !disabled && !isStreaming && !isSending; - const showToolbar = isFocused || hasText || upload.attachments.length > 0; - // isSending locks the input and attachment controls too — otherwise text or - // attachments added while the send is in flight get wiped by the success path. + // Compute base composer disabled before the voice hook so voice can react to it. const toolbarDisabled = disabled || isStreaming || isSending; - const paperclipDisabled = - toolbarDisabled || upload.attachments.length >= AGENT_ATTACHMENT_MAX_FILES; + const voiceDisabled = toolbarDisabled; + + const voiceInput = useVoiceInput({ + disabled: voiceDisabled, + getDraft: () => textRef.current, + onDraftChange: draft => { + applyVoiceDraftToInput({ + input: inputRef.current, + draft, + maxLength: 4000, + onChangeText: handleChangeText, + }); + }, + }); + + const control = resolveChatComposerControlState({ + attachmentsCount: upload.attachments.length, + attachmentMax: AGENT_ATTACHMENT_MAX_FILES, + disabled, + hasText, + isFocused, + isSending, + isStreaming, + voiceInputActive: voiceInput.isActive, + }); function handleChangeText(value: string) { textRef.current = value; @@ -107,7 +128,7 @@ export function ChatComposer({ async function handleSend() { const trimmed = textRef.current.trim(); - if (!trimmed || !canSend) { + if (!trimmed || !control.canSend) { return; } if (upload.isUploading) { @@ -144,6 +165,15 @@ export function ChatComposer({ } } + async function submit() { + await settleVoiceInputBeforeSubmit({ + settleVoiceInput: voiceInput.settleBeforeSubmit, + submit: () => { + void handleSend(); + }, + }); + } + function handleStop() { void Haptics.impactAsync(Haptics.ImpactFeedbackStyle.Medium); void onStop?.(); @@ -162,7 +192,7 @@ export function ChatComposer({ const textInputStyle: TextStyle = { color: colors.foreground, - fontSize: 16, + fontSize: TEXT_INPUT_FONT_SIZE, height: measure.height, includeFontPadding: false, lineHeight: TEXT_INPUT_LINE_HEIGHT, @@ -176,7 +206,7 @@ export function ChatComposer({ {measure.measureElement} - {showToolbar ? ( + {control.showToolbar ? ( ) : null} @@ -198,98 +228,46 @@ export function ChatComposer({ /> ) : null} - - {attachmentsEnabled ? ( - { - void handleAddAttachment(); - }} - disabled={paperclipDisabled} - hitSlop={PAPERCLIP_HIT_SLOP} - className={cn( - 'h-8 w-8 items-center justify-center rounded-full active:opacity-70', - paperclipDisabled && 'opacity-50' - )} - accessibilityRole="button" - accessibilityLabel="Add attachment" - accessibilityState={{ disabled: paperclipDisabled }} - > - - - ) : null} - - - { - setIsFocused(true); - }} - onBlur={() => { - setIsFocused(false); - }} - style={textInputStyle} - scrollEnabled={measure.height >= TEXT_INPUT_MAX_HEIGHT} - editable={!toolbarDisabled} - accessibilityState={{ disabled: toolbarDisabled }} - returnKeyType="default" - submitBehavior="newline" - autoCapitalize="sentences" - autoCorrect - /> - - - {isStreaming ? ( - - - - ) : ( - { - void handleSend(); - }} - disabled={!canSend} - hitSlop={6} - accessibilityRole="button" - accessibilityLabel="Send message" - accessibilityState={{ disabled: !canSend, busy: isSending }} - className={`h-8 w-8 items-center justify-center rounded-full active:opacity-70 ${ - canSend ? 'bg-accent-soft' : 'bg-muted' - }`} - > - {isSending ? ( - - ) : ( - - )} - - )} + + + + { + void handleAddAttachment(); + }} + onChangeText={handleChangeText} + onInputBlur={() => { + setIsFocused(false); + }} + onInputFocus={() => { + setIsFocused(true); + }} + onInputLayout={handleInputLayout} + onStop={handleStop} + onSubmit={() => { + void submit(); + }} + onToggleVoice={() => { + void voiceInput.toggle(); + }} + paperclipDisabled={control.paperclipDisabled} + placeholder={placeholder} + textInputStyle={textInputStyle} + voiceDisabled={control.voiceDisabled} + voiceInputAvailable={voiceInput.available} + voiceInputStatus={voiceInput.status} + /> ); } diff --git a/apps/mobile/src/components/kilo-chat/message-input-content.tsx b/apps/mobile/src/components/kilo-chat/message-input-content.tsx index 3198816bb0..11718d82ea 100644 --- a/apps/mobile/src/components/kilo-chat/message-input-content.tsx +++ b/apps/mobile/src/components/kilo-chat/message-input-content.tsx @@ -32,11 +32,11 @@ import { canSubmitMessageInputContent, clearSubmittedMessageInputDraft, isMessageInputOverLimit, - settleVoiceInputBeforeSubmit, shouldShowMessageInputCounter, submitMessageInputDraft, } from './message-input-state'; import { MessageInputView } from './message-input-view'; +import { settleVoiceInputBeforeSubmit } from '@/lib/voice-input/voice-input-submit'; const MESSAGE_INPUT_FOCUS_RESTORE_DELAY_MS = 100; const EMPTY_READY_ATTACHMENT_BLOCKS: readonly AttachmentBlock[] = []; @@ -99,8 +99,7 @@ export function MessageInputContent({ const controlsDisabled = disabled === true || submitDisabled === true; const showAttachmentButton = attachmentQueue !== null && hasAttachmentsCapability && isEditing !== true && disabled !== true; - const voiceDisabled = - disabled === true || hasUploadingAttachment || isEditing === true; + const voiceDisabled = disabled === true || hasUploadingAttachment || isEditing === true; const inputMeasure = useTextHeight({ minHeight: MESSAGE_INPUT_MIN_HEIGHT, maxHeight: MESSAGE_INPUT_MAX_HEIGHT, diff --git a/apps/mobile/src/components/kilo-chat/message-input-state.ts b/apps/mobile/src/components/kilo-chat/message-input-state.ts index 5a47390720..b85d3ca5bd 100644 --- a/apps/mobile/src/components/kilo-chat/message-input-state.ts +++ b/apps/mobile/src/components/kilo-chat/message-input-state.ts @@ -85,29 +85,6 @@ export function clearSubmittedMessageInputDraft({ return cleared; } -/** - * Awaits a voice-input settle callback (e.g. waiting for the final transcript of - * an in-flight listening session) and only invokes `submit` when settle resolves - * truthy. Returning `false` from settle aborts the submit so the caller can - * surface controller-reported feedback. The helper intentionally does not - * swallow rejections: a throw from settle is treated as a programmer or native - * bridge bug and propagated to the caller, again without invoking submit. - */ -export async function settleVoiceInputBeforeSubmit({ - settleVoiceInput, - submit, -}: { - settleVoiceInput: () => Promise; - submit: () => void; -}): Promise { - const settled = await settleVoiceInput(); - if (!settled) { - return false; - } - submit(); - return true; -} - export function applyMessageInputTextChange({ text, valueRef, diff --git a/apps/mobile/src/components/kilo-chat/message-input-voice-submit.test.ts b/apps/mobile/src/components/kilo-chat/message-input-voice-submit.test.ts index de57775bca..27fb0d4b2f 100644 --- a/apps/mobile/src/components/kilo-chat/message-input-voice-submit.test.ts +++ b/apps/mobile/src/components/kilo-chat/message-input-voice-submit.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from 'vitest'; -import { settleVoiceInputBeforeSubmit } from './message-input-state'; +import { settleVoiceInputBeforeSubmit } from '@/lib/voice-input/voice-input-submit'; describe('settleVoiceInputBeforeSubmit', () => { it('settles before submitting', async () => { diff --git a/apps/mobile/src/lib/voice-input/voice-input-submit.test.ts b/apps/mobile/src/lib/voice-input/voice-input-submit.test.ts new file mode 100644 index 0000000000..d3a323cedc --- /dev/null +++ b/apps/mobile/src/lib/voice-input/voice-input-submit.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it, vi } from 'vitest'; + +import { settleVoiceInputBeforeSubmit } from './voice-input-submit'; + +describe('settleVoiceInputBeforeSubmit', () => { + it('settles before submitting and reports true', async () => { + const order: string[] = []; + + await expect( + settleVoiceInputBeforeSubmit({ + settleVoiceInput: vi.fn().mockImplementationOnce(async () => { + await Promise.resolve(); + order.push('settle'); + return true; + }), + submit: () => { + order.push('submit'); + }, + }) + ).resolves.toBe(true); + + expect(order).toEqual(['settle', 'submit']); + }); + + it('does not submit when settlement returns false', async () => { + const submit = vi.fn<() => void>(); + + await expect( + settleVoiceInputBeforeSubmit({ + settleVoiceInput: vi.fn().mockResolvedValueOnce(false), + submit, + }) + ).resolves.toBe(false); + + expect(submit).not.toHaveBeenCalled(); + }); + + it('propagates settlement errors without submitting', async () => { + const failure = new Error('native recognition crashed'); + const submit = vi.fn<() => void>(); + + await expect( + settleVoiceInputBeforeSubmit({ + settleVoiceInput: vi.fn().mockRejectedValueOnce(failure), + submit, + }) + ).rejects.toBe(failure); + + expect(submit).not.toHaveBeenCalled(); + }); + + it('invokes submit exactly once when settlement returns true', async () => { + const submit = vi.fn<() => void>(); + + await expect( + settleVoiceInputBeforeSubmit({ + settleVoiceInput: vi.fn().mockResolvedValueOnce(true), + submit, + }) + ).resolves.toBe(true); + + expect(submit).toHaveBeenCalledTimes(1); + }); +}); diff --git a/apps/mobile/src/lib/voice-input/voice-input-submit.ts b/apps/mobile/src/lib/voice-input/voice-input-submit.ts new file mode 100644 index 0000000000..a1b27b940d --- /dev/null +++ b/apps/mobile/src/lib/voice-input/voice-input-submit.ts @@ -0,0 +1,29 @@ +/** + * Awaits a voice-input settle callback (e.g. waiting for the final transcript + * of an in-flight listening session) and only invokes `submit` when settle + * resolves truthy. Returning `false` from settle aborts the submit so the + * caller can surface controller-reported feedback. The helper intentionally + * does not swallow rejections: a throw from settle is treated as a programmer + * or native bridge bug and propagated to the caller, again without invoking + * submit. + * + * This is a domain-agnostic submit race guarantee. Both the Kilo Chat message + * input and the Cloud Agent chat composer call into `useVoiceInput`'s + * `settleBeforeSubmit` before their respective submit paths so an in-flight + * recognition session can deliver its final transcript before the message is + * flushed. + */ +export async function settleVoiceInputBeforeSubmit({ + settleVoiceInput, + submit, +}: { + settleVoiceInput: () => Promise; + submit: () => void; +}): Promise { + const settled = await settleVoiceInput(); + if (!settled) { + return false; + } + submit(); + return true; +} From ffce7beab2ada9829310e410071f5751e5c8ac8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20=C5=A0=C4=87eki=C4=87?= Date: Tue, 14 Jul 2026 16:04:46 +0200 Subject: [PATCH 09/14] feat(mobile): add voice input to new sessions --- apps/mobile/src/app/(app)/agent-chat/new.tsx | 379 +++++------------- .../agents/new-session-prompt-state.test.ts | 121 ++++++ .../agents/new-session-prompt-state.ts | 66 +++ .../components/agents/new-session-prompt.tsx | 247 ++++++++++++ .../agents/new-session-repository-section.tsx | 98 +++++ .../agents/use-new-session-creator.ts | 145 +++++++ 6 files changed, 767 insertions(+), 289 deletions(-) create mode 100644 apps/mobile/src/components/agents/new-session-prompt-state.test.ts create mode 100644 apps/mobile/src/components/agents/new-session-prompt-state.ts create mode 100644 apps/mobile/src/components/agents/new-session-prompt.tsx create mode 100644 apps/mobile/src/components/agents/new-session-repository-section.tsx create mode 100644 apps/mobile/src/components/agents/use-new-session-creator.ts diff --git a/apps/mobile/src/app/(app)/agent-chat/new.tsx b/apps/mobile/src/app/(app)/agent-chat/new.tsx index 0205427387..f11795d998 100644 --- a/apps/mobile/src/app/(app)/agent-chat/new.tsx +++ b/apps/mobile/src/app/(app)/agent-chat/new.tsx @@ -1,78 +1,36 @@ -/* eslint-disable max-lines -- New-session screen bundles closely related prompt/toolbar/repository concerns in a single component to keep navigation props colocated. */ import { useCallback, useMemo, useRef, useState } from 'react'; -import { - ActivityIndicator, - type LayoutChangeEvent, - Platform, - Pressable, - ScrollView, - TextInput, - type TextStyle, - View, -} from 'react-native'; -import { type Href, useLocalSearchParams, useNavigation, useRouter } from 'expo-router'; +import { ActivityIndicator, ScrollView, View } from 'react-native'; +import { useLocalSearchParams } from 'expo-router'; import { useActionSheet } from '@expo/react-native-action-sheet'; -import { useQuery, useQueryClient } from '@tanstack/react-query'; -import { generateMessageId } from 'cloud-agent-sdk/message-id'; -import * as Haptics from 'expo-haptics'; +import { useQuery } from '@tanstack/react-query'; import * as WebBrowser from 'expo-web-browser'; -import { ExternalLink, Paperclip, RefreshCw } from 'lucide-react-native'; import { toast } from 'sonner-native'; -import { AttachmentPreviewStrip } from '@/components/agents/attachment-preview-strip'; +import { NewSessionPrompt } from '@/components/agents/new-session-prompt'; +import { NewSessionRepositorySection } from '@/components/agents/new-session-repository-section'; +import { useNewSessionCreator } from '@/components/agents/use-new-session-creator'; import { pickAgentAttachments } from '@/components/agents/attachment-picker'; -import { ChatToolbar } from '@/components/agents/chat-toolbar'; import { type AgentMode } from '@/components/agents/mode-selector'; -import { RepoSelector } from '@/components/agents/repo-selector'; -import { useTextHeight } from '@/components/agents/use-text-height'; import { Button } from '@/components/ui/button'; import { Text } from '@/components/ui/text'; -import { QueryError } from '@/components/query-error'; import { ScreenHeader } from '@/components/screen-header'; -import { invalidateAgentSessionQueries } from '@/lib/agent-session-cache'; import { getGitHubIntegrationUrl, shouldShowGitHubIntegrationPrompt, } from '@/lib/agent-github-integration'; import { AGENT_ATTACHMENT_MAX_FILES } from '@/lib/agent-attachments/constants'; -import { captureEvent, SESSION_CREATED_EVENT } from '@/lib/analytics/posthog'; -import { - type AgentAttachmentWire, - useAgentAttachmentUpload, -} from '@/lib/agent-attachments/use-agent-attachment-upload'; +import { useAgentAttachmentUpload } from '@/lib/agent-attachments/use-agent-attachment-upload'; import { WEB_BASE_URL } from '@/lib/config'; import { useAvailableModels } from '@/lib/hooks/use-available-models'; import { useAutoSelectModel } from '@/lib/hooks/use-auto-select-model'; import { useModelPreferences } from '@/lib/hooks/use-model-preferences'; import { usePersistedAgentModel } from '@/lib/hooks/use-persisted-agent-model'; import { useThemeColors } from '@/lib/hooks/use-theme-colors'; -import { trpcClient, useTRPC } from '@/lib/trpc'; -import { cn } from '@/lib/utils'; - -const PROMPT_INPUT_DEFAULT_LINES = 3; -const PROMPT_INPUT_MAX_LINES = 6; -const PROMPT_INPUT_LINE_HEIGHT = 24; -// Must mirror the TextInput's actual padding: py-2 (16 total) and px-2 on -// iOS (16 total) / the 24pt-per-side Android inset (48 total). -const PROMPT_INPUT_VERTICAL_PADDING = 16; -const PROMPT_INPUT_HORIZONTAL_PADDING = Platform.OS === 'android' ? 48 : 16; -const PROMPT_INPUT_ANDROID_HORIZONTAL_INSET = 24; -const PROMPT_INPUT_MIN_HEIGHT = - PROMPT_INPUT_LINE_HEIGHT * PROMPT_INPUT_DEFAULT_LINES + PROMPT_INPUT_VERTICAL_PADDING; -const PROMPT_INPUT_MAX_HEIGHT = - PROMPT_INPUT_LINE_HEIGHT * PROMPT_INPUT_MAX_LINES + PROMPT_INPUT_VERTICAL_PADDING; - -const promptInputStyle = { - includeFontPadding: false, - lineHeight: PROMPT_INPUT_LINE_HEIGHT, - textAlignVertical: 'top', -} satisfies TextStyle; +import { useTRPC } from '@/lib/trpc'; +import { settleVoiceInputBeforeSubmit } from '@/lib/voice-input/voice-input-submit'; export default function NewSessionScreen() { - const router = useRouter(); - const navigation = useNavigation(); const colors = useThemeColors(); - const queryClient = useQueryClient(); const { showActionSheetWithOptions } = useActionSheet(); const { organizationId } = useLocalSearchParams<{ organizationId?: string }>(); @@ -82,19 +40,8 @@ export default function NewSessionScreen() { const [variant, setVariant] = useState(''); const [selectedRepo, setSelectedRepo] = useState(''); const [isCreating, setIsCreating] = useState(false); - - // Prompt ref (uncontrolled TextInput on iOS) - const promptRef = useRef(''); const [hasPrompt, setHasPrompt] = useState(false); - const [promptInputWidth, setPromptInputWidth] = useState(0); - const promptMeasure = useTextHeight({ - minHeight: PROMPT_INPUT_MIN_HEIGHT, - maxHeight: PROMPT_INPUT_MAX_HEIGHT, - verticalPadding: PROMPT_INPUT_VERTICAL_PADDING, - textContentWidth: promptInputWidth - PROMPT_INPUT_HORIZONTAL_PADDING, - fontSize: 16, - lineHeight: PROMPT_INPUT_LINE_HEIGHT, - }); + const voiceInputSettlerRef = useRef<(() => Promise) | null>(null); // ── Models ─────────────────────────────────────────────────────── const { @@ -151,6 +98,17 @@ export default function NewSessionScreen() { })); }, [repoData]); + // ── Session creator ────────────────────────────────────────────── + const { createSessionFromDraft, promptRef } = useNewSessionCreator({ + attachments, + mode, + model, + organizationId, + selectedRepo, + setIsCreating, + variant, + }); + // ── Handlers ───────────────────────────────────────────────────── const handleModelSelect = useCallback( (modelId: string, newVariant: string) => { @@ -171,102 +129,43 @@ export default function NewSessionScreen() { } }, [organizationId, refetchRepos]); - const handleCreate = useCallback(async () => { - const prompt = promptRef.current.trim(); - if (prompt.startsWith('/') && attachments.attachments.length > 0) { - toast.error('Attachments cannot be sent with slash commands.'); - return; - } - - setIsCreating(true); - - try { - const initialMessageId = generateMessageId(); - const baseInput: { - prompt: string; - initialMessageId: string; - mode: AgentMode; - model: string; - variant: string | undefined; - githubRepo: string; - autoCommit: boolean; - autoInitiate: boolean; - attachments?: AgentAttachmentWire; - } = { - prompt, - initialMessageId, - mode, - model, - variant: variant || undefined, - githubRepo: selectedRepo, - autoCommit: true, - autoInitiate: true, - }; - const wireAttachments = attachments.toWirePayload(); - if (wireAttachments) { - baseInput.attachments = wireAttachments; - } + function handlePromptChange(text: string) { + promptRef.current = text; + const nextHasPrompt = text.trim().length > 0; + setHasPrompt(current => (current === nextHasPrompt ? current : nextHasPrompt)); + } - const result = organizationId - ? await trpcClient.organizations.cloudAgentNext.prepareSession.mutate({ - ...baseInput, - organizationId, - }) - : await trpcClient.cloudAgentNext.prepareSession.mutate(baseInput); - - captureEvent(SESSION_CREATED_EVENT, { surface: 'cloud-agent' }); - await invalidateAgentSessionQueries(queryClient, trpc); - void Haptics.notificationAsync(Haptics.NotificationFeedbackType.Success); - const path = organizationId - ? `/(app)/agent-chat/${result.kiloSessionId}?organizationId=${organizationId}` - : `/(app)/agent-chat/${result.kiloSessionId}`; - router.push(path as Href); - requestAnimationFrame(() => { - navigation.dispatch(state => { - const routes = state.routes.filter((r: { name: string }) => r.name !== 'agent-chat/new'); - return { - type: 'RESET' as const, - payload: { ...state, routes, index: routes.length - 1 }, - }; - }); - }); - } catch (error) { - const message = error instanceof Error ? error.message : 'Failed to create session'; - toast.error(message); - } finally { - setIsCreating(false); - } - }, [ - selectedRepo, - model, - mode, - variant, - organizationId, - queryClient, - trpc, - router, - navigation, - attachments, - ]); + const submitCreate = useCallback(async () => { + await settleVoiceInputBeforeSubmit({ + settleVoiceInput: async () => { + const settleVoiceInput = voiceInputSettlerRef.current; + if (settleVoiceInput === null) { + return true; + } + const settled = await settleVoiceInput(); + return settled; + }, + submit: () => { + void createSessionFromDraft(); + }, + }); + }, [createSessionFromDraft]); + + const handleStartSession = useCallback(() => { + void submitCreate(); + }, [submitCreate]); const { addCandidates } = attachments; const handleAddAttachment = useCallback(async () => { addCandidates(await pickAgentAttachments(showActionSheetWithOptions)); }, [addCandidates, showActionSheetWithOptions]); - function handlePromptInputLayout(event: LayoutChangeEvent) { - const nextWidth = Math.max(Math.round(event.nativeEvent.layout.width), 0); - setPromptInputWidth(current => (current === nextWidth ? current : nextWidth)); - } - const canCreate = hasPrompt && Boolean(selectedRepo) && Boolean(model) && !attachments.isUploading && !attachments.hasFailedAttachments; - const paperclipDisabled = - isCreating || attachments.attachments.length >= AGENT_ATTACHMENT_MAX_FILES; return ( @@ -278,154 +177,56 @@ export default function NewSessionScreen() { keyboardShouldPersistTaps="handled" automaticallyAdjustKeyboardInsets > - - { - attachments.removeAttachment(id); - }} - onRetry={id => { - attachments.retryAttachment(id); - }} - /> - - { - void handleAddAttachment(); - }} - disabled={paperclipDisabled} - hitSlop={{ top: 8, bottom: 8, left: 8, right: 8 }} - className={cn( - 'h-9 w-9 items-center justify-center rounded-full active:opacity-70', - paperclipDisabled && 'opacity-50' - )} - accessibilityRole="button" - accessibilityLabel="Add attachment" - accessibilityState={{ disabled: paperclipDisabled }} - > - - - {promptMeasure.measureElement} - { - promptRef.current = text; - promptMeasure.setText(text); - const nextHasPrompt = text.trim().length > 0; - setHasPrompt(current => (current === nextHasPrompt ? current : nextHasPrompt)); - }} - onLayout={handlePromptInputLayout} - scrollEnabled={promptMeasure.height >= PROMPT_INPUT_MAX_HEIGHT} - editable={!isCreating} - accessibilityState={{ disabled: isCreating }} - autoFocus - /> - - {isModelsError && models.length === 0 ? ( - void refetchModels()} - className="border-t border-border py-4" - /> - ) : ( - - )} - - - - Repository - {isReposError && repoData === undefined ? ( - void refetchRepos()} - isRetrying={isRefetchingRepos} - /> - ) : ( - <> - - {showGitHubIntegrationPrompt ? ( - - - Connect GitHub - - Connect GitHub in your browser, then return here to pick a repository. - - - - - - - - ) : null} - - )} - + { + void handleAddAttachment(); + }} + onRemoveAttachment={id => { + attachments.removeAttachment(id); + }} + onRetryAttachment={id => { + attachments.retryAttachment(id); + }} + onRefetchModels={() => { + void refetchModels(); + }} + voiceInputSettlerRef={voiceInputSettlerRef} + /> + + { + void handleOpenGitHubIntegration(); + }} + onRefetch={() => { + void refetchRepos(); + }} + repositories={repositories} + showGitHubIntegrationPrompt={showGitHubIntegrationPrompt} + value={selectedRepo} + /> + + + + ) : null} + + )} + + ); +} diff --git a/apps/mobile/src/components/agents/use-new-session-creator.ts b/apps/mobile/src/components/agents/use-new-session-creator.ts new file mode 100644 index 0000000000..c9f5a47654 --- /dev/null +++ b/apps/mobile/src/components/agents/use-new-session-creator.ts @@ -0,0 +1,145 @@ +import { type RefObject, useCallback, useRef } from 'react'; +import { type Href, useNavigation, useRouter } from 'expo-router'; +import { useQueryClient } from '@tanstack/react-query'; +import { generateMessageId } from 'cloud-agent-sdk/message-id'; +import * as Haptics from 'expo-haptics'; +import { toast } from 'sonner-native'; + +import { type AgentMode } from '@/components/agents/mode-selector'; +import { resolveNewSessionPromptForCreate } from '@/components/agents/new-session-prompt-state'; +import { invalidateAgentSessionQueries } from '@/lib/agent-session-cache'; +import { captureEvent, SESSION_CREATED_EVENT } from '@/lib/analytics/posthog'; +import { + type AgentAttachmentWire, + type useAgentAttachmentUpload, +} from '@/lib/agent-attachments/use-agent-attachment-upload'; +import { trpcClient, useTRPC } from '@/lib/trpc'; + +type UseNewSessionCreatorInput = { + attachments: ReturnType; + mode: AgentMode; + model: string; + organizationId?: string; + selectedRepo: string; + setIsCreating: (value: boolean) => void; + variant: string; +}; + +type UseNewSessionCreatorResult = { + createSessionFromDraft: () => Promise; + promptRef: RefObject; +}; + +/** + * Owns the side effects of starting a new Cloud Agent session: validating + * the draft, calling the tRPC `prepareSession` mutation, navigating to the + * session, and reporting the analytics event. The route supplies the live + * draft through `promptRef` so the caller can read the post-settle value + * without re-rendering the parent. + */ +export function useNewSessionCreator({ + attachments, + mode, + model, + organizationId, + selectedRepo, + setIsCreating, + variant, +}: UseNewSessionCreatorInput): UseNewSessionCreatorResult { + const router = useRouter(); + const navigation = useNavigation(); + const queryClient = useQueryClient(); + const trpc = useTRPC(); + const promptRef = useRef(''); + + const createSessionFromDraft = useCallback(async () => { + // Read the live, post-settlement draft (see `settleVoiceInputBeforeSubmit` + // in `useNewSessionCreator` callers). An interim voice transcript can be + // replaced by an empty final transcript when no speech was recognized; + // reject empty/whitespace drafts before doing anything else so we never + // call prepareSession with an empty prompt. The voice controller has + // already presented its own feedback, so a no-op here preserves the + // user's draft and screen state without toasting. + const prompt = resolveNewSessionPromptForCreate(promptRef.current); + if (prompt === null) { + return; + } + if (prompt.startsWith('/') && attachments.attachments.length > 0) { + toast.error('Attachments cannot be sent with slash commands.'); + return; + } + + setIsCreating(true); + + try { + const initialMessageId = generateMessageId(); + const baseInput: { + prompt: string; + initialMessageId: string; + mode: AgentMode; + model: string; + variant: string | undefined; + githubRepo: string; + autoCommit: boolean; + autoInitiate: boolean; + attachments?: AgentAttachmentWire; + } = { + prompt, + initialMessageId, + mode, + model, + variant: variant || undefined, + githubRepo: selectedRepo, + autoCommit: true, + autoInitiate: true, + }; + const wireAttachments = attachments.toWirePayload(); + if (wireAttachments) { + baseInput.attachments = wireAttachments; + } + + const result = organizationId + ? await trpcClient.organizations.cloudAgentNext.prepareSession.mutate({ + ...baseInput, + organizationId, + }) + : await trpcClient.cloudAgentNext.prepareSession.mutate(baseInput); + + captureEvent(SESSION_CREATED_EVENT, { surface: 'cloud-agent' }); + await invalidateAgentSessionQueries(queryClient, trpc); + void Haptics.notificationAsync(Haptics.NotificationFeedbackType.Success); + const path = organizationId + ? `/(app)/agent-chat/${result.kiloSessionId}?organizationId=${organizationId}` + : `/(app)/agent-chat/${result.kiloSessionId}`; + router.push(path as Href); + requestAnimationFrame(() => { + navigation.dispatch(state => { + const routes = state.routes.filter((r: { name: string }) => r.name !== 'agent-chat/new'); + return { + type: 'RESET' as const, + payload: { ...state, routes, index: routes.length - 1 }, + }; + }); + }); + } catch (error) { + const message = error instanceof Error ? error.message : 'Failed to create session'; + toast.error(message); + } finally { + setIsCreating(false); + } + }, [ + selectedRepo, + model, + mode, + variant, + organizationId, + queryClient, + trpc, + router, + navigation, + attachments, + setIsCreating, + ]); + + return { createSessionFromDraft, promptRef }; +} From b0536436c15de7b4248005b36197a9dd28334e9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20=C5=A0=C4=87eki=C4=87?= Date: Tue, 14 Jul 2026 16:05:54 +0200 Subject: [PATCH 10/14] style(mobile): format voice input files --- apps/mobile/src/components/kilo-chat/message-input-view.tsx | 5 ++++- .../src/lib/voice-input/use-voice-input-feedback.test.ts | 5 ++++- apps/mobile/src/lib/voice-input/use-voice-input.test.ts | 1 - 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/apps/mobile/src/components/kilo-chat/message-input-view.tsx b/apps/mobile/src/components/kilo-chat/message-input-view.tsx index c654ae2140..5ee70025a9 100644 --- a/apps/mobile/src/components/kilo-chat/message-input-view.tsx +++ b/apps/mobile/src/components/kilo-chat/message-input-view.tsx @@ -8,7 +8,10 @@ import { Text } from '@/components/ui/text'; import { useThemeColors } from '@/lib/hooks/use-theme-colors'; import { type VoiceInputStatus } from '@/lib/voice-input/voice-input-state'; import { cn } from '@/lib/utils'; -import { VoiceInputButton, VoiceInputStatus as VoiceInputStatusView } from '@/components/voice-input-control'; +import { + VoiceInputButton, + VoiceInputStatus as VoiceInputStatusView, +} from '@/components/voice-input-control'; import { MessageAttachmentPreviewStrip } from './message-attachment-preview-strip'; import { messageInputKeyboardProps, messageInputTextStyle } from './message-input-layout'; import { type ComposerAttachmentQueue } from './message-input-types'; diff --git a/apps/mobile/src/lib/voice-input/use-voice-input-feedback.test.ts b/apps/mobile/src/lib/voice-input/use-voice-input-feedback.test.ts index cea87e80be..2cdee1a704 100644 --- a/apps/mobile/src/lib/voice-input/use-voice-input-feedback.test.ts +++ b/apps/mobile/src/lib/voice-input/use-voice-input-feedback.test.ts @@ -39,7 +39,10 @@ describe('voice input feedback side effects', () => { expect(alertMock.alert).toHaveBeenCalledWith( 'Microphone access is off', 'Microphone access is off. Enable it in Settings.', - [{ text: 'Cancel', style: 'cancel' }, { text: 'Open Settings', onPress: expect.any(Function) }] + [ + { text: 'Cancel', style: 'cancel' }, + { text: 'Open Settings', onPress: expect.any(Function) }, + ] ); buttons?.find(button => button.text === 'Open Settings')?.onPress?.(); expect(linkingMock.openSettings).toHaveBeenCalled(); diff --git a/apps/mobile/src/lib/voice-input/use-voice-input.test.ts b/apps/mobile/src/lib/voice-input/use-voice-input.test.ts index b5599916b5..00b063c9b4 100644 --- a/apps/mobile/src/lib/voice-input/use-voice-input.test.ts +++ b/apps/mobile/src/lib/voice-input/use-voice-input.test.ts @@ -268,5 +268,4 @@ describe('useVoiceInput integration', () => { ).toBe(false); }); }); - }); From 6a88fa969e31c65960c07498c3713cea536124b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20=C5=A0=C4=87eki=C4=87?= Date: Tue, 14 Jul 2026 16:23:33 +0200 Subject: [PATCH 11/14] fix(mobile): harden voice input errors --- .../message-input-voice-submit.test.ts | 51 ------------------- .../src/components/voice-input-control.tsx | 2 +- .../voice-input-controller-results.test.ts | 21 ++++++++ .../lib/voice-input/voice-input-controller.ts | 3 +- .../lib/voice-input/voice-input-state.test.ts | 9 ++++ .../src/lib/voice-input/voice-input-state.ts | 12 +++-- 6 files changed, 39 insertions(+), 59 deletions(-) delete mode 100644 apps/mobile/src/components/kilo-chat/message-input-voice-submit.test.ts diff --git a/apps/mobile/src/components/kilo-chat/message-input-voice-submit.test.ts b/apps/mobile/src/components/kilo-chat/message-input-voice-submit.test.ts deleted file mode 100644 index 27fb0d4b2f..0000000000 --- a/apps/mobile/src/components/kilo-chat/message-input-voice-submit.test.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { describe, expect, it, vi } from 'vitest'; - -import { settleVoiceInputBeforeSubmit } from '@/lib/voice-input/voice-input-submit'; - -describe('settleVoiceInputBeforeSubmit', () => { - it('settles before submitting', async () => { - const order: string[] = []; - - await expect( - settleVoiceInputBeforeSubmit({ - settleVoiceInput: vi.fn().mockImplementationOnce(async () => { - await Promise.resolve(); - order.push('settle'); - return true; - }), - submit: () => { - order.push('submit'); - }, - }) - ).resolves.toBe(true); - - expect(order).toEqual(['settle', 'submit']); - }); - - it('does not submit when settlement fails', async () => { - const submit = vi.fn<() => void>(); - - await expect( - settleVoiceInputBeforeSubmit({ - settleVoiceInput: vi.fn().mockResolvedValueOnce(false), - submit, - }) - ).resolves.toBe(false); - - expect(submit).not.toHaveBeenCalled(); - }); - - it('propagates settlement errors without submitting', async () => { - const failure = new Error('native recognition crashed'); - const submit = vi.fn<() => void>(); - - await expect( - settleVoiceInputBeforeSubmit({ - settleVoiceInput: vi.fn().mockRejectedValueOnce(failure), - submit, - }) - ).rejects.toBe(failure); - - expect(submit).not.toHaveBeenCalled(); - }); -}); diff --git a/apps/mobile/src/components/voice-input-control.tsx b/apps/mobile/src/components/voice-input-control.tsx index 2adc8c6eed..620f26795b 100644 --- a/apps/mobile/src/components/voice-input-control.tsx +++ b/apps/mobile/src/components/voice-input-control.tsx @@ -13,7 +13,7 @@ type VoiceInputButtonProps = { }; const ICON_SIZE = 18; -const HIT_SLOP = 2; +const HIT_SLOP = { top: 2, bottom: 2, left: 2, right: 2 } as const; const LISTENING_BG = 'bg-red-600 dark:bg-red-500'; const RESTING_BG = 'bg-secondary'; diff --git a/apps/mobile/src/lib/voice-input/voice-input-controller-results.test.ts b/apps/mobile/src/lib/voice-input/voice-input-controller-results.test.ts index 30cfe9843d..35356335a7 100644 --- a/apps/mobile/src/lib/voice-input/voice-input-controller-results.test.ts +++ b/apps/mobile/src/lib/voice-input/voice-input-controller-results.test.ts @@ -135,6 +135,27 @@ describe('createVoiceInputController - events and serialization', () => { ]); }); + it('maps an unknown native error code to a generic retryable feedback without throwing', async () => { + const { harness, controller } = build(); + const fb = recordFeedback(); + await controller.start(makeStartOptions({ onFeedback: fb.onFeedback })); + harness.emit('start', null); + + harness.emit('error', { + error: 'future-native-error', + message: 'future', + }); + + expect(fb.feedback).toEqual([ + { + action: 'none', + availability: 'available', + message: 'Voice input stopped. Tap the microphone to try again.', + retryable: true, + }, + ]); + }); + it('permanently flips availability to unavailable when a service-not-allowed feedback is reported', async () => { const { harness, controller } = build(); const fb = recordFeedback(); diff --git a/apps/mobile/src/lib/voice-input/voice-input-controller.ts b/apps/mobile/src/lib/voice-input/voice-input-controller.ts index 6319d46f5c..36e1fd9f41 100644 --- a/apps/mobile/src/lib/voice-input/voice-input-controller.ts +++ b/apps/mobile/src/lib/voice-input/voice-input-controller.ts @@ -1,5 +1,4 @@ import { - type ExpoSpeechRecognitionErrorEvent, type ExpoSpeechRecognitionResultEvent, } from 'expo-speech-recognition'; @@ -27,7 +26,7 @@ export type VoiceInputNativeEvent = { start: null; result: ExpoSpeechRecognitionResultEvent; nomatch: null; - error: ExpoSpeechRecognitionErrorEvent; + error: { code?: number; error: string; message: string }; end: null; }; diff --git a/apps/mobile/src/lib/voice-input/voice-input-state.test.ts b/apps/mobile/src/lib/voice-input/voice-input-state.test.ts index 0a548f12d1..ef7b0ea46e 100644 --- a/apps/mobile/src/lib/voice-input/voice-input-state.test.ts +++ b/apps/mobile/src/lib/voice-input/voice-input-state.test.ts @@ -246,6 +246,15 @@ describe('classifyVoiceInputError', () => { retryable: false, }); }); + + it('maps an unknown error string to a generic retryable available feedback', () => { + expect(classifyVoiceInputError('future-native-error')).toEqual({ + action: 'none', + availability: 'available', + message: 'Voice input stopped. Tap the microphone to try again.', + retryable: true, + }); + }); }); describe('shouldAbortVoiceInput', () => { diff --git a/apps/mobile/src/lib/voice-input/voice-input-state.ts b/apps/mobile/src/lib/voice-input/voice-input-state.ts index bd2d07ebe8..5281100124 100644 --- a/apps/mobile/src/lib/voice-input/voice-input-state.ts +++ b/apps/mobile/src/lib/voice-input/voice-input-state.ts @@ -1,5 +1,3 @@ -import { type ExpoSpeechRecognitionErrorCode } from 'expo-speech-recognition'; - export type VoiceInputStatus = 'idle' | 'starting' | 'listening' | 'stopping'; export type VoiceInputAvailability = 'available' | 'unavailable'; @@ -111,7 +109,7 @@ export function classifyVoiceInputPermission( }; } -export function classifyVoiceInputError(code: ExpoSpeechRecognitionErrorCode): VoiceInputFeedback { +export function classifyVoiceInputError(code: string): VoiceInputFeedback { switch (code) { case 'no-speech': case 'speech-timeout': { @@ -176,8 +174,12 @@ export function classifyVoiceInputError(code: ExpoSpeechRecognitionErrorCode): V }; } default: { - const unhandled: never = code; - throw new Error(`Unhandled ExpoSpeechRecognitionErrorCode: ${String(unhandled)}`); + return { + action: 'none', + availability: 'available', + message: 'Voice input stopped. Tap the microphone to try again.', + retryable: true, + }; } } } From a4ad1f1d6bca505a4711d8159e90cf2110d649e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20=C5=A0=C4=87eki=C4=87?= Date: Tue, 14 Jul 2026 17:23:29 +0200 Subject: [PATCH 12/14] fix(mobile): cache voice input snapshots --- .../voice-input-controller-snapshot.test.ts | 51 +++++++++++++++++++ .../lib/voice-input/voice-input-controller.ts | 18 +++++-- 2 files changed, 64 insertions(+), 5 deletions(-) create mode 100644 apps/mobile/src/lib/voice-input/voice-input-controller-snapshot.test.ts diff --git a/apps/mobile/src/lib/voice-input/voice-input-controller-snapshot.test.ts b/apps/mobile/src/lib/voice-input/voice-input-controller-snapshot.test.ts new file mode 100644 index 0000000000..2eadc1e7af --- /dev/null +++ b/apps/mobile/src/lib/voice-input/voice-input-controller-snapshot.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from 'vitest'; + +import { createVoiceInputController } from './voice-input-controller'; +import { + createVoiceInputNativeHarness, + makeStartOptions, + type VoiceInputNativeHarness, +} from './voice-input-controller-test-helpers'; + +describe('createVoiceInputController - snapshot identity', () => { + const build = (overrides: Partial = {}) => { + const harness = createVoiceInputNativeHarness(overrides); + const controller = createVoiceInputController(harness.native); + return { harness, controller }; + }; + + it('returns the same snapshot object from consecutive getSnapshot() calls without a state transition', () => { + const { controller } = build(); + + const first = controller.getSnapshot(); + const second = controller.getSnapshot(); + + expect(second).toBe(first); + }); + + it('changes snapshot identity after a state transition and keeps it stable until the next transition', async () => { + const { harness, controller } = build(); + + const idleSnapshot = controller.getSnapshot(); + expect(idleSnapshot.status).toBe('idle'); + + const started = await controller.start(makeStartOptions({ owner: 'A' })); + expect(started).toBe(true); + + const startingSnapshot = controller.getSnapshot(); + expect(startingSnapshot).not.toBe(idleSnapshot); + expect(startingSnapshot.status).toBe('starting'); + expect(startingSnapshot.owner).toBe('A'); + + // Same object after the transition until the next one. + expect(controller.getSnapshot()).toBe(startingSnapshot); + + harness.emit('start', null); + + const listeningSnapshot = controller.getSnapshot(); + expect(listeningSnapshot).not.toBe(startingSnapshot); + expect(listeningSnapshot.status).toBe('listening'); + + expect(controller.getSnapshot()).toBe(listeningSnapshot); + }); +}); diff --git a/apps/mobile/src/lib/voice-input/voice-input-controller.ts b/apps/mobile/src/lib/voice-input/voice-input-controller.ts index 36e1fd9f41..ab7d9b41bc 100644 --- a/apps/mobile/src/lib/voice-input/voice-input-controller.ts +++ b/apps/mobile/src/lib/voice-input/voice-input-controller.ts @@ -1,6 +1,4 @@ -import { - type ExpoSpeechRecognitionResultEvent, -} from 'expo-speech-recognition'; +import { type ExpoSpeechRecognitionResultEvent } from 'expo-speech-recognition'; import { classifyVoiceInputError, @@ -92,8 +90,18 @@ export function createVoiceInputController(native: VoiceInputNative): VoiceInput let session: VoiceInputSession | null = null; const lifecycle: Lifecycle = { disposed: false }; + let snapshot: VoiceInputControllerSnapshot = { availability, owner, status }; + const notify = (): void => { - const snapshot: VoiceInputControllerSnapshot = { availability, owner, status }; + const next: VoiceInputControllerSnapshot = { availability, owner, status }; + if ( + next.availability === snapshot.availability && + next.owner === snapshot.owner && + next.status === snapshot.status + ) { + return; + } + snapshot = next; for (const subscriber of subscribers) { subscriber(snapshot); } @@ -301,7 +309,7 @@ export function createVoiceInputController(native: VoiceInputNative): VoiceInput return { abort, dispose, - getSnapshot: () => ({ availability, owner, status }), + getSnapshot: () => snapshot, start, stop, subscribe, From 39db7b5a6b4a953e9c040840dd5362206b6e666d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20=C5=A0=C4=87eki=C4=87?= Date: Tue, 14 Jul 2026 21:39:57 +0200 Subject: [PATCH 13/14] fix(mobile): close voice input races --- apps/mobile/src/app/(app)/agent-chat/new.tsx | 10 ++- .../src/components/agents/chat-composer.tsx | 10 +-- .../agents/new-session-repository-section.tsx | 7 +- .../new-session-repository-state.test.ts | 13 +++ .../agents/new-session-repository-state.ts | 9 +++ .../use-conversation-message-controller.ts | 64 +++++++-------- .../kilo-chat/message-input-content.tsx | 27 ++++--- .../kilo-chat/message-input-state.test.ts | 8 +- .../kilo-chat/message-input-state.ts | 24 +++--- .../kilo-chat/message-input-types.ts | 4 +- .../kilo-chat/message-input-view.tsx | 4 +- .../voice-input/use-voice-input-actions.ts | 5 -- .../lib/voice-input/use-voice-input.test.ts | 5 +- .../voice-input-controller-helpers.ts | 48 +++++++++++ ...ice-input-controller-pending-start.test.ts | 79 +++++++++++++++++++ .../lib/voice-input/voice-input-controller.ts | 45 +++++++---- .../voice-input/voice-input-submit.test.ts | 70 ++++++++++++++++ .../src/lib/voice-input/voice-input-submit.ts | 32 +++++--- 18 files changed, 355 insertions(+), 109 deletions(-) create mode 100644 apps/mobile/src/components/agents/new-session-repository-state.test.ts create mode 100644 apps/mobile/src/components/agents/new-session-repository-state.ts create mode 100644 apps/mobile/src/lib/voice-input/voice-input-controller-pending-start.test.ts diff --git a/apps/mobile/src/app/(app)/agent-chat/new.tsx b/apps/mobile/src/app/(app)/agent-chat/new.tsx index f11795d998..c33f965c7d 100644 --- a/apps/mobile/src/app/(app)/agent-chat/new.tsx +++ b/apps/mobile/src/app/(app)/agent-chat/new.tsx @@ -40,7 +40,9 @@ export default function NewSessionScreen() { const [variant, setVariant] = useState(''); const [selectedRepo, setSelectedRepo] = useState(''); const [isCreating, setIsCreating] = useState(false); + const [isSubmitting, setIsSubmitting] = useState(false); const [hasPrompt, setHasPrompt] = useState(false); + const submissionLockRef = useRef(false); const voiceInputSettlerRef = useRef<(() => Promise) | null>(null); // ── Models ─────────────────────────────────────────────────────── @@ -137,6 +139,8 @@ export default function NewSessionScreen() { const submitCreate = useCallback(async () => { await settleVoiceInputBeforeSubmit({ + lock: submissionLockRef, + onPendingChange: setIsSubmitting, settleVoiceInput: async () => { const settleVoiceInput = voiceInputSettlerRef.current; if (settleVoiceInput === null) { @@ -145,9 +149,7 @@ export default function NewSessionScreen() { const settled = await settleVoiceInput(); return settled; }, - submit: () => { - void createSessionFromDraft(); - }, + submit: createSessionFromDraft, }); }, [createSessionFromDraft]); @@ -225,7 +227,7 @@ export default function NewSessionScreen() {