diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml new file mode 100644 index 0000000..0c44d0d --- /dev/null +++ b/android/app/src/main/AndroidManifest.xml @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/app/i18n/api.ts b/app/i18n/api.ts index 4b8b2e7..9317ae5 100644 --- a/app/i18n/api.ts +++ b/app/i18n/api.ts @@ -76,5 +76,6 @@ export class CachedTranslator extends Translator { const tr2 = await super.translate(text, target); const key2 = `${this.source}::${targetKey}::${text}` await cache.set(key2, tr2); + return tr2; } } \ No newline at end of file diff --git a/app/index.tsx b/app/index.tsx index ff5b1fe..851a26e 100644 --- a/app/index.tsx +++ b/app/index.tsx @@ -32,6 +32,10 @@ export default function Home() { ); } + function onGoBack() { + setConversation(undefined); + } + return ( Home Screen {conversation ? ( - + ) : ( )} diff --git a/app/lib/conversation.ts b/app/lib/conversation.ts index ad77239..dfa6512 100644 --- a/app/lib/conversation.ts +++ b/app/lib/conversation.ts @@ -14,9 +14,10 @@ export class Message { constructor (public conversation : Conversation, public speaker : Speaker, public text? : string) {} - public async translate(translator : Translator, language? : string) { + public async translate() { + const translator = this.conversation.translator if (!this.text) throw new Error("No text") - this.translation = await translator.translate(this.text, language); + this.translation = await translator.translate(this.text, this.otherLanguage); } get otherSpeaker() { @@ -34,7 +35,7 @@ export class Conversation extends Array { public onTranslationDone? : (conversation : Conversation) => any; constructor ( - private translator : Translator, + public translator : Translator, public host : Speaker, public guest : Speaker, ) { @@ -48,7 +49,7 @@ export class Conversation extends Array { public async translateMessage(i : number) { if (!this[i]) throw new Error(`${i} is not a valid message number`); console.log(`Translating sentence to %s: %s`, this[i].otherLanguage, this[i].text) - await this[i].translate(this.translator, this[i].otherLanguage); + await this[i].translate(); } get lastMessage() { diff --git a/components/ConversationThread.tsx b/components/ConversationThread.tsx index ebc175c..1d2ca4d 100644 --- a/components/ConversationThread.tsx +++ b/components/ConversationThread.tsx @@ -1,20 +1,41 @@ -import React, { useState, useEffect } from 'react'; -import { View } from 'react-native'; -import { Conversation, Message } from '@/app/lib/conversation'; -import MessageBubble from '@/components/ui/MessageBubble'; -import { NavigationProp, ParamListBase } from '@react-navigation/native'; -import { language_matrix_entry, Translator } from '@/app/i18n/api'; -import { getDb } from '@/app/lib/db'; +import React, { useState, useEffect } from "react"; +import { ScrollView, Text, TouchableHighlight, View } from "react-native"; +import { Conversation, Message } from "@/app/lib/conversation"; +import MessageBubble from "@/components/ui/MessageBubble"; +import { WhisperContext } from "whisper.rn"; +import { NavigationProp, ParamListBase } from "@react-navigation/native"; +import { + CachedTranslator, + language_matrix_entry, + Translator, +} from "@/app/i18n/api"; +import { getDb } from "@/app/lib/db"; +import LiveAudioStream from 'react-native-live-audio-stream'; + +const lasOptions = { + sampleRate: 32000, // default is 44100 but 32000 is adequate for accurate voice recognition + channels: 1, // 1 or 2, default 1 + bitsPerSample: 16, // 8 or 16, default 16 + audioSource: 6, // android only (see below) + bufferSize: 4096 // default is 2048 +}; +// LiveAudioStream.init(lasOptions as any); interface ConversationThreadProps { conversation: Conversation; + whisperContext: WhisperContext; + onGoBack?: () => any; } -const ConversationThread = (p : ConversationThreadProps) => { +const ConversationThread = (p: ConversationThreadProps) => { const [messages, setMessages] = useState([]); + const [guestSpeak, setGuestSpeak] = useState(); + const [guestSpeakLoaded, setGuestSpeakLoaded] = useState(false); + const ct = new CachedTranslator("en", p.conversation.guest.language); useEffect(() => { - const updateMessages = (c : Conversation) => { + + const updateMessages = (c: Conversation) => { setMessages([...c]); }; @@ -25,19 +46,59 @@ const ConversationThread = (p : ConversationThreadProps) => { p.conversation.onAddMessage = undefined; p.conversation.onTranslationDone = undefined; }; - }, [p.conversation]); + }, [p.conversation, guestSpeak]); - const renderMessages = () => ( + useEffect(() => { + const fetchData = async () => { + setGuestSpeak(await ct.translate("Speak")); + } + + fetchData(); + }, [guestSpeak]) + + const renderMessages = () => messages.map((message, index) => ( - )) - ); + )); + + function onGoBack() { + p.onGoBack && p.onGoBack(); + } return ( - - {renderMessages()} + + + {renderMessages()} + + + + Speak + + + Go Back + + + + {guestSpeak ? guestSpeak : "Speak"} + + + ); }; -export default ConversationThread; \ No newline at end of file +export default ConversationThread; diff --git a/package.json b/package.json index d4f17e5..d17e1aa 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "scripts": { "start": "expo start", "reset-project": "node ./scripts/reset-project.js", - "android": "expo start --android", + "android": "expo start --offline --android", "ios": "expo start --ios", "web": "expo start --offline --web", "test": "jest --watchAll", @@ -36,12 +36,14 @@ "react-native-cache": "^2.0.3", "react-native-country-flag": "^2.0.2", "react-native-gesture-handler": "~2.20.2", + "react-native-live-audio-stream": "^1.1.1", "react-native-reanimated": "~3.16.7", "react-native-safe-area-context": "4.12.0", "react-native-screens": "~4.4.0", "react-native-sqlite-storage": "^6.0.1", "react-native-web": "~0.19.13", - "react-native-webview": "13.12.5" + "react-native-webview": "13.12.5", + "whisper.rn": "^0.3.9" }, "devDependencies": { "@babel/core": "^7.26.7", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b140042..d9c9df1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -77,6 +77,9 @@ dependencies: react-native-gesture-handler: specifier: ~2.20.2 version: 2.20.2(react-native@0.76.6)(react@18.3.1) + react-native-live-audio-stream: + specifier: ^1.1.1 + version: 1.1.1 react-native-reanimated: specifier: ~3.16.7 version: 3.16.7(@babel/core@7.26.7)(react-native@0.76.6)(react@18.3.1) @@ -95,6 +98,9 @@ dependencies: react-native-webview: specifier: 13.12.5 version: 13.12.5(react-native@0.76.6)(react@18.3.1) + whisper.rn: + specifier: ^0.3.9 + version: 0.3.9(react-native@0.76.6)(react@18.3.1) devDependencies: '@babel/core': @@ -1587,7 +1593,7 @@ packages: /@expo/bunyan@4.0.1: resolution: {integrity: sha512-+Lla7nYSiHZirgK+U/uYzsLv/X+HaJienbD5AKX1UQZHYfWaP+9uuQluRB4GrEVWF0GZ7vEVp/jzaOT9k/SQlg==, tarball: https://registry.npmjs.org/@expo/bunyan/-/bunyan-4.0.1.tgz} - engines: {node: '>=0.10.0'} + engines: {'0': node >=0.10.0} dependencies: uuid: 8.3.2 @@ -7030,6 +7036,10 @@ packages: react-native: 0.76.6(@babel/core@7.26.7)(@babel/preset-env@7.26.7)(@types/react@18.3.18)(react@18.3.1) dev: false + /react-native-live-audio-stream@1.1.1: + resolution: {integrity: sha512-Yk0O51hY7eFMUv1umYxGDs4SJVPHyhUX6uz4jI+GiowOwSqIzLLRNh03hJjCVZRFXTWLPCntqOKZ+N8fVAc6BQ==, tarball: https://registry.npmjs.org/react-native-live-audio-stream/-/react-native-live-audio-stream-1.1.1.tgz} + dev: false + /react-native-reanimated@3.16.7(@babel/core@7.26.7)(react-native@0.76.6)(react@18.3.1): resolution: {integrity: sha512-qoUUQOwE1pHlmQ9cXTJ2MX9FQ9eHllopCLiWOkDkp6CER95ZWeXhJCP4cSm6AD4jigL5jHcZf/SkWrg8ttZUsw==, tarball: https://registry.npmjs.org/react-native-reanimated/-/react-native-reanimated-3.16.7.tgz} peerDependencies: @@ -8377,6 +8387,17 @@ packages: dependencies: isexe: 2.0.0 + /whisper.rn@0.3.9(react-native@0.76.6)(react@18.3.1): + resolution: {integrity: sha512-y2hsJ6IpUqtYUZA7YrtGqU3pTXNFzF8Piu8Ch4yAhBB6tyZcEl113e/X10xkd+bVfeIbrSZS2QJZQ4CBfbXS3g==, tarball: https://registry.npmjs.org/whisper.rn/-/whisper.rn-0.3.9.tgz} + engines: {node: '>= 16.0.0'} + peerDependencies: + react: '*' + react-native: '*' + dependencies: + react: 18.3.1 + react-native: 0.76.6(@babel/core@7.26.7)(@babel/preset-env@7.26.7)(@types/react@18.3.18)(react@18.3.1) + dev: false + /wonka@6.3.4: resolution: {integrity: sha512-CjpbqNtBGNAeyNS/9W6q3kSkKE52+FjIj7AkFlLr11s/VWGUu6a2CdYSdGxocIhIVjaW/zchesBQUKPVU69Cqg==, tarball: https://registry.npmjs.org/wonka/-/wonka-6.3.4.tgz}