start to integrate whisper.

android/app/src/main/AndroidManifest.xml

@@ -0,0 +1 @@
+<uses-permission android:name="android.permission.RECORD_AUDIO" />

app/i18n/api.ts

@@ -76,5 +76,6 @@ export class CachedTranslator extends Translator {
         const tr2 = await super.translate(text, target);
         const key2 = `${this.source}::${targetKey}::${text}`
         await cache.set(key2, tr2);
+        return tr2;
     }
 }

app/index.tsx

@@ -32,6 +32,10 @@ export default function Home() {
     );
   }
 
+  function onGoBack() {
+    setConversation(undefined);
+  }
+
   return (
     <View style={styles.container}>
       <Stack.Screen
@@ -46,7 +50,7 @@ export default function Home() {
       />
       <Text>Home Screen</Text>
       {conversation ? (
-        <ConversationThread conversation={conversation} />
+        <ConversationThread conversation={conversation} onGoBack={onGoBack} />
       ) : (
         <LanguageSelection onLangSelected={onLangSelected} />
       )}

app/lib/conversation.ts

@@ -14,9 +14,10 @@ export class Message {
     
     constructor (public conversation : Conversation, public speaker : Speaker, public text? : string) {}
 
-    public async translate(translator : Translator, language? : string) {
+    public async translate() {
+        const translator = this.conversation.translator
         if (!this.text) throw new Error("No text")
-        this.translation = await translator.translate(this.text, language);
+        this.translation = await translator.translate(this.text, this.otherLanguage);
     }
 
     get otherSpeaker() {
@@ -34,7 +35,7 @@ export class Conversation extends Array<Message> {
     public onTranslationDone? : (conversation : Conversation) => any;
 
     constructor (
-        private translator : Translator,
+        public translator : Translator,
         public host : Speaker,
         public guest : Speaker,
     ) {
@@ -48,7 +49,7 @@ export class Conversation extends Array<Message> {
     public async translateMessage(i : number) {
         if (!this[i]) throw new Error(`${i} is not a valid message number`);
         console.log(`Translating sentence to %s: %s`, this[i].otherLanguage, this[i].text)
-        await this[i].translate(this.translator, this[i].otherLanguage);
+        await this[i].translate();
     }
 
     get lastMessage() {

components/ConversationThread.tsx

@@ -1,20 +1,41 @@
-import React, { useState, useEffect } from 'react';
-import { View } from 'react-native';
-import { Conversation, Message } from '@/app/lib/conversation';
-import MessageBubble from '@/components/ui/MessageBubble';
-import { NavigationProp, ParamListBase } from '@react-navigation/native';
-import { language_matrix_entry, Translator } from '@/app/i18n/api';
-import { getDb } from '@/app/lib/db';
+import React, { useState, useEffect } from "react";
+import { ScrollView, Text, TouchableHighlight, View } from "react-native";
+import { Conversation, Message } from "@/app/lib/conversation";
+import MessageBubble from "@/components/ui/MessageBubble";
+import { WhisperContext } from "whisper.rn";
+import { NavigationProp, ParamListBase } from "@react-navigation/native";
+import {
+  CachedTranslator,
+  language_matrix_entry,
+  Translator,
+} from "@/app/i18n/api";
+import { getDb } from "@/app/lib/db";
+import LiveAudioStream from 'react-native-live-audio-stream';
+
+const lasOptions = {
+  sampleRate: 32000,  // default is 44100 but 32000 is adequate for accurate voice recognition
+  channels: 1,        // 1 or 2, default 1
+  bitsPerSample: 16,  // 8 or 16, default 16
+  audioSource: 6,     // android only (see below)
+  bufferSize: 4096    // default is 2048
+};
+// LiveAudioStream.init(lasOptions as any);
 
 interface ConversationThreadProps {
   conversation: Conversation;
+  whisperContext: WhisperContext;
+  onGoBack?: () => any;
 }
 
-const ConversationThread = (p : ConversationThreadProps) => {
+const ConversationThread = (p: ConversationThreadProps) => {
   const [messages, setMessages] = useState<Message[]>([]);
+  const [guestSpeak, setGuestSpeak] = useState<string | undefined>();
+  const [guestSpeakLoaded, setGuestSpeakLoaded] = useState<boolean>(false);
+  const ct = new CachedTranslator("en", p.conversation.guest.language);
 
   useEffect(() => {
-    const updateMessages = (c : Conversation) => {
+
+    const updateMessages = (c: Conversation) => {
       setMessages([...c]);
     };
 
@@ -25,17 +46,57 @@ const ConversationThread = (p : ConversationThreadProps) => {
       p.conversation.onAddMessage = undefined;
       p.conversation.onTranslationDone = undefined;
     };
-  }, [p.conversation]);
+  }, [p.conversation, guestSpeak]);
 
-  const renderMessages = () => (
+  useEffect(() => {
+    const fetchData = async () => {
+      setGuestSpeak(await ct.translate("Speak"));
+    }
+
+    fetchData();
+  }, [guestSpeak])
+
+  const renderMessages = () =>
     messages.map((message, index) => (
       <MessageBubble key={index} message={message} />
-    ))
-  );
+    ));
+
+  function onGoBack() {
+    p.onGoBack && p.onGoBack();
+  }
 
   return (
-    <View style={{ flex: 1 }}>
-      {renderMessages()}
+    <View style={{ flex: 1, flexDirection: "column" }}>
+      <ScrollView
+        style={{
+          borderColor: "black",
+          borderWidth: 1,
+          borderStyle: "solid",
+          height: "90%",
+        }}
+      >
+        {renderMessages()}
+      </ScrollView>
+      <View style={{ alignSelf: "center", flexDirection: "row" }}>
+        <TouchableHighlight
+          style={{ backgroundColor: "blue", padding: 3, borderRadius: 5 }}
+        >
+          <Text style={{ color: "white", fontSize: 30 }}>Speak</Text>
+        </TouchableHighlight>
+        <TouchableHighlight
+          style={{ backgroundColor: "gray", padding: 3, borderRadius: 5 }}
+          onPress={onGoBack}
+        >
+          <Text style={{ color: "white", fontSize: 30 }}>Go Back</Text>
+        </TouchableHighlight>
+        <TouchableHighlight
+          style={{ backgroundColor: "blue", padding: 3, borderRadius: 5 }}
+        >
+          <Text style={{ color: "white", fontSize: 30 }}>
+            {guestSpeak ? guestSpeak : "Speak"}
+          </Text>
+        </TouchableHighlight>
+      </View>
     </View>
   );
 };

package.json

@@ -5,7 +5,7 @@
   "scripts": {
     "start": "expo start",
     "reset-project": "node ./scripts/reset-project.js",
-    "android": "expo start --android",
+    "android": "expo start --offline --android",
     "ios": "expo start --ios",
     "web": "expo start --offline --web",
     "test": "jest --watchAll",
@@ -36,12 +36,14 @@
     "react-native-cache": "^2.0.3",
     "react-native-country-flag": "^2.0.2",
     "react-native-gesture-handler": "~2.20.2",
+    "react-native-live-audio-stream": "^1.1.1",
     "react-native-reanimated": "~3.16.7",
     "react-native-safe-area-context": "4.12.0",
     "react-native-screens": "~4.4.0",
     "react-native-sqlite-storage": "^6.0.1",
     "react-native-web": "~0.19.13",
-    "react-native-webview": "13.12.5"
+    "react-native-webview": "13.12.5",
+    "whisper.rn": "^0.3.9"
   },
   "devDependencies": {
     "@babel/core": "^7.26.7",

pnpm-lock.yaml

@@ -77,6 +77,9 @@ dependencies:
   react-native-gesture-handler:
     specifier: ~2.20.2
     version: 2.20.2(react-native@0.76.6)(react@18.3.1)
+  react-native-live-audio-stream:
+    specifier: ^1.1.1
+    version: 1.1.1
   react-native-reanimated:
     specifier: ~3.16.7
     version: 3.16.7(@babel/core@7.26.7)(react-native@0.76.6)(react@18.3.1)
@@ -95,6 +98,9 @@ dependencies:
   react-native-webview:
     specifier: 13.12.5
     version: 13.12.5(react-native@0.76.6)(react@18.3.1)
+  whisper.rn:
+    specifier: ^0.3.9
+    version: 0.3.9(react-native@0.76.6)(react@18.3.1)
 
 devDependencies:
   '@babel/core':
@@ -1587,7 +1593,7 @@ packages:
 
   /@expo/bunyan@4.0.1:
     resolution: {integrity: sha512-+Lla7nYSiHZirgK+U/uYzsLv/X+HaJienbD5AKX1UQZHYfWaP+9uuQluRB4GrEVWF0GZ7vEVp/jzaOT9k/SQlg==, tarball: https://registry.npmjs.org/@expo/bunyan/-/bunyan-4.0.1.tgz}
-    engines: {node: '>=0.10.0'}
+    engines: {'0': node >=0.10.0}
     dependencies:
       uuid: 8.3.2
 
@@ -7030,6 +7036,10 @@ packages:
       react-native: 0.76.6(@babel/core@7.26.7)(@babel/preset-env@7.26.7)(@types/react@18.3.18)(react@18.3.1)
     dev: false
 
+  /react-native-live-audio-stream@1.1.1:
+    resolution: {integrity: sha512-Yk0O51hY7eFMUv1umYxGDs4SJVPHyhUX6uz4jI+GiowOwSqIzLLRNh03hJjCVZRFXTWLPCntqOKZ+N8fVAc6BQ==, tarball: https://registry.npmjs.org/react-native-live-audio-stream/-/react-native-live-audio-stream-1.1.1.tgz}
+    dev: false
+
   /react-native-reanimated@3.16.7(@babel/core@7.26.7)(react-native@0.76.6)(react@18.3.1):
     resolution: {integrity: sha512-qoUUQOwE1pHlmQ9cXTJ2MX9FQ9eHllopCLiWOkDkp6CER95ZWeXhJCP4cSm6AD4jigL5jHcZf/SkWrg8ttZUsw==, tarball: https://registry.npmjs.org/react-native-reanimated/-/react-native-reanimated-3.16.7.tgz}
     peerDependencies:
@@ -8377,6 +8387,17 @@ packages:
     dependencies:
       isexe: 2.0.0
 
+  /whisper.rn@0.3.9(react-native@0.76.6)(react@18.3.1):
+    resolution: {integrity: sha512-y2hsJ6IpUqtYUZA7YrtGqU3pTXNFzF8Piu8Ch4yAhBB6tyZcEl113e/X10xkd+bVfeIbrSZS2QJZQ4CBfbXS3g==, tarball: https://registry.npmjs.org/whisper.rn/-/whisper.rn-0.3.9.tgz}
+    engines: {node: '>= 16.0.0'}
+    peerDependencies:
+      react: '*'
+      react-native: '*'
+    dependencies:
+      react: 18.3.1
+      react-native: 0.76.6(@babel/core@7.26.7)(@babel/preset-env@7.26.7)(@types/react@18.3.18)(react@18.3.1)
+    dev: false
+
   /wonka@6.3.4:
     resolution: {integrity: sha512-CjpbqNtBGNAeyNS/9W6q3kSkKE52+FjIj7AkFlLr11s/VWGUu6a2CdYSdGxocIhIVjaW/zchesBQUKPVU69Cqg==, tarball: https://registry.npmjs.org/wonka/-/wonka-6.3.4.tgz}