add expo-audio. work on memory issue when reading file.

android/app/src/main/AndroidManifest.xml

@@ -1,7 +1,9 @@
 <manifest xmlns:android="http://schemas.android.com/apk/res/android">
   <uses-permission android:name="android.permission.INTERNET"/>
+  <uses-permission android:name="android.permission.MODIFY_AUDIO_SETTINGS"/>
   <uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE"/>
   <uses-permission android:name="android.permission.RECEIVE_BOOT_COMPLETED"/>
+  <uses-permission android:name="android.permission.RECORD_AUDIO"/>
   <uses-permission android:name="android.permission.SYSTEM_ALERT_WINDOW"/>
   <uses-permission android:name="android.permission.VIBRATE"/>
   <uses-permission android:name="android.permission.WAKE_LOCK"/>

app.json

@@ -57,10 +57,11 @@
             ]
           }
         }
-      ]
+      ],
+      "expo-audio"
     ],
     "experiments": {
       "typedRoutes": true
     }
   }
-}
+}

app/lib/util.ts

@@ -1,5 +1,9 @@
 import { TextDecoder } from "util";
 
-export async function arrbufToStr(arrayBuffer : ArrayBuffer) {
+export function arrbufToStr(arrayBuffer : ArrayBuffer) {
     return new TextDecoder().decode(new Uint8Array(arrayBuffer));
+}
+
+export function strToArrBuf(input : string) : Uint8Array<ArrayBufferLike> {
+    return new TextEncoder().encode(input)
 }

app/lib/whisper.ts

@@ -3,7 +3,7 @@ import * as FileSystem from "expo-file-system";
 import { File, Paths } from "expo-file-system/next";
 import { getDb } from "./db";
 import * as Crypto from "expo-crypto";
-import { arrbufToStr } from "./util";
+import { arrbufToStr, strToArrBuf } from "./util";
 
 export const WHISPER_MODEL_PATH = Paths.join(
   FileSystem.documentDirectory || "file:///",
@@ -154,16 +154,24 @@ export class WhisperFile {
       console.debug("%s does not exist", this.targetPath);
       return undefined;
     }
-    return await Crypto.digest(
+
+    const strData = await FileSystem.readAsStringAsync(this.targetPath, {
+      encoding: FileSystem.EncodingType.Base64,
+    });
+    const data = strToArrBuf(strData);
+
+    const digest = await Crypto.digest(
       Crypto.CryptoDigestAlgorithm.SHA256,
-      this.targetFile.bytes()
+      data
     );
+
+    return digest;
   }
 
   public async updateTargetHash() {
     const targetSha = await this.getTargetSha();
     if (!targetSha) return;
-    this.target_hash = await arrbufToStr(targetSha);
+    this.target_hash = arrbufToStr(targetSha);
   }
 
   get isHashValid() {
@@ -239,8 +247,8 @@ export class WhisperFile {
     options: {
       onData?: DownloadCallback | undefined;
     } = {
-      onData: undefined,
-    }
+        onData: undefined,
+      }
   ) {
     await this.syncHfMetadata();
 
@@ -254,24 +262,51 @@ export class WhisperFile {
     // Check for the existence of the target file
     // If it exists, load the existing data.
     await this.updateTargetExistence();
-    const existingData = this.does_target_exist
-      ? this.targetFile.text()
-      : undefined;
 
-    // Create the resumable.
-    return FileSystem.createDownloadResumable(
-      this.modelUrl,
-      this.targetPath,
-      {},
-      async (data: FileSystem.DownloadProgressData) => {
-        this.download_data = data;
-        await this.syncHfMetadata();
-        await this.updateTargetHash();
-        await this.updateTargetExistence();
-        if (options.onData) await options.onData(this);
-      },
-      existingData ? existingData : undefined
-    );
+    try {
+      const existingData = this.does_target_exist
+        ? await FileSystem.readAsStringAsync(this.targetPath, {
+          encoding: FileSystem.EncodingType.Base64,
+        })
+        : undefined;
+
+      // Create the resumable.
+      return FileSystem.createDownloadResumable(
+        this.modelUrl,
+        this.targetPath,
+        {},
+        async (data: FileSystem.DownloadProgressData) => {
+          // console.debug("yes, I'm still downloading");
+          try {
+            this.download_data = data;
+          } catch (err) {
+            console.error("Failed to set downloadData: %s", err);
+          }
+
+          try {
+            await this.syncHfMetadata();
+          } catch (err) {
+            console.error("Failed to update HuggingFace metadata: %s", err)
+          }
+
+          try {
+            await this.updateTargetHash();
+          } catch (er) {
+            console.error("Failed to update target hash: %s", er);
+          }
+
+          try {
+            await this.updateTargetExistence();
+          } catch (err) {
+            console.error("Failed to update target existence: %s", err)
+          }
+          if (options.onData) await options.onData(this);
+        },
+        existingData ? existingData : undefined
+      );
+    } catch (err) {
+      console.error("Could not read %s: %s", this.targetPath, err);
+    }
   }
 }
 

components/ConversationThread.tsx

@@ -1,9 +1,14 @@
 import React, { useState, useEffect } from "react";
-import { ScrollView, StyleSheet, Text, TouchableHighlight, View } from "react-native";
+import { Alert, ScrollView, StyleSheet, Text, TouchableHighlight, View } from "react-native";
 import { useNavigation, Route } from "@react-navigation/native";
 import { Conversation, Message } from "@/app/lib/conversation";
 import MessageBubble from "@/components/ui/MessageBubble";
 import { CachedTranslator, LanguageServer, language_matrix_entry } from "@/app/i18n/api";
+import { Settings } from "@/app/lib/settings";
+import { WHISPER_FILES } from "@/app/lib/whisper";
+import { initWhisper, WhisperContext } from 'whisper.rn'
+import { useAudioRecorder, AudioModule, RecordingPresets } from 'expo-audio';
+
 
 const lasOptions = {
   sampleRate: 32000, // default is 44100 but 32000 is adequate for accurate voice recognition
@@ -14,9 +19,9 @@ const lasOptions = {
 };
 // LiveAudioStream.init(lasOptions as any);
 
-const ConversationThread = ({ route } : {route?: Route<"Conversation", {conversation : Conversation}>}) => {
+const ConversationThread = ({ route }: { route?: Route<"Conversation", { conversation: Conversation }> }) => {
   const navigation = useNavigation();
-  
+
   if (!route) {
     return (<View><Text>Missing Params!</Text></View>)
   }
@@ -27,6 +32,7 @@ const ConversationThread = ({ route } : {route?: Route<"Conversation", {conversa
   const [messages, setMessages] = useState<Message[]>([]);
   const [guestSpeak, setGuestSpeak] = useState<string | undefined>();
   const [guestSpeakLoaded, setGuestSpeakLoaded] = useState<boolean>(false);
+  const [whisperContext, setWhisperContext] = useState<WhisperContext | undefined>();
   const [cachedTranslator, setCachedTranslator] = useState<
     undefined | CachedTranslator
   >();
@@ -42,17 +48,50 @@ const ConversationThread = ({ route } : {route?: Route<"Conversation", {conversa
     }
   }>()
 
+  // recorder settings
+  const audioRecorder = useAudioRecorder(RecordingPresets.HIGH_QUALITY);
+
+  const record = async () => {
+    await audioRecorder.prepareToRecordAsync();
+    audioRecorder.record();
+  };
+
+  const stopRecording = async () => {
+    // The recording will be available on `audioRecorder.uri`.
+    await audioRecorder.stop();
+  };
+
   useEffect(() => {
-    (async () => {
+    (async function () {
 
       const languageServer = await LanguageServer.getDefault();
-      const languages = await languageServer.fetchLanguages();
-      const cc = new CachedTranslator(
-        "en",
-        conversation.guest.language,
-        languageServer,
-      )
-      setCachedTranslator(cc);
+      try {
+        const languages = await languageServer.fetchLanguages(5000);
+        const cc = new CachedTranslator(
+          "en",
+          conversation.guest.language,
+          languageServer,
+        )
+        console.log("Set cached translator from %s", languageServer.baseUrl)
+        setCachedTranslator(cc);
+      } catch (err) {
+        console.error("Could not set translator from %s: %s", languageServer.baseUrl, err)
+      }
+
+      const settings = await Settings.getDefault();
+      const whisperFile = WHISPER_FILES[await settings.getWhisperModel() || "en"];
+      setWhisperContext(await initWhisper({
+        filePath: whisperFile.targetPath,
+      }));
+
+      // recorder settings
+
+      (async () => {
+        const status = await AudioModule.requestRecordingPermissionsAsync();
+        if (!status.granted) {
+          Alert.alert('Permission to access microphone was denied');
+        }
+      })();
       setGuestSpeak(await cc.translate("Speak"));
       const hostLang1 = languages[conversation.host.language].name;
       const guestLang1 = languages[conversation.host.language].name;
@@ -69,18 +108,22 @@ const ConversationThread = ({ route } : {route?: Route<"Conversation", {conversa
         }
       })
     })();
-    
+
     const updateMessages = (c: Conversation) => {
       setMessages([...c]);
     };
 
-    conversation.onAddMessage = updateMessages;
-    conversation.onTranslationDone = updateMessages;
+    if (!conversation) {
+      console.warn("Conversation is null or undefined.")
+    }
 
-    return () => {
-      conversation.onAddMessage = undefined;
-      conversation.onTranslationDone = undefined;
-    };
+    conversation.on("add_message", updateMessages);
+    conversation.on("translation_done", updateMessages);
+
+    // return () => {
+    //   conversation.on("add_message", undefined);
+    //   conversation.on("translation_done", undefined);
+    // };
   }, [conversation, guestSpeak]);
 
   const renderMessages = () =>
@@ -91,8 +134,8 @@ const ConversationThread = ({ route } : {route?: Route<"Conversation", {conversa
   return cachedTranslator ? (
     <View style={{ flex: 1, flexDirection: "column" }}>
       {languageLabels && (<View style={styles.languageLabels}>
-        <Text style={styles.nativeHostLabel}>{ languageLabels.hostNative.host } / { languageLabels.hostNative.guest }</Text>
-        <Text style={styles.nativeGuestLabel}>{ languageLabels.guestNative.host } / { languageLabels.guestNative.guest }</Text>
+        <Text style={styles.nativeHostLabel}>{languageLabels.hostNative.host} / {languageLabels.hostNative.guest}</Text>
+        <Text style={styles.nativeGuestLabel}>{languageLabels.guestNative.host} / {languageLabels.guestNative.guest}</Text>
       </View>)
       }
       <ScrollView

components/LanguageSelection.tsx

@@ -18,7 +18,7 @@ export function LanguageSelection(props: {
 }) {
   const [languages, setLanguages] = useState<language_matrix | undefined>();
   const [languagesLoaded, setLanguagesLoaded] = useState<boolean>(false);
-  const [translator, setTranslator] = useState<Translator|undefined>();
+  const [translator, setTranslator] = useState<Translator | undefined>();
 
   const nav = useNavigation();
 
@@ -56,7 +56,7 @@ export function LanguageSelection(props: {
             {(languages && languagesLoaded) ? Object.entries(languages).filter((l) => (LANG_FLAGS as any)[l[0]] !== undefined).map(
               ([lang, lang_entry]) => {
                 return (
-                    <ISpeakButton language={lang_entry} key={lang_entry.code} onLangSelected={onLangSelected} translator={translator} />
+                  <ISpeakButton language={lang_entry} key={lang_entry.code} onLangSelected={onLangSelected} translator={translator} />
                 );
               }
             ) : <Text>Waiting...</Text>

components/Settings.tsx

@@ -54,9 +54,12 @@ const SettingsComponent = () => {
       );
       setWhisperModel((await settings.getWhisperModel()) || "small");
       setWhisperFile(WHISPER_FILES[whisperModel]);
-      await whisperFile?.syncHfMetadata();
-      await whisperFile?.updateTargetExistence();
-      await whisperFile?.updateTargetHash();
+      if (whisperFile) {
+        await whisperFile.syncHfMetadata();
+        await whisperFile.updateTargetExistence();
+        await whisperFile.updateTargetHash();
+        setWhisperFileExists(whisperFile.does_target_exist)
+      }
     })();
   }, [whisperFile]);
 
@@ -94,8 +97,8 @@ const SettingsComponent = () => {
     const wFile = WHISPER_FILES[whisperModel];
     await wFile.syncHfMetadata();
     await wFile.updateTargetExistence();
-    await wFile.updateTargetHash();
-    setIsWhisperHashValid(wFile.isHashValid);
+    // await wFile.updateTargetHash();
+    // setIsWhisperHashValid(wFile.isHashValid);
     setWhisperFile(wFile);
     setWhisperFileExists(wFile.does_target_exist);
   };
@@ -184,19 +187,30 @@ const SettingsComponent = () => {
           />
         ))}
       </Picker>
-      <View>
-        {/* <Text>whisper file: { whisperFile?.tag }</Text> */}
-        {whisperFile &&
-        ( whisperFileExists && (<Pressable onPress={doDelete} style={styles.deleteButton}>
-          <Text>DELETE {whisperModel.toUpperCase()}</Text>
-        </Pressable>))
-}
-            <Pressable onPress={doDownload} style={styles.pauseDownloadButton}>
-              <Text>DOWNLOAD {whisperModel.toUpperCase()}</Text>
+      <View style={styles.downloadButtonWrapper}>
+        {((!downloader) && whisperFile) &&
+          (whisperFile.does_target_exist && (<Pressable onPress={doDelete} style={styles.deleteButton}>
+            <Text style={styles.buttonText}>DELETE {whisperModel.toUpperCase()}</Text>
+          </Pressable>))
+        }
+        <Pressable onPress={doDownload} style={styles.pauseDownloadButton}>
+          <Text style={styles.buttonText}>DOWNLOAD {whisperModel.toUpperCase()}</Text>
+        </Pressable>
+        ))
+
+        {
+          downloader && (
+            <Pressable onPress={doStopDownload} style={styles.pauseDownloadButton}>
+              <Text style={styles.buttonText}>STOP DOWNLOAD</Text>
             </Pressable>
-          ))}
+          )
+        }
         {bytesDone && bytesRemaining && (
           <View>
+            {whisperFile &&
+              (<Text>
+                Downloading to {whisperFile.targetPath}
+              </Text>)}
             <Text>
               {bytesDone} of{" "}
               {bytesRemaining} (

components/ui/ISpeakButton.tsx

@@ -92,7 +92,7 @@ const ISpeakButton = (props: ISpeakButtonProps) => {
   }, []);
 
   const countries =
-  // @ts-ignore
+    // @ts-ignore
     DEFAULT_FLAGS[props.language.code] || chooseCountry(props.language.code);
 
   return title ? (
@@ -107,10 +107,7 @@ const ISpeakButton = (props: ISpeakButtonProps) => {
           {countries &&
             countries.map((c) => {
               return (
-                <View>
-                  <Text>{c}</Text>
-                  <CountryFlag isoCode={c} size={25} key={c} />
-                </View>
+                <CountryFlag isoCode={c} size={25} key={c} />
               );
             })}
         </View>

package-lock.json

@@ -17,6 +17,7 @@
         "@react-navigation/bottom-tabs": "^7.2.0",
         "@react-navigation/native-stack": "^7.2.0",
         "expo": "~52.0.28",
+        "expo-audio": "~0.3.4",
         "expo-background-fetch": "~13.0.5",
         "expo-blur": "~14.0.3",
         "expo-constants": "~17.0.6",
@@ -8162,6 +8163,17 @@
         "react-native": "*"
       }
     },
+    "node_modules/expo-audio": {
+      "version": "0.3.5",
+      "resolved": "https://registry.npmjs.org/expo-audio/-/expo-audio-0.3.5.tgz",
+      "integrity": "sha512-gzpDH3vZI1FDL1Q8pXryACtNIW+idZ/zIZ8WqdTRzJuzxucazrG2gLXUS2ngcXQBn09Jyz4RUnU10Tu2N7/Hgg==",
+      "license": "MIT",
+      "peerDependencies": {
+        "expo": "*",
+        "react": "*",
+        "react-native": "*"
+      }
+    },
     "node_modules/expo-background-fetch": {
       "version": "13.0.5",
       "resolved": "https://registry.npmjs.org/expo-background-fetch/-/expo-background-fetch-13.0.5.tgz",

package.json

@@ -58,7 +58,8 @@
     "react-native-webview": "13.12.5",
     "sqlite": "^5.1.1",
     "sqlite3": "^5.1.7",
-    "whisper.rn": "^0.3.9"
+    "whisper.rn": "^0.3.9",
+    "expo-audio": "~0.3.4"
   },
   "jest": {
     "preset": "jest-expo",