add expo-audio. work on memory issue when reading file.

2025-03-11 18:35:37 -07:00
parent dca3987e18
commit f0a722b3fb
10 changed files with 178 additions and 69 deletions
--- a/components/ConversationThread.tsx
+++ b/components/ConversationThread.tsx
@ -1,9 +1,14 @@
 import React, { useState, useEffect } from "react";
-import { ScrollView, StyleSheet, Text, TouchableHighlight, View } from "react-native";
+import { Alert, ScrollView, StyleSheet, Text, TouchableHighlight, View } from "react-native";
 import { useNavigation, Route } from "@react-navigation/native";
 import { Conversation, Message } from "@/app/lib/conversation";
 import MessageBubble from "@/components/ui/MessageBubble";
 import { CachedTranslator, LanguageServer, language_matrix_entry } from "@/app/i18n/api";
+import { Settings } from "@/app/lib/settings";
+import { WHISPER_FILES } from "@/app/lib/whisper";
+import { initWhisper, WhisperContext } from 'whisper.rn'
+import { useAudioRecorder, AudioModule, RecordingPresets } from 'expo-audio';
+

 const lasOptions = {
  sampleRate: 32000, // default is 44100 but 32000 is adequate for accurate voice recognition
@ -14,9 +19,9 @@ const lasOptions = {
 };
 // LiveAudioStream.init(lasOptions as any);

-const ConversationThread = ({ route } : {route?: Route<"Conversation", {conversation : Conversation}>}) => {
+const ConversationThread = ({ route }: { route?: Route<"Conversation", { conversation: Conversation }> }) => {
  const navigation = useNavigation();
-  
+
  if (!route) {
    return (<View><Text>Missing Params!</Text></View>)
  }
@ -27,6 +32,7 @@ const ConversationThread = ({ route } : {route?: Route<"Conversation", {conversa
  const [messages, setMessages] = useState<Message[]>([]);
  const [guestSpeak, setGuestSpeak] = useState<string | undefined>();
  const [guestSpeakLoaded, setGuestSpeakLoaded] = useState<boolean>(false);
+  const [whisperContext, setWhisperContext] = useState<WhisperContext | undefined>();
  const [cachedTranslator, setCachedTranslator] = useState<
    undefined | CachedTranslator
  >();
@ -42,17 +48,50 @@ const ConversationThread = ({ route } : {route?: Route<"Conversation", {conversa
    }
  }>()

+  // recorder settings
+  const audioRecorder = useAudioRecorder(RecordingPresets.HIGH_QUALITY);
+
+  const record = async () => {
+    await audioRecorder.prepareToRecordAsync();
+    audioRecorder.record();
+  };
+
+  const stopRecording = async () => {
+    // The recording will be available on `audioRecorder.uri`.
+    await audioRecorder.stop();
+  };
+
  useEffect(() => {
-    (async () => {
+    (async function () {

      const languageServer = await LanguageServer.getDefault();
-      const languages = await languageServer.fetchLanguages();
-      const cc = new CachedTranslator(
-        "en",
-        conversation.guest.language,
-        languageServer,
-      )
-      setCachedTranslator(cc);
+      try {
+        const languages = await languageServer.fetchLanguages(5000);
+        const cc = new CachedTranslator(
+          "en",
+          conversation.guest.language,
+          languageServer,
+        )
+        console.log("Set cached translator from %s", languageServer.baseUrl)
+        setCachedTranslator(cc);
+      } catch (err) {
+        console.error("Could not set translator from %s: %s", languageServer.baseUrl, err)
+      }
+
+      const settings = await Settings.getDefault();
+      const whisperFile = WHISPER_FILES[await settings.getWhisperModel() || "en"];
+      setWhisperContext(await initWhisper({
+        filePath: whisperFile.targetPath,
+      }));
+
+      // recorder settings
+
+      (async () => {
+        const status = await AudioModule.requestRecordingPermissionsAsync();
+        if (!status.granted) {
+          Alert.alert('Permission to access microphone was denied');
+        }
+      })();
      setGuestSpeak(await cc.translate("Speak"));
      const hostLang1 = languages[conversation.host.language].name;
      const guestLang1 = languages[conversation.host.language].name;
@ -69,18 +108,22 @@ const ConversationThread = ({ route } : {route?: Route<"Conversation", {conversa
        }
      })
    })();
-    
+
    const updateMessages = (c: Conversation) => {
      setMessages([...c]);
    };

-    conversation.onAddMessage = updateMessages;
-    conversation.onTranslationDone = updateMessages;
+    if (!conversation) {
+      console.warn("Conversation is null or undefined.")
+    }

-    return () => {
-      conversation.onAddMessage = undefined;
-      conversation.onTranslationDone = undefined;
-    };
+    conversation.on("add_message", updateMessages);
+    conversation.on("translation_done", updateMessages);
+
+    // return () => {
+    //   conversation.on("add_message", undefined);
+    //   conversation.on("translation_done", undefined);
+    // };
  }, [conversation, guestSpeak]);

  const renderMessages = () =>
@ -91,8 +134,8 @@ const ConversationThread = ({ route } : {route?: Route<"Conversation", {conversa
  return cachedTranslator ? (
    <View style={{ flex: 1, flexDirection: "column" }}>
      {languageLabels && (<View style={styles.languageLabels}>
-        <Text style={styles.nativeHostLabel}>{ languageLabels.hostNative.host } / { languageLabels.hostNative.guest }</Text>
-        <Text style={styles.nativeGuestLabel}>{ languageLabels.guestNative.host } / { languageLabels.guestNative.guest }</Text>
+        <Text style={styles.nativeHostLabel}>{languageLabels.hostNative.host} / {languageLabels.hostNative.guest}</Text>
+        <Text style={styles.nativeGuestLabel}>{languageLabels.guestNative.host} / {languageLabels.guestNative.guest}</Text>
      </View>)
      }
      <ScrollView
--- a/components/LanguageSelection.tsx
+++ b/components/LanguageSelection.tsx
@ -18,7 +18,7 @@ export function LanguageSelection(props: {
 }) {
  const [languages, setLanguages] = useState<language_matrix | undefined>();
  const [languagesLoaded, setLanguagesLoaded] = useState<boolean>(false);
-  const [translator, setTranslator] = useState<Translator|undefined>();
+  const [translator, setTranslator] = useState<Translator | undefined>();

  const nav = useNavigation();

@ -56,7 +56,7 @@ export function LanguageSelection(props: {
            {(languages && languagesLoaded) ? Object.entries(languages).filter((l) => (LANG_FLAGS as any)[l[0]] !== undefined).map(
              ([lang, lang_entry]) => {
                return (
-                    <ISpeakButton language={lang_entry} key={lang_entry.code} onLangSelected={onLangSelected} translator={translator} />
+                  <ISpeakButton language={lang_entry} key={lang_entry.code} onLangSelected={onLangSelected} translator={translator} />
                );
              }
            ) : <Text>Waiting...</Text>
--- a/components/Settings.tsx
+++ b/components/Settings.tsx
@ -54,9 +54,12 @@ const SettingsComponent = () => {
      );
      setWhisperModel((await settings.getWhisperModel()) || "small");
      setWhisperFile(WHISPER_FILES[whisperModel]);
-      await whisperFile?.syncHfMetadata();
-      await whisperFile?.updateTargetExistence();
-      await whisperFile?.updateTargetHash();
+      if (whisperFile) {
+        await whisperFile.syncHfMetadata();
+        await whisperFile.updateTargetExistence();
+        await whisperFile.updateTargetHash();
+        setWhisperFileExists(whisperFile.does_target_exist)
+      }
    })();
  }, [whisperFile]);

@ -94,8 +97,8 @@ const SettingsComponent = () => {
    const wFile = WHISPER_FILES[whisperModel];
    await wFile.syncHfMetadata();
    await wFile.updateTargetExistence();
-    await wFile.updateTargetHash();
-    setIsWhisperHashValid(wFile.isHashValid);
+    // await wFile.updateTargetHash();
+    // setIsWhisperHashValid(wFile.isHashValid);
    setWhisperFile(wFile);
    setWhisperFileExists(wFile.does_target_exist);
  };
@ -184,19 +187,30 @@ const SettingsComponent = () => {
          />
        ))}
      </Picker>
-      <View>
-        {/* <Text>whisper file: { whisperFile?.tag }</Text> */}
-        {whisperFile &&
-        ( whisperFileExists && (<Pressable onPress={doDelete} style={styles.deleteButton}>
-          <Text>DELETE {whisperModel.toUpperCase()}</Text>
-        </Pressable>))
-}
-            <Pressable onPress={doDownload} style={styles.pauseDownloadButton}>
-              <Text>DOWNLOAD {whisperModel.toUpperCase()}</Text>
+      <View style={styles.downloadButtonWrapper}>
+        {((!downloader) && whisperFile) &&
+          (whisperFile.does_target_exist && (<Pressable onPress={doDelete} style={styles.deleteButton}>
+            <Text style={styles.buttonText}>DELETE {whisperModel.toUpperCase()}</Text>
+          </Pressable>))
+        }
+        <Pressable onPress={doDownload} style={styles.pauseDownloadButton}>
+          <Text style={styles.buttonText}>DOWNLOAD {whisperModel.toUpperCase()}</Text>
+        </Pressable>
+        ))
+
+        {
+          downloader && (
+            <Pressable onPress={doStopDownload} style={styles.pauseDownloadButton}>
+              <Text style={styles.buttonText}>STOP DOWNLOAD</Text>
            </Pressable>
-          ))}
+          )
+        }
        {bytesDone && bytesRemaining && (
          <View>
+            {whisperFile &&
+              (<Text>
+                Downloading to {whisperFile.targetPath}
+              </Text>)}
            <Text>
              {bytesDone} of{" "}
              {bytesRemaining} (
--- a/components/ui/ISpeakButton.tsx
+++ b/components/ui/ISpeakButton.tsx
@ -92,7 +92,7 @@ const ISpeakButton = (props: ISpeakButtonProps) => {
  }, []);

  const countries =
-  // @ts-ignore
+    // @ts-ignore
    DEFAULT_FLAGS[props.language.code] || chooseCountry(props.language.code);

  return title ? (
@ -107,10 +107,7 @@ const ISpeakButton = (props: ISpeakButtonProps) => {
          {countries &&
            countries.map((c) => {
              return (
-                <View>
-                  <Text>{c}</Text>
-                  <CountryFlag isoCode={c} size={25} key={c} />
-                </View>
+                <CountryFlag isoCode={c} size={25} key={c} />
              );
            })}
        </View>