diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml index ad88360..f77ba4d 100644 --- a/android/app/src/main/AndroidManifest.xml +++ b/android/app/src/main/AndroidManifest.xml @@ -1,7 +1,9 @@ + + diff --git a/app.json b/app.json index 6737023..eb6b8db 100644 --- a/app.json +++ b/app.json @@ -57,10 +57,11 @@ ] } } - ] + ], + "expo-audio" ], "experiments": { "typedRoutes": true } } -} \ No newline at end of file +} diff --git a/app/lib/util.ts b/app/lib/util.ts index b81db9f..f5f2902 100644 --- a/app/lib/util.ts +++ b/app/lib/util.ts @@ -1,5 +1,9 @@ import { TextDecoder } from "util"; -export async function arrbufToStr(arrayBuffer : ArrayBuffer) { +export function arrbufToStr(arrayBuffer : ArrayBuffer) { return new TextDecoder().decode(new Uint8Array(arrayBuffer)); +} + +export function strToArrBuf(input : string) : Uint8Array { + return new TextEncoder().encode(input) } \ No newline at end of file diff --git a/app/lib/whisper.ts b/app/lib/whisper.ts index 503dc30..2255fe7 100644 --- a/app/lib/whisper.ts +++ b/app/lib/whisper.ts @@ -3,7 +3,7 @@ import * as FileSystem from "expo-file-system"; import { File, Paths } from "expo-file-system/next"; import { getDb } from "./db"; import * as Crypto from "expo-crypto"; -import { arrbufToStr } from "./util"; +import { arrbufToStr, strToArrBuf } from "./util"; export const WHISPER_MODEL_PATH = Paths.join( FileSystem.documentDirectory || "file:///", @@ -154,16 +154,24 @@ export class WhisperFile { console.debug("%s does not exist", this.targetPath); return undefined; } - return await Crypto.digest( + + const strData = await FileSystem.readAsStringAsync(this.targetPath, { + encoding: FileSystem.EncodingType.Base64, + }); + const data = strToArrBuf(strData); + + const digest = await Crypto.digest( Crypto.CryptoDigestAlgorithm.SHA256, - this.targetFile.bytes() + data ); + + return digest; } public async updateTargetHash() { const targetSha = await this.getTargetSha(); if (!targetSha) return; - this.target_hash = await arrbufToStr(targetSha); + this.target_hash = arrbufToStr(targetSha); } get isHashValid() { @@ -239,8 +247,8 @@ export class WhisperFile { options: { onData?: DownloadCallback | undefined; } = { - onData: undefined, - } + onData: undefined, + } ) { await this.syncHfMetadata(); @@ -254,24 +262,51 @@ export class WhisperFile { // Check for the existence of the target file // If it exists, load the existing data. await this.updateTargetExistence(); - const existingData = this.does_target_exist - ? this.targetFile.text() - : undefined; - // Create the resumable. - return FileSystem.createDownloadResumable( - this.modelUrl, - this.targetPath, - {}, - async (data: FileSystem.DownloadProgressData) => { - this.download_data = data; - await this.syncHfMetadata(); - await this.updateTargetHash(); - await this.updateTargetExistence(); - if (options.onData) await options.onData(this); - }, - existingData ? existingData : undefined - ); + try { + const existingData = this.does_target_exist + ? await FileSystem.readAsStringAsync(this.targetPath, { + encoding: FileSystem.EncodingType.Base64, + }) + : undefined; + + // Create the resumable. + return FileSystem.createDownloadResumable( + this.modelUrl, + this.targetPath, + {}, + async (data: FileSystem.DownloadProgressData) => { + // console.debug("yes, I'm still downloading"); + try { + this.download_data = data; + } catch (err) { + console.error("Failed to set downloadData: %s", err); + } + + try { + await this.syncHfMetadata(); + } catch (err) { + console.error("Failed to update HuggingFace metadata: %s", err) + } + + try { + await this.updateTargetHash(); + } catch (er) { + console.error("Failed to update target hash: %s", er); + } + + try { + await this.updateTargetExistence(); + } catch (err) { + console.error("Failed to update target existence: %s", err) + } + if (options.onData) await options.onData(this); + }, + existingData ? existingData : undefined + ); + } catch (err) { + console.error("Could not read %s: %s", this.targetPath, err); + } } } diff --git a/components/ConversationThread.tsx b/components/ConversationThread.tsx index c7d873a..b864e63 100644 --- a/components/ConversationThread.tsx +++ b/components/ConversationThread.tsx @@ -1,9 +1,14 @@ import React, { useState, useEffect } from "react"; -import { ScrollView, StyleSheet, Text, TouchableHighlight, View } from "react-native"; +import { Alert, ScrollView, StyleSheet, Text, TouchableHighlight, View } from "react-native"; import { useNavigation, Route } from "@react-navigation/native"; import { Conversation, Message } from "@/app/lib/conversation"; import MessageBubble from "@/components/ui/MessageBubble"; import { CachedTranslator, LanguageServer, language_matrix_entry } from "@/app/i18n/api"; +import { Settings } from "@/app/lib/settings"; +import { WHISPER_FILES } from "@/app/lib/whisper"; +import { initWhisper, WhisperContext } from 'whisper.rn' +import { useAudioRecorder, AudioModule, RecordingPresets } from 'expo-audio'; + const lasOptions = { sampleRate: 32000, // default is 44100 but 32000 is adequate for accurate voice recognition @@ -14,9 +19,9 @@ const lasOptions = { }; // LiveAudioStream.init(lasOptions as any); -const ConversationThread = ({ route } : {route?: Route<"Conversation", {conversation : Conversation}>}) => { +const ConversationThread = ({ route }: { route?: Route<"Conversation", { conversation: Conversation }> }) => { const navigation = useNavigation(); - + if (!route) { return (Missing Params!) } @@ -27,6 +32,7 @@ const ConversationThread = ({ route } : {route?: Route<"Conversation", {conversa const [messages, setMessages] = useState([]); const [guestSpeak, setGuestSpeak] = useState(); const [guestSpeakLoaded, setGuestSpeakLoaded] = useState(false); + const [whisperContext, setWhisperContext] = useState(); const [cachedTranslator, setCachedTranslator] = useState< undefined | CachedTranslator >(); @@ -42,17 +48,50 @@ const ConversationThread = ({ route } : {route?: Route<"Conversation", {conversa } }>() + // recorder settings + const audioRecorder = useAudioRecorder(RecordingPresets.HIGH_QUALITY); + + const record = async () => { + await audioRecorder.prepareToRecordAsync(); + audioRecorder.record(); + }; + + const stopRecording = async () => { + // The recording will be available on `audioRecorder.uri`. + await audioRecorder.stop(); + }; + useEffect(() => { - (async () => { + (async function () { const languageServer = await LanguageServer.getDefault(); - const languages = await languageServer.fetchLanguages(); - const cc = new CachedTranslator( - "en", - conversation.guest.language, - languageServer, - ) - setCachedTranslator(cc); + try { + const languages = await languageServer.fetchLanguages(5000); + const cc = new CachedTranslator( + "en", + conversation.guest.language, + languageServer, + ) + console.log("Set cached translator from %s", languageServer.baseUrl) + setCachedTranslator(cc); + } catch (err) { + console.error("Could not set translator from %s: %s", languageServer.baseUrl, err) + } + + const settings = await Settings.getDefault(); + const whisperFile = WHISPER_FILES[await settings.getWhisperModel() || "en"]; + setWhisperContext(await initWhisper({ + filePath: whisperFile.targetPath, + })); + + // recorder settings + + (async () => { + const status = await AudioModule.requestRecordingPermissionsAsync(); + if (!status.granted) { + Alert.alert('Permission to access microphone was denied'); + } + })(); setGuestSpeak(await cc.translate("Speak")); const hostLang1 = languages[conversation.host.language].name; const guestLang1 = languages[conversation.host.language].name; @@ -69,18 +108,22 @@ const ConversationThread = ({ route } : {route?: Route<"Conversation", {conversa } }) })(); - + const updateMessages = (c: Conversation) => { setMessages([...c]); }; - conversation.onAddMessage = updateMessages; - conversation.onTranslationDone = updateMessages; + if (!conversation) { + console.warn("Conversation is null or undefined.") + } - return () => { - conversation.onAddMessage = undefined; - conversation.onTranslationDone = undefined; - }; + conversation.on("add_message", updateMessages); + conversation.on("translation_done", updateMessages); + + // return () => { + // conversation.on("add_message", undefined); + // conversation.on("translation_done", undefined); + // }; }, [conversation, guestSpeak]); const renderMessages = () => @@ -91,8 +134,8 @@ const ConversationThread = ({ route } : {route?: Route<"Conversation", {conversa return cachedTranslator ? ( {languageLabels && ( - { languageLabels.hostNative.host } / { languageLabels.hostNative.guest } - { languageLabels.guestNative.host } / { languageLabels.guestNative.guest } + {languageLabels.hostNative.host} / {languageLabels.hostNative.guest} + {languageLabels.guestNative.host} / {languageLabels.guestNative.guest} ) } (); const [languagesLoaded, setLanguagesLoaded] = useState(false); - const [translator, setTranslator] = useState(); + const [translator, setTranslator] = useState(); const nav = useNavigation(); @@ -56,7 +56,7 @@ export function LanguageSelection(props: { {(languages && languagesLoaded) ? Object.entries(languages).filter((l) => (LANG_FLAGS as any)[l[0]] !== undefined).map( ([lang, lang_entry]) => { return ( - + ); } ) : Waiting... diff --git a/components/Settings.tsx b/components/Settings.tsx index 5c8d10d..49df516 100644 --- a/components/Settings.tsx +++ b/components/Settings.tsx @@ -54,9 +54,12 @@ const SettingsComponent = () => { ); setWhisperModel((await settings.getWhisperModel()) || "small"); setWhisperFile(WHISPER_FILES[whisperModel]); - await whisperFile?.syncHfMetadata(); - await whisperFile?.updateTargetExistence(); - await whisperFile?.updateTargetHash(); + if (whisperFile) { + await whisperFile.syncHfMetadata(); + await whisperFile.updateTargetExistence(); + await whisperFile.updateTargetHash(); + setWhisperFileExists(whisperFile.does_target_exist) + } })(); }, [whisperFile]); @@ -94,8 +97,8 @@ const SettingsComponent = () => { const wFile = WHISPER_FILES[whisperModel]; await wFile.syncHfMetadata(); await wFile.updateTargetExistence(); - await wFile.updateTargetHash(); - setIsWhisperHashValid(wFile.isHashValid); + // await wFile.updateTargetHash(); + // setIsWhisperHashValid(wFile.isHashValid); setWhisperFile(wFile); setWhisperFileExists(wFile.does_target_exist); }; @@ -184,19 +187,30 @@ const SettingsComponent = () => { /> ))} - - {/* whisper file: { whisperFile?.tag } */} - {whisperFile && - ( whisperFileExists && ( - DELETE {whisperModel.toUpperCase()} - )) -} - - DOWNLOAD {whisperModel.toUpperCase()} + + {((!downloader) && whisperFile) && + (whisperFile.does_target_exist && ( + DELETE {whisperModel.toUpperCase()} + )) + } + + DOWNLOAD {whisperModel.toUpperCase()} + + )) + + { + downloader && ( + + STOP DOWNLOAD - ))} + ) + } {bytesDone && bytesRemaining && ( + {whisperFile && + ( + Downloading to {whisperFile.targetPath} + )} {bytesDone} of{" "} {bytesRemaining} ( diff --git a/components/ui/ISpeakButton.tsx b/components/ui/ISpeakButton.tsx index 2072d9f..525ac45 100644 --- a/components/ui/ISpeakButton.tsx +++ b/components/ui/ISpeakButton.tsx @@ -92,7 +92,7 @@ const ISpeakButton = (props: ISpeakButtonProps) => { }, []); const countries = - // @ts-ignore + // @ts-ignore DEFAULT_FLAGS[props.language.code] || chooseCountry(props.language.code); return title ? ( @@ -107,10 +107,7 @@ const ISpeakButton = (props: ISpeakButtonProps) => { {countries && countries.map((c) => { return ( - - {c} - - + ); })} diff --git a/package-lock.json b/package-lock.json index 3956983..a26b98e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,6 +17,7 @@ "@react-navigation/bottom-tabs": "^7.2.0", "@react-navigation/native-stack": "^7.2.0", "expo": "~52.0.28", + "expo-audio": "~0.3.4", "expo-background-fetch": "~13.0.5", "expo-blur": "~14.0.3", "expo-constants": "~17.0.6", @@ -8162,6 +8163,17 @@ "react-native": "*" } }, + "node_modules/expo-audio": { + "version": "0.3.5", + "resolved": "https://registry.npmjs.org/expo-audio/-/expo-audio-0.3.5.tgz", + "integrity": "sha512-gzpDH3vZI1FDL1Q8pXryACtNIW+idZ/zIZ8WqdTRzJuzxucazrG2gLXUS2ngcXQBn09Jyz4RUnU10Tu2N7/Hgg==", + "license": "MIT", + "peerDependencies": { + "expo": "*", + "react": "*", + "react-native": "*" + } + }, "node_modules/expo-background-fetch": { "version": "13.0.5", "resolved": "https://registry.npmjs.org/expo-background-fetch/-/expo-background-fetch-13.0.5.tgz", diff --git a/package.json b/package.json index ea03ce2..b85a4ea 100644 --- a/package.json +++ b/package.json @@ -58,7 +58,8 @@ "react-native-webview": "13.12.5", "sqlite": "^5.1.1", "sqlite3": "^5.1.7", - "whisper.rn": "^0.3.9" + "whisper.rn": "^0.3.9", + "expo-audio": "~0.3.4" }, "jest": { "preset": "jest-expo",