srcrr/translation-terrace
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add expo-audio. work on memory issue when reading file.

Browse Source
This commit is contained in:
Jordan Hewitt 2025-03-11 18:35:37 -07:00
parent dca3987e18
commit f0a722b3fb
10 changed files with 178 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
android/app/src/main/AndroidManifest.xml
View File

@ -1,7 +1,9 @@
<manifest xmlns:android="http://schemas.android.com/apk/res/android"> <manifest xmlns:android="http://schemas.android.com/apk/res/android">
<uses-permission android:name="android.permission.INTERNET"/> <uses-permission android:name="android.permission.INTERNET"/>
<uses-permission android:name="android.permission.MODIFY_AUDIO_SETTINGS"/>
<uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE"/> <uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE"/>
<uses-permission android:name="android.permission.RECEIVE_BOOT_COMPLETED"/> <uses-permission android:name="android.permission.RECEIVE_BOOT_COMPLETED"/>
<uses-permission android:name="android.permission.RECORD_AUDIO"/>
<uses-permission android:name="android.permission.SYSTEM_ALERT_WINDOW"/> <uses-permission android:name="android.permission.SYSTEM_ALERT_WINDOW"/>
<uses-permission android:name="android.permission.VIBRATE"/> <uses-permission android:name="android.permission.VIBRATE"/>
<uses-permission android:name="android.permission.WAKE_LOCK"/> <uses-permission android:name="android.permission.WAKE_LOCK"/>

5
app.json
View File

@ -57,10 +57,11 @@
] ]
} }
} }
] ],
"expo-audio"
], ],
"experiments": { "experiments": {
"typedRoutes": true "typedRoutes": true
} }
} }
} }

6
app/lib/util.ts
View File

@ -1,5 +1,9 @@
import { TextDecoder } from "util"; import { TextDecoder } from "util";
export async function arrbufToStr(arrayBuffer : ArrayBuffer) { export function arrbufToStr(arrayBuffer : ArrayBuffer) {
return new TextDecoder().decode(new Uint8Array(arrayBuffer)); return new TextDecoder().decode(new Uint8Array(arrayBuffer));
}
export function strToArrBuf(input : string) : Uint8Array<ArrayBufferLike> {
return new TextEncoder().encode(input)
} }

81
app/lib/whisper.ts
View File

@ -3,7 +3,7 @@ import * as FileSystem from "expo-file-system";
import { File, Paths } from "expo-file-system/next"; import { File, Paths } from "expo-file-system/next";
import { getDb } from "./db"; import { getDb } from "./db";
import * as Crypto from "expo-crypto"; import * as Crypto from "expo-crypto";
import { arrbufToStr } from "./util"; import { arrbufToStr, strToArrBuf } from "./util";
export const WHISPER_MODEL_PATH = Paths.join( export const WHISPER_MODEL_PATH = Paths.join(
FileSystem.documentDirectory || "file:///", FileSystem.documentDirectory || "file:///",
@ -154,16 +154,24 @@ export class WhisperFile {
console.debug("%s does not exist", this.targetPath); console.debug("%s does not exist", this.targetPath);
return undefined; return undefined;
} }
return await Crypto.digest(
const strData = await FileSystem.readAsStringAsync(this.targetPath, {
encoding: FileSystem.EncodingType.Base64,
});
const data = strToArrBuf(strData);
const digest = await Crypto.digest(
Crypto.CryptoDigestAlgorithm.SHA256, Crypto.CryptoDigestAlgorithm.SHA256,
this.targetFile.bytes() data
); );
return digest;
} }
public async updateTargetHash() { public async updateTargetHash() {
const targetSha = await this.getTargetSha(); const targetSha = await this.getTargetSha();
if (!targetSha) return; if (!targetSha) return;
this.target_hash = await arrbufToStr(targetSha); this.target_hash = arrbufToStr(targetSha);
} }
get isHashValid() { get isHashValid() {
@ -239,8 +247,8 @@ export class WhisperFile {
options: { options: {
onData?: DownloadCallback | undefined; onData?: DownloadCallback | undefined;
} = { } = {
onData: undefined, onData: undefined,
} }
) { ) {
await this.syncHfMetadata(); await this.syncHfMetadata();
@ -254,24 +262,51 @@ export class WhisperFile {
// Check for the existence of the target file // Check for the existence of the target file
// If it exists, load the existing data. // If it exists, load the existing data.
await this.updateTargetExistence(); await this.updateTargetExistence();
const existingData = this.does_target_exist
? this.targetFile.text()
: undefined;
// Create the resumable. try {
return FileSystem.createDownloadResumable( const existingData = this.does_target_exist
this.modelUrl, ? await FileSystem.readAsStringAsync(this.targetPath, {
this.targetPath, encoding: FileSystem.EncodingType.Base64,
{}, })
async (data: FileSystem.DownloadProgressData) => { : undefined;
this.download_data = data;
await this.syncHfMetadata(); // Create the resumable.
await this.updateTargetHash(); return FileSystem.createDownloadResumable(
await this.updateTargetExistence(); this.modelUrl,
if (options.onData) await options.onData(this); this.targetPath,
}, {},
existingData ? existingData : undefined async (data: FileSystem.DownloadProgressData) => {
); // console.debug("yes, I'm still downloading");
try {
this.download_data = data;
} catch (err) {
console.error("Failed to set downloadData: %s", err);
}
try {
await this.syncHfMetadata();
} catch (err) {
console.error("Failed to update HuggingFace metadata: %s", err)
}
try {
await this.updateTargetHash();
} catch (er) {
console.error("Failed to update target hash: %s", er);
}
try {
await this.updateTargetExistence();
} catch (err) {
console.error("Failed to update target existence: %s", err)
}
if (options.onData) await options.onData(this);
},
existingData ? existingData : undefined
);
} catch (err) {
console.error("Could not read %s: %s", this.targetPath, err);
}
} }
} }

83
components/ConversationThread.tsx
View File

@ -1,9 +1,14 @@
import React, { useState, useEffect } from "react"; import React, { useState, useEffect } from "react";
import { ScrollView, StyleSheet, Text, TouchableHighlight, View } from "react-native"; import { Alert, ScrollView, StyleSheet, Text, TouchableHighlight, View } from "react-native";
import { useNavigation, Route } from "@react-navigation/native"; import { useNavigation, Route } from "@react-navigation/native";
import { Conversation, Message } from "@/app/lib/conversation"; import { Conversation, Message } from "@/app/lib/conversation";
import MessageBubble from "@/components/ui/MessageBubble"; import MessageBubble from "@/components/ui/MessageBubble";
import { CachedTranslator, LanguageServer, language_matrix_entry } from "@/app/i18n/api"; import { CachedTranslator, LanguageServer, language_matrix_entry } from "@/app/i18n/api";
import { Settings } from "@/app/lib/settings";
import { WHISPER_FILES } from "@/app/lib/whisper";
import { initWhisper, WhisperContext } from 'whisper.rn'
import { useAudioRecorder, AudioModule, RecordingPresets } from 'expo-audio';
const lasOptions = { const lasOptions = {
sampleRate: 32000, // default is 44100 but 32000 is adequate for accurate voice recognition sampleRate: 32000, // default is 44100 but 32000 is adequate for accurate voice recognition
@ -14,9 +19,9 @@ const lasOptions = {
}; };
// LiveAudioStream.init(lasOptions as any); // LiveAudioStream.init(lasOptions as any);
const ConversationThread = ({ route } : {route?: Route<"Conversation", {conversation : Conversation}>}) => { const ConversationThread = ({ route }: { route?: Route<"Conversation", { conversation: Conversation }> }) => {
const navigation = useNavigation(); const navigation = useNavigation();
if (!route) { if (!route) {
return (<View><Text>Missing Params!</Text></View>) return (<View><Text>Missing Params!</Text></View>)
} }
@ -27,6 +32,7 @@ const ConversationThread = ({ route } : {route?: Route<"Conversation", {conversa
const [messages, setMessages] = useState<Message[]>([]); const [messages, setMessages] = useState<Message[]>([]);
const [guestSpeak, setGuestSpeak] = useState<string | undefined>(); const [guestSpeak, setGuestSpeak] = useState<string | undefined>();
const [guestSpeakLoaded, setGuestSpeakLoaded] = useState<boolean>(false); const [guestSpeakLoaded, setGuestSpeakLoaded] = useState<boolean>(false);
const [whisperContext, setWhisperContext] = useState<WhisperContext | undefined>();
const [cachedTranslator, setCachedTranslator] = useState< const [cachedTranslator, setCachedTranslator] = useState<
undefined | CachedTranslator undefined | CachedTranslator
>(); >();
@ -42,17 +48,50 @@ const ConversationThread = ({ route } : {route?: Route<"Conversation", {conversa
} }
}>() }>()
// recorder settings
const audioRecorder = useAudioRecorder(RecordingPresets.HIGH_QUALITY);
const record = async () => {
await audioRecorder.prepareToRecordAsync();
audioRecorder.record();
};
const stopRecording = async () => {
// The recording will be available on `audioRecorder.uri`.
await audioRecorder.stop();
};
useEffect(() => { useEffect(() => {
(async () => { (async function () {
const languageServer = await LanguageServer.getDefault(); const languageServer = await LanguageServer.getDefault();
const languages = await languageServer.fetchLanguages(); try {
const cc = new CachedTranslator( const languages = await languageServer.fetchLanguages(5000);
"en", const cc = new CachedTranslator(
conversation.guest.language, "en",
languageServer, conversation.guest.language,
) languageServer,
setCachedTranslator(cc); )
console.log("Set cached translator from %s", languageServer.baseUrl)
setCachedTranslator(cc);
} catch (err) {
console.error("Could not set translator from %s: %s", languageServer.baseUrl, err)
}
const settings = await Settings.getDefault();
const whisperFile = WHISPER_FILES[await settings.getWhisperModel() || "en"];
setWhisperContext(await initWhisper({
filePath: whisperFile.targetPath,
}));
// recorder settings
(async () => {
const status = await AudioModule.requestRecordingPermissionsAsync();
if (!status.granted) {
Alert.alert('Permission to access microphone was denied');
}
})();
setGuestSpeak(await cc.translate("Speak")); setGuestSpeak(await cc.translate("Speak"));
const hostLang1 = languages[conversation.host.language].name; const hostLang1 = languages[conversation.host.language].name;
const guestLang1 = languages[conversation.host.language].name; const guestLang1 = languages[conversation.host.language].name;
@ -69,18 +108,22 @@ const ConversationThread = ({ route } : {route?: Route<"Conversation", {conversa
} }
}) })
})(); })();
const updateMessages = (c: Conversation) => { const updateMessages = (c: Conversation) => {
setMessages([...c]); setMessages([...c]);
}; };
conversation.onAddMessage = updateMessages; if (!conversation) {
conversation.onTranslationDone = updateMessages; console.warn("Conversation is null or undefined.")
}
return () => { conversation.on("add_message", updateMessages);
conversation.onAddMessage = undefined; conversation.on("translation_done", updateMessages);
conversation.onTranslationDone = undefined;
}; // return () => {
// conversation.on("add_message", undefined);
// conversation.on("translation_done", undefined);
// };
}, [conversation, guestSpeak]); }, [conversation, guestSpeak]);
const renderMessages = () => const renderMessages = () =>
@ -91,8 +134,8 @@ const ConversationThread = ({ route } : {route?: Route<"Conversation", {conversa
return cachedTranslator ? ( return cachedTranslator ? (
<View style={{ flex: 1, flexDirection: "column" }}> <View style={{ flex: 1, flexDirection: "column" }}>
{languageLabels && (<View style={styles.languageLabels}> {languageLabels && (<View style={styles.languageLabels}>
<Text style={styles.nativeHostLabel}>{ languageLabels.hostNative.host } / { languageLabels.hostNative.guest }</Text> <Text style={styles.nativeHostLabel}>{languageLabels.hostNative.host} / {languageLabels.hostNative.guest}</Text>
<Text style={styles.nativeGuestLabel}>{ languageLabels.guestNative.host } / { languageLabels.guestNative.guest }</Text> <Text style={styles.nativeGuestLabel}>{languageLabels.guestNative.host} / {languageLabels.guestNative.guest}</Text>
</View>) </View>)
} }
<ScrollView <ScrollView

4
components/LanguageSelection.tsx
View File

@ -18,7 +18,7 @@ export function LanguageSelection(props: {
}) { }) {
const [languages, setLanguages] = useState<language_matrix | undefined>(); const [languages, setLanguages] = useState<language_matrix | undefined>();
const [languagesLoaded, setLanguagesLoaded] = useState<boolean>(false); const [languagesLoaded, setLanguagesLoaded] = useState<boolean>(false);
const [translator, setTranslator] = useState<Translator|undefined>(); const [translator, setTranslator] = useState<Translator | undefined>();
const nav = useNavigation(); const nav = useNavigation();
@ -56,7 +56,7 @@ export function LanguageSelection(props: {
{(languages && languagesLoaded) ? Object.entries(languages).filter((l) => (LANG_FLAGS as any)[l[0]] !== undefined).map( {(languages && languagesLoaded) ? Object.entries(languages).filter((l) => (LANG_FLAGS as any)[l[0]] !== undefined).map(
([lang, lang_entry]) => { ([lang, lang_entry]) => {
return ( return (
<ISpeakButton language={lang_entry} key={lang_entry.code} onLangSelected={onLangSelected} translator={translator} /> <ISpeakButton language={lang_entry} key={lang_entry.code} onLangSelected={onLangSelected} translator={translator} />
); );
} }
) : <Text>Waiting...</Text> ) : <Text>Waiting...</Text>

44
components/Settings.tsx
View File

@ -54,9 +54,12 @@ const SettingsComponent = () => {
); );
setWhisperModel((await settings.getWhisperModel()) || "small"); setWhisperModel((await settings.getWhisperModel()) || "small");
setWhisperFile(WHISPER_FILES[whisperModel]); setWhisperFile(WHISPER_FILES[whisperModel]);
await whisperFile?.syncHfMetadata(); if (whisperFile) {
await whisperFile?.updateTargetExistence(); await whisperFile.syncHfMetadata();
await whisperFile?.updateTargetHash(); await whisperFile.updateTargetExistence();
await whisperFile.updateTargetHash();
setWhisperFileExists(whisperFile.does_target_exist)
}
})(); })();
}, [whisperFile]); }, [whisperFile]);
@ -94,8 +97,8 @@ const SettingsComponent = () => {
const wFile = WHISPER_FILES[whisperModel]; const wFile = WHISPER_FILES[whisperModel];
await wFile.syncHfMetadata(); await wFile.syncHfMetadata();
await wFile.updateTargetExistence(); await wFile.updateTargetExistence();
await wFile.updateTargetHash(); // await wFile.updateTargetHash();
setIsWhisperHashValid(wFile.isHashValid); // setIsWhisperHashValid(wFile.isHashValid);
setWhisperFile(wFile); setWhisperFile(wFile);
setWhisperFileExists(wFile.does_target_exist); setWhisperFileExists(wFile.does_target_exist);
}; };
@ -184,19 +187,30 @@ const SettingsComponent = () => {
/> />
))} ))}
</Picker> </Picker>
<View> <View style={styles.downloadButtonWrapper}>
{/* <Text>whisper file: { whisperFile?.tag }</Text> */} {((!downloader) && whisperFile) &&
{whisperFile && (whisperFile.does_target_exist && (<Pressable onPress={doDelete} style={styles.deleteButton}>
( whisperFileExists && (<Pressable onPress={doDelete} style={styles.deleteButton}> <Text style={styles.buttonText}>DELETE {whisperModel.toUpperCase()}</Text>
<Text>DELETE {whisperModel.toUpperCase()}</Text> </Pressable>))
</Pressable>)) }
} <Pressable onPress={doDownload} style={styles.pauseDownloadButton}>
<Pressable onPress={doDownload} style={styles.pauseDownloadButton}> <Text style={styles.buttonText}>DOWNLOAD {whisperModel.toUpperCase()}</Text>
<Text>DOWNLOAD {whisperModel.toUpperCase()}</Text> </Pressable>
))
{
downloader && (
<Pressable onPress={doStopDownload} style={styles.pauseDownloadButton}>
<Text style={styles.buttonText}>STOP DOWNLOAD</Text>
</Pressable> </Pressable>
))} )
}
{bytesDone && bytesRemaining && ( {bytesDone && bytesRemaining && (
<View> <View>
{whisperFile &&
(<Text>
Downloading to {whisperFile.targetPath}
</Text>)}
<Text> <Text>
{bytesDone} of{" "} {bytesDone} of{" "}
{bytesRemaining} ( {bytesRemaining} (

7
components/ui/ISpeakButton.tsx
View File

@ -92,7 +92,7 @@ const ISpeakButton = (props: ISpeakButtonProps) => {
}, []); }, []);
const countries = const countries =
// @ts-ignore // @ts-ignore
DEFAULT_FLAGS[props.language.code] || chooseCountry(props.language.code); DEFAULT_FLAGS[props.language.code] || chooseCountry(props.language.code);
return title ? ( return title ? (
@ -107,10 +107,7 @@ const ISpeakButton = (props: ISpeakButtonProps) => {
{countries && {countries &&
countries.map((c) => { countries.map((c) => {
return ( return (
<View> <CountryFlag isoCode={c} size={25} key={c} />
<Text>{c}</Text>
<CountryFlag isoCode={c} size={25} key={c} />
</View>
); );
})} })}
</View> </View>

12
package-lock.json generated
View File

@ -17,6 +17,7 @@
"@react-navigation/bottom-tabs": "^7.2.0", "@react-navigation/bottom-tabs": "^7.2.0",
"@react-navigation/native-stack": "^7.2.0", "@react-navigation/native-stack": "^7.2.0",
"expo": "~52.0.28", "expo": "~52.0.28",
"expo-audio": "~0.3.4",
"expo-background-fetch": "~13.0.5", "expo-background-fetch": "~13.0.5",
"expo-blur": "~14.0.3", "expo-blur": "~14.0.3",
"expo-constants": "~17.0.6", "expo-constants": "~17.0.6",
@ -8162,6 +8163,17 @@
"react-native": "*" "react-native": "*"
} }
}, },
"node_modules/expo-audio": {
"version": "0.3.5",
"resolved": "https://registry.npmjs.org/expo-audio/-/expo-audio-0.3.5.tgz",
"integrity": "sha512-gzpDH3vZI1FDL1Q8pXryACtNIW+idZ/zIZ8WqdTRzJuzxucazrG2gLXUS2ngcXQBn09Jyz4RUnU10Tu2N7/Hgg==",
"license": "MIT",
"peerDependencies": {
"expo": "*",
"react": "*",
"react-native": "*"
}
},
"node_modules/expo-background-fetch": { "node_modules/expo-background-fetch": {
"version": "13.0.5", "version": "13.0.5",
"resolved": "https://registry.npmjs.org/expo-background-fetch/-/expo-background-fetch-13.0.5.tgz", "resolved": "https://registry.npmjs.org/expo-background-fetch/-/expo-background-fetch-13.0.5.tgz",

3
package.json
View File

@ -58,7 +58,8 @@
"react-native-webview": "13.12.5", "react-native-webview": "13.12.5",
"sqlite": "^5.1.1", "sqlite": "^5.1.1",
"sqlite3": "^5.1.7", "sqlite3": "^5.1.7",
"whisper.rn": "^0.3.9" "whisper.rn": "^0.3.9",
"expo-audio": "~0.3.4"
}, },
"jest": { "jest": {
"preset": "jest-expo", "preset": "jest-expo",