Compare commits

6 Commits

25 changed files with 495 additions and 616 deletions

1
.gitignore vendored
View File

@ -36,3 +36,4 @@ yarn-error.*
*.tsbuildinfo
coverage/**/*
assets/whisper

3
.gitmodules vendored
View File

@ -4,3 +4,6 @@
[submodule "knex-expo-sqlite-dialect"]
path = knex-expo-sqlite-dialect
url = https://github.com/expo/knex-expo-sqlite-dialect.git
[submodule "whisper_android"]
path = whisper_android
url = https://github.com/vilassn/whisper_android.git

View File

@ -1,2 +1,3 @@
{
"java.compile.nullAnalysis.mode": "disabled"
}

View File

@ -21,11 +21,14 @@ react {
/* Folders */
// The root of your project, i.e. where "package.json" lives. Default is '../..'
// root = file("../../")
root = file("../../")
// The folder where the react-native NPM package is. Default is ../../node_modules/react-native
// reactNativeDir = file("../../node_modules/react-native")
// The folder where the react-native Codegen package is. Default is ../../node_modules/@react-native/codegen
// codegenDir = file("../../node_modules/@react-native/codegen")
// the sub-project for whisper-native
// whisper_android = file("$root/whisper-android/")
/* Variants */
// The list of variants to that are debuggable. For those we're going to

View File

@ -12,3 +12,5 @@
-keep class com.facebook.react.turbomodule.** { *; }
# Add any project specific keep options here:
# whisper.rn
-keep class com.rnwhisper.** { *; }

View File

@ -3,5 +3,5 @@
<uses-permission android:name="android.permission.SYSTEM_ALERT_WINDOW"/>
<application android:usesCleartextTraffic="true" tools:targetApi="28" tools:ignore="GoogleAppIndexingWarning" tools:replace="android:usesCleartextTraffic" />
<application android:largeHeap="true" android:usesCleartextTraffic="true" tools:targetApi="28" tools:ignore="GoogleAppIndexingWarning" tools:replace="android:usesCleartextTraffic" />
</manifest>

View File

@ -1,7 +1,9 @@
<manifest xmlns:android="http://schemas.android.com/apk/res/android">
<uses-permission android:name="android.permission.INTERNET"/>
<uses-permission android:name="android.permission.MODIFY_AUDIO_SETTINGS"/>
<uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE"/>
<uses-permission android:name="android.permission.RECEIVE_BOOT_COMPLETED"/>
<uses-permission android:name="android.permission.RECORD_AUDIO"/>
<uses-permission android:name="android.permission.SYSTEM_ALERT_WINDOW"/>
<uses-permission android:name="android.permission.VIBRATE"/>
<uses-permission android:name="android.permission.WAKE_LOCK"/>
@ -13,7 +15,7 @@
<data android:scheme="https"/>
</intent>
</queries>
<application android:name=".MainApplication" android:label="@string/app_name" android:icon="@mipmap/ic_launcher" android:roundIcon="@mipmap/ic_launcher_round" android:allowBackup="true" android:theme="@style/AppTheme" android:supportsRtl="true">
<application android:name=".MainApplication" android:label="@string/app_name" android:icon="@mipmap/ic_launcher" android:roundIcon="@mipmap/ic_launcher_round" android:allowBackup="true" android:theme="@style/AppTheme" android:supportsRtl="true" android:largeHeap="true">
<meta-data android:name="expo.modules.updates.ENABLED" android:value="false"/>
<meta-data android:name="expo.modules.updates.EXPO_UPDATES_CHECK_ON_LAUNCH" android:value="ALWAYS"/>
<meta-data android:name="expo.modules.updates.EXPO_UPDATES_LAUNCH_WAIT_MS" android:value="0"/>

View File

@ -1,65 +0,0 @@
package com.anonymous.translationterrace
import expo.modules.splashscreen.SplashScreenManager
import android.os.Build
import android.os.Bundle
import com.facebook.react.ReactActivity
import com.facebook.react.ReactActivityDelegate
import com.facebook.react.defaults.DefaultNewArchitectureEntryPoint.fabricEnabled
import com.facebook.react.defaults.DefaultReactActivityDelegate
import expo.modules.ReactActivityDelegateWrapper
class MainActivity : ReactActivity() {
override fun onCreate(savedInstanceState: Bundle?) {
// Set the theme to AppTheme BEFORE onCreate to support
// coloring the background, status bar, and navigation bar.
// This is required for expo-splash-screen.
// setTheme(R.style.AppTheme);
// @generated begin expo-splashscreen - expo prebuild (DO NOT MODIFY) sync-f3ff59a738c56c9a6119210cb55f0b613eb8b6af
SplashScreenManager.registerOnActivity(this)
// @generated end expo-splashscreen
super.onCreate(null)
}
/**
* Returns the name of the main component registered from JavaScript. This is used to schedule
* rendering of the component.
*/
override fun getMainComponentName(): String = "main"
/**
* Returns the instance of the [ReactActivityDelegate]. We use [DefaultReactActivityDelegate]
* which allows you to enable New Architecture with a single boolean flags [fabricEnabled]
*/
override fun createReactActivityDelegate(): ReactActivityDelegate {
return ReactActivityDelegateWrapper(
this,
BuildConfig.IS_NEW_ARCHITECTURE_ENABLED,
object : DefaultReactActivityDelegate(
this,
mainComponentName,
fabricEnabled
){})
}
/**
* Align the back button behavior with Android S
* where moving root activities to background instead of finishing activities.
* @see <a href="https://developer.android.com/reference/android/app/Activity#onBackPressed()">onBackPressed</a>
*/
override fun invokeDefaultOnBackPressed() {
if (Build.VERSION.SDK_INT <= Build.VERSION_CODES.R) {
if (!moveTaskToBack(false)) {
// For non-root activities, use the default implementation to finish them.
super.invokeDefaultOnBackPressed()
}
return
}
// Use the default back button implementation on Android S
// because it's doing more than [Activity.moveTaskToBack] in fact.
super.invokeDefaultOnBackPressed()
}
}

View File

@ -1,57 +0,0 @@
package com.anonymous.translationterrace
import android.app.Application
import android.content.res.Configuration
import com.facebook.react.PackageList
import com.facebook.react.ReactApplication
import com.facebook.react.ReactNativeHost
import com.facebook.react.ReactPackage
import com.facebook.react.ReactHost
import com.facebook.react.defaults.DefaultNewArchitectureEntryPoint.load
import com.facebook.react.defaults.DefaultReactNativeHost
import com.facebook.react.soloader.OpenSourceMergedSoMapping
import com.facebook.soloader.SoLoader
import expo.modules.ApplicationLifecycleDispatcher
import expo.modules.ReactNativeHostWrapper
class MainApplication : Application(), ReactApplication {
override val reactNativeHost: ReactNativeHost = ReactNativeHostWrapper(
this,
object : DefaultReactNativeHost(this) {
override fun getPackages(): List<ReactPackage> {
val packages = PackageList(this).packages
// Packages that cannot be autolinked yet can be added manually here, for example:
// packages.add(new MyReactNativePackage());
return packages
}
override fun getJSMainModuleName(): String = ".expo/.virtual-metro-entry"
override fun getUseDeveloperSupport(): Boolean = BuildConfig.DEBUG
override val isNewArchEnabled: Boolean = BuildConfig.IS_NEW_ARCHITECTURE_ENABLED
override val isHermesEnabled: Boolean = BuildConfig.IS_HERMES_ENABLED
}
)
override val reactHost: ReactHost
get() = ReactNativeHostWrapper.createReactHost(applicationContext, reactNativeHost)
override fun onCreate() {
super.onCreate()
SoLoader.init(this, OpenSourceMergedSoMapping)
if (BuildConfig.IS_NEW_ARCHITECTURE_ENABLED) {
// If you opted-in for the New Architecture, we load the native entry point for this app.
load()
}
ApplicationLifecycleDispatcher.onApplicationCreate(this)
}
override fun onConfigurationChanged(newConfig: Configuration) {
super.onConfigurationChanged(newConfig)
ApplicationLifecycleDispatcher.onConfigurationChanged(this, newConfig)
}
}

View File

@ -0,0 +1,30 @@
package com.nativewhisperengine;
// react-native turbo spec
import com.nativewhisperengine.NativeWhisperEngineSpec;
// inherent context
import android.content.Context;
import com.facebook.react.bridge.ReactApplicationContext;
// packages we'll be using.
import com.whispertflite.engine.WhisperEngine;
public class NativeWhisperModule extends NativeLocalStorageSpec {
public static final String NAME = "NativeLocalStorage";
private WhisperEngineNative mNativeEngine;
public NativeWhisperModule(ReactApplicationContext reactContext) {
super(reactContext);
mNativeEngine = new WhisperEngineNative(reactContext);
}
@Override
public void loadWhisper(String path) {}
@Override
public void translate(String text, String language) {
}
}

View File

@ -0,0 +1,41 @@
package com.nativewhisperengine;
import com.facebook.react.BaseReactPackage;
import com.facebook.react.bridge.NativeModule;
import com.facebook.react.bridge.ReactApplicationContext;
import com.facebook.react.module.model.ReactModuleInfo;
import com.facebook.react.module.model.ReactModuleInfoProvider;
import java.util.HashMap;
import java.util.Map;
public class NativeWhisperPackage extends BaseReactPackage {
@Override
public NativeModule getModule(String name, ReactApplicationContext reactContext) {
if (name.equals(NativeWhisperModule.NAME)) {
return new NativeWhisperModule(reactContext);
} else {
return null;
}
}
@Override
public ReactModuleInfoProvider getReactModuleInfoProvider() {
return new ReactModuleInfoProvider() {
@Override
public Map<String, ReactModuleInfo> getReactModuleInfos() {
Map<String, ReactModuleInfo> map = new HashMap<>();
map.put(NativeWhisperModule.NAME, new ReactModuleInfo(
NativeWhisperModule.NAME, // name
NativeWhisperModule.NAME, // className
false, // canOverrideExistingModule
false, // needsEagerInit
false, // isCXXModule
true // isTurboModule
));
return map;
}
};
}
}

View File

@ -1,5 +1,5 @@
pluginManagement {
includeBuild(new File(["node", "--print", "require.resolve('@react-native/gradle-plugin/package.json', { paths: [require.resolve('react-native/package.json')] })"].execute(null, rootDir).text.trim()).getParentFile().toString())
includeBuild(file(["node", "--print", "require.resolve('@react-native/gradle-plugin/package.json', { paths: [require.resolve('react-native/package.json')] })"].execute(null, rootDir).text.trim()).getParentFile().toString())
}
plugins { id("com.facebook.react.settings") }
@ -26,13 +26,17 @@ rootProject.name = 'translation-terrace'
dependencyResolutionManagement {
versionCatalogs {
reactAndroidLibs {
from(files(new File(["node", "--print", "require.resolve('react-native/package.json')"].execute(null, rootDir).text.trim(), "../gradle/libs.versions.toml")))
from(files(file(["node", "--print", "require.resolve('react-native/package.json')"].execute(null, rootDir).text.trim(), "../gradle/libs.versions.toml")))
}
}
}
apply from: new File(["node", "--print", "require.resolve('expo/package.json')"].execute(null, rootDir).text.trim(), "../scripts/autolinking.gradle");
apply from: file(["node", "--print", "require.resolve('expo/package.json')"].execute(null, rootDir).text.trim(), "../scripts/autolinking.gradle")
useExpoModules()
include ':whisper-android'
project(':whisper-android').projectDir = file(rootProject.projectDir, '../whisper_native')
include ':app'
includeBuild(new File(["node", "--print", "require.resolve('@react-native/gradle-plugin/package.json', { paths: [require.resolve('react-native/package.json')] })"].execute(null, rootDir).text.trim()).getParentFile())
includeBuild(new File(["node", "--print", "require.resolve('@react-native/gradle-plugin/package.json', { paths: [require.resolve('react-native/package.json')] })"].execute(null, rootDir).text.trim()).getParentFile())

View File

@ -57,10 +57,11 @@
]
}
}
]
],
"expo-audio"
],
"experiments": {
"typedRoutes": true
}
}
}
}

View File

@ -115,13 +115,13 @@ export class Translator {
console.log(data);
return data.translatedText;
} else {
console.error(data);
console.error("Status %d: %s", res.status, JSON.stringify(data));
}
}
static async getDefault(defaultTarget: string | undefined = undefined) {
const settings = await Settings.getDefault();
const source = await settings.getHostLanguage();
const source = await settings.getHostLanguage() || "en";
return new Translator(
source,
defaultTarget,

61
app/lib/readstream.ts Normal file
View File

@ -0,0 +1,61 @@
/* eslint-disable unicorn/no-null */
import * as fs from 'expo-file-system';
import { Readable } from 'readable-stream';
class ExpoReadStream extends Readable {
private readonly fileUri: string;
private fileSize: number;
private currentPosition: number;
private readonly chunkSize: number;
constructor(fileUri: string, options: fs.ReadingOptions) {
super();
this.fileUri = fileUri;
this.fileSize = 0; // Initialize file size (could be fetched if necessary)
this.currentPosition = options.position ?? 0;
/**
* Default chunk size in bytes. React Native Expo will OOM at 110MB, so we set this to 1/100 of it to balance speed and memory usage and importantly the feedback for user.
* If this is too large, the progress bar will be stuck when down stream processing this chunk.
*/
this.chunkSize = options.length ?? 1024 * 1024;
void this._init();
}
async _init() {
try {
const fileInfo = await fs.getInfoAsync(this.fileUri, { size: true });
if (fileInfo.exists) {
this.fileSize = fileInfo.size ?? 0;
} else {
this.fileSize = 0;
}
} catch (error) {
this.emit('error', error);
}
}
_read() {
const readingOptions = {
encoding: fs.EncodingType.Base64,
position: this.currentPosition,
length: this.chunkSize,
} satisfies fs.ReadingOptions;
fs.readAsStringAsync(this.fileUri, readingOptions).then(chunk => {
if (chunk.length === 0) {
// End of the stream
this.emit('progress', 1);
this.push(null);
} else {
this.currentPosition = Math.min(this.chunkSize + this.currentPosition, this.fileSize);
this.emit('progress', this.fileSize === 0 ? 0.5 : (this.currentPosition / this.fileSize));
this.push(Buffer.from(chunk, 'base64'));
}
}).catch(error => {
this.emit('error', error);
});
}
}
export function createReadStream(fileUri: string, options: { encoding?: fs.EncodingType; end?: number; highWaterMark?: number; start?: number } = {}): ExpoReadStream {
return new ExpoReadStream(fileUri, options);
}

View File

@ -1,5 +1,9 @@
import { TextDecoder } from "util";
export async function arrbufToStr(arrayBuffer : ArrayBuffer) {
export function arrbufToStr(arrayBuffer : ArrayBuffer) {
return new TextDecoder().decode(new Uint8Array(arrayBuffer));
}
export function strToArrBuf(input : string) : Uint8Array<ArrayBufferLike> {
return new TextEncoder().encode(input)
}

View File

@ -1,284 +1,14 @@
import { Platform } from "react-native";
import * as FileSystem from "expo-file-system";
import { File, Paths } from "expo-file-system/next";
import { getDb } from "./db";
import * as Crypto from "expo-crypto";
import { arrbufToStr } from "./util";
import FileSystem from "expo-file-system"
import { pathToFileURLString } from "expo-file-system/src/next/pathUtilities/url";
export const WHISPER_MODEL_PATH = Paths.join(
FileSystem.documentDirectory || "file:///",
"whisper"
);
export const WHISPER_MODEL_PATH = Paths.join("..", "..", "assets", "whisper");
export const WHISPER_MODEL_DIR = new File(WHISPER_MODEL_PATH);
// Thanks to https://medium.com/@fabi.mofar/downloading-and-saving-files-in-react-native-expo-5b3499adda84
export const WHISPER_MODEL_SMALL_PATH = "file://../../assets/whisper/whisper-small.bin";
export async function saveFile(
uri: string,
filename: string,
mimetype: string
) {
if (Platform.OS === "android") {
const permissions =
await FileSystem.StorageAccessFramework.requestDirectoryPermissionsAsync();
if (permissions.granted) {
const base64 = await FileSystem.readAsStringAsync(uri, {
encoding: FileSystem.EncodingType.Base64,
});
await FileSystem.StorageAccessFramework.createFileAsync(
permissions.directoryUri,
filename,
mimetype
)
.then(async (uri) => {
await FileSystem.writeAsStringAsync(uri, base64, {
encoding: FileSystem.EncodingType.Base64,
});
})
.catch((e) => console.log(e));
} else {
shareAsync(uri);
}
} else {
shareAsync(uri);
}
}
function shareAsync(uri: string) {
throw new Error("Function not implemented.");
}
export const WHISPER_MODEL_TAGS = ["small", "medium", "large"];
export type whisper_model_tag_t = "small" | "medium" | "large";
export const WHISPER_MODELS = {
small: {
source:
"https://huggingface.co/openai/whisper-small/blob/resolve/pytorch_model.bin",
target: "small.bin",
label: "Small",
size: 967092419,
},
medium: {
source:
"https://huggingface.co/openai/whisper-medium/resolve/main/pytorch_model.bin",
target: "medium.bin",
label: "Medium",
size: 3055735323,
},
large: {
source:
"https://huggingface.co/openai/whisper-large/resolve/main/pytorch_model.bin",
target: "large.bin",
label: "Large",
size: 6173629930,
},
} as {
[key: whisper_model_tag_t]: {
source: string;
target: string;
label: string;
size: number;
};
};
export type whisper_tag_t = "small" | "medium" | "large";
export type hf_channel_t = "raw" | "resolve";
export const HF_URL_BASE = "https://huggingface.co/openai/whisper-";
export const HF_URL_RAW = "raw";
export const HF_URL_RESOLVE = "resolve";
export const HF_URL_END = "/main/pytorch_model.bin";
export function create_hf_url(tag: whisper_tag_t, channel: hf_channel_t) {
return `${HF_URL_BASE}${tag}/${channel}${HF_URL_END}`;
}
export type hf_metadata_t = {
version: string;
oid: string;
size: string;
};
export type download_status_t = {
doesTargetExist: boolean;
isDownloadComplete: boolean;
hasDownloadStarted: boolean;
progress?: {
current: number;
total: number;
remaining: number;
percentRemaining: number;
};
};
export class WhisperFile {
hf_metadata: hf_metadata_t | undefined;
target_hash: string | undefined;
does_target_exist: boolean = false;
download_data: FileSystem.DownloadProgressData | undefined;
constructor(
public tag: whisper_model_tag_t,
private targetFileName?: string,
public label?: string,
public size?: number
) {
this.targetFileName = this.targetFileName || `${tag}.bin`;
this.label =
this.label || `${tag[0].toUpperCase()}${tag.substring(1).toLowerCase()}`;
}
get targetPath() {
return Paths.join(WHISPER_MODEL_PATH, this.targetFileName as string);
}
get targetFile() {
return new File(this.targetPath);
}
async getTargetInfo() {
return await FileSystem.getInfoAsync(this.targetPath);
}
async updateTargetExistence() {
this.does_target_exist = (await this.getTargetInfo()).exists;
}
public async getTargetSha() {
await this.updateTargetExistence();
if (!this.does_target_exist) {
console.debug("%s does not exist", this.targetPath);
return undefined;
}
return await Crypto.digest(
Crypto.CryptoDigestAlgorithm.SHA256,
this.targetFile.bytes()
);
}
public async updateTargetHash() {
const targetSha = await this.getTargetSha();
if (!targetSha) return;
this.target_hash = await arrbufToStr(targetSha);
}
get isHashValid() {
return this.target_hash === this.hf_metadata?.oid;
}
delete(ignoreErrors = true) {
try {
this.targetFile.delete();
} catch (err) {
console.error(err);
if (!ignoreErrors) {
throw err;
}
}
console.debug("Created %s", WHISPER_MODEL_DIR);
}
get modelUrl() {
return create_hf_url(this.tag, "resolve");
}
get metadataUrl() {
return create_hf_url(this.tag, "raw");
}
get percentDone() {
if (!this.download_data) return 0;
return (
(this.download_data.totalBytesWritten /
this.download_data.totalBytesExpectedToWrite) *
100
);
}
get percentLeft() {
if (!this.download_data) return 0;
return 100 - this.percentDone;
}
public async syncHfMetadata() {
try {
const resp = await fetch(this.metadataUrl, {
credentials: "include",
headers: {
"User-Agent":
"Mozilla/5.0 (X11; Linux x86_64; rv:135.0) Gecko/20100101 Firefox/135.0",
Accept:
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.5",
"Sec-GPC": "1",
"Upgrade-Insecure-Requests": "1",
"Sec-Fetch-Dest": "document",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Site": "cross-site",
"If-None-Match": '"8fa71cbce85078986b46fb97caec22039e73351a"',
Priority: "u=0, i",
},
method: "GET",
mode: "cors",
});
const text = await resp.text();
this.hf_metadata = Object.fromEntries(
text.split("\n").map((line) => line.split(" "))
) as hf_metadata_t;
} catch (err) {
console.error("Failed to fetch %s: %s", this.metadataUrl, err);
throw err;
}
}
async createDownloadResumable(
options: {
onData?: DownloadCallback | undefined;
} = {
onData: undefined,
}
) {
await this.syncHfMetadata();
// If the whisper model dir doesn't exist, create it.
if (!WHISPER_MODEL_DIR.exists) {
FileSystem.makeDirectoryAsync(WHISPER_MODEL_PATH, {
intermediates: true,
});
}
// Check for the existence of the target file
// If it exists, load the existing data.
await this.updateTargetExistence();
const existingData = this.does_target_exist
? this.targetFile.text()
: undefined;
// Create the resumable.
return FileSystem.createDownloadResumable(
this.modelUrl,
this.targetPath,
{},
async (data: FileSystem.DownloadProgressData) => {
this.download_data = data;
await this.syncHfMetadata();
await this.updateTargetHash();
await this.updateTargetExistence();
if (options.onData) await options.onData(this);
},
existingData ? existingData : undefined
);
}
}
export type DownloadCallback = (arg0: WhisperFile) => any;
export const WHISPER_FILES = {
small: new WhisperFile("small"),
medium: new WhisperFile("medium"),
large: new WhisperFile("large"),
};
export async function whisperModelExists() {
const file = new File(WHISPER_MODEL_PATH);
return file.exists;
}

View File

@ -1,9 +1,27 @@
import React, { useState, useEffect } from "react";
import { ScrollView, StyleSheet, Text, TouchableHighlight, View } from "react-native";
import {
Alert,
ScrollView,
StyleSheet,
Text,
TouchableHighlight,
View,
} from "react-native";
import { useNavigation, Route } from "@react-navigation/native";
import { Conversation, Message } from "@/app/lib/conversation";
import MessageBubble from "@/components/ui/MessageBubble";
import { CachedTranslator, LanguageServer, language_matrix_entry } from "@/app/i18n/api";
import {
CachedTranslator,
LanguageServer,
language_matrix_entry,
} from "@/app/i18n/api";
import {
WHISPER_MODEL_SMALL_PATH,
whisperModelExists,
} from "@/app/lib/whisper";
import { initWhisper, WhisperContext } from "whisper.rn";
import { useAudioRecorder, AudioModule, RecordingPresets } from "expo-audio";
import FileSystem from "expo-file-system";
const lasOptions = {
sampleRate: 32000, // default is 44100 but 32000 is adequate for accurate voice recognition
@ -14,11 +32,19 @@ const lasOptions = {
};
// LiveAudioStream.init(lasOptions as any);
const ConversationThread = ({ route } : {route?: Route<"Conversation", {conversation : Conversation}>}) => {
const ConversationThread = ({
route,
}: {
route?: Route<"Conversation", { conversation: Conversation }>;
}) => {
const navigation = useNavigation();
if (!route) {
return (<View><Text>Missing Params!</Text></View>)
return (
<View>
<Text>Missing Params!</Text>
</View>
);
}
/* 2. Get the param */
@ -27,60 +53,128 @@ const ConversationThread = ({ route } : {route?: Route<"Conversation", {conversa
const [messages, setMessages] = useState<Message[]>([]);
const [guestSpeak, setGuestSpeak] = useState<string | undefined>();
const [guestSpeakLoaded, setGuestSpeakLoaded] = useState<boolean>(false);
const [whisperContext, setWhisperContext] = useState<
WhisperContext | undefined
>();
const [cachedTranslator, setCachedTranslator] = useState<
undefined | CachedTranslator
>();
const [languageLabels, setLanguageLabels] = useState<undefined | {
hostNative: {
host: string,
guest: string,
},
guestNative: {
host: string,
guest: string,
}
}>()
const [languageLabels, setLanguageLabels] = useState<
| undefined
| {
hostNative: {
host: string;
guest: string;
};
guestNative: {
host: string;
guest: string;
};
}
>();
// recorder settings
const audioRecorder = useAudioRecorder(RecordingPresets.HIGH_QUALITY);
const record = async () => {
await audioRecorder.prepareToRecordAsync();
audioRecorder.record();
};
const stopRecording = async () => {
// The recording will be available on `audioRecorder.uri`.
await audioRecorder.stop();
};
useEffect(() => {
(async () => {
(async function () {
const languageServer = await LanguageServer.getDefault();
const languages = await languageServer.fetchLanguages();
const cc = new CachedTranslator(
"en",
conversation.guest.language,
languageServer,
)
setCachedTranslator(cc);
setGuestSpeak(await cc.translate("Speak"));
const hostLang1 = languages[conversation.host.language].name;
const guestLang1 = languages[conversation.host.language].name;
const hostLang2 = await cc.translate(languages[conversation.host.language].name);
const guestLang2 = await cc.translate(languages[conversation.host.language].name);
setLanguageLabels({
hostNative: {
host: hostLang1,
guest: guestLang1,
},
guestNative: {
host: hostLang2,
guest: guestLang2,
try {
const languages = await languageServer.fetchLanguages(5000);
const cachedTranslator = new CachedTranslator(
"en",
conversation.guest.language,
languageServer
);
console.log("Set cached translator from %s", languageServer.baseUrl);
setCachedTranslator(cachedTranslator);
try {
if (!(await whisperModelExists())) {
throw new Error(`${WHISPER_MODEL_SMALL_PATH} does not exist`);
}
} catch (err) {
console.error(
`Could not determine if %s exists: %s`,
WHISPER_MODEL_SMALL_PATH,
err
);
throw err;
}
})
try {
setWhisperContext(
await initWhisper({
filePath: WHISPER_MODEL_SMALL_PATH,
})
);
} catch (err) {
console.error(err);
throw err;
}
// recorder settings
(async () => {
const status = await AudioModule.requestRecordingPermissionsAsync();
if (!status.granted) {
Alert.alert("Permission to access microphone was denied");
}
})();
setGuestSpeak(await cachedTranslator.translate("Speak"));
const hostLang1 = languages[conversation.host.language].name;
const guestLang1 = languages[conversation.host.language].name;
const hostLang2 = await cachedTranslator.translate(
languages[conversation.host.language].name
);
const guestLang2 = await cachedTranslator.translate(
languages[conversation.host.language].name
);
setLanguageLabels({
hostNative: {
host: hostLang1,
guest: guestLang1,
},
guestNative: {
host: hostLang2,
guest: guestLang2,
},
});
} catch (err) {
console.error(
"Could not set translator from %s: %s",
languageServer.baseUrl,
err
);
}
})();
const updateMessages = (c: Conversation) => {
setMessages([...c]);
};
conversation.onAddMessage = updateMessages;
conversation.onTranslationDone = updateMessages;
if (!conversation) {
console.warn("Conversation is null or undefined.");
}
return () => {
conversation.onAddMessage = undefined;
conversation.onTranslationDone = undefined;
};
conversation.on("add_message", updateMessages);
conversation.on("translation_done", updateMessages);
// return () => {
// conversation.on("add_message", undefined);
// conversation.on("translation_done", undefined);
// };
}, [conversation, guestSpeak]);
const renderMessages = () =>
@ -90,11 +184,17 @@ const ConversationThread = ({ route } : {route?: Route<"Conversation", {conversa
return cachedTranslator ? (
<View style={{ flex: 1, flexDirection: "column" }}>
{languageLabels && (<View style={styles.languageLabels}>
<Text style={styles.nativeHostLabel}>{ languageLabels.hostNative.host } / { languageLabels.hostNative.guest }</Text>
<Text style={styles.nativeGuestLabel}>{ languageLabels.guestNative.host } / { languageLabels.guestNative.guest }</Text>
</View>)
}
{languageLabels && (
<View style={styles.languageLabels}>
<Text style={styles.nativeHostLabel}>
{languageLabels.hostNative.host} / {languageLabels.hostNative.guest}
</Text>
<Text style={styles.nativeGuestLabel}>
{languageLabels.guestNative.host} /{" "}
{languageLabels.guestNative.guest}
</Text>
</View>
)}
<ScrollView
style={{
borderColor: "black",
@ -134,15 +234,9 @@ const ConversationThread = ({ route } : {route?: Route<"Conversation", {conversa
};
const styles = StyleSheet.create({
languageLabels: {
},
nativeHostLabel: {
},
nativeGuestLabel: {
},
})
languageLabels: {},
nativeHostLabel: {},
nativeGuestLabel: {},
});
export default ConversationThread;

View File

@ -18,7 +18,7 @@ export function LanguageSelection(props: {
}) {
const [languages, setLanguages] = useState<language_matrix | undefined>();
const [languagesLoaded, setLanguagesLoaded] = useState<boolean>(false);
const [translator, setTranslator] = useState<Translator|undefined>();
const [translator, setTranslator] = useState<Translator | undefined>();
const nav = useNavigation();
@ -36,7 +36,7 @@ export function LanguageSelection(props: {
// Replace with your actual async data fetching logic
setTranslator(await CachedTranslator.getDefault());
const languageServer = await LanguageServer.getDefault();
const languages = await languageServer.fetchLanguages(5000);
const languages = await languageServer.fetchLanguages(10000);
setLanguages(languages);
setLanguagesLoaded(true);
} catch (error) {
@ -56,7 +56,7 @@ export function LanguageSelection(props: {
{(languages && languagesLoaded) ? Object.entries(languages).filter((l) => (LANG_FLAGS as any)[l[0]] !== undefined).map(
([lang, lang_entry]) => {
return (
<ISpeakButton language={lang_entry} key={lang_entry.code} onLangSelected={onLangSelected} translator={translator} />
<ISpeakButton language={lang_entry} key={lang_entry.code} onLangSelected={onLangSelected} translator={translator} />
);
}
) : <Text>Waiting...</Text>

View File

@ -1,23 +1,11 @@
import React, { useState, useEffect } from "react";
import { View, Text, TextInput, Pressable, StyleSheet } from "react-native";
import {
WHISPER_FILES,
WhisperFile,
download_status_t,
whisper_tag_t,
} from "@/app/lib/whisper";
import { View, Text, TextInput, StyleSheet } from "react-native";
import { Settings } from "@/app/lib/settings";
import { Picker } from "@react-native-picker/picker";
import {
LanguageServer,
language_matrix,
language_matrix_entry,
} from "@/app/i18n/api";
const WHISPER_MODELS = {
small: new WhisperFile("small"),
medium: new WhisperFile("medium"),
large: new WhisperFile("large"),
};
const LIBRETRANSLATE_BASE_URL = "https://translate.argosopentech.com/translate";
@ -33,17 +21,6 @@ const SettingsComponent = () => {
success: boolean;
error?: string;
} | null>(null);
const [whisperModel, setWhisperModel] =
useState<keyof typeof WHISPER_MODELS>("small");
const [whisperFile, setWhisperFile] = useState<WhisperFile | undefined>();
const [whisperFileExists, setWhisperFileExists] = useState<boolean>(false);
const [isWhisperHashValid, setIsWhisperHashValid] = useState<boolean>(false);
const [downloader, setDownloader] = useState<any>(null);
const [bytesDone, setBytesDone] = useState<number | undefined>();
const [bytesRemaining, setBytesRemaining] = useState<number | undefined>();
const [statusTimeout, setStatusTimeout] = useState<
NodeJS.Timeout | undefined
>();
useEffect(() => {
(async function () {
@ -52,18 +29,9 @@ const SettingsComponent = () => {
setLibretranslateBaseUrl(
(await settings.getLibretranslateBaseUrl()) || LIBRETRANSLATE_BASE_URL
);
setWhisperModel((await settings.getWhisperModel()) || "small");
setWhisperFile(WHISPER_FILES[whisperModel]);
await whisperFile?.syncHfMetadata();
await whisperFile?.updateTargetExistence();
await whisperFile?.updateTargetHash();
})();
}, [whisperFile]);
});
const getLanguageOptions = async () => {
const languageServer = await LanguageServer.getDefault();
setLanguageOptions(await languageServer.fetchLanguages());
};
const handleHostLanguageChange = async (lang: string) => {
const settings = await Settings.getDefault();
@ -81,63 +49,17 @@ const SettingsComponent = () => {
const checkLangServerConnection = async (baseUrl: string) => {
try {
// Replace with actual connection check logic
setLangServerConn({ success: true });
const testResult = await fetch(baseUrl, {
method: "HEAD",
});
if (testResult.status !== 200) {
setLangServerConn({ success: true, error: testResult.statusText });
}
} catch (error) {
setLangServerConn({ success: false, error: `${error}` });
}
};
const handleWhisperModelChange = async (model: whisper_tag_t) => {
const settings = await Settings.getDefault();
await settings.setWhisperModel(model);
setWhisperModel(model);
const wFile = WHISPER_FILES[whisperModel];
await wFile.syncHfMetadata();
await wFile.updateTargetExistence();
await wFile.updateTargetHash();
setIsWhisperHashValid(wFile.isHashValid);
setWhisperFile(wFile);
setWhisperFileExists(wFile.does_target_exist);
};
const doSetDownloadStatus = (arg0: WhisperFile) => {
console.log("Downloading ....")
setIsWhisperHashValid(arg0.isHashValid);
setBytesDone(arg0.download_data?.totalBytesWritten);
setBytesRemaining(arg0.download_data?.totalBytesExpectedToWrite);
};
const doDownload = async () => {
if (!whisperModel) {
throw new Error("Could not start download because whisperModel not set.");
}
console.log("Starting download of %s", whisperModel);
if (!whisperFile) throw new Error("No whisper file");
try {
const resumable = await whisperFile.createDownloadResumable({
onData: doSetDownloadStatus,
});
setDownloader(resumable);
await resumable.resumeAsync();
} catch (error) {
console.error("Failed to download whisper model:", error);
}
};
const doStopDownload = async () => {
downloader.cancelAsync();
setDownloader(null);
};
const doDelete = async () => {
const whisperFile = WHISPER_MODELS[whisperModel];
whisperFile.delete();
setStatusTimeout(undefined);
};
return hostLanguage && libretranslateBaseUrl ? (
<View style={styles.container}>
<Text style={styles.label}>Host Language:</Text>
@ -170,41 +92,6 @@ const SettingsComponent = () => {
Error connecting to {libretranslateBaseUrl}: {langServerConn.error}
</Text>
))}
<Picker
selectedValue={whisperModel}
style={{ height: 50, width: "100%" }}
onValueChange={handleWhisperModelChange}
accessibilityHint="whisper models"
>
{Object.entries(WHISPER_MODELS).map(([key, whisperFile]) => (
<Picker.Item
key={whisperFile.tag}
label={whisperFile.label}
value={key}
/>
))}
</Picker>
<View>
{/* <Text>whisper file: { whisperFile?.tag }</Text> */}
{whisperFile &&
( whisperFileExists && (<Pressable onPress={doDelete} style={styles.deleteButton}>
<Text>DELETE {whisperModel.toUpperCase()}</Text>
</Pressable>))
}
<Pressable onPress={doDownload} style={styles.pauseDownloadButton}>
<Text>DOWNLOAD {whisperModel.toUpperCase()}</Text>
</Pressable>
))}
{bytesDone && bytesRemaining && (
<View>
<Text>
{bytesDone} of{" "}
{bytesRemaining} (
{bytesDone / bytesRemaining * 100} %){" "}
</Text>
</View>
)}
</View>
</View>
) : (
<View>
@ -219,11 +106,12 @@ const styles = StyleSheet.create({
flexDirection: "row",
},
downloadButton: {
backgroundColor: "darkblue",
backgroundColor: "#236b9f",
padding: 20,
margin: 10,
flex: 3,
flex: 1,
flexDirection: "column",
alignItems: "center",
},
deleteButton: {
backgroundColor: "darkred",
@ -241,11 +129,11 @@ const styles = StyleSheet.create({
},
buttonText: {
color: "#fff",
flex: 1,
fontSize: 16,
alignSelf: "center",
textAlign: "center",
textAlignVertical: "top",
// flex: 1,
// fontSize: 16,
// alignSelf: "center",
// textAlign: "center",
// textAlignVertical: "top",
},
container: {
flex: 1,

View File

@ -92,7 +92,7 @@ const ISpeakButton = (props: ISpeakButtonProps) => {
}, []);
const countries =
// @ts-ignore
// @ts-ignore
DEFAULT_FLAGS[props.language.code] || chooseCountry(props.language.code);
return title ? (
@ -107,10 +107,7 @@ const ISpeakButton = (props: ISpeakButtonProps) => {
{countries &&
countries.map((c) => {
return (
<View>
<Text>{c}</Text>
<CountryFlag isoCode={c} size={25} key={c} />
</View>
<CountryFlag isoCode={c} size={25} key={c} />
);
})}
</View>

129
package-lock.json generated
View File

@ -17,6 +17,7 @@
"@react-navigation/bottom-tabs": "^7.2.0",
"@react-navigation/native-stack": "^7.2.0",
"expo": "~52.0.28",
"expo-audio": "~0.3.4",
"expo-background-fetch": "~13.0.5",
"expo-blur": "~14.0.3",
"expo-constants": "~17.0.6",
@ -49,6 +50,7 @@
"react-native-sqlite-storage": "^6.0.1",
"react-native-web": "~0.19.13",
"react-native-webview": "13.12.5",
"readable-stream": "^4.7.0",
"sqlite": "^5.1.1",
"sqlite3": "^5.1.7",
"whisper.rn": "^0.3.9"
@ -65,6 +67,7 @@
"@types/react-native-sqlite-storage": "^6.0.5",
"@types/react-navigation": "^3.0.8",
"@types/react-test-renderer": "^18.3.1",
"@types/readable-stream": "^4.0.18",
"babel-jest": "^29.7.0",
"babel-plugin-module-resolver": "^5.0.2",
"expo": "~52.0.28",
@ -5046,6 +5049,24 @@
"@types/react": "^18"
}
},
"node_modules/@types/readable-stream": {
"version": "4.0.18",
"resolved": "https://registry.npmjs.org/@types/readable-stream/-/readable-stream-4.0.18.tgz",
"integrity": "sha512-21jK/1j+Wg+7jVw1xnSwy/2Q1VgVjWuFssbYGTREPUBeZ+rqVFl2udq0IkxzPC0ZhOzVceUbyIACFZKLqKEBlA==",
"dev": true,
"license": "MIT",
"dependencies": {
"@types/node": "*",
"safe-buffer": "~5.1.1"
}
},
"node_modules/@types/readable-stream/node_modules/safe-buffer": {
"version": "5.1.2",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
"dev": true,
"license": "MIT"
},
"node_modules/@types/stack-utils": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.3.tgz",
@ -5589,6 +5610,21 @@
"node": "^12.13.0 || ^14.15.0 || >=16.0.0"
}
},
"node_modules/are-we-there-yet/node_modules/readable-stream": {
"version": "3.6.2",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
"integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==",
"license": "MIT",
"optional": true,
"dependencies": {
"inherits": "^2.0.3",
"string_decoder": "^1.1.1",
"util-deprecate": "^1.0.1"
},
"engines": {
"node": ">= 6"
}
},
"node_modules/arg": {
"version": "5.0.2",
"resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz",
@ -6030,6 +6066,20 @@
"readable-stream": "^3.4.0"
}
},
"node_modules/bl/node_modules/readable-stream": {
"version": "3.6.2",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
"integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==",
"license": "MIT",
"dependencies": {
"inherits": "^2.0.3",
"string_decoder": "^1.1.1",
"util-deprecate": "^1.0.1"
},
"engines": {
"node": ">= 6"
}
},
"node_modules/bn.js": {
"version": "5.2.1",
"resolved": "https://registry.npmjs.org/bn.js/-/bn.js-5.2.1.tgz",
@ -8162,6 +8212,17 @@
"react-native": "*"
}
},
"node_modules/expo-audio": {
"version": "0.3.5",
"resolved": "https://registry.npmjs.org/expo-audio/-/expo-audio-0.3.5.tgz",
"integrity": "sha512-gzpDH3vZI1FDL1Q8pXryACtNIW+idZ/zIZ8WqdTRzJuzxucazrG2gLXUS2ngcXQBn09Jyz4RUnU10Tu2N7/Hgg==",
"license": "MIT",
"peerDependencies": {
"expo": "*",
"react": "*",
"react-native": "*"
}
},
"node_modules/expo-background-fetch": {
"version": "13.0.5",
"resolved": "https://registry.npmjs.org/expo-background-fetch/-/expo-background-fetch-13.0.5.tgz",
@ -15204,17 +15265,43 @@
}
},
"node_modules/readable-stream": {
"version": "3.6.2",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
"integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==",
"version": "4.7.0",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz",
"integrity": "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==",
"license": "MIT",
"dependencies": {
"inherits": "^2.0.3",
"string_decoder": "^1.1.1",
"util-deprecate": "^1.0.1"
"abort-controller": "^3.0.0",
"buffer": "^6.0.3",
"events": "^3.3.0",
"process": "^0.11.10",
"string_decoder": "^1.3.0"
},
"engines": {
"node": ">= 6"
"node": "^12.22.0 || ^14.17.0 || >=16.0.0"
}
},
"node_modules/readable-stream/node_modules/buffer": {
"version": "6.0.3",
"resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz",
"integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/feross"
},
{
"type": "patreon",
"url": "https://www.patreon.com/feross"
},
{
"type": "consulting",
"url": "https://feross.org/support"
}
],
"license": "MIT",
"dependencies": {
"base64-js": "^1.3.1",
"ieee754": "^1.2.1"
}
},
"node_modules/readline": {
@ -16439,6 +16526,20 @@
"readable-stream": "^3.5.0"
}
},
"node_modules/stream-browserify/node_modules/readable-stream": {
"version": "3.6.2",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
"integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==",
"license": "MIT",
"dependencies": {
"inherits": "^2.0.3",
"string_decoder": "^1.1.1",
"util-deprecate": "^1.0.1"
},
"engines": {
"node": ">= 6"
}
},
"node_modules/stream-buffers": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/stream-buffers/-/stream-buffers-2.2.0.tgz",
@ -16837,6 +16938,20 @@
"node": ">=6"
}
},
"node_modules/tar-stream/node_modules/readable-stream": {
"version": "3.6.2",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
"integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==",
"license": "MIT",
"dependencies": {
"inherits": "^2.0.3",
"string_decoder": "^1.1.1",
"util-deprecate": "^1.0.1"
},
"engines": {
"node": ">= 6"
}
},
"node_modules/tar/node_modules/fs-minipass": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz",

View File

@ -9,7 +9,8 @@
"updateSnapshots": "jest -u --coverage=false",
"start": "expo start",
"reset-project": "node ./scripts/reset-project.js",
"android": "expo prebuild --npm -p android",
"prebuild:android": "expo prebuild --npm -p android",
"android": "expo run:android",
"ios": "expo prebuild --npm -p android --offline",
"web": "expo start --offline --web",
"lint": "expo lint"
@ -24,6 +25,7 @@
"@react-navigation/bottom-tabs": "^7.2.0",
"@react-navigation/native-stack": "^7.2.0",
"expo": "~52.0.28",
"expo-audio": "~0.3.4",
"expo-background-fetch": "~13.0.5",
"expo-blur": "~14.0.3",
"expo-constants": "~17.0.6",
@ -56,10 +58,19 @@
"react-native-sqlite-storage": "^6.0.1",
"react-native-web": "~0.19.13",
"react-native-webview": "13.12.5",
"readable-stream": "^4.7.0",
"sqlite": "^5.1.1",
"sqlite3": "^5.1.7",
"whisper.rn": "^0.3.9"
},
"codegenConfig": {
"name": "NativeWhisperEngineSpec",
"type": "modules",
"jsSrcsDir": "specs",
"android": {
"javaPackageName": "com.nativewhisperengine"
}
},
"jest": {
"preset": "jest-expo",
"testPathIgnorePatterns": [
@ -85,6 +96,7 @@
"@types/react-native-sqlite-storage": "^6.0.5",
"@types/react-navigation": "^3.0.8",
"@types/react-test-renderer": "^18.3.1",
"@types/readable-stream": "^4.0.18",
"babel-jest": "^29.7.0",
"babel-plugin-module-resolver": "^5.0.2",
"expo": "~52.0.28",

11
specs/WhisperEngine.ts Normal file
View File

@ -0,0 +1,11 @@
import type {TurboModule} from 'react-native';
import {TurboModuleRegistry} from 'react-native';
export interface Spec extends TurboModule {
loadWhisper(path: string): void;
translate() : void;
}
export default TurboModuleRegistry.getEnforcing<Spec>(
'NativeWhisperEngine',
);

1
whisper_android Submodule

Submodule whisper_android added at 08b766af19