Compare commits
2 Commits
123933d459
...
0ba5c4b309
Author | SHA1 | Date | |
---|---|---|---|
|
0ba5c4b309 | ||
|
e61fb43ee3 |
1
.gitignore
vendored
1
.gitignore
vendored
@ -36,3 +36,4 @@ yarn-error.*
|
|||||||
*.tsbuildinfo
|
*.tsbuildinfo
|
||||||
|
|
||||||
coverage/**/*
|
coverage/**/*
|
||||||
|
assets/whisper
|
||||||
|
2
android/app/proguard-rules.pro
vendored
2
android/app/proguard-rules.pro
vendored
@ -12,3 +12,5 @@
|
|||||||
-keep class com.facebook.react.turbomodule.** { *; }
|
-keep class com.facebook.react.turbomodule.** { *; }
|
||||||
|
|
||||||
# Add any project specific keep options here:
|
# Add any project specific keep options here:
|
||||||
|
# whisper.rn
|
||||||
|
-keep class com.rnwhisper.** { *; }
|
@ -115,13 +115,13 @@ export class Translator {
|
|||||||
console.log(data);
|
console.log(data);
|
||||||
return data.translatedText;
|
return data.translatedText;
|
||||||
} else {
|
} else {
|
||||||
console.error(data);
|
console.error("Status %d: %s", res.status, JSON.stringify(data));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static async getDefault(defaultTarget: string | undefined = undefined) {
|
static async getDefault(defaultTarget: string | undefined = undefined) {
|
||||||
const settings = await Settings.getDefault();
|
const settings = await Settings.getDefault();
|
||||||
const source = await settings.getHostLanguage();
|
const source = await settings.getHostLanguage() || "en";
|
||||||
return new Translator(
|
return new Translator(
|
||||||
source,
|
source,
|
||||||
defaultTarget,
|
defaultTarget,
|
||||||
|
@ -1,348 +1,14 @@
|
|||||||
import { Platform } from "react-native";
|
|
||||||
import * as FileSystem from "expo-file-system";
|
|
||||||
import { File, Paths } from "expo-file-system/next";
|
import { File, Paths } from "expo-file-system/next";
|
||||||
import { getDb } from "./db";
|
import FileSystem from "expo-file-system"
|
||||||
import * as Crypto from "expo-crypto";
|
import { pathToFileURLString } from "expo-file-system/src/next/pathUtilities/url";
|
||||||
import { arrbufToStr, strToArrBuf } from "./util";
|
|
||||||
import { createReadStream } from "./readstream";
|
|
||||||
|
|
||||||
export const WHISPER_MODEL_PATH = Paths.join(
|
export const WHISPER_MODEL_PATH = Paths.join("..", "..", "assets", "whisper");
|
||||||
FileSystem.documentDirectory || "file:///",
|
|
||||||
"whisper"
|
|
||||||
);
|
|
||||||
export const WHISPER_MODEL_DIR = new File(WHISPER_MODEL_PATH);
|
export const WHISPER_MODEL_DIR = new File(WHISPER_MODEL_PATH);
|
||||||
|
|
||||||
// Thanks to https://medium.com/@fabi.mofar/downloading-and-saving-files-in-react-native-expo-5b3499adda84
|
export const WHISPER_MODEL_SMALL_PATH = "file://../../assets/whisper/whisper-small.bin";
|
||||||
|
|
||||||
export async function saveFile(
|
|
||||||
uri: string,
|
|
||||||
filename: string,
|
|
||||||
mimetype: string
|
|
||||||
) {
|
|
||||||
if (Platform.OS === "android") {
|
|
||||||
const permissions =
|
|
||||||
await FileSystem.StorageAccessFramework.requestDirectoryPermissionsAsync();
|
|
||||||
|
|
||||||
if (permissions.granted) {
|
|
||||||
const base64 = await FileSystem.readAsStringAsync(uri, {
|
|
||||||
encoding: FileSystem.EncodingType.Base64,
|
|
||||||
});
|
|
||||||
|
|
||||||
await FileSystem.StorageAccessFramework.createFileAsync(
|
|
||||||
permissions.directoryUri,
|
|
||||||
filename,
|
|
||||||
mimetype
|
|
||||||
)
|
|
||||||
.then(async (uri) => {
|
|
||||||
await FileSystem.writeAsStringAsync(uri, base64, {
|
|
||||||
encoding: FileSystem.EncodingType.Base64,
|
|
||||||
});
|
|
||||||
})
|
|
||||||
.catch((e) => console.log(e));
|
|
||||||
} else {
|
|
||||||
shareAsync(uri);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
shareAsync(uri);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function shareAsync(uri: string) {
|
|
||||||
throw new Error("Function not implemented.");
|
|
||||||
}
|
|
||||||
|
|
||||||
export const WHISPER_MODEL_TAGS = ["small", "medium", "large"];
|
|
||||||
export type whisper_model_tag_t = "small" | "medium" | "large";
|
|
||||||
|
|
||||||
export const WHISPER_MODELS = {
|
|
||||||
small: {
|
|
||||||
source:
|
|
||||||
"https://huggingface.co/openai/whisper-small/blob/resolve/pytorch_model.bin",
|
|
||||||
target: "small.bin",
|
|
||||||
label: "Small",
|
|
||||||
size: 967092419,
|
|
||||||
},
|
|
||||||
medium: {
|
|
||||||
source:
|
|
||||||
"https://huggingface.co/openai/whisper-medium/resolve/main/pytorch_model.bin",
|
|
||||||
target: "medium.bin",
|
|
||||||
label: "Medium",
|
|
||||||
size: 3055735323,
|
|
||||||
},
|
|
||||||
large: {
|
|
||||||
source:
|
|
||||||
"https://huggingface.co/openai/whisper-large/resolve/main/pytorch_model.bin",
|
|
||||||
target: "large.bin",
|
|
||||||
label: "Large",
|
|
||||||
size: 6173629930,
|
|
||||||
},
|
|
||||||
} as {
|
|
||||||
[key: whisper_model_tag_t]: {
|
|
||||||
source: string;
|
|
||||||
target: string;
|
|
||||||
label: string;
|
|
||||||
size: number;
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
export type whisper_tag_t = "small" | "medium" | "large";
|
|
||||||
export type hf_channel_t = "raw" | "resolve";
|
|
||||||
|
|
||||||
export const HF_URL_BASE = "https://huggingface.co/openai/whisper-";
|
|
||||||
export const HF_URL_RAW = "raw";
|
|
||||||
export const HF_URL_RESOLVE = "resolve";
|
|
||||||
export const HF_URL_END = "/main/pytorch_model.bin";
|
|
||||||
|
|
||||||
export function create_hf_url(tag: whisper_tag_t, channel: hf_channel_t) {
|
|
||||||
return `${HF_URL_BASE}${tag}/${channel}${HF_URL_END}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
export type hf_metadata_t = {
|
|
||||||
version: string;
|
|
||||||
oid: string;
|
|
||||||
size: string;
|
|
||||||
};
|
|
||||||
|
|
||||||
export type download_status_t = {
|
|
||||||
doesTargetExist: boolean;
|
|
||||||
isDownloadComplete: boolean;
|
|
||||||
hasDownloadStarted: boolean;
|
|
||||||
progress?: {
|
|
||||||
current: number;
|
|
||||||
total: number;
|
|
||||||
remaining: number;
|
|
||||||
percentRemaining: number;
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
export class WhisperFile {
|
|
||||||
hf_metadata: hf_metadata_t | undefined;
|
|
||||||
|
|
||||||
target_hash: string | undefined;
|
|
||||||
does_target_exist: boolean = false;
|
|
||||||
does_part_target_exist: boolean = false;
|
|
||||||
download_data: FileSystem.DownloadProgressData | undefined;
|
|
||||||
|
|
||||||
constructor(
|
|
||||||
public tag: whisper_model_tag_t,
|
|
||||||
private targetFileName?: string,
|
|
||||||
public label?: string,
|
|
||||||
public size?: number
|
|
||||||
) {
|
|
||||||
this.targetFileName = this.targetFileName || `${tag}.bin`;
|
|
||||||
this.label =
|
|
||||||
this.label || `${tag[0].toUpperCase()}${tag.substring(1).toLowerCase()}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
get targetPath() {
|
|
||||||
return Paths.join(WHISPER_MODEL_PATH, this.targetFileName as string);
|
|
||||||
}
|
|
||||||
|
|
||||||
get targetPartPath () {
|
|
||||||
return this.targetPath + ".part";
|
|
||||||
}
|
|
||||||
|
|
||||||
get targetFile() {
|
|
||||||
return new File(this.targetPath);
|
|
||||||
}
|
|
||||||
|
|
||||||
async getTargetInfo() {
|
|
||||||
return await FileSystem.getInfoAsync(this.targetPath);
|
|
||||||
}
|
|
||||||
|
|
||||||
async getTargetPartInfo() {
|
|
||||||
return await FileSystem.getInfoAsync(this.targetPartPath);
|
|
||||||
}
|
|
||||||
|
|
||||||
async updateTargetExistence() {
|
|
||||||
this.does_target_exist = (await this.getTargetInfo()).exists;
|
|
||||||
this.does_part_target_exist = (await this.getTargetPartInfo()).exists;
|
|
||||||
}
|
|
||||||
|
|
||||||
public async getTargetSha() {
|
|
||||||
await this.updateTargetExistence();
|
|
||||||
if (!this.does_target_exist) {
|
|
||||||
console.debug("%s does not exist", this.targetPath);
|
|
||||||
return undefined;
|
|
||||||
}
|
|
||||||
|
|
||||||
const strData = await FileSystem.readAsStringAsync(this.targetPath, {
|
|
||||||
encoding: FileSystem.EncodingType.Base64,
|
|
||||||
});
|
|
||||||
const data = strToArrBuf(strData);
|
|
||||||
|
|
||||||
|
|
||||||
const digest = await Crypto.digest(
|
export async function whisperModelExists() {
|
||||||
Crypto.CryptoDigestAlgorithm.SHA256,
|
const file = new File(WHISPER_MODEL_PATH);
|
||||||
data
|
return file.exists;
|
||||||
);
|
}
|
||||||
|
|
||||||
return digest;
|
|
||||||
}
|
|
||||||
|
|
||||||
public async updateTargetHash() {
|
|
||||||
const targetSha = await this.getTargetSha();
|
|
||||||
if (!targetSha) return;
|
|
||||||
this.target_hash = arrbufToStr(targetSha);
|
|
||||||
}
|
|
||||||
|
|
||||||
get isHashValid() {
|
|
||||||
return this.target_hash === this.hf_metadata?.oid;
|
|
||||||
}
|
|
||||||
|
|
||||||
delete(ignoreErrors = true) {
|
|
||||||
try {
|
|
||||||
this.targetFile.delete();
|
|
||||||
} catch (err) {
|
|
||||||
console.error(err);
|
|
||||||
if (!ignoreErrors) {
|
|
||||||
throw err;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
console.debug("Created %s", WHISPER_MODEL_DIR);
|
|
||||||
}
|
|
||||||
|
|
||||||
get modelUrl() {
|
|
||||||
return create_hf_url(this.tag, "resolve");
|
|
||||||
}
|
|
||||||
|
|
||||||
get metadataUrl() {
|
|
||||||
return create_hf_url(this.tag, "raw");
|
|
||||||
}
|
|
||||||
|
|
||||||
get percentDone() {
|
|
||||||
if (!this.download_data) return 0;
|
|
||||||
return (
|
|
||||||
(this.download_data.totalBytesWritten /
|
|
||||||
this.download_data.totalBytesExpectedToWrite) *
|
|
||||||
100
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
get percentLeft() {
|
|
||||||
if (!this.download_data) return 0;
|
|
||||||
return 100 - this.percentDone;
|
|
||||||
}
|
|
||||||
|
|
||||||
public async syncHfMetadata() {
|
|
||||||
try {
|
|
||||||
const resp = await fetch(this.metadataUrl, {
|
|
||||||
credentials: "include",
|
|
||||||
headers: {
|
|
||||||
"User-Agent":
|
|
||||||
"Mozilla/5.0 (X11; Linux x86_64; rv:135.0) Gecko/20100101 Firefox/135.0",
|
|
||||||
Accept:
|
|
||||||
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
||||||
"Accept-Language": "en-US,en;q=0.5",
|
|
||||||
"Sec-GPC": "1",
|
|
||||||
"Upgrade-Insecure-Requests": "1",
|
|
||||||
"Sec-Fetch-Dest": "document",
|
|
||||||
"Sec-Fetch-Mode": "navigate",
|
|
||||||
"Sec-Fetch-Site": "cross-site",
|
|
||||||
"If-None-Match": '"8fa71cbce85078986b46fb97caec22039e73351a"',
|
|
||||||
Priority: "u=0, i",
|
|
||||||
},
|
|
||||||
method: "GET",
|
|
||||||
mode: "cors",
|
|
||||||
});
|
|
||||||
const text = await resp.text();
|
|
||||||
this.hf_metadata = Object.fromEntries(
|
|
||||||
text.split("\n").map((line) => line.split(" "))
|
|
||||||
) as hf_metadata_t;
|
|
||||||
} catch (err) {
|
|
||||||
console.error("Failed to fetch %s: %s", this.metadataUrl, err);
|
|
||||||
throw err;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async createDownloadResumable(
|
|
||||||
options: {
|
|
||||||
onData?: DownloadCallback | undefined;
|
|
||||||
onComplete?: CompletionCallback | undefined;
|
|
||||||
} = {
|
|
||||||
onData: undefined,
|
|
||||||
onComplete: undefined,
|
|
||||||
}
|
|
||||||
) {
|
|
||||||
await this.syncHfMetadata();
|
|
||||||
|
|
||||||
// If the whisper model dir doesn't exist, create it.
|
|
||||||
if (!WHISPER_MODEL_DIR.exists) {
|
|
||||||
FileSystem.makeDirectoryAsync(WHISPER_MODEL_PATH, {
|
|
||||||
intermediates: true,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check for the existence of the target file
|
|
||||||
// If it exists, load the existing data.
|
|
||||||
await this.updateTargetExistence();
|
|
||||||
|
|
||||||
if (this.does_part_target_exist) {
|
|
||||||
options.onComplete && options.onComplete(this)
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
const existingData = this.does_target_exist
|
|
||||||
? await FileSystem.readAsStringAsync(this.targetPath, {
|
|
||||||
encoding: FileSystem.EncodingType.Base64,
|
|
||||||
})
|
|
||||||
: undefined;
|
|
||||||
|
|
||||||
// Create the resumable.
|
|
||||||
return FileSystem.createDownloadResumable(
|
|
||||||
this.modelUrl,
|
|
||||||
this.targetPath,
|
|
||||||
{},
|
|
||||||
async (data: FileSystem.DownloadProgressData) => {
|
|
||||||
// console.debug("yes, I'm still downloading");
|
|
||||||
try {
|
|
||||||
this.download_data = data;
|
|
||||||
} catch (err) {
|
|
||||||
console.error("Failed to set downloadData: %s", err);
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
await this.syncHfMetadata();
|
|
||||||
} catch (err) {
|
|
||||||
console.error("Failed to update HuggingFace metadata: %s", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// try {
|
|
||||||
// await this.updateTargetHash();
|
|
||||||
// } catch (er) {
|
|
||||||
// console.error("Failed to update target hash: %s", er);
|
|
||||||
// }
|
|
||||||
|
|
||||||
try {
|
|
||||||
await this.updateTargetExistence();
|
|
||||||
} catch (err) {
|
|
||||||
console.error("Failed to update target existence: %s", err)
|
|
||||||
}
|
|
||||||
if (options.onData) await options.onData(this);
|
|
||||||
|
|
||||||
if (data.totalBytesExpectedToWrite === 0) {
|
|
||||||
console.debug("Finalizing; copying from %s -> %s", this.targetPartPath, this.targetPath);
|
|
||||||
await FileSystem.copyAsync({
|
|
||||||
from: this.targetPartPath,
|
|
||||||
to: this.targetPath,
|
|
||||||
});
|
|
||||||
await this.updateTargetExistence();
|
|
||||||
options.onComplete && options.onComplete(this);
|
|
||||||
}
|
|
||||||
},
|
|
||||||
existingData ? existingData : undefined
|
|
||||||
);
|
|
||||||
} catch (err) {
|
|
||||||
console.error("Could not read %s: %s", this.targetPath, err);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export type DownloadCallback = (arg0: WhisperFile) => any;
|
|
||||||
export type CompletionCallback = (arg0: WhisperFile) => any;
|
|
||||||
|
|
||||||
export const WHISPER_FILES = {
|
|
||||||
small: new WhisperFile("small"),
|
|
||||||
medium: new WhisperFile("medium"),
|
|
||||||
large: new WhisperFile("large"),
|
|
||||||
};
|
|
@ -1,14 +1,27 @@
|
|||||||
import React, { useState, useEffect } from "react";
|
import React, { useState, useEffect } from "react";
|
||||||
import { Alert, ScrollView, StyleSheet, Text, TouchableHighlight, View } from "react-native";
|
import {
|
||||||
|
Alert,
|
||||||
|
ScrollView,
|
||||||
|
StyleSheet,
|
||||||
|
Text,
|
||||||
|
TouchableHighlight,
|
||||||
|
View,
|
||||||
|
} from "react-native";
|
||||||
import { useNavigation, Route } from "@react-navigation/native";
|
import { useNavigation, Route } from "@react-navigation/native";
|
||||||
import { Conversation, Message } from "@/app/lib/conversation";
|
import { Conversation, Message } from "@/app/lib/conversation";
|
||||||
import MessageBubble from "@/components/ui/MessageBubble";
|
import MessageBubble from "@/components/ui/MessageBubble";
|
||||||
import { CachedTranslator, LanguageServer, language_matrix_entry } from "@/app/i18n/api";
|
import {
|
||||||
import { Settings } from "@/app/lib/settings";
|
CachedTranslator,
|
||||||
import { WHISPER_FILES } from "@/app/lib/whisper";
|
LanguageServer,
|
||||||
import { initWhisper, WhisperContext } from 'whisper.rn'
|
language_matrix_entry,
|
||||||
import { useAudioRecorder, AudioModule, RecordingPresets } from 'expo-audio';
|
} from "@/app/i18n/api";
|
||||||
|
import {
|
||||||
|
WHISPER_MODEL_SMALL_PATH,
|
||||||
|
whisperModelExists,
|
||||||
|
} from "@/app/lib/whisper";
|
||||||
|
import { initWhisper, WhisperContext } from "whisper.rn";
|
||||||
|
import { useAudioRecorder, AudioModule, RecordingPresets } from "expo-audio";
|
||||||
|
import FileSystem from "expo-file-system";
|
||||||
|
|
||||||
const lasOptions = {
|
const lasOptions = {
|
||||||
sampleRate: 32000, // default is 44100 but 32000 is adequate for accurate voice recognition
|
sampleRate: 32000, // default is 44100 but 32000 is adequate for accurate voice recognition
|
||||||
@ -19,11 +32,19 @@ const lasOptions = {
|
|||||||
};
|
};
|
||||||
// LiveAudioStream.init(lasOptions as any);
|
// LiveAudioStream.init(lasOptions as any);
|
||||||
|
|
||||||
const ConversationThread = ({ route }: { route?: Route<"Conversation", { conversation: Conversation }> }) => {
|
const ConversationThread = ({
|
||||||
|
route,
|
||||||
|
}: {
|
||||||
|
route?: Route<"Conversation", { conversation: Conversation }>;
|
||||||
|
}) => {
|
||||||
const navigation = useNavigation();
|
const navigation = useNavigation();
|
||||||
|
|
||||||
if (!route) {
|
if (!route) {
|
||||||
return (<View><Text>Missing Params!</Text></View>)
|
return (
|
||||||
|
<View>
|
||||||
|
<Text>Missing Params!</Text>
|
||||||
|
</View>
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* 2. Get the param */
|
/* 2. Get the param */
|
||||||
@ -32,21 +53,26 @@ const ConversationThread = ({ route }: { route?: Route<"Conversation", { convers
|
|||||||
const [messages, setMessages] = useState<Message[]>([]);
|
const [messages, setMessages] = useState<Message[]>([]);
|
||||||
const [guestSpeak, setGuestSpeak] = useState<string | undefined>();
|
const [guestSpeak, setGuestSpeak] = useState<string | undefined>();
|
||||||
const [guestSpeakLoaded, setGuestSpeakLoaded] = useState<boolean>(false);
|
const [guestSpeakLoaded, setGuestSpeakLoaded] = useState<boolean>(false);
|
||||||
const [whisperContext, setWhisperContext] = useState<WhisperContext | undefined>();
|
const [whisperContext, setWhisperContext] = useState<
|
||||||
|
WhisperContext | undefined
|
||||||
|
>();
|
||||||
const [cachedTranslator, setCachedTranslator] = useState<
|
const [cachedTranslator, setCachedTranslator] = useState<
|
||||||
undefined | CachedTranslator
|
undefined | CachedTranslator
|
||||||
>();
|
>();
|
||||||
|
|
||||||
const [languageLabels, setLanguageLabels] = useState<undefined | {
|
const [languageLabels, setLanguageLabels] = useState<
|
||||||
hostNative: {
|
| undefined
|
||||||
host: string,
|
| {
|
||||||
guest: string,
|
hostNative: {
|
||||||
},
|
host: string;
|
||||||
guestNative: {
|
guest: string;
|
||||||
host: string,
|
};
|
||||||
guest: string,
|
guestNative: {
|
||||||
}
|
host: string;
|
||||||
}>()
|
guest: string;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
>();
|
||||||
|
|
||||||
// recorder settings
|
// recorder settings
|
||||||
const audioRecorder = useAudioRecorder(RecordingPresets.HIGH_QUALITY);
|
const audioRecorder = useAudioRecorder(RecordingPresets.HIGH_QUALITY);
|
||||||
@ -63,50 +89,75 @@ const ConversationThread = ({ route }: { route?: Route<"Conversation", { convers
|
|||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
(async function () {
|
(async function () {
|
||||||
|
|
||||||
const languageServer = await LanguageServer.getDefault();
|
const languageServer = await LanguageServer.getDefault();
|
||||||
try {
|
try {
|
||||||
const languages = await languageServer.fetchLanguages(5000);
|
const languages = await languageServer.fetchLanguages(5000);
|
||||||
const cc = new CachedTranslator(
|
const cachedTranslator = new CachedTranslator(
|
||||||
"en",
|
"en",
|
||||||
conversation.guest.language,
|
conversation.guest.language,
|
||||||
languageServer,
|
languageServer
|
||||||
)
|
);
|
||||||
console.log("Set cached translator from %s", languageServer.baseUrl)
|
console.log("Set cached translator from %s", languageServer.baseUrl);
|
||||||
setCachedTranslator(cc);
|
setCachedTranslator(cachedTranslator);
|
||||||
|
|
||||||
|
try {
|
||||||
|
if (!(await whisperModelExists())) {
|
||||||
|
throw new Error(`${WHISPER_MODEL_SMALL_PATH} does not exist`);
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
console.error(
|
||||||
|
`Could not determine if %s exists: %s`,
|
||||||
|
WHISPER_MODEL_SMALL_PATH,
|
||||||
|
err
|
||||||
|
);
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
setWhisperContext(
|
||||||
|
await initWhisper({
|
||||||
|
filePath: WHISPER_MODEL_SMALL_PATH,
|
||||||
|
})
|
||||||
|
);
|
||||||
|
} catch (err) {
|
||||||
|
console.error(err);
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
|
||||||
|
// recorder settings
|
||||||
|
|
||||||
|
(async () => {
|
||||||
|
const status = await AudioModule.requestRecordingPermissionsAsync();
|
||||||
|
if (!status.granted) {
|
||||||
|
Alert.alert("Permission to access microphone was denied");
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
setGuestSpeak(await cachedTranslator.translate("Speak"));
|
||||||
|
const hostLang1 = languages[conversation.host.language].name;
|
||||||
|
const guestLang1 = languages[conversation.host.language].name;
|
||||||
|
const hostLang2 = await cachedTranslator.translate(
|
||||||
|
languages[conversation.host.language].name
|
||||||
|
);
|
||||||
|
const guestLang2 = await cachedTranslator.translate(
|
||||||
|
languages[conversation.host.language].name
|
||||||
|
);
|
||||||
|
setLanguageLabels({
|
||||||
|
hostNative: {
|
||||||
|
host: hostLang1,
|
||||||
|
guest: guestLang1,
|
||||||
|
},
|
||||||
|
guestNative: {
|
||||||
|
host: hostLang2,
|
||||||
|
guest: guestLang2,
|
||||||
|
},
|
||||||
|
});
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error("Could not set translator from %s: %s", languageServer.baseUrl, err)
|
console.error(
|
||||||
|
"Could not set translator from %s: %s",
|
||||||
|
languageServer.baseUrl,
|
||||||
|
err
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
const settings = await Settings.getDefault();
|
|
||||||
const whisperFile = WHISPER_FILES[await settings.getWhisperModel() || "en"];
|
|
||||||
setWhisperContext(await initWhisper({
|
|
||||||
filePath: whisperFile.targetPath,
|
|
||||||
}));
|
|
||||||
|
|
||||||
// recorder settings
|
|
||||||
|
|
||||||
(async () => {
|
|
||||||
const status = await AudioModule.requestRecordingPermissionsAsync();
|
|
||||||
if (!status.granted) {
|
|
||||||
Alert.alert('Permission to access microphone was denied');
|
|
||||||
}
|
|
||||||
})();
|
|
||||||
setGuestSpeak(await cc.translate("Speak"));
|
|
||||||
const hostLang1 = languages[conversation.host.language].name;
|
|
||||||
const guestLang1 = languages[conversation.host.language].name;
|
|
||||||
const hostLang2 = await cc.translate(languages[conversation.host.language].name);
|
|
||||||
const guestLang2 = await cc.translate(languages[conversation.host.language].name);
|
|
||||||
setLanguageLabels({
|
|
||||||
hostNative: {
|
|
||||||
host: hostLang1,
|
|
||||||
guest: guestLang1,
|
|
||||||
},
|
|
||||||
guestNative: {
|
|
||||||
host: hostLang2,
|
|
||||||
guest: guestLang2,
|
|
||||||
}
|
|
||||||
})
|
|
||||||
})();
|
})();
|
||||||
|
|
||||||
const updateMessages = (c: Conversation) => {
|
const updateMessages = (c: Conversation) => {
|
||||||
@ -114,7 +165,7 @@ const ConversationThread = ({ route }: { route?: Route<"Conversation", { convers
|
|||||||
};
|
};
|
||||||
|
|
||||||
if (!conversation) {
|
if (!conversation) {
|
||||||
console.warn("Conversation is null or undefined.")
|
console.warn("Conversation is null or undefined.");
|
||||||
}
|
}
|
||||||
|
|
||||||
conversation.on("add_message", updateMessages);
|
conversation.on("add_message", updateMessages);
|
||||||
@ -133,11 +184,17 @@ const ConversationThread = ({ route }: { route?: Route<"Conversation", { convers
|
|||||||
|
|
||||||
return cachedTranslator ? (
|
return cachedTranslator ? (
|
||||||
<View style={{ flex: 1, flexDirection: "column" }}>
|
<View style={{ flex: 1, flexDirection: "column" }}>
|
||||||
{languageLabels && (<View style={styles.languageLabels}>
|
{languageLabels && (
|
||||||
<Text style={styles.nativeHostLabel}>{languageLabels.hostNative.host} / {languageLabels.hostNative.guest}</Text>
|
<View style={styles.languageLabels}>
|
||||||
<Text style={styles.nativeGuestLabel}>{languageLabels.guestNative.host} / {languageLabels.guestNative.guest}</Text>
|
<Text style={styles.nativeHostLabel}>
|
||||||
</View>)
|
{languageLabels.hostNative.host} / {languageLabels.hostNative.guest}
|
||||||
}
|
</Text>
|
||||||
|
<Text style={styles.nativeGuestLabel}>
|
||||||
|
{languageLabels.guestNative.host} /{" "}
|
||||||
|
{languageLabels.guestNative.guest}
|
||||||
|
</Text>
|
||||||
|
</View>
|
||||||
|
)}
|
||||||
<ScrollView
|
<ScrollView
|
||||||
style={{
|
style={{
|
||||||
borderColor: "black",
|
borderColor: "black",
|
||||||
@ -177,15 +234,9 @@ const ConversationThread = ({ route }: { route?: Route<"Conversation", { convers
|
|||||||
};
|
};
|
||||||
|
|
||||||
const styles = StyleSheet.create({
|
const styles = StyleSheet.create({
|
||||||
languageLabels: {
|
languageLabels: {},
|
||||||
|
nativeHostLabel: {},
|
||||||
},
|
nativeGuestLabel: {},
|
||||||
nativeHostLabel: {
|
});
|
||||||
|
|
||||||
},
|
|
||||||
nativeGuestLabel: {
|
|
||||||
|
|
||||||
},
|
|
||||||
})
|
|
||||||
|
|
||||||
export default ConversationThread;
|
export default ConversationThread;
|
||||||
|
@ -36,7 +36,7 @@ export function LanguageSelection(props: {
|
|||||||
// Replace with your actual async data fetching logic
|
// Replace with your actual async data fetching logic
|
||||||
setTranslator(await CachedTranslator.getDefault());
|
setTranslator(await CachedTranslator.getDefault());
|
||||||
const languageServer = await LanguageServer.getDefault();
|
const languageServer = await LanguageServer.getDefault();
|
||||||
const languages = await languageServer.fetchLanguages(5000);
|
const languages = await languageServer.fetchLanguages(10000);
|
||||||
setLanguages(languages);
|
setLanguages(languages);
|
||||||
setLanguagesLoaded(true);
|
setLanguagesLoaded(true);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
@ -1,23 +1,11 @@
|
|||||||
import React, { useState, useEffect } from "react";
|
import React, { useState, useEffect } from "react";
|
||||||
import { View, Text, TextInput, Pressable, StyleSheet } from "react-native";
|
import { View, Text, TextInput, StyleSheet } from "react-native";
|
||||||
import {
|
|
||||||
WHISPER_FILES,
|
|
||||||
WhisperFile,
|
|
||||||
download_status_t,
|
|
||||||
whisper_tag_t,
|
|
||||||
} from "@/app/lib/whisper";
|
|
||||||
import { Settings } from "@/app/lib/settings";
|
import { Settings } from "@/app/lib/settings";
|
||||||
import { Picker } from "@react-native-picker/picker";
|
import { Picker } from "@react-native-picker/picker";
|
||||||
import {
|
import {
|
||||||
LanguageServer,
|
LanguageServer,
|
||||||
language_matrix,
|
language_matrix,
|
||||||
language_matrix_entry,
|
|
||||||
} from "@/app/i18n/api";
|
} from "@/app/i18n/api";
|
||||||
const WHISPER_MODELS = {
|
|
||||||
small: new WhisperFile("small"),
|
|
||||||
medium: new WhisperFile("medium"),
|
|
||||||
large: new WhisperFile("large"),
|
|
||||||
};
|
|
||||||
|
|
||||||
const LIBRETRANSLATE_BASE_URL = "https://translate.argosopentech.com/translate";
|
const LIBRETRANSLATE_BASE_URL = "https://translate.argosopentech.com/translate";
|
||||||
|
|
||||||
@ -33,17 +21,6 @@ const SettingsComponent = () => {
|
|||||||
success: boolean;
|
success: boolean;
|
||||||
error?: string;
|
error?: string;
|
||||||
} | null>(null);
|
} | null>(null);
|
||||||
const [whisperModel, setWhisperModel] =
|
|
||||||
useState<keyof typeof WHISPER_MODELS>("small");
|
|
||||||
const [whisperFile, setWhisperFile] = useState<WhisperFile | undefined>();
|
|
||||||
const [whisperFileExists, setWhisperFileExists] = useState<boolean>(false);
|
|
||||||
const [isWhisperHashValid, setIsWhisperHashValid] = useState<boolean>(false);
|
|
||||||
const [downloader, setDownloader] = useState<any>(null);
|
|
||||||
const [bytesDone, setBytesDone] = useState<number | undefined>();
|
|
||||||
const [bytesRemaining, setBytesRemaining] = useState<number | undefined>();
|
|
||||||
const [statusTimeout, setStatusTimeout] = useState<
|
|
||||||
NodeJS.Timeout | undefined
|
|
||||||
>();
|
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
(async function () {
|
(async function () {
|
||||||
@ -52,21 +29,9 @@ const SettingsComponent = () => {
|
|||||||
setLibretranslateBaseUrl(
|
setLibretranslateBaseUrl(
|
||||||
(await settings.getLibretranslateBaseUrl()) || LIBRETRANSLATE_BASE_URL
|
(await settings.getLibretranslateBaseUrl()) || LIBRETRANSLATE_BASE_URL
|
||||||
);
|
);
|
||||||
setWhisperModel((await settings.getWhisperModel()) || "small");
|
|
||||||
setWhisperFile(WHISPER_FILES[whisperModel]);
|
|
||||||
if (whisperFile) {
|
|
||||||
await whisperFile.syncHfMetadata();
|
|
||||||
await whisperFile.updateTargetExistence();
|
|
||||||
await whisperFile.updateTargetHash();
|
|
||||||
setWhisperFileExists(whisperFile.does_target_exist)
|
|
||||||
}
|
|
||||||
})();
|
})();
|
||||||
}, [whisperFile]);
|
});
|
||||||
|
|
||||||
const getLanguageOptions = async () => {
|
|
||||||
const languageServer = await LanguageServer.getDefault();
|
|
||||||
setLanguageOptions(await languageServer.fetchLanguages());
|
|
||||||
};
|
|
||||||
|
|
||||||
const handleHostLanguageChange = async (lang: string) => {
|
const handleHostLanguageChange = async (lang: string) => {
|
||||||
const settings = await Settings.getDefault();
|
const settings = await Settings.getDefault();
|
||||||
@ -84,69 +49,17 @@ const SettingsComponent = () => {
|
|||||||
const checkLangServerConnection = async (baseUrl: string) => {
|
const checkLangServerConnection = async (baseUrl: string) => {
|
||||||
try {
|
try {
|
||||||
// Replace with actual connection check logic
|
// Replace with actual connection check logic
|
||||||
setLangServerConn({ success: true });
|
const testResult = await fetch(baseUrl, {
|
||||||
|
method: "HEAD",
|
||||||
|
});
|
||||||
|
if (testResult.status !== 200) {
|
||||||
|
setLangServerConn({ success: true, error: testResult.statusText });
|
||||||
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
setLangServerConn({ success: false, error: `${error}` });
|
setLangServerConn({ success: false, error: `${error}` });
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const handleWhisperModelChange = async (model: whisper_tag_t) => {
|
|
||||||
const settings = await Settings.getDefault();
|
|
||||||
await settings.setWhisperModel(model);
|
|
||||||
setWhisperModel(model);
|
|
||||||
const wFile = WHISPER_FILES[whisperModel];
|
|
||||||
await wFile.syncHfMetadata();
|
|
||||||
await wFile.updateTargetExistence();
|
|
||||||
// await wFile.updateTargetHash();
|
|
||||||
// setIsWhisperHashValid(wFile.isHashValid);
|
|
||||||
setWhisperFile(wFile);
|
|
||||||
setWhisperFileExists(wFile.does_target_exist);
|
|
||||||
};
|
|
||||||
|
|
||||||
const doSetDownloadStatus = (arg0: WhisperFile) => {
|
|
||||||
console.log("Downloading ....")
|
|
||||||
setIsWhisperHashValid(arg0.isHashValid);
|
|
||||||
setBytesDone(arg0.download_data?.totalBytesWritten);
|
|
||||||
setBytesRemaining(arg0.download_data?.totalBytesExpectedToWrite);
|
|
||||||
};
|
|
||||||
|
|
||||||
const doOnComplete = (arg0: WhisperFile) => {
|
|
||||||
setWhisperFile(arg0);
|
|
||||||
}
|
|
||||||
|
|
||||||
const doDownload = async () => {
|
|
||||||
if (!whisperModel) {
|
|
||||||
throw new Error("Could not start download because whisperModel not set.");
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log("Starting download of %s", whisperModel);
|
|
||||||
|
|
||||||
if (!whisperFile) throw new Error("No whisper file");
|
|
||||||
|
|
||||||
try {
|
|
||||||
const resumable = await whisperFile.createDownloadResumable({
|
|
||||||
onData: doSetDownloadStatus,
|
|
||||||
onComplete: doOnComplete,
|
|
||||||
});
|
|
||||||
setDownloader(resumable);
|
|
||||||
if (!resumable) throw new Error("Could not construct resumable");
|
|
||||||
await resumable.resumeAsync();
|
|
||||||
} catch (error) {
|
|
||||||
console.error("Failed to download whisper model:", error);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
const doStopDownload = async () => {
|
|
||||||
downloader.cancelAsync();
|
|
||||||
setDownloader(null);
|
|
||||||
};
|
|
||||||
|
|
||||||
const doDelete = async () => {
|
|
||||||
const whisperFile = WHISPER_MODELS[whisperModel];
|
|
||||||
whisperFile.delete();
|
|
||||||
setStatusTimeout(undefined);
|
|
||||||
};
|
|
||||||
|
|
||||||
return hostLanguage && libretranslateBaseUrl ? (
|
return hostLanguage && libretranslateBaseUrl ? (
|
||||||
<View style={styles.container}>
|
<View style={styles.container}>
|
||||||
<Text style={styles.label}>Host Language:</Text>
|
<Text style={styles.label}>Host Language:</Text>
|
||||||
@ -179,52 +92,6 @@ const SettingsComponent = () => {
|
|||||||
Error connecting to {libretranslateBaseUrl}: {langServerConn.error}
|
Error connecting to {libretranslateBaseUrl}: {langServerConn.error}
|
||||||
</Text>
|
</Text>
|
||||||
))}
|
))}
|
||||||
<Picker
|
|
||||||
selectedValue={whisperModel}
|
|
||||||
style={{ height: 50, width: "100%" }}
|
|
||||||
onValueChange={handleWhisperModelChange}
|
|
||||||
accessibilityHint="whisper models"
|
|
||||||
>
|
|
||||||
{Object.entries(WHISPER_MODELS).map(([key, whisperFile]) => (
|
|
||||||
<Picker.Item
|
|
||||||
key={whisperFile.tag}
|
|
||||||
label={whisperFile.label}
|
|
||||||
value={key}
|
|
||||||
/>
|
|
||||||
))}
|
|
||||||
</Picker>
|
|
||||||
<View style={styles.downloadButtonWrapper}>
|
|
||||||
{((!downloader) && whisperFile) &&
|
|
||||||
(whisperFile.does_target_exist && (<Pressable onPress={doDelete} style={styles.deleteButton}>
|
|
||||||
<Text style={styles.buttonText}>DELETE {whisperModel.toUpperCase()}</Text>
|
|
||||||
</Pressable>))
|
|
||||||
}
|
|
||||||
<Pressable onPress={doDownload} style={styles.downloadButton}>
|
|
||||||
<Text style={styles.buttonText}>DOWNLOAD {whisperModel.toUpperCase()}</Text>
|
|
||||||
</Pressable>
|
|
||||||
))
|
|
||||||
|
|
||||||
{
|
|
||||||
downloader && (
|
|
||||||
<Pressable onPress={doStopDownload} style={styles.pauseDownloadButton}>
|
|
||||||
<Text style={styles.buttonText}>STOP DOWNLOAD</Text>
|
|
||||||
</Pressable>
|
|
||||||
)
|
|
||||||
}
|
|
||||||
{bytesDone && bytesRemaining && whisperFile?.does_part_target_exist && (
|
|
||||||
<View>
|
|
||||||
{whisperFile &&
|
|
||||||
(<Text>
|
|
||||||
Downloading to {whisperFile.targetPath}
|
|
||||||
</Text>)}
|
|
||||||
<Text>
|
|
||||||
{bytesDone} of{" "}
|
|
||||||
{bytesRemaining} (
|
|
||||||
{bytesDone / bytesRemaining * 100} %){" "}
|
|
||||||
</Text>
|
|
||||||
</View>
|
|
||||||
)}
|
|
||||||
</View>
|
|
||||||
</View>
|
</View>
|
||||||
) : (
|
) : (
|
||||||
<View>
|
<View>
|
||||||
@ -239,11 +106,12 @@ const styles = StyleSheet.create({
|
|||||||
flexDirection: "row",
|
flexDirection: "row",
|
||||||
},
|
},
|
||||||
downloadButton: {
|
downloadButton: {
|
||||||
backgroundColor: "darkblue",
|
backgroundColor: "#236b9f",
|
||||||
padding: 20,
|
padding: 20,
|
||||||
margin: 10,
|
margin: 10,
|
||||||
flex: 3,
|
flex: 1,
|
||||||
flexDirection: "column",
|
flexDirection: "column",
|
||||||
|
alignItems: "center",
|
||||||
},
|
},
|
||||||
deleteButton: {
|
deleteButton: {
|
||||||
backgroundColor: "darkred",
|
backgroundColor: "darkred",
|
||||||
@ -261,11 +129,11 @@ const styles = StyleSheet.create({
|
|||||||
},
|
},
|
||||||
buttonText: {
|
buttonText: {
|
||||||
color: "#fff",
|
color: "#fff",
|
||||||
flex: 1,
|
// flex: 1,
|
||||||
fontSize: 16,
|
// fontSize: 16,
|
||||||
alignSelf: "center",
|
// alignSelf: "center",
|
||||||
textAlign: "center",
|
// textAlign: "center",
|
||||||
textAlignVertical: "top",
|
// textAlignVertical: "top",
|
||||||
},
|
},
|
||||||
container: {
|
container: {
|
||||||
flex: 1,
|
flex: 1,
|
||||||
|
@ -9,7 +9,8 @@
|
|||||||
"updateSnapshots": "jest -u --coverage=false",
|
"updateSnapshots": "jest -u --coverage=false",
|
||||||
"start": "expo start",
|
"start": "expo start",
|
||||||
"reset-project": "node ./scripts/reset-project.js",
|
"reset-project": "node ./scripts/reset-project.js",
|
||||||
"android": "expo prebuild --npm -p android",
|
"prebuild:android": "expo prebuild --npm -p android",
|
||||||
|
"android": "expo run:android",
|
||||||
"ios": "expo prebuild --npm -p android --offline",
|
"ios": "expo prebuild --npm -p android --offline",
|
||||||
"web": "expo start --offline --web",
|
"web": "expo start --offline --web",
|
||||||
"lint": "expo lint"
|
"lint": "expo lint"
|
||||||
|
Loading…
x
Reference in New Issue
Block a user