From 0ba5c4b309cfbd8e9e9f858dd36618c27e20391a Mon Sep 17 00:00:00 2001 From: Jordan Date: Mon, 17 Mar 2025 06:56:05 -0700 Subject: [PATCH] encountered weird network error. --- .gitignore | 1 + app/i18n/api.ts | 4 +- app/lib/whisper.ts | 363 +----------------------------- components/ConversationThread.tsx | 28 ++- components/LanguageSelection.tsx | 2 +- components/Settings.tsx | 196 +--------------- package.json | 3 +- 7 files changed, 40 insertions(+), 557 deletions(-) diff --git a/.gitignore b/.gitignore index ca7038f..9c287a8 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,4 @@ yarn-error.* *.tsbuildinfo coverage/**/* +assets/whisper diff --git a/app/i18n/api.ts b/app/i18n/api.ts index 76a7582..21d56ed 100644 --- a/app/i18n/api.ts +++ b/app/i18n/api.ts @@ -115,13 +115,13 @@ export class Translator { console.log(data); return data.translatedText; } else { - console.error(data); + console.error("Status %d: %s", res.status, JSON.stringify(data)); } } static async getDefault(defaultTarget: string | undefined = undefined) { const settings = await Settings.getDefault(); - const source = await settings.getHostLanguage(); + const source = await settings.getHostLanguage() || "en"; return new Translator( source, defaultTarget, diff --git a/app/lib/whisper.ts b/app/lib/whisper.ts index 9ad4c6a..9a35efb 100644 --- a/app/lib/whisper.ts +++ b/app/lib/whisper.ts @@ -1,361 +1,14 @@ -import { Platform } from "react-native"; -import * as FileSystem from "expo-file-system"; import { File, Paths } from "expo-file-system/next"; -import { getDb } from "./db"; -import * as Crypto from "expo-crypto"; -import { arrbufToStr, strToArrBuf } from "./util"; -import { createReadStream } from "./readstream"; +import FileSystem from "expo-file-system" +import { pathToFileURLString } from "expo-file-system/src/next/pathUtilities/url"; -export const WHISPER_MODEL_PATH = Paths.join( - FileSystem.documentDirectory || "file:///", - "whisper" -); +export const WHISPER_MODEL_PATH = Paths.join("..", "..", "assets", "whisper"); export const WHISPER_MODEL_DIR = new File(WHISPER_MODEL_PATH); -// Thanks to https://medium.com/@fabi.mofar/downloading-and-saving-files-in-react-native-expo-5b3499adda84 +export const WHISPER_MODEL_SMALL_PATH = "file://../../assets/whisper/whisper-small.bin"; -export async function saveFile( - uri: string, - filename: string, - mimetype: string -) { - if (Platform.OS === "android") { - const permissions = - await FileSystem.StorageAccessFramework.requestDirectoryPermissionsAsync(); - if (permissions.granted) { - const base64 = await FileSystem.readAsStringAsync(uri, { - encoding: FileSystem.EncodingType.Base64, - }); - - await FileSystem.StorageAccessFramework.createFileAsync( - permissions.directoryUri, - filename, - mimetype - ) - .then(async (uri) => { - await FileSystem.writeAsStringAsync(uri, base64, { - encoding: FileSystem.EncodingType.Base64, - }); - }) - .catch((e) => console.log(e)); - } else { - shareAsync(uri); - } - } else { - shareAsync(uri); - } -} - -function shareAsync(uri: string) { - throw new Error("Function not implemented."); -} - -export const WHISPER_MODEL_TAGS = ["small", "medium", "large"]; -export type whisper_model_tag_t = "small" | "medium" | "large"; - -export const WHISPER_MODELS = { - small: { - source: - "https://huggingface.co/openai/whisper-small/blob/resolve/pytorch_model.bin", - target: "small.bin", - label: "Small", - size: 967092419, - }, - medium: { - source: - "https://huggingface.co/openai/whisper-medium/resolve/main/pytorch_model.bin", - target: "medium.bin", - label: "Medium", - size: 3055735323, - }, - large: { - source: - "https://huggingface.co/openai/whisper-large/resolve/main/pytorch_model.bin", - target: "large.bin", - label: "Large", - size: 6173629930, - }, -} as { - [key: whisper_model_tag_t]: { - source: string; - target: string; - label: string; - size: number; - }; -}; - -export type whisper_tag_t = "small" | "medium" | "large"; -export type hf_channel_t = "raw" | "resolve"; - -export const HF_URL_BASE = "https://huggingface.co/openai/whisper-"; -export const HF_URL_RAW = "raw"; -export const HF_URL_RESOLVE = "resolve"; -export const HF_URL_END = "/main/pytorch_model.bin"; - -export function create_hf_url(tag: whisper_tag_t, channel: hf_channel_t) { - return `${HF_URL_BASE}${tag}/${channel}${HF_URL_END}`; -} - -export type hf_metadata_t = { - version: string; - oid: string; - size: string; -}; - -export type download_status_t = { - doesTargetExist: boolean; - isDownloadComplete: boolean; - hasDownloadStarted: boolean; - progress?: { - current: number; - total: number; - remaining: number; - percentRemaining: number; - }; -}; - -export class WhisperFile { - hf_metadata: hf_metadata_t | undefined; - - target_hash: string | undefined; - does_target_exist: boolean = false; - does_part_target_exist: boolean = false; - download_data: FileSystem.DownloadProgressData | undefined; - - constructor( - public tag: whisper_model_tag_t, - private targetFileName?: string, - public label?: string, - public size?: number - ) { - this.targetFileName = this.targetFileName || `${tag}.bin`; - this.label = - this.label || `${tag[0].toUpperCase()}${tag.substring(1).toLowerCase()}`; - } - - get targetPath() { - return Paths.join(WHISPER_MODEL_PATH, this.targetFileName as string); - } - - get targetPartPath() { - return this.targetPath + ".part"; - } - - get targetFile() { - return new File(this.targetPath); - } - - get targetPartFile() { - return new File(this.targetPartPath); - } - - async getTargetInfo() { - return await FileSystem.getInfoAsync(this.targetPath); - } - - async getTargetPartInfo() { - return await FileSystem.getInfoAsync(this.targetPartPath); - } - - async updateTargetExistence() { - this.does_target_exist = (await this.getTargetInfo()).exists; - console.log("Determining if %s exists: %s", this.targetPath, this.does_target_exist) - this.does_part_target_exist = (await this.getTargetPartInfo()).exists; - console.log("Determining if %s exists: %s", this.targetPartPath, this.does_part_target_exist) - } - - public async getTargetSha() { - await this.updateTargetExistence(); - if (!this.does_target_exist) { - console.debug("%s does not exist", this.targetPath); - return undefined; - } - - const strData = await FileSystem.readAsStringAsync(this.targetPath, { - encoding: FileSystem.EncodingType.Base64, - }); - const data = strToArrBuf(strData); - - const digest = await Crypto.digest( - Crypto.CryptoDigestAlgorithm.SHA256, - data - ); - - return digest; - } - - public async updateTargetHash() { - const targetSha = await this.getTargetSha(); - if (!targetSha) return; - this.target_hash = arrbufToStr(targetSha); - } - - get isHashValid() { - return this.target_hash === this.hf_metadata?.oid; - } - - delete(ignoreErrors = true) { - try { - this.does_target_exist && this.targetFile.delete(); - this.does_part_target_exist && this.targetPartFile.delete(); - } catch (err) { - console.error(err); - if (!ignoreErrors) { - throw err; - } - } - console.debug("Successfully deleted %s and %s", this.targetPartPath, this.targetPath); - } - - get modelUrl() { - return create_hf_url(this.tag, "resolve"); - } - - get metadataUrl() { - return create_hf_url(this.tag, "raw"); - } - - get percentDone() { - if (!this.download_data) return 0; - return ( - (this.download_data.totalBytesWritten / - this.download_data.totalBytesExpectedToWrite) * - 100 - ); - } - - get percentLeft() { - if (!this.download_data) return 0; - return 100 - this.percentDone; - } - - public async syncHfMetadata() { - try { - const resp = await fetch(this.metadataUrl, { - credentials: "include", - headers: { - "User-Agent": - "Mozilla/5.0 (X11; Linux x86_64; rv:135.0) Gecko/20100101 Firefox/135.0", - Accept: - "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "Accept-Language": "en-US,en;q=0.5", - "Sec-GPC": "1", - "Upgrade-Insecure-Requests": "1", - "Sec-Fetch-Dest": "document", - "Sec-Fetch-Mode": "navigate", - "Sec-Fetch-Site": "cross-site", - "If-None-Match": '"8fa71cbce85078986b46fb97caec22039e73351a"', - Priority: "u=0, i", - }, - method: "GET", - mode: "cors", - }); - const text = await resp.text(); - this.hf_metadata = Object.fromEntries( - text.split("\n").map((line) => line.split(" ")) - ) as hf_metadata_t; - } catch (err) { - console.error("Failed to fetch %s: %s", this.metadataUrl, err); - throw err; - } - } - - async createDownloadResumable( - options: { - onData?: DownloadCallback | undefined; - onComplete?: CompletionCallback | undefined; - } = { - onData: undefined, - onComplete: undefined, - } - ) { - await this.syncHfMetadata(); - - // If the whisper model dir doesn't exist, create it. - if (!WHISPER_MODEL_DIR.exists) { - FileSystem.makeDirectoryAsync(WHISPER_MODEL_PATH, { - intermediates: true, - }); - } - - // Check for the existence of the target file - // If it exists, load the existing data. - await this.updateTargetExistence(); - - try { - // const existingData = this.does_target_exist - // ? await FileSystem.readAsStringAsync(this.targetPath, { - // encoding: FileSystem.EncodingType.Base64, - // }) - // : undefined; - - // Create the resumable. - return FileSystem.createDownloadResumable( - this.modelUrl, - this.targetPartPath, - {}, - async (data: FileSystem.DownloadProgressData) => { - console.log( - "Downloading %s: %d of %d", - this.targetPartPath, - data.totalBytesExpectedToWrite, - data.totalBytesWritten - ); - - // console.debug("yes, I'm still downloading"); - try { - this.download_data = data; - } catch (err) { - console.error("Failed to set downloadData: %s", err); - } - - try { - await this.syncHfMetadata(); - } catch (err) { - console.error("Failed to update HuggingFace metadata: %s", err); - } - - // try { - // await this.updateTargetHash(); - // } catch (er) { - // console.error("Failed to update target hash: %s", er); - // } - - try { - await this.updateTargetExistence(); - } catch (err) { - console.error("Failed to update target existence: %s", err); - } - if (options.onData) await options.onData(this); - - if (data.totalBytesExpectedToWrite === data.totalBytesWritten) { - console.debug( - "Finalizing; copying from %s -> %s", - this.targetPartPath, - this.targetPath - ); - await FileSystem.moveAsync({ - from: this.targetPartPath, - to: this.targetPath, - }); - await this.updateTargetExistence(); - options.onComplete && options.onComplete(this); - } - }, - // existingData ? existingData : undefined - ); - } catch (err) { - console.error("Could not read %s: %s", this.targetPath, err); - } - } -} - -export type DownloadCallback = (arg0: WhisperFile) => any; -export type CompletionCallback = (arg0: WhisperFile) => any; - -export const WHISPER_FILES = { - small: new WhisperFile("small"), - medium: new WhisperFile("medium"), - large: new WhisperFile("large"), -}; +export async function whisperModelExists() { + const file = new File(WHISPER_MODEL_PATH); + return file.exists; +} \ No newline at end of file diff --git a/components/ConversationThread.tsx b/components/ConversationThread.tsx index 832d7a4..8d42db7 100644 --- a/components/ConversationThread.tsx +++ b/components/ConversationThread.tsx @@ -15,10 +15,13 @@ import { LanguageServer, language_matrix_entry, } from "@/app/i18n/api"; -import { Settings } from "@/app/lib/settings"; -import { WHISPER_FILES } from "@/app/lib/whisper"; +import { + WHISPER_MODEL_SMALL_PATH, + whisperModelExists, +} from "@/app/lib/whisper"; import { initWhisper, WhisperContext } from "whisper.rn"; import { useAudioRecorder, AudioModule, RecordingPresets } from "expo-audio"; +import FileSystem from "expo-file-system"; const lasOptions = { sampleRate: 32000, // default is 44100 but 32000 is adequate for accurate voice recognition @@ -97,22 +100,27 @@ const ConversationThread = ({ console.log("Set cached translator from %s", languageServer.baseUrl); setCachedTranslator(cachedTranslator); - const settings = await Settings.getDefault(); - const whisperFileLabel = (await settings.getWhisperModel()) || "small"; - const whisperFile = WHISPER_FILES[whisperFileLabel]; - - if (!whisperFile) { - throw new Error(`Could not find the whisper file with the label ${whisperFileLabel}`); + try { + if (!(await whisperModelExists())) { + throw new Error(`${WHISPER_MODEL_SMALL_PATH} does not exist`); + } + } catch (err) { + console.error( + `Could not determine if %s exists: %s`, + WHISPER_MODEL_SMALL_PATH, + err + ); + throw err; } try { setWhisperContext( await initWhisper({ - filePath: whisperFile.targetPath, + filePath: WHISPER_MODEL_SMALL_PATH, }) ); } catch (err) { - console.error(err) + console.error(err); throw err; } diff --git a/components/LanguageSelection.tsx b/components/LanguageSelection.tsx index 48f7b16..412fe71 100644 --- a/components/LanguageSelection.tsx +++ b/components/LanguageSelection.tsx @@ -36,7 +36,7 @@ export function LanguageSelection(props: { // Replace with your actual async data fetching logic setTranslator(await CachedTranslator.getDefault()); const languageServer = await LanguageServer.getDefault(); - const languages = await languageServer.fetchLanguages(5000); + const languages = await languageServer.fetchLanguages(10000); setLanguages(languages); setLanguagesLoaded(true); } catch (error) { diff --git a/components/Settings.tsx b/components/Settings.tsx index 867704d..1650945 100644 --- a/components/Settings.tsx +++ b/components/Settings.tsx @@ -1,23 +1,11 @@ import React, { useState, useEffect } from "react"; -import { View, Text, TextInput, Pressable, StyleSheet } from "react-native"; -import { - WHISPER_FILES, - WhisperFile, - download_status_t, - whisper_tag_t, -} from "@/app/lib/whisper"; +import { View, Text, TextInput, StyleSheet } from "react-native"; import { Settings } from "@/app/lib/settings"; import { Picker } from "@react-native-picker/picker"; import { LanguageServer, language_matrix, - language_matrix_entry, } from "@/app/i18n/api"; -const WHISPER_MODELS = { - small: new WhisperFile("small"), - medium: new WhisperFile("medium"), - large: new WhisperFile("large"), -}; const LIBRETRANSLATE_BASE_URL = "https://translate.argosopentech.com/translate"; @@ -33,15 +21,6 @@ const SettingsComponent = () => { success: boolean; error?: string; } | null>(null); - const [whisperModel, setWhisperModel] = - useState("small"); - const [whisperFile, setWhisperFile] = useState(); - const [downloader, setDownloader] = useState(null); - const [bytesDone, setBytesDone] = useState(); - const [bytesRemaining, setBytesRemaining] = useState(); - const [statusTimeout, setStatusTimeout] = useState< - NodeJS.Timeout | undefined - >(); useEffect(() => { (async function () { @@ -50,22 +29,9 @@ const SettingsComponent = () => { setLibretranslateBaseUrl( (await settings.getLibretranslateBaseUrl()) || LIBRETRANSLATE_BASE_URL ); - setWhisperModel((await settings.getWhisperModel()) || "small"); - setWhisperFile(WHISPER_FILES[whisperModel]); - if (!whisperFile) { - throw new Error("Invalid Whisper file!"); - } - await whisperFile.syncHfMetadata(); - await whisperFile.updateTargetExistence(); - await whisperFile.updateTargetHash(); - console.log("Does %s exist? part=%s, target=%s", whisperFile.label, whisperFile.does_part_target_exist, whisperFile.does_target_exist) })(); - }, [whisperFile]); + }); - const getLanguageOptions = async () => { - const languageServer = await LanguageServer.getDefault(); - setLanguageOptions(await languageServer.fetchLanguages()); - }; const handleHostLanguageChange = async (lang: string) => { const settings = await Settings.getDefault(); @@ -83,71 +49,17 @@ const SettingsComponent = () => { const checkLangServerConnection = async (baseUrl: string) => { try { // Replace with actual connection check logic - setLangServerConn({ success: true }); + const testResult = await fetch(baseUrl, { + method: "HEAD", + }); + if (testResult.status !== 200) { + setLangServerConn({ success: true, error: testResult.statusText }); + } } catch (error) { setLangServerConn({ success: false, error: `${error}` }); } }; - const handleWhisperModelChange = async (model: whisper_tag_t) => { - const settings = await Settings.getDefault(); - await settings.setWhisperModel(model); - setWhisperModel(model); - const wFile = WHISPER_FILES[whisperModel]; - await wFile.syncHfMetadata(); - await wFile.updateTargetExistence(); - // await wFile.updateTargetHash(); - // setIsWhisperHashValid(wFile.isHashValid); - setWhisperFile(wFile); - }; - - const doSetDownloadStatus = (arg0: WhisperFile) => { - // console.log("Downloading ...."); - setBytesDone(arg0.download_data?.totalBytesWritten); - setBytesRemaining(arg0.download_data?.totalBytesExpectedToWrite); - }; - - const doOnComplete = async (arg0: WhisperFile) => { - console.log("✅ Download complete."); - setDownloader(undefined); - await arg0.updateTargetExistence(); - setWhisperFile(arg0); - await whisperFile?.updateTargetExistence(); - }; - - const doDownload = async () => { - if (!whisperModel) { - throw new Error("Could not start download because whisperModel not set."); - } - - console.log("Starting download of %s", whisperModel); - - if (!whisperFile) throw new Error("No whisper file"); - - try { - const resumable = await whisperFile.createDownloadResumable({ - onData: doSetDownloadStatus, - onComplete: doOnComplete, - }); - setDownloader(resumable); - if (!resumable) throw new Error("Could not construct resumable"); - await resumable.resumeAsync(); - } catch (error) { - console.error("Failed to download whisper model:", error); - } - }; - - const doStopDownload = async () => { - downloader.cancelAsync(); - setDownloader(null); - }; - - const doDelete = async () => { - const whisperFile = WHISPER_MODELS[whisperModel]; - whisperFile.delete(); - await whisperFile.updateTargetExistence(); - }; - return hostLanguage && libretranslateBaseUrl ? ( Host Language: @@ -180,98 +92,6 @@ const SettingsComponent = () => { Error connecting to {libretranslateBaseUrl}: {langServerConn.error} ))} - - {Object.entries(WHISPER_MODELS).map(([key, whisperFile]) => ( - - ))} - - - {/* The target is completely downloaded */} - {!downloader && whisperFile?.does_target_exist && ( - - - DELETE {whisperModel.toUpperCase()} - - - )} - - {/* The target "part" is present and is downloading */} - - {downloader && whisperFile?.does_part_target_exist && ( - - STOP DOWNLOAD - - )} - - {/* The target "part" is present and we are NOT downloading */} - - {!downloader && whisperFile?.does_part_target_exist && ( - <> - - RESUME DOWNLOAD - - - - DELETE {whisperModel.toUpperCase()} - - - - )} - - {/* Anything else -- usually if the file has not yet been downloaded */} - - {!( - downloader || - whisperFile?.does_target_exist || - whisperFile?.does_part_target_exist - ) && ( - - - DOWNLOAD {whisperModel.toUpperCase()} - - - )} - - {downloader && - bytesDone && - bytesRemaining && - whisperFile?.does_part_target_exist && ( - - {whisperFile && ( - Downloading to {whisperFile.targetPath} - )} - - {bytesDone} of {bytesRemaining} ( - {(bytesDone / bytesRemaining) * 100} %){" "} - - - )} - - - - Debug Panel - downloader is null? {downloader ? "no" : "yes"} - - whisperFile.does_target_exist{" "} - {whisperFile?.does_target_exist ? "yes" : "no"} - - - whisperFile.does_part_target_exist{" "} - {whisperFile?.does_part_target_exist ? "yes" : "no"} - - ) : ( diff --git a/package.json b/package.json index 8a95a6b..6868c9e 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,8 @@ "updateSnapshots": "jest -u --coverage=false", "start": "expo start", "reset-project": "node ./scripts/reset-project.js", - "android": "expo prebuild --npm -p android", + "prebuild:android": "expo prebuild --npm -p android", + "android": "expo run:android", "ios": "expo prebuild --npm -p android --offline", "web": "expo start --offline --web", "lint": "expo lint"