updating code using knex.
This commit is contained in:
Jordan
@ -1,32 +1,26 @@
|
||||
import {describe, expect, beforeEach} from '@jest/globals';
|
||||
import {Settings} from '@/app/lib/settings';
|
||||
import { getDb, migrateDb } from '@/app/lib/db';
|
||||
import { Settings } from '@/app/lib/settings';
|
||||
import { getDb } from '@/app/lib/db';
|
||||
import { Knex } from 'knex';
|
||||
|
||||
describe('Settings', () => {
|
||||
let settings: Settings;
|
||||
let settings : Settings;
|
||||
let db : Knex;
|
||||
|
||||
beforeEach(async () => {
|
||||
// Initialize your Settings class here with a fresh database instance
|
||||
await migrateDb();
|
||||
const db = await getDb();
|
||||
if (!db) throw new Error("Could not get db");
|
||||
settings = new Settings(db);
|
||||
db = await getDb("development");
|
||||
settings = new Settings(db)
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
// Clean up the database after each test
|
||||
settings && await settings.db.executeSql('DELETE FROM settings');
|
||||
await db.migrate.down();
|
||||
});
|
||||
|
||||
describe('setHostLanguage', () => {
|
||||
it('should set the host language in the database', async () => {
|
||||
const value = 'en';
|
||||
await settings.db.runAsync("REPLACE INTO settings (host_language) VALUES (?)", "en");
|
||||
await settings.setHostLanguage(value);
|
||||
it('should set the host language in the database', async () => {
|
||||
const value = 'en';
|
||||
await settings.setHostLanguage(value);
|
||||
|
||||
const result = await settings.getHostLanguage();
|
||||
expect(result).toEqual(value);
|
||||
});
|
||||
const result = await settings.getHostLanguage();
|
||||
expect(result).toEqual(value);
|
||||
});
|
||||
|
||||
describe('getHostLanguage', () => {
|
||||
@ -40,7 +34,7 @@ describe('Settings', () => {
|
||||
|
||||
it('should return null if the host language is not set', async () => {
|
||||
const result = await settings.getHostLanguage();
|
||||
expect(result).toBeNull();
|
||||
expect(result).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
@ -57,7 +51,34 @@ describe('Settings', () => {
|
||||
describe('getLibretranslateBaseUrl', () => {
|
||||
it('should return null if the LibreTranslate base URL is not set', async () => {
|
||||
const result = await settings.getLibretranslateBaseUrl();
|
||||
expect(result).toBeNull();
|
||||
expect(result).toBeUndefined();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('setWhisperModel', () => {
|
||||
it('should set the Whisper model in the database', async () => {
|
||||
const value = 'base';
|
||||
await settings.setWhisperModel(value);
|
||||
|
||||
const result = await settings.getWhisperModel();
|
||||
expect(result).toEqual(value);
|
||||
});
|
||||
});
|
||||
describe('setWhisperModel', () => {
|
||||
it('should set the Whisper model in the database', async () => {
|
||||
const value = 'base';
|
||||
await settings.setWhisperModel(value);
|
||||
|
||||
const result = await settings.getWhisperModel();
|
||||
expect(result).toEqual(value);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getWhisperModel', () => {
|
||||
it('should return null if the Whisper model is not set', async () => {
|
||||
const result = await settings.getWhisperModel();
|
||||
expect(result).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
101
app/lib/__tests__/whisper.spec.tsx
Normal file
101
app/lib/__tests__/whisper.spec.tsx
Normal file
@ -0,0 +1,101 @@
|
||||
// components/ui/__tests__/WhisperFile.spec.tsx
|
||||
|
||||
import React from "react";
|
||||
import { render, act } from "@testing-library/react-native";
|
||||
import { WhisperFile } from "@/app/lib/whisper"; // Adjust the import path as necessary
|
||||
|
||||
describe("WhisperFile", () => {
|
||||
let whisperFile: WhisperFile;
|
||||
|
||||
beforeEach(() => {
|
||||
whisperFile = new WhisperFile("small");
|
||||
});
|
||||
|
||||
it("should initialize correctly", () => {
|
||||
expect(whisperFile).toBeInstanceOf(WhisperFile);
|
||||
});
|
||||
|
||||
describe("getModelFileSize", () => {
|
||||
it("should return the correct model file size", async () => {
|
||||
expect(whisperFile.size).toBeUndefined();
|
||||
await whisperFile.updateMetadata();
|
||||
expect(whisperFile.size).toBeGreaterThan(1000);
|
||||
});
|
||||
});
|
||||
|
||||
describe("getWhisperDownloadStatus", () => {
|
||||
it("should return the correct download status", async () => {
|
||||
const mockStatus = {
|
||||
doesTargetExist: true,
|
||||
isDownloadComplete: false,
|
||||
hasDownloadStarted: true,
|
||||
progress: {
|
||||
current: 100,
|
||||
total: 200,
|
||||
remaining: 100,
|
||||
percentRemaining: 50.0,
|
||||
},
|
||||
};
|
||||
jest
|
||||
.spyOn(whisperFile, "getDownloadStatus")
|
||||
.mockResolvedValue(mockStatus);
|
||||
|
||||
const result = await whisperFile.getDownloadStatus();
|
||||
|
||||
expect(result).toEqual(mockStatus);
|
||||
});
|
||||
});
|
||||
|
||||
describe("initiateWhisperDownload", () => {
|
||||
it("should initiate the download with default options", async () => {
|
||||
const mockModelLabel = "small";
|
||||
jest
|
||||
.spyOn(whisperFile, "createDownloadResumable")
|
||||
.mockResolvedValue(true);
|
||||
|
||||
await whisperFile.initiateWhisperDownload(mockModelLabel);
|
||||
|
||||
expect(whisperFile.createDownloadResumable).toHaveBeenCalledWith(
|
||||
mockModelLabel
|
||||
);
|
||||
});
|
||||
|
||||
it("should initiate the download with custom options", async () => {
|
||||
const mockModelLabel = "small";
|
||||
const mockOptions = { force_redownload: true };
|
||||
jest
|
||||
.spyOn(whisperFile, "createDownloadResumable")
|
||||
.mockResolvedValue(true);
|
||||
|
||||
await whisperFile.initiateWhisperDownload(mockModelLabel, mockOptions);
|
||||
|
||||
expect(whisperFile.createDownloadResumable).toHaveBeenCalledWith(
|
||||
mockModelLabel,
|
||||
mockOptions
|
||||
);
|
||||
});
|
||||
|
||||
it("should return the correct download status when target exists and is complete", async () => {
|
||||
jest.spyOn(whisperFile, "doesTargetExist").mockResolvedValue(true);
|
||||
jest.spyOn(whisperFile, "isDownloadComplete").mockResolvedValue(true);
|
||||
|
||||
expect(await whisperFile.doesTargetExist()).toEqual(true);
|
||||
expect(await whisperFile.isDownloadComplete()).toEqual(true);
|
||||
});
|
||||
|
||||
it("should return the correct download status when target does not exist", async () => {
|
||||
jest.spyOn(whisperFile, "doesTargetExist").mockResolvedValue(false);
|
||||
|
||||
const result = await whisperFile.getDownloadStatus();
|
||||
|
||||
expect(result).toEqual({
|
||||
doesTargetExist: false,
|
||||
isDownloadComplete: false,
|
||||
hasDownloadStarted: false,
|
||||
progress: undefined,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// Add more tests as needed for other methods in WhisperFile
|
||||
});
|
||||
@ -1,26 +1,11 @@
|
||||
import * as SQLite from "expo-sqlite";
|
||||
import { MIGRATE_UP, MIGRATE_DOWN } from "./migrations";
|
||||
import config from "@/knexfile";
|
||||
import Knex from "knex";
|
||||
|
||||
export async function getDb() {
|
||||
return await SQLite.openDatabaseAsync("translation_terrace");
|
||||
export async function getDb(
|
||||
environment: "development" | "staging" | "production" = "production",
|
||||
automigrate = true
|
||||
) {
|
||||
const k = Knex(config[environment]);
|
||||
if (automigrate) await k.migrate.up();
|
||||
return k;
|
||||
}
|
||||
|
||||
|
||||
export async function migrateDb(direction: "up" | "down" = "up") {
|
||||
|
||||
const db = await getDb();
|
||||
|
||||
const m = direction === "up" ? MIGRATE_UP : MIGRATE_DOWN;
|
||||
|
||||
for (let [migration, statements] of Object.entries(m)) {
|
||||
for (let statement of statements) {
|
||||
console.log(statement);
|
||||
try {
|
||||
const result = await db.runAsync(statement);
|
||||
console.log(result);
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,23 +0,0 @@
|
||||
|
||||
export const MIGRATE_UP = {
|
||||
1: [
|
||||
`CREATE TABLE IF NOT EXISTS settings (
|
||||
host_language TEXT,
|
||||
libretranslate_base_url TEXT,
|
||||
ui_direction INTEGER,
|
||||
whisper_model TEXT
|
||||
)`,
|
||||
],
|
||||
2: [
|
||||
`CREATE TABLE IF NOT EXISTS whisper_models (
|
||||
model TEXT PRIMARY KEY,
|
||||
bytes_done INTEGER,
|
||||
bytes_total INTEGER
|
||||
)`,
|
||||
],
|
||||
};
|
||||
|
||||
export const MIGRATE_DOWN = {
|
||||
1: [`DROP TABLE IF EXISTS settings`],
|
||||
2: [`DROP TABLE IF EXISTS whisper_models`],
|
||||
};
|
||||
0
app/lib/models.ts
Normal file
0
app/lib/models.ts
Normal file
@ -1,6 +1,5 @@
|
||||
import { SQLiteDatabase } from "expo-sqlite";
|
||||
import FileSystem from "expo-file-system"
|
||||
import { getDb } from "./db";
|
||||
import { Knex } from "knex";
|
||||
|
||||
export class Settings {
|
||||
|
||||
@ -11,7 +10,7 @@ export class Settings {
|
||||
"whisper_model",
|
||||
]
|
||||
|
||||
constructor(public db: SQLiteDatabase) {
|
||||
constructor(public db: Knex) {
|
||||
|
||||
}
|
||||
|
||||
@ -21,23 +20,29 @@ export class Settings {
|
||||
throw new Error(`Invalid setting: '${key}'`)
|
||||
}
|
||||
|
||||
const query = `
|
||||
SELECT ${key}
|
||||
FROM settings
|
||||
LIMIT 1`
|
||||
const result = await this.db.getFirstAsync(
|
||||
query
|
||||
);
|
||||
|
||||
return result ? (result as any)[key] : null;
|
||||
const row = await this.db("settings").select(key).limit(1).first();
|
||||
if (!(row && row[key])) return undefined;
|
||||
return row[key];
|
||||
}
|
||||
|
||||
|
||||
private async setValue(key: string, value: any) {
|
||||
if (!Settings.KEYS.includes(key)) {
|
||||
throw new Error(`Invalid setting: '${key}'`)
|
||||
throw new Error(`Invalid setting: '${key}'`);
|
||||
}
|
||||
|
||||
// Check if the key already exists
|
||||
const [exists] = await this.db("settings").select(1).whereNotNull(key).limit(1);
|
||||
|
||||
if (exists) {
|
||||
// Update the existing column
|
||||
await this.db("settings").update({ [key]: value });
|
||||
} else {
|
||||
// Insert a new value into the specified column
|
||||
const insertData: { [key: string]: any } = {};
|
||||
insertData[key] = value;
|
||||
await this.db("settings").insert(insertData);
|
||||
}
|
||||
const statement = `REPLACE INTO settings (${key}) VALUES (?)`
|
||||
await this.db.runAsync(statement, value);
|
||||
}
|
||||
|
||||
async setHostLanguage(value: string) {
|
||||
|
||||
@ -1,9 +1,13 @@
|
||||
import { Platform } from "react-native";
|
||||
import * as FileSystem from "expo-file-system";
|
||||
import { File, Paths } from 'expo-file-system/next';
|
||||
import { File, Paths } from "expo-file-system/next";
|
||||
import { getDb } from "./db";
|
||||
import * as Crypto from "expo-crypto";
|
||||
|
||||
export const WHISPER_MODEL_PATH = Paths.join(FileSystem.documentDirectory || "file:///", "whisper");
|
||||
export const WHISPER_MODEL_PATH = Paths.join(
|
||||
FileSystem.documentDirectory || "file:///",
|
||||
"whisper"
|
||||
);
|
||||
export const WHISPER_MODEL_DIR = new File(WHISPER_MODEL_PATH);
|
||||
|
||||
// Thanks to https://medium.com/@fabi.mofar/downloading-and-saving-files-in-react-native-expo-5b3499adda84
|
||||
@ -46,7 +50,7 @@ function shareAsync(uri: string) {
|
||||
}
|
||||
|
||||
export const WHISPER_MODEL_TAGS = ["small", "medium", "large"];
|
||||
export type whisper_model_tag_t = (typeof WHISPER_MODEL_TAGS)[number];
|
||||
export type whisper_model_tag_t = "small" | "medium" | "large";
|
||||
|
||||
export const WHISPER_MODELS = {
|
||||
small: {
|
||||
@ -54,164 +58,290 @@ export const WHISPER_MODELS = {
|
||||
"https://huggingface.co/openai/whisper-small/blob/resolve/pytorch_model.bin",
|
||||
target: "small.bin",
|
||||
label: "Small",
|
||||
size: 967092419,
|
||||
},
|
||||
medium: {
|
||||
source:
|
||||
"https://huggingface.co/openai/whisper-medium/resolve/main/pytorch_model.bin",
|
||||
target: "medium.bin",
|
||||
label: "Medium",
|
||||
size: 3055735323,
|
||||
},
|
||||
large: {
|
||||
source:
|
||||
"https://huggingface.co/openai/whisper-large/resolve/main/pytorch_model.bin",
|
||||
target: "large.bin",
|
||||
label: "Large",
|
||||
size: 6173629930,
|
||||
},
|
||||
} as {
|
||||
[key: whisper_model_tag_t]: { source: string; target: string; label: string };
|
||||
[key:whisper_model_tag_t]: {
|
||||
source: string;
|
||||
target: string;
|
||||
label: string;
|
||||
size: number;
|
||||
};
|
||||
};
|
||||
|
||||
export function getWhisperTarget(key : whisper_model_tag_t) {
|
||||
const path = Paths.join(WHISPER_MODEL_DIR, WHISPER_MODELS[key].target);
|
||||
return new File(path)
|
||||
export type whisper_tag_t = "small" | "medium" | "large";
|
||||
export type hf_channel_t = "raw" | "resolve";
|
||||
|
||||
export const HF_URL_BASE = "https://huggingface.co/openai/whisper-";
|
||||
export const HF_URL_RAW = "raw";
|
||||
export const HF_URL_RESOLVE = "resolve";
|
||||
export const HF_URL_END = "/main/pytorch_model.bin";
|
||||
|
||||
export function create_hf_url(tag: whisper_tag_t, channel: hf_channel_t) {
|
||||
return `${HF_URL_BASE}${tag}/${channel}${HF_URL_END}`;
|
||||
}
|
||||
|
||||
export type download_status =
|
||||
| {
|
||||
status: "not_started" | "complete";
|
||||
}
|
||||
| {
|
||||
status: "in_progress";
|
||||
bytes: {
|
||||
total: number;
|
||||
done: number;
|
||||
};
|
||||
};
|
||||
export type hf_metadata_t = {
|
||||
version: string;
|
||||
oid: string;
|
||||
size: string;
|
||||
};
|
||||
|
||||
export async function getModelFileSize(whisper_model: whisper_model_tag_t) {
|
||||
const target = getWhisperTarget(whisper_model)
|
||||
if (!target.exists) return undefined;
|
||||
return target.size;
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param whisper_model The whisper model key to check (e.g. `"small"`)
|
||||
* @returns
|
||||
*/
|
||||
export async function getWhisperDownloadStatus(
|
||||
whisper_model: whisper_model_tag_t
|
||||
): Promise<download_status> {
|
||||
// const files = await FileSystem.readDirectoryAsync("file:///whisper");
|
||||
const result = (await (
|
||||
await getDb()
|
||||
).getFirstSync(
|
||||
`
|
||||
SELECT (bytes_done, total) WHERE model = ?
|
||||
`,
|
||||
[whisper_model]
|
||||
)) as { bytes_done: number; total: number } | undefined;
|
||||
|
||||
if (!result)
|
||||
return {
|
||||
status: "not_started",
|
||||
};
|
||||
|
||||
if (result.bytes_done < result.total)
|
||||
return {
|
||||
status: "in_progress",
|
||||
bytes: {
|
||||
done: result.bytes_done,
|
||||
total: result.total,
|
||||
},
|
||||
};
|
||||
|
||||
return {
|
||||
status: "complete",
|
||||
export type download_status_t = {
|
||||
doesTargetExist: boolean;
|
||||
isDownloadComplete: boolean;
|
||||
hasDownloadStarted: boolean;
|
||||
progress?: {
|
||||
current: number;
|
||||
total: number;
|
||||
remaining: number;
|
||||
percentRemaining: number;
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
export function whisperFileExists(whisper_model : whisper_model_tag_t) {
|
||||
const target = getWhisperTarget(whisper_model);
|
||||
return target.exists
|
||||
}
|
||||
|
||||
export type DownloadCallback = (arg0 : FileSystem.DownloadProgressData) => any;
|
||||
|
||||
async function updateModelSize(model_label : string, size : number) {
|
||||
const db = await getDb();
|
||||
const query = "INSERT OR REPLACE INTO whisper_models (model, bytes_total) VALUES (?, ?)"
|
||||
const stmt = db.prepareSync(query);
|
||||
stmt.executeSync(model_label, size);
|
||||
}
|
||||
|
||||
async function getExpectedModelSize(model_label : string) : Promise<number | undefined> {
|
||||
const db = await getDb();
|
||||
const query = "SELECT bytes_total FROM whisper_models WHERE model = ?"
|
||||
const stmt = db.prepareSync(query);
|
||||
const curs = stmt.executeSync(model_label);
|
||||
const row = curs.getFirstSync()
|
||||
return row ? row.bytes_total : undefined;
|
||||
}
|
||||
|
||||
export async function initiateWhisperDownload(
|
||||
whisper_model: whisper_model_tag_t,
|
||||
options: {
|
||||
force_redownload?: boolean;
|
||||
onDownload?: DownloadCallback | undefined;
|
||||
} = {
|
||||
force_redownload: false,
|
||||
onDownload: undefined,
|
||||
export class WhisperFile {
|
||||
constructor(
|
||||
public tag: whisper_model_tag_t,
|
||||
private targetFileName?: string,
|
||||
public label?: string,
|
||||
public size?: number
|
||||
) {
|
||||
this.targetFileName = this.targetFileName || `${tag}.bin`;
|
||||
this.label =
|
||||
this.label || `${tag[0].toUpperCase}${tag.substring(1).toLowerCase()}`;
|
||||
}
|
||||
) {
|
||||
|
||||
console.debug("Starting download of %s", whisper_model);
|
||||
get targetPath() {
|
||||
return Paths.join(WHISPER_MODEL_DIR, this.targetFileName as string);
|
||||
}
|
||||
|
||||
await FileSystem.makeDirectoryAsync(WHISPER_MODEL_PATH, {
|
||||
intermediates: true,
|
||||
});
|
||||
get targetFile() {
|
||||
return new File(this.targetPath);
|
||||
}
|
||||
|
||||
const whisperTarget = getWhisperTarget(whisper_model);
|
||||
async getTargetInfo() {
|
||||
return await FileSystem.getInfoAsync(this.targetPath);
|
||||
}
|
||||
|
||||
// If the target file exists, delete it.
|
||||
if (whisperTarget.exists) {
|
||||
if (options.force_redownload) {
|
||||
whisperTarget.delete()
|
||||
} else {
|
||||
const expected = await getExpectedModelSize(whisper_model);
|
||||
if (whisperTarget.size === expected) {
|
||||
console.warn("Whisper model for %s already exists", whisper_model);
|
||||
return undefined;
|
||||
async doesTargetExist() {
|
||||
return (await this.getTargetInfo()).exists;
|
||||
}
|
||||
|
||||
public async recordLatestTargetHash() {
|
||||
if (!(await this.doesTargetExist())) {
|
||||
console.debug("%s does not exist", this.targetPath);
|
||||
}
|
||||
const digest1Str = await this.getActualTargetHash();
|
||||
if (!digest1Str) {
|
||||
return;
|
||||
}
|
||||
const db = await getDb();
|
||||
await db("whisper_models")
|
||||
.upsert({
|
||||
model: this.tag,
|
||||
last_hash: digest1Str,
|
||||
})
|
||||
.where({ model: this.tag });
|
||||
}
|
||||
|
||||
public async getRecordedTargetHash(): Promise<string> {
|
||||
const db = await getDb();
|
||||
const row = await db("whisper_models").select("last_hash").where({
|
||||
model: this.tag,
|
||||
}).first();
|
||||
return row["last_hash"]
|
||||
}
|
||||
|
||||
public async getActualTargetHash(): Promise<string | undefined> {
|
||||
if (!(await this.doesTargetExist())) {
|
||||
console.debug("%s does not exist", this.targetPath);
|
||||
return undefined;
|
||||
}
|
||||
const digest1 = await Crypto.digest(
|
||||
Crypto.CryptoDigestAlgorithm.SHA256,
|
||||
this.targetFile.bytes()
|
||||
);
|
||||
const digest1Str = new TextDecoder().decode(new Uint8Array(digest1));
|
||||
return digest1Str;
|
||||
}
|
||||
|
||||
async isTargetCorrupted() {
|
||||
const recordedTargetHash = await this.getRecordedTargetHash();
|
||||
const actualTargetHash = await this.getActualTargetHash();
|
||||
if (!(actualTargetHash || recordedTargetHash)) return false;
|
||||
return actualTargetHash !== recordedTargetHash;
|
||||
}
|
||||
|
||||
async isDownloadComplete() {
|
||||
if (!(await this.doesTargetExist())) {
|
||||
console.debug("%s does not exist", this.targetPath);
|
||||
return false;
|
||||
}
|
||||
const data = this.targetFile.bytes();
|
||||
const meta = await this.fetchMetadata();
|
||||
const expectedHash = meta.oid;
|
||||
const digest1: ArrayBuffer = await Crypto.digest(
|
||||
Crypto.CryptoDigestAlgorithm.SHA256,
|
||||
data
|
||||
);
|
||||
const digest1Str = new TextDecoder().decode(new Uint8Array(digest1));
|
||||
const doesMatch = digest1Str === expectedHash;
|
||||
if (!doesMatch) {
|
||||
console.debug(
|
||||
"sha256 of '%s' does not match expected '%s'",
|
||||
digest1Str,
|
||||
expectedHash
|
||||
);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
delete(ignoreErrors = true) {
|
||||
try {
|
||||
this.targetFile.delete();
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
if (!ignoreErrors) {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Initiate a new resumable download.
|
||||
const spec = WHISPER_MODELS[whisper_model];
|
||||
get modelUrl() {
|
||||
return create_hf_url(this.tag, "resolve");
|
||||
}
|
||||
|
||||
console.log("Downloading %s", spec.source);
|
||||
get metadataUrl() {
|
||||
return create_hf_url(this.tag, "raw");
|
||||
}
|
||||
|
||||
const resumable = FileSystem.createDownloadResumable(
|
||||
spec.source,
|
||||
whisperTarget.uri,
|
||||
{
|
||||
md5: true,
|
||||
headers: {
|
||||
'Content-Type': 'application/octet-stream',
|
||||
'Accept': 'application/octet-stream',
|
||||
private async fetchMetadata(): Promise<hf_metadata_t> {
|
||||
try {
|
||||
const resp = await fetch(this.metadataUrl, {
|
||||
credentials: "include",
|
||||
headers: {
|
||||
"User-Agent":
|
||||
"Mozilla/5.0 (X11; Linux x86_64; rv:135.0) Gecko/20100101 Firefox/135.0",
|
||||
Accept:
|
||||
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "en-US,en;q=0.5",
|
||||
"Sec-GPC": "1",
|
||||
"Upgrade-Insecure-Requests": "1",
|
||||
"Sec-Fetch-Dest": "document",
|
||||
"Sec-Fetch-Mode": "navigate",
|
||||
"Sec-Fetch-Site": "cross-site",
|
||||
"If-None-Match": '"8fa71cbce85078986b46fb97caec22039e73351a"',
|
||||
Priority: "u=0, i",
|
||||
},
|
||||
method: "GET",
|
||||
mode: "cors",
|
||||
});
|
||||
const text = await resp.text();
|
||||
return Object.fromEntries(
|
||||
text.split("\n").map((line) => line.split(" "))
|
||||
) as hf_metadata_t;
|
||||
} catch (err) {
|
||||
console.error("Failed to fetch %s: %s", this.metadataUrl, err);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async updateMetadata() {
|
||||
const metadata = await this.fetchMetadata();
|
||||
this.size = Number.parseInt(metadata.size);
|
||||
}
|
||||
|
||||
async addToDatabase() {
|
||||
const db = await getDb();
|
||||
await db("whisper_models").upsert({
|
||||
model: this.tag,
|
||||
expected_size: this.size,
|
||||
}).where({
|
||||
model: this.tag,
|
||||
});
|
||||
}
|
||||
|
||||
async createDownloadResumable(
|
||||
options: {
|
||||
onData?: DownloadCallback | undefined;
|
||||
} = {
|
||||
onData: undefined,
|
||||
}
|
||||
) {
|
||||
const existingData = (await this.doesTargetExist())
|
||||
? this.targetFile.text()
|
||||
: undefined;
|
||||
|
||||
if (await this.doesTargetExist()) {
|
||||
}
|
||||
|
||||
return FileSystem.createDownloadResumable(
|
||||
this.modelUrl,
|
||||
this.targetPath,
|
||||
{},
|
||||
async (data: FileSystem.DownloadProgressData) => {
|
||||
const db = await getDb();
|
||||
await db.upsert({
|
||||
model: this.tag,
|
||||
download_status: "active",
|
||||
})
|
||||
await this.recordLatestTargetHash();
|
||||
if (options.onData) await options.onData(data);
|
||||
},
|
||||
sessionType: FileSystem.FileSystemSessionType.BACKGROUND
|
||||
},
|
||||
// On each data write, update the whisper model download status.
|
||||
// Note that since createDownloadResumable callback only works in the foreground,
|
||||
// a background process will also be updating the file size.
|
||||
async (data) => {
|
||||
console.log("%s: %d bytes of %d", whisperTarget.uri, data.totalBytesWritten, data.totalBytesExpectedToWrite);
|
||||
await updateModelSize(whisper_model, data.totalBytesExpectedToWrite)
|
||||
if (options.onDownload) await options.onDownload(data);
|
||||
},
|
||||
whisperTarget.exists ? whisperTarget.base64() : undefined,
|
||||
);
|
||||
existingData ? existingData : undefined
|
||||
);
|
||||
}
|
||||
|
||||
return resumable;
|
||||
}
|
||||
async getDownloadStatus() : Promise<download_status_t> {
|
||||
const doesTargetExist = await this.doesTargetExist();
|
||||
const isDownloadComplete = await this.isDownloadComplete();
|
||||
const hasDownloadStarted = doesTargetExist && !isDownloadComplete;
|
||||
|
||||
if (!this.size) {
|
||||
return {
|
||||
doesTargetExist: false,
|
||||
isDownloadComplete: false,
|
||||
hasDownloadStarted: false,
|
||||
progress: undefined,
|
||||
}
|
||||
}
|
||||
|
||||
const remaining = hasDownloadStarted
|
||||
? this.size - (this.targetFile.size as number)
|
||||
: 0;
|
||||
|
||||
const progress = hasDownloadStarted
|
||||
? {
|
||||
current: this.targetFile.size || 0,
|
||||
total: this.size,
|
||||
remaining: this.size - (this.targetFile.size as number),
|
||||
percentRemaining: (remaining / this.size) * 100.0,
|
||||
}
|
||||
: undefined;
|
||||
|
||||
return {
|
||||
doesTargetExist,
|
||||
isDownloadComplete,
|
||||
hasDownloadStarted,
|
||||
progress,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
export type DownloadCallback = (arg0: FileSystem.DownloadProgressData) => any;
|
||||
Reference in New Issue
Block a user