start to integrate whisper.

This commit is contained in:
Jordan
2025-02-02 06:09:21 -08:00
parent 718d8e034f
commit 013578778c
7 changed files with 115 additions and 24 deletions

View File

@ -1,20 +1,41 @@
import React, { useState, useEffect } from 'react';
import { View } from 'react-native';
import { Conversation, Message } from '@/app/lib/conversation';
import MessageBubble from '@/components/ui/MessageBubble';
import { NavigationProp, ParamListBase } from '@react-navigation/native';
import { language_matrix_entry, Translator } from '@/app/i18n/api';
import { getDb } from '@/app/lib/db';
import React, { useState, useEffect } from "react";
import { ScrollView, Text, TouchableHighlight, View } from "react-native";
import { Conversation, Message } from "@/app/lib/conversation";
import MessageBubble from "@/components/ui/MessageBubble";
import { WhisperContext } from "whisper.rn";
import { NavigationProp, ParamListBase } from "@react-navigation/native";
import {
CachedTranslator,
language_matrix_entry,
Translator,
} from "@/app/i18n/api";
import { getDb } from "@/app/lib/db";
import LiveAudioStream from 'react-native-live-audio-stream';
const lasOptions = {
sampleRate: 32000, // default is 44100 but 32000 is adequate for accurate voice recognition
channels: 1, // 1 or 2, default 1
bitsPerSample: 16, // 8 or 16, default 16
audioSource: 6, // android only (see below)
bufferSize: 4096 // default is 2048
};
// LiveAudioStream.init(lasOptions as any);
interface ConversationThreadProps {
conversation: Conversation;
whisperContext: WhisperContext;
onGoBack?: () => any;
}
const ConversationThread = (p : ConversationThreadProps) => {
const ConversationThread = (p: ConversationThreadProps) => {
const [messages, setMessages] = useState<Message[]>([]);
const [guestSpeak, setGuestSpeak] = useState<string | undefined>();
const [guestSpeakLoaded, setGuestSpeakLoaded] = useState<boolean>(false);
const ct = new CachedTranslator("en", p.conversation.guest.language);
useEffect(() => {
const updateMessages = (c : Conversation) => {
const updateMessages = (c: Conversation) => {
setMessages([...c]);
};
@ -25,19 +46,59 @@ const ConversationThread = (p : ConversationThreadProps) => {
p.conversation.onAddMessage = undefined;
p.conversation.onTranslationDone = undefined;
};
}, [p.conversation]);
}, [p.conversation, guestSpeak]);
const renderMessages = () => (
useEffect(() => {
const fetchData = async () => {
setGuestSpeak(await ct.translate("Speak"));
}
fetchData();
}, [guestSpeak])
const renderMessages = () =>
messages.map((message, index) => (
<MessageBubble key={index} message={message} />
))
);
));
function onGoBack() {
p.onGoBack && p.onGoBack();
}
return (
<View style={{ flex: 1 }}>
{renderMessages()}
<View style={{ flex: 1, flexDirection: "column" }}>
<ScrollView
style={{
borderColor: "black",
borderWidth: 1,
borderStyle: "solid",
height: "90%",
}}
>
{renderMessages()}
</ScrollView>
<View style={{ alignSelf: "center", flexDirection: "row" }}>
<TouchableHighlight
style={{ backgroundColor: "blue", padding: 3, borderRadius: 5 }}
>
<Text style={{ color: "white", fontSize: 30 }}>Speak</Text>
</TouchableHighlight>
<TouchableHighlight
style={{ backgroundColor: "gray", padding: 3, borderRadius: 5 }}
onPress={onGoBack}
>
<Text style={{ color: "white", fontSize: 30 }}>Go Back</Text>
</TouchableHighlight>
<TouchableHighlight
style={{ backgroundColor: "blue", padding: 3, borderRadius: 5 }}
>
<Text style={{ color: "white", fontSize: 30 }}>
{guestSpeak ? guestSpeak : "Speak"}
</Text>
</TouchableHighlight>
</View>
</View>
);
};
export default ConversationThread;
export default ConversationThread;