From f440c2204760ffdf9ef76750d3e226dde50062e4 Mon Sep 17 00:00:00 2001 From: Ishan Mishra Date: Tue, 20 Feb 2024 14:52:14 +0530 Subject: [PATCH] immediate display transcribed text added: frontend changes (#179) * immediate display transcribed text added * error handling --- .../[project_id]/chat/[chat_id]/page.tsx | 22 +++++++- src/app/(main)/project/[project_id]/page.tsx | 55 ++++++++++++++----- src/utils/api.ts | 9 +++ 3 files changed, 70 insertions(+), 16 deletions(-) diff --git a/src/app/(main)/project/[project_id]/chat/[chat_id]/page.tsx b/src/app/(main)/project/[project_id]/chat/[chat_id]/page.tsx index 1e3310a7..1bb8ffda 100644 --- a/src/app/(main)/project/[project_id]/chat/[chat_id]/page.tsx +++ b/src/app/(main)/project/[project_id]/chat/[chat_id]/page.tsx @@ -25,6 +25,7 @@ export default function Chat( const [chatMessage, setChatMessage] = useState(""); const [storage] = useAtom(storageAtom); const [isTyping, setIsTyping] = useState(false); + const [apiError, setApiError] = useState(undefined); const chatQuery = useQuery({ queryKey: ["chat", chat_id], @@ -120,8 +121,23 @@ export default function Chat( const handleAudio = async (blobUrl: string) => { setIsTyping(true); - const fd = await getFormData(storage, blobUrl); - converseMutation.mutate({ formdata: fd }); + const sttFormData = await getFormData(storage, blobUrl) + try{ + const {transcript, stats} = await API.chat.speechToText( + project_id, + chat_id, + sttFormData, + ) + setNewChat(transcript); + + const fd = await getFormData(storage, undefined, transcript); + fd.append("transcript_start_time", stats.transcript_start_time.toString()); + fd.append("transcript_end_time", stats.transcript_end_time.toString()); + converseMutation.mutate({ formdata: fd }); + } + catch(e: any){ + setApiError(e.message); + } }; const messagesContainerRef = useRef(null); @@ -186,7 +202,7 @@ export default function Chat( onChange={(e) => setNewChat(e.target.value)} onSubmit={handleSubmit} onAudio={handleAudio} - errors={[(converseMutation.error as any)?.error?.error]} + errors={[(converseMutation.error as any)?.error?.error, apiError]} loading={converseMutation.isPending || isTyping} projectId={project_id} forceLanguage={chat_language} diff --git a/src/app/(main)/project/[project_id]/page.tsx b/src/app/(main)/project/[project_id]/page.tsx index b0dfe689..bbbde1fd 100644 --- a/src/app/(main)/project/[project_id]/page.tsx +++ b/src/app/(main)/project/[project_id]/page.tsx @@ -20,6 +20,7 @@ export default function Chat(params: { params: { project_id: string } }) { const [chatMessage, setChatMessage] = useState(""); const [isTyping, setIsTyping] = useState(false); const [chatID, setChatID] = useState(""); + const [apiError, setApiError] = useState(undefined); const openai_key = !storage?.user?.allow_key || storage?.override_api_key @@ -55,20 +56,13 @@ export default function Chat(params: { params: { project_id: string } }) { const newChatMutation = useMutation( { - mutationFn: (params: { formdata: FormData }) => + mutationFn: () => API.chat.create( project_id, chat !== "" ? chat.slice(0, 50) : "new chat", storage.openai_api_key, ), retry: false, - onSuccess: async (data, vars) => { - setChatID(data.external_id); - await converseMutation.mutateAsync({ - external_id: data.external_id, - formdata: vars.formdata, - }); - }, }, ); @@ -113,14 +107,41 @@ export default function Chat(params: { params: { project_id: string } }) { const handleSubmit = async (e: React.FormEvent) => { setIsTyping(true); e.preventDefault(); - const fd = await getFormData(undefined, chat); - newChatMutation.mutate({ formdata: fd }); + + try{ + const {external_id} = await newChatMutation.mutateAsync(); + setChatID(external_id); + + const fd = await getFormData(undefined, chat); + converseMutation.mutate({ external_id, formdata: fd }); + }catch(e: any){ + setApiError(e.message); + } + }; const handleAudio = async (blobUrl: string) => { setIsTyping(true); - const fd = await getFormData(blobUrl); - newChatMutation.mutate({ formdata: fd }); + try{ + const {external_id} = await newChatMutation.mutateAsync(); + setChatID(external_id); + + const sttFormData = await getFormData(blobUrl); + const {transcript, stats} = await API.chat.speechToText( + project_id, + external_id, + sttFormData, + ) + setChat(transcript); + + const fd = await getFormData(undefined, transcript); + fd.append("transcript_start_time", stats.transcript_start_time.toString()); + fd.append("transcript_end_time", stats.transcript_end_time.toString()); + converseMutation.mutate({ external_id, formdata: fd }); + } + catch(e: any){ + setApiError(e.message); + } }; return ( @@ -144,7 +165,14 @@ export default function Chat(params: { params: { project_id: string } }) { setChat(prompt); setIsTyping(true); const fd = await getFormData(undefined, prompt); - newChatMutation.mutate({ formdata: fd }); + const {external_id} = await newChatMutation.mutateAsync(); + if(external_id === "") return; + setChatID(external_id); + + converseMutation.mutate({ + external_id, + formdata: fd, + }); }} disabled={newChatMutation.isPending} className="bg-white hover:shadow-lg hover:bg-gray-100 hover:text-indigo-500 text-left border border-gray-200 rounded-lg p-4 transition disabled:opacity-50 disabled:hover:text-gray-400" @@ -203,6 +231,7 @@ export default function Chat(params: { params: { project_id: string } }) { errors={[ (newChatMutation.error as any)?.error?.error, (newChatMutation.error as any)?.error?.non_field_errors, + apiError, ]} loading={ newChatMutation.isPending || converseMutation.isPending || isTyping diff --git a/src/utils/api.ts b/src/utils/api.ts index d87c2f92..555eb185 100644 --- a/src/utils/api.ts +++ b/src/utils/api.ts @@ -295,6 +295,15 @@ export const API = { request(`projects/${project_id}/chats/${id}`, "PATCH", fields), delete: (project_id: string, id: string) => request(`projects/${project_id}/chats/${id}`, "DELETE"), + speechToText: (project_id: string, chat_id: string, formdata: FormData) => + request( + `projects/${project_id}/chats/${chat_id}/speech_to_text`, + "POST", + formdata, + { + formdata: true, + }, + ), converse: ( project_id: string, chat_id: string,