feat: Optimize the voice mode to make the conversation more natural a…

…nd concise feat: Optimize the copy code logic to avoid including markdown syntax
u14app · Feb 3, 2025 · 5c4a5d1 · 5c4a5d1
1 parent 7c7bdba
commit 5c4a5d1
Show file tree

Hide file tree

Showing 8 changed files with 147 additions and 52 deletions.
diff --git a/app/page.tsx b/app/page.tsx
@@ -190,6 +190,9 @@ export default function Home() {
         safety,
       }
       if (systemInstruction) config.systemInstruction = systemInstruction
+      if (talkMode === 'voice') {
+        config.systemInstruction = `${getVoiceModelPrompt()}\n\n${systemInstruction}`
+      }
       if (tools.length > 0 && !isThinkingModel) config.tools = [{ functionDeclarations: tools }]
       if (apiKey !== '') {
         config.baseUrl = apiProxy || GEMINI_API_BASE_URL
@@ -277,7 +280,7 @@ export default function Home() {
         }
       }
     },
-    [systemInstruction, isThinkingModel],
+    [systemInstruction, isThinkingModel, talkMode],
   )
 
   const summarize = useCallback(
@@ -326,9 +329,7 @@ export default function Home() {
         },
         onStatement: (statement) => {
           if (talkMode === 'voice') {
-            // Remove list symbols and adjust layout
-            const audioText = statement.replaceAll('*', '').replaceAll('\n\n', '\n')
-            speech(audioText)
+            speech(statement)
           }
         },
         onFinish: async () => {
@@ -596,7 +597,6 @@ export default function Home() {
         messages = getTalkAudioPrompt(messages)
       }
       if (talkMode === 'voice') {
-        messages = getVoiceModelPrompt(messages)
         setStatus('thinkng')
         setSubtitle('')
       }

diff --git a/components/FileList.tsx b/components/FileList.tsx
@@ -34,7 +34,7 @@ function FileList({ fileList, onRemove }: Props) {
         return (
           <div
             className={cn(
-              'flex rounded-md border p-1.5',
+              'flex rounded-md border p-1.5 text-left',
               file.status === 'FAILED' ? 'border-red-500 text-red-500' : '',
             )}
             key={file.id}

diff --git a/components/Magicdown/Code.tsx b/components/Magicdown/Code.tsx
@@ -12,15 +12,14 @@ import './highlight.css'
 
 type Props = {
   children: ReactNode
-  content: string | null | undefined
   lang: string
 }
 
 function getLangAlias(lang: string): string {
   return get(langAlias, lang, capitalize(lang)) || ''
 }
 
-function Code({ children, content, lang }: Props) {
+function Code({ children, lang }: Props) {
   const { t } = useTranslation()
   const codeWrapperRef = useRef<HTMLDivElement>(null)
   const [waitingCopy, setWaitingCopy] = useState<boolean>(false)
@@ -29,9 +28,9 @@ function Code({ children, content, lang }: Props) {
   const isMobile = useIsMobile(450)
 
   const handleCopy = () => {
-    if (content) {
+    if (codeWrapperRef.current) {
       setWaitingCopy(true)
-      copy(content)
+      copy(codeWrapperRef.current.innerText)
       setTimeout(() => {
         setWaitingCopy(false)
       }, 1200)

diff --git a/components/Magicdown/index.tsx b/components/Magicdown/index.tsx
@@ -14,14 +14,6 @@ import 'katex/dist/katex.min.css'
 const Code = dynamic(() => import('./Code'))
 const Mermaid = dynamic(() => import('./Mermaid'))
 
-function getContent(content: string | null | undefined, start?: number, end?: number): string {
-  if (content && start && end) {
-    return content.substring(start, end)
-  } else {
-    return ''
-  }
-}
-
 function Magicdown({ children: content, className, ...rest }: Options) {
   const remarkPlugins = useMemo(() => rest.remarkPlugins ?? [], [rest.remarkPlugins])
   const rehypePlugins = useMemo(() => rest.rehypePlugins ?? [], [rest.rehypePlugins])
@@ -50,10 +42,7 @@ function Magicdown({ children: content, className, ...rest }: Options) {
               return <Mermaid>{children}</Mermaid>
             }
             return (
-              <Code
-                content={getContent(content, node?.position?.start.offset, node?.position?.end.offset)}
-                lang={lang ? lang[1] : ''}
-              >
+              <Code lang={lang ? lang[1] : ''}>
                 <code {...rest} className={clsx('break-all', className)}>
                   {children}
                 </code>

diff --git a/components/MessageItem.tsx b/components/MessageItem.tsx
@@ -1,6 +1,6 @@
 'use client'
 import dynamic from 'next/dynamic'
-import { useEffect, useState, useCallback, useMemo, memo } from 'react'
+import { useEffect, useState, useCallback, useRef, useMemo, memo } from 'react'
 import { useTranslation } from 'react-i18next'
 import Lightbox from 'yet-another-react-lightbox'
 import LightboxFullscreen from 'yet-another-react-lightbox/plugins/fullscreen'
@@ -20,6 +20,7 @@ import {
 } from 'lucide-react'
 import { EdgeSpeech } from '@xiangfa/polly'
 import copy from 'copy-to-clipboard'
+import { convert } from 'html-to-text'
 import { Avatar, AvatarFallback } from '@/components/ui/avatar'
 import { Accordion, AccordionContent, AccordionItem, AccordionTrigger } from '@/components/ui/accordion'
 import BubblesLoading from '@/components/BubblesLoading'
@@ -71,10 +72,10 @@ function mergeSentences(sentences: string[], sentenceLength = 20): string[] {
 function MessageItem(props: Props) {
   const { id, role, parts, attachments, onRegenerate } = props
   const { t } = useTranslation()
+  const contentRef = useRef<HTMLDivElement>(null)
   const [html, setHtml] = useState<string>('')
   const [thoughtsHtml, setThoughtsHtml] = useState<string>('')
   const chatLayout = useMessageStore((state) => state.chatLayout)
-  const [hasTextContent, setHasTextContent] = useState<boolean>(false)
   const [isEditing, setIsEditing] = useState<boolean>(false)
   const [isCopyed, setIsCopyed] = useState<boolean>(false)
   const [showLightbox, setShowLightbox] = useState<boolean>(false)
@@ -155,8 +156,14 @@ function MessageItem(props: Props) {
     }, 1200)
   }, [content])
 
-  const handleSpeak = useCallback(async (content: string) => {
+  const handleSpeak = useCallback(async () => {
+    if (!contentRef.current) return false
+
     const { lang, ttsLang, ttsVoice } = useSettingStore.getState()
+    const content = convert(contentRef.current.innerHTML, {
+      wordwrap: false,
+      selectors: [{ selector: 'ul', options: { itemPrefix: '  ' } }],
+    })
     const sentences = mergeSentences(sentenceSegmentation(content, lang), 100)
     const edgeSpeech = new EdgeSpeech({ locale: ttsLang })
     const audioStream = new AudioStream()
@@ -307,7 +314,9 @@ function MessageItem(props: Props) {
                   </AccordionItem>
                 </Accordion>
               ) : null}
-              <Magicdown>{html}</Magicdown>
+              <div ref={contentRef}>
+                <Magicdown>{html}</Magicdown>
+              </div>
               <div
                 className={cn(
                   'flex gap-1 text-right opacity-0 transition-opacity duration-300 group-hover:opacity-100 max-md:opacity-30',
@@ -325,13 +334,13 @@ function MessageItem(props: Props) {
                     <IconButton title={t('edit')} onClick={() => setIsEditing(true)}>
                       <PencilLine className="h-4 w-4" />
                     </IconButton>
-                    <IconButton title={t('copy')} className={`copy-${id}`} onClick={() => handleCopy()}>
+                    <IconButton title={t('copy')} onClick={() => handleCopy()}>
                       {isCopyed ? <CopyCheck className="h-4 w-4" /> : <Copy className="h-4 w-4" />}
                     </IconButton>
                     <IconButton title={t('delete')} onClick={() => handleDelete(id)}>
                       <Eraser className="h-4 w-4" />
                     </IconButton>
-                    <IconButton title={t('speak')} onClick={() => handleSpeak(content)}>
+                    <IconButton title={t('speak')} onClick={() => handleSpeak()}>
                       <Volume2 className="h-4 w-4" />
                     </IconButton>
                   </>
@@ -358,15 +367,13 @@ function MessageItem(props: Props) {
         setThoughtsHtml(textParts[0].text)
       }
       if (textParts[1].text) {
-        setHasTextContent(true)
         setHtml(textParts[1].text)
       }
     } else {
       const messageParts: string[] = []
       parts.forEach(async (part) => {
         if (part.text) {
           messageParts.push(part.text)
-          setHasTextContent(true)
         }
       })
       setHtml(messageParts.join(''))

diff --git a/package.json b/package.json
@@ -44,6 +44,7 @@
     "copy-to-clipboard": "^3.3.3",
     "dayjs": "^1.11.13",
     "fix-webm-duration": "^1.0.5",
+    "html-to-text": "^9.0.5",
     "i18next": "^23.11.5",
     "i18next-browser-languagedetector": "^7.2.1",
     "i18next-resources-to-backend": "^1.2.1",
@@ -78,6 +79,7 @@
   },
   "devDependencies": {
     "@tauri-apps/cli": "^2.1.0",
+    "@types/html-to-text": "^9.0.4",
     "@types/lodash-es": "^4.17.12",
     "@types/node": "^20.14.2",
     "@types/react": "^18.3.3",