bug fix for file upload & session switch (#67)

* fix aws file upload bugs Signed-off-by: cbh778899 <[email protected]> * edit styles of code section Signed-off-by: cbh778899 <[email protected]> * add padding for user input element Signed-off-by: cbh778899 <[email protected]> * add MIN_TOKENS Signed-off-by: cbh778899 <[email protected]> * if max_tokens set to 0, there will be no limitations Signed-off-by: cbh778899 <[email protected]> * remove output to console Signed-off-by: cbh778899 <[email protected]> * add special value to bypass checkValue Signed-off-by: cbh778899 <[email protected]> * change default max_tokens to 1024 as we have gpu inference now Signed-off-by: cbh778899 <[email protected]> * fix engine switching bugs, engines are not shared between sessions any more Signed-off-by: cbh778899 <[email protected]> --------- Signed-off-by: cbh778899 <[email protected]>
SkywardAI · Oct 17, 2024 · c8113d2 · c8113d2
1 parent 28881aa
commit c8113d2
Show file tree

Hide file tree

Showing 10 changed files with 53 additions and 30 deletions.
diff --git a/preloader/node-llama-cpp-preloader.js b/preloader/node-llama-cpp-preloader.js
@@ -106,10 +106,10 @@ async function chatCompletions(latest_message, cb=null) {
     const options = {
         signal: stop_signal.signal,
         stopOnAbortSignal: true,
-        maxTokens: max_tokens,
         topP: top_p,
         temperature
     }
+    if(max_tokens) options.maxTokens = max_tokens
     let resp_text = ''
     if(cb) options.onTextChunk = chunk => {
         resp_text += chunk;

diff --git a/src/components/chat/UserMessage.jsx b/src/components/chat/UserMessage.jsx
@@ -16,7 +16,7 @@ export default function UserMessage({ uid, enable_send, file_available, abort_co
         event.preventDefault();
         send(message, files);
         setMessage('');
-        setFiles('');
+        setFiles([]);
     }
 
     // update when uid changed, means we entered a new conversation
@@ -41,7 +41,7 @@ export default function UserMessage({ uid, enable_send, file_available, abort_co
                         <input 
                             type="file" className="clickable" 
                             title={files.length ? `Append file ${files.map(e=>e.name).join('; ')}` : "Select file to append"}
-                            onChange={evt=>setFiles(evt.target.files.length ? evt.target.files[0] : null)} />
+                            onChange={evt=>setFiles([...evt.target.files])} />
                     </div>
                 }
                 <input type="text" ref={inputRef} value={message} onChange={evt=>setMessage(evt.target.value)}/>

diff --git a/src/components/chat/index.jsx b/src/components/chat/index.jsx
@@ -1,11 +1,11 @@
-import { useEffect, useState } from "react";
+import { useEffect, useRef, useState } from "react";
 import Tickets from "./Tickets";
 // import Conversation from "./Conversation";
 import useIDB from "../../utils/idb";
 import DeleteConfirm from "./DeleteConfirm";
 import ChatPage from "./ChatPage";
-import { useRef } from "react";
 import { getCompletionFunctions } from "../../utils/workers";
+import { getPlatformSettings } from "../../utils/general_settings";
 
 export default function Chat() {
 
@@ -20,8 +20,8 @@ export default function Chat() {
     const [pending_message, setPendingMessage] = useState(null);
 
     const idb = useIDB();
-    // const settings = useRef(getCompletionFunctions());
-    const settings = useRef(getCompletionFunctions());
+    const platform = useRef(getPlatformSettings().enabled_platform);
+    const [session_setting, setSessionSetting] = useState({});
 
     async function sendMessage(message, files) {
         // save user messages
@@ -59,11 +59,11 @@ export default function Chat() {
 
         // start inference
         const send_message = (
-            settings.current.formator ? 
-            await settings.current.formator(history_save, files) : history_save
+            session_setting.formator ? 
+            await session_setting.formator(history_save, files) : history_save
         )
         setPendingMessage('')
-        await settings.current.completions(send_message, cb)
+        await session_setting.completions(send_message, cb)
     }
 
     function updateChatClient(client) {
@@ -127,27 +127,29 @@ export default function Chat() {
                 message_history = messages;
                 setChatHistory(messages)
             }).finally(()=>{
-                const client = settings.current.initClient(chat.client || null, message_history)
+                const ss = getCompletionFunctions(chat.platform);
+                const client = ss.initClient(chat.client || null, message_history)
                 if(!chat.client) {
                     updateChatClient(client)
                 }
+                setSessionSetting(ss);
             })
         }
     // eslint-disable-next-line
     }, [chat])
 
     return (
-        settings.current ?
+        platform.current ?
         <div className="chat">
             <Tickets 
                 selectChat={selectChat} current_chat={chat} 
                 setHistory={setTickets} history={tickets} 
-                deleteHistory={requestDelete} platform={settings.current.platform}
+                deleteHistory={requestDelete} platform={platform.current}
             />
             <ChatPage 
                 updateTitle={updateTitle}
                 chat={chat} chat_history={chat_history}
-                pending_message={pending_message} abort={settings.current.abort}
+                pending_message={pending_message} abort={session_setting.abort}
                 sendMessage={sendMessage}
             />
             <DeleteConfirm 

diff --git a/src/components/settings/ModelSettings.jsx b/src/components/settings/ModelSettings.jsx
@@ -2,6 +2,7 @@ import { useEffect, useState } from "react";
 import ScrollBarComponent from "./components/ScrollBarComponent";
 import SettingSection from "./SettingSection";
 import { getModelSettings, updateModelSettings } from "../../utils/general_settings";
+import { MIN_TOKENS } from "../../utils/types";
 
 export default function ModelSettings({ trigger, updateState }) {
 
@@ -10,8 +11,11 @@ export default function ModelSettings({ trigger, updateState }) {
     const [temperature, setTemperature] = useState(0);
 
     function saveSettings() {
+        let validate_max_token = max_tokens;
+        console.log(max_tokens)
+        if(max_tokens < MIN_TOKENS && max_tokens !== 0) validate_max_token = MIN_TOKENS;
         updateModelSettings({
-            max_tokens, top_p, temperature
+            max_tokens: validate_max_token, top_p, temperature
         })
         updateState()
     }
@@ -32,9 +36,9 @@ export default function ModelSettings({ trigger, updateState }) {
         <SettingSection title={'General Model Settings'}>
             <ScrollBarComponent
                 title={'Set Max Tokens'}
-                description={'The max tokens AI can generate'}
+                description={'The max tokens AI can generate, if set to 0 there will be no limitations.'}
                 value={max_tokens} cb={setMaxTokens}
-                max={2048} min={32}
+                max={2048} min={32} special={0}
             />
             <ScrollBarComponent
                 title={'Set Top P'}

diff --git a/src/components/settings/components/ScrollBarComponent.jsx b/src/components/settings/components/ScrollBarComponent.jsx
@@ -1,13 +1,13 @@
 import { useEffect, useState } from "react"
 
-export default function ScrollBarComponent({ cb, value, disabled, title, description, min, max, times_10, step }) {
+export default function ScrollBarComponent({ cb, value, disabled, title, description, min, max, times_10, step, special }) {
 
     const [scrollValue, setScrollValue] = useState((times_10 ? 10 : 1) * value);
     const [textValue, setTextValue] = useState(value);
 
     function checkValue(v) {
         v = v || +textValue;
-        return v <= max && v >= min;
+        return (v <= max && v >= min) || v === special;
     }
 
     function setValue(value, is_scroll = false) {

diff --git a/src/styles/chat.css b/src/styles/chat.css
@@ -248,12 +248,21 @@
     padding: 10px;
     overflow: auto;
 }
-.chat > .conversation-main > .bubbles > .bubble :not(pre) code {
+
+.chat > .conversation-main > .bubbles > .bubble code {
     padding: 0px 5px;
     background-color: rgb(227, 227, 227);
     border-radius: 5px;
 }
 
+.chat > .conversation-main > .bubbles > .bubble.user code {
+    background-color: rgb(22, 113, 203);
+}
+
+.chat > .conversation-main > .bubbles > .bubble pre code {
+    background-color: unset;
+}
+
 @keyframes dotAnimation {
     0% { color: rgb(90, 90, 90); }
     50% { color: rgb(150, 150, 150); }
@@ -290,7 +299,7 @@ input[type="text"] {
     height: 100%;
     position: relative;
     border: none;
-    padding: 0px var(--elem-size) 0px 10px;
+    padding: 0px calc(var(--elem-size) + 5px) 0px 10px;
 }
 
 .chat > .conversation-main > .send-message-form > .input-container >

diff --git a/src/utils/general_settings.js b/src/utils/general_settings.js
@@ -34,7 +34,7 @@ const MODEL_SETTINGS_KEY = 'general-model-settings'
  * @property {Number} temperature
  */
 const DEFAULT_MODEL_SETTINGS = {
-    max_tokens: 128,
+    max_tokens: 1024,
     top_p: 0.9,
     temperature: 0.7
 }

diff --git a/src/utils/types.js b/src/utils/types.js
@@ -6,4 +6,6 @@ export const LOAD_FINISHED = 1;
 export const LOAD_SET_SETTINGS = 2;
 export const LOAD_SKIP_SETTINGS = 3;
 
-export const DEFAULT_LLAMA_CPP_MODEL_URL = "https://huggingface.co/aisuko/Phi-3-mini-4k-instruct-gguf/resolve/main/Phi3-mini-4k-instruct-Q4.gguf"
+export const DEFAULT_LLAMA_CPP_MODEL_URL = "https://huggingface.co/aisuko/Phi-3-mini-4k-instruct-gguf/resolve/main/Phi3-mini-4k-instruct-Q4.gguf"
+
+export const MIN_TOKENS = 32;
diff --git a/src/utils/workers/aws-worker.js b/src/utils/workers/aws-worker.js
@@ -125,15 +125,19 @@ export async function chatCompletions(messages, cb = null) {
         }
     })
 
-    const { max_tokens:maxTokens, top_p:topP, temperature } = getModelSettings();
+    const { max_tokens, top_p:topP, temperature } = getModelSettings();
     const input = {
         modelId: aws_model_id,
         messages: normal_messages,
         inferenceConfig: {
-            maxTokens, temperature, topP
+            temperature, topP
         }
     }
 
+    if(max_tokens) {
+        input.inferenceConfig.maxTokens = max_tokens
+    }
+
     if(system.length) input.system = system;
     let response_text = '', usage = {}
 
@@ -195,9 +199,9 @@ export async function formator(messages, files = []) {
     if(files.length) {
         for(const file of files) {
             const file_info = file.name.split('.')
-            const extension = file_info.pop();
+            const extension = file_info.pop().toLowerCase();
             const filename = file_info.join('_');
-            const bytes = await file.arrayBuffer()
+            const bytes = new Uint8Array(await file.arrayBuffer())
 
             if(/^image\/.+/.test(file.type)) {
                 common_messages[common_messages.length - 1].content.push(
@@ -209,11 +213,12 @@ export async function formator(messages, files = []) {
                     }
                 )
             } else {
+                const is_valid_format = /^(docx|csv|html|txt|pdf|md|doc|xlsx|xls)$/.test(extension)
                 common_messages[common_messages.length - 1].content.push(
                     {
                         document: {
-                            name: filename,
-                            format: extension,
+                            name: filename + (is_valid_format ? '' : `_${extension}`),
+                            format: is_valid_format ? extension : 'txt' ,
                             source: { bytes  }
                         }
                     }

diff --git a/src/utils/workers/wllama-worker.js b/src/utils/workers/wllama-worker.js
@@ -35,10 +35,11 @@ export function loadModelSamplingSettings() {
         n_threads: wllama_threads, 
         n_batch: wllama_batch_size, 
         n_ctx: wllama_context_length,
-        nPredict: max_tokens,
         temp: temperature,
         top_p
     }
+
+    if(max_tokens) model_sampling_settings.nPredict = max_tokens;
 }
 loadModelSamplingSettings();