From c27754125781819352063be348aac363e9b996e5 Mon Sep 17 00:00:00 2001 From: Bohan Cheng <47214785+cbh778899@users.noreply.github.com> Date: Mon, 21 Oct 2024 17:32:04 +1100 Subject: [PATCH] add function to switch between downloaded models (#70) * add dropdown element for select model Signed-off-by: cbh778899 * implement model switch & hide reset everytime Signed-off-by: cbh778899 * update style Signed-off-by: cbh778899 * fix llama_reset_everytime issue so we can normally reset now Signed-off-by: cbh778899 --------- Signed-off-by: cbh778899 --- preloader/index.js | 5 +- preloader/node-llama-cpp-preloader.js | 63 +++++++++++++------ src/components/settings/LlamaSettings.jsx | 41 +++++++++--- .../settings/components/DropdownComponent.jsx | 25 ++++++++ src/styles/settings.css | 6 ++ src/utils/workers/index.js | 3 +- 6 files changed, 112 insertions(+), 31 deletions(-) create mode 100644 src/components/settings/components/DropdownComponent.jsx diff --git a/preloader/index.js b/preloader/index.js index 90a1552..1ece726 100644 --- a/preloader/index.js +++ b/preloader/index.js @@ -5,7 +5,8 @@ const { abortCompletion, setClient, downloadModel, - updateModelSettings + updateModelSettings, + formator } = require("./node-llama-cpp-preloader.js") const { @@ -14,7 +15,7 @@ const { contextBridge.exposeInMainWorld('node-llama-cpp', { loadModel, chatCompletions, updateModelSettings, - abortCompletion, setClient, downloadModel + abortCompletion, setClient, downloadModel, formator }) contextBridge.exposeInMainWorld('file-handler', { diff --git a/preloader/node-llama-cpp-preloader.js b/preloader/node-llama-cpp-preloader.js index ceae26f..bad950c 100644 --- a/preloader/node-llama-cpp-preloader.js +++ b/preloader/node-llama-cpp-preloader.js @@ -40,9 +40,18 @@ async function loadModel(model_name = '') { }) } + + +/** + * @typedef Message + * @property {"user"|"assistant"|"system"} role Sender + * @property {String} content Message content + */ + /** * Set the session, basically reset the history and return a static string 'fake-client' * @param {String} client a fake client, everything using the same client and switch between clients just simply reset the chat history + * @param {Message[]} history the history to load * @returns {String} */ async function setClient(client, history = []) { @@ -66,20 +75,20 @@ async function setClient(client, history = []) { * @param {any} message The latest user message, either string or any can be converted to string. If is array in the format of {content:String}, it will retrieve the last content * @returns {String} the retrived message */ -function findMessageText(message) { - if(typeof message === "string") return message; - else if(typeof message === "object") { - if(Array.isArray(message)) { - while(message.length) { - message = message.pop(); - if(typeof message === 'object' && message.role && message.role === 'user' && message.content) { - return message.content; - } - } - } - } - return `${message}` -} +// function findMessageText(message) { +// if(typeof message === "string") return message; +// else if(typeof message === "object") { +// if(Array.isArray(message)) { +// while(message.length) { +// message = message.pop(); +// if(typeof message === 'object' && message.role && message.role === 'user' && message.content) { +// return message.content; +// } +// } +// } +// } +// return `${message}` +// } let model_settings = {}; function updateModelSettings(settings) { @@ -99,15 +108,12 @@ function updateModelSettings(settings) { * @returns {Promise} the response text */ async function chatCompletions(latest_message, cb=null) { - const {max_tokens, top_p, temperature, llama_reset_everytime} = model_settings; + const {max_tokens, top_p, temperature} = model_settings; if(!llama_session) { cb && cb('> **ERROR: MODEL NOT LOADED**', true); return ''; } - latest_message = findMessageText(latest_message); - if(llama_reset_everytime) { - setClient(null, llama_session.getChatHistory().filter(({type})=>type === 'system')) - } + // latest_message = findMessageText(latest_message);} stop_signal = new AbortController(); @@ -210,11 +216,28 @@ function downloadModel(url, cb=null) { }) } +/** + * format messages, reset history if needed + * @param {Message[]} messages + */ +function formator(messages) { + const user_messages = messages.filter(e=>e.role === 'user'); + const system_messages = messages.filter(e=>e.role === 'system'); + + const {llama_reset_everytime} = model_settings; + if(llama_reset_everytime) { + setClient(null, system_messages) + } + + return user_messages.pop().content; +} + module.exports = { loadModel, chatCompletions, abortCompletion, setClient, downloadModel, - updateModelSettings + updateModelSettings, + formator } \ No newline at end of file diff --git a/src/components/settings/LlamaSettings.jsx b/src/components/settings/LlamaSettings.jsx index 763ed3d..167e386 100644 --- a/src/components/settings/LlamaSettings.jsx +++ b/src/components/settings/LlamaSettings.jsx @@ -5,11 +5,13 @@ import TextComponent from "./components/TextComponent"; import useIDB from "../../utils/idb"; import { getPlatformSettings, updatePlatformSettings } from "../../utils/general_settings"; import { DEFAULT_LLAMA_CPP_MODEL_URL } from "../../utils/types"; +import DropdownComponent from "./components/DropdownComponent"; export default function LlamaSettings({ trigger, enabled, updateEnabled, openDownloadProtector, updateState }) { const [model_download_link, setModelDownloadLink] = useState(''); const [reset_everytime, setResetEveryTime] = useState(false); + const [downloaded_models, setDownloadedModels] = useState([]) const idb = useIDB(); async function saveSettings() { @@ -39,14 +41,23 @@ export default function LlamaSettings({ trigger, enabled, updateEnabled, openDow platform: 'Llama' } await idb.insert("downloaded-models", stored_model) + setDownloadedModels([...downloaded_models, { title: stored_model['model-name'], value: stored_model.url }]) } } ) } - // load model using the model name retrieved - await window['node-llama-cpp'].loadModel(stored_model['model-name']) + await openDownloadProtector( + 'Loading model...', + `Loading model ${stored_model['model-name']}`, + async callback => { + callback(100, false); + // load model using the model name retrieved + await window['node-llama-cpp'].loadModel(stored_model['model-name']) + updateState(); + callback(100, true) + } + ) } - updateState(); } useEffect(()=>{ @@ -55,9 +66,18 @@ export default function LlamaSettings({ trigger, enabled, updateEnabled, openDow }, [trigger]) useEffect(()=>{ - const { llama_model_url, llama_reset_everytime } = getPlatformSettings(); - setModelDownloadLink(llama_model_url || DEFAULT_LLAMA_CPP_MODEL_URL); - setResetEveryTime(llama_reset_everytime); + (async function() { + const { llama_model_url, llama_reset_everytime } = getPlatformSettings(); + setModelDownloadLink(llama_model_url || DEFAULT_LLAMA_CPP_MODEL_URL); + setResetEveryTime(llama_reset_everytime); + + const models = await idb.getAll("downloaded-models", { + where: [{'platform': 'Llama'}], + select: ["model-name", "url"] + }); + setDownloadedModels(models.map(e=>{return { title: e['model-name'], value: e.url }})) + })() + // eslint-disable-next-line }, []) return ( @@ -67,9 +87,14 @@ export default function LlamaSettings({ trigger, enabled, updateEnabled, openDow description={"Llama.cpp Engine is powerful and it allows you to run your own .gguf model using GPU on your local machine."} value={enabled} cb={updateEnabled} /> + diff --git a/src/components/settings/components/DropdownComponent.jsx b/src/components/settings/components/DropdownComponent.jsx new file mode 100644 index 0000000..35be96a --- /dev/null +++ b/src/components/settings/components/DropdownComponent.jsx @@ -0,0 +1,25 @@ +export default function DropdownComponent({ cb, value, title, description }) { + return ( +
+
{title}
+ { description &&
{description}
} + +
+ ) +} \ No newline at end of file diff --git a/src/styles/settings.css b/src/styles/settings.css index f628fd4..999feb7 100644 --- a/src/styles/settings.css +++ b/src/styles/settings.css @@ -164,6 +164,12 @@ margin-left: var(--elem-margin); } +.setting-page > .setting-section > .component > select.main-part { + border-radius: 7px; + background-color: white; + padding: 0px 7px; +} + dialog:has(.download-protector) { border: none; border-radius: 15px; diff --git a/src/utils/workers/index.js b/src/utils/workers/index.js index dc8cd21..f5e1db5 100644 --- a/src/utils/workers/index.js +++ b/src/utils/workers/index.js @@ -40,7 +40,8 @@ export function getCompletionFunctions(platform = null) { completions: window['node-llama-cpp'].chatCompletions, abort: window['node-llama-cpp'].abortCompletion, platform: 'Llama', - initClient: window['node-llama-cpp'].setClient + initClient: window['node-llama-cpp'].setClient, + formator: window['node-llama-cpp'].formator } case "Wllama": return {