From 460cc49da944c69ece86cfbcd090ea7bfe94e9b6 Mon Sep 17 00:00:00 2001 From: cybercoder Date: Sun, 21 Jan 2024 14:06:36 +0000 Subject: [PATCH 1/6] Update package.json update the keyworkds --- IntelliNode/package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/IntelliNode/package.json b/IntelliNode/package.json index cb7765e..5be2b00 100644 --- a/IntelliNode/package.json +++ b/IntelliNode/package.json @@ -17,7 +17,8 @@ "prompt", "automation", "mistralai", - "gemini" + "gemini", + "robotics" ], "author": "IntelliNode", "license": "Apache", From 17c63685caead7869bea9c2b9dfb05387a839da1 Mon Sep 17 00:00:00 2001 From: cybercoder Date: Sun, 21 Jan 2024 21:07:13 +0000 Subject: [PATCH 2/6] Update ChatModelInput.js add extra parameters --- IntelliNode/model/input/ChatModelInput.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/IntelliNode/model/input/ChatModelInput.js b/IntelliNode/model/input/ChatModelInput.js index 69cefa6..1eeb2db 100644 --- a/IntelliNode/model/input/ChatModelInput.js +++ b/IntelliNode/model/input/ChatModelInput.js @@ -180,6 +180,8 @@ class GeminiInput extends ChatModelInput { constructor(systemMessage, options = {}) { super(options); this.messages = []; + this.maxOutputTokens = options.maxTokens + this.temperature = options.temperature if (systemMessage && typeof systemMessage === 'string') { this.addUserMessage(systemMessage); @@ -207,7 +209,11 @@ class GeminiInput extends ChatModelInput { getChatInput() { return { - contents: this.messages + contents: this.messages, + generationConfig: { + ...(this.temperature && { temperature: this.temperature }), + ...(this.maxOutputTokens && { maxOutputTokens: this.maxOutputTokens }), + } }; } From 8c288f6df0b795be82e4634698aa3076530c1ce5 Mon Sep 17 00:00:00 2001 From: cybercoder Date: Sun, 21 Jan 2024 21:47:17 +0000 Subject: [PATCH 3/6] Update LLMEvaluation.js add more supported chatbots --- IntelliNode/utils/LLMEvaluation.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/IntelliNode/utils/LLMEvaluation.js b/IntelliNode/utils/LLMEvaluation.js index f806a8a..1c74fe8 100644 --- a/IntelliNode/utils/LLMEvaluation.js +++ b/IntelliNode/utils/LLMEvaluation.js @@ -2,7 +2,7 @@ const { RemoteEmbedModel, SupportedEmbedModels } = require('../controller/Remote const LanguageModelInput = require('../model/input/LanguageModelInput'); const { Chatbot, SupportedChatModels } = require("../function/Chatbot"); const { RemoteLanguageModel, SupportedLangModels } = require("../controller/RemoteLanguageModel"); -const { ChatGPTInput, LLamaReplicateInput, LLamaSageInput } = require("../model/input/ChatModelInput"); +const { ChatGPTInput, LLamaReplicateInput, LLamaSageInput, GeminiInput, CohereInput, MistralInput } = require("../model/input/ChatModelInput"); const MatchHelpers = require('../utils/MatchHelpers'); const EmbedInput = require('../model/input/EmbedInput'); const { ModelEvaluation } = require('./ModelEvaluation'); @@ -26,7 +26,7 @@ class LLMEvaluation extends ModelEvaluation { } async generateText(apiKey, inputString, provider, modelName, type, - maxTokens = 400, custom_url = null) { + maxTokens = 500, custom_url = null) { if (type == 'chat' && Object.values(SupportedChatModels).includes(provider.toLowerCase())) { @@ -40,6 +40,12 @@ class LLMEvaluation extends ModelEvaluation { input = new LLamaReplicateInput("provide direct answer", { model: modelName, maxTokens: maxTokens}); } else if (SupportedChatModels.SAGEMAKER == provider.toLowerCase()) { input = new LLamaSageInput("provide direct answer", {maxTokens: maxTokens}); + } else if (SupportedChatModels.GEMINI == provider.toLowerCase()) { + input = new GeminiInput("provide direct answer", {maxTokens: maxTokens}); + } else if (SupportedChatModels.COHERE == provider.toLowerCase()) { + input = new CohereInput("provide direct answer", {maxTokens: maxTokens}); + } else if (SupportedChatModels.MISTRAL == provider.toLowerCase()) { + input = new MistralInput("provide direct answer", {maxTokens: maxTokens}); } else { input = new ChatGPTInput("provide direct answer", { model: modelName, maxTokens: maxTokens}); } From b41ced74ad5c76879d6b7cfdebbfe830d6cbae88 Mon Sep 17 00:00:00 2001 From: cybercoder Date: Sun, 21 Jan 2024 21:47:26 +0000 Subject: [PATCH 4/6] Update ModelEvaluation.test.js add gemini to the testing --- IntelliNode/test/integration/ModelEvaluation.test.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/IntelliNode/test/integration/ModelEvaluation.test.js b/IntelliNode/test/integration/ModelEvaluation.test.js index 79b2f36..30f2d29 100644 --- a/IntelliNode/test/integration/ModelEvaluation.test.js +++ b/IntelliNode/test/integration/ModelEvaluation.test.js @@ -11,6 +11,8 @@ const openaiChat = { apiKey: process.env.OPENAI_API_KEY, provider: SupportedChat type: 'chat', model:'gpt-3.5-turbo', maxTokens: 50}; const cohereCompletion = { apiKey: process.env.COHERE_API_KEY, provider: SupportedLangModels.COHERE, type:'completion', model: 'command', maxTokens: 50}; +const geminiChat = { apiKey: process.env.GEMINI_API_KEY, provider: SupportedChatModels.GEMINI, + type:'chat', model: 'gemini'}; // create the evaluation object const llmEvaluation = new LLMEvaluation(process.env.OPENAI_API_KEY, 'openai'); @@ -20,7 +22,7 @@ async function testLLMEvaluation() { const targetAnswers = ["Photosynthesis is the process where green plants use sunlight to turn carbon dioxide and water into glucose and oxygen. The glucose provides food for the plant, and the oxygen gets released back into the air.", "Photosynthesis is how plants make their own food. They take in water and carbon dioxide, use the energy from sunlight to transform them into glucose (their food) and oxygen, which they release into the air.", "In simple terms, photosynthesis is like cooking for plants but instead of a stove, they use sunlight. They mix water and carbon dioxide with the sunlight to create glucose, which is their food, and also produce oxygen."]; - const providerSets = [llamaChat, openaiChat, cohereCompletion]; + const providerSets = [llamaChat, openaiChat, cohereCompletion, geminiChat]; const results = await llmEvaluation.compareModels(inputString, targetAnswers, providerSets); From 942dd9e09bac79703192d265124bc5c2b65a873f Mon Sep 17 00:00:00 2001 From: cybercoder Date: Sun, 21 Jan 2024 22:14:52 +0000 Subject: [PATCH 5/6] format, handle errors and add mistral model --- .../test/integration/ModelEvaluation.test.js | 42 ++++--- IntelliNode/utils/LLMEvaluation.js | 116 ++++++++++-------- 2 files changed, 89 insertions(+), 69 deletions(-) diff --git a/IntelliNode/test/integration/ModelEvaluation.test.js b/IntelliNode/test/integration/ModelEvaluation.test.js index 30f2d29..12f7281 100644 --- a/IntelliNode/test/integration/ModelEvaluation.test.js +++ b/IntelliNode/test/integration/ModelEvaluation.test.js @@ -5,14 +5,26 @@ const { SupportedChatModels } = require('../../function/Chatbot'); const { SupportedLangModels } = require('../../controller/RemoteLanguageModel'); // prepare the evaluation settings -const llamaChat = { apiKey: process.env.REPLICATE_API_KEY, provider: SupportedChatModels.REPLICATE, - type:'chat', model: '13b-chat', maxTokens: 50}; -const openaiChat = { apiKey: process.env.OPENAI_API_KEY, provider: SupportedChatModels.OPENAI, - type: 'chat', model:'gpt-3.5-turbo', maxTokens: 50}; -const cohereCompletion = { apiKey: process.env.COHERE_API_KEY, provider: SupportedLangModels.COHERE, - type:'completion', model: 'command', maxTokens: 50}; -const geminiChat = { apiKey: process.env.GEMINI_API_KEY, provider: SupportedChatModels.GEMINI, - type:'chat', model: 'gemini'}; +const llamaChat = { + apiKey: process.env.REPLICATE_API_KEY, provider: SupportedChatModels.REPLICATE, + type: 'chat', model: '13b-chat', maxTokens: 50 +}; +const openaiChat = { + apiKey: process.env.OPENAI_API_KEY, provider: SupportedChatModels.OPENAI, + type: 'chat', model: 'gpt-3.5-turbo', maxTokens: 50 +}; +const cohereCompletion = { + apiKey: process.env.COHERE_API_KEY, provider: SupportedLangModels.COHERE, + type: 'completion', model: 'command', maxTokens: 50 +}; +const geminiChat = { + apiKey: process.env.GEMINI_API_KEY, provider: SupportedChatModels.GEMINI, + type: 'chat', model: 'gemini' +}; +const mistralChat = { + apiKey: process.env.MISTRAL_API_KEY, provider: SupportedChatModels.MISTRAL, + type: 'chat', model: 'mistral-medium', maxTokens: 50 +}; // create the evaluation object const llmEvaluation = new LLMEvaluation(process.env.OPENAI_API_KEY, 'openai'); @@ -20,15 +32,15 @@ const llmEvaluation = new LLMEvaluation(process.env.OPENAI_API_KEY, 'openai'); async function testLLMEvaluation() { const inputString = "Explain the process of photosynthesis in simple terms."; const targetAnswers = ["Photosynthesis is the process where green plants use sunlight to turn carbon dioxide and water into glucose and oxygen. The glucose provides food for the plant, and the oxygen gets released back into the air.", - "Photosynthesis is how plants make their own food. They take in water and carbon dioxide, use the energy from sunlight to transform them into glucose (their food) and oxygen, which they release into the air.", - "In simple terms, photosynthesis is like cooking for plants but instead of a stove, they use sunlight. They mix water and carbon dioxide with the sunlight to create glucose, which is their food, and also produce oxygen."]; - const providerSets = [llamaChat, openaiChat, cohereCompletion, geminiChat]; + "Photosynthesis is how plants make their own food. They take in water and carbon dioxide, use the energy from sunlight to transform them into glucose (their food) and oxygen, which they release into the air.", + "In simple terms, photosynthesis is like cooking for plants but instead of a stove, they use sunlight. They mix water and carbon dioxide with the sunlight to create glucose, which is their food, and also produce oxygen."]; + const providerSets = [llamaChat, openaiChat, cohereCompletion, geminiChat, mistralChat]; const results = await llmEvaluation.compareModels(inputString, targetAnswers, providerSets); console.log('OpenAI Chat and Cohere Completion ModelEvaluation Results:', results); - assert(Object.keys(results).length === providerSets.length+1, 'Test failed'); + assert(Object.keys(results).length === providerSets.length + 1, 'Test failed'); } @@ -37,15 +49,15 @@ async function testLLMEvaluationJson() { const inputString = "Explain the process of photosynthesis in simple terms."; const targetAnswers = ["Photosynthesis is the process where green plants use sunlight to turn carbon dioxide and water into glucose and oxygen. The glucose provides food for the plant, and the oxygen gets released back into the air.", - "Photosynthesis is how plants make their own food. They take in water and carbon dioxide, use the energy from sunlight to transform them into glucose (their food) and oxygen, which they release into the air.", - "In simple terms, photosynthesis is like cooking for plants but instead of a stove, they use sunlight. They mix water and carbon dioxide with the sunlight to create glucose, which is their food, and also produce oxygen."]; + "Photosynthesis is how plants make their own food. They take in water and carbon dioxide, use the energy from sunlight to transform them into glucose (their food) and oxygen, which they release into the air.", + "In simple terms, photosynthesis is like cooking for plants but instead of a stove, they use sunlight. They mix water and carbon dioxide with the sunlight to create glucose, which is their food, and also produce oxygen."]; const providerSets = [llamaChat, openaiChat, cohereCompletion]; const results = await llmEvaluation.compareModels(inputString, targetAnswers, providerSets, true); console.log('Json Results:', results); - + } (async () => { diff --git a/IntelliNode/utils/LLMEvaluation.js b/IntelliNode/utils/LLMEvaluation.js index 1c74fe8..cad274d 100644 --- a/IntelliNode/utils/LLMEvaluation.js +++ b/IntelliNode/utils/LLMEvaluation.js @@ -26,42 +26,42 @@ class LLMEvaluation extends ModelEvaluation { } async generateText(apiKey, inputString, provider, modelName, type, - maxTokens = 500, custom_url = null) { + maxTokens = 500, custom_url = null) { if (type == 'chat' && Object.values(SupportedChatModels).includes(provider.toLowerCase())) { - const customProxy = (custom_url != undefined && custom_url != null && custom_url != '') ? {url: custom_url } : null; - - const chatbot = new Chatbot(apiKey, provider, customProxy); - - // define the chat input - let input; - if (SupportedChatModels.REPLICATE == provider.toLowerCase()) { - input = new LLamaReplicateInput("provide direct answer", { model: modelName, maxTokens: maxTokens}); - } else if (SupportedChatModels.SAGEMAKER == provider.toLowerCase()) { - input = new LLamaSageInput("provide direct answer", {maxTokens: maxTokens}); - } else if (SupportedChatModels.GEMINI == provider.toLowerCase()) { - input = new GeminiInput("provide direct answer", {maxTokens: maxTokens}); - } else if (SupportedChatModels.COHERE == provider.toLowerCase()) { - input = new CohereInput("provide direct answer", {maxTokens: maxTokens}); - } else if (SupportedChatModels.MISTRAL == provider.toLowerCase()) { - input = new MistralInput("provide direct answer", {maxTokens: maxTokens}); - } else { - input = new ChatGPTInput("provide direct answer", { model: modelName, maxTokens: maxTokens}); - } + const customProxy = (custom_url != undefined && custom_url != null && custom_url != '') ? { url: custom_url } : null; + + const chatbot = new Chatbot(apiKey, provider, customProxy); + + // define the chat input + let input; + if (SupportedChatModels.REPLICATE == provider.toLowerCase()) { + input = new LLamaReplicateInput("provide direct answer", { model: modelName, maxTokens: maxTokens }); + } else if (SupportedChatModels.SAGEMAKER == provider.toLowerCase()) { + input = new LLamaSageInput("provide direct answer", { maxTokens: maxTokens }); + } else if (SupportedChatModels.GEMINI == provider.toLowerCase()) { + input = new GeminiInput("provide direct answer", { maxTokens: maxTokens }); + } else if (SupportedChatModels.COHERE == provider.toLowerCase()) { + input = new CohereInput("provide direct answer", { maxTokens: maxTokens }); + } else if (SupportedChatModels.MISTRAL == provider.toLowerCase()) { + input = new MistralInput("provide direct answer", { maxTokens: maxTokens }); + } else { + input = new ChatGPTInput("provide direct answer", { model: modelName, maxTokens: maxTokens }); + } - input.addUserMessage(inputString); - const responses = await chatbot.chat(input); + input.addUserMessage(inputString); + const responses = await chatbot.chat(input); - return responses[0].trim(); + return responses[0].trim(); } else if (type == 'completion' && Object.values(SupportedLangModels).includes(provider.toLowerCase())) { - const languageModel = new RemoteLanguageModel(apiKey, provider); - const langInput = new LanguageModelInput({ prompt: inputString, model: modelName, maxTokens: maxTokens }); - langInput.setDefaultValues(provider, maxTokens); + const languageModel = new RemoteLanguageModel(apiKey, provider); + const langInput = new LanguageModelInput({ prompt: inputString, model: modelName, maxTokens: maxTokens }); + langInput.setDefaultValues(provider, maxTokens); - const responses = await languageModel.generateText(langInput); - return responses[0].trim(); + const responses = await languageModel.generateText(langInput); + return responses[0].trim(); } else { throw new Error('Provider not supported'); } @@ -92,45 +92,53 @@ class LLMEvaluation extends ModelEvaluation { let targetEmbeddings = []; // Initiate Embedding for targets - for(let target of targetAnswers) { + for (let target of targetAnswers) { const embedding = await this.generateEmbedding(target); targetEmbeddings.push(embedding); } - for(let provider of providerSets) { + for (let provider of providerSets) { console.log(`- start ${provider.model} evaluation`) let predictions = []; - let prediction = await this.generateText(provider.apiKey, inputString, provider.provider, - provider.model, provider.type, - provider.maxTokens, provider.url); - const predictionEmbedding = await this.generateEmbedding(prediction); - - let cosineSum = 0, euclideanSum = 0, manhattanSum = 0; - for(let targetEmbedding of targetEmbeddings) { - cosineSum += MatchHelpers.cosineSimilarity(predictionEmbedding, targetEmbedding); - euclideanSum += MatchHelpers.euclideanDistance(predictionEmbedding, targetEmbedding); - manhattanSum += MatchHelpers.manhattanDistance(predictionEmbedding, targetEmbedding); - } - - const avgCosine = cosineSum / targetEmbeddings.length; - const avgEuclidean = euclideanSum / targetEmbeddings.length; - const avgManhattan = manhattanSum / targetEmbeddings.length; + try { + let prediction = await this.generateText(provider.apiKey, inputString, provider.provider, + provider.model, provider.type, + provider.maxTokens, provider.url); + const predictionEmbedding = await this.generateEmbedding(prediction); + + let cosineSum = 0, euclideanSum = 0, manhattanSum = 0; + for (let targetEmbedding of targetEmbeddings) { + cosineSum += MatchHelpers.cosineSimilarity(predictionEmbedding, targetEmbedding); + euclideanSum += MatchHelpers.euclideanDistance(predictionEmbedding, targetEmbedding); + manhattanSum += MatchHelpers.manhattanDistance(predictionEmbedding, targetEmbedding); + } - predictions.push({ - prediction: prediction, - score_cosine_similarity: avgCosine, - score_euclidean_distance: avgEuclidean, - score_manhattan_distance: avgManhattan - }); + const avgCosine = cosineSum / targetEmbeddings.length; + const avgEuclidean = euclideanSum / targetEmbeddings.length; + const avgManhattan = manhattanSum / targetEmbeddings.length; + + predictions.push({ + prediction: prediction, + score_cosine_similarity: avgCosine, + score_euclidean_distance: avgEuclidean, + score_manhattan_distance: avgManhattan, + stop_reason: "complete" + }); + } catch (error) { + console.error(error); + predictions.push({ + stop_reason: "error" + }); + } results[`${provider.provider}/${provider.model}`] = predictions; } results['lookup'] = { - 'cosine_similarity': 'a value closer to 1 indicates a higher degree of similarity between two vectors.', - 'euclidean_distance': 'the lower the value, the closer the two points.', - 'manhattan_distance': 'the lower the value, the closer the two vectors.' + 'cosine_similarity': 'a value closer to 1 indicates a higher degree of similarity between two vectors.', + 'euclidean_distance': 'the lower the value, the closer the two points.', + 'manhattan_distance': 'the lower the value, the closer the two vectors.' } if (isJson) { From 9340babbdc689cae814e8921f5d05bc47accee28 Mon Sep 17 00:00:00 2001 From: cybercoder Date: Sun, 21 Jan 2024 22:15:45 +0000 Subject: [PATCH 6/6] Update package.json update the version --- IntelliNode/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IntelliNode/package.json b/IntelliNode/package.json index 5be2b00..34123f5 100644 --- a/IntelliNode/package.json +++ b/IntelliNode/package.json @@ -1,6 +1,6 @@ { "name": "intellinode", - "version": "1.7.8", + "version": "1.7.9", "description": "Integrate and evaluate various AI models, such as ChatGPT, Llama, Diffusion, Cohere, Gemini and Hugging Face.", "main": "index.js", "keywords": [