[GSK-3846] Add support to LiteLLM

docs/open_source/scan/scan_llm/index.md

@@ -87,7 +87,7 @@ os.environ['OPENAI_API_VERSION'] = '2023-07-01-preview'
 # You'll need to provide the name of the model that you've deployed
 # Beware, the model provided must be capable of using function calls
 giskard.llm.set_llm_model('my-gpt-4-model')
-giskard.llm.embeddings.set_embedding_model('my-embedding-model')
+giskard.llm.set_embedding_model('my-embedding-model')
 ```
 
 ::::::

docs/open_source/setting_up/index.md

@@ -10,193 +10,131 @@ This guide focuses primarily on configuring and using various LLM clients suppor
 
 ## OpenAI GPT-4 Client Setup
 
+More information on [litellm documentation](https://docs.litellm.ai/docs/providers/openai)
+
 ```python
 import os
 import giskard
-from giskard.llm.client.openai import OpenAIClient
-
-# Set the OpenAI API key
-os.environ["OPENAI_API_KEY"] = "sk-…"
 
-# Create a giskard OpenAI client
-openai_client = OpenAIClient(model="gpt-4o")
+os.environ["OPENAI_API_KEY"] = "your-api-key"
 
-# Set the default client
-giskard.llm.set_llm_api("openai")
-giskard.llm.set_default_client(openai_client)
+# Optional, setup a model (default model is gpt-4)
+giskard.llm.set_llm_model("gpt-4")
+giskard.llm.set_embedding_model("text-embedding-ada-002")
 ```
 
 ## Azure OpenAI Client Setup
 
+More information on [litellm documentation](https://docs.litellm.ai/docs/providers/azure)
+
+
 ```python
 import os
 import giskard
 
-# Set the Azure OpenAI API key and endpoint
-os.environ['AZURE_OPENAI_API_KEY'] = '...'
-os.environ['AZURE_OPENAI_ENDPOINT'] = 'https://xxx.openai.azure.com'
-os.environ['OPENAI_API_VERSION'] = '2023-07-01-preview'
+os.environ["AZURE_API_KEY"] = "" # "my-azure-api-key"
+os.environ["AZURE_API_BASE"] = "" # "https://example-endpoint.openai.azure.com"
+os.environ["AZURE_API_VERSION"] = "" # "2023-05-15"
+giskard.llm.set_llm_model("azure/<your_deployment_name>")
+giskard.llm.set_embedding_model("azure/<your_deployment_name>")
 
-# You'll need to provide the name of the model that you've deployed
-# Beware, the model provided must be capable of using function calls
-giskard.llm.set_llm_model('my-gpt-4-model')
-giskard.llm.embeddings.set_embedding_model('my-embedding-model')
+# optional
+os.environ["AZURE_AD_TOKEN"] = ""
+os.environ["AZURE_API_TYPE"] = ""
 ```
 
 ## Mistral Client Setup
 
+More information on [litellm documentation](https://docs.litellm.ai/docs/providers/mistral)
+
 ```python
 import os
 import giskard
-from giskard.llm.client.mistral import MistralClient
-
-# Set the Mistral API key
-os.environ["MISTRAL_API_KEY"] = "…"
 
-# Create a giskard Mistral client
-mistral_client = MistralClient()
+os.environ['MISTRAL_API_KEY'] = ""
+giskard.llm.set_llm_model("mistral/mistral-tiny")
+giskard.llm.set_embedding_model("mistral/mistral-embed")
 
-# Set the default client
-giskard.llm.set_default_client(mistral_client)
-
-# You may also want to set the default embedding model
-# Check the Custom Client Setup section for more details
 ```
 
 ## Ollama Client Setup
 
-The Ollama setup involves configuring an OpenAI client customized for the Ollama API:
+More information on [litellm documentation](https://docs.litellm.ai/docs/providers/ollama)
 
 ```python
+import litellm
 import giskard
-from openai import OpenAI
-from giskard.llm.client.openai import OpenAIClient
-from giskard.llm.embeddings.openai import OpenAIEmbedding
-
-# Setup the OpenAI client with API key and base URL for Ollama
-_client = OpenAI(base_url="http://localhost:11434/v1/", api_key="ollama")
 
-# Wrap the original OpenAI client with giskard OpenAI client and embedding
-llm_client = OpenAIClient(model="llama3.2", client=_client)
-embed_client = OpenAIEmbedding(model="nomic-embed-text", client=_client)
+giskard.llm.set_llm_model("ollama/llama2") # See supported models here: https://docs.litellm.ai/docs/providers/ollama#ollama-models
+litellm.api_base = "http://localhost:11434"
 
-# Set the default client and embedding
-giskard.llm.set_default_client(llm_client)
-giskard.llm.embeddings.set_default_embedding(embed_client)
+# TODO: embedding
 ```
 
-## Claude 3 Client Setup
+## AWS Bedrock Client Setup
 
-The Claude 3 setup involves configuring a Bedrock client:
+More information on [litellm documentation](https://docs.litellm.ai/docs/providers/bedrock)
 
 ```python
 import os
-import boto3
 import giskard
 
-from giskard.llm.client.bedrock import ClaudeBedrockClient
-from giskard.llm.embeddings.bedrock import BedrockEmbedding
-
-# Create a Bedrock client
-bedrock_runtime = boto3.client("bedrock-runtime", region_name=os.environ["AWS_DEFAULT_REGION"])
-
-# Wrap the Beddock client with giskard Bedrock client and embedding
-claude_client = ClaudeBedrockClient(bedrock_runtime, model="anthropic.claude-3-haiku-20240307-v1:0")
-embed_client = BedrockEmbedding(bedrock_runtime, model="amazon.titan-embed-text-v1")
-
-# Set the default client and embedding
-giskard.llm.set_default_client(claude_client)
-giskard.llm.embeddings.set_default_embedding(embed_client)
+os.environ["AWS_ACCESS_KEY_ID"] = ""
+os.environ["AWS_SECRET_ACCESS_KEY"] = ""
+os.environ["AWS_REGION_NAME"] = ""
+giskard.llm.set_llm_model("bedrock/anthropic.claude-3-sonnet-20240229-v1:0")
+giskard.llm.set_embedding_model("bedrock/amazon.titan-embed-text-v1")
 ```
 
 ## Gemini Client Setup
 
+More information on [litellm documentation](https://docs.litellm.ai/docs/providers/gemini)
+
 ```python
 import os
 import giskard
-import google.generativeai as genai
-from giskard.llm.client.gemini import GeminiClient
-
-# Set the Gemini API key
-os.environ["GEMINI_API_KEY"] = "…"
-
-# Configure the Gemini API
-genai.configure(api_key=os.environ["GEMINI_API_KEY"])
 
-# Create a giskard Gemini client
-gemini_client = GeminiClient()
+os.environ["GEMINI_API_KEY"] = "your-api-key"
 
-# Set the default client
-giskard.llm.set_default_client(gemini_client)
-
-# You may also want to set the default embedding model
-# Check the Custom Client Setup section for more details
+giskard.llm.set_llm_model("gemini/gemini-pro")
+# TODO: embedding
 ```
 
 ## Custom Client Setup
 
+More information on [litellm documentation](https://docs.litellm.ai/docs/providers/custom_llm_server    )
+
 ```python
 import giskard
-from typing import Sequence, Optional
-from giskard.llm.client import set_default_client
-from giskard.llm.client.base import LLMClient, ChatMessage
-
-# Create a custom client by extending the LLMClient class
-class MyLLMClient(LLMClient):
-    def __init__(self, my_client):
-        self._client = my_client
-
-    def complete(
-            self,
-            messages: Sequence[ChatMessage],
-            temperature: float = 1,
-            max_tokens: Optional[int] = None,
-            caller_id: Optional[str] = None,
-            seed: Optional[int] = None,
-            format=None,
-    ) -> ChatMessage:
-        # Create the prompt
-        prompt = ""
-        for msg in messages:
-            if msg.role.lower() == "assistant":
-                prefix = "\n\nAssistant: "
-            else:
-                prefix = "\n\nHuman: "
-
-            prompt += prefix + msg.content
-
-        prompt += "\n\nAssistant: "
-
-        # Create the body
-        params = {
-            "prompt": prompt,
-            "max_tokens_to_sample": max_tokens or 1000,
-            "temperature": temperature,
-            "top_p": 0.9,
-        }
-        body = json.dumps(params)
-
-        response = self._client.invoke_model(
-            body=body,
-            modelId=self._model_id,
-            accept="application/json",
-            contentType="application/json",
+import litellm
+from litellm import CustomLLM, completion, get_llm_provider
+
+
+class MyCustomLLM(CustomLLM):
+    def completion(self, *args, **kwargs) -> litellm.ModelResponse:
+        return litellm.completion(
+            model="gpt-3.5-turbo",
+            messages=[{"role": "user", "content": "Hello world"}],
+            mock_response="Hi!",
         )
-        data = json.loads(response.get("body").read())
 
-        return ChatMessage(role="assistant", message=data["completion"])
+    def embedding(self, *args, **kwargs) -> litellm.EmbeddingResponse:
+        return litellm.embedding(
+            model="openai/text-embedding-ada-002",
+            input=["Hello world"]
+
+        )
+
+my_custom_llm = MyCustomLLM()
+
+litellm.custom_provider_map = [ # 👈 KEY STEP - REGISTER HANDLER
+    {"provider": "my-custom-llm", "custom_handler": my_custom_llm}
+]
 
-# Create an instance of the custom client
-llm_client = MyLLMClient()
+giskard.llm.set_llm_model("my-custom-llm/my-fake-llm-model")
+giskard.llm.set_embedding_model("my-custom-llm/my-fake-embedding-model")
 
-# Set the default client
-set_default_client(llm_client)
 
-# It's also possible to create a custom embedding class extending BaseEmbedding
-# Or you can use FastEmbed for a pre-built embedding model:
-from giskard.llm.embeddings.fastembed import try_get_fastembed_embeddings
-embed_client = try_get_fastembed_embeddings()
-giskard.llm.embeddings.set_default_embedding(embed_client)
 ```
 
 If you run into any issues configuring the LLM client, don't hesitate to [ask us on Discord](https://discord.com/invite/ABvfpbu69R) or open a new issue on [our GitHub repo](https://github.com/Giskard-AI/giskard).

docs/open_source/testset_generation/testset_generation/index.md

@@ -121,7 +121,7 @@ os.environ['OPENAI_API_VERSION'] = '2023-07-01-preview'
 # You'll need to provide the name of the model that you've deployed
 # Beware, the model provided must be capable of using function calls
 giskard.llm.set_llm_model('my-gpt-4-model')
-giskard.llm.embeddings.set_embedding_model('my-embedding-model')
+giskard.llm.set_embedding_model('my-embedding-model')
 ```
 
 ::::::

giskard/llm/__init__.py

@@ -1,4 +1,5 @@
 from .client import get_default_client, set_default_client, set_llm_api, set_llm_model
+from .embeddings import set_default_embedding, set_embedding_model
 from .errors import LLMImportError
 
 __all__ = [
@@ -7,4 +8,6 @@
     "set_default_client",
     "set_llm_api",
     "set_llm_model",
+    "set_default_embedding",
+    "set_embedding_model",
 ]

giskard/llm/client/__init__.py

@@ -1,4 +1,5 @@
 from typing import Optional
+from typing_extensions import deprecated
 
 import logging
 import os
@@ -12,16 +13,25 @@
 _default_llm_base_url = os.getenv("GSK_LLM_BASE_URL", None)
 
 
+@deprecated(
+    "set_default_client is deprecated, check documentation to setup llm: https://docs.giskard.ai/en/latest/open_source/setting_up/index.html"
+)
 def set_default_client(client: LLMClient):
     global _default_client
     _default_client = client
 
 
+@deprecated(
+    "_unset_default_client is deprecated, check documentation to setup llm: https://docs.giskard.ai/en/latest/open_source/setting_up/index.html"
+)
 def _unset_default_client():
     global _default_client
     _default_client = None
 
 
+@deprecated(
+    "set_llm_api is deprecated, check documentation to setup llm: https://docs.giskard.ai/en/latest/open_source/setting_up/index.html"
+)
 def set_llm_api(llm_api: str):
     if llm_api.lower() not in {"azure", "openai"}:
         raise ValueError("Giskard LLM-based evaluators is only working with `azure` and `openai`")
@@ -32,6 +42,9 @@ def set_llm_api(llm_api: str):
     _unset_default_client()
 
 
+@deprecated(
+    "set_llm_base_url is deprecated, check documentation to setup llm: https://docs.giskard.ai/en/latest/open_source/setting_up/index.html"
+)
 def set_llm_base_url(llm_base_url: Optional[str]):
     global _default_llm_base_url
     _default_llm_base_url = llm_base_url
@@ -68,20 +81,12 @@ def get_default_client() -> LLMClient:
     if _default_client is not None:
         return _default_client
 
-    # Setup the default client
-    from .openai import OpenAIClient
-
-    default_llm_api = get_default_llm_api()
-
     try:
-        # For openai>=1.0.0
-        from openai import AzureOpenAI, OpenAI
-
-        client = AzureOpenAI() if default_llm_api == "azure" else OpenAI(base_url=_default_llm_base_url)
+        from .litellm import LiteLLMClient
 
-        _default_client = OpenAIClient(model=_default_llm_model, client=client)
+        _default_client = LiteLLMClient(_default_llm_model)
     except ImportError:
-        raise ValueError(f"LLM scan using {default_llm_api.name} require openai>=1.0.0")
+        raise ValueError(f"LLM scan using {_default_llm_model} requires litellm")
 
     return _default_client
 

giskard/llm/client/bedrock.py

@@ -1,4 +1,5 @@
 from typing import Dict, List, Optional, Sequence
+from typing_extensions import deprecated
 
 import json
 from abc import ABC, abstractmethod
@@ -16,6 +17,9 @@
     ) from err
 
 
+@deprecated(
+    "BaseBedrockClient is deprecated, check documentation to setup llm: https://docs.giskard.ai/en/latest/open_source/setting_up/index.html"
+)
 class BaseBedrockClient(LLMClient, ABC):
     def __init__(self, bedrock_runtime_client, model: str):
         self._client = bedrock_runtime_client

giskard/llm/client/copilot.py

@@ -1,4 +1,5 @@
 from typing import Optional, Sequence
+from typing_extensions import deprecated
 
 from dataclasses import dataclass
 from logging import warning
@@ -42,6 +43,9 @@ def _format_message(msg: ChatMessage) -> dict:
     return fmt_msg
 
 
+@deprecated(
+    "GiskardCopilotClient is deprecated, check documentation to setup llm: https://docs.giskard.ai/en/latest/open_source/setting_up/index.html"
+)
 class GiskardCopilotClient(OpenAIClient):
     def complete(
         self,