diff --git a/.github/workflows/upload-pypi-dev.yml b/.github/workflows/upload-pypi-dev.yml index 8304eb18..dcc4d365 100644 --- a/.github/workflows/upload-pypi-dev.yml +++ b/.github/workflows/upload-pypi-dev.yml @@ -1,4 +1,4 @@ -name: Upload Python package to PyPI as dev pre-release +name: Upload Python package to PyPI as dev release and build/push Docker image. on: workflow_dispatch: @@ -39,3 +39,48 @@ jobs: git add pyproject.toml git commit -m "[fix] bump prerelease version in pyproject.toml" git push + + # Wait for PyPI to update + - name: Wait for PyPI to update + run: | + VERSION=$(poetry version --short) + echo "Checking for llmstudio==$VERSION on PyPI..." + for i in {1..10}; do + if python -m pip install llmstudio==${VERSION} --dry-run >/dev/null 2>&1; then + echo "Package llmstudio==${VERSION} is available on PyPI." + break + else + echo "Package llmstudio==${VERSION} not available yet. Waiting 15 seconds..." + sleep 15 + fi + if [ $i -eq 10 ]; then + echo "Package did not become available in time." + exit 1 + fi + done + + # Docker build and push section + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Log in to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Extract version for tagging Docker image + id: get_version + run: | + echo "VERSION=$(poetry version --short)" >> $GITHUB_ENV + + - name: Build and tag Docker image + run: | + docker build \ + --build-arg LLMSTUDIO_VERSION=${{ env.VERSION }} \ + -t tensoropsai/llmstudio:${{ env.VERSION }} \ + . + + - name: Push Docker image to Docker Hub + run: | + docker push tensoropsai/llmstudio:${{ env.VERSION }} \ No newline at end of file diff --git a/.github/workflows/upload-pypi.yml b/.github/workflows/upload-pypi.yml index ec06e30b..15952239 100644 --- a/.github/workflows/upload-pypi.yml +++ b/.github/workflows/upload-pypi.yml @@ -1,4 +1,4 @@ -name: Upload Python package to PyPI +name: Upload Python package to PyPI and build/push Docker images. on: push: @@ -11,23 +11,77 @@ jobs: deploy: runs-on: ubuntu-latest steps: + # Checkout the code - name: Checkout code uses: actions/checkout@v2 + # Set up Python environment - name: Set up Python uses: actions/setup-python@v2 with: python-version: "3.x" + # Install Poetry - name: Install Poetry run: | curl -sSL https://install.python-poetry.org | python3 - + # Configure Poetry with PyPI token - name: Configure Poetry run: | poetry config pypi-token.pypi ${{ secrets.PYPI_API_TOKEN }} + # Build and publish package to PyPI - name: Build and publish to PyPI run: | poetry build poetry publish + + # Extract the new version number from pyproject.toml + - name: Extract version for tagging Docker image + run: | + echo "VERSION=$(poetry version --short)" >> $GITHUB_ENV + + # Wait for the package to become available on PyPI + - name: Wait for PyPI to update + run: | + echo "Checking for llmstudio==${{ env.VERSION }} on PyPI..." + for i in {1..10}; do + if python -m pip install llmstudio==${{ env.VERSION }} --dry-run >/dev/null 2>&1; then + echo "Package llmstudio==${{ env.VERSION }} is available on PyPI." + break + else + echo "Package llmstudio==${{ env.VERSION }} not available yet. Waiting 15 seconds..." + sleep 15 + fi + if [ $i -eq 10 ]; then + echo "Package did not become available in time." + exit 1 + fi + done + + # Set up Docker Buildx + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + # Log in to Docker Hub + - name: Log in to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + # Build and tag Docker images with both :latest and :[NEW_VERSION] + - name: Build and tag Docker images + run: | + docker build \ + --build-arg LLMSTUDIO_VERSION=${{ env.VERSION }} \ + -t tensoropsai/llmstudio:latest \ + -t tensoropsai/llmstudio:${{ env.VERSION }} \ + . + + # Push both Docker images to Docker Hub + - name: Push Docker images to Docker Hub + run: | + docker push tensoropsai/llmstudio:${{ env.VERSION }} + docker push tensoropsai/llmstudio:latest \ No newline at end of file diff --git a/docs/how-to/build-a-tool-agent.mdx b/docs/how-to/build-a-tool-agent.mdx new file mode 100644 index 00000000..d3a5e741 --- /dev/null +++ b/docs/how-to/build-a-tool-agent.mdx @@ -0,0 +1,80 @@ +This guide outlines how to build a tool calling agent using Langchain + LLMstudio. + +## 1. Set up your tools +Start by defining the tools your agent is going to have access to. +```python +from langchain.tools import tool + +@tool +def buy_ticket(destination: str): + """Use this to buy a ticket""" + return "Bought ticket number 270924" + +@tool +def get_departure(ticket_number: str): + """Use this to fetch the departure time of a train""" + return "8:25 AM" +``` + +## 2. Setup your .env +Create a `.env` file on the root of your project with the the credentials for the providers you want to use. + + + + ``` + OPENAI_API_KEY="YOUR_API_KEY" + ``` + + + ``` + GOOGLE_API_KEY="YOUR_API_KEY" + ``` + + + ``` + AZURE_BASE_URL="YOUR_MODEL_ENDPOINT" + AZURE_API_KEY="YOUR_API_KEY" + ``` + + + +## 3. Set up your model using LLMstudio +Use LLMstudio to choose the provider and model you want to use. + + + ```python + model = ChatLLMstudio(model_id='openai/gpt-4o') + ``` + + + ```python + model = ChatLLMstudio(model_id='vertexai/gemini-1.5-flash') + ``` + + + ```python + model = ChatLLMstudio(model_id='azure/Meta-Llama-3.1-70B-Instruct') + ``` + + + +## 4. Build the agent +Set up your agent and agent executor using Langchain. + +```python +from langchain import hub +from langchain.agents import AgentExecutor, create_openai_tools_agent + +prompt = hub.pull("hwchase17/openai-tools-agent") +agent = create_openai_tools_agent(model, tools, prompt) +agent_executor = AgentExecutor(agent=agent, tools=tools) + +input = "Can you buy me a ticket to madrid?" + +# Using with chat history +agent_executor.invoke( + { + "input": input, + } +) +``` \ No newline at end of file diff --git a/docs/how-to/deploy-on-gke/deploy-on-google-kubernetes-engine.mdx b/docs/how-to/deploy-on-gke/deploy-on-google-kubernetes-engine.mdx new file mode 100644 index 00000000..cad6cb04 --- /dev/null +++ b/docs/how-to/deploy-on-gke/deploy-on-google-kubernetes-engine.mdx @@ -0,0 +1,156 @@ +Learn how to deploy LLMstudio as a containerized application on Google Kubernetes Engine and make calls from a local repository. + + +## Prerequisites +To follow this guide you need to have the following set-up: + +- A **project** on google cloud platform. +- **Kubernetes Engine** API enabled on your project. +- **Kubernetes Engine Admin** role for the user performing the guide. + +## Deploy LLMstudio + +This example demonstrates a public deployment. For a private service accessible only within your enterprise infrastructure, deploy it within your own Virtual Private Cloud (VPC). + + + Begin by navigating to the Kubernetes Engine page. + + + Go to **Workloads** and **Create a new Deployment**. + + + + + + Rename your project. We will call the one in this guide **llmstudio-on-gcp**. + + + + + + Choose between **creating a new cluster** or **using an existing cluster**. + For this guide, we will create a new cluster and use the default region. + + + + + + Once done done with the **Deployment configuration**, proceed to **Container details**. + + + In the new container section, select **Existing container image**. + + + Copy the path to LLMstudio's image available on Docker Hub. + ```bash Image Path + tensoropsai/llmstudio:latest + ``` + Set it as the **Image path** to your container. + + + + + + Configure the following mandatory environment variables: +| Environment Variable | Value | +|----------------------------|-----------| +| `LLMSTUDIO_ENGINE_HOST` | 0.0.0.0 | +| `LLMSTUDIO_ENGINE_PORT` | 8001 | +| `LLMSTUDIO_TRACKING_HOST` | 0.0.0.0 | +| `LLMSTUDIO_TRACKING_PORT` | 8002 | + +Additionally, set the `GOOGLE_API_KEY` environment variable to enable calls to Google's Gemini models. +Refer to **SDK/LLM/Providers** for instructions on setting up other providers. + + + + + + + + After configuring your container, proceed to **Expose (Optional)**. + + + Select **Expose deployment as a new service** and leave the first item as is. + + + + + + Add two other items, and expose the ports defined in the **Set Environment Variables** step. + + + + + + + After setting up and exposing the ports, press **Deploy**. + You have successfully deployed **LLMstudio on Google Cloud Platform**! + + + + +## Make a Call +Now let's make a call to our LLMstudio instance on GCP! + + + + + + Setup a simple project with this two files: + 1. `calls.ipynb` + 2. `.env` + + + + + + + Go to your newly deployed **Workload**, scroll to the **Exposing services** section, and take note of the Host of your endpoint. + + + + + Create your `.env` file with the following: + + ```env .env + LLMSTUDIO_ENGINE_HOST = "YOUR_HOST" + LLMSTUDIO_ENGINE_PORT = "8001" + LLMSTUDIO_TRACKING_HOST = "YOUR_HOST" + LLMSTUDIO_TRACKING_PORT = "8002" + ``` + + You are done seting up you **.env** file! + + + + Start by importing llmstudio: + ```python 1st cell + from llmstudio import LLM + ``` + + Set up your LLM. We will be using `gemini-1.5-flash` for this guide. + ```python 2nd cell + llm = LLM('vertexai/gemini-1.5-flash') + ``` + + Chat with your model. + ```python 3rd cell + llm.chat('Hello!') + print(response.chat_output) + ``` + + + + + + + You are done calling llmstudio on GCP! + + + + + + + + \ No newline at end of file diff --git a/docs/how-to/deploy-on-gke/images/step-2.png b/docs/how-to/deploy-on-gke/images/step-2.png new file mode 100644 index 00000000..4d2bb3ee Binary files /dev/null and b/docs/how-to/deploy-on-gke/images/step-2.png differ diff --git a/docs/how-to/deploy-on-gke/images/step-3.png b/docs/how-to/deploy-on-gke/images/step-3.png new file mode 100644 index 00000000..adb4eac3 Binary files /dev/null and b/docs/how-to/deploy-on-gke/images/step-3.png differ diff --git a/docs/how-to/deploy-on-gke/images/step-4.png b/docs/how-to/deploy-on-gke/images/step-4.png new file mode 100644 index 00000000..cc18c845 Binary files /dev/null and b/docs/how-to/deploy-on-gke/images/step-4.png differ diff --git a/docs/how-to/deploy-on-gke/images/step-6.png b/docs/how-to/deploy-on-gke/images/step-6.png new file mode 100644 index 00000000..6ac8b8b1 Binary files /dev/null and b/docs/how-to/deploy-on-gke/images/step-6.png differ diff --git a/docs/how-to/deploy-on-gke/images/step-7-1.png b/docs/how-to/deploy-on-gke/images/step-7-1.png new file mode 100644 index 00000000..e3e523a3 Binary files /dev/null and b/docs/how-to/deploy-on-gke/images/step-7-1.png differ diff --git a/docs/how-to/deploy-on-gke/images/step-7-2.png b/docs/how-to/deploy-on-gke/images/step-7-2.png new file mode 100644 index 00000000..b8325854 Binary files /dev/null and b/docs/how-to/deploy-on-gke/images/step-7-2.png differ diff --git a/docs/how-to/deploy-on-gke/images/step-9-1.png b/docs/how-to/deploy-on-gke/images/step-9-1.png new file mode 100644 index 00000000..328ca889 Binary files /dev/null and b/docs/how-to/deploy-on-gke/images/step-9-1.png differ diff --git a/docs/how-to/deploy-on-gke/images/step-9-2.png b/docs/how-to/deploy-on-gke/images/step-9-2.png new file mode 100644 index 00000000..64d9ddb6 Binary files /dev/null and b/docs/how-to/deploy-on-gke/images/step-9-2.png differ diff --git a/docs/how-to/deploy-on-gke/images/step-env.png b/docs/how-to/deploy-on-gke/images/step-env.png new file mode 100644 index 00000000..392a49ad Binary files /dev/null and b/docs/how-to/deploy-on-gke/images/step-env.png differ diff --git a/docs/how-to/deploy-on-gke/images/step-llmstudio-call.png b/docs/how-to/deploy-on-gke/images/step-llmstudio-call.png new file mode 100644 index 00000000..e05609fb Binary files /dev/null and b/docs/how-to/deploy-on-gke/images/step-llmstudio-call.png differ diff --git a/docs/llm/anthropic.mdx b/docs/llm/anthropic.mdx new file mode 100644 index 00000000..2ef11bfa --- /dev/null +++ b/docs/llm/anthropic.mdx @@ -0,0 +1,101 @@ +Interact with your Anthropic models using LLMstudios LLM. + +## Supported models +1. `claude-3-opus-20240229` +2. `claude-3-sonnet-2024022` +3. `claude-3-haiku-20240307` +4. `claude-2.1` +5. `claude-2` +6. `claude-instant-1.2` + +## Parameters +An Anthropic LLM interface can have the following parameters: +| Parameter | Type | Description | +|-------------------|--------|-----------------------------------------------------------------------------| +| `api_key` | str | The API key for authentication. | +| `temperature` | float | The temperature parameter for the model. | +| `top_p` | float | The top-p parameter for the model. | +| `max_tokens` | int | The maximum number of tokens for the model's output. | +| `top_k` | int | The top-k parameter for the model. | + + +## Usage +Here is how you setup an interface to interact with your Anthropic models. + + + + + + Create a `.env` file with you `ANTHROPIC_API_KEY` + + Make sure you call your environment variable ANTHROPIC_API_KEY + ```bash + ANTHROPIC_API_KEY="YOUR-KEY" + ``` + + + In your python code, import LLM from llmstudio. + ```python + from llmstudio import LLM + ``` + + + Create your **llm** instance. + ```python + llm = LLM('anthropic/{model}') + ``` + + + **Optional:** You can add your parameters as follows: + ```python + llm = LLM('anthropic/model', + temperature= ..., + max_tokens= ..., + top_p= ..., + frequency_penalty= ..., + presence_penalty= ...) + ``` + You are done setting up your **Anthropic LLM**! + + + + + + + In your python code, import LLM from llmstudio. + ```python + from llmstudio import LLM + ``` + + + Create your **llm** instance. + ```python + llm = LLM('anthropic/{model}',api_key="YOUR_API_KEY") + ``` + + + **Optional:** You can add your parameters as follows: + ```python + llm = LLM('anthropic/model', + temperature= ..., + max_tokens= ..., + top_p= ..., + frequency_penalty= ..., + presence_penalty= ...) + ``` + You are done setting up your **Anthropic LLM**! + + + + + + +## What's next? + + + Learn how to send messeges and recieve responses next! + + + Learn how to build a tool calling agent using llmstudio. + + \ No newline at end of file diff --git a/docs/llm/azure.mdx b/docs/llm/azure.mdx new file mode 100644 index 00000000..1942ffac --- /dev/null +++ b/docs/llm/azure.mdx @@ -0,0 +1,126 @@ +Interact with your Azure models using LLM. + +## Parameters +An Azure LLM interface can have the following parameters: +| Parameter | Type | Description | +|---------------------|--------|-----------------------------------------------------------------------------| +| `temperature` | float | The temperature parameter for the model. | +| `max_tokens` | int | The maximum number of tokens to generate. | +| `top_p` | float | The top-p parameter for the model. | +| `frequency_penalty` | float | The frequency penalty parameter for the model. | +| `presence_penalty` | float | The presence penalty parameter for the model. | + + +## Usage +Here is how you setup an interface to interact with your Azure models. + + + + + Create a `config.yaml` file in the same directory as your code. + 1. 📁 src + 1. 🐍 PythonCode.py + 2. 🐍 PyNotebook.ipynb + 3. 📄 **config.yaml** + + + Define your Azure OpenAI provider and models inside the `config.yaml` file. + ```yaml + providers: + azure: + id: azure + name: Azure + chat: true + embed: true + models: + YOUR_MODEL: <- Replace with your model name + mode: chat + max_tokens: ... + input_token_cost: ... + output_token_cost: ... + ``` + If you are not sure, you can leave `max_tokens`, `input_tokens` and the other parameters as **0** + + + Create your **llm** instance. + ```python + llm = LLM('azure/YOUR_MODEL', + api_key = YOUR_API_KEY, + api_endpoint = YOUR_ENDPOINT, + api_version = YOUR_API_VERSION) + ``` + + + **Optional:** You can add your parameters as follows: + ```python + llm = LLM('azure/model', + temperature= ..., + max_tokens= ..., + top_p= ..., + frequency_penalty= ..., + presence_penalty= ...) + ``` + You are done setting up your **Azure LLM**! + + + + + + + Create a `config.yaml` file in the same directory as your code. + 1. 📁 src + 1. 🐍 PythonCode.py + 2. 🐍 PyNotebook.ipynb + 3. 📄 **config.yaml** + + + Define your Azure provider and models inside the `config.yaml` file. + ```yaml + providers: + azure: + id: azure + name: Azure + chat: true + embed: true + models: + YOUR_MODEL: <- Replace with your model name + mode: chat + max_tokens: ... + input_token_cost: ... + output_token_cost: ... + ``` + If you are not sure, you can leave `max_tokens`, `input_tokens` and the other parameters as **0** + + + Create your **llm** instance. + ```python + llm = LLM('azure/YOUR_MODEL', + api_key = YOUR_API_KEY, + base_url = YOUR_ENDPOINT) + ``` + + + **Optional:** You can add your parameters as follows: + ```python + llm = LLM('azure/model', + temperature= ..., + max_tokens= ..., + top_p= ..., + frequency_penalty= ..., + presence_penalty= ...) + ``` + You are done setting up your **Azure LLM**! + + + + + +## What's next? + + + Learn how to send messeges and recieve responses next! + + + Learn how to build a tool calling agent using llmstudio. + + \ No newline at end of file diff --git a/docs/llm/chat.mdx b/docs/llm/chat.mdx new file mode 100644 index 00000000..cc8852f4 --- /dev/null +++ b/docs/llm/chat.mdx @@ -0,0 +1,86 @@ +Make chat calls using your LLM. + +## Parameters +The llm.chat method can have the following parameters. +| Parameter | Type | Description | +|-------------------|--------|-----------------------------------------------------------------------------| +| `input ` | str | The input message to send to the chat model. | +| `is_stream` | bool | The temperature parameter for the model. | +| `**kwargs` | dict | Additional parameters to pass to the chat model. | + +Refer to your provider-specific documentation for additional kwargs you can use. + +## Returns +| Output | Type | Description | +|-------------------|--------|-----------------------------------------------------------------------------| +| `ChatCompletion` | object | A chat completion object in the OpenAI format + metrics computed by LLMstudio.| + + +## Usage +Here's how to use `.chat()` to make calls to your LLM. + + + + Start by importing LLM. + ```python + from llmstudio import LLM + ``` + + + Set up an LLM from your desired provider. + ```python + llm = LLM('openai/gpt-4o') + ``` + + + Create your message. Your message can be a simple `string` or a message in the `OpenAI format`. + + + + ```python + message = "Hello, how are you today?" + ``` + + + ```python + message = [ + {"role": "system", "content": "You are a helpfull assistant."}, + {"role": "user", "content": "Hello, how are you today?"} + ] + ``` + + + + + + + + + Get your response. + ```python + response = llm.chat(message) + ``` + + Vizualize your response. + ```python + print(response) + ``` + + + Get your response. + ```python + response = llm.chat(message, is_stream = True) + ``` + + Vizualize your response. + ```python + for chunk in response: + print(chunk) + ``` + + + + You are done chating with your **LLMstudio LLM**! + + + \ No newline at end of file diff --git a/docs/llm/ollama.mdx b/docs/llm/ollama.mdx new file mode 100644 index 00000000..ea4b1fea --- /dev/null +++ b/docs/llm/ollama.mdx @@ -0,0 +1,70 @@ +Interact with your Ollama models using LLM. + +## Parameters +An Ollama LLM interface can have the following parameters: +| Parameter | Type | Description | +|-------------------|--------|-----------------------------------------------------------------------------| +| `temperature` | float | The temperature parameter for the model. | +| `top_p` | float | The top-p parameter for the model. | +| `num_predict` | int | The number of tokens to predict. | +| `top_k` | int | The top-k parameter for the model. | + + +## Usage +Here is how you setup an interface to interact with your Ollama models. + + + + Create a `config.yaml` in the same directory your code is in. + 1. src + 1. yourPythonCode.py + 2. yourPyNotebook.py + 3. **config.yaml** + + + Define your Ollama provider and models inside the `config.yaml` file. + ```yaml + providers: + ollama: + id: ollama + name: Ollama + chat: true + embed: true + keys: + models: + YOUR_MODEL: <- Replace with your model name + mode: chat + max_tokens: ... + input_token_cost: ... + output_token_cost: ... + ``` + If you are not sure about any of these parameters, you can just leave them as **0** + + + Create your **llm** instance. + ```python + llm = LLM('ollama/{YOUR_MODEL}') + ``` + + + **Optional:** You can add your parameters as follows: + ```python + llm = LLM('ollama/model', + temperature= ..., + num_predict= ..., + top_p= ..., + top_k= ...,) + ``` + You are done setting up your **Ollama LLM**! + + + +## What's next? + + + Learn how to send messeges and recieve responses next! + + + Learn how to build a tool calling agent using llmstudio. + + \ No newline at end of file diff --git a/docs/llm/openai.mdx b/docs/llm/openai.mdx new file mode 100644 index 00000000..60a21bd3 --- /dev/null +++ b/docs/llm/openai.mdx @@ -0,0 +1,100 @@ +Interact with your OpenAI models using LLM. + +## Supported models +1. `gpt-4o` +2. `gpt-4-turbo` +3. `gpt-4` +4. `gpt-3.5-turbo` +5. `gpt-3.5-turbo-instruct` + +## Parameters +An OpenAI LLM interface can have the following parameters: +| Parameter | Type | Description | +|-------------------|--------|-----------------------------------------------------------------------------| +| `api_key` | str | The API key for authentication. | +| `temperature` | float | The temperature parameter for the model. | +| `top_p` | float | The top-p parameter for the model. | +| `max_tokens` | int | The maximum number of tokens for the model's output. | +| `frequency_penalty` | float | The frequency penalty parameter for the model. | +| `presence_penalty` | float | The presence penalty parameter for the model. | + + +## Usage +Here is how you setup an interface to interact with your OpenAI models. + + + + + + Create a `.env` file with you `OPENAI_API_KEY` + + Make sure you call your environment variable OPENAI_API_KEY + ```bash + OPENAI_API_KEY="YOUR-KEY" + ``` + + + In your python code, import LLM from llmstudio. + ```python + from llmstudio import LLM + ``` + + + Create your **llm** instance. + ```python + llm = LLM('openai/{model}') + ``` + + + **Optional:** You can add your parameters as follows: + ```python + llm = LLM('openai/model', + temperature= ..., + max_tokens= ..., + top_p= ..., + frequency_penalty= ..., + presence_penalty= ...) + ``` + You are done setting up your **OpenAI LLM**! + + + + + + + In your python code, import LLM from llmstudio. + ```python + from llmstudio import LLM + ``` + + + Create your **llm** instance. + ```python + llm = LLM('openai/{model}',api_key="YOUR_API_KEY") + ``` + + + **Optional:** You can add your parameters as follows: + ```python + llm = LLM('openai/model', + temperature= ..., + max_tokens= ..., + top_p= ..., + frequency_penalty= ..., + presence_penalty= ...) + ``` + You are done setting up your **OpenAI LLM**! + + + + + +## What's next? + + + Learn how to send messeges and recieve responses next! + + + Learn how to build a tool calling agent using llmstudio. + + \ No newline at end of file diff --git a/docs/llm/vertexai.mdx b/docs/llm/vertexai.mdx new file mode 100644 index 00000000..1f5a1c81 --- /dev/null +++ b/docs/llm/vertexai.mdx @@ -0,0 +1,98 @@ +Interact with your VertexAI models using LLM. + +## Supported models +1. `gemini-1.5-flash` +2. `gemini-1.5-pro` +3. `gemini-1.0-pro` + +## Parameters +A VertexAI LLM interface can have the following parameters: +| Parameter | Type | Description | +|-------------------|--------|-----------------------------------------------------------------------------| +| `api_key` | str | The API key for authentication. | +| `temperature` | float | The temperature parameter for the model. | +| `top_p` | float | The top-p parameter for the model. | +| `max_tokens` | int | The maximum number of tokens for the model's output. | +| `frequency_penalty` | float | The frequency penalty parameter for the model. | +| `presence_penalty` | float | The presence penalty parameter for the model. | + + +## Usage +Here is how you setup an interface to interact with your VertexAI models. + + + + + + Create a `.env` file with you `GOOGLE_API_KEY` + + Make sure you call your environment variable GOOGLE_API_KEY + ```bash + GOOGLE_API_KEY="YOUR-KEY" + ``` + + + In your python code, import LLM from llmstudio. + ```python + from llmstudio import LLM + ``` + + + Create your **llm** instance. + ```python + llm = LLM('vertexai/{model}') + ``` + + + **Optional:** You can add your parameters as follows: + ```python + llm = LLM('vertexai/model', + temperature= ..., + max_tokens= ..., + top_p= ..., + frequency_penalty= ..., + presence_penalty= ...) + ``` + You are done setting up your **VertexAI LLM**! + + + + + + + In your python code, import LLM from llmstudio. + ```python + from llmstudio import LLM + ``` + + + Create your **llm** instance. + ```python + llm = LLM('vertexai/{model}',api_key="YOUR_API_KEY") + ``` + + + **Optional:** You can add your parameters as follows: + ```python + llm = LLM('vertexai/model', + temperature= ..., + max_tokens= ..., + top_p= ..., + frequency_penalty= ..., + presence_penalty= ...) + ``` + You are done setting up your **VertexAI LLM**! + + + + + +## What's next? + + + Learn how to send messeges and recieve responses next! + + + Learn how to build a tool calling agent using llmstudio. + + \ No newline at end of file diff --git a/docs/mint.json b/docs/mint.json index 284081e2..63b3defc 100644 --- a/docs/mint.json +++ b/docs/mint.json @@ -43,6 +43,25 @@ "group": "Get Started", "pages": ["quickstart", "support"] }, + { + "group": "LLM", + "pages": [ + "llm/anthropic", + "llm/azure", + "llm/ollama", + "llm/openai", + "llm/vertexai", + "llm/chat" + + ] + }, + { + "group": "How to", + "pages": [ + "how-to/deploy-on-gke/deploy-on-google-kubernetes-engine", + "how-to/build-a-tool-agent" + ] + }, { "group": "Endpoint Examples", "pages": [ diff --git a/llmstudio/cli.py b/llmstudio/cli.py index 2c6197ec..57944402 100644 --- a/llmstudio/cli.py +++ b/llmstudio/cli.py @@ -1,5 +1,6 @@ import os import signal +import threading import click @@ -25,8 +26,9 @@ def server(ui): print("Servers are running. Press CTRL+C to stop.") + stop_event = threading.Event() try: - signal.pause() + stop_event.wait() # Wait indefinitely until the event is set except KeyboardInterrupt: print("Shutting down servers...")