Merge branch 'main' into grit-prod

.circleci/config.yml

@@ -1,4 +1,4 @@
-version: 2.1
+version: 4.3.4
 jobs:
   local_testing:
     docker:
@@ -8,6 +8,11 @@ jobs:
     steps:
       - checkout
 
+      - run:
+          name: Show git commit hash
+          command: |
+            echo "Git commit hash: $CIRCLE_SHA1"
+
       - run:
           name: Check if litellm dir was updated or if pyproject.toml was modified
           command: |
@@ -28,17 +33,21 @@ jobs:
             pip install "pytest==7.3.1"
             pip install "pytest-asyncio==0.21.1"
             pip install mypy
-            pip install "google-generativeai>=0.3.2"
-            pip install "google-cloud-aiplatform>=1.38.0"
-            pip install "boto3>=1.28.57"
-            pip install "aioboto3>=12.3.0"
+            pip install "google-generativeai==0.3.2"
+            pip install "google-cloud-aiplatform==1.43.0"
+            pip install pyarrow
+            pip install "boto3==1.34.34"
+            pip install "aioboto3==12.3.0"
             pip install langchain
-            pip install "langfuse>=2.0.0"
+            pip install lunary==0.2.5
+            pip install "langfuse==2.27.1"
+            pip install "logfire==0.29.0"
             pip install numpydoc
-            pip install traceloop-sdk==0.0.69
+            pip install traceloop-sdk==0.18.2
             pip install openai
             pip install prisma            
             pip install "httpx==0.24.1"
+            pip install fastapi
             pip install "gunicorn==21.2.0"
             pip install "anyio==3.7.1"
             pip install "aiodynamo==23.10.1"
@@ -48,6 +57,10 @@ jobs:
             pip install argon2-cffi
             pip install "pytest-mock==3.12.0"
             pip install python-multipart
+            pip install google-cloud-aiplatform
+            pip install prometheus-client==0.20.0
+            pip install "pydantic==2.7.1"
+            pip install "diskcache==5.6.1"
       - save_cache:
           paths:
             - ./venv
@@ -70,21 +83,20 @@ jobs:
           name: Linting Testing
           command: |
             cd litellm
-            python -m pip install types-requests types-setuptools types-redis
+            python -m pip install types-requests types-setuptools types-redis types-PyYAML
             if ! python -m mypy . --ignore-missing-imports; then
               echo "mypy detected errors"
               exit 1
             fi
             cd ..
-  
 
       # Run pytest and generate JUnit XML report
       - run:
           name: Run tests
           command: |
             pwd
             ls
-            python -m pytest -vv litellm/tests/ -x --junitxml=test-results/junit.xml --durations=5 
+            python -m pytest -vv litellm/tests/ -x --junitxml=test-results/junit.xml --durations=5
           no_output_timeout: 120m
 
       # Store test results
@@ -120,6 +132,7 @@ jobs:
   build_and_test:
     machine:
       image: ubuntu-2204:2023.10.1
+    resource_class: xlarge
     working_directory: ~/project
     steps:
       - checkout
@@ -152,12 +165,14 @@ jobs:
             pip install "pytest-mock==3.12.0"
             pip install "pytest-asyncio==0.21.1"
             pip install mypy
-            pip install "google-generativeai>=0.3.2"
-            pip install "google-cloud-aiplatform>=1.38.0"
-            pip install "boto3>=1.28.57"
-            pip install "aioboto3>=12.3.0"
+            pip install "google-generativeai==0.3.2"
+            pip install "google-cloud-aiplatform==1.43.0"
+            pip install pyarrow
+            pip install "boto3==1.34.34"
+            pip install "aioboto3==12.3.0"
             pip install langchain
             pip install "langfuse>=2.0.0"
+            pip install "logfire==0.29.0"
             pip install numpydoc
             pip install prisma            
             pip install fastapi            
@@ -176,22 +191,28 @@ jobs:
           command: |
             docker run -d \
               -p 4000:4000 \
-              -e DATABASE_URL=$PROXY_DOCKER_DB_URL \
+              -e DATABASE_URL=$PROXY_DATABASE_URL \
               -e AZURE_API_KEY=$AZURE_API_KEY \
+              -e REDIS_HOST=$REDIS_HOST \
+              -e REDIS_PASSWORD=$REDIS_PASSWORD \
+              -e REDIS_PORT=$REDIS_PORT \
               -e AZURE_FRANCE_API_KEY=$AZURE_FRANCE_API_KEY \
               -e AZURE_EUROPE_API_KEY=$AZURE_EUROPE_API_KEY \
               -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
               -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
               -e AWS_REGION_NAME=$AWS_REGION_NAME \
+              -e AUTO_INFER_REGION=True \
               -e OPENAI_API_KEY=$OPENAI_API_KEY \
+              -e LANGFUSE_PROJECT1_PUBLIC=$LANGFUSE_PROJECT1_PUBLIC \
+              -e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \
+              -e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \
+              -e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \
               --name my-app \
               -v $(pwd)/proxy_server_config.yaml:/app/config.yaml \
               my-app:latest \
               --config /app/config.yaml \
               --port 4000 \
-              --num_workers 8 \
               --detailed_debug \
-              --run_gunicorn \
       - run:
           name: Install curl and dockerize
           command: |
@@ -204,15 +225,15 @@ jobs:
           name: Start outputting logs
           command: docker logs -f my-app
           background: true
-      - run: 
+      - run:
           name: Wait for app to be ready
-          command: dockerize -wait http://localhost:4000 -timeout 1m
+          command: dockerize -wait http://localhost:4000 -timeout 5m
       - run:
           name: Run tests
           command: |
             pwd
             ls
-            python -m pytest -vv tests/ -x --junitxml=test-results/junit.xml --durations=5 
+            python -m pytest -vv tests/ -x --junitxml=test-results/junit.xml --durations=5
           no_output_timeout: 120m
 
       # Store test results
@@ -234,7 +255,7 @@ jobs:
           name: Copy model_prices_and_context_window File to model_prices_and_context_window_backup
           command: |
             cp model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
-      
+
       - run:
           name: Check if litellm dir was updated or if pyproject.toml was modified
           command: |
@@ -288,7 +309,7 @@ jobs:
               -H "Accept: application/vnd.github.v3+json" \
               -H "Authorization: Bearer $GITHUB_TOKEN" \
               "https://api.github.com/repos/BerriAI/litellm/actions/workflows/ghcr_deploy.yml/dispatches" \
-              -d "{\"ref\":\"main\", \"inputs\":{\"tag\":\"v${VERSION}\"}}"
+              -d "{\"ref\":\"main\", \"inputs\":{\"tag\":\"v${VERSION}\", \"commit_hash\":\"$CIRCLE_SHA1\"}}"
 
 workflows:
   version: 2
@@ -319,4 +340,4 @@ workflows:
           filters:
             branches:
               only:
-                - main
+                - main

.circleci/requirements.txt

@@ -3,12 +3,10 @@ openai
 python-dotenv
 tiktoken
 importlib_metadata
-baseten
 cohere
 redis
 anthropic
-boto3
 orjson
-pydantic
-google-cloud-aiplatform
+pydantic==1.10.14
+google-cloud-aiplatform==1.43.0
 redisvl==0.0.7 # semantic caching

.devcontainer/devcontainer.json

@@ -0,0 +1,51 @@
+{
+	"name": "Python 3.11",
+	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+	"image": "mcr.microsoft.com/devcontainers/python:3.11-bookworm",
+	// https://github.com/devcontainers/images/tree/main/src/python
+	// https://mcr.microsoft.com/en-us/product/devcontainers/python/tags
+
+	// "build": {
+	// 	"dockerfile": "Dockerfile",
+	// 	"context": ".."
+	// },
+
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	// "features": {},
+
+	// Configure tool-specific properties.
+	"customizations": {
+		// Configure properties specific to VS Code.
+		"vscode": {
+			"settings": {},
+			"extensions": [
+				"ms-python.python",
+				"ms-python.vscode-pylance",
+				"GitHub.copilot",
+				"GitHub.copilot-chat"
+			]
+		}
+	},
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	"forwardPorts": [4000],
+
+	"containerEnv": {
+		"LITELLM_LOG": "DEBUG"
+	},
+
+	// Use 'portsAttributes' to set default properties for specific forwarded ports. 
+	// More info: https://containers.dev/implementors/json_reference/#port-attributes
+	"portsAttributes": {
+		"4000": {
+			"label": "LiteLLM Server",
+			"onAutoForward": "notify"
+		}
+	},
+
+	// More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "litellm",
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	"postCreateCommand": "pipx install poetry && poetry install -E extra_proxy -E proxy"
+}

.dockerignore

@@ -1,5 +1,5 @@
-/docs
-/cookbook
-/.circleci
-/.github
-/tests
+docs
+cookbook
+.circleci
+.github
+tests

.git-blame-ignore-revs

@@ -0,0 +1,10 @@
+# Add the commit hash of any commit you want to ignore in `git blame` here.
+# One commit hash per line.
+#
+# The GitHub Blame UI will use this file automatically!
+#
+# Run this command to always ignore formatting commits in `git blame`
+#     git config blame.ignoreRevsFile .git-blame-ignore-revs
+
+# Update pydantic code to fix warnings (GH-3600)
+876840e9957bc7e9f7d6a2b58c4d7c53dad16481

.github/pull_request_template.md

@@ -0,0 +1,29 @@
+## Title
+
+<!-- e.g. "Implement user authentication feature" -->
+
+## Relevant issues
+
+<!-- e.g. "Fixes #000" -->
+
+## Type
+
+<!-- Select the type of Pull Request -->
+<!-- Keep only the necessary ones -->
+
+🆕 New Feature
+🐛 Bug Fix
+🧹 Refactoring
+📖 Documentation
+🚄 Infrastructure
+✅ Test
+
+## Changes
+
+<!-- List of changes -->
+
+## [REQUIRED] Testing - Attach a screenshot of any new tests passing locall
+If UI changes, send a screenshot/GIF of working UI fixes
+
+<!-- Test procedure -->
+