Merge branch 'main' into litellm_bedrock_document_processing_improvem…

…ents

docs/my-website/docs/proxy/jwt_auth_arch.md

@@ -0,0 +1,116 @@
+import Image from '@theme/IdealImage';
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Control Model Access with SSO (Azure AD/Keycloak/etc.)
+
+:::info
+
+✨ JWT Auth is on LiteLLM Enterprise
+
+[Enterprise Pricing](https://www.litellm.ai/#pricing)
+
+[Get free 7-day trial key](https://www.litellm.ai/#trial)
+
+:::
+
+<Image img={require('../../img/control_model_access_jwt.png')} style={{ width: '100%', maxWidth: '4000px' }} />
+
+## Example Token 
+
+<Tabs>
+<TabItem value="Azure AD">
+
+```bash
+{
+  "sub": "1234567890",
+  "name": "John Doe",
+  "email": "[email protected]",
+  "roles": ["basic_user"] # 👈 ROLE
+}
+```
+</TabItem>
+<TabItem value="Keycloak">
+
+```bash
+{
+  "sub": "1234567890",
+  "name": "John Doe",
+  "email": "[email protected]",
+  "resource_access": {
+    "litellm-test-client-id": {
+      "roles": ["basic_user"] # 👈 ROLE
+    }
+  }
+}
+```
+</TabItem>
+</Tabs>
+
+## Proxy Configuration
+
+<Tabs>
+<TabItem value="Azure AD">
+
+```yaml
+general_settings:
+  enable_jwt_auth: True 
+  litellm_jwtauth:
+    user_roles_jwt_field: "roles" # the field in the JWT that contains the roles 
+    user_allowed_roles: ["basic_user"] # roles that map to an 'internal_user' role on LiteLLM 
+    enforce_rbac: true # if true, will check if the user has the correct role to access the model
+
+  role_permissions: # control what models are allowed for each role
+    - role: internal_user
+      models: ["anthropic-claude"]
+
+model_list:
+    - model: anthropic-claude
+      litellm_params:
+        model: claude-3-5-haiku-20241022
+    - model: openai-gpt-4o
+      litellm_params:
+        model: gpt-4o
+```
+
+</TabItem>
+<TabItem value="Keycloak">
+
+```yaml
+general_settings:
+  enable_jwt_auth: True 
+  litellm_jwtauth:
+    user_roles_jwt_field: "resource_access.litellm-test-client-id.roles" # the field in the JWT that contains the roles
+    user_allowed_roles: ["basic_user"] # roles that map to an 'internal_user' role on LiteLLM 
+    enforce_rbac: true # if true, will check if the user has the correct role to access the model
+
+  role_permissions: # control what models are allowed for each role
+    - role: internal_user
+      models: ["anthropic-claude"]
+
+model_list:
+    - model: anthropic-claude
+      litellm_params:
+        model: claude-3-5-haiku-20241022
+    - model: openai-gpt-4o
+      litellm_params:
+        model: gpt-4o
+```
+
+</TabItem>
+</Tabs>
+
+
+## How it works
+
+1. Specify JWT_PUBLIC_KEY_URL - This is the public keys endpoint of your OpenID provider. For Azure AD it's `https://login.microsoftonline.com/{tenant_id}/discovery/v2.0/keys`. For Keycloak it's `{keycloak_base_url}/realms/{your-realm}/protocol/openid-connect/certs`.
+
+1. Map JWT roles to LiteLLM roles - Done via `user_roles_jwt_field` and `user_allowed_roles`
+    -  Currently just `internal_user` is supported for role mapping. 
+2. Specify model access: 
+    - `role_permissions`: control what models are allowed for each role. 
+        - `role`: the LiteLLM role to control access for. Allowed roles = ["internal_user", "proxy_admin", "team"]
+        - `models`: list of models that the role is allowed to access. 
+    - `model_list`: parent list of models on the proxy. [Learn more](./configs.md#llm-configs-model_list)
+
+3. Model Checks: The proxy will run validation checks on the received JWT. [Code](https://github.com/BerriAI/litellm/blob/3a4f5b23b5025b87b6d969f2485cc9bc741f9ba6/litellm/proxy/auth/user_api_key_auth.py#L284)

docs/my-website/docs/proxy/model_access.md

@@ -344,3 +344,6 @@ curl -i http://localhost:4000/v1/chat/completions \
 
 </TabItem>
 </Tabs>
+
+
+## [Role Based Access Control (RBAC)](./jwt_auth_arch)

docs/my-website/docs/proxy/token_auth.md

@@ -1,7 +1,7 @@
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 
-# JWT-based Auth 
+# SSO - JWT-based Auth 
 
 Use JWT's to auth admins / projects into the proxy.
 
@@ -183,6 +183,24 @@ Expected Scope in JWT:
 }
 ```
 
+### Control Model Access 
+
+```yaml
+general_settings:
+  enable_jwt_auth: True 
+  litellm_jwtauth:
+    user_roles_jwt_field: "resource_access.litellm-test-client-id.roles"
+    user_allowed_roles: ["basic_user"] # roles that map to an 'internal_user' role on LiteLLM 
+    enforce_rbac: true # if true, will check if the user has the correct role to access the model + endpoint
+
+  role_permissions: # control what models + endpointsare allowed for each role
+    - role: internal_user
+      models: ["anthropic-claude"]
+```
+
+
+**[Architecture Diagram (Control Model Access)](./jwt_auth_arch)**
+
 ## Advanced - Allowed Routes 
 
 Configure which routes a JWT can access via the config.

docs/my-website/sidebars.js

@@ -51,7 +51,7 @@ const sidebars = {
         {
           type: "category",
           label: "Architecture",
-          items: ["proxy/architecture", "proxy/db_info", "router_architecture", "proxy/user_management_heirarchy"],
+          items: ["proxy/architecture", "proxy/db_info", "router_architecture", "proxy/user_management_heirarchy", "proxy/jwt_auth_arch"],
         },
         {
           type: "link",

litellm/litellm_core_utils/dot_notation_indexing.py

@@ -0,0 +1,59 @@
+"""
+This file contains the logic for dot notation indexing.
+
+Used by JWT Auth to get the user role from the token.
+"""
+
+from typing import Any, Dict, Optional, TypeVar
+
+T = TypeVar("T")
+
+
+def get_nested_value(
+    data: Dict[str, Any], key_path: str, default: Optional[T] = None
+) -> Optional[T]:
+    """
+    Retrieves a value from a nested dictionary using dot notation.
+
+    Args:
+        data: The dictionary to search in
+        key_path: The path to the value using dot notation (e.g., "a.b.c")
+        default: The default value to return if the path is not found
+
+    Returns:
+        The value at the specified path, or the default value if not found
+
+    Example:
+        >>> data = {"a": {"b": {"c": "value"}}}
+        >>> get_nested_value(data, "a.b.c")
+        'value'
+        >>> get_nested_value(data, "a.b.d", "default")
+        'default'
+    """
+    if not key_path:
+        return default
+
+    # Remove metadata. prefix if it exists
+    key_path = (
+        key_path.replace("metadata.", "", 1)
+        if key_path.startswith("metadata.")
+        else key_path
+    )
+
+    # Split the key path into parts
+    parts = key_path.split(".")
+
+    # Traverse through the dictionary
+    current: Any = data
+    for part in parts:
+        try:
+            current = current[part]
+        except (KeyError, TypeError):
+            return default
+
+    # If default is None, we can return any type
+    if default is None:
+        return current
+
+    # Otherwise, ensure the type matches the default
+    return current if isinstance(current, type(default)) else default

litellm/proxy/_new_secret_config.yaml

@@ -1,18 +1,16 @@
 model_list:
-  - model_name: gpt-3.5-turbo-end-user-test
-    litellm_params:
-      model: gpt-3.5-turbo
-      region_name: "eu"
-    model_info:
-      id: "1"
   - model_name: gpt-3.5-turbo
     litellm_params:
       model: gpt-3.5-turbo
-      timeout: 2
-      num_retries: 0
   - model_name: anthropic-claude
     litellm_params:
-      model: anthropic.claude-3-sonnet-20240229-v1:0
-
-litellm_settings:
-  callbacks: ["langsmith"]
+      model: claude-3-5-haiku-20241022
+  - model_name: groq/*
+    litellm_params:
+      model: groq/*
+      api_key: os.environ/GROQ_API_KEY
+      mock_response: Hi!
+  - model_name: deepseek/*
+    litellm_params:
+      model: deepseek/*
+      api_key: os.environ/DEEPSEEK_API_KEY

litellm/proxy/_types.py

@@ -445,6 +445,8 @@ class LiteLLM_JWTAuth(LiteLLMPydanticObjectBase):
     user_id_jwt_field: Optional[str] = None
     user_email_jwt_field: Optional[str] = None
     user_allowed_email_domain: Optional[str] = None
+    user_roles_jwt_field: Optional[str] = None
+    user_allowed_roles: Optional[List[str]] = None
     user_id_upsert: bool = Field(
         default=False, description="If user doesn't exist, upsert them into the db."
     )
@@ -458,11 +460,19 @@ def __init__(self, **kwargs: Any) -> None:
         allowed_keys = self.__annotations__.keys()
 
         invalid_keys = set(kwargs.keys()) - allowed_keys
+        user_roles_jwt_field = kwargs.get("user_roles_jwt_field")
+        user_allowed_roles = kwargs.get("user_allowed_roles")
 
         if invalid_keys:
             raise ValueError(
                 f"Invalid arguments provided: {', '.join(invalid_keys)}. Allowed arguments are: {', '.join(allowed_keys)}."
             )
+        if (user_roles_jwt_field is not None and user_allowed_roles is None) or (
+            user_roles_jwt_field is None and user_allowed_roles is not None
+        ):
+            raise ValueError(
+                "user_allowed_roles must be provided if user_roles_jwt_field is set."
+            )
 
         super().__init__(**kwargs)
 
@@ -2335,3 +2345,15 @@ class ClientSideFallbackModel(TypedDict, total=False):
 
 
 ALL_FALLBACK_MODEL_VALUES = Union[str, ClientSideFallbackModel]
+
+
+RBAC_ROLES = Literal[
+    LitellmUserRoles.PROXY_ADMIN,
+    LitellmUserRoles.TEAM,
+    LitellmUserRoles.INTERNAL_USER,
+]
+
+
+class RoleBasedPermissions(TypedDict):
+    role: Required[RBAC_ROLES]
+    models: Required[List[str]]