Merge CD/CI fixes from Main

.github/workflows/test-litellm.yml

@@ -0,0 +1,35 @@
+name: LiteLLM Tests
+
+on:
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Thank You Message
+      run: |
+        echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY
+        echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.12'
+
+    - name: Install Poetry
+      uses: snok/install-poetry@v1
+
+    - name: Install dependencies
+      run: |
+        poetry install --with dev,proxy-dev --extras proxy
+        poetry run pip install pytest-xdist
+
+    - name: Run tests
+      run: |
+        poetry run pytest tests/litellm -x -vv -n 4 

Makefile

@@ -14,6 +14,9 @@ help:
 install-dev:
 	poetry install --with dev
 
+install-proxy-dev:
+	poetry install --with dev,proxy-dev
+
 lint: install-dev
 	poetry run pip install types-requests types-setuptools types-redis types-PyYAML
 	cd litellm && poetry run mypy . --ignore-missing-imports

deploy/charts/litellm-helm/templates/migrations-job.yaml

@@ -65,6 +65,6 @@ spec:
       tolerations:
         {{- toYaml . | nindent 8 }}
       {{- end }}
-      ttlSecondsAfterFinished: {{ .Values.migrationJob.ttlSecondsAfterFinished }}
+  ttlSecondsAfterFinished: {{ .Values.migrationJob.ttlSecondsAfterFinished }}
   backoffLimit: {{ .Values.migrationJob.backoffLimit }}
 {{- end }}

litellm/_logging.py

@@ -1,6 +1,7 @@
 import json
 import logging
 import os
+import sys
 from datetime import datetime
 from logging import Formatter
 
@@ -40,9 +41,56 @@ def format(self, record):
         return json.dumps(json_record)
 
 
+# Function to set up exception handlers for JSON logging
+def _setup_json_exception_handlers(formatter):
+    # Create a handler with JSON formatting for exceptions
+    error_handler = logging.StreamHandler()
+    error_handler.setFormatter(formatter)
+
+    # Setup excepthook for uncaught exceptions
+    def json_excepthook(exc_type, exc_value, exc_traceback):
+        record = logging.LogRecord(
+            name="LiteLLM",
+            level=logging.ERROR,
+            pathname="",
+            lineno=0,
+            msg=str(exc_value),
+            args=(),
+            exc_info=(exc_type, exc_value, exc_traceback),
+        )
+        error_handler.handle(record)
+
+    sys.excepthook = json_excepthook
+
+    # Configure asyncio exception handler if possible
+    try:
+        import asyncio
+
+        def async_json_exception_handler(loop, context):
+            exception = context.get("exception")
+            if exception:
+                record = logging.LogRecord(
+                    name="LiteLLM",
+                    level=logging.ERROR,
+                    pathname="",
+                    lineno=0,
+                    msg=str(exception),
+                    args=(),
+                    exc_info=None,
+                )
+                error_handler.handle(record)
+            else:
+                loop.default_exception_handler(context)
+
+        asyncio.get_event_loop().set_exception_handler(async_json_exception_handler)
+    except Exception:
+        pass
+
+
 # Create a formatter and set it for the handler
 if json_logs:
     handler.setFormatter(JsonFormatter())
+    _setup_json_exception_handlers(JsonFormatter())
 else:
     formatter = logging.Formatter(
         "\033[92m%(asctime)s - %(name)s:%(levelname)s\033[0m: %(filename)s:%(lineno)s - %(message)s",
@@ -65,18 +113,24 @@ def _turn_on_json():
     handler = logging.StreamHandler()
     handler.setFormatter(JsonFormatter())
 
-    # Define a list of the loggers to update
-    loggers = [verbose_router_logger, verbose_proxy_logger, verbose_logger]
+    # Define all loggers to update, including root logger
+    loggers = [logging.getLogger()] + [
+        verbose_router_logger,
+        verbose_proxy_logger,
+        verbose_logger,
+    ]
 
     # Iterate through each logger and update its handlers
     for logger in loggers:
         # Remove all existing handlers
         for h in logger.handlers[:]:
             logger.removeHandler(h)
-
         # Add the new handler
         logger.addHandler(handler)
 
+    # Set up exception handlers
+    _setup_json_exception_handlers(JsonFormatter())
+
 
 def _turn_on_debug():
     verbose_logger.setLevel(level=logging.DEBUG)  # set package log to debug

litellm/llms/xai/common_utils.py

@@ -1,4 +1,4 @@
-from typing import Optional
+from typing import List, Optional
 
 import httpx
 
@@ -22,7 +22,7 @@ def get_base_model(model: str) -> Optional[str]:
 
     def get_models(
         self, api_key: Optional[str] = None, api_base: Optional[str] = None
-    ) -> list[str]:
+    ) -> List[str]:
         api_base = self.get_api_base(api_base)
         api_key = self.get_api_key(api_key)
         if api_base is None or api_key is None:

litellm/proxy/_new_secret_config.yaml

@@ -20,6 +20,7 @@ model_list:
 litellm_settings:
   num_retries: 0
   callbacks: ["prometheus"]
+  json_logs: true
 
 router_settings:
   routing_strategy: usage-based-routing-v2 # 👈 KEY CHANGE

litellm/proxy/proxy_server.py

@@ -462,6 +462,8 @@ async def proxy_startup_event(app: FastAPI):
     if premium_user is False:
         premium_user = _license_check.is_premium()
 
+    ## CHECK MASTER KEY IN ENVIRONMENT ##
+    master_key = get_secret_str("LITELLM_MASTER_KEY")
     ### LOAD CONFIG ###
     worker_config: Optional[Union[str, dict]] = get_secret("WORKER_CONFIG")  # type: ignore
     env_config_yaml: Optional[str] = get_secret_str("CONFIG_FILE_PATH")