From 340505ac1ef29f87ed450f1d793941feaa4737e1 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 20 Dec 2024 21:27:45 -0800 Subject: [PATCH 1/5] (fix) LiteLLM Proxy fix GET `/files/{file_id:path}/content"` endpoint (#7342) * fix order of get_file_content * update e2 files tests * add e2 batches endpoint testing * update config.yml * write content to file * use correct oai_misc_config * fixes for openai batches endpoint testing * remove extra out file * fix input.jsonl --- .circleci/config.yml | 125 ++++++++++++++++++ .../example_config_yaml/oai_misc_config.yaml | 62 +++++++++ .../openai_files_endpoints/files_endpoints.py | 123 ++++++++--------- litellm/proxy/proxy_config.yaml | 5 + tests/openai_misc_endpoints_tests/input.jsonl | 1 + .../openai_batch_completions.jsonl | 2 + tests/openai_misc_endpoints_tests/out.jsonl | 1 + .../test_openai_batches_endpoint.py | 75 +++++++++++ .../test_openai_files_endpoints.py | 29 ++-- .../test_openai_fine_tuning.py | 0 10 files changed, 351 insertions(+), 72 deletions(-) create mode 100644 litellm/proxy/example_config_yaml/oai_misc_config.yaml create mode 100644 tests/openai_misc_endpoints_tests/input.jsonl create mode 100644 tests/openai_misc_endpoints_tests/openai_batch_completions.jsonl create mode 100644 tests/openai_misc_endpoints_tests/out.jsonl rename tests/{ => openai_misc_endpoints_tests}/test_openai_batches_endpoint.py (68%) rename tests/{ => openai_misc_endpoints_tests}/test_openai_files_endpoints.py (75%) rename tests/{ => openai_misc_endpoints_tests}/test_openai_fine_tuning.py (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7789d57c1ff7..cb887077de9f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -998,6 +998,124 @@ jobs: python -m pytest -s -vv tests/*.py -x --junitxml=test-results/junit.xml --durations=5 --ignore=tests/otel_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests --ignore=tests/load_tests --ignore=tests/llm_translation --ignore=tests/image_gen_tests --ignore=tests/pass_through_unit_tests no_output_timeout: 120m + # Store test results + - store_test_results: + path: test-results + e2e_openai_misc_endpoints: + machine: + image: ubuntu-2204:2023.10.1 + resource_class: xlarge + working_directory: ~/project + steps: + - checkout + - run: + name: Install Docker CLI (In case it's not already installed) + command: | + sudo apt-get update + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + - run: + name: Install Python 3.9 + command: | + curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh + bash miniconda.sh -b -p $HOME/miniconda + export PATH="$HOME/miniconda/bin:$PATH" + conda init bash + source ~/.bashrc + conda create -n myenv python=3.9 -y + conda activate myenv + python --version + - run: + name: Install Dependencies + command: | + pip install "pytest==7.3.1" + pip install "pytest-asyncio==0.21.1" + pip install aiohttp + python -m pip install --upgrade pip + python -m pip install -r .circleci/requirements.txt + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-mock==3.12.0" + pip install "pytest-asyncio==0.21.1" + pip install mypy + pip install "jsonlines==4.0.0" + pip install "google-generativeai==0.3.2" + pip install "google-cloud-aiplatform==1.43.0" + pip install pyarrow + pip install "boto3==1.34.34" + pip install "aioboto3==12.3.0" + pip install langchain + pip install "langfuse>=2.0.0" + pip install "logfire==0.29.0" + pip install numpydoc + pip install prisma + pip install fastapi + pip install jsonschema + pip install "httpx==0.24.1" + pip install "gunicorn==21.2.0" + pip install "anyio==3.7.1" + pip install "aiodynamo==23.10.1" + pip install "asyncio==3.4.3" + pip install "PyGithub==1.59.1" + pip install "openai==1.54.0 " + # Run pytest and generate JUnit XML report + - run: + name: Build Docker image + command: docker build -t my-app:latest -f ./docker/Dockerfile.database . + - run: + name: Run Docker container + command: | + docker run -d \ + -p 4000:4000 \ + -e DATABASE_URL=$PROXY_DATABASE_URL \ + -e AZURE_API_KEY=$AZURE_API_KEY \ + -e REDIS_HOST=$REDIS_HOST \ + -e REDIS_PASSWORD=$REDIS_PASSWORD \ + -e REDIS_PORT=$REDIS_PORT \ + -e AZURE_FRANCE_API_KEY=$AZURE_FRANCE_API_KEY \ + -e AZURE_EUROPE_API_KEY=$AZURE_EUROPE_API_KEY \ + -e MISTRAL_API_KEY=$MISTRAL_API_KEY \ + -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ + -e GROQ_API_KEY=$GROQ_API_KEY \ + -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ + -e COHERE_API_KEY=$COHERE_API_KEY \ + -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ + -e AWS_REGION_NAME=$AWS_REGION_NAME \ + -e AUTO_INFER_REGION=True \ + -e OPENAI_API_KEY=$OPENAI_API_KEY \ + -e LITELLM_LICENSE=$LITELLM_LICENSE \ + -e LANGFUSE_PROJECT1_PUBLIC=$LANGFUSE_PROJECT1_PUBLIC \ + -e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \ + -e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \ + -e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \ + --name my-app \ + -v $(pwd)/litellm/proxy/example_config_yaml/oai_misc_config.yaml:/app/config.yaml \ + my-app:latest \ + --config /app/config.yaml \ + --port 4000 \ + --detailed_debug \ + - run: + name: Install curl and dockerize + command: | + sudo apt-get update + sudo apt-get install -y curl + sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + sudo rm dockerize-linux-amd64-v0.6.1.tar.gz + - run: + name: Start outputting logs + command: docker logs -f my-app + background: true + - run: + name: Wait for app to be ready + command: dockerize -wait http://localhost:4000 -timeout 5m + - run: + name: Run tests + command: | + pwd + ls + python -m pytest -s -vv tests/openai_misc_endpoints_tests --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 120m + # Store test results - store_test_results: path: test-results @@ -1572,6 +1690,12 @@ workflows: only: - main - /litellm_.*/ + - e2e_openai_misc_endpoints: + filters: + branches: + only: + - main + - /litellm_.*/ - proxy_logging_guardrails_model_info_tests: filters: branches: @@ -1655,6 +1779,7 @@ workflows: requires: - local_testing - build_and_test + - e2e_openai_misc_endpoints - load_testing - test_bad_database_url - llm_translation_testing diff --git a/litellm/proxy/example_config_yaml/oai_misc_config.yaml b/litellm/proxy/example_config_yaml/oai_misc_config.yaml new file mode 100644 index 000000000000..4785b0e7949d --- /dev/null +++ b/litellm/proxy/example_config_yaml/oai_misc_config.yaml @@ -0,0 +1,62 @@ +model_list: + - model_name: gpt-3.5-turbo-end-user-test + litellm_params: + model: gpt-3.5-turbo + region_name: "eu" + model_info: + id: "1" + - model_name: "*" + litellm_params: + model: openai/* + api_key: os.environ/OPENAI_API_KEY + # provider specific wildcard routing + - model_name: "anthropic/*" + litellm_params: + model: "anthropic/*" + api_key: os.environ/ANTHROPIC_API_KEY + - model_name: "groq/*" + litellm_params: + model: "groq/*" + api_key: os.environ/GROQ_API_KEY +litellm_settings: + # set_verbose: True # Uncomment this if you want to see verbose logs; not recommended in production + drop_params: True + # max_budget: 100 + # budget_duration: 30d + num_retries: 5 + request_timeout: 600 + telemetry: False + context_window_fallbacks: [{"gpt-3.5-turbo": ["gpt-3.5-turbo-large"]}] + default_team_settings: + - team_id: team-1 + success_callback: ["langfuse"] + failure_callback: ["langfuse"] + langfuse_public_key: os.environ/LANGFUSE_PROJECT1_PUBLIC # Project 1 + langfuse_secret: os.environ/LANGFUSE_PROJECT1_SECRET # Project 1 + - team_id: team-2 + success_callback: ["langfuse"] + failure_callback: ["langfuse"] + langfuse_public_key: os.environ/LANGFUSE_PROJECT2_PUBLIC # Project 2 + langfuse_secret: os.environ/LANGFUSE_PROJECT2_SECRET # Project 2 + langfuse_host: https://us.cloud.langfuse.com + +# For /fine_tuning/jobs endpoints +finetune_settings: + - custom_llm_provider: azure + api_base: https://exampleopenaiendpoint-production.up.railway.app + api_key: fake-key + api_version: "2023-03-15-preview" + - custom_llm_provider: openai + api_key: os.environ/OPENAI_API_KEY + +# for /files endpoints +files_settings: + - custom_llm_provider: azure + api_base: https://exampleopenaiendpoint-production.up.railway.app + api_key: fake-key + api_version: "2023-03-15-preview" + - custom_llm_provider: openai + api_key: os.environ/OPENAI_API_KEY + +general_settings: + master_key: sk-1234 # [OPTIONAL] Use to enforce auth on proxy. See - https://docs.litellm.ai/docs/proxy/virtual_keys \ No newline at end of file diff --git a/litellm/proxy/openai_files_endpoints/files_endpoints.py b/litellm/proxy/openai_files_endpoints/files_endpoints.py index 19b176730bd7..c1943636269d 100644 --- a/litellm/proxy/openai_files_endpoints/files_endpoints.py +++ b/litellm/proxy/openai_files_endpoints/files_endpoints.py @@ -266,21 +266,21 @@ async def create_file( @router.get( - "/{provider}/v1/files/{file_id:path}", + "/{provider}/v1/files/{file_id:path}/content", dependencies=[Depends(user_api_key_auth)], tags=["files"], ) @router.get( - "/v1/files/{file_id:path}", + "/v1/files/{file_id:path}/content", dependencies=[Depends(user_api_key_auth)], tags=["files"], ) @router.get( - "/files/{file_id:path}", + "/files/{file_id:path}/content", dependencies=[Depends(user_api_key_auth)], tags=["files"], ) -async def get_file( +async def get_file_content( request: Request, fastapi_response: Response, file_id: str, @@ -289,13 +289,13 @@ async def get_file( ): """ Returns information about a specific file. that can be used across - Assistants API, Batch API - This is the equivalent of GET https://api.openai.com/v1/files/{file_id} + This is the equivalent of GET https://api.openai.com/v1/files/{file_id}/content - Supports Identical Params as: https://platform.openai.com/docs/api-reference/files/retrieve + Supports Identical Params as: https://platform.openai.com/docs/api-reference/files/retrieve-contents Example Curl ``` - curl http://localhost:4000/v1/files/file-abc123 \ + curl http://localhost:4000/v1/files/file-abc123/content \ -H "Authorization: Bearer sk-1234" ``` @@ -322,9 +322,9 @@ async def get_file( proxy_config=proxy_config, ) - if provider is None: # default to openai + if provider is None: provider = "openai" - response = await litellm.afile_retrieve( + response = await litellm.afile_content( custom_llm_provider=provider, file_id=file_id, **data # type: ignore ) @@ -351,14 +351,24 @@ async def get_file( model_region=getattr(user_api_key_dict, "allowed_model_region", ""), ) ) - return response + httpx_response: Optional[httpx.Response] = getattr(response, "response", None) + if httpx_response is None: + raise ValueError( + f"Invalid response - response.response is None - got {response}" + ) + + return Response( + content=httpx_response.content, + status_code=httpx_response.status_code, + headers=httpx_response.headers, + ) except Exception as e: await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) verbose_proxy_logger.error( - "litellm.proxy.proxy_server.retrieve_file(): Exception occured - {}".format( + "litellm.proxy.proxy_server.retrieve_file_content(): Exception occured - {}".format( str(e) ) ) @@ -380,22 +390,22 @@ async def get_file( ) -@router.delete( +@router.get( "/{provider}/v1/files/{file_id:path}", dependencies=[Depends(user_api_key_auth)], tags=["files"], ) -@router.delete( +@router.get( "/v1/files/{file_id:path}", dependencies=[Depends(user_api_key_auth)], tags=["files"], ) -@router.delete( +@router.get( "/files/{file_id:path}", dependencies=[Depends(user_api_key_auth)], tags=["files"], ) -async def delete_file( +async def get_file( request: Request, fastapi_response: Response, file_id: str, @@ -403,16 +413,15 @@ async def delete_file( user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ - Deletes a specified file. that can be used across - Assistants API, Batch API - This is the equivalent of DELETE https://api.openai.com/v1/files/{file_id} + Returns information about a specific file. that can be used across - Assistants API, Batch API + This is the equivalent of GET https://api.openai.com/v1/files/{file_id} - Supports Identical Params as: https://platform.openai.com/docs/api-reference/files/delete + Supports Identical Params as: https://platform.openai.com/docs/api-reference/files/retrieve Example Curl ``` curl http://localhost:4000/v1/files/file-abc123 \ - -X DELETE \ - -H "Authorization: Bearer $OPENAI_API_KEY" + -H "Authorization: Bearer sk-1234" ``` """ @@ -440,7 +449,7 @@ async def delete_file( if provider is None: # default to openai provider = "openai" - response = await litellm.afile_delete( + response = await litellm.afile_retrieve( custom_llm_provider=provider, file_id=file_id, **data # type: ignore ) @@ -496,38 +505,39 @@ async def delete_file( ) -@router.get( - "/{provider}/v1/files", +@router.delete( + "/{provider}/v1/files/{file_id:path}", dependencies=[Depends(user_api_key_auth)], tags=["files"], ) -@router.get( - "/v1/files", +@router.delete( + "/v1/files/{file_id:path}", dependencies=[Depends(user_api_key_auth)], tags=["files"], ) -@router.get( - "/files", +@router.delete( + "/files/{file_id:path}", dependencies=[Depends(user_api_key_auth)], tags=["files"], ) -async def list_files( +async def delete_file( request: Request, fastapi_response: Response, - user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + file_id: str, provider: Optional[str] = None, - purpose: Optional[str] = None, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ - Returns information about a specific file. that can be used across - Assistants API, Batch API - This is the equivalent of GET https://api.openai.com/v1/files/ + Deletes a specified file. that can be used across - Assistants API, Batch API + This is the equivalent of DELETE https://api.openai.com/v1/files/{file_id} - Supports Identical Params as: https://platform.openai.com/docs/api-reference/files/list + Supports Identical Params as: https://platform.openai.com/docs/api-reference/files/delete Example Curl ``` - curl http://localhost:4000/v1/files\ - -H "Authorization: Bearer sk-1234" + curl http://localhost:4000/v1/files/file-abc123 \ + -X DELETE \ + -H "Authorization: Bearer $OPENAI_API_KEY" ``` """ @@ -553,10 +563,10 @@ async def list_files( proxy_config=proxy_config, ) - if provider is None: + if provider is None: # default to openai provider = "openai" - response = await litellm.afile_list( - custom_llm_provider=provider, purpose=purpose, **data # type: ignore + response = await litellm.afile_delete( + custom_llm_provider=provider, file_id=file_id, **data # type: ignore ) ### ALERTING ### @@ -589,7 +599,7 @@ async def list_files( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) verbose_proxy_logger.error( - "litellm.proxy.proxy_server.list_files(): Exception occured - {}".format( + "litellm.proxy.proxy_server.retrieve_file(): Exception occured - {}".format( str(e) ) ) @@ -612,36 +622,36 @@ async def list_files( @router.get( - "/{provider}/v1/files/{file_id:path}/content", + "/{provider}/v1/files", dependencies=[Depends(user_api_key_auth)], tags=["files"], ) @router.get( - "/v1/files/{file_id:path}/content", + "/v1/files", dependencies=[Depends(user_api_key_auth)], tags=["files"], ) @router.get( - "/files/{file_id:path}/content", + "/files", dependencies=[Depends(user_api_key_auth)], tags=["files"], ) -async def get_file_content( +async def list_files( request: Request, fastapi_response: Response, - file_id: str, - provider: Optional[str] = None, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + provider: Optional[str] = None, + purpose: Optional[str] = None, ): """ Returns information about a specific file. that can be used across - Assistants API, Batch API - This is the equivalent of GET https://api.openai.com/v1/files/{file_id}/content + This is the equivalent of GET https://api.openai.com/v1/files/ - Supports Identical Params as: https://platform.openai.com/docs/api-reference/files/retrieve-contents + Supports Identical Params as: https://platform.openai.com/docs/api-reference/files/list Example Curl ``` - curl http://localhost:4000/v1/files/file-abc123/content \ + curl http://localhost:4000/v1/files\ -H "Authorization: Bearer sk-1234" ``` @@ -670,8 +680,8 @@ async def get_file_content( if provider is None: provider = "openai" - response = await litellm.afile_content( - custom_llm_provider=provider, file_id=file_id, **data # type: ignore + response = await litellm.afile_list( + custom_llm_provider=provider, purpose=purpose, **data # type: ignore ) ### ALERTING ### @@ -697,23 +707,14 @@ async def get_file_content( model_region=getattr(user_api_key_dict, "allowed_model_region", ""), ) ) - httpx_response: Optional[httpx.Response] = getattr(response, "response", None) - if httpx_response is None: - raise ValueError( - f"Invalid response - response.response is None - got {response}" - ) - return Response( - content=httpx_response.content, - status_code=httpx_response.status_code, - headers=httpx_response.headers, - ) + return response except Exception as e: await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) verbose_proxy_logger.error( - "litellm.proxy.proxy_server.retrieve_file_content(): Exception occured - {}".format( + "litellm.proxy.proxy_server.list_files(): Exception occured - {}".format( str(e) ) ) diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 2cacf1e08ecc..2861177f1b10 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -4,3 +4,8 @@ model_list: model: openai/o1-preview api_key: os.environ/OPENAI_API_KEY + +# for /files endpoints +files_settings: + - custom_llm_provider: openai + api_key: os.environ/OPENAI_API_KEY \ No newline at end of file diff --git a/tests/openai_misc_endpoints_tests/input.jsonl b/tests/openai_misc_endpoints_tests/input.jsonl new file mode 100644 index 000000000000..2cb5dada55fe --- /dev/null +++ b/tests/openai_misc_endpoints_tests/input.jsonl @@ -0,0 +1 @@ +{"custom_id": "ae006110bb364606||/workspace/saved_models/meta-llama/Meta-Llama-3.1-8B-Instruct", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-4o-2024-05-13", "temperature": 0, "max_tokens": 1024, "response_format": {"type": "json_object"}, "messages": [{"role": "user", "content": "# Instruction \n\nYou are an expert evaluator. Your task is to evaluate the quality of the responses generated by AI models. \nWe will provide you with the user query and an AI-generated responses.\nYo must respond in json"}]}} \ No newline at end of file diff --git a/tests/openai_misc_endpoints_tests/openai_batch_completions.jsonl b/tests/openai_misc_endpoints_tests/openai_batch_completions.jsonl new file mode 100644 index 000000000000..8b17a304a455 --- /dev/null +++ b/tests/openai_misc_endpoints_tests/openai_batch_completions.jsonl @@ -0,0 +1,2 @@ +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "my-custom-name", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "my-custom-name", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} \ No newline at end of file diff --git a/tests/openai_misc_endpoints_tests/out.jsonl b/tests/openai_misc_endpoints_tests/out.jsonl new file mode 100644 index 000000000000..3e6aa688d6e7 --- /dev/null +++ b/tests/openai_misc_endpoints_tests/out.jsonl @@ -0,0 +1 @@ +{"id": "batch_req_6765ed82629c8190b70c10c183b5e994", "custom_id": "ae006110bb364606||/workspace/saved_models/meta-llama/Meta-Llama-3.1-8B-Instruct", "response": {"status_code": 200, "request_id": "36bbc935dec50094e84af1db52cf2cc7", "body": {"id": "chatcmpl-AgfdQmdwJQ0NrQManGI8ecwMvF0ZC", "object": "chat.completion", "created": 1734733184, "model": "gpt-4o-2024-05-13", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\n \"user_query\": \"What are the benefits of using renewable energy sources?\",\n \"ai_response\": \"Renewable energy sources, such as solar, wind, and hydroelectric power, offer numerous benefits. They are sustainable and can be replenished naturally, reducing the reliance on finite fossil fuels. Additionally, renewable energy sources produce little to no greenhouse gas emissions, helping to combat climate change and reduce air pollution. They also create jobs in the renewable energy sector and can lead to energy independence for countries that invest in their development. Furthermore, renewable energy technologies often have lower operating costs once established, providing long-term economic benefits.\"\n}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 51, "completion_tokens": 128, "total_tokens": 179, "prompt_tokens_details": {"cached_tokens": 0, "audio_tokens": 0}, "completion_tokens_details": {"reasoning_tokens": 0, "audio_tokens": 0, "accepted_prediction_tokens": 0, "rejected_prediction_tokens": 0}}, "system_fingerprint": "fp_20cb129c3a"}}, "error": null} diff --git a/tests/test_openai_batches_endpoint.py b/tests/openai_misc_endpoints_tests/test_openai_batches_endpoint.py similarity index 68% rename from tests/test_openai_batches_endpoint.py rename to tests/openai_misc_endpoints_tests/test_openai_batches_endpoint.py index 64e3277e9287..9564243311f8 100644 --- a/tests/test_openai_batches_endpoint.py +++ b/tests/openai_misc_endpoints_tests/test_openai_batches_endpoint.py @@ -6,6 +6,9 @@ from openai import OpenAI, AsyncOpenAI from typing import Optional, List, Union from test_openai_files_endpoints import upload_file, delete_file +import os +import sys +import time BASE_URL = "http://localhost:4000" # Replace with your actual base URL @@ -87,6 +90,78 @@ async def test_batches_operations(): await delete_file(session, file_id) +from openai import OpenAI + +client = OpenAI(base_url=BASE_URL, api_key=API_KEY) + + +def create_batch_oai_sdk(filepath) -> str: + batch_input_file = client.files.create(file=open(filepath, "rb"), purpose="batch") + batch_input_file_id = batch_input_file.id + + rq = client.batches.create( + input_file_id=batch_input_file_id, + endpoint="/v1/chat/completions", + completion_window="24h", + metadata={ + "description": filepath, + }, + ) + + print(f"Batch submitted. ID: {rq.id}") + return rq.id + + +def await_batch_completion(batch_id: str): + while True: + batch = client.batches.retrieve(batch_id) + if batch.status == "completed": + print(f"Batch {batch_id} completed.") + return + + print("waiting for batch to complete...") + time.sleep(10) + + +def write_content_to_file(batch_id: str, output_path: str) -> str: + batch = client.batches.retrieve(batch_id) + content = client.files.content(batch.output_file_id) + print("content from files.content", content.content) + content.write_to_file(output_path) + + +import jsonlines + + +def read_jsonl(filepath: str): + results = [] + with jsonlines.open(filepath) as f: + for line in f: + results.append(line) + + for item in results: + print(item) + custom_id = item["custom_id"] + print(custom_id) + + +def test_e2e_batches_files(): + """ + [PROD Test] Ensures OpenAI Batches + files work with OpenAI SDK + """ + input_path = "input.jsonl" + output_path = "out.jsonl" + + _current_dir = os.path.dirname(os.path.abspath(__file__)) + input_file_path = os.path.join(_current_dir, input_path) + output_file_path = os.path.join(_current_dir, output_path) + + batch_id = create_batch_oai_sdk(input_file_path) + await_batch_completion(batch_id) + write_content_to_file(batch_id, output_file_path) + read_jsonl(output_file_path) + + @pytest.mark.skip(reason="Local only test to verify if things work well") def test_vertex_batches_endpoint(): """ diff --git a/tests/test_openai_files_endpoints.py b/tests/openai_misc_endpoints_tests/test_openai_files_endpoints.py similarity index 75% rename from tests/test_openai_files_endpoints.py rename to tests/openai_misc_endpoints_tests/test_openai_files_endpoints.py index 1444b8a7064d..5299cfc5376d 100644 --- a/tests/test_openai_files_endpoints.py +++ b/tests/openai_misc_endpoints_tests/test_openai_files_endpoints.py @@ -13,21 +13,27 @@ @pytest.mark.asyncio async def test_file_operations(): - async with aiohttp.ClientSession() as session: - # Test file upload and get file_id - file_id = await upload_file(session) + openai_client = AsyncOpenAI(api_key=API_KEY, base_url=BASE_URL) + file_content = b'{"prompt": "Hello", "completion": "Hi"}' + uploaded_file = await openai_client.files.create( + purpose="fine-tune", + file=file_content, + ) + list_files = await openai_client.files.list() + print("list_files=", list_files) - # Test list files - await list_files(session) + get_file = await openai_client.files.retrieve(file_id=uploaded_file.id) + print("get_file=", get_file) - # Test get file - await get_file(session, file_id) + get_file_content = await openai_client.files.content(file_id=uploaded_file.id) + print("get_file_content=", get_file_content.content) - # Test get file content - await get_file_content(session, file_id) + assert get_file_content.content == file_content + # try get_file_content.write_to_file + get_file_content.write_to_file("get_file_content.jsonl") - # Test delete file - await delete_file(session, file_id) + delete_file = await openai_client.files.delete(file_id=uploaded_file.id) + print("delete_file=", delete_file) async def upload_file(session, purpose="fine-tune"): @@ -81,6 +87,7 @@ async def get_file_content(session, file_id): async with session.get(url, headers=headers) as response: assert response.status == 200 content = await response.text() + print("content from /files/{file_id}/content=", content) assert content # Check if content is not empty print(f"Get file content successful for file ID: {file_id}") diff --git a/tests/test_openai_fine_tuning.py b/tests/openai_misc_endpoints_tests/test_openai_fine_tuning.py similarity index 100% rename from tests/test_openai_fine_tuning.py rename to tests/openai_misc_endpoints_tests/test_openai_fine_tuning.py From 52d14819c8e1f2528cf29d884cd7432327faa3b9 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 20 Dec 2024 21:28:32 -0800 Subject: [PATCH 2/5] use helper for image gen tests (#7343) --- litellm/cost_calculator.py | 107 ++++++++++++------ .../image_gen_tests/test_image_generation.py | 18 ++- 2 files changed, 84 insertions(+), 41 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 234ca1a1d437..3dbb9aad7db9 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -473,8 +473,8 @@ def completion_cost( # noqa: PLR0915 region_name=None, # used for bedrock pricing ### IMAGE GEN ### size: Optional[str] = None, - quality=None, - n=None, # number of images + quality: Optional[str] = None, + n: Optional[int] = None, # number of images ### CUSTOM PRICING ### custom_cost_per_token: Optional[CostPerToken] = None, custom_cost_per_second: Optional[float] = None, @@ -640,41 +640,14 @@ def completion_cost( # noqa: PLR0915 raise TypeError( "completion_response must be of type ImageResponse for bedrock image cost calculation" ) - if size is None: - size = "1024-x-1024" # openai default - # fix size to match naming convention - if "x" in size and "-x-" not in size: - size = size.replace("x", "-x-") - image_gen_model_name = f"{size}/{model}" - image_gen_model_name_with_quality = image_gen_model_name - if quality is not None: - image_gen_model_name_with_quality = f"{quality}/{image_gen_model_name}" - size_parts = size.split("-x-") - height = int(size_parts[0]) # if it's 1024-x-1024 vs. 1024x1024 - width = int(size_parts[1]) - verbose_logger.debug(f"image_gen_model_name: {image_gen_model_name}") - verbose_logger.debug( - f"image_gen_model_name_with_quality: {image_gen_model_name_with_quality}" - ) - if image_gen_model_name in litellm.model_cost: - return ( - litellm.model_cost[image_gen_model_name]["input_cost_per_pixel"] - * height - * width - * n - ) - elif image_gen_model_name_with_quality in litellm.model_cost: - return ( - litellm.model_cost[image_gen_model_name_with_quality][ - "input_cost_per_pixel" - ] - * height - * width - * n - ) else: - raise Exception( - f"Model={image_gen_model_name} not found in completion cost model map" + return default_image_cost_calculator( + model=model, + quality=quality, + custom_llm_provider=custom_llm_provider, + n=n, + size=size, + optional_params=optional_params, ) elif ( call_type == CallTypes.speech.value or call_type == CallTypes.aspeech.value @@ -869,3 +842,65 @@ def transcription_cost( return openai_cost_per_second( model=model, custom_llm_provider=custom_llm_provider, duration=duration ) + + +def default_image_cost_calculator( + model: str, + custom_llm_provider: Optional[str] = None, + quality: Optional[str] = None, + n: Optional[int] = 1, # Default to 1 image + size: Optional[str] = "1024-x-1024", # OpenAI default + optional_params: Optional[dict] = None, +) -> float: + """ + Default image cost calculator for image generation + + Args: + model (str): Model name + image_response (ImageResponse): Response from image generation + quality (Optional[str]): Image quality setting + n (Optional[int]): Number of images generated + size (Optional[str]): Image size (e.g. "1024x1024" or "1024-x-1024") + + Returns: + float: Cost in USD for the image generation + + Raises: + Exception: If model pricing not found in cost map + """ + # Standardize size format to use "-x-" + size_str: str = size or "1024-x-1024" + size_str = ( + size_str.replace("x", "-x-") + if "x" in size_str and "-x-" not in size_str + else size_str + ) + + # Parse dimensions + height, width = map(int, size_str.split("-x-")) + + # Build model names for cost lookup + base_model_name = f"{size_str}/{model}" + if custom_llm_provider and model.startswith(custom_llm_provider): + base_model_name = ( + f"{custom_llm_provider}/{size_str}/{model.replace(custom_llm_provider, '')}" + ) + model_name_with_quality = ( + f"{quality}/{base_model_name}" if quality else base_model_name + ) + + verbose_logger.debug( + f"Looking up cost for models: {model_name_with_quality}, {base_model_name}" + ) + + # Try model with quality first, fall back to base model name + if model_name_with_quality in litellm.model_cost: + cost_info = litellm.model_cost[model_name_with_quality] + elif base_model_name in litellm.model_cost: + cost_info = litellm.model_cost[base_model_name] + else: + raise Exception( + f"Model not found in cost map. Tried {model_name_with_quality} and {base_model_name}" + ) + + return cost_info["input_cost_per_pixel"] * height * width * n diff --git a/tests/image_gen_tests/test_image_generation.py b/tests/image_gen_tests/test_image_generation.py index 6605b3e3d443..bad6e4b72fcd 100644 --- a/tests/image_gen_tests/test_image_generation.py +++ b/tests/image_gen_tests/test_image_generation.py @@ -6,6 +6,11 @@ import sys import traceback + +sys.path.insert( + 0, os.path.abspath("../..") +) # Adds the parent directory to the system path + from dotenv import load_dotenv from openai.types.image import Image from litellm.caching import InMemoryCache @@ -14,10 +19,6 @@ load_dotenv() import asyncio import os - -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import pytest import litellm @@ -142,7 +143,7 @@ def get_base_image_generation_call_args(self) -> dict: "api_version": "2023-09-01-preview", "metadata": { "model_info": { - "base_model": "dall-e-3", + "base_model": "azure/dall-e-3", } }, } @@ -158,8 +159,15 @@ def test_image_generation_azure_dall_e_3(): api_version="2023-12-01-preview", api_base=os.getenv("AZURE_SWEDEN_API_BASE"), api_key=os.getenv("AZURE_SWEDEN_API_KEY"), + metadata={ + "model_info": { + "base_model": "azure/dall-e-3", + } + }, ) print(f"response: {response}") + + print("response", response._hidden_params) assert len(response.data) > 0 except litellm.InternalServerError as e: pass From e00dc3a6ffedb15e0f77c0a0f8a395ea2e9398d9 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 21 Dec 2024 18:53:28 -0800 Subject: [PATCH 3/5] update test name --- tests/local_testing/test_amazing_vertex_completion.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/local_testing/test_amazing_vertex_completion.py b/tests/local_testing/test_amazing_vertex_completion.py index bb7b38addc91..519237ddbc31 100644 --- a/tests/local_testing/test_amazing_vertex_completion.py +++ b/tests/local_testing/test_amazing_vertex_completion.py @@ -343,7 +343,7 @@ async def test_aaavertex_ai_anthropic_async_streaming(): @pytest.mark.flaky(retries=3, delay=1) -def test_vertex_ai(): +def test_aavertex_ai(): import random litellm.num_retries = 3 @@ -394,7 +394,7 @@ def test_vertex_ai(): @pytest.mark.flaky(retries=3, delay=1) -def test_vertex_ai_stream(): +def test_aavertex_ai_stream(): load_vertex_ai_credentials() litellm.set_verbose = True litellm.vertex_project = "adroit-crow-413218" From 7f8897947890f283ac31e7006d1fdfb026cd90be Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 21 Dec 2024 19:00:04 -0800 Subject: [PATCH 4/5] run ci/cd again --- tests/local_testing/test_completion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/local_testing/test_completion.py b/tests/local_testing/test_completion.py index 5902047b70ca..e819bd80f01e 100644 --- a/tests/local_testing/test_completion.py +++ b/tests/local_testing/test_completion.py @@ -3097,7 +3097,7 @@ def test_completion_azure_with_litellm_key(): # Add any assertions here to check the response print(response) - ######### RESET ENV VARs for this ################ + ######### RESET ENV VARS for this ################ os.environ["AZURE_API_BASE"] = litellm.api_base os.environ["AZURE_API_VERSION"] = litellm.api_version os.environ["AZURE_API_KEY"] = litellm.api_key From 1dc58f3845e48dc7105bfde8da98afa46f68b632 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 20 Feb 2025 18:30:23 -0800 Subject: [PATCH 5/5] (Infra/DB) - Allow running older litellm version when out of sync with current state of DB (#8695) * fix check migration * clean up should_update_prisma_schema * update test * db_migration_disable_update_check * Check container logs for expected message * db_migration_disable_update_check * test_check_migration_out_of_sync * test_should_update_prisma_schema * db_migration_disable_update_check * pip install aiohttp --- .circleci/config.yml | 56 +++++++++++++++---- litellm/proxy/db/check_migration.py | 7 ++- litellm/proxy/db/prisma_client.py | 32 ++++++++--- .../disable_schema_update.yaml | 5 ++ litellm/proxy/proxy_cli.py | 4 +- .../litellm/proxy/db/test_check_migration.py | 51 +++++++++++++++++ tests/litellm/proxy/db/test_prisma_client.py | 35 ++++++++++++ 7 files changed, 168 insertions(+), 22 deletions(-) create mode 100644 tests/litellm/proxy/db/test_check_migration.py create mode 100644 tests/litellm/proxy/db/test_prisma_client.py diff --git a/.circleci/config.yml b/.circleci/config.yml index cb887077de9f..bade03f0058e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -853,6 +853,23 @@ jobs: working_directory: ~/project steps: - checkout + - run: + name: Install Python 3.9 + command: | + curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh + bash miniconda.sh -b -p $HOME/miniconda + export PATH="$HOME/miniconda/bin:$PATH" + conda init bash + source ~/.bashrc + conda create -n myenv python=3.9 -y + conda activate myenv + python --version + - run: + name: Install Dependencies + command: | + pip install "pytest==7.3.1" + pip install "pytest-asyncio==0.21.1" + pip install aiohttp - run: name: Build Docker image command: | @@ -860,30 +877,49 @@ jobs: - run: name: Run Docker container command: | - docker run --name my-app \ + docker run -d \ -p 4000:4000 \ -e DATABASE_URL=$PROXY_DATABASE_URL \ -e DISABLE_SCHEMA_UPDATE="True" \ -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/schema.prisma \ -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/litellm/proxy/schema.prisma \ -v $(pwd)/litellm/proxy/example_config_yaml/disable_schema_update.yaml:/app/config.yaml \ + --name my-app \ myapp:latest \ --config /app/config.yaml \ - --port 4000 > docker_output.log 2>&1 || true + --port 4000 - run: - name: Display Docker logs - command: cat docker_output.log + name: Install curl and dockerize + command: | + sudo apt-get update + sudo apt-get install -y curl + sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + sudo rm dockerize-linux-amd64-v0.6.1.tar.gz + - run: - name: Check for expected error + name: Wait for container to be ready + command: dockerize -wait http://localhost:4000 -timeout 1m + - run: + name: Check container logs for expected message command: | - if grep -q "prisma schema out of sync with db. Consider running these sql_commands to sync the two" docker_output.log; then - echo "Expected error found. Test passed." + echo "=== Printing Full Container Startup Logs ===" + docker logs my-app + echo "=== End of Full Container Startup Logs ===" + + if docker logs my-app 2>&1 | grep -q "prisma schema out of sync with db. Consider running these sql_commands to sync the two"; then + echo "Expected message found in logs. Test passed." else - echo "Expected error not found. Test failed." - cat docker_output.log + echo "Expected message not found in logs. Test failed." exit 1 fi - + - run: + name: Run Basic Proxy Startup Tests (Health Readiness and Chat Completion) + command: | + python -m pytest -vv tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5 + no_output_timeout: 120m + + build_and_test: machine: image: ubuntu-2204:2023.10.1 diff --git a/litellm/proxy/db/check_migration.py b/litellm/proxy/db/check_migration.py index ecb503db8ff5..bf180c1132d4 100644 --- a/litellm/proxy/db/check_migration.py +++ b/litellm/proxy/db/check_migration.py @@ -4,6 +4,8 @@ import subprocess from typing import List, Optional, Tuple +from litellm._logging import verbose_logger + def extract_sql_commands(diff_output: str) -> List[str]: """ @@ -52,6 +54,7 @@ def check_prisma_schema_diff_helper(db_url: str) -> Tuple[bool, List[str]]: subprocess.CalledProcessError: If the Prisma command fails. Exception: For any other errors during execution. """ + verbose_logger.debug("Checking for Prisma schema diff...") # noqa: T201 try: result = subprocess.run( [ @@ -94,8 +97,8 @@ def check_prisma_schema_diff(db_url: Optional[str] = None) -> None: raise Exception("DATABASE_URL not set") has_diff, message = check_prisma_schema_diff_helper(db_url) if has_diff: - raise Exception( - "prisma schema out of sync with db. Consider running these sql_commands to sync the two - {}".format( + verbose_logger.exception( + "🚨🚨🚨 prisma schema out of sync with db. Consider running these sql_commands to sync the two - {}".format( message ) ) diff --git a/litellm/proxy/db/prisma_client.py b/litellm/proxy/db/prisma_client.py index 54d59bd34700..f8bb6b09fd9b 100644 --- a/litellm/proxy/db/prisma_client.py +++ b/litellm/proxy/db/prisma_client.py @@ -7,7 +7,7 @@ import urllib import urllib.parse from datetime import datetime, timedelta -from typing import Any, Optional +from typing import Any, Optional, Union from litellm.secret_managers.main import str_to_bool @@ -112,12 +112,28 @@ def __getattr__(self, name: str): return original_attr -def should_update_schema(disable_prisma_schema_update: Optional[bool]): +def should_update_prisma_schema( + disable_updates: Optional[Union[bool, str]] = None +) -> bool: """ - This function is used to determine if the Prisma schema should be updated. + Determines if Prisma Schema updates should be applied during startup. + + Args: + disable_updates: Controls whether schema updates are disabled. + Accepts boolean or string ('true'/'false'). Defaults to checking DISABLE_SCHEMA_UPDATE env var. + + Returns: + bool: True if schema updates should be applied, False if updates are disabled. + + Examples: + >>> should_update_prisma_schema() # Checks DISABLE_SCHEMA_UPDATE env var + >>> should_update_prisma_schema(True) # Explicitly disable updates + >>> should_update_prisma_schema("false") # Enable updates using string """ - if disable_prisma_schema_update is None: - disable_prisma_schema_update = str_to_bool(os.getenv("DISABLE_SCHEMA_UPDATE")) - if disable_prisma_schema_update is True: - return False - return True + if disable_updates is None: + disable_updates = os.getenv("DISABLE_SCHEMA_UPDATE", "false") + + if isinstance(disable_updates, str): + disable_updates = str_to_bool(disable_updates) + + return not bool(disable_updates) diff --git a/litellm/proxy/example_config_yaml/disable_schema_update.yaml b/litellm/proxy/example_config_yaml/disable_schema_update.yaml index cc56b9516f2a..5dcbd0dbd575 100644 --- a/litellm/proxy/example_config_yaml/disable_schema_update.yaml +++ b/litellm/proxy/example_config_yaml/disable_schema_update.yaml @@ -4,6 +4,11 @@ model_list: model: openai/fake api_key: fake-key api_base: https://exampleopenaiendpoint-production.up.railway.app/ + - model_name: gpt-4 + litellm_params: + model: openai/gpt-4 + api_key: fake-key + api_base: https://exampleopenaiendpoint-production.up.railway.app/ litellm_settings: callbacks: ["gcs_bucket"] diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 5c4b04fb701a..431e03ed15a4 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -653,10 +653,10 @@ def _make_openai_completion(): if is_prisma_runnable: from litellm.proxy.db.check_migration import check_prisma_schema_diff - from litellm.proxy.db.prisma_client import should_update_schema + from litellm.proxy.db.prisma_client import should_update_prisma_schema if ( - should_update_schema( + should_update_prisma_schema( general_settings.get("disable_prisma_schema_update") ) is False diff --git a/tests/litellm/proxy/db/test_check_migration.py b/tests/litellm/proxy/db/test_check_migration.py new file mode 100644 index 000000000000..ad72a0d1195c --- /dev/null +++ b/tests/litellm/proxy/db/test_check_migration.py @@ -0,0 +1,51 @@ +import json +import os +import sys + +import pytest +from fastapi.testclient import TestClient + +sys.path.insert( + 0, os.path.abspath("../../../..") +) # Adds the parent directory to the system path + + +import json +import os +import sys +import time + +import pytest +from fastapi.testclient import TestClient + +import litellm + + +def test_check_migration_out_of_sync(mocker): + """ + Test that the check_prisma_schema_diff function + - 🚨 [IMPORTANT] Does NOT Raise an Exception when the Prisma schema is out of sync with the database. + - logs an error when the Prisma schema is out of sync with the database. + """ + # Mock the logger BEFORE importing the function + mock_logger = mocker.patch("litellm._logging.verbose_logger") + + # Import the function after mocking the logger + from litellm.proxy.db.check_migration import check_prisma_schema_diff + + # Mock the helper function to simulate out-of-sync state + mock_diff_helper = mocker.patch( + "litellm.proxy.db.check_migration.check_prisma_schema_diff_helper", + return_value=(True, ["ALTER TABLE users ADD COLUMN new_field TEXT;"]), + ) + + # Run the function - it should not raise an error + try: + check_prisma_schema_diff(db_url="mock_url") + except Exception as e: + pytest.fail(f"check_prisma_schema_diff raised an unexpected exception: {e}") + + # Verify the logger was called with the expected message + mock_logger.exception.assert_called_once() + actual_message = mock_logger.exception.call_args[0][0] + assert "prisma schema out of sync with db" in actual_message diff --git a/tests/litellm/proxy/db/test_prisma_client.py b/tests/litellm/proxy/db/test_prisma_client.py new file mode 100644 index 000000000000..c7e99aa75406 --- /dev/null +++ b/tests/litellm/proxy/db/test_prisma_client.py @@ -0,0 +1,35 @@ +import json +import os +import sys + +import pytest +from fastapi.testclient import TestClient + +sys.path.insert( + 0, os.path.abspath("../../../..") +) # Adds the parent directory to the system path + + +from litellm.proxy.db.prisma_client import should_update_prisma_schema + + +def test_should_update_prisma_schema(monkeypatch): + # CASE 1: Environment variable behavior + # When DISABLE_SCHEMA_UPDATE is not set -> should update + monkeypatch.setenv("DISABLE_SCHEMA_UPDATE", None) + assert should_update_prisma_schema() == True + + # When DISABLE_SCHEMA_UPDATE="true" -> should not update + monkeypatch.setenv("DISABLE_SCHEMA_UPDATE", "true") + assert should_update_prisma_schema() == False + + # When DISABLE_SCHEMA_UPDATE="false" -> should update + monkeypatch.setenv("DISABLE_SCHEMA_UPDATE", "false") + assert should_update_prisma_schema() == True + + # CASE 2: Explicit parameter behavior (overrides env var) + monkeypatch.setenv("DISABLE_SCHEMA_UPDATE", None) + assert should_update_prisma_schema(True) == False # Param True -> should not update + + monkeypatch.setenv("DISABLE_SCHEMA_UPDATE", None) # Set env var opposite to param + assert should_update_prisma_schema(False) == True # Param False -> should update