diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index f38883bb1791..28d4fca6fa9e 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -2335,4 +2335,4 @@ def _add_key_name_and_team_to_alert(request_info: str, metadata: dict) -> str: return request_info except Exception: - return request_info + return request_info \ No newline at end of file diff --git a/tests/local_testing/test_exceptions.py b/tests/local_testing/test_exceptions.py index 6851325cc1fc..67506f27692b 100644 --- a/tests/local_testing/test_exceptions.py +++ b/tests/local_testing/test_exceptions.py @@ -252,30 +252,7 @@ def test_invalid_request_error(model): completion(model=model, messages=messages, max_tokens="hello world") -def test_completion_azure_exception(): - try: - import openai - print("azure gpt-3.5 test\n\n") - litellm.set_verbose = True - ## Test azure call - old_azure_key = os.environ["AZURE_API_KEY"] - os.environ["AZURE_API_KEY"] = "good morning" - response = completion( - model="azure/chatgpt-v-3", - messages=[{"role": "user", "content": "hello"}], - ) - os.environ["AZURE_API_KEY"] = old_azure_key - print(f"response: {response}") - print(response) - except openai.AuthenticationError as e: - os.environ["AZURE_API_KEY"] = old_azure_key - print("good job got the correct error for azure when key not set") - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -# test_completion_azure_exception() def test_azure_embedding_exceptions(): @@ -414,31 +391,7 @@ def test_completion_openai_exception(): # test_completion_openai_exception() -def test_anthropic_openai_exception(): - # test if anthropic raises litellm.AuthenticationError - try: - litellm.set_verbose = True - ## Test azure call - old_azure_key = os.environ["ANTHROPIC_API_KEY"] - os.environ.pop("ANTHROPIC_API_KEY") - response = completion( - model="anthropic/claude-3-sonnet-20240229", - messages=[{"role": "user", "content": "hello"}], - ) - print(f"response: {response}") - print(response) - except litellm.AuthenticationError as e: - os.environ["ANTHROPIC_API_KEY"] = old_azure_key - print("Exception vars=", vars(e)) - assert ( - "Missing Anthropic API Key - A call is being made to anthropic but no key is set either in the environment variables or via params" - in e.message - ) - print( - "ANTHROPIC_API_KEY: good job got the correct error for ANTHROPIC_API_KEY when key not set" - ) - except Exception as e: - pytest.fail(f"Error occurred: {e}") + def test_completion_mistral_exception(): diff --git a/tests/test_litellm/litellm_core_utils/test_exceptions.py b/tests/test_litellm/litellm_core_utils/test_exceptions.py new file mode 100644 index 000000000000..3540a125d28d --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/test_exceptions.py @@ -0,0 +1,99 @@ +import os +import sys + +sys.path.insert( + 0, os.path.abspath("../..") +) # Adds the parent directory to the system path + +import pytest + +def test_fireworks_ai_exception_mapping(): + """ + Comprehensive test for Fireworks AI exception mapping, including: + 1. Standard 429 rate limit errors + 2. Text-based rate limit detection (the main issue fixed) + 3. Generic 400 errors that should NOT be rate limits + 4. ExceptionCheckers utility function + + Related to: https://github.com/BerriAI/litellm/pull/11455 + Based on Fireworks AI documentation: https://docs.fireworks.ai/tools-sdks/python-client/api-reference + """ + import litellm + from litellm.llms.fireworks_ai.common_utils import FireworksAIException + from litellm.litellm_core_utils.exception_mapping_utils import ExceptionCheckers + + # Test scenarios covering all important cases + test_scenarios = [ + { + "name": "Standard 429 rate limit with proper status code", + "status_code": 429, + "message": "Rate limit exceeded. Please try again in 60 seconds.", + "expected_exception": litellm.RateLimitError, + }, + { + "name": "Status 400 with rate limit text (the main issue fixed)", + "status_code": 400, + "message": '{"error":{"object":"error","type":"invalid_request_error","message":"rate limit exceeded, please try again later"}}', + "expected_exception": litellm.RateLimitError, + }, + { + "name": "Status 400 with generic invalid request (should NOT be rate limit)", + "status_code": 400, + "message": '{"error":{"type":"invalid_request_error","message":"Invalid parameter value"}}', + "expected_exception": litellm.BadRequestError, + }, + ] + + # Test each scenario + for scenario in test_scenarios: + mock_exception = FireworksAIException( + status_code=scenario["status_code"], + message=scenario["message"], + headers={} + ) + + try: + response = litellm.completion( + model="fireworks_ai/llama-v3p1-70b-instruct", + messages=[{"role": "user", "content": "Hello"}], + mock_response=mock_exception, + ) + pytest.fail(f"Expected {scenario['expected_exception'].__name__} to be raised") + except scenario["expected_exception"] as e: + if scenario["expected_exception"] == litellm.RateLimitError: + assert "rate limit" in str(e).lower() or "429" in str(e) + except Exception as e: + pytest.fail(f"Expected {scenario['expected_exception'].__name__} but got {type(e).__name__}: {e}") + + # Test ExceptionCheckers.is_error_str_rate_limit() method directly + + # Test cases that should return True (rate limit detected) + rate_limit_strings = [ + "429 rate limit exceeded", + "Rate limit exceeded, please try again later", + "RATE LIMIT ERROR", + "Error 429: rate limit", + '{"error":{"type":"invalid_request_error","message":"rate limit exceeded, please try again later"}}', + "HTTP 429 Too Many Requests", + ] + + for error_str in rate_limit_strings: + assert ExceptionCheckers.is_error_str_rate_limit(error_str), f"Should detect rate limit in: {error_str}" + + # Test cases that should return False (not rate limit) + non_rate_limit_strings = [ + "400 Bad Request", + "Authentication failed", + "Invalid model specified", + "Context window exceeded", + "Internal server error", + "", + "Some other error message", + ] + + for error_str in non_rate_limit_strings: + assert not ExceptionCheckers.is_error_str_rate_limit(error_str), f"Should NOT detect rate limit in: {error_str}" + + # Test edge cases + assert not ExceptionCheckers.is_error_str_rate_limit(None) # type: ignore + assert not ExceptionCheckers.is_error_str_rate_limit(42) # type: ignore \ No newline at end of file