From 914c87c387a8e48aa0d82b55d593f8fc8855ca73 Mon Sep 17 00:00:00 2001 From: David Tam Date: Tue, 9 Jul 2024 14:59:19 -0700 Subject: [PATCH 1/4] add handling for validation errors to return 400 and message --- guardrails_api/utils/handle_error.py | 5 +++ tests/blueprints/test_guards.py | 66 ++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/guardrails_api/utils/handle_error.py b/guardrails_api/utils/handle_error.py index 403d48f..6dc0ad2 100644 --- a/guardrails_api/utils/handle_error.py +++ b/guardrails_api/utils/handle_error.py @@ -3,6 +3,7 @@ from werkzeug.exceptions import HTTPException from guardrails_api.classes.http_error import HttpError from guardrails_api.utils.logger import logger +from guardrails.errors import ValidationError def handle_error(fn): @@ -10,6 +11,10 @@ def handle_error(fn): def decorator(*args, **kwargs): try: return fn(*args, **kwargs) + except ValidationError as validation_error: + logger.error(validation_error) + traceback.print_exception(type(validation_error), validation_error, validation_error.__traceback__) + return str(validation_error), 400 except HttpError as http_error: logger.error(http_error) traceback.print_exception(http_error) diff --git a/tests/blueprints/test_guards.py b/tests/blueprints/test_guards.py index 1b9e927..9c1fc93 100644 --- a/tests/blueprints/test_guards.py +++ b/tests/blueprints/test_guards.py @@ -11,6 +11,7 @@ from guardrails.classes.generic import Stack from guardrails.classes.history import Call, Iteration from guardrails_api.app import register_config +from guardrails.errors import ValidationError # TODO: Should we mock this somehow? # Right now it's just empty, but it technically does a file read @@ -548,6 +549,71 @@ def test_validate__call(mocker): del os.environ["PGHOST"] +def test_validate__call_throws_validation_error(mocker): + os.environ["PGHOST"] = "localhost" + + mock___call__ = mocker.patch.object(MockGuardStruct, "__call__") + mock___call__.side_effect = ValidationError("Test guard validation error") + + mock_guard = MockGuardStruct() + mock_from_dict = mocker.patch("guardrails_api.blueprints.guards.Guard.from_dict") + mock_from_dict.return_value = mock_guard + + # mock_tracer = MockTracer() + mock_request = MockRequest( + "POST", + json={ + "llmApi": "openai.Completion.create", + "promptParams": {"p1": "bar"}, + "args": [1, 2, 3], + "some_kwarg": "foo", + "prompt": "Hello world!", + }, + headers={"x-openai-api-key": "mock-key"}, + ) + + mocker.patch("flask.Blueprint", new=MockBlueprint) + mocker.patch("guardrails_api.blueprints.guards.request", mock_request) + mock_get_guard = mocker.patch( + "guardrails_api.blueprints.guards.guard_client.get_guard", + return_value=mock_guard, + ) + mocker.patch( + "guardrails_api.blueprints.guards.get_llm_callable", + return_value="openai.Completion.create", + ) + + mocker.patch("guardrails_api.blueprints.guards.CacheClient.set") + + mock_status = mocker.patch( + "guardrails.classes.history.call.Call.status", new_callable=PropertyMock + ) + mock_status.return_value = "fail" + mock_guard.history = Stack(Call()) + from guardrails_api.blueprints.guards import validate + + response = validate("My%20Guard's%20Name") + + mock_get_guard.assert_called_once_with("My Guard's Name") + + assert mock___call__.call_count == 1 + + mock___call__.assert_called_once_with( + 1, + 2, + 3, + llm_api="openai.Completion.create", + prompt_params={"p1": "bar"}, + num_reasks=None, + some_kwarg="foo", + api_key="mock-key", + prompt="Hello world!", + ) + + assert response == ('Test guard validation error', 400) + + del os.environ["PGHOST"] + def test_openai_v1_chat_completions__call(mocker): from guardrails_api.blueprints.guards import openai_v1_chat_completions os.environ["PGHOST"] = "localhost" From eb05c63f5a73bf2821c9e9b0d636f9a7226cabb1 Mon Sep 17 00:00:00 2001 From: David Tam Date: Tue, 9 Jul 2024 15:10:39 -0700 Subject: [PATCH 2/4] remove unecessary openai error handling code now we have the base handler --- guardrails_api/blueprints/guards.py | 33 ++++++++++++----------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/guardrails_api/blueprints/guards.py b/guardrails_api/blueprints/guards.py index d84e08c..328fa46 100644 --- a/guardrails_api/blueprints/guards.py +++ b/guardrails_api/blueprints/guards.py @@ -191,26 +191,19 @@ def openai_v1_chat_completions(guard_name: str): pass if not stream: - try: - validation_outcome: ValidationOutcome = guard( - # todo make this come from the guard struct? - # currently we dont support .configure - num_reasks=0, - **payload, - ) - llm_response = guard.history[-1].iterations[-1].outputs.llm_response_info - result = outcome_to_chat_completion( - validation_outcome=validation_outcome, - llm_response=llm_response, - has_tool_gd_tool_call=has_tool_gd_tool_call, - ) - return result - except Exception as e: - raise HttpError( - status=400, - message="BadRequest", - cause=(str(e)), - ) + validation_outcome: ValidationOutcome = guard( + # todo make this come from the guard struct? + # currently we dont support .configure + num_reasks=0, + **payload, + ) + llm_response = guard.history[-1].iterations[-1].outputs.llm_response_info + result = outcome_to_chat_completion( + validation_outcome=validation_outcome, + llm_response=llm_response, + has_tool_gd_tool_call=has_tool_gd_tool_call, + ) + return result else: # need to return validated chunks that look identical to openai's From e2c2e4657550561aa1451b90dbc120aafa4f9c5b Mon Sep 17 00:00:00 2001 From: David Tam Date: Wed, 10 Jul 2024 17:04:13 -0700 Subject: [PATCH 3/4] make history actually stacks --- guardrails_api/blueprints/guards.py | 2 +- tests/blueprints/test_guards.py | 4 ++-- tests/mocks/mock_guard_client.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/guardrails_api/blueprints/guards.py b/guardrails_api/blueprints/guards.py index 328fa46..4c8c115 100644 --- a/guardrails_api/blueprints/guards.py +++ b/guardrails_api/blueprints/guards.py @@ -197,7 +197,7 @@ def openai_v1_chat_completions(guard_name: str): num_reasks=0, **payload, ) - llm_response = guard.history[-1].iterations[-1].outputs.llm_response_info + llm_response = guard.history.last.iterations.last.outputs.llm_response_info result = outcome_to_chat_completion( validation_outcome=validation_outcome, llm_response=llm_response, diff --git a/tests/blueprints/test_guards.py b/tests/blueprints/test_guards.py index 9c1fc93..768017c 100644 --- a/tests/blueprints/test_guards.py +++ b/tests/blueprints/test_guards.py @@ -22,7 +22,7 @@ "id": "mock-guard-id", "name": "mock-guard", "description": "mock guard description", - "history": [], + "history": Stack(), } @@ -185,7 +185,7 @@ def test_guard__put_pg(mocker): "name": "mock-guard", "id": "mock-guard-id", "description": "mock guard description", - "history": [], + "history": Stack(), } mock_request = MockRequest("PUT", json=json_guard) diff --git a/tests/mocks/mock_guard_client.py b/tests/mocks/mock_guard_client.py index 3f90c40..8ec3ee3 100644 --- a/tests/mocks/mock_guard_client.py +++ b/tests/mocks/mock_guard_client.py @@ -1,7 +1,7 @@ from typing import Any, List from guardrails_api_client import Guard as GuardStruct from pydantic import ConfigDict - +from guardrails.classes.generic import Stack class MockGuardStruct(GuardStruct): # Pydantic Config @@ -10,7 +10,7 @@ class MockGuardStruct(GuardStruct): id: str = "mock-guard-id" name: str = "mock-guard" description: str = "mock guard description" - history: List[Any] = [] + history: Stack[Any] = Stack() def to_guard(self, *args): return self From 1ddc1d0f412c689d81a6417cd25ef27732d22cf7 Mon Sep 17 00:00:00 2001 From: David Tam Date: Wed, 10 Jul 2024 17:07:56 -0700 Subject: [PATCH 4/4] lint --- tests/mocks/mock_guard_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/mocks/mock_guard_client.py b/tests/mocks/mock_guard_client.py index 8ec3ee3..04bb77f 100644 --- a/tests/mocks/mock_guard_client.py +++ b/tests/mocks/mock_guard_client.py @@ -1,4 +1,4 @@ -from typing import Any, List +from typing import Any from guardrails_api_client import Guard as GuardStruct from pydantic import ConfigDict from guardrails.classes.generic import Stack