diff --git a/Makefile b/Makefile index 2802dd935..6fb50b3e2 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,5 @@ MKDOCS_SERVE_ADDR ?= localhost:8000 # Default address for mkdocs serve, format: :, override with `make docs-serve MKDOCS_SERVE_ADDR=:` -# Extract major package versions for OpenAI and Pydantic -OPENAI_VERSION_MAJOR := $(shell poetry run python -c 'import openai; print(openai.__version__.split(".")[0])') -PYDANTIC_VERSION_MAJOR := $(shell poetry run python -c 'import pydantic; print(pydantic.__version__.split(".")[0])') - -# Construct the typing command using only major versions -TYPING_CMD := type-pydantic-v$(PYDANTIC_VERSION_MAJOR)-openai-v$(OPENAI_VERSION_MAJOR) - autoformat: poetry run ruff check guardrails/ tests/ --fix poetry run ruff format guardrails/ tests/ @@ -14,27 +7,7 @@ autoformat: .PHONY: type type: - @make $(TYPING_CMD) - -type-pydantic-v1-openai-v0: - echo '{"reportDeprecated": true, "exclude": ["guardrails/utils/pydantic_utils/v2.py", "guardrails/utils/openai_utils/v1.py"]}' > pyrightconfig.json - poetry run pyright guardrails/ - rm pyrightconfig.json - -type-pydantic-v1-openai-v1: - echo '{"reportDeprecated": true, "exclude": ["guardrails/utils/pydantic_utils/v2.py", "guardrails/utils/openai_utils/v0.py"]}' > pyrightconfig.json - poetry run pyright guardrails/ - rm pyrightconfig.json - -type-pydantic-v2-openai-v0: - echo '{"reportDeprecated": true, "exclude": ["guardrails/utils/pydantic_utils/v1.py", "guardrails/utils/openai_utils/v1.py"]}' > pyrightconfig.json - poetry run pyright guardrails/ - rm pyrightconfig.json - -type-pydantic-v2-openai-v1: - echo '{"reportDeprecated": true, "exclude": ["guardrails/utils/pydantic_utils/v1.py", "guardrails/utils/openai_utils/v0.py"]}' > pyrightconfig.json poetry run pyright guardrails/ - rm pyrightconfig.json lint: poetry run ruff check guardrails/ tests/ diff --git a/guardrails/actions/reask.py b/guardrails/actions/reask.py index b657c7c6c..3ffa8b0ec 100644 --- a/guardrails/actions/reask.py +++ b/guardrails/actions/reask.py @@ -20,6 +20,44 @@ class ReAsk(IReask): incorrect_value: Any fail_results: List[FailResult] + @classmethod + def from_interface(cls, reask: IReask) -> "ReAsk": + fail_results = [] + if reask.fail_results: + fail_results: List[FailResult] = [ + FailResult.from_interface(fail_result) + for fail_result in reask.fail_results + ] + + if reask.additional_properties.get("path"): + return FieldReAsk( + incorrect_value=reask.incorrect_value, + fail_results=fail_results, + path=reask.additional_properties["path"], + ) + + if len(fail_results) == 1: + error_message = fail_results[0].error_message + if error_message == "Output is not parseable as JSON": + return NonParseableReAsk( + incorrect_value=reask.incorrect_value, + fail_results=fail_results, + ) + elif "JSON does not match schema" in error_message: + return SkeletonReAsk( + incorrect_value=reask.incorrect_value, + fail_results=fail_results, + ) + + return cls(incorrect_value=reask.incorrect_value, fail_results=fail_results) + + @classmethod + def from_dict(cls, obj: Dict[str, Any]) -> Optional["ReAsk"]: + i_reask = super().from_dict(obj) + if not i_reask: + return None + return cls.from_interface(i_reask) + class FieldReAsk(ReAsk): # FIXME: This shouldn't be optional @@ -363,7 +401,7 @@ def get_reask_setup_for_json( def reask_decoder(obj: ReAsk): decoded = {} for k, v in obj.__dict__.items(): - if k in ["path"]: + if k in ["path", "additional_properties"]: continue if k == "fail_results": k = "error_messages" diff --git a/guardrails/api_client.py b/guardrails/api_client.py index 8b81c494e..84eb4df66 100644 --- a/guardrails/api_client.py +++ b/guardrails/api_client.py @@ -9,6 +9,8 @@ from guardrails_api_client.api.validate_api import ValidateApi from guardrails_api_client.models import Guard, ValidatePayload +from guardrails.logger import logger + class GuardrailsApiClient: _api_client: ApiClient @@ -39,6 +41,13 @@ def upsert_guard(self, guard: Guard): guard_name=guard.name, body=guard, _request_timeout=self.timeout ) + def fetch_guard(self, guard_name: str) -> Optional[Guard]: + try: + return self._guard_api.get_guard(guard_name=guard_name) + except Exception as e: + logger.debug(f"Error fetching guard {guard_name}: {e}") + return None + def validate( self, guard: Guard, @@ -86,3 +95,6 @@ def stream_validate( if line: json_output = json.loads(line) yield json_output + + def get_history(self, guard_name: str, call_id: str): + return self._guard_api.get_guard_history(guard_name, call_id) diff --git a/guardrails/async_guard.py b/guardrails/async_guard.py index 8476ab694..fca49a948 100644 --- a/guardrails/async_guard.py +++ b/guardrails/async_guard.py @@ -15,7 +15,10 @@ cast, ) -from guardrails_api_client.models import ValidatePayload +from guardrails_api_client.models import ( + ValidatePayload, + ValidationOutcome as IValidationOutcome, +) from guardrails import Guard from guardrails.classes import OT, ValidationOutcome @@ -273,9 +276,6 @@ async def __exec( args=list(args), kwargs=kwargs, ) - call_log = Call(inputs=call_inputs) - set_scope(str(object_id(call_log))) - self._history.push(call_log) if self._api_client is not None and model_is_supported_server_side( llm_api, *args, **kwargs @@ -287,13 +287,15 @@ async def __exec( prompt_params=prompt_params, metadata=metadata, full_schema_reask=full_schema_reask, - call_log=call_log, *args, **kwargs, ) # If the LLM API is async, return a coroutine else: + call_log = Call(inputs=call_inputs) + set_scope(str(object_id(call_log))) + self.history.push(call_log) result = await self._exec( llm_api=llm_api, llm_output=llm_output, @@ -538,20 +540,12 @@ async def parse( ) async def _stream_server_call( - self, - *, - payload: Dict[str, Any], - llm_output: Optional[str] = None, - num_reasks: Optional[int] = None, - prompt_params: Optional[Dict] = None, - metadata: Optional[Dict] = {}, - full_schema_reask: Optional[bool] = True, - call_log: Optional[Call], + self, *, payload: Dict[str, Any] ) -> AsyncIterable[ValidationOutcome[OT]]: # TODO: Once server side supports async streaming, this function will need to # yield async generators, not generators if self._api_client: - validation_output: Optional[Any] = None + validation_output: Optional[IValidationOutcome] = None response = self._api_client.stream_validate( guard=self, # type: ignore payload=ValidatePayload.from_dict(payload), # type: ignore @@ -561,26 +555,30 @@ async def _stream_server_call( validation_output = fragment if validation_output is None: yield ValidationOutcome[OT]( + call_id="0", # type: ignore raw_llm_output=None, validated_output=None, validation_passed=False, error="The response from the server was empty!", ) else: + validated_output = ( + cast(OT, validation_output.validated_output.actual_instance) + if validation_output.validated_output + else None + ) yield ValidationOutcome[OT]( - raw_llm_output=validation_output.raw_llm_response, # type: ignore - validated_output=cast(OT, validation_output.validated_output), - validation_passed=validation_output.result, + call_id=validation_output.call_id, # type: ignore + raw_llm_output=validation_output.raw_llm_output, # type: ignore + validated_output=validated_output, + validation_passed=(validation_output.validation_passed is True), ) if validation_output: - self._construct_history_from_server_response( - validation_output=validation_output, - llm_output=llm_output, - num_reasks=num_reasks, - prompt_params=prompt_params, - metadata=metadata, - full_schema_reask=full_schema_reask, - call_log=call_log, + guard_history = self._api_client.get_history( + self.name, validation_output.call_id + ) + self.history.extend( + [Call.from_interface(call) for call in guard_history] ) else: raise ValueError("AsyncGuard does not have an api client!") diff --git a/guardrails/classes/history/call.py b/guardrails/classes/history/call.py index a4ecd3212..99252830f 100644 --- a/guardrails/classes/history/call.py +++ b/guardrails/classes/history/call.py @@ -1,11 +1,11 @@ from typing import Any, Dict, List, Optional, Union - -from pydantic import Field, PrivateAttr +from builtins import id as object_id +from pydantic import Field from rich.panel import Panel from rich.pretty import pretty_repr from rich.tree import Tree -from guardrails_api_client import Call as ICall, CallException +from guardrails_api_client import Call as ICall from guardrails.actions.filter import Filter from guardrails.actions.refrain import Refrain from guardrails.actions.reask import merge_reask_output @@ -36,7 +36,10 @@ class Call(ICall, ArbitraryModel): inputs: CallInputs = Field( description="The inputs as passed in to Guard.__call__ or Guard.parse" ) - _exception: Optional[Exception] = PrivateAttr() + exception: Optional[Exception] = Field( + description="The exception that interrupted the run.", + default=None, + ) # Prevent Pydantic from changing our types # Without this, Pydantic casts iterations to a list @@ -46,16 +49,13 @@ def __init__( inputs: Optional[CallInputs] = None, exception: Optional[Exception] = None, ): + call_id = str(object_id(self)) iterations = iterations or Stack() inputs = inputs or CallInputs() - super().__init__( - iterations=iterations, # type: ignore - inputs=inputs, # type: ignore - i_exception=CallException(message=str(exception)), # type: ignore - ) + super().__init__(id=call_id, iterations=iterations, inputs=inputs) # type: ignore - pyright doesn't understand pydantic overrides self.iterations = iterations self.inputs = inputs - self._exception = exception + self.exception = exception @property def prompt(self) -> Optional[str]: @@ -312,25 +312,12 @@ def validator_logs(self) -> Stack[ValidatorLogs]: def error(self) -> Optional[str]: """The error message from any exception that raised and interrupted the run.""" - if self._exception: - return str(self._exception) + if self.exception: + return str(self.exception) elif self.iterations.empty(): return None return self.iterations.last.error # type: ignore - @property - def exception(self) -> Optional[Exception]: - """The exception that interrupted the run.""" - if self._exception: - return self._exception - elif self.iterations.empty(): - return None - return self.iterations.last.exception # type: ignore - - def _set_exception(self, exception: Optional[Exception]): - self._exception = exception - self.i_exception = CallException(message=str(exception)) - @property def failed_validations(self) -> Stack[ValidatorLogs]: """The validator logs for any validations that failed during the @@ -408,14 +395,35 @@ def tree(self) -> Tree: def __str__(self) -> str: return pretty_repr(self) - def to_dict(self) -> Dict[str, Any]: - i_call = ICall( - iterations=list(self.iterations), - inputs=self.inputs, + def to_interface(self) -> ICall: + return ICall( + id=self.id, + iterations=[i.to_interface() for i in self.iterations], + inputs=self.inputs.to_interface(), + exception=self.error, ) - i_call_dict = i_call.to_dict() + def to_dict(self) -> Dict[str, Any]: + return self.to_interface().to_dict() - if self._exception: - i_call_dict["exception"] = str(self._exception) - return i_call_dict + @classmethod + def from_interface(cls, i_call: ICall) -> "Call": + iterations = Stack( + *[Iteration.from_interface(i) for i in (i_call.iterations or [])] + ) + inputs = ( + CallInputs.from_interface(i_call.inputs) if i_call.inputs else CallInputs() + ) + exception = Exception(i_call.exception) if i_call.exception else None + call_inst = cls(iterations=iterations, inputs=inputs, exception=exception) + call_inst.id = i_call.id + return call_inst + + # TODO: Necessary to GET /guards/{guard_name}/history/{call_id} + @classmethod + def from_dict(cls, obj: Dict[str, Any]) -> "Call": + i_call = ICall.from_dict(obj) + + if i_call: + return cls.from_interface(i_call) + return Call() diff --git a/guardrails/classes/history/call_inputs.py b/guardrails/classes/history/call_inputs.py index 0ff92d3ad..c62fe905f 100644 --- a/guardrails/classes/history/call_inputs.py +++ b/guardrails/classes/history/call_inputs.py @@ -27,3 +27,45 @@ class CallInputs(Inputs, ICallInputs, ArbitraryModel): description="Additional keyword-arguments for the LLM as provided by the user.", default_factory=dict, ) + + def to_interface(self) -> ICallInputs: + inputs = super().to_interface().to_dict() or {} + inputs["args"] = self.args + # TODO: Better way to prevent creds from being logged, + # if they're passed in as kwargs to the LLM + redacted_kwargs = {} + for k, v in self.kwargs.items(): + if "key" in k.lower() or "token" in k.lower(): + redaction_length = len(v) - 4 + stars = "*" * redaction_length + redacted_kwargs[k] = f"{stars}{v[-4:]}" + else: + redacted_kwargs[k] = v + inputs["kwargs"] = redacted_kwargs + return ICallInputs(**inputs) + + def to_dict(self) -> Dict[str, Any]: + return self.to_interface().to_dict() + + @classmethod + def from_interface(cls, i_call_inputs: ICallInputs) -> "CallInputs": + return cls( + llm_api=None, + llm_output=i_call_inputs.llm_output, + instructions=i_call_inputs.instructions, + prompt=i_call_inputs.prompt, + msg_history=i_call_inputs.msg_history, + prompt_params=i_call_inputs.prompt_params, + num_reasks=i_call_inputs.num_reasks, + metadata=i_call_inputs.metadata, + full_schema_reask=(i_call_inputs.full_schema_reask is True), + stream=(i_call_inputs.stream is True), + args=(i_call_inputs.args or []), + kwargs=(i_call_inputs.kwargs or {}), + ) + + @classmethod + def from_dict(cls, obj: Dict[str, Any]): + i_call_inputs = ICallInputs.from_dict(obj) or ICallInputs() + + return cls.from_interface(i_call_inputs) diff --git a/guardrails/classes/history/inputs.py b/guardrails/classes/history/inputs.py index 0e6809acf..f9ee44423 100644 --- a/guardrails/classes/history/inputs.py +++ b/guardrails/classes/history/inputs.py @@ -34,7 +34,7 @@ class Inputs(IInputs, ArbitraryModel): "that will be formatted into the final LLM prompt.", default=None, ) - num_reasks: int = Field( + num_reasks: Optional[int] = Field( description="The total number of reasks allowed; user provided or defaulted.", default=None, ) @@ -42,7 +42,7 @@ class Inputs(IInputs, ArbitraryModel): description="The metadata provided by the user to be used during validation.", default=None, ) - full_schema_reask: bool = Field( + full_schema_reask: Optional[bool] = Field( description="Whether to perform reasks across the entire schema" "or at the field level.", default=None, @@ -51,3 +51,78 @@ class Inputs(IInputs, ArbitraryModel): description="Whether to use streaming.", default=False, ) + + def to_interface(self) -> IInputs: + serialized_msg_history = None + if self.msg_history: + serialized_msg_history = [] + for msg in self.msg_history: + ser_msg = {**msg} + content = ser_msg.get("content") + if content: + ser_msg["content"] = ( + content.source if isinstance(content, Prompt) else content + ) + serialized_msg_history.append(ser_msg) + + instructions = ( + self.instructions.source + if isinstance(self.instructions, Instructions) + else self.instructions + ) + + prompt = self.prompt.source if isinstance(self.prompt, Prompt) else self.prompt + + return IInputs( + llm_api=str(self.llm_api) if self.llm_api else None, # type: ignore - pyright doesn't understand aliases + llm_output=self.llm_output, # type: ignore - pyright doesn't understand aliases + instructions=instructions, + prompt=prompt, + msg_history=serialized_msg_history, # type: ignore - pyright doesn't understand aliases + prompt_params=self.prompt_params, # type: ignore - pyright doesn't understand aliases + num_reasks=self.num_reasks, # type: ignore - pyright doesn't understand aliases + metadata=self.metadata, + full_schema_reask=self.full_schema_reask, # type: ignore - pyright doesn't understand aliases + stream=self.stream, + ) + + def to_dict(self) -> Dict[str, Any]: + return self.to_interface().to_dict() + + @classmethod + def from_interface(cls, i_inputs: IInputs) -> "Inputs": + deserialized_msg_history = None + if i_inputs.msg_history: + deserialized_msg_history = [] + for msg in i_inputs.msg_history: + ser_msg = {**msg} + content = ser_msg.get("content") + if content: + ser_msg["content"] = Prompt(content) + deserialized_msg_history.append(ser_msg) + + instructions = ( + Instructions(i_inputs.instructions) if i_inputs.instructions else None + ) + + prompt = Prompt(i_inputs.prompt) if i_inputs.prompt else None + num_reasks = ( + int(i_inputs.num_reasks) if i_inputs.num_reasks is not None else None + ) + return cls( + llm_api=None, + llm_output=i_inputs.llm_output, + instructions=instructions, + prompt=prompt, + msg_history=deserialized_msg_history, + prompt_params=i_inputs.prompt_params, + num_reasks=num_reasks, + metadata=i_inputs.metadata, + full_schema_reask=(i_inputs.full_schema_reask is True), + stream=(i_inputs.stream is True), + ) + + @classmethod + def from_dict(cls, obj: Dict[str, Any]) -> "Inputs": + i_inputs = IInputs.from_dict(obj) or IInputs() + return cls.from_interface(i_inputs) diff --git a/guardrails/classes/history/iteration.py b/guardrails/classes/history/iteration.py index adf9e6168..c440c7e95 100644 --- a/guardrails/classes/history/iteration.py +++ b/guardrails/classes/history/iteration.py @@ -1,5 +1,5 @@ -from typing import Dict, List, Optional, Sequence, Union - +from typing import Any, Dict, List, Optional, Sequence, Union +from builtins import id as object_id from pydantic import Field from rich.console import Group from rich.panel import Panel @@ -29,6 +29,26 @@ class Iteration(IIteration, ArbitraryModel): description="The outputs from the iteration/step.", default_factory=Outputs ) + def __init__( + self, + call_id: str, + index: int, + inputs: Optional[Inputs] = None, + outputs: Optional[Outputs] = None, + ): + iteration_id = str(object_id(self)) + inputs = inputs or Inputs() + outputs = outputs or Outputs() + super().__init__( + id=iteration_id, + call_id=call_id, # type: ignore + index=index, + inputs=inputs, + outputs=outputs, + ) + self.inputs = inputs + self.outputs = outputs + @property def logs(self) -> Stack[str]: """Returns the logs from this iteration as a stack.""" @@ -216,3 +236,45 @@ def create_msg_history_table( def __str__(self) -> str: return pretty_repr(self) + + def to_interface(self) -> IIteration: + return IIteration( + id=self.id, + call_id=self.call_id, # type: ignore + index=self.index, + inputs=self.inputs.to_interface(), + outputs=self.outputs.to_interface(), + ) + + def to_dict(self) -> Dict: + return self.to_interface().to_dict() + + @classmethod + def from_interface(cls, i_iteration: IIteration) -> "Iteration": + inputs = ( + Inputs.from_interface(i_iteration.inputs) if i_iteration.inputs else None + ) + outputs = ( + Outputs.from_interface(i_iteration.outputs) if i_iteration.outputs else None + ) + iteration = cls( + call_id=i_iteration.call_id, + index=i_iteration.index, + inputs=inputs, + outputs=outputs, + ) + iteration.id = i_iteration.id + return iteration + + @classmethod + def from_dict(cls, obj: Dict[str, Any]) -> "Iteration": + id = obj.get("id", "0") + call_id = obj.get("callId", obj.get("call_id", "0")) + index = obj.get("index", 0) + i_iteration = IIteration.from_dict(obj) or IIteration( + id=id, + call_id=call_id, # type: ignore + index=index, # type: ignore + ) + + return cls.from_interface(i_iteration) diff --git a/guardrails/classes/history/outputs.py b/guardrails/classes/history/outputs.py index 922d05be1..f5083a803 100644 --- a/guardrails/classes/history/outputs.py +++ b/guardrails/classes/history/outputs.py @@ -4,7 +4,6 @@ from guardrails_api_client import ( Outputs as IOutputs, - OutputsException, OutputsParsedOutput, OutputsValidationResponse, ) @@ -136,16 +135,46 @@ def status(self) -> str: return fail_status return pass_status - def to_dict(self) -> Dict[str, Any]: - i_outputs = IOutputs( - llm_response_info=self.llm_response_info, # type: ignore + def to_interface(self) -> IOutputs: + return IOutputs( + llm_response_info=self.llm_response_info.to_interface(), # type: ignore raw_output=self.raw_output, # type: ignore parsed_output=OutputsParsedOutput(self.parsed_output), # type: ignore validation_response=OutputsValidationResponse(self.validation_response), # type: ignore guarded_output=OutputsParsedOutput(self.guarded_output), # type: ignore reasks=self.reasks, # type: ignore - validator_logs=self.validator_logs, # type: ignore + validator_logs=[v.to_interface() for v in self.validator_logs], # type: ignore error=self.error, - exception=OutputsException(message=str(self.exception)), ) - return i_outputs.to_dict() + + def to_dict(self) -> Dict[str, Any]: + return self.to_interface().to_dict() + + @classmethod + def from_interface(cls, i_outputs: IOutputs) -> "Outputs": + reasks = [] + if i_outputs.reasks: + reasks = [ReAsk.from_interface(r) for r in i_outputs.reasks] + + validator_logs = [] + if i_outputs.validator_logs: + validator_logs = [ + ValidatorLogs.from_interface(v) for v in i_outputs.validator_logs + ] + + return cls( + llm_response_info=LLMResponse.from_interface(i_outputs.llm_response_info), # type: ignore + raw_output=i_outputs.raw_output, # type: ignore + parsed_output=i_outputs.parsed_output.actual_instance, # type: ignore + validation_response=i_outputs.validation_response.actual_instance, # type: ignore + guarded_output=i_outputs.guarded_output.actual_instance, # type: ignore + reasks=reasks, # type: ignore + validator_logs=validator_logs, # type: ignore + error=i_outputs.error, + ) + + @classmethod + def from_dict(cls, obj: Dict[str, Any]) -> "Outputs": + i_outputs = IOutputs.from_dict(obj) or IOutputs() + + return cls.from_interface(i_outputs) diff --git a/guardrails/classes/llm/llm_response.py b/guardrails/classes/llm/llm_response.py index f1851e405..6051d2fca 100644 --- a/guardrails/classes/llm/llm_response.py +++ b/guardrails/classes/llm/llm_response.py @@ -1,9 +1,16 @@ -from typing import Iterable, Optional, AsyncIterable +import asyncio +from typing import Any, Dict, Iterable, Optional, AsyncIterable from guardrails_api_client import LLMResponse as ILLMResponse from pydantic.config import ConfigDict +# TODO: Move this somewhere that makes sense +def async_to_sync(awaitable): + loop = asyncio.get_event_loop() + return loop.run_until_complete(awaitable) + + # TODO: We might be able to delete this class LLMResponse(ILLMResponse): # Pydantic Config @@ -14,3 +21,52 @@ class LLMResponse(ILLMResponse): output: str stream_output: Optional[Iterable] = None async_stream_output: Optional[AsyncIterable] = None + + def to_interface(self) -> ILLMResponse: + stream_output = None + if self.stream_output: + stream_output = [str(so) for so in self.stream_output] + + async_stream_output = None + if self.async_stream_output: + async_stream_output = [str(async_to_sync(so)) for so in self.stream_output] # type: ignore - we just established it isn't None + + return ILLMResponse( + prompt_token_count=self.prompt_token_count, # type: ignore - pyright doesn't understand aliases + response_token_count=self.response_token_count, # type: ignore - pyright doesn't understand aliases + output=self.output, + stream_output=stream_output, # type: ignore - pyright doesn't understand aliases + async_stream_output=async_stream_output, # type: ignore - pyright doesn't understand aliases + ) + + def to_dict(self) -> Dict[str, Any]: + return self.to_interface().to_dict() + + @classmethod + def from_interface(cls, i_llm_response: ILLMResponse) -> "LLMResponse": + stream_output = None + if i_llm_response.stream_output: + stream_output = [so for so in i_llm_response.stream_output] + + async_stream_output = None + if i_llm_response.async_stream_output: + + async def async_iter(): + for aso in i_llm_response.async_stream_output: # type: ignore - just verified it isn't None... + yield aso + + async_stream_output = async_iter() + + return cls( + prompt_token_count=i_llm_response.prompt_token_count, + response_token_count=i_llm_response.response_token_count, + output=i_llm_response.output, + stream_output=stream_output, + async_stream_output=async_stream_output, + ) + + @classmethod + def from_dict(cls, obj: Dict[str, Any]) -> "LLMResponse": + i_llm_response = super().from_dict(obj) or ILLMResponse(output="") + + return cls.from_interface(i_llm_response) diff --git a/guardrails/classes/schema/model_schema.py b/guardrails/classes/schema/model_schema.py index 7f7e2cc8e..cb28770ee 100644 --- a/guardrails/classes/schema/model_schema.py +++ b/guardrails/classes/schema/model_schema.py @@ -1,5 +1,5 @@ from typing import Any, Dict, Optional -from guardrails_api_client import ModelSchema as IModelSchema +from guardrails_api_client import ModelSchema as IModelSchema, ValidationType # Because pydantic insists on including None values in the serialized dictionary @@ -8,12 +8,21 @@ def to_dict(self) -> Dict[str, Any]: super_dict = super().to_dict() return {k: v for k, v in super_dict.items() if v is not None} - def from_dict(self, d: Dict[str, Any]) -> Optional["ModelSchema"]: - i_model_schema = super().from_dict(d) + @classmethod + def from_dict(cls, obj: Optional[Dict[str, Any]]) -> "ModelSchema": + if not obj: + obj = {"type": "string"} - if not i_model_schema: - return i_model_schema + i_model_schema = super().from_dict(obj) - trimmed = {k: v for k, v in i_model_schema.to_dict().items() if v is not None} + i_model_schema_dict = ( + i_model_schema.to_dict() if i_model_schema else {"type": "string"} + ) - return ModelSchema(**trimmed) + trimmed = {k: v for k, v in i_model_schema_dict.items() if v is not None} + + output_schema_type = trimmed.get("type") + if output_schema_type: + trimmed["type"] = ValidationType.from_dict(output_schema_type) # type: ignore + + return cls(**trimmed) # type: ignore diff --git a/guardrails/classes/validation/validation_result.py b/guardrails/classes/validation/validation_result.py index 9b194f8ea..316f3c415 100644 --- a/guardrails/classes/validation/validation_result.py +++ b/guardrails/classes/validation/validation_result.py @@ -1,9 +1,10 @@ -from typing import Any, Dict, List, Literal, Optional +from typing import Any, Dict, List, Literal, Optional, Union from pydantic import Field from guardrails_api_client import ( ValidationResult as IValidationResult, # noqa PassResult as IPassResult, FailResult as IFailResult, + ErrorSpan as IErrorSpan, ) from guardrails.classes.generic.arbitrary_model import ArbitraryModel @@ -14,9 +15,34 @@ class ValidationResult(IValidationResult, ArbitraryModel): # value argument passed to validator.validate # or validator.validate_stream - # FIXME: Add this to json schema validated_chunk: Optional[Any] = None + @classmethod + def from_interface( + cls, i_validation_result: Union[IValidationResult, IPassResult, IFailResult] + ) -> "ValidationResult": + if i_validation_result.outcome == "pass": + return PassResult( + outcome=i_validation_result.outcome, + metadata=i_validation_result.metadata, + validated_chunk=i_validation_result.validated_chunk, + ) + elif i_validation_result.outcome == "fail": + return FailResult.from_dict(i_validation_result.to_dict()) + + return cls( + outcome=i_validation_result.outcome or "", + metadata=i_validation_result.metadata, + validated_chunk=i_validation_result.validated_chunk, + ) + + @classmethod + def from_dict(cls, obj: Dict[str, Any]) -> "ValidationResult": + i_validation_result = IValidationResult.from_dict(obj) or IValidationResult( + outcome="pail" + ) + return cls.from_interface(i_validation_result) + class PassResult(ValidationResult, IPassResult): outcome: Literal["pass"] = "pass" @@ -27,17 +53,31 @@ class ValueOverrideSentinel: # should only be used if Validator.override_value_on_pass is True value_override: Optional[Any] = Field(default=ValueOverrideSentinel) - def to_dict(self) -> Dict[str, Any]: + def to_interface(self) -> IPassResult: i_pass_result = IPassResult(outcome=self.outcome, metadata=self.metadata) if self.value_override is not self.ValueOverrideSentinel: i_pass_result.value_override = self.value_override - return i_pass_result.to_dict() + return i_pass_result + + def to_dict(self) -> Dict[str, Any]: + # Pydantic's model_dump method isn't working properly + _dict = { + "outcome": self.outcome, + "metadata": self.metadata, + "validatedChunk": self.validated_chunk, + "valueOverride": ( + self.value_override + if self.value_override is not self.ValueOverrideSentinel + else None + ), + } + return _dict # FIXME: Add this to json schema -class ErrorSpan(ArbitraryModel): +class ErrorSpan(IErrorSpan, ArbitraryModel): start: int end: int # reason validation failed, specific to this chunk @@ -49,6 +89,51 @@ class FailResult(ValidationResult, IFailResult): error_message: str fix_value: Optional[Any] = None - # FIXME: Add this to json schema # segments that caused validation to fail error_spans: Optional[List[ErrorSpan]] = None + + @classmethod + def from_interface(cls, i_fail_result: IFailResult) -> "FailResult": + error_spans = None + if i_fail_result.error_spans: + error_spans = [ + ErrorSpan( + start=error_span.start, + end=error_span.end, + reason=error_span.reason, + ) + for error_span in i_fail_result.error_spans + ] + + return cls( + outcome="fail", + metadata=i_fail_result.metadata, + validated_chunk=i_fail_result.validated_chunk, + error_message=i_fail_result.error_message or "", + fix_value=i_fail_result.fix_value, + error_spans=error_spans, + ) + + @classmethod + def from_dict(cls, obj: Dict[str, Any]) -> "FailResult": + i_fail_result = IFailResult.from_dict(obj) or IFailResult( + outcome="Fail", + error_message="", # type: ignore - pyright doesn't understand aliases + ) + return cls.from_interface(i_fail_result) + + def to_dict(self) -> Dict[str, Any]: + # Pydantic's model_dump method isn't working properly + _dict = { + "outcome": self.outcome, + "metadata": self.metadata, + "validatedChunk": self.validated_chunk, + "errorMessage": self.error_message, + "fixValue": self.fix_value, + "errorSpans": ( + [error_span.to_dict() for error_span in self.error_spans] + if self.error_spans + else [] + ), + } + return _dict diff --git a/guardrails/classes/validation/validator_logs.py b/guardrails/classes/validation/validator_logs.py index dad0dc378..7cf31f193 100644 --- a/guardrails/classes/validation/validator_logs.py +++ b/guardrails/classes/validation/validator_logs.py @@ -1,12 +1,17 @@ from datetime import datetime from typing import Any, Dict, Optional -from guardrails_api_client import ValidatorLog +from guardrails_api_client import ( + ValidatorLog as IValidatorLog, + ValidatorLogInstanceId, + ValidatorLogValidationResult, +) from guardrails.classes.generic.arbitrary_model import ArbitraryModel from guardrails.classes.validation.validation_result import ValidationResult +from guardrails.utils.casting_utils import to_int -class ValidatorLogs(ValidatorLog, ArbitraryModel): +class ValidatorLogs(IValidatorLog, ArbitraryModel): """Logs for a single validator.""" validator_name: str @@ -19,26 +24,63 @@ class ValidatorLogs(ValidatorLog, ArbitraryModel): instance_id: Optional[int] = None property_path: str + def to_interface(self) -> IValidatorLog: + start_time = self.start_time.isoformat() if self.start_time else None + end_time = self.end_time.isoformat() if self.end_time else None + return IValidatorLog( + validator_name=self.validator_name, # type: ignore - pyright doesn't understand aliases + registered_name=self.registered_name, # type: ignore - pyright doesn't understand aliases + instance_id=ValidatorLogInstanceId(self.instance_id), # type: ignore - pyright doesn't understand aliases + property_path=self.property_path, # type: ignore - pyright doesn't understand aliases + value_before_validation=self.value_before_validation, # type: ignore - pyright doesn't understand aliases + value_after_validation=self.value_after_validation, # type: ignore - pyright doesn't understand aliases + validation_result=ValidatorLogValidationResult(self.validation_result), # type: ignore - pyright doesn't understand aliases + start_time=start_time, # type: ignore - pyright doesn't understand aliases + end_time=end_time, # type: ignore - pyright doesn't understand aliases + ) + def to_dict(self) -> Dict[str, Any]: - i_validator_logs = ValidatorLog( - validator_name=self.validator_name, # type: ignore - registered_name=self.registered_name, # type: ignore - value_before_validation=self.value_before_validation, # type: ignore - value_after_validation=self.value_after_validation, # type: ignore - property_path=self.property_path, # type: ignore + return self.to_interface().to_dict() + + @classmethod + def from_interface(cls, i_validator_log: IValidatorLog) -> "ValidatorLogs": + instance_id = ( + i_validator_log.instance_id.actual_instance + if i_validator_log.instance_id + else None + ) + validation_result = ( + ValidationResult.from_interface( + i_validator_log.validation_result.actual_instance + ) + if ( + i_validator_log.validation_result + and i_validator_log.validation_result.actual_instance + ) + else None ) - i_validator_log = i_validator_logs.model_dump( - by_alias=True, - exclude_none=True, + start_time = i_validator_log.start_time + if isinstance(start_time, str): + start_time = datetime.fromisoformat(start_time) + + end_time = i_validator_log.end_time + if isinstance(end_time, str): + end_time = datetime.fromisoformat(end_time) + + return cls( + validator_name=i_validator_log.validator_name, + registered_name=i_validator_log.registered_name, + instance_id=to_int(instance_id) or 0, + property_path=i_validator_log.property_path, + value_before_validation=i_validator_log.value_before_validation, + value_after_validation=i_validator_log.value_after_validation, + validation_result=validation_result, + start_time=start_time, + end_time=end_time, ) - if self.instance_id: - i_validator_log["instanceId"] = self.instance_id - if self.validation_result: - i_validator_log["validation_result"] = self.validation_result.to_dict() - if self.start_time: - i_validator_log["start_time"] = self.start_time.isoformat() - if self.end_time: - i_validator_log["end_time"] = self.end_time.isoformat() - - return i_validator_log + + @classmethod + def from_dict(cls, obj: Dict[str, Any]) -> "ValidatorLogs": + i_validator_log = IValidatorLog.from_dict(obj) + return cls.from_interface(i_validator_log) # type: ignore diff --git a/guardrails/classes/validation_outcome.py b/guardrails/classes/validation_outcome.py index 12f199c46..fe23a1e91 100644 --- a/guardrails/classes/validation_outcome.py +++ b/guardrails/classes/validation_outcome.py @@ -55,7 +55,7 @@ class ValidationOutcome(IValidationOutcome, ArbitraryModel, Generic[OT]): @classmethod def from_guard_history(cls, call: Call): """Create a ValidationOutcome from a history Call object.""" - last_iteration = call.iterations.last or Iteration() + last_iteration = call.iterations.last or Iteration(call_id=call.id, index=0) last_output = last_iteration.validation_response or safe_get( list(last_iteration.reasks), 0 ) @@ -64,6 +64,7 @@ def from_guard_history(cls, call: Call): error = call.error output = cast(OT, call.guarded_output) return cls( + call_id=call.id, # type: ignore raw_llm_output=call.raw_outputs.last, validated_output=output, reask=reask, @@ -97,6 +98,7 @@ def __str__(self) -> str: def to_dict(self): i_validation_outcome = IValidationOutcome( + call_id=self.call_id, # type: ignore raw_llm_output=self.raw_llm_output, # type: ignore validated_output=ValidationOutcomeValidatedOutput(self.validated_output), # type: ignore reask=self.reask, diff --git a/guardrails/cli/__init__.py b/guardrails/cli/__init__.py index f1b21618b..d16b49c17 100644 --- a/guardrails/cli/__init__.py +++ b/guardrails/cli/__init__.py @@ -1,4 +1,5 @@ import guardrails.cli.configure # noqa +import guardrails.cli.start # noqa import guardrails.cli.validate # noqa from guardrails.cli.guardrails import guardrails as cli from guardrails.cli.hub import hub_command diff --git a/guardrails/cli/start.py b/guardrails/cli/start.py new file mode 100644 index 000000000..556a0c6d5 --- /dev/null +++ b/guardrails/cli/start.py @@ -0,0 +1,49 @@ +from typing import Optional +import typer + +from guardrails.cli.guardrails import guardrails +from guardrails.cli.hub.utils import pip_process +from guardrails.cli.logger import logger + + +def api_is_installed() -> bool: + try: + import guardrails_api # type: ignore # noqa + + return True + except ImportError: + return False + + +@guardrails.command() +def start( + env: Optional[str] = typer.Option( + default="", + help="An env file to load environment variables from.", + ), + config: Optional[str] = typer.Option( + default="", + help="A config file to load Guards from.", + ), + timeout: Optional[int] = typer.Option( + default=5, + help="Gunicorn worker timeout.", + ), + threads: Optional[int] = typer.Option( + default=10, + help="Number of Gunicorn worker threads.", + ), + port: Optional[int] = typer.Option( + default=8000, + help="The port to run the server on.", + ), +): + logger.debug("Checking for prerequisites...") + if not api_is_installed(): + package_name = 'guardrails-api>="^0.0.0a0"' + pip_process("install", package_name) + + from guardrails_api.cli.start import start # type: ignore + + logger.info("Starting Guardrails server") + start(env, config, timeout, threads, port) diff --git a/guardrails/formatters/base_formatter.py b/guardrails/formatters/base_formatter.py index c767ccd0f..dc409e8c4 100644 --- a/guardrails/formatters/base_formatter.py +++ b/guardrails/formatters/base_formatter.py @@ -7,9 +7,12 @@ class BaseFormatter(ABC): - """A Formatter takes an LLM Callable and wraps the method into an abstract callable. - Used to perform manipulations of the input or the output, like JSON constrained- - decoding.""" + """A Formatter takes an LLM Callable and wraps the method into an abstract + callable. + + Used to perform manipulations of the input or the output, like JSON + constrained- decoding. + """ @abstractmethod def wrap_callable(self, llm_callable: PromptCallableBase) -> ArbitraryCallable: ... diff --git a/guardrails/formatters/json_formatter.py b/guardrails/formatters/json_formatter.py index b98fbe32c..000423851 100644 --- a/guardrails/formatters/json_formatter.py +++ b/guardrails/formatters/json_formatter.py @@ -1,8 +1,6 @@ import json from typing import Optional, Union -from jsonformer import Jsonformer - from guardrails.formatters.base_formatter import BaseFormatter from guardrails.llm_providers import ( ArbitraryCallable, @@ -94,6 +92,8 @@ def __init__(self, schema: dict): def wrap_callable(self, llm_callable) -> ArbitraryCallable: # JSON Schema enforcement experiment. + from jsonformer import Jsonformer + if isinstance(llm_callable, HuggingFacePipelineCallable): model = llm_callable.init_kwargs["pipeline"] return ArbitraryCallable( diff --git a/guardrails/guard.py b/guardrails/guard.py index f66369c1c..f0a5a6d62 100644 --- a/guardrails/guard.py +++ b/guardrails/guard.py @@ -21,11 +21,10 @@ from guardrails_api_client import ( Guard as IGuard, - GuardHistory, ValidatorReference, ValidatePayload, - ValidationType, SimpleTypes, + ValidationOutcome as IValidationOutcome, ) from pydantic import field_validator from pydantic.config import ConfigDict @@ -33,15 +32,11 @@ from guardrails.api_client import GuardrailsApiClient from guardrails.classes.output_type import OT from guardrails.classes.validation_outcome import ValidationOutcome -from guardrails.classes.validation.validation_result import FailResult from guardrails.classes.credentials import Credentials from guardrails.classes.execution import GuardExecutionOptions from guardrails.classes.generic import Stack from guardrails.classes.history import Call from guardrails.classes.history.call_inputs import CallInputs -from guardrails.classes.history.inputs import Inputs -from guardrails.classes.history.iteration import Iteration -from guardrails.classes.history.outputs import Outputs from guardrails.classes.output_type import OutputTypes from guardrails.classes.schema.processed_schema import ProcessedSchema from guardrails.classes.schema.model_schema import ModelSchema @@ -52,7 +47,6 @@ model_is_supported_server_side, ) from guardrails.logger import logger, set_scope -from guardrails.prompt import Instructions, Prompt from guardrails.run import Runner, StreamRunner from guardrails.schema.primitive_schema import primitive_to_schema from guardrails.schema.pydantic_schema import pydantic_model_to_schema @@ -72,8 +66,6 @@ from guardrails.utils.naming_utils import random_id from guardrails.utils.api_utils import extract_serializeable_metadata from guardrails.utils.hub_telemetry_utils import HubTelemetry -from guardrails.classes.llm.llm_response import LLMResponse -from guardrails.actions.reask import FieldReAsk from guardrails.utils.validator_utils import ( get_validator, parse_validator_reference, @@ -112,6 +104,7 @@ class that contains the raw output from validators: List[ValidatorReference] output_schema: ModelSchema + history: Stack[Call] # Pydantic Config model_config = ConfigDict(arbitrary_types_allowed=True) @@ -127,6 +120,8 @@ def __init__( ): """Initialize the Guard with validators and an output schema.""" + _try_to_load = name is not None + # Shared Interface Properties id = id or random_id() name = name or f"gr-{id}" @@ -136,11 +131,14 @@ def __init__( output_schema = output_schema or {"type": "string"} # Init ModelSchema class - schema_with_type = {**output_schema} - output_schema_type = output_schema.get("type") - if output_schema_type: - schema_with_type["type"] = ValidationType.from_dict(output_schema_type) - model_schema = ModelSchema(**schema_with_type) + # schema_with_type = {**output_schema} + # output_schema_type = output_schema.get("type") + # if output_schema_type: + # schema_with_type["type"] = ValidationType.from_dict(output_schema_type) + model_schema = ModelSchema.from_dict(output_schema) + + # TODO: Support a sink for history so that it is not solely held in memory + history: Stack[Call] = Stack() # Super Init super().__init__( @@ -149,7 +147,7 @@ def __init__( description=description, validators=validators, output_schema=model_schema, - i_history=GuardHistory([]), # type: ignore + history=history, # type: ignore - pyright doesn't understand pydantic overrides ) ### Public ### @@ -159,6 +157,7 @@ def __init__( # self.description: Optional[str] = None # self.validators: Optional[List[ValidatorReference]] = [] # self.output_schema: Optional[ModelSchema] = None + # self.history = history ### Legacy ## self._num_reasks = None @@ -178,18 +177,29 @@ def __init__( self._allow_metrics_collection: Optional[bool] = None self._output_formatter: Optional[BaseFormatter] = None - # TODO: Support a sink for history so that it is not solely held in memory - self._history: Stack[Call] = Stack() - # Gaurdrails As A Service Initialization api_key = os.environ.get("GUARDRAILS_API_KEY") if api_key is not None: self._api_client = GuardrailsApiClient(api_key=api_key) - self.upsert_guard() - - @property - def history(self): - return self._history + _loaded = False + if _try_to_load: + loaded_guard = self._api_client.fetch_guard(self.name) + if loaded_guard: + self.id = loaded_guard.id + self.description = loaded_guard.description + self.validators = loaded_guard.validators or [] + + loaded_output_schema = ( + ModelSchema.from_dict( # trims out extra keys + loaded_guard.output_schema.to_dict() + if loaded_guard.output_schema + else {"type": "string"} + ) + ) + self.output_schema = loaded_output_schema + _loaded = True + if not _loaded: + self._save() @field_validator("output_schema") @classmethod @@ -657,9 +667,6 @@ def __exec( args=list(args), kwargs=kwargs, ) - call_log = Call(inputs=call_inputs) - set_scope(str(object_id(call_log))) - self._history.push(call_log) if self._api_client is not None and model_is_supported_server_side( llm_api, *args, **kwargs @@ -671,11 +678,13 @@ def __exec( prompt_params=prompt_params, metadata=metadata, full_schema_reask=full_schema_reask, - call_log=call_log, *args, **kwargs, ) + call_log = Call(inputs=call_inputs) + set_scope(str(object_id(call_log))) + self.history.push(call_log) # Otherwise, call the LLM synchronously return self._exec( llm_api=llm_api, @@ -999,141 +1008,42 @@ def upsert_guard(self): else: raise ValueError("Guard does not have an api client!") - def _construct_history_from_server_response( - self, - *, - validation_output: Optional[Any] = None, - llm_api: Optional[Callable] = None, - llm_output: Optional[str] = None, - num_reasks: Optional[int] = None, - prompt_params: Optional[Dict] = None, - metadata: Optional[Dict] = None, - full_schema_reask: Optional[bool] = True, - call_log: Optional[Call], - stream: Optional[bool] = False, - ): - # TODO: GET /guard/{guard-name}/history - call_log = call_log or Call() - if llm_api is not None: - llm_api = get_llm_ask(llm_api) - session_history = ( - validation_output.session_history - if validation_output is not None and validation_output.session_history - else [] - ) - history: List[Call] - for history in session_history: - history_events: Optional[List[Any]] = ( # type: ignore - history.history # type: ignore - ) - if history_events is None: - continue - - iterations = [ - Iteration( - inputs=Inputs( - llm_api=llm_api, - llm_output=llm_output, - instructions=( - Instructions(h.instructions) if h.instructions else None - ), - prompt=( - Prompt(h.prompt.source) # type: ignore - if h.prompt - else None - ), - prompt_params=prompt_params, - num_reasks=(num_reasks or 0), - metadata=metadata, - full_schema_reask=full_schema_reask, # type: ignore - ), - outputs=Outputs( - llm_response_info=LLMResponse( - output=h.output # type: ignore - ), - raw_output=h.output, - parsed_output=( - h.parsed_output.to_dict() - if isinstance(h.parsed_output, Any) - else h.parsed_output - ), - validation_output=( # type: ignore - h.validated_output.to_dict() - if isinstance(h.validated_output, Any) - else h.validated_output - ), - reasks=list( - [ - FieldReAsk( - incorrect_value=r.to_dict().get("incorrect_value"), - path=r.to_dict().get("path"), - fail_results=[ - FailResult( - error_message=r.to_dict().get( - "error_message" - ), - fix_value=r.to_dict().get("fix_value"), - ) - ], - ) - for r in h.reasks # type: ignore - ] - if h.reasks is not None - else [] - ), - ), - ) - for h in history_events - ] - call_log.iterations.extend(iterations) - if self._history.length == 0: - self._history.push(call_log) - - def _single_server_call( - self, - *, - payload: Dict[str, Any], - llm_output: Optional[str] = None, - num_reasks: Optional[int] = None, - prompt_params: Optional[Dict] = None, - metadata: Optional[Dict] = {}, - full_schema_reask: Optional[bool] = True, - call_log: Optional[Call], - stream: Optional[bool] = False, - ) -> ValidationOutcome[OT]: + def _single_server_call(self, *, payload: Dict[str, Any]) -> ValidationOutcome[OT]: if self._api_client: - validation_output: ValidationOutcome = self._api_client.validate( + validation_output: IValidationOutcome = self._api_client.validate( guard=self, # type: ignore payload=ValidatePayload.from_dict(payload), # type: ignore openai_api_key=get_call_kwarg("api_key"), ) if not validation_output: return ValidationOutcome[OT]( + call_id="0", # type: ignore raw_llm_output=None, validated_output=None, validation_passed=False, error="The response from the server was empty!", ) - # TODO: Replace this with GET /guard/{guard_name}/history - self._construct_history_from_server_response( - validation_output=validation_output, - llm_output=llm_output, - num_reasks=num_reasks, - prompt_params=prompt_params, - metadata=metadata, - full_schema_reask=full_schema_reask, - call_log=call_log, - stream=stream, + + guard_history = self._api_client.get_history( + self.name, validation_output.call_id ) + self.history.extend([Call.from_interface(call) for call in guard_history]) + # TODO: See if the below statement is still true # Our interfaces are too different for this to work right now. # Once we move towards shared interfaces for both the open source # and the api we can re-enable this. # return ValidationOutcome[OT].from_guard_history(call_log) + validated_output = ( + cast(OT, validation_output.validated_output.actual_instance) + if validation_output.validated_output + else None + ) return ValidationOutcome[OT]( + call_id=validation_output.call_id, # type: ignore raw_llm_output=validation_output.raw_llm_output, - validated_output=cast(OT, validation_output.validated_output), - validation_passed=validation_output.validation_passed, + validated_output=validated_output, + validation_passed=(validation_output.validation_passed is True), ) else: raise ValueError("Guard does not have an api client!") @@ -1142,16 +1052,9 @@ def _stream_server_call( self, *, payload: Dict[str, Any], - llm_output: Optional[str] = None, - num_reasks: Optional[int] = None, - prompt_params: Optional[Dict] = None, - metadata: Optional[Dict] = {}, - full_schema_reask: Optional[bool] = True, - call_log: Optional[Call], - stream: Optional[bool] = False, ) -> Iterable[ValidationOutcome[OT]]: if self._api_client: - validation_output: Optional[ValidationOutcome] = None + validation_output: Optional[IValidationOutcome] = None response = self._api_client.stream_validate( guard=self, # type: ignore payload=ValidatePayload.from_dict(payload), # type: ignore @@ -1161,28 +1064,30 @@ def _stream_server_call( validation_output = fragment if validation_output is None: yield ValidationOutcome[OT]( + call_id="0", # type: ignore raw_llm_output=None, validated_output=None, validation_passed=False, error="The response from the server was empty!", ) else: + validated_output = ( + cast(OT, validation_output.validated_output.actual_instance) + if validation_output.validated_output + else None + ) yield ValidationOutcome[OT]( + call_id=validation_output.call_id, # type: ignore raw_llm_output=validation_output.raw_llm_output, - validated_output=cast(OT, validation_output.validated_output), - validation_passed=validation_output.validation_passed, + validated_output=validated_output, + validation_passed=(validation_output.validation_passed is True), ) if validation_output: - # TODO: Replace this with GET /guard/{guard_name}/history - self._construct_history_from_server_response( - validation_output=validation_output, - llm_output=llm_output, - num_reasks=num_reasks, - prompt_params=prompt_params, - metadata=metadata, - full_schema_reask=full_schema_reask, - call_log=call_log, - stream=stream, + guard_history = self._api_client.get_history( + self.name, validation_output.call_id + ) + self.history.extend( + [Call.from_interface(call) for call in guard_history] ) else: raise ValueError("Guard does not have an api client!") @@ -1196,11 +1101,13 @@ def _call_server( prompt_params: Optional[Dict] = None, metadata: Optional[Dict] = {}, full_schema_reask: Optional[bool] = True, - call_log: Optional[Call], **kwargs, ) -> Union[ValidationOutcome[OT], Iterable[ValidationOutcome[OT]]]: if self._api_client: - payload: Dict[str, Any] = {"args": list(args)} + payload: Dict[str, Any] = { + "args": list(args), + "full_schema_reask": full_schema_reask, + } payload.update(**kwargs) if metadata: payload["metadata"] = extract_serializeable_metadata(metadata) @@ -1215,26 +1122,10 @@ def _call_server( should_stream = kwargs.get("stream", False) if should_stream: - return self._stream_server_call( - payload=payload, - llm_output=llm_output, - num_reasks=num_reasks, - prompt_params=prompt_params, - metadata=metadata, - full_schema_reask=full_schema_reask, - call_log=call_log, - stream=should_stream, - ) + return self._stream_server_call(payload=payload) else: return self._single_server_call( payload=payload, - llm_output=llm_output, - num_reasks=num_reasks, - prompt_params=prompt_params, - metadata=metadata, - full_schema_reask=full_schema_reask, - call_log=call_log, - stream=should_stream, ) else: raise ValueError("Guard does not have an api client!") @@ -1267,15 +1158,10 @@ def to_dict(self) -> Dict[str, Any]: description=self.description, validators=self.validators, output_schema=self.output_schema, - i_history=GuardHistory(list(self.history)), # type: ignore + history=[c.to_interface() for c in self.history], # type: ignore ) - i_guard_dict = i_guard.to_dict() - - i_guard_dict["history"] = [ - call.to_dict() for call in i_guard_dict.get("history", []) - ] - return i_guard_dict + return i_guard.to_dict() def add_json_function_calling_tool( self, @@ -1307,10 +1193,11 @@ def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional["Guard"]: validators=i_guard.validators, output_schema=output_schema, ) - i_history = ( - i_guard.i_history.actual_instance - if i_guard.i_history and i_guard.i_history.actual_instance + + history = ( + [Call.from_interface(i_call) for i_call in i_guard.history] + if i_guard.history else [] ) - guard._history = Stack(*i_history) + guard.history = Stack(*history) return guard diff --git a/guardrails/run/async_runner.py b/guardrails/run/async_runner.py index 7e91d135c..c3b7a779c 100644 --- a/guardrails/run/async_runner.py +++ b/guardrails/run/async_runner.py @@ -139,11 +139,11 @@ async def async_run( ) except UserFacingException as e: # Because Pydantic v1 doesn't respect property setters - call_log._set_exception(e.original_exception) + call_log.exception = e.original_exception raise e.original_exception except Exception as e: # Because Pydantic v1 doesn't respect property setters - call_log._set_exception(e) + call_log.exception = e raise e return call_log @@ -177,7 +177,9 @@ async def async_step( full_schema_reask=self.full_schema_reask, ) outputs = Outputs() - iteration = Iteration(inputs=inputs, outputs=outputs) + iteration = Iteration( + call_id=call_log.id, index=index, inputs=inputs, outputs=outputs + ) set_scope(str(id(iteration))) call_log.iterations.push(iteration) @@ -370,7 +372,9 @@ async def async_prepare( inputs = Inputs( llm_output=msg_str, ) - iteration = Iteration(inputs=inputs) + iteration = Iteration( + call_id=call_log.id, index=attempt_number, inputs=inputs + ) call_log.iterations.insert(0, iteration) value, _metadata = await validator_service.async_validate( value=msg_str, @@ -426,7 +430,9 @@ async def async_prepare( inputs = Inputs( llm_output=prompt.source, ) - iteration = Iteration(inputs=inputs) + iteration = Iteration( + call_id=call_log.id, index=attempt_number, inputs=inputs + ) call_log.iterations.insert(0, iteration) value, _metadata = await validator_service.async_validate( value=prompt.source, @@ -455,7 +461,9 @@ async def async_prepare( inputs = Inputs( llm_output=instructions.source, ) - iteration = Iteration(inputs=inputs) + iteration = Iteration( + call_id=call_log.id, index=attempt_number, inputs=inputs + ) call_log.iterations.insert(0, iteration) value, _metadata = await validator_service.async_validate( value=instructions.source, diff --git a/guardrails/run/async_stream_runner.py b/guardrails/run/async_stream_runner.py index 35321317c..cb89f0f9e 100644 --- a/guardrails/run/async_stream_runner.py +++ b/guardrails/run/async_stream_runner.py @@ -90,7 +90,9 @@ async def async_step( stream=True, ) outputs = Outputs() - iteration = Iteration(inputs=inputs, outputs=outputs) + iteration = Iteration( + call_id=call_log.id, index=index, inputs=inputs, outputs=outputs + ) set_scope(str(id(iteration))) call_log.iterations.push(iteration) if output: @@ -159,6 +161,7 @@ async def async_step( ) passed = call_log.status == pass_status yield ValidationOutcome( + call_id=call_log.id, # type: ignore raw_llm_output=chunk_text, validated_output=validated_fragment, validation_passed=passed, @@ -194,6 +197,7 @@ async def async_step( ) yield ValidationOutcome( + call_id=call_log.id, # type: ignore raw_llm_output=fragment, validated_output=chunk_text, validation_passed=validated_fragment is not None, diff --git a/guardrails/run/runner.py b/guardrails/run/runner.py index 827e85b75..58839f410 100644 --- a/guardrails/run/runner.py +++ b/guardrails/run/runner.py @@ -237,11 +237,11 @@ def __call__(self, call_log: Call, prompt_params: Optional[Dict] = None) -> Call except UserFacingException as e: # Because Pydantic v1 doesn't respect property setters - call_log._set_exception(e.original_exception) + call_log.exception = e.original_exception raise e.original_exception except Exception as e: # Because Pydantic v1 doesn't respect property setters - call_log._set_exception(e) + call_log.exception = e raise e return call_log @@ -273,7 +273,9 @@ def step( full_schema_reask=self.full_schema_reask, ) outputs = Outputs() - iteration = Iteration(inputs=inputs, outputs=outputs) + iteration = Iteration( + call_id=call_log.id, index=index, inputs=inputs, outputs=outputs + ) set_scope(str(id(iteration))) call_log.iterations.push(iteration) @@ -343,7 +345,7 @@ def validate_msg_history( inputs = Inputs( llm_output=msg_str, ) - iteration = Iteration(inputs=inputs) + iteration = Iteration(call_id=call_log.id, index=attempt_number, inputs=inputs) call_log.iterations.insert(0, iteration) value, _metadata = validator_service.validate( value=msg_str, @@ -389,7 +391,7 @@ def validate_prompt(self, call_log: Call, prompt: Prompt, attempt_number: int): inputs = Inputs( llm_output=prompt.source, ) - iteration = Iteration(inputs=inputs) + iteration = Iteration(call_id=call_log.id, index=attempt_number, inputs=inputs) call_log.iterations.insert(0, iteration) value, _metadata = validator_service.validate( value=prompt.source, @@ -418,7 +420,7 @@ def validate_instructions( inputs = Inputs( llm_output=instructions.source, ) - iteration = Iteration(inputs=inputs) + iteration = Iteration(call_id=call_log.id, index=attempt_number, inputs=inputs) call_log.iterations.insert(0, iteration) value, _metadata = validator_service.validate( value=instructions.source, diff --git a/guardrails/run/stream_runner.py b/guardrails/run/stream_runner.py index 71de8ba2d..a968548e3 100644 --- a/guardrails/run/stream_runner.py +++ b/guardrails/run/stream_runner.py @@ -98,7 +98,9 @@ def step( stream=True, ) outputs = Outputs() - iteration = Iteration(inputs=inputs, outputs=outputs) + iteration = Iteration( + call_id=call_log.id, index=index, inputs=inputs, outputs=outputs + ) call_log.iterations.push(iteration) # Prepare: run pre-processing, and input validation. @@ -184,6 +186,7 @@ def step( # 5. Convert validated fragment to a pretty JSON string passed = call_log.status == pass_status yield ValidationOutcome( + call_id=call_log.id, # type: ignore # The chunk or the whole output? raw_llm_output=chunk_text, validated_output=validated_text, @@ -212,6 +215,7 @@ def step( reask = last_result yield ValidationOutcome( + call_id=call_log.id, # type: ignore raw_llm_output=last_chunk_text, validated_output=validated_output, reask=reask, @@ -256,6 +260,7 @@ def step( # 5. Convert validated fragment to a pretty JSON string yield ValidationOutcome( + call_id=call_log.id, # type: ignore raw_llm_output=fragment, validated_output=validated_fragment, validation_passed=validated_fragment is not None, diff --git a/poetry.lock b/poetry.lock index 20294a95f..bfae6c428 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,10 +1,10 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "aiohttp" version = "3.9.5" description = "Async http client/server framework (asyncio)" -optional = true +optional = false python-versions = ">=3.8" files = [ {file = "aiohttp-3.9.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:fcde4c397f673fdec23e6b05ebf8d4751314fa7c24f93334bf1f1364c1c69ac7"}, @@ -100,7 +100,7 @@ speedups = ["Brotli", "aiodns", "brotlicffi"] name = "aiosignal" version = "1.3.1" description = "aiosignal: a list of registered asynchronous callbacks" -optional = true +optional = false python-versions = ">=3.7" files = [ {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"}, @@ -297,7 +297,7 @@ typing-extensions = {version = ">=4.0.0", markers = "python_version < \"3.11\""} name = "async-timeout" version = "4.0.3" description = "Timeout context manager for asyncio programs" -optional = true +optional = false python-versions = ">=3.7" files = [ {file = "async-timeout-4.0.3.tar.gz", hash = "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f"}, @@ -450,6 +450,69 @@ webencodings = "*" [package.extras] css = ["tinycss2 (>=1.1.0,<1.3)"] +[[package]] +name = "blinker" +version = "1.8.2" +description = "Fast, simple object-to-object and broadcast signaling" +optional = false +python-versions = ">=3.8" +files = [ + {file = "blinker-1.8.2-py3-none-any.whl", hash = "sha256:1779309f71bf239144b9399d06ae925637cf6634cf6bd131104184531bf67c01"}, + {file = "blinker-1.8.2.tar.gz", hash = "sha256:8f77b09d3bf7c795e969e9486f39c2c5e9c39d4ee07424be2bc594ece9642d83"}, +] + +[[package]] +name = "boto3" +version = "1.34.132" +description = "The AWS SDK for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "boto3-1.34.132-py3-none-any.whl", hash = "sha256:b5d1681a0d8bf255787c8b37f911d706672d5722c9ace5342cd283a3cdb04820"}, + {file = "boto3-1.34.132.tar.gz", hash = "sha256:3b2964060620f1bbe9574b5f8d3fb2a4e087faacfc6023c24154b184f1b16443"}, +] + +[package.dependencies] +botocore = ">=1.34.132,<1.35.0" +jmespath = ">=0.7.1,<2.0.0" +s3transfer = ">=0.10.0,<0.11.0" + +[package.extras] +crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] + +[[package]] +name = "botocore" +version = "1.34.132" +description = "Low-level, data-driven core of boto 3." +optional = false +python-versions = ">=3.8" +files = [ + {file = "botocore-1.34.132-py3-none-any.whl", hash = "sha256:06ef8b4bd3b3cb5a9b9a4273a543b257be3304030978ba51516b576a65156c39"}, + {file = "botocore-1.34.132.tar.gz", hash = "sha256:372a6cfce29e5de9bcf8c95af901d0bc3e27d8aa2295fadee295424f95f43f16"}, +] + +[package.dependencies] +jmespath = ">=0.7.1,<2.0.0" +python-dateutil = ">=2.1,<3.0.0" +urllib3 = [ + {version = ">=1.25.4,<1.27", markers = "python_version < \"3.10\""}, + {version = ">=1.25.4,<2.2.0 || >2.2.0,<3", markers = "python_version >= \"3.10\""}, +] + +[package.extras] +crt = ["awscrt (==0.20.11)"] + +[[package]] +name = "cachelib" +version = "0.9.0" +description = "A collection of cache libraries in the same API interface." +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachelib-0.9.0-py3-none-any.whl", hash = "sha256:811ceeb1209d2fe51cd2b62810bd1eccf70feba5c52641532498be5c675493b3"}, + {file = "cachelib-0.9.0.tar.gz", hash = "sha256:38222cc7c1b79a23606de5c2607f4925779e37cdcea1c2ad21b8bae94b5425a5"}, +] + [[package]] name = "cairocffi" version = "1.7.0" @@ -1245,6 +1308,73 @@ docs = ["furo (>=2023.9.10)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1 testing = ["covdefaults (>=2.3)", "coverage (>=7.3.2)", "diff-cover (>=8.0.1)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)", "pytest-timeout (>=2.2)"] typing = ["typing-extensions (>=4.8)"] +[[package]] +name = "flask" +version = "3.0.3" +description = "A simple framework for building complex web applications." +optional = false +python-versions = ">=3.8" +files = [ + {file = "flask-3.0.3-py3-none-any.whl", hash = "sha256:34e815dfaa43340d1d15a5c3a02b8476004037eb4840b34910c6e21679d288f3"}, + {file = "flask-3.0.3.tar.gz", hash = "sha256:ceb27b0af3823ea2737928a4d99d125a06175b8512c445cbd9a9ce200ef76842"}, +] + +[package.dependencies] +blinker = ">=1.6.2" +click = ">=8.1.3" +importlib-metadata = {version = ">=3.6.0", markers = "python_version < \"3.10\""} +itsdangerous = ">=2.1.2" +Jinja2 = ">=3.1.2" +Werkzeug = ">=3.0.0" + +[package.extras] +async = ["asgiref (>=3.2)"] +dotenv = ["python-dotenv"] + +[[package]] +name = "flask-caching" +version = "2.3.0" +description = "Adds caching support to Flask applications." +optional = false +python-versions = ">=3.8" +files = [ + {file = "Flask_Caching-2.3.0-py3-none-any.whl", hash = "sha256:51771c75682e5abc1483b78b96d9131d7941dc669b073852edfa319dd4e29b6e"}, + {file = "flask_caching-2.3.0.tar.gz", hash = "sha256:d7e4ca64a33b49feb339fcdd17e6ba25f5e01168cf885e53790e885f83a4d2cf"}, +] + +[package.dependencies] +cachelib = ">=0.9.0,<0.10.0" +Flask = "*" + +[[package]] +name = "flask-cors" +version = "4.0.1" +description = "A Flask extension adding a decorator for CORS support" +optional = false +python-versions = "*" +files = [ + {file = "Flask_Cors-4.0.1-py2.py3-none-any.whl", hash = "sha256:f2a704e4458665580c074b714c4627dd5a306b333deb9074d0b1794dfa2fb677"}, + {file = "flask_cors-4.0.1.tar.gz", hash = "sha256:eeb69b342142fdbf4766ad99357a7f3876a2ceb77689dc10ff912aac06c389e4"}, +] + +[package.dependencies] +Flask = ">=0.9" + +[[package]] +name = "flask-sqlalchemy" +version = "3.1.1" +description = "Add SQLAlchemy support to your Flask application." +optional = false +python-versions = ">=3.8" +files = [ + {file = "flask_sqlalchemy-3.1.1-py3-none-any.whl", hash = "sha256:4ba4be7f419dc72f4efd8802d69974803c37259dd42f3913b0dcf75c9447e0a0"}, + {file = "flask_sqlalchemy-3.1.1.tar.gz", hash = "sha256:e4b68bb881802dda1a7d878b2fc84c06d1ee57fb40b874d3dc97dabfa36b8312"}, +] + +[package.dependencies] +flask = ">=2.2.5" +sqlalchemy = ">=2.0.16" + [[package]] name = "fqdn" version = "1.5.1" @@ -1260,7 +1390,7 @@ files = [ name = "frozenlist" version = "1.4.1" description = "A list-like structure which implements collections.abc.MutableSequence" -optional = true +optional = false python-versions = ">=3.8" files = [ {file = "frozenlist-1.4.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:f9aa1878d1083b276b0196f2dfbe00c9b7e752475ed3b682025ff20c1c1f51ac"}, @@ -1346,7 +1476,7 @@ files = [ name = "fsspec" version = "2024.6.0" description = "File-system specification" -optional = true +optional = false python-versions = ">=3.8" files = [ {file = "fsspec-2024.6.0-py3-none-any.whl", hash = "sha256:58d7122eb8a1a46f7f13453187bfea4972d66bf01618d37366521b1998034cee"}, @@ -1470,7 +1600,7 @@ grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"] name = "greenlet" version = "3.0.3" description = "Lightweight in-process concurrent programming" -optional = true +optional = false python-versions = ">=3.7" files = [ {file = "greenlet-3.0.3-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:9da2bd29ed9e4f15955dd1595ad7bc9320308a3b766ef7f837e23ad4b4aac31a"}, @@ -1609,20 +1739,75 @@ files = [ [package.extras] protobuf = ["grpcio-tools (>=1.64.0)"] +[[package]] +name = "guardrails-api" +version = "0.0.0a0" +description = "Guardrails API" +optional = false +python-versions = "<4,>=3.8" +files = [ + {file = "guardrails_api-0.0.0a0-py3-none-any.whl", hash = "sha256:d0cbd26b755a5d3b932d6b17e9a0982c8c011517e4904aca8eab75ab471b6ca9"}, + {file = "guardrails_api-0.0.0a0.tar.gz", hash = "sha256:e6ca674ce1627b273a2fe0f1a61cb8c2c331f733d2c1ad09cd92be36f5da7bec"}, +] + +[package.dependencies] +boto3 = ">=1.34.115,<2" +flask = ">=3.0.3,<4" +Flask-Caching = ">=2.3.0,<3" +Flask-Cors = ">=4.0.1,<5" +Flask-SQLAlchemy = ">=3.1.1,<4" +guardrails-ai = ">=0.5.0a2" +gunicorn = ">=22.0.0,<23" +jsonschema = ">=4.22.0,<5" +litellm = ">=1.39.3,<2" +opentelemetry-api = ">=1.0.0,<2" +opentelemetry-exporter-otlp-proto-grpc = ">=1.0.0,<2" +opentelemetry-exporter-otlp-proto-http = ">=1.0.0,<2" +opentelemetry-instrumentation-flask = ">=0.12b0,<1" +opentelemetry-sdk = ">=1.0.0,<2" +psycopg2-binary = ">=2.9.9,<3" +referencing = ">=0.35.1,<1" +typer = ">=0.9.4,<1" +Werkzeug = ">=3.0.3,<4" + +[package.extras] +dev = ["coverage", "pytest", "pytest-mock", "ruff"] + [[package]] name = "guardrails-api-client" -version = "0.3.4" +version = "0.3.8" description = "Guardrails API Client." optional = false python-versions = "<4,>=3.8" files = [ - {file = "guardrails_api_client-0.3.4-py3-none-any.whl", hash = "sha256:4a6b28d11848c129d5474663e3bed9be3180c25a303709c388da62054b2c0621"}, - {file = "guardrails_api_client-0.3.4.tar.gz", hash = "sha256:8b2da58baaa5449c06a4f74828078a79ce8e27121f600764e2a5b7be6aa0355c"}, + {file = "guardrails_api_client-0.3.8-py3-none-any.whl", hash = "sha256:2becd5ac9c720879a997e50e2a813d9e77a6fb1a7068a96b1b9712dd6dd9efb8"}, + {file = "guardrails_api_client-0.3.8.tar.gz", hash = "sha256:2e1e45cddf727534a378bc99f7f98da0800960918c75bda010b8bc493b946d16"}, ] [package.extras] dev = ["pyright", "pytest", "pytest-cov", "ruff"] +[[package]] +name = "gunicorn" +version = "22.0.0" +description = "WSGI HTTP Server for UNIX" +optional = false +python-versions = ">=3.7" +files = [ + {file = "gunicorn-22.0.0-py3-none-any.whl", hash = "sha256:350679f91b24062c86e386e198a15438d53a7a8207235a78ba1b53df4c4378d9"}, + {file = "gunicorn-22.0.0.tar.gz", hash = "sha256:4a0b436239ff76fb33f11c07a16482c521a7e09c1ce3cc293c2330afe01bec63"}, +] + +[package.dependencies] +packaging = "*" + +[package.extras] +eventlet = ["eventlet (>=0.24.1,!=0.36.0)"] +gevent = ["gevent (>=1.4.0)"] +setproctitle = ["setproctitle"] +testing = ["coverage", "eventlet", "gevent", "pytest", "pytest-cov"] +tornado = ["tornado (>=0.2)"] + [[package]] name = "h11" version = "0.14.0" @@ -1683,7 +1868,7 @@ socks = ["socksio (==1.*)"] name = "huggingface-hub" version = "0.19.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" -optional = true +optional = false python-versions = ">=3.8.0" files = [ {file = "huggingface_hub-0.19.4-py3-none-any.whl", hash = "sha256:dba013f779da16f14b606492828f3760600a1e1801432d09fe1c33e50b825bb5"}, @@ -1913,6 +2098,17 @@ files = [ [package.dependencies] arrow = ">=0.15.0" +[[package]] +name = "itsdangerous" +version = "2.2.0" +description = "Safely pass data to untrusted environments and back." +optional = false +python-versions = ">=3.8" +files = [ + {file = "itsdangerous-2.2.0-py3-none-any.whl", hash = "sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef"}, + {file = "itsdangerous-2.2.0.tar.gz", hash = "sha256:e0050c0b7da1eea53ffaf149c0cfbb5c6e2e2b69c4bef22c81fa6eb73e5f6173"}, +] + [[package]] name = "jaraco-classes" version = "3.4.0" @@ -2018,6 +2214,17 @@ MarkupSafe = ">=2.0" [package.extras] i18n = ["Babel (>=2.7)"] +[[package]] +name = "jmespath" +version = "1.0.1" +description = "JSON Matching Expressions" +optional = false +python-versions = ">=3.7" +files = [ + {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, + {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, +] + [[package]] name = "joblib" version = "1.4.2" @@ -2508,7 +2715,7 @@ requests = ">=2,<3" name = "litellm" version = "1.40.2" description = "Library to easily interface with LLM API providers" -optional = true +optional = false python-versions = "!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*,>=3.8" files = [ {file = "litellm-1.40.2-py3-none-any.whl", hash = "sha256:56ee777eed30ee9acb86e74401d090dcac4adb57b5c8a8714f791b0c97a34afc"}, @@ -3101,7 +3308,7 @@ tests = ["pytest (>=4.6)"] name = "multidict" version = "6.0.5" description = "multidict implementation" -optional = true +optional = false python-versions = ">=3.7" files = [ {file = "multidict-6.0.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:228b644ae063c10e7f324ab1ab6b548bdf6f8b47f3ec234fef1093bc2735e5f9"}, @@ -3799,6 +4006,62 @@ opentelemetry-proto = "1.24.0" opentelemetry-sdk = ">=1.24.0,<1.25.0" requests = ">=2.7,<3.0" +[[package]] +name = "opentelemetry-instrumentation" +version = "0.45b0" +description = "Instrumentation Tools & Auto Instrumentation for OpenTelemetry Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "opentelemetry_instrumentation-0.45b0-py3-none-any.whl", hash = "sha256:06c02e2c952c1b076e8eaedf1b82f715e2937ba7eeacab55913dd434fbcec258"}, + {file = "opentelemetry_instrumentation-0.45b0.tar.gz", hash = "sha256:6c47120a7970bbeb458e6a73686ee9ba84b106329a79e4a4a66761f933709c7e"}, +] + +[package.dependencies] +opentelemetry-api = ">=1.4,<2.0" +setuptools = ">=16.0" +wrapt = ">=1.0.0,<2.0.0" + +[[package]] +name = "opentelemetry-instrumentation-flask" +version = "0.45b0" +description = "Flask instrumentation for OpenTelemetry" +optional = false +python-versions = ">=3.8" +files = [ + {file = "opentelemetry_instrumentation_flask-0.45b0-py3-none-any.whl", hash = "sha256:4a07d1bca110dff0e2ce51a2930df497e90982e3a36e0362272fa5080db7f851"}, + {file = "opentelemetry_instrumentation_flask-0.45b0.tar.gz", hash = "sha256:70875ad03da6e4e07aada6795c65d6b919024a741f9295aa19a022f7f7afc900"}, +] + +[package.dependencies] +importlib-metadata = ">=4.0" +opentelemetry-api = ">=1.12,<2.0" +opentelemetry-instrumentation = "0.45b0" +opentelemetry-instrumentation-wsgi = "0.45b0" +opentelemetry-semantic-conventions = "0.45b0" +opentelemetry-util-http = "0.45b0" +packaging = ">=21.0" + +[package.extras] +instruments = ["flask (>=1.0)"] + +[[package]] +name = "opentelemetry-instrumentation-wsgi" +version = "0.45b0" +description = "WSGI Middleware for OpenTelemetry" +optional = false +python-versions = ">=3.8" +files = [ + {file = "opentelemetry_instrumentation_wsgi-0.45b0-py3-none-any.whl", hash = "sha256:7a6f9c71b25f5c5e112827540008882f6a9088447cb65745e7f2083749516663"}, + {file = "opentelemetry_instrumentation_wsgi-0.45b0.tar.gz", hash = "sha256:f53a2a38e6582406e207d404e4c1b859b83bec11a68ad6c7366642d01c873ad0"}, +] + +[package.dependencies] +opentelemetry-api = ">=1.12,<2.0" +opentelemetry-instrumentation = "0.45b0" +opentelemetry-semantic-conventions = "0.45b0" +opentelemetry-util-http = "0.45b0" + [[package]] name = "opentelemetry-proto" version = "1.24.0" @@ -3840,6 +4103,17 @@ files = [ {file = "opentelemetry_semantic_conventions-0.45b0.tar.gz", hash = "sha256:7c84215a44ac846bc4b8e32d5e78935c5c43482e491812a0bb8aaf87e4d92118"}, ] +[[package]] +name = "opentelemetry-util-http" +version = "0.45b0" +description = "Web util for OpenTelemetry" +optional = false +python-versions = ">=3.8" +files = [ + {file = "opentelemetry_util_http-0.45b0-py3-none-any.whl", hash = "sha256:6628868b501b3004e1860f976f410eeb3d3499e009719d818000f24ce17b6e33"}, + {file = "opentelemetry_util_http-0.45b0.tar.gz", hash = "sha256:4ce08b6a7d52dd7c96b7705b5b4f06fdb6aa3eac1233b3b0bfef8a0cab9a92cd"}, +] + [[package]] name = "orjson" version = "3.10.3" @@ -4225,6 +4499,87 @@ files = [ [package.extras] test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] +[[package]] +name = "psycopg2-binary" +version = "2.9.9" +description = "psycopg2 - Python-PostgreSQL Database Adapter" +optional = false +python-versions = ">=3.7" +files = [ + {file = "psycopg2-binary-2.9.9.tar.gz", hash = "sha256:7f01846810177d829c7692f1f5ada8096762d9172af1b1a28d4ab5b77c923c1c"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c2470da5418b76232f02a2fcd2229537bb2d5a7096674ce61859c3229f2eb202"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c6af2a6d4b7ee9615cbb162b0738f6e1fd1f5c3eda7e5da17861eacf4c717ea7"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:75723c3c0fbbf34350b46a3199eb50638ab22a0228f93fb472ef4d9becc2382b"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:83791a65b51ad6ee6cf0845634859d69a038ea9b03d7b26e703f94c7e93dbcf9"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0ef4854e82c09e84cc63084a9e4ccd6d9b154f1dbdd283efb92ecd0b5e2b8c84"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ed1184ab8f113e8d660ce49a56390ca181f2981066acc27cf637d5c1e10ce46e"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d2997c458c690ec2bc6b0b7ecbafd02b029b7b4283078d3b32a852a7ce3ddd98"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:b58b4710c7f4161b5e9dcbe73bb7c62d65670a87df7bcce9e1faaad43e715245"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:0c009475ee389757e6e34611d75f6e4f05f0cf5ebb76c6037508318e1a1e0d7e"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8dbf6d1bc73f1d04ec1734bae3b4fb0ee3cb2a493d35ede9badbeb901fb40f6f"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-win32.whl", hash = "sha256:3f78fd71c4f43a13d342be74ebbc0666fe1f555b8837eb113cb7416856c79682"}, + {file = "psycopg2_binary-2.9.9-cp310-cp310-win_amd64.whl", hash = "sha256:876801744b0dee379e4e3c38b76fc89f88834bb15bf92ee07d94acd06ec890a0"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ee825e70b1a209475622f7f7b776785bd68f34af6e7a46e2e42f27b659b5bc26"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1ea665f8ce695bcc37a90ee52de7a7980be5161375d42a0b6c6abedbf0d81f0f"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:143072318f793f53819048fdfe30c321890af0c3ec7cb1dfc9cc87aa88241de2"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c332c8d69fb64979ebf76613c66b985414927a40f8defa16cf1bc028b7b0a7b0"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f7fc5a5acafb7d6ccca13bfa8c90f8c51f13d8fb87d95656d3950f0158d3ce53"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:977646e05232579d2e7b9c59e21dbe5261f403a88417f6a6512e70d3f8a046be"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b6356793b84728d9d50ead16ab43c187673831e9d4019013f1402c41b1db9b27"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:bc7bb56d04601d443f24094e9e31ae6deec9ccb23581f75343feebaf30423359"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:77853062a2c45be16fd6b8d6de2a99278ee1d985a7bd8b103e97e41c034006d2"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:78151aa3ec21dccd5cdef6c74c3e73386dcdfaf19bced944169697d7ac7482fc"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-win32.whl", hash = "sha256:dc4926288b2a3e9fd7b50dc6a1909a13bbdadfc67d93f3374d984e56f885579d"}, + {file = "psycopg2_binary-2.9.9-cp311-cp311-win_amd64.whl", hash = "sha256:b76bedd166805480ab069612119ea636f5ab8f8771e640ae103e05a4aae3e417"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:8532fd6e6e2dc57bcb3bc90b079c60de896d2128c5d9d6f24a63875a95a088cf"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0605eaed3eb239e87df0d5e3c6489daae3f7388d455d0c0b4df899519c6a38d"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f8544b092a29a6ddd72f3556a9fcf249ec412e10ad28be6a0c0d948924f2212"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2d423c8d8a3c82d08fe8af900ad5b613ce3632a1249fd6a223941d0735fce493"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2e5afae772c00980525f6d6ecf7cbca55676296b580c0e6abb407f15f3706996"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e6f98446430fdf41bd36d4faa6cb409f5140c1c2cf58ce0bbdaf16af7d3f119"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c77e3d1862452565875eb31bdb45ac62502feabbd53429fdc39a1cc341d681ba"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:cb16c65dcb648d0a43a2521f2f0a2300f40639f6f8c1ecbc662141e4e3e1ee07"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:911dda9c487075abd54e644ccdf5e5c16773470a6a5d3826fda76699410066fb"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:57fede879f08d23c85140a360c6a77709113efd1c993923c59fde17aa27599fe"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-win32.whl", hash = "sha256:64cf30263844fa208851ebb13b0732ce674d8ec6a0c86a4e160495d299ba3c93"}, + {file = "psycopg2_binary-2.9.9-cp312-cp312-win_amd64.whl", hash = "sha256:81ff62668af011f9a48787564ab7eded4e9fb17a4a6a74af5ffa6a457400d2ab"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2293b001e319ab0d869d660a704942c9e2cce19745262a8aba2115ef41a0a42a"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:03ef7df18daf2c4c07e2695e8cfd5ee7f748a1d54d802330985a78d2a5a6dca9"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a602ea5aff39bb9fac6308e9c9d82b9a35c2bf288e184a816002c9fae930b77"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8359bf4791968c5a78c56103702000105501adb557f3cf772b2c207284273984"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:275ff571376626195ab95a746e6a04c7df8ea34638b99fc11160de91f2fef503"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:f9b5571d33660d5009a8b3c25dc1db560206e2d2f89d3df1cb32d72c0d117d52"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:420f9bbf47a02616e8554e825208cb947969451978dceb77f95ad09c37791dae"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:4154ad09dac630a0f13f37b583eae260c6aa885d67dfbccb5b02c33f31a6d420"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a148c5d507bb9b4f2030a2025c545fccb0e1ef317393eaba42e7eabd28eb6041"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-win32.whl", hash = "sha256:68fc1f1ba168724771e38bee37d940d2865cb0f562380a1fb1ffb428b75cb692"}, + {file = "psycopg2_binary-2.9.9-cp37-cp37m-win_amd64.whl", hash = "sha256:281309265596e388ef483250db3640e5f414168c5a67e9c665cafce9492eda2f"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:60989127da422b74a04345096c10d416c2b41bd7bf2a380eb541059e4e999980"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:246b123cc54bb5361588acc54218c8c9fb73068bf227a4a531d8ed56fa3ca7d6"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:34eccd14566f8fe14b2b95bb13b11572f7c7d5c36da61caf414d23b91fcc5d94"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:18d0ef97766055fec15b5de2c06dd8e7654705ce3e5e5eed3b6651a1d2a9a152"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d3f82c171b4ccd83bbaf35aa05e44e690113bd4f3b7b6cc54d2219b132f3ae55"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ead20f7913a9c1e894aebe47cccf9dc834e1618b7aa96155d2091a626e59c972"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:ca49a8119c6cbd77375ae303b0cfd8c11f011abbbd64601167ecca18a87e7cdd"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:323ba25b92454adb36fa425dc5cf6f8f19f78948cbad2e7bc6cdf7b0d7982e59"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:1236ed0952fbd919c100bc839eaa4a39ebc397ed1c08a97fc45fee2a595aa1b3"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:729177eaf0aefca0994ce4cffe96ad3c75e377c7b6f4efa59ebf003b6d398716"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-win32.whl", hash = "sha256:804d99b24ad523a1fe18cc707bf741670332f7c7412e9d49cb5eab67e886b9b5"}, + {file = "psycopg2_binary-2.9.9-cp38-cp38-win_amd64.whl", hash = "sha256:a6cdcc3ede532f4a4b96000b6362099591ab4a3e913d70bcbac2b56c872446f7"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:72dffbd8b4194858d0941062a9766f8297e8868e1dd07a7b36212aaa90f49472"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:30dcc86377618a4c8f3b72418df92e77be4254d8f89f14b8e8f57d6d43603c0f"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:31a34c508c003a4347d389a9e6fcc2307cc2150eb516462a7a17512130de109e"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:15208be1c50b99203fe88d15695f22a5bed95ab3f84354c494bcb1d08557df67"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1873aade94b74715be2246321c8650cabf5a0d098a95bab81145ffffa4c13876"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a58c98a7e9c021f357348867f537017057c2ed7f77337fd914d0bedb35dace7"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4686818798f9194d03c9129a4d9a702d9e113a89cb03bffe08c6cf799e053291"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:ebdc36bea43063116f0486869652cb2ed7032dbc59fbcb4445c4862b5c1ecf7f"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:ca08decd2697fdea0aea364b370b1249d47336aec935f87b8bbfd7da5b2ee9c1"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ac05fb791acf5e1a3e39402641827780fe44d27e72567a000412c648a85ba860"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-win32.whl", hash = "sha256:9dba73be7305b399924709b91682299794887cbbd88e38226ed9f6712eabee90"}, + {file = "psycopg2_binary-2.9.9-cp39-cp39-win_amd64.whl", hash = "sha256:f7ae5d65ccfbebdfa761585228eb4d0df3a8b15cfb53bd953e713e09fbb12957"}, +] + [[package]] name = "ptyprocess" version = "0.7.0" @@ -4591,7 +4946,7 @@ six = ">=1.5" name = "python-dotenv" version = "1.0.1" description = "Read key-value pairs from a .env file and set them as environment variables" -optional = true +optional = false python-versions = ">=3.8" files = [ {file = "python-dotenv-1.0.1.tar.gz", hash = "sha256:e324ee90a023d808f1959c46bcbc04446a10ced277783dc6ee09987c37ec10ca"}, @@ -5279,6 +5634,23 @@ files = [ {file = "ruff-0.4.7.tar.gz", hash = "sha256:2331d2b051dc77a289a653fcc6a42cce357087c5975738157cd966590b18b5e1"}, ] +[[package]] +name = "s3transfer" +version = "0.10.2" +description = "An Amazon S3 Transfer Manager" +optional = false +python-versions = ">=3.8" +files = [ + {file = "s3transfer-0.10.2-py3-none-any.whl", hash = "sha256:eca1c20de70a39daee580aef4986996620f365c4e0fda6a86100231d62f1bf69"}, + {file = "s3transfer-0.10.2.tar.gz", hash = "sha256:0711534e9356d3cc692fdde846b4a1e4b0cb6519971860796e6bc4c7aea00ef6"}, +] + +[package.dependencies] +botocore = ">=1.33.2,<2.0a.0" + +[package.extras] +crt = ["botocore[crt] (>=1.33.2,<2.0a.0)"] + [[package]] name = "safetensors" version = "0.4.3" @@ -5641,7 +6013,7 @@ test = ["pytest"] name = "sqlalchemy" version = "2.0.30" description = "Database Abstraction Library" -optional = true +optional = false python-versions = ">=3.7" files = [ {file = "SQLAlchemy-2.0.30-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3b48154678e76445c7ded1896715ce05319f74b1e73cf82d4f8b59b46e9c0ddc"}, @@ -5943,7 +6315,7 @@ files = [ name = "tokenizers" version = "0.15.2" description = "" -optional = true +optional = false python-versions = ">=3.7" files = [ {file = "tokenizers-0.15.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:52f6130c9cbf70544287575a985bf44ae1bda2da7e8c24e97716080593638012"}, @@ -6394,6 +6766,22 @@ files = [ [package.extras] dev = ["flake8", "flake8-annotations", "flake8-bandit", "flake8-bugbear", "flake8-commas", "flake8-comprehensions", "flake8-continuation", "flake8-datetimez", "flake8-docstrings", "flake8-import-order", "flake8-literal", "flake8-modern-annotations", "flake8-noqa", "flake8-pyproject", "flake8-requirements", "flake8-typechecking-import", "flake8-use-fstring", "mypy", "pep8-naming", "types-PyYAML"] +[[package]] +name = "urllib3" +version = "1.26.19" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +files = [ + {file = "urllib3-1.26.19-py2.py3-none-any.whl", hash = "sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3"}, + {file = "urllib3-1.26.19.tar.gz", hash = "sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429"}, +] + +[package.extras] +brotli = ["brotli (==1.0.9)", "brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + [[package]] name = "urllib3" version = "2.2.1" @@ -6528,6 +6916,23 @@ docs = ["Sphinx (>=6.0)", "myst-parser (>=2.0.0)", "sphinx-rtd-theme (>=1.1.0)"] optional = ["python-socks", "wsaccel"] test = ["websockets"] +[[package]] +name = "werkzeug" +version = "3.0.3" +description = "The comprehensive WSGI web application library." +optional = false +python-versions = ">=3.8" +files = [ + {file = "werkzeug-3.0.3-py3-none-any.whl", hash = "sha256:fc9645dc43e03e4d630d23143a04a7f947a9a3b5727cd535fdfe155a17cc48c8"}, + {file = "werkzeug-3.0.3.tar.gz", hash = "sha256:097e5bfda9f0aba8da6b8545146def481d06aa7d3266e7448e2cccf67dd8bd18"}, +] + +[package.dependencies] +MarkupSafe = ">=2.1.1" + +[package.extras] +watchdog = ["watchdog (>=2.3)"] + [[package]] name = "widgetsnbextension" version = "4.0.11" @@ -6755,7 +7160,7 @@ tomli = ">=2.0.1" name = "yarl" version = "1.9.4" description = "Yet another URL library" -optional = true +optional = false python-versions = ">=3.7" files = [ {file = "yarl-1.9.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a8c1df72eb746f4136fe9a2e72b0c9dc1da1cbd23b5372f94b5820ff8ae30e0e"}, @@ -6881,4 +7286,4 @@ vectordb = ["faiss-cpu", "numpy"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "78c2c1087b57fdef24163ed7bb78b3b8098f19a46efe369c3408b28cc550f823" +content-hash = "516e84db2a745f6f4063db8e501def11c1a070afbf04e24fae619e50ab462941" diff --git a/pyproject.toml b/pyproject.toml index d541b58bd..9138d73ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "guardrails-ai" -version = "0.5.0a1" +version = "0.5.0a3" description = "Adding guardrails to large language models." authors = ["Guardrails AI "] license = "Apache License 2.0" @@ -56,7 +56,7 @@ pip = ">=22" opentelemetry-sdk = "^1.24.0" opentelemetry-exporter-otlp-proto-grpc = "^1.24.0" opentelemetry-exporter-otlp-proto-http = "^1.24.0" -guardrails-api-client = "0.3.4" +guardrails-api-client = ">=0.3.8" [tool.poetry.extras] sql = ["sqlvalidator", "sqlalchemy", "sqlglot"] @@ -81,6 +81,12 @@ pyright = "1.1.334" lxml-stubs = "^0.4.0" ruff = ">=0.4.1" +[tool.poetry.group.api] +optional = true + +[tool.poetry.group.api.dependencies] +guardrails-api = "^0.0.0a0" + [tool.poetry.group.docs] optional = true diff --git a/pyrightconfig.json b/pyrightconfig.json new file mode 100644 index 000000000..101df4801 --- /dev/null +++ b/pyrightconfig.json @@ -0,0 +1,3 @@ +{ + "reportDeprecated": true +} \ No newline at end of file diff --git a/tests/integration_tests/test_guard.py b/tests/integration_tests/test_guard.py index c1ce66167..f781fe695 100644 --- a/tests/integration_tests/test_guard.py +++ b/tests/integration_tests/test_guard.py @@ -6,10 +6,11 @@ import pytest from pydantic import BaseModel, Field -from guardrails_api_client import Guard as IGuard, GuardHistory, ValidatorReference +from guardrails_api_client import Guard as IGuard, ValidatorReference import guardrails as gd from guardrails.actions.reask import SkeletonReAsk +from guardrails.classes.generic.stack import Stack from guardrails.classes.llm.llm_response import LLMResponse from guardrails.classes.validation_outcome import ValidationOutcome from guardrails.classes.validation.validation_result import FailResult @@ -1178,7 +1179,7 @@ def test_guard_i_guard(self): description=guard.description, validators=guard.validators, output_schema=guard.output_schema, - history=GuardHistory(guard.history), + history=guard.history, ) cls_guard = Guard( @@ -1188,6 +1189,7 @@ def test_guard_i_guard(self): output_schema=i_guard.output_schema.to_dict(), validators=i_guard.validators, ) + cls_guard.history = Stack(*i_guard.history) assert cls_guard == guard diff --git a/tests/integration_tests/test_run.py b/tests/integration_tests/test_run.py index 7e07d94f5..7190151d5 100644 --- a/tests/integration_tests/test_run.py +++ b/tests/integration_tests/test_run.py @@ -64,7 +64,10 @@ async def test_sync_async_validate_equivalence(mocker): ) ] - iteration = Iteration() + iteration = Iteration( + call_id="mock-call", + index=0, + ) parsed_output, _ = runner_instance(True).parse(OUTPUT, OUTPUT_SCHEMA) diff --git a/tests/unit_tests/classes/history/test_call.py b/tests/unit_tests/classes/history/test_call.py index 39735ad93..f1f2eb034 100644 --- a/tests/unit_tests/classes/history/test_call.py +++ b/tests/unit_tests/classes/history/test_call.py @@ -117,7 +117,9 @@ def custom_llm(): validator_logs=first_validator_logs, ) - first_iteration = Iteration(inputs=inputs, outputs=first_outputs) + first_iteration = Iteration( + call_id="mock-call", index=0, inputs=inputs, outputs=first_outputs + ) second_iter_prompt = Prompt(source="That wasn't quite right. Try again.") @@ -155,7 +157,9 @@ def custom_llm(): validator_logs=second_validator_logs, ) - second_iteration = Iteration(inputs=second_inputs, outputs=second_outputs) + second_iteration = Iteration( + call_id="mock-call", index=0, inputs=second_inputs, outputs=second_outputs + ) iterations: Stack[Iteration] = Stack(first_iteration, second_iteration) diff --git a/tests/unit_tests/classes/history/test_iteration.py b/tests/unit_tests/classes/history/test_iteration.py index 1dde35b0c..38bc05d73 100644 --- a/tests/unit_tests/classes/history/test_iteration.py +++ b/tests/unit_tests/classes/history/test_iteration.py @@ -13,7 +13,10 @@ def test_empty_initialization(): - iteration = Iteration() + iteration = Iteration( + call_id="mock-call", + index=0, + ) assert iteration.inputs == Inputs() assert iteration.outputs == Outputs() @@ -95,7 +98,7 @@ def test_non_empty_initialization(): error=error, ) - iteration = Iteration(inputs=inputs, outputs=outputs) + iteration = Iteration(call_id="mock-call", index=0, inputs=inputs, outputs=outputs) assert iteration.inputs == inputs assert iteration.outputs == outputs diff --git a/tests/unit_tests/cli/test_validate.py b/tests/unit_tests/cli/test_validate.py index 22a9ebc90..78cf4cffa 100644 --- a/tests/unit_tests/cli/test_validate.py +++ b/tests/unit_tests/cli/test_validate.py @@ -49,6 +49,7 @@ def parse(self, *args): parse_mock = mocker.patch.object(mock_guard, "parse") parse_mock.return_value = ValidationOutcome( + call_id="mock-call", raw_llm_output="output", validated_output="validated output", validation_passed=True, diff --git a/tests/unit_tests/test_async_validator_service.py b/tests/unit_tests/test_async_validator_service.py index e01beb02e..2c566035a 100644 --- a/tests/unit_tests/test_async_validator_service.py +++ b/tests/unit_tests/test_async_validator_service.py @@ -15,7 +15,10 @@ def test_validate_with_running_loop(mocker): - iteration = Iteration() + iteration = Iteration( + call_id="mock-call", + index=0, + ) with pytest.raises(RuntimeError) as e_info: mock_loop = MockLoop(True) mocker.patch("asyncio.get_event_loop", return_value=mock_loop) @@ -43,7 +46,10 @@ def test_validate_without_running_loop(mocker): mocker.patch.object(avs, "async_validate", async_validate_mock) loop_spy = mocker.spy(mock_loop, "run_until_complete") - iteration = Iteration() + iteration = Iteration( + call_id="mock-call", + index=0, + ) validated_value, validated_metadata = avs.validate( value=True, @@ -71,7 +77,10 @@ async def test_async_validate_with_children(mocker): value = {"a": 1} - iteration = Iteration() + iteration = Iteration( + call_id="mock-call", + index=0, + ) validated_value, validated_metadata = await avs.async_validate( value=value, @@ -103,7 +112,10 @@ async def test_async_validate_without_children(mocker): run_validators_mock = mocker.patch.object(avs, "run_validators") run_validators_mock.return_value = ("run_validators_mock", {"async": True}) - iteration = Iteration() + iteration = Iteration( + call_id="mock-call", + index=0, + ) validated_value, validated_metadata = await avs.async_validate( value="Hello world!", @@ -149,7 +161,10 @@ async def mock_async_validate(v, md, *args, **kwargs): } } - iteration = Iteration() + iteration = Iteration( + call_id="mock-call", + index=0, + ) validated_value, validated_metadata = await avs.validate_children( value=value.get("mock-parent-key"), @@ -230,7 +245,10 @@ async def mock_gather(*args): asyancio_gather_mock = mocker.patch("asyncio.gather", side_effect=mock_gather) - iteration = Iteration() + iteration = Iteration( + call_id="mock-call", + index=0, + ) value, metadata = await avs.run_validators( iteration=iteration, @@ -292,7 +310,10 @@ async def test_run_validators_with_override(mocker): asyancio_gather_mock = mocker.patch("asyncio.gather") - iteration = Iteration() + iteration = Iteration( + call_id="mock-call", + index=0, + ) value, metadata = await avs.run_validators( iteration=iteration, diff --git a/tests/unit_tests/test_validator_base.py b/tests/unit_tests/test_validator_base.py index ade7bfbd7..70fbee750 100644 --- a/tests/unit_tests/test_validator_base.py +++ b/tests/unit_tests/test_validator_base.py @@ -508,11 +508,11 @@ async def mock_llm_api(*args, **kwargs): [ ( OnFailAction.REASK, - "Prompt validation failed: incorrect_value='What kind of pet should I get?' fail_results=[FailResult(outcome='fail', error_message='must be exactly two words', fix_value='What kind', metadata=None, validated_chunk=None, error_spans=None)] path=None", # noqa - "Instructions validation failed: incorrect_value='What kind of pet should I get?' fail_results=[FailResult(outcome='fail', error_message='must be exactly two words', fix_value='What kind', metadata=None, validated_chunk=None, error_spans=None)] path=None", # noqa - "Message history validation failed: incorrect_value='What kind of pet should I get?' fail_results=[FailResult(outcome='fail', error_message='must be exactly two words', fix_value='What kind', metadata=None, validated_chunk=None, error_spans=None)] path=None", # noqa - "Prompt validation failed: incorrect_value='\\nThis is not two words\\n' fail_results=[FailResult(outcome='fail', error_message='must be exactly two words', fix_value='This is', metadata=None, validated_chunk=None, error_spans=None)] path=None", # noqa - "Instructions validation failed: incorrect_value='\\nThis also is not two words\\n' fail_results=[FailResult(outcome='fail', error_message='must be exactly two words', fix_value='This also', metadata=None, validated_chunk=None, error_spans=None)] path=None", # noqa + "Prompt validation failed: incorrect_value='What kind of pet should I get?' fail_results=[FailResult(outcome='fail', error_message='must be exactly two words', fix_value='What kind', error_spans=None, metadata=None, validated_chunk=None)] additional_properties={} path=None", # noqa + "Instructions validation failed: incorrect_value='What kind of pet should I get?' fail_results=[FailResult(outcome='fail', error_message='must be exactly two words', fix_value='What kind', error_spans=None, metadata=None, validated_chunk=None)] additional_properties={} path=None", # noqa + "Message history validation failed: incorrect_value='What kind of pet should I get?' fail_results=[FailResult(outcome='fail', error_message='must be exactly two words', fix_value='What kind', error_spans=None, metadata=None, validated_chunk=None)] additional_properties={} path=None", # noqa + "Prompt validation failed: incorrect_value='\\nThis is not two words\\n' fail_results=[FailResult(outcome='fail', error_message='must be exactly two words', fix_value='This is', error_spans=None, metadata=None, validated_chunk=None)] additional_properties={} path=None", # noqa + "Instructions validation failed: incorrect_value='\\nThis also is not two words\\n' fail_results=[FailResult(outcome='fail', error_message='must be exactly two words', fix_value='This also', error_spans=None, metadata=None, validated_chunk=None)] additional_properties={} path=None", # noqa ), ( OnFailAction.FILTER, @@ -660,36 +660,36 @@ def custom_llm(*args, **kwargs): [ ( OnFailAction.REASK, - "Prompt validation failed: incorrect_value='What kind of pet should I get?\\n\\nJson Output:\\n\\n' fail_results=[FailResult(outcome='fail', error_message='must be exactly two words', fix_value='What kind', metadata=None, validated_chunk=None, error_spans=None)] path=None", # noqa - "Instructions validation failed: incorrect_value='What kind of pet should I get?' fail_results=[FailResult(outcome='fail', error_message='must be exactly two words', fix_value='What kind', metadata=None, validated_chunk=None, error_spans=None)] path=None", # noqa - "Message history validation failed: incorrect_value='What kind of pet should I get?' fail_results=[FailResult(outcome='fail', error_message='must be exactly two words', fix_value='What kind', metadata=None, validated_chunk=None, error_spans=None)] path=None", # noqa - "Prompt validation failed: incorrect_value='\\nThis is not two words\\n\\n\\nString Output:\\n\\n' fail_results=[FailResult(outcome='fail', error_message='must be exactly two words', fix_value='This is', metadata=None, validated_chunk=None, error_spans=None)] path=None", # noqa - "Instructions validation failed: incorrect_value='\\nThis also is not two words\\n' fail_results=[FailResult(outcome='fail', error_message='must be exactly two words', fix_value='This also', metadata=None, validated_chunk=None, error_spans=None)] path=None", # noqa + "Prompt validation failed: incorrect_value='What kind of pet should I get?\\n\\nJson Output:\\n\\n' fail_results=[FailResult(outcome='fail', error_message='must be exactly two words', fix_value='What kind', error_spans=None, metadata=None, validated_chunk=None)] additional_properties={} path=None", # noqa + "Instructions validation failed: incorrect_value='What kind of pet should I get?' fail_results=[FailResult(outcome='fail', error_message='must be exactly two words', fix_value='What kind', error_spans=None, metadata=None, validated_chunk=None)] additional_properties={} path=None", # noqa + "Message history validation failed: incorrect_value='What kind of pet should I get?' fail_results=[FailResult(outcome='fail', error_message='must be exactly two words', fix_value='What kind', error_spans=None, metadata=None, validated_chunk=None)] additional_properties={} path=None", # noqa + "Prompt validation failed: incorrect_value='\\nThis is not two words\\n\\n\\nString Output:\\n\\n' fail_results=[FailResult(outcome='fail', error_message='must be exactly two words', fix_value='This is', error_spans=None, metadata=None, validated_chunk=None)] additional_properties={} path=None", # noqa + "Instructions validation failed: incorrect_value='\\nThis also is not two words\\n' fail_results=[FailResult(outcome='fail', error_message='must be exactly two words', fix_value='This also', error_spans=None, metadata=None, validated_chunk=None)] additional_properties={} path=None", # noqa + ), + ( + OnFailAction.FILTER, + "Prompt validation failed", + "Instructions validation failed", + "Message history validation failed", + "Prompt validation failed", + "Instructions validation failed", + ), + ( + OnFailAction.REFRAIN, + "Prompt validation failed", + "Instructions validation failed", + "Message history validation failed", + "Prompt validation failed", + "Instructions validation failed", + ), + ( + OnFailAction.EXCEPTION, + "Validation failed for field with errors: must be exactly two words", + "Validation failed for field with errors: must be exactly two words", + "Validation failed for field with errors: must be exactly two words", + "Validation failed for field with errors: must be exactly two words", + "Validation failed for field with errors: must be exactly two words", ), - # ( - # OnFailAction.FILTER, - # "Prompt validation failed", - # "Instructions validation failed", - # "Message history validation failed", - # "Prompt validation failed", - # "Instructions validation failed", - # ), - # ( - # OnFailAction.REFRAIN, - # "Prompt validation failed", - # "Instructions validation failed", - # "Message history validation failed", - # "Prompt validation failed", - # "Instructions validation failed", - # ), - # ( - # OnFailAction.EXCEPTION, - # "Validation failed for field with errors: must be exactly two words", - # "Validation failed for field with errors: must be exactly two words", - # "Validation failed for field with errors: must be exactly two words", - # "Validation failed for field with errors: must be exactly two words", - # "Validation failed for field with errors: must be exactly two words", - # ), ], ) @pytest.mark.asyncio diff --git a/tests/unit_tests/test_validator_service.py b/tests/unit_tests/test_validator_service.py index aebdfad01..36b723382 100644 --- a/tests/unit_tests/test_validator_service.py +++ b/tests/unit_tests/test_validator_service.py @@ -6,7 +6,10 @@ from .mocks import MockAsyncValidatorService, MockLoop, MockSequentialValidatorService -iteration = Iteration() +iteration = Iteration( + call_id="mock-call", + index=0, +) @pytest.mark.asyncio