From 29ee128ec7ee2f05cf3e399ab6ae2f9b62e427aa Mon Sep 17 00:00:00 2001
From: Nicholas Chen <nichwch@gmail.com>
Date: Tue, 7 May 2024 12:31:11 -0400
Subject: [PATCH 01/85] preliminary code and pseudocode

---
 guardrails/run/stream_runner.py    | 16 ++++++++++++----
 guardrails/schema/string_schema.py |  3 +++
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/guardrails/run/stream_runner.py b/guardrails/run/stream_runner.py
index 890c3fd9c..7ce27dfa3 100644
--- a/guardrails/run/stream_runner.py
+++ b/guardrails/run/stream_runner.py
@@ -158,19 +158,27 @@ def step(
             chunk_text = self.get_chunk_text(chunk, api)
             fragment += chunk_text
 
-            # 2. Parse the fragment
-            parsed_fragment, move_to_next = self.parse(
-                index, fragment, output_schema, verified
+            # 2. Parse the chunk
+            # I assume we have to parse the chunk before validating it...
+            parsed_chunk, move_to_next = self.parse(
+                index, chunk, output_schema, verified
             )
             if move_to_next:
                 # Continue to next chunk
                 continue
 
             # 3. Run output validation
+            # If validator chunk size is smaller than LLM chunk size:
+            # split llm chunk down into validator-sized chunks
+            # Question: How can I tell what the validator chunk size is?
+
+            # If validator chunk size is larger, pass to validator.
+            # Validator will return None until it's accumulated enough
+            # Don't forget to validate incomplete chunks at the end.
             validated_fragment = self.validate(
                 iteration,
                 index,
-                parsed_fragment,
+                parsed_chunk,
                 output_schema,
                 validate_subschema=True,
             )
diff --git a/guardrails/schema/string_schema.py b/guardrails/schema/string_schema.py
index cbff5d5d1..23683082f 100644
--- a/guardrails/schema/string_schema.py
+++ b/guardrails/schema/string_schema.py
@@ -136,6 +136,9 @@ def validate(
         disable_tracer: Optional[bool] = True,
         **kwargs,
     ) -> Any:
+        # TODO: add class field to track number of chunks accumulated
+        # If not enough chunks have been accumulated, emit None
+        # Once enough chunks have been accumulated, validate and emit the result
         """Validate a dictionary of data against the schema.
 
         Args:

From 66a53333fb250b4ecc488ff71ebc5c91f96809b5 Mon Sep 17 00:00:00 2001
From: Nicholas Chen <nichwch@gmail.com>
Date: Tue, 7 May 2024 19:25:07 -0400
Subject: [PATCH 02/85] add chunk accumulation strategy to Validator base class

---
 guardrails/run/stream_runner.py |  2 +-
 guardrails/validator_base.py    | 45 +++++++++++++++++++++++++++++++++
 2 files changed, 46 insertions(+), 1 deletion(-)

diff --git a/guardrails/run/stream_runner.py b/guardrails/run/stream_runner.py
index 7ce27dfa3..b4683c63e 100644
--- a/guardrails/run/stream_runner.py
+++ b/guardrails/run/stream_runner.py
@@ -172,7 +172,7 @@ def step(
             # split llm chunk down into validator-sized chunks
             # Question: How can I tell what the validator chunk size is?
 
-            # If validator chunk size is larger, pass to validator.
+            # If validator chunk size is larger, pass to validator. 
             # Validator will return None until it's accumulated enough
             # Don't forget to validate incomplete chunks at the end.
             validated_fragment = self.validate(
diff --git a/guardrails/validator_base.py b/guardrails/validator_base.py
index 2b92915d9..c40e47680 100644
--- a/guardrails/validator_base.py
+++ b/guardrails/validator_base.py
@@ -7,6 +7,7 @@
     Any,
     Callable,
     Dict,
+    Iterable,
     List,
     Literal,
     Optional,
@@ -26,6 +27,8 @@
 from guardrails.errors import ValidationError
 from guardrails.utils.dataclass import dataclass
 
+VALIDATOR_CHUNKING_STRATEGIES = Enum('VALIDATOR_CHUNKING_STRATEGIES', ['WORD', 'SENTENCE', 'PARAGRAPH'])
+
 VALIDATOR_IMPORT_WARNING = """Accessing `{validator_name}` using
 `from guardrails.validators import {validator_name}` is deprecated and
 support will be removed after version 0.5.x. Please switch to the Guardrails Hub syntax:
@@ -174,6 +177,14 @@ class Filter:
 class Refrain:
     pass
 
+def is_word(chunk:str) -> bool:
+    return ' ' in chunk
+
+def is_sentence(chunk:str) -> bool:
+    return '.' in chunk
+
+def is_paragraph(chunk:str) -> bool:
+    return '\n' in chunk
 
 def check_refrain_in_list(schema: List) -> bool:
     """Checks if a Refrain object exists in a list.
@@ -390,6 +401,8 @@ class Validator(Runnable):
 
     rail_alias: str = ""
 
+    chunking_strategy=VALIDATOR_CHUNKING_STRATEGIES.SENTENCE
+    accumulated_chunks = []
     run_in_separate_process = False
     override_value_on_pass = False
     required_metadata_keys = []
@@ -452,6 +465,38 @@ def validate(self, value: Any, metadata: Dict[str, Any]) -> ValidationResult:
         """Validates a value and return a validation result."""
         raise NotImplementedError
 
+    def validate_stream(self, chunk:Any, metadata: Dict[str, Any]) -> ValidationResult:
+        """Validates a chunk emitted by an LLM.
+        If the LLM chunk is smaller than the validator's chunking strategy, 
+        it will be accumulated until it reaches the desired size. In the meantime, 
+        the validator will return None.
+
+        Otherwise, the validator will validate the chunk and return the result.
+        """
+        # combine accumulated chunks and new chunk
+        self.accumulated_chunks.append(chunk)
+        # check if enough chunks have accumulated for validation
+        accumulated_enough = self.accumulated_enough_to_validate()
+        if not accumulated_enough:
+            return None
+        # if we've accumulated enough chunks, validate the accumulated chunks
+        accumulated_text = ''.join(self.accumulated_chunks)
+        # remove the accummulated chunks
+        self.accumulated_chunks = []
+        return self.validate(accumulated_text, metadata)
+
+    
+    def accumulated_enough_to_validate(self) -> bool:
+        accumulated_text = ''.join(self.accumulated_chunks)
+        """Check if the accumulated chunks are large enough to be validated."""
+        if(self.chunking_strategy == VALIDATOR_CHUNKING_STRATEGIES.WORD):
+            return is_word(accumulated_text)
+        if(self.chunking_strategy == VALIDATOR_CHUNKING_STRATEGIES.SENTENCE):
+            return is_sentence(accumulated_text)
+        if(self.chunking_strategy == VALIDATOR_CHUNKING_STRATEGIES.PARAGRAPH):
+            return is_paragraph(accumulated_text)
+
+
     def to_prompt(self, with_keywords: bool = True) -> str:
         """Convert the validator to a prompt.
 

From 7831466b7b5f3a0cfc60c7026f78f5351189bbed Mon Sep 17 00:00:00 2001
From: Nicholas Chen <nichwch@gmail.com>
Date: Wed, 8 May 2024 18:08:59 -0400
Subject: [PATCH 03/85] handle case where llm chunk > validator chunk in
 validator class

---
 guardrails/run/stream_runner.py    | 10 +---
 guardrails/schema/string_schema.py |  2 +-
 guardrails/validator_base.py       | 92 +++++++++++++++++++-----------
 3 files changed, 61 insertions(+), 43 deletions(-)

diff --git a/guardrails/run/stream_runner.py b/guardrails/run/stream_runner.py
index b4683c63e..c4a5584c5 100644
--- a/guardrails/run/stream_runner.py
+++ b/guardrails/run/stream_runner.py
@@ -167,14 +167,6 @@ def step(
                 # Continue to next chunk
                 continue
 
-            # 3. Run output validation
-            # If validator chunk size is smaller than LLM chunk size:
-            # split llm chunk down into validator-sized chunks
-            # Question: How can I tell what the validator chunk size is?
-
-            # If validator chunk size is larger, pass to validator. 
-            # Validator will return None until it's accumulated enough
-            # Don't forget to validate incomplete chunks at the end.
             validated_fragment = self.validate(
                 iteration,
                 index,
@@ -198,7 +190,7 @@ def step(
 
             # 5. Convert validated fragment to a pretty JSON string
             yield ValidationOutcome(
-                raw_llm_output=fragment,
+                raw_llm_output=chunk,
                 validated_output=validated_fragment,
                 validation_passed=validated_fragment is not None,
             )
diff --git a/guardrails/schema/string_schema.py b/guardrails/schema/string_schema.py
index 23683082f..6a09a0be3 100644
--- a/guardrails/schema/string_schema.py
+++ b/guardrails/schema/string_schema.py
@@ -175,7 +175,7 @@ def validate(
         validated_response = {dummy_key: validated_response}
 
         if check_refrain_in_dict(validated_response):
-            # If the data contains a `Refain` value, we return an empty
+            # If the data contains a `Refrain` value, we return an empty
             # dictionary.
             logger.debug("Refrain detected.")
             validated_response = {}
diff --git a/guardrails/validator_base.py b/guardrails/validator_base.py
index c40e47680..a7de679b9 100644
--- a/guardrails/validator_base.py
+++ b/guardrails/validator_base.py
@@ -27,7 +27,9 @@
 from guardrails.errors import ValidationError
 from guardrails.utils.dataclass import dataclass
 
-VALIDATOR_CHUNKING_STRATEGIES = Enum('VALIDATOR_CHUNKING_STRATEGIES', ['WORD', 'SENTENCE', 'PARAGRAPH'])
+VALIDATOR_CHUNKING_STRATEGIES = Enum(
+    "VALIDATOR_CHUNKING_STRATEGIES", ["WORD", "SENTENCE", "PARAGRAPH"]
+)
 
 VALIDATOR_IMPORT_WARNING = """Accessing `{validator_name}` using
 `from guardrails.validators import {validator_name}` is deprecated and
@@ -177,14 +179,17 @@ class Filter:
 class Refrain:
     pass
 
-def is_word(chunk:str) -> bool:
-    return ' ' in chunk
+# functions to get chunks
+def split_word(chunk: str) -> bool:
+    return list(map( lambda x: x + ' ',chunk.split(" ") ))[:-1]
+
+
+def split_sentence(chunk: str) -> bool:
+    return list(map( lambda x: x + '.',chunk.split(".") ))[:-1]
 
-def is_sentence(chunk:str) -> bool:
-    return '.' in chunk
+def split_paragraph(chunk: str) -> bool:
+    return list(map( lambda x: x + '\n',chunk.split("\n") ))[:-1]
 
-def is_paragraph(chunk:str) -> bool:
-    return '\n' in chunk
 
 def check_refrain_in_list(schema: List) -> bool:
     """Checks if a Refrain object exists in a list.
@@ -401,7 +406,7 @@ class Validator(Runnable):
 
     rail_alias: str = ""
 
-    chunking_strategy=VALIDATOR_CHUNKING_STRATEGIES.SENTENCE
+    chunking_strategy = VALIDATOR_CHUNKING_STRATEGIES.SENTENCE
     accumulated_chunks = []
     run_in_separate_process = False
     override_value_on_pass = False
@@ -465,37 +470,58 @@ def validate(self, value: Any, metadata: Dict[str, Any]) -> ValidationResult:
         """Validates a value and return a validation result."""
         raise NotImplementedError
 
-    def validate_stream(self, chunk:Any, metadata: Dict[str, Any]) -> ValidationResult:
+    def validate_stream(self, stream: Iterable[Any], metadata: Dict[str, Any]) -> Iterable[ValidationResult]:
         """Validates a chunk emitted by an LLM.
-        If the LLM chunk is smaller than the validator's chunking strategy, 
-        it will be accumulated until it reaches the desired size. In the meantime, 
+        If the LLM chunk is smaller than the validator's chunking strategy,
+        it will be accumulated until it reaches the desired size. In the meantime,
         the validator will return None.
 
+        If the LLM chunk is larger than the validator's chunking strategy, 
+        it will split it into validator-sized chunks and validate each one,
+        returning an array of validation results.
+
         Otherwise, the validator will validate the chunk and return the result.
         """
-        # combine accumulated chunks and new chunk
-        self.accumulated_chunks.append(chunk)
-        # check if enough chunks have accumulated for validation
-        accumulated_enough = self.accumulated_enough_to_validate()
-        if not accumulated_enough:
-            return None
-        # if we've accumulated enough chunks, validate the accumulated chunks
-        accumulated_text = ''.join(self.accumulated_chunks)
-        # remove the accummulated chunks
-        self.accumulated_chunks = []
-        return self.validate(accumulated_text, metadata)
-
-    
-    def accumulated_enough_to_validate(self) -> bool:
-        accumulated_text = ''.join(self.accumulated_chunks)
+        for chunk in stream:
+            # combine accumulated chunks and new chunk
+            self.accumulated_chunks.append(chunk)
+            # check if enough chunks have accumulated for validation
+            val_chunks_to_validate = self.get_validator_chunks()
+            if len(val_chunks_to_validate) == 0:
+                yield None
+            # exclude last chunk, because it may not be a complete chunk
+            for val_chunk in val_chunks_to_validate[:-1]:
+                yield self.validate(val_chunk, metadata)
+            # we now need to check if the last chunk is a complete chunk
+            # if so, we process it
+            # otherwise, we keep it around
+            # TODO: how to figure out if a word chunk is complete?
+            # is it gauranteed that the LLM emits complete words? 
+            last_chunk = val_chunks_to_validate[-1]
+            if self.chunking_strategy == VALIDATOR_CHUNKING_STRATEGIES.SENTENCE:
+               if '.' in last_chunk: 
+                   yield self.validate(last_chunk, metadata) 
+                   self.accumulated_chunks = []
+               else:
+                   self.accumulated_chunks = [last_chunk]
+            if self.chunking_strategy == VALIDATOR_CHUNKING_STRATEGIES.PARAGRAPH:
+               if '\n' in last_chunk: 
+                   yield self.validate(last_chunk, metadata) 
+                   self.accumulated_chunks = []
+               else:
+                   self.accumulated_chunks = [last_chunk]
+
+
+
+    def get_validator_chunks(self) -> list[str]:
+        accumulated_text = "".join(self.accumulated_chunks)
         """Check if the accumulated chunks are large enough to be validated."""
-        if(self.chunking_strategy == VALIDATOR_CHUNKING_STRATEGIES.WORD):
-            return is_word(accumulated_text)
-        if(self.chunking_strategy == VALIDATOR_CHUNKING_STRATEGIES.SENTENCE):
-            return is_sentence(accumulated_text)
-        if(self.chunking_strategy == VALIDATOR_CHUNKING_STRATEGIES.PARAGRAPH):
-            return is_paragraph(accumulated_text)
-
+        if self.chunking_strategy == VALIDATOR_CHUNKING_STRATEGIES.WORD:
+            return split_word(accumulated_text)
+        if self.chunking_strategy == VALIDATOR_CHUNKING_STRATEGIES.SENTENCE:
+            return split_sentence(accumulated_text)
+        if self.chunking_strategy == VALIDATOR_CHUNKING_STRATEGIES.PARAGRAPH:
+            return split_paragraph(accumulated_text)
 
     def to_prompt(self, with_keywords: bool = True) -> str:
         """Convert the validator to a prompt.

From 2dbae2ebb9dea710c96e9659c92bf5390cd3b2fb Mon Sep 17 00:00:00 2001
From: Nicholas Chen <nichwch@gmail.com>
Date: Wed, 8 May 2024 18:18:28 -0400
Subject: [PATCH 04/85] added some questions

---
 guardrails/validator_base.py | 43 +++++++++++++++++++++---------------
 1 file changed, 25 insertions(+), 18 deletions(-)

diff --git a/guardrails/validator_base.py b/guardrails/validator_base.py
index a7de679b9..bd9d2f531 100644
--- a/guardrails/validator_base.py
+++ b/guardrails/validator_base.py
@@ -179,16 +179,18 @@ class Filter:
 class Refrain:
     pass
 
+
 # functions to get chunks
 def split_word(chunk: str) -> bool:
-    return list(map( lambda x: x + ' ',chunk.split(" ") ))[:-1]
+    return list(map(lambda x: x + " ", chunk.split(" ")))[:-1]
 
 
 def split_sentence(chunk: str) -> bool:
-    return list(map( lambda x: x + '.',chunk.split(".") ))[:-1]
+    return list(map(lambda x: x + ".", chunk.split(".")))[:-1]
+
 
 def split_paragraph(chunk: str) -> bool:
-    return list(map( lambda x: x + '\n',chunk.split("\n") ))[:-1]
+    return list(map(lambda x: x + "\n", chunk.split("\n")))[:-1]
 
 
 def check_refrain_in_list(schema: List) -> bool:
@@ -470,13 +472,15 @@ def validate(self, value: Any, metadata: Dict[str, Any]) -> ValidationResult:
         """Validates a value and return a validation result."""
         raise NotImplementedError
 
-    def validate_stream(self, stream: Iterable[Any], metadata: Dict[str, Any]) -> Iterable[ValidationResult]:
+    def validate_stream(
+        self, stream: Iterable[Any], metadata: Dict[str, Any]
+    ) -> Iterable[ValidationResult]:
         """Validates a chunk emitted by an LLM.
         If the LLM chunk is smaller than the validator's chunking strategy,
         it will be accumulated until it reaches the desired size. In the meantime,
         the validator will return None.
 
-        If the LLM chunk is larger than the validator's chunking strategy, 
+        If the LLM chunk is larger than the validator's chunking strategy,
         it will split it into validator-sized chunks and validate each one,
         returning an array of validation results.
 
@@ -484,6 +488,7 @@ def validate_stream(self, stream: Iterable[Any], metadata: Dict[str, Any]) -> It
         """
         for chunk in stream:
             # combine accumulated chunks and new chunk
+            # TODO: Question: I'm assuming chunks are strings here. I'm not sure this is true
             self.accumulated_chunks.append(chunk)
             # check if enough chunks have accumulated for validation
             val_chunks_to_validate = self.get_validator_chunks()
@@ -495,22 +500,24 @@ def validate_stream(self, stream: Iterable[Any], metadata: Dict[str, Any]) -> It
             # we now need to check if the last chunk is a complete chunk
             # if so, we process it
             # otherwise, we keep it around
-            # TODO: how to figure out if a word chunk is complete?
-            # is it gauranteed that the LLM emits complete words? 
+            # TODO: Question: how to figure out if a word chunk is complete?
+            # is it gauranteed that the LLM emits complete words?
             last_chunk = val_chunks_to_validate[-1]
             if self.chunking_strategy == VALIDATOR_CHUNKING_STRATEGIES.SENTENCE:
-               if '.' in last_chunk: 
-                   yield self.validate(last_chunk, metadata) 
-                   self.accumulated_chunks = []
-               else:
-                   self.accumulated_chunks = [last_chunk]
+                if "." in last_chunk:
+                    yield self.validate(last_chunk, metadata)
+                    self.accumulated_chunks = []
+                else:
+                    self.accumulated_chunks = [last_chunk]
             if self.chunking_strategy == VALIDATOR_CHUNKING_STRATEGIES.PARAGRAPH:
-               if '\n' in last_chunk: 
-                   yield self.validate(last_chunk, metadata) 
-                   self.accumulated_chunks = []
-               else:
-                   self.accumulated_chunks = [last_chunk]
-
+                if "\n" in last_chunk:
+                    yield self.validate(last_chunk, metadata)
+                    self.accumulated_chunks = []
+                else:
+                    self.accumulated_chunks = [last_chunk]
+        # after llm stream has been exhausted, we need to validate everything that's left
+        str_from_leftover_chunks = "".join(self.accumulated_chunks)
+        yield self.validate(str_from_leftover_chunks, metadata)
 
 
     def get_validator_chunks(self) -> list[str]:

From 1e3544daf96397967569cbf9fc43e9fb23b29574 Mon Sep 17 00:00:00 2001
From: Nicholas Chen <nichwch@gmail.com>
Date: Wed, 8 May 2024 18:22:44 -0400
Subject: [PATCH 05/85] change stream_runner to handle the result of iterable
 validate

---
 guardrails/run/stream_runner.py | 42 ++++++++++++++++++---------------
 guardrails/validator_base.py    |  1 -
 2 files changed, 23 insertions(+), 20 deletions(-)

diff --git a/guardrails/run/stream_runner.py b/guardrails/run/stream_runner.py
index c4a5584c5..f16124cec 100644
--- a/guardrails/run/stream_runner.py
+++ b/guardrails/run/stream_runner.py
@@ -167,33 +167,37 @@ def step(
                 # Continue to next chunk
                 continue
 
-            validated_fragment = self.validate(
+             # TODO: change these to generators all the way down
+            validated_fragments = self.validate(
                 iteration,
                 index,
                 parsed_chunk,
                 output_schema,
                 validate_subschema=True,
             )
-            if isinstance(validated_fragment, SkeletonReAsk):
-                raise ValueError(
-                    "Received fragment schema is an invalid sub-schema "
-                    "of the expected output JSON schema."
-                )
+            for validated_fragment in validated_fragments:
+                if isinstance(validated_fragment, SkeletonReAsk):
+                    raise ValueError(
+                        "Received fragment schema is an invalid sub-schema "
+                        "of the expected output JSON schema."
+                    )
 
-            # 4. Introspect: inspect the validated fragment for reasks
-            reasks, valid_op = self.introspect(index, validated_fragment, output_schema)
-            if reasks:
-                raise ValueError(
-                    "Reasks are not yet supported with streaming. Please "
-                    "remove reasks from schema or disable streaming."
-                )
+                # 4. Introspect: inspect the validated fragment for reasks
+                reasks, valid_op = self.introspect(index, validated_fragment, output_schema)
+                if reasks:
+                    raise ValueError(
+                        "Reasks are not yet supported with streaming. Please "
+                        "remove reasks from schema or disable streaming."
+                    )
 
-            # 5. Convert validated fragment to a pretty JSON string
-            yield ValidationOutcome(
-                raw_llm_output=chunk,
-                validated_output=validated_fragment,
-                validation_passed=validated_fragment is not None,
-            )
+                # 5. Convert validated fragment to a pretty JSON string
+                yield ValidationOutcome(
+                    # TODO: question: What do we want raw_llm_output to be here?
+                    #  The chunk or the whole output?
+                    raw_llm_output=chunk,
+                    validated_output=validated_fragment,
+                    validation_passed=validated_fragment is not None,
+                )
 
         # Finally, add to logs
         iteration.outputs.raw_output = fragment
diff --git a/guardrails/validator_base.py b/guardrails/validator_base.py
index bd9d2f531..a836ff1f1 100644
--- a/guardrails/validator_base.py
+++ b/guardrails/validator_base.py
@@ -519,7 +519,6 @@ def validate_stream(
         str_from_leftover_chunks = "".join(self.accumulated_chunks)
         yield self.validate(str_from_leftover_chunks, metadata)
 
-
     def get_validator_chunks(self) -> list[str]:
         accumulated_text = "".join(self.accumulated_chunks)
         """Check if the accumulated chunks are large enough to be validated."""

From e9084e14c5176c949900b8cadf09f2ed06bb0749 Mon Sep 17 00:00:00 2001
From: Nicholas Chen <nichwch@gmail.com>
Date: Wed, 8 May 2024 18:24:46 -0400
Subject: [PATCH 06/85] format

---
 guardrails/run/stream_runner.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/guardrails/run/stream_runner.py b/guardrails/run/stream_runner.py
index f16124cec..f1d015551 100644
--- a/guardrails/run/stream_runner.py
+++ b/guardrails/run/stream_runner.py
@@ -167,7 +167,7 @@ def step(
                 # Continue to next chunk
                 continue
 
-             # TODO: change these to generators all the way down
+            # TODO: change these to generators all the way down
             validated_fragments = self.validate(
                 iteration,
                 index,
@@ -183,7 +183,9 @@ def step(
                     )
 
                 # 4. Introspect: inspect the validated fragment for reasks
-                reasks, valid_op = self.introspect(index, validated_fragment, output_schema)
+                reasks, valid_op = self.introspect(
+                    index, validated_fragment, output_schema
+                )
                 if reasks:
                     raise ValueError(
                         "Reasks are not yet supported with streaming. Please "

From 1269f6840a2cb25ec6f1f12969e67b24da999ef4 Mon Sep 17 00:00:00 2001
From: Nicholas Chen <nichwch@gmail.com>
Date: Fri, 10 May 2024 11:55:17 -0400
Subject: [PATCH 07/85] change validator base to use a chunking function
 instead of specifying a chunking strategy

---
 guardrails/run/runner.py           |  2 +
 guardrails/run/stream_runner.py    | 12 +++--
 guardrails/schema/string_schema.py | 25 +++++++----
 guardrails/validator_base.py       | 72 ++++++++++--------------------
 guardrails/validator_service.py    | 67 +++++++++++++++++++++++++--
 5 files changed, 111 insertions(+), 67 deletions(-)

diff --git a/guardrails/run/runner.py b/guardrails/run/runner.py
index 9688d43ea..2fd1cba3b 100644
--- a/guardrails/run/runner.py
+++ b/guardrails/run/runner.py
@@ -547,6 +547,7 @@ def validate(
         index: int,
         parsed_output: Any,
         output_schema: Schema,
+        stream: Optional[bool] = False,
         **kwargs,
     ):
         """Validate the output."""
@@ -554,6 +555,7 @@ def validate(
             iteration,
             parsed_output,
             self.metadata,
+            stream,
             attempt_number=index,
             disable_tracer=self._disable_tracer,
             **kwargs,
diff --git a/guardrails/run/stream_runner.py b/guardrails/run/stream_runner.py
index f1d015551..a53a86967 100644
--- a/guardrails/run/stream_runner.py
+++ b/guardrails/run/stream_runner.py
@@ -1,4 +1,4 @@
-from typing import Any, Dict, Generator, List, Optional, Union
+from typing import Any, Dict, Generator, Iterable, List, Optional, Union
 
 from guardrails.classes.history import Call, Inputs, Iteration, Outputs
 from guardrails.classes.output_type import OT
@@ -167,15 +167,15 @@ def step(
                 # Continue to next chunk
                 continue
 
-            # TODO: change these to generators all the way down
-            validated_fragments = self.validate(
+            validated_fragments_iterable = self.validate(
                 iteration,
                 index,
                 parsed_chunk,
                 output_schema,
+                True,
                 validate_subschema=True,
             )
-            for validated_fragment in validated_fragments:
+            for validated_fragment in validated_fragments_iterable:
                 if isinstance(validated_fragment, SkeletonReAsk):
                     raise ValueError(
                         "Received fragment schema is an invalid sub-schema "
@@ -183,9 +183,7 @@ def step(
                     )
 
                 # 4. Introspect: inspect the validated fragment for reasks
-                reasks, valid_op = self.introspect(
-                    index, validated_fragment, output_schema
-                )
+                reasks, valid_op = self.introspect(index, validated_fragment, output_schema)
                 if reasks:
                     raise ValueError(
                         "Reasks are not yet supported with streaming. Please "
diff --git a/guardrails/schema/string_schema.py b/guardrails/schema/string_schema.py
index 6a09a0be3..b663b9417 100644
--- a/guardrails/schema/string_schema.py
+++ b/guardrails/schema/string_schema.py
@@ -134,6 +134,7 @@ def validate(
         metadata: Dict,
         attempt_number: int = 0,
         disable_tracer: Optional[bool] = True,
+        stream: Optional[bool] = False,
         **kwargs,
     ) -> Any:
         # TODO: add class field to track number of chunks accumulated
@@ -163,14 +164,22 @@ def validate(
                 dummy_key: data,
             },
         )
-
-        validated_response, metadata = validator_service.validate(
-            value=data,
-            metadata=metadata,
-            validator_setup=validation,
-            iteration=iteration,
-            disable_tracer=disable_tracer,
-        )
+        if(stream):
+            validated_response, metadata = validator_service.validate_stream(
+                value=data,
+                metadata=metadata,
+                validator_setup=validation,
+                iteration=iteration,
+                disable_tracer=disable_tracer,
+            )
+        else:
+            validated_response, metadata = validator_service.validate(
+                value=data,
+                metadata=metadata,
+                validator_setup=validation,
+                iteration=iteration,
+                disable_tracer=disable_tracer,
+            )
 
         validated_response = {dummy_key: validated_response}
 
diff --git a/guardrails/validator_base.py b/guardrails/validator_base.py
index a836ff1f1..296e590b3 100644
--- a/guardrails/validator_base.py
+++ b/guardrails/validator_base.py
@@ -27,10 +27,6 @@
 from guardrails.errors import ValidationError
 from guardrails.utils.dataclass import dataclass
 
-VALIDATOR_CHUNKING_STRATEGIES = Enum(
-    "VALIDATOR_CHUNKING_STRATEGIES", ["WORD", "SENTENCE", "PARAGRAPH"]
-)
-
 VALIDATOR_IMPORT_WARNING = """Accessing `{validator_name}` using
 `from guardrails.validators import {validator_name}` is deprecated and
 support will be removed after version 0.5.x. Please switch to the Guardrails Hub syntax:
@@ -179,7 +175,6 @@ class Filter:
 class Refrain:
     pass
 
-
 # functions to get chunks
 def split_word(chunk: str) -> bool:
     return list(map(lambda x: x + " ", chunk.split(" ")))[:-1]
@@ -408,7 +403,10 @@ class Validator(Runnable):
 
     rail_alias: str = ""
 
-    chunking_strategy = VALIDATOR_CHUNKING_STRATEGIES.SENTENCE
+    # chunking function returns empty list or list of 2 chunks 
+    # first chunk is the chunk to validate
+    # second chunk is incomplete chunk that needs further accumulation 
+    chunking_function = split_sentence
     accumulated_chunks = []
     run_in_separate_process = False
     override_value_on_pass = False
@@ -473,8 +471,8 @@ def validate(self, value: Any, metadata: Dict[str, Any]) -> ValidationResult:
         raise NotImplementedError
 
     def validate_stream(
-        self, stream: Iterable[Any], metadata: Dict[str, Any]
-    ) -> Iterable[ValidationResult]:
+        self, chunk: Any, metadata: Dict[str, Any]
+    ) -> ValidationResult:
         """Validates a chunk emitted by an LLM.
         If the LLM chunk is smaller than the validator's chunking strategy,
         it will be accumulated until it reaches the desired size. In the meantime,
@@ -486,48 +484,24 @@ def validate_stream(
 
         Otherwise, the validator will validate the chunk and return the result.
         """
-        for chunk in stream:
-            # combine accumulated chunks and new chunk
-            # TODO: Question: I'm assuming chunks are strings here. I'm not sure this is true
-            self.accumulated_chunks.append(chunk)
-            # check if enough chunks have accumulated for validation
-            val_chunks_to_validate = self.get_validator_chunks()
-            if len(val_chunks_to_validate) == 0:
-                yield None
-            # exclude last chunk, because it may not be a complete chunk
-            for val_chunk in val_chunks_to_validate[:-1]:
-                yield self.validate(val_chunk, metadata)
-            # we now need to check if the last chunk is a complete chunk
-            # if so, we process it
-            # otherwise, we keep it around
-            # TODO: Question: how to figure out if a word chunk is complete?
-            # is it gauranteed that the LLM emits complete words?
-            last_chunk = val_chunks_to_validate[-1]
-            if self.chunking_strategy == VALIDATOR_CHUNKING_STRATEGIES.SENTENCE:
-                if "." in last_chunk:
-                    yield self.validate(last_chunk, metadata)
-                    self.accumulated_chunks = []
-                else:
-                    self.accumulated_chunks = [last_chunk]
-            if self.chunking_strategy == VALIDATOR_CHUNKING_STRATEGIES.PARAGRAPH:
-                if "\n" in last_chunk:
-                    yield self.validate(last_chunk, metadata)
-                    self.accumulated_chunks = []
-                else:
-                    self.accumulated_chunks = [last_chunk]
-        # after llm stream has been exhausted, we need to validate everything that's left
-        str_from_leftover_chunks = "".join(self.accumulated_chunks)
-        yield self.validate(str_from_leftover_chunks, metadata)
-
-    def get_validator_chunks(self) -> list[str]:
+        # combine accumulated chunks and new chunk
+        # TODO: Question: I'm assuming chunks are strings here. I'm not sure this is true
+        self.accumulated_chunks.append(chunk)
         accumulated_text = "".join(self.accumulated_chunks)
-        """Check if the accumulated chunks are large enough to be validated."""
-        if self.chunking_strategy == VALIDATOR_CHUNKING_STRATEGIES.WORD:
-            return split_word(accumulated_text)
-        if self.chunking_strategy == VALIDATOR_CHUNKING_STRATEGIES.SENTENCE:
-            return split_sentence(accumulated_text)
-        if self.chunking_strategy == VALIDATOR_CHUNKING_STRATEGIES.PARAGRAPH:
-            return split_paragraph(accumulated_text)
+        # check if enough chunks have accumulated for validation
+        [chunk_to_validate, new_accumulated_chunks] = self.chunking_function(accumulated_text)
+        if len(chunk_to_validate) == 0:
+            return None
+        self.accumulated_chunks = new_accumulated_chunks
+        # exclude last chunk, because it may not be a complete chunk
+        validation_result = self.validate(chunk_to_validate, metadata)
+        return validation_result
+
+        # TODO: the following logic needs to be moved up the chain to stream_runner
+        # TODO: maybe implement a function validateRemainder that validates remaining chunks
+        # after llm stream has been exhausted, we need to validate everything that's left
+        # str_from_leftover_chunks = "".join(self.accumulated_chunks)
+        # yield self.validate(str_from_leftover_chunks, metadata)
 
     def to_prompt(self, with_keywords: bool = True) -> str:
         """Convert the validator to a prompt.
diff --git a/guardrails/validator_service.py b/guardrails/validator_service.py
index 79b61ee0c..395203551 100644
--- a/guardrails/validator_service.py
+++ b/guardrails/validator_service.py
@@ -113,6 +113,7 @@ def run_validator(
         value: Any,
         metadata: Dict,
         property_path: str,
+        stream:bool = False
     ) -> ValidatorLogs:
         validator_class_name = validator.__class__.__name__
         validator_logs = ValidatorLogs(
@@ -124,7 +125,10 @@ def run_validator(
         iteration.outputs.validator_logs.append(validator_logs)
 
         start_time = datetime.now()
-        result = self.execute_validator(validator, value, metadata)
+        if stream:  
+            result = self.execute_validator(validator, value, metadata)
+        else:
+            result = self.execute_validator(validator, value, metadata)
         end_time = datetime.now()
 
         result = validator.validate(value, metadata)
@@ -164,13 +168,14 @@ def run_validators(
         value: Any,
         metadata: Dict[str, Any],
         property_path: str,
+        stream:bool = False
     ) -> Tuple[Any, Dict[str, Any]]:
         # Validate the field
         for validator in validator_setup.validators:
             validator_logs = self.run_validator(
-                iteration, validator, value, metadata, property_path
+                iteration, validator, value, metadata, property_path, stream
             )
-
+            # if stream is true, validator_logs will be a list of 
             result = validator_logs.validation_result
             if isinstance(result, FailResult):
                 value = self.perform_correction(
@@ -193,6 +198,7 @@ def run_validators(
                 return value, metadata
         return value, metadata
 
+    
     def validate_dependents(
         self,
         value: Any,
@@ -208,6 +214,7 @@ def validate_dependents(
             )
             value[child_setup.key] = child_schema
 
+    
     def validate(
         self,
         value: Any,
@@ -234,6 +241,30 @@ def validate(
 
         return value, metadata
 
+    def validate_stream(
+        self,
+        value: Any,
+        metadata: dict,
+        validator_setup: FieldValidation,
+        iteration: Iteration,
+        path: str = "$",
+    ) -> Tuple[Any, dict]:
+        property_path = (
+            f"{path}.{validator_setup.key}"
+            if key_not_empty(validator_setup.key)
+            else path
+        )
+        # I assume validate stream doesn't need validate_dependents
+        # because right now we're only handling StringSchema
+
+        # Validate the field
+        value, metadata = self.run_validators(
+            iteration, validator_setup, value, metadata, property_path, True
+        )
+
+        return value, metadata
+
+
 
 class MultiprocMixin:
     multiprocessing_executor: Optional[ProcessPoolExecutor] = None
@@ -442,6 +473,36 @@ def validate(
         iteration,
     )
 
+def validate_stream(
+    value: Any,
+    metadata: dict,
+    validator_setup: FieldValidation,
+    iteration: Iteration,
+    disable_tracer: Optional[bool] = True,
+):
+    process_count = int(os.environ.get("GUARDRAILS_PROCESS_COUNT", 10))
+
+    # try:
+        # loop = asyncio.get_event_loop()
+    # except RuntimeError:
+        # loop = None
+
+    if process_count == 1:
+        logger.warning(
+            "Process count was set to 1 via the GUARDRAILS_PROCESS_COUNT"
+            "environment variable."
+            "This will cause all validations to run synchronously."
+            "To run asynchronously, specify a process count"
+            "greater than 1 or unset this environment variable."
+        )
+    sequential_validator_service = SequentialValidatorService(disable_tracer)
+    return sequential_validator_service.validate_stream(
+        value,
+        metadata,
+        validator_setup,
+        iteration,
+    )
+
 
 async def async_validate(
     value: Any,

From b454cd5ddde49c34e80988f084c565ebbd56daa8 Mon Sep 17 00:00:00 2001
From: Nicholas Chen <nichwch@gmail.com>
Date: Fri, 10 May 2024 13:25:47 -0400
Subject: [PATCH 08/85] connect streaming all the way down call chain, include
 validated chunk in ValidationResult metadata

---
 guardrails/schema/string_schema.py | 3 ---
 guardrails/validator_base.py       | 2 ++
 guardrails/validator_service.py    | 7 +++----
 3 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/guardrails/schema/string_schema.py b/guardrails/schema/string_schema.py
index b663b9417..66b94f353 100644
--- a/guardrails/schema/string_schema.py
+++ b/guardrails/schema/string_schema.py
@@ -137,9 +137,6 @@ def validate(
         stream: Optional[bool] = False,
         **kwargs,
     ) -> Any:
-        # TODO: add class field to track number of chunks accumulated
-        # If not enough chunks have been accumulated, emit None
-        # Once enough chunks have been accumulated, validate and emit the result
         """Validate a dictionary of data against the schema.
 
         Args:
diff --git a/guardrails/validator_base.py b/guardrails/validator_base.py
index 296e590b3..c27766c16 100644
--- a/guardrails/validator_base.py
+++ b/guardrails/validator_base.py
@@ -495,6 +495,8 @@ def validate_stream(
         self.accumulated_chunks = new_accumulated_chunks
         # exclude last chunk, because it may not be a complete chunk
         validation_result = self.validate(chunk_to_validate, metadata)
+        # include the chunk that we've validated in the metadata
+        validation_result.metadata['validated_chunk'] = chunk_to_validate
         return validation_result
 
         # TODO: the following logic needs to be moved up the chain to stream_runner
diff --git a/guardrails/validator_service.py b/guardrails/validator_service.py
index 395203551..58696715a 100644
--- a/guardrails/validator_service.py
+++ b/guardrails/validator_service.py
@@ -128,10 +128,9 @@ def run_validator(
         if stream:  
             result = self.execute_validator(validator, value, metadata)
         else:
-            result = self.execute_validator(validator, value, metadata)
+            result = validator.validate_stream(value, metadata)
         end_time = datetime.now()
 
-        result = validator.validate(value, metadata)
         if result is None:
             result = PassResult()
 
@@ -175,9 +174,9 @@ def run_validators(
             validator_logs = self.run_validator(
                 iteration, validator, value, metadata, property_path, stream
             )
-            # if stream is true, validator_logs will be a list of 
             result = validator_logs.validation_result
-            if isinstance(result, FailResult):
+            # I assume we would want to wait until the end to do reasks for streams
+            if isinstance(result, FailResult) and not stream:
                 value = self.perform_correction(
                     [result], value, validator, validator.on_fail_descriptor
                 )

From b64ab4ea72e8ac3178a8e0fe8075a569dd6d55b3 Mon Sep 17 00:00:00 2001
From: Nicholas Chen <nichwch@gmail.com>
Date: Fri, 10 May 2024 14:44:48 -0400
Subject: [PATCH 09/85] change execute_validator to handle streaming

---
 guardrails/validator_service.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/guardrails/validator_service.py b/guardrails/validator_service.py
index 58696715a..a9a6341cf 100644
--- a/guardrails/validator_service.py
+++ b/guardrails/validator_service.py
@@ -42,8 +42,9 @@ def __init__(self, disable_tracer: Optional[bool] = True):
     #       Using `fork` instead of `spawn` may alleviate the symptom for POSIX systems,
     #       but is relatively unsupported on Windows.
     def execute_validator(
-        self, validator: Validator, value: Any, metadata: Optional[Dict]
+        self, validator: Validator, value: Any, metadata: Optional[Dict], stream:Optional[bool] = False
     ) -> ValidationResult:
+        validate_func = validator.validate_stream if stream else validator.validate
         traced_validator = trace_validator(
             validator_name=validator.rail_alias,
             obj_id=id(validator),
@@ -51,7 +52,7 @@ def execute_validator(
             # namespace=validator.namespace,
             on_fail_descriptor=validator.on_fail_descriptor,
             **validator._kwargs,
-        )(validator.validate)
+        )(validate_func)
         result = traced_validator(value, metadata)
         return result
 
@@ -113,7 +114,7 @@ def run_validator(
         value: Any,
         metadata: Dict,
         property_path: str,
-        stream:bool = False
+        stream:Optional[bool] = False
     ) -> ValidatorLogs:
         validator_class_name = validator.__class__.__name__
         validator_logs = ValidatorLogs(
@@ -125,10 +126,10 @@ def run_validator(
         iteration.outputs.validator_logs.append(validator_logs)
 
         start_time = datetime.now()
-        if stream:  
-            result = self.execute_validator(validator, value, metadata)
-        else:
+        if stream:
             result = validator.validate_stream(value, metadata)
+        else:
+            result = self.execute_validator(validator, value, metadata)
         end_time = datetime.now()
 
         if result is None:
@@ -167,7 +168,7 @@ def run_validators(
         value: Any,
         metadata: Dict[str, Any],
         property_path: str,
-        stream:bool = False
+        stream:Optional[bool] = False
     ) -> Tuple[Any, Dict[str, Any]]:
         # Validate the field
         for validator in validator_setup.validators:

From bf2bd32030203fc2fed2111a57696f80aa9418f1 Mon Sep 17 00:00:00 2001
From: Nicholas Chen <nichwch@gmail.com>
Date: Fri, 10 May 2024 15:20:52 -0400
Subject: [PATCH 10/85] make validate take stream parameter, remove
 validate_stream in top level of validator_service

---
 guardrails/validator_service.py | 52 ++++++++++-----------------------
 1 file changed, 15 insertions(+), 37 deletions(-)

diff --git a/guardrails/validator_service.py b/guardrails/validator_service.py
index a9a6341cf..28a2c57a2 100644
--- a/guardrails/validator_service.py
+++ b/guardrails/validator_service.py
@@ -445,14 +445,9 @@ def validate(
     validator_setup: FieldValidation,
     iteration: Iteration,
     disable_tracer: Optional[bool] = True,
+    stream: Optional[bool] = False,
 ):
     process_count = int(os.environ.get("GUARDRAILS_PROCESS_COUNT", 10))
-
-    try:
-        loop = asyncio.get_event_loop()
-    except RuntimeError:
-        loop = None
-
     if process_count == 1:
         logger.warning(
             "Process count was set to 1 via the GUARDRAILS_PROCESS_COUNT"
@@ -461,6 +456,20 @@ def validate(
             "To run asynchronously, specify a process count"
             "greater than 1 or unset this environment variable."
         )
+    if stream: 
+        sequential_validator_service = SequentialValidatorService(disable_tracer)
+        return sequential_validator_service.validate_stream(
+            value,
+            metadata,
+            validator_setup,
+            iteration
+        )
+    try:
+        loop = asyncio.get_event_loop()
+    except RuntimeError:
+        loop = None
+
+    if process_count == 1:
         validator_service = SequentialValidatorService(disable_tracer)
     elif loop is not None and not loop.is_running():
         validator_service = AsyncValidatorService(disable_tracer)
@@ -473,37 +482,6 @@ def validate(
         iteration,
     )
 
-def validate_stream(
-    value: Any,
-    metadata: dict,
-    validator_setup: FieldValidation,
-    iteration: Iteration,
-    disable_tracer: Optional[bool] = True,
-):
-    process_count = int(os.environ.get("GUARDRAILS_PROCESS_COUNT", 10))
-
-    # try:
-        # loop = asyncio.get_event_loop()
-    # except RuntimeError:
-        # loop = None
-
-    if process_count == 1:
-        logger.warning(
-            "Process count was set to 1 via the GUARDRAILS_PROCESS_COUNT"
-            "environment variable."
-            "This will cause all validations to run synchronously."
-            "To run asynchronously, specify a process count"
-            "greater than 1 or unset this environment variable."
-        )
-    sequential_validator_service = SequentialValidatorService(disable_tracer)
-    return sequential_validator_service.validate_stream(
-        value,
-        metadata,
-        validator_setup,
-        iteration,
-    )
-
-
 async def async_validate(
     value: Any,
     metadata: dict,

From c79e9b2fb7e7726d96db8751c6fe4823468536ea Mon Sep 17 00:00:00 2001
From: Nicholas Chen <nichwch@gmail.com>
Date: Fri, 10 May 2024 16:24:08 -0400
Subject: [PATCH 11/85] use wyatts sentence splitting strategy

---
 guardrails/validator_base.py | 33 +++++++++++++++++++++------------
 1 file changed, 21 insertions(+), 12 deletions(-)

diff --git a/guardrails/validator_base.py b/guardrails/validator_base.py
index c27766c16..c4634ad09 100644
--- a/guardrails/validator_base.py
+++ b/guardrails/validator_base.py
@@ -7,7 +7,6 @@
     Any,
     Callable,
     Dict,
-    Iterable,
     List,
     Literal,
     Optional,
@@ -175,16 +174,26 @@ class Filter:
 class Refrain:
     pass
 
+
 # functions to get chunks
-def split_word(chunk: str) -> bool:
+def split_word(chunk: str):
     return list(map(lambda x: x + " ", chunk.split(" ")))[:-1]
 
 
-def split_sentence(chunk: str) -> bool:
-    return list(map(lambda x: x + ".", chunk.split(".")))[:-1]
+def split_sentence(chunk: str):
+    # using the sentence tokenizer is expensive
+    # we check for a . to avoid wastefully calling the tokenizer
+    if "." not in chunk:
+        return []
+    sentences = nltk.sent_tokenize(chunk)
+    if len(sentences) == 0:
+        return []
+    # return the sentence
+    # then the remaining chunks that aren't finished accumulating
+    return [sentences[0], "".join(sentences[1:])]
 
 
-def split_paragraph(chunk: str) -> bool:
+def split_paragraph(chunk: str):
     return list(map(lambda x: x + "\n", chunk.split("\n")))[:-1]
 
 
@@ -403,9 +412,9 @@ class Validator(Runnable):
 
     rail_alias: str = ""
 
-    # chunking function returns empty list or list of 2 chunks 
+    # chunking function returns empty list or list of 2 chunks
     # first chunk is the chunk to validate
-    # second chunk is incomplete chunk that needs further accumulation 
+    # second chunk is incomplete chunk that needs further accumulation
     chunking_function = split_sentence
     accumulated_chunks = []
     run_in_separate_process = False
@@ -470,9 +479,7 @@ def validate(self, value: Any, metadata: Dict[str, Any]) -> ValidationResult:
         """Validates a value and return a validation result."""
         raise NotImplementedError
 
-    def validate_stream(
-        self, chunk: Any, metadata: Dict[str, Any]
-    ) -> ValidationResult:
+    def validate_stream(self, chunk: Any, metadata: Dict[str, Any]) -> ValidationResult:
         """Validates a chunk emitted by an LLM.
         If the LLM chunk is smaller than the validator's chunking strategy,
         it will be accumulated until it reaches the desired size. In the meantime,
@@ -489,14 +496,16 @@ def validate_stream(
         self.accumulated_chunks.append(chunk)
         accumulated_text = "".join(self.accumulated_chunks)
         # check if enough chunks have accumulated for validation
-        [chunk_to_validate, new_accumulated_chunks] = self.chunking_function(accumulated_text)
+        [chunk_to_validate, new_accumulated_chunks] = self.chunking_function(
+            accumulated_text
+        )
         if len(chunk_to_validate) == 0:
             return None
         self.accumulated_chunks = new_accumulated_chunks
         # exclude last chunk, because it may not be a complete chunk
         validation_result = self.validate(chunk_to_validate, metadata)
         # include the chunk that we've validated in the metadata
-        validation_result.metadata['validated_chunk'] = chunk_to_validate
+        validation_result.metadata["validated_chunk"] = chunk_to_validate
         return validation_result
 
         # TODO: the following logic needs to be moved up the chain to stream_runner

From 4583cb9c4e702db86664d5263402468be3c5a8cd Mon Sep 17 00:00:00 2001
From: Nicholas Chen <nichwch@gmail.com>
Date: Fri, 10 May 2024 17:35:43 -0400
Subject: [PATCH 12/85] import nltk

---
 guardrails/validator_base.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/guardrails/validator_base.py b/guardrails/validator_base.py
index c4634ad09..cca815c1b 100644
--- a/guardrails/validator_base.py
+++ b/guardrails/validator_base.py
@@ -1,4 +1,5 @@
 import inspect
+import nltk
 from collections import defaultdict
 from copy import deepcopy
 from enum import Enum

From f1b4a88bc7ab8119f4a7bccb00a9dff39febed8f Mon Sep 17 00:00:00 2001
From: Nicholas Chen <nichwch@gmail.com>
Date: Tue, 14 May 2024 11:38:57 -0400
Subject: [PATCH 13/85] use stream-enabled execute_validator

---
 guardrails/validator_service.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/guardrails/validator_service.py b/guardrails/validator_service.py
index 28a2c57a2..9ec39f471 100644
--- a/guardrails/validator_service.py
+++ b/guardrails/validator_service.py
@@ -126,10 +126,7 @@ def run_validator(
         iteration.outputs.validator_logs.append(validator_logs)
 
         start_time = datetime.now()
-        if stream:
-            result = validator.validate_stream(value, metadata)
-        else:
-            result = self.execute_validator(validator, value, metadata)
+        result = self.execute_validator(validator, value, metadata, stream)
         end_time = datetime.now()
 
         if result is None:

From 289745cccee7477713e9afb443dd2506b5a83e6a Mon Sep 17 00:00:00 2001
From: Nicholas Chen <nichwch@gmail.com>
Date: Tue, 14 May 2024 12:53:48 -0400
Subject: [PATCH 14/85] format

---
 guardrails/validator_service.py | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/guardrails/validator_service.py b/guardrails/validator_service.py
index 9ec39f471..cb8c51819 100644
--- a/guardrails/validator_service.py
+++ b/guardrails/validator_service.py
@@ -42,7 +42,11 @@ def __init__(self, disable_tracer: Optional[bool] = True):
     #       Using `fork` instead of `spawn` may alleviate the symptom for POSIX systems,
     #       but is relatively unsupported on Windows.
     def execute_validator(
-        self, validator: Validator, value: Any, metadata: Optional[Dict], stream:Optional[bool] = False
+        self,
+        validator: Validator,
+        value: Any,
+        metadata: Optional[Dict],
+        stream: Optional[bool] = False,
     ) -> ValidationResult:
         validate_func = validator.validate_stream if stream else validator.validate
         traced_validator = trace_validator(
@@ -114,7 +118,7 @@ def run_validator(
         value: Any,
         metadata: Dict,
         property_path: str,
-        stream:Optional[bool] = False
+        stream: Optional[bool] = False,
     ) -> ValidatorLogs:
         validator_class_name = validator.__class__.__name__
         validator_logs = ValidatorLogs(
@@ -165,7 +169,7 @@ def run_validators(
         value: Any,
         metadata: Dict[str, Any],
         property_path: str,
-        stream:Optional[bool] = False
+        stream: Optional[bool] = False,
     ) -> Tuple[Any, Dict[str, Any]]:
         # Validate the field
         for validator in validator_setup.validators:
@@ -195,7 +199,6 @@ def run_validators(
                 return value, metadata
         return value, metadata
 
-    
     def validate_dependents(
         self,
         value: Any,
@@ -211,7 +214,6 @@ def validate_dependents(
             )
             value[child_setup.key] = child_schema
 
-    
     def validate(
         self,
         value: Any,
@@ -262,7 +264,6 @@ def validate_stream(
         return value, metadata
 
 
-
 class MultiprocMixin:
     multiprocessing_executor: Optional[ProcessPoolExecutor] = None
     process_count = int(os.environ.get("GUARDRAILS_PROCESS_COUNT", 10))
@@ -453,13 +454,10 @@ def validate(
             "To run asynchronously, specify a process count"
             "greater than 1 or unset this environment variable."
         )
-    if stream: 
+    if stream:
         sequential_validator_service = SequentialValidatorService(disable_tracer)
         return sequential_validator_service.validate_stream(
-            value,
-            metadata,
-            validator_setup,
-            iteration
+            value, metadata, validator_setup, iteration
         )
     try:
         loop = asyncio.get_event_loop()
@@ -479,6 +477,7 @@ def validate(
         iteration,
     )
 
+
 async def async_validate(
     value: Any,
     metadata: dict,

From 58d8eed6cf6804f3b625595dd4fa0a803139cdbf Mon Sep 17 00:00:00 2001
From: Nicholas Chen <nichwch@gmail.com>
Date: Wed, 15 May 2024 16:41:39 -0400
Subject: [PATCH 15/85] fix bug where json_schema was being called with
 streaming

---
 guardrails/run/runner.py | 28 +++++++++++++++++++---------
 1 file changed, 19 insertions(+), 9 deletions(-)

diff --git a/guardrails/run/runner.py b/guardrails/run/runner.py
index 2fd1cba3b..eba815973 100644
--- a/guardrails/run/runner.py
+++ b/guardrails/run/runner.py
@@ -551,15 +551,25 @@ def validate(
         **kwargs,
     ):
         """Validate the output."""
-        validated_output = output_schema.validate(
-            iteration,
-            parsed_output,
-            self.metadata,
-            stream,
-            attempt_number=index,
-            disable_tracer=self._disable_tracer,
-            **kwargs,
-        )
+        if isinstance(output_schema, StringSchema):
+            validated_output = output_schema.validate(
+                iteration,
+                parsed_output,
+                self.metadata,
+                index,
+                self._disable_tracer,
+                stream,
+                **kwargs,
+            )
+        else:
+            validated_output = output_schema.validate(
+                iteration,
+                parsed_output,
+                self.metadata,
+                attempt_number=index,
+                disable_tracer=self._disable_tracer,
+                **kwargs,
+            )
 
         return validated_output
 

From 947f47698f1d9aed4ad2b0c07cb2be5da2be30fa Mon Sep 17 00:00:00 2001
From: Nicholas Chen <nichwch@gmail.com>
Date: Thu, 16 May 2024 12:35:25 -0400
Subject: [PATCH 16/85] conditionally use old logic for json_schema to avoid
 breaking json_schema parse

---
 guardrails/run/stream_runner.py | 95 ++++++++++++++++++++++++---------
 1 file changed, 69 insertions(+), 26 deletions(-)

diff --git a/guardrails/run/stream_runner.py b/guardrails/run/stream_runner.py
index a53a86967..08714ef7d 100644
--- a/guardrails/run/stream_runner.py
+++ b/guardrails/run/stream_runner.py
@@ -1,4 +1,4 @@
-from typing import Any, Dict, Generator, Iterable, List, Optional, Union
+from typing import Any, Dict, Generator, List, Optional, Union
 
 from guardrails.classes.history import Call, Inputs, Iteration, Outputs
 from guardrails.classes.output_type import OT
@@ -153,29 +153,72 @@ def step(
         verified = set()
         # Loop over the stream
         # and construct "fragments" of concatenated chunks
-        for chunk in stream:
-            # 1. Get the text from the chunk and append to fragment
-            chunk_text = self.get_chunk_text(chunk, api)
-            fragment += chunk_text
+        # for now, handle string and json schema differently
 
-            # 2. Parse the chunk
-            # I assume we have to parse the chunk before validating it...
-            parsed_chunk, move_to_next = self.parse(
-                index, chunk, output_schema, verified
-            )
-            if move_to_next:
-                # Continue to next chunk
-                continue
+        if isinstance(output_schema, StringSchema):
+            for chunk in stream:
+                # 1. Get the text from the chunk and append to fragment
+                chunk_text = self.get_chunk_text(chunk, api)
+                fragment += chunk_text
 
-            validated_fragments_iterable = self.validate(
-                iteration,
-                index,
-                parsed_chunk,
-                output_schema,
-                True,
-                validate_subschema=True,
-            )
-            for validated_fragment in validated_fragments_iterable:
+                # 2. Parse the chunk
+                parsed_chunk, move_to_next = self.parse(
+                    index, chunk_text, output_schema, verified
+                )
+                if move_to_next:
+                    # Continue to next chunk
+                    continue
+                validated_fragments_iterable = self.validate(
+                    iteration,
+                    index,
+                    parsed_chunk,
+                    output_schema,
+                    True,
+                    validate_subschema=True,
+                )
+                for validated_fragment in validated_fragments_iterable:
+                    if isinstance(validated_fragment, SkeletonReAsk):
+                        raise ValueError(
+                            "Received fragment schema is an invalid sub-schema "
+                            "of the expected output JSON schema."
+                        )
+
+                    # 4. Introspect: inspect the validated fragment for reasks
+                    reasks, valid_op = self.introspect(
+                        index, validated_fragment, output_schema
+                    )
+                    if reasks:
+                        raise ValueError(
+                            "Reasks are not yet supported with streaming. Please "
+                            "remove reasks from schema or disable streaming."
+                        )
+
+                    # 5. Convert validated fragment to a pretty JSON string
+                    yield ValidationOutcome(
+                        #  The chunk or the whole output?
+                        raw_llm_output=chunk,
+                        validated_output=validated_fragment,
+                        validation_passed=validated_fragment is not None,
+                    )
+        else:
+            for chunk in stream:
+                # 1. Get the text from the chunk and append to fragment
+                chunk_text = self.get_chunk_text(chunk, api)
+                fragment += chunk_text
+
+                parsed_fragment, move_to_next = self.parse(
+                    index, fragment, output_schema, verified
+                )
+                if move_to_next:
+                    # Continue to next chunk
+                    continue
+                validated_fragment = self.validate(
+                    iteration,
+                    index,
+                    parsed_fragment,
+                    output_schema,
+                    validate_subschema=True,
+                )
                 if isinstance(validated_fragment, SkeletonReAsk):
                     raise ValueError(
                         "Received fragment schema is an invalid sub-schema "
@@ -183,7 +226,9 @@ def step(
                     )
 
                 # 4. Introspect: inspect the validated fragment for reasks
-                reasks, valid_op = self.introspect(index, validated_fragment, output_schema)
+                reasks, valid_op = self.introspect(
+                    index, validated_fragment, output_schema
+                )
                 if reasks:
                     raise ValueError(
                         "Reasks are not yet supported with streaming. Please "
@@ -192,9 +237,7 @@ def step(
 
                 # 5. Convert validated fragment to a pretty JSON string
                 yield ValidationOutcome(
-                    # TODO: question: What do we want raw_llm_output to be here?
-                    #  The chunk or the whole output?
-                    raw_llm_output=chunk,
+                    raw_llm_output=fragment,
                     validated_output=validated_fragment,
                     validation_passed=validated_fragment is not None,
                 )

From 8b2c15473148f76e5e7da7c47e31fcc6ff19b992 Mon Sep 17 00:00:00 2001
From: Nicholas Chen <nichwch@gmail.com>
Date: Thu, 16 May 2024 18:39:44 -0400
Subject: [PATCH 17/85] validate remainders

---
 guardrails/run/stream_runner.py    | 77 +++++++++++++++++++++---------
 guardrails/schema/string_schema.py | 25 ++++------
 guardrails/validator_base.py       | 23 +++++----
 guardrails/validator_service.py    | 15 ++++--
 4 files changed, 85 insertions(+), 55 deletions(-)

diff --git a/guardrails/run/stream_runner.py b/guardrails/run/stream_runner.py
index 08714ef7d..9e91bbc4a 100644
--- a/guardrails/run/stream_runner.py
+++ b/guardrails/run/stream_runner.py
@@ -168,7 +168,7 @@ def step(
                 if move_to_next:
                     # Continue to next chunk
                     continue
-                validated_fragments_iterable = self.validate(
+                validated_result = self.validate(
                     iteration,
                     index,
                     parsed_chunk,
@@ -176,30 +176,63 @@ def step(
                     True,
                     validate_subschema=True,
                 )
-                for validated_fragment in validated_fragments_iterable:
-                    if isinstance(validated_fragment, SkeletonReAsk):
-                        raise ValueError(
-                            "Received fragment schema is an invalid sub-schema "
-                            "of the expected output JSON schema."
-                        )
-
-                    # 4. Introspect: inspect the validated fragment for reasks
-                    reasks, valid_op = self.introspect(
-                        index, validated_fragment, output_schema
+                if isinstance(validated_result, SkeletonReAsk):
+                    raise ValueError(
+                        "Received fragment schema is an invalid sub-schema "
+                        "of the expected output JSON schema."
                     )
-                    if reasks:
-                        raise ValueError(
-                            "Reasks are not yet supported with streaming. Please "
-                            "remove reasks from schema or disable streaming."
-                        )
 
-                    # 5. Convert validated fragment to a pretty JSON string
-                    yield ValidationOutcome(
-                        #  The chunk or the whole output?
-                        raw_llm_output=chunk,
-                        validated_output=validated_fragment,
-                        validation_passed=validated_fragment is not None,
+                # 4. Introspect: inspect the validated fragment for reasks
+                reasks, valid_op = self.introspect(
+                    index, validated_result, output_schema
+                )
+                if reasks:
+                    raise ValueError(
+                        "Reasks are not yet supported with streaming. Please "
+                        "remove reasks from schema or disable streaming."
                     )
+                # 5. Convert validated fragment to a pretty JSON string
+                yield ValidationOutcome(
+                    #  The chunk or the whole output?
+                    raw_llm_output=chunk,
+                    validated_output=validated_result,
+                    validation_passed=validated_fragment is not None,
+                )
+            ######################################
+            # need to validate remainder of chunks
+            ######################################
+            remainder_validation = self.validate(
+                iteration,
+                index,
+                "",
+                output_schema,
+                True,
+                validate_subschema=True,
+                remainder=True,
+            )
+            if isinstance(remainder_validation, SkeletonReAsk):
+                raise ValueError(
+                    "Received fragment schema is an invalid sub-schema "
+                    "of the expected output JSON schema."
+                )
+
+            # 4. Introspect: inspect the validated fragment for reasks
+            reasks, valid_op = self.introspect(
+                index, remainder_validation, output_schema
+            )
+            if reasks:
+                raise ValueError(
+                    "Reasks are not yet supported with streaming. Please "
+                    "remove reasks from schema or disable streaming."
+                )
+            # 5. Convert validated fragment to a pretty JSON string
+            yield ValidationOutcome(
+                #  The chunk or the whole output?
+                raw_llm_output=chunk,
+                validated_output=remainder_validation,
+                validation_passed=remainder_validation is not None,
+            )
+        # handle non string schema
         else:
             for chunk in stream:
                 # 1. Get the text from the chunk and append to fragment
diff --git a/guardrails/schema/string_schema.py b/guardrails/schema/string_schema.py
index 66b94f353..200666041 100644
--- a/guardrails/schema/string_schema.py
+++ b/guardrails/schema/string_schema.py
@@ -161,22 +161,15 @@ def validate(
                 dummy_key: data,
             },
         )
-        if(stream):
-            validated_response, metadata = validator_service.validate_stream(
-                value=data,
-                metadata=metadata,
-                validator_setup=validation,
-                iteration=iteration,
-                disable_tracer=disable_tracer,
-            )
-        else:
-            validated_response, metadata = validator_service.validate(
-                value=data,
-                metadata=metadata,
-                validator_setup=validation,
-                iteration=iteration,
-                disable_tracer=disable_tracer,
-            )
+        validated_response, metadata = validator_service.validate(
+            value=data,
+            metadata=metadata,
+            validator_setup=validation,
+            iteration=iteration,
+            disable_tracer=disable_tracer,
+            stream=stream,
+            **kwargs,
+        )
 
         validated_response = {dummy_key: validated_response}
 
diff --git a/guardrails/validator_base.py b/guardrails/validator_base.py
index cca815c1b..ca1a54ea5 100644
--- a/guardrails/validator_base.py
+++ b/guardrails/validator_base.py
@@ -480,7 +480,9 @@ def validate(self, value: Any, metadata: Dict[str, Any]) -> ValidationResult:
         """Validates a value and return a validation result."""
         raise NotImplementedError
 
-    def validate_stream(self, chunk: Any, metadata: Dict[str, Any]) -> ValidationResult:
+    def validate_stream(
+        self, chunk: Any, metadata: Dict[str, Any], **kwargs
+    ) -> ValidationResult:
         """Validates a chunk emitted by an LLM.
         If the LLM chunk is smaller than the validator's chunking strategy,
         it will be accumulated until it reaches the desired size. In the meantime,
@@ -493,15 +495,18 @@ def validate_stream(self, chunk: Any, metadata: Dict[str, Any]) -> ValidationRes
         Otherwise, the validator will validate the chunk and return the result.
         """
         # combine accumulated chunks and new chunk
-        # TODO: Question: I'm assuming chunks are strings here. I'm not sure this is true
         self.accumulated_chunks.append(chunk)
         accumulated_text = "".join(self.accumulated_chunks)
         # check if enough chunks have accumulated for validation
-        [chunk_to_validate, new_accumulated_chunks] = self.chunking_function(
-            accumulated_text
-        )
-        if len(chunk_to_validate) == 0:
+        splitcontents = self.chunking_function(accumulated_text)
+
+        # if remainder kwargs is passed, validate remainder regardless
+        remainder = kwargs.get("remainder", False)
+        if remainder:
+            splitcontents = [accumulated_text, []]
+        if len(splitcontents) == 0:
             return None
+        [chunk_to_validate, new_accumulated_chunks] = splitcontents
         self.accumulated_chunks = new_accumulated_chunks
         # exclude last chunk, because it may not be a complete chunk
         validation_result = self.validate(chunk_to_validate, metadata)
@@ -509,12 +514,6 @@ def validate_stream(self, chunk: Any, metadata: Dict[str, Any]) -> ValidationRes
         validation_result.metadata["validated_chunk"] = chunk_to_validate
         return validation_result
 
-        # TODO: the following logic needs to be moved up the chain to stream_runner
-        # TODO: maybe implement a function validateRemainder that validates remaining chunks
-        # after llm stream has been exhausted, we need to validate everything that's left
-        # str_from_leftover_chunks = "".join(self.accumulated_chunks)
-        # yield self.validate(str_from_leftover_chunks, metadata)
-
     def to_prompt(self, with_keywords: bool = True) -> str:
         """Convert the validator to a prompt.
 
diff --git a/guardrails/validator_service.py b/guardrails/validator_service.py
index cb8c51819..3e7fa930d 100644
--- a/guardrails/validator_service.py
+++ b/guardrails/validator_service.py
@@ -47,6 +47,7 @@ def execute_validator(
         value: Any,
         metadata: Optional[Dict],
         stream: Optional[bool] = False,
+        **kwargs,
     ) -> ValidationResult:
         validate_func = validator.validate_stream if stream else validator.validate
         traced_validator = trace_validator(
@@ -57,7 +58,7 @@ def execute_validator(
             on_fail_descriptor=validator.on_fail_descriptor,
             **validator._kwargs,
         )(validate_func)
-        result = traced_validator(value, metadata)
+        result = traced_validator(value, metadata, **kwargs)
         return result
 
     def perform_correction(
@@ -119,6 +120,7 @@ def run_validator(
         metadata: Dict,
         property_path: str,
         stream: Optional[bool] = False,
+        **kwargs,
     ) -> ValidatorLogs:
         validator_class_name = validator.__class__.__name__
         validator_logs = ValidatorLogs(
@@ -130,7 +132,7 @@ def run_validator(
         iteration.outputs.validator_logs.append(validator_logs)
 
         start_time = datetime.now()
-        result = self.execute_validator(validator, value, metadata, stream)
+        result = self.execute_validator(validator, value, metadata, stream, **kwargs)
         end_time = datetime.now()
 
         if result is None:
@@ -170,11 +172,12 @@ def run_validators(
         metadata: Dict[str, Any],
         property_path: str,
         stream: Optional[bool] = False,
+        **kwargs,
     ) -> Tuple[Any, Dict[str, Any]]:
         # Validate the field
         for validator in validator_setup.validators:
             validator_logs = self.run_validator(
-                iteration, validator, value, metadata, property_path, stream
+                iteration, validator, value, metadata, property_path, stream, **kwargs
             )
             result = validator_logs.validation_result
             # I assume we would want to wait until the end to do reasks for streams
@@ -247,6 +250,7 @@ def validate_stream(
         validator_setup: FieldValidation,
         iteration: Iteration,
         path: str = "$",
+        **kwargs,
     ) -> Tuple[Any, dict]:
         property_path = (
             f"{path}.{validator_setup.key}"
@@ -258,7 +262,7 @@ def validate_stream(
 
         # Validate the field
         value, metadata = self.run_validators(
-            iteration, validator_setup, value, metadata, property_path, True
+            iteration, validator_setup, value, metadata, property_path, True, **kwargs
         )
 
         return value, metadata
@@ -444,6 +448,7 @@ def validate(
     iteration: Iteration,
     disable_tracer: Optional[bool] = True,
     stream: Optional[bool] = False,
+    **kwargs,
 ):
     process_count = int(os.environ.get("GUARDRAILS_PROCESS_COUNT", 10))
     if process_count == 1:
@@ -457,7 +462,7 @@ def validate(
     if stream:
         sequential_validator_service = SequentialValidatorService(disable_tracer)
         return sequential_validator_service.validate_stream(
-            value, metadata, validator_setup, iteration
+            value, metadata, validator_setup, iteration, **kwargs
         )
     try:
         loop = asyncio.get_event_loop()

From 0ab245ca9fa2dfa38fa3f493fb5bfa9b6b631013 Mon Sep 17 00:00:00 2001
From: Nicholas Chen <nichwch@gmail.com>
Date: Thu, 16 May 2024 18:52:14 -0400
Subject: [PATCH 18/85] new chunk span validation schema

---
 guardrails/validator_base.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/guardrails/validator_base.py b/guardrails/validator_base.py
index ca1a54ea5..d97ee83aa 100644
--- a/guardrails/validator_base.py
+++ b/guardrails/validator_base.py
@@ -378,6 +378,9 @@ def get_validator(name: str):
 class ValidationResult(BaseModel):
     outcome: str
     metadata: Optional[Dict[str, Any]] = None
+    # value argument passed to validator.validate
+    # or validator.validate_stream
+    validated_chunk: Any
 
 
 class PassResult(ValidationResult):
@@ -390,11 +393,19 @@ class ValueOverrideSentinel:
     value_override: Optional[Any] = Field(default=ValueOverrideSentinel)
 
 
+# specifies the start and end of segment of validate_chunk
+class ErrorSpan:
+    start: int
+    end: int
+
+
 class FailResult(ValidationResult):
     outcome: Literal["fail"] = "fail"
 
     error_message: str
     fix_value: Optional[Any] = None
+    # segments that caused validation to fail
+    error_spans: List[ErrorSpan]
 
 
 class OnFailAction(str, Enum):

From a3204645316129c098b0b4f85d5eac9852c6fd4d Mon Sep 17 00:00:00 2001
From: Nicholas Chen <nichwch@gmail.com>
Date: Thu, 16 May 2024 19:54:34 -0400
Subject: [PATCH 19/85] field for reason that validation failed for a given
 span

---
 guardrails/validator_base.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/guardrails/validator_base.py b/guardrails/validator_base.py
index d97ee83aa..9a2c520e2 100644
--- a/guardrails/validator_base.py
+++ b/guardrails/validator_base.py
@@ -397,6 +397,8 @@ class ValueOverrideSentinel:
 class ErrorSpan:
     start: int
     end: int
+    # reason validation failed, specific to this chunk
+    reason: str
 
 
 class FailResult(ValidationResult):

From 93bb7813b7f553dfaf1f4ddb8a3a65a3b1eadbbb Mon Sep 17 00:00:00 2001
From: Nicholas Chen <nichwch@gmail.com>
Date: Thu, 16 May 2024 20:00:57 -0400
Subject: [PATCH 20/85] add validated_chunk to ValidationResult

---
 guardrails/validator_base.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/guardrails/validator_base.py b/guardrails/validator_base.py
index 9a2c520e2..1d41dcb70 100644
--- a/guardrails/validator_base.py
+++ b/guardrails/validator_base.py
@@ -524,7 +524,8 @@ def validate_stream(
         # exclude last chunk, because it may not be a complete chunk
         validation_result = self.validate(chunk_to_validate, metadata)
         # include the chunk that we've validated in the metadata
-        validation_result.metadata["validated_chunk"] = chunk_to_validate
+        # TODO: Can I count on the validator to do this?
+        validation_result.validated_chunk = chunk_to_validate
         return validation_result
 
     def to_prompt(self, with_keywords: bool = True) -> str:

From 1381821e7209664a1b7073af617e9c5582eb1962 Mon Sep 17 00:00:00 2001
From: Nicholas Chen <nichwch@gmail.com>
Date: Fri, 17 May 2024 12:35:42 -0400
Subject: [PATCH 21/85] add helper method to get a list of error spans relative
 to llm output

---
 guardrails/classes/history/iteration.py |  7 +++++++
 guardrails/classes/history/outputs.py   | 22 +++++++++++++++++++++-
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/guardrails/classes/history/iteration.py b/guardrails/classes/history/iteration.py
index 1eb9e12dd..12120d914 100644
--- a/guardrails/classes/history/iteration.py
+++ b/guardrails/classes/history/iteration.py
@@ -15,6 +15,7 @@
 from guardrails.utils.logs_utils import ValidatorLogs
 from guardrails.utils.pydantic_utils import ArbitraryModel
 from guardrails.utils.reask_utils import ReAsk
+from guardrails.validator_base import ErrorSpan
 
 
 class Iteration(ArbitraryModel):
@@ -155,6 +156,12 @@ def failed_validations(self) -> List[ValidatorLogs]:
         iteration."""
         return self.outputs.failed_validations
 
+    @property
+    def error_spans_in_output(self) -> List[ErrorSpan]:
+        """The error spans from the LLM response.
+        These indices are relative to the complete LLM output."""
+        return self.outputs.error_spans_in_output
+
     @property
     def status(self) -> str:
         """Representation of the end state of this iteration.
diff --git a/guardrails/classes/history/outputs.py b/guardrails/classes/history/outputs.py
index 9d4d19544..699e6bad8 100644
--- a/guardrails/classes/history/outputs.py
+++ b/guardrails/classes/history/outputs.py
@@ -8,7 +8,7 @@
 from guardrails.utils.logs_utils import ValidatorLogs
 from guardrails.utils.pydantic_utils import ArbitraryModel
 from guardrails.utils.reask_utils import ReAsk
-from guardrails.validator_base import FailResult
+from guardrails.validator_base import ErrorSpan, FailResult
 
 
 class Outputs(ArbitraryModel):
@@ -75,6 +75,26 @@ def failed_validations(self) -> List[ValidatorLogs]:
             ]
         )
 
+    @property
+    def error_spans_in_output(self) -> List[ErrorSpan]:
+        """The error spans from the LLM response.
+        These indices are relative to the complete LLM output."""
+        total_len = 0
+        spans_in_output = []
+        for log in self.validator_logs:
+            result = log.validation_result
+            if isinstance(result, FailResult):
+                for error_span in result.error_spans:
+                    spans_in_output.append(
+                        ErrorSpan(
+                            start=error_span.start + total_len,
+                            end=error_span.end + total_len,
+                            message=error_span.message,
+                        )
+                    )
+            total_len += len(log.value_before_validation)
+        return spans_in_output
+
     @property
     def status(self) -> str:
         """Representation of the end state of the validation run.

From 3ccdda1d1aca54ffa2b1626ec43cdfbf48b3fe64 Mon Sep 17 00:00:00 2001
From: Nicholas Chen <nichwch@gmail.com>
Date: Fri, 17 May 2024 12:36:57 -0400
Subject: [PATCH 22/85] conceptual question

---
 guardrails/classes/history/outputs.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/guardrails/classes/history/outputs.py b/guardrails/classes/history/outputs.py
index 699e6bad8..ec9e34a92 100644
--- a/guardrails/classes/history/outputs.py
+++ b/guardrails/classes/history/outputs.py
@@ -75,6 +75,7 @@ def failed_validations(self) -> List[ValidatorLogs]:
             ]
         )
 
+    # TODO: Is it possible the user wants the entire ValidationResult here too?
     @property
     def error_spans_in_output(self) -> List[ErrorSpan]:
         """The error spans from the LLM response.

From 74485eb8f0ec3f8564cb8b3f2da903d7082cf825 Mon Sep 17 00:00:00 2001
From: Nicholas Chen <nichwch@gmail.com>
Date: Fri, 17 May 2024 15:44:55 -0400
Subject: [PATCH 23/85] turn chunking_function into class method

---
 guardrails/validator_base.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/guardrails/validator_base.py b/guardrails/validator_base.py
index ca1a54ea5..c5661d4ce 100644
--- a/guardrails/validator_base.py
+++ b/guardrails/validator_base.py
@@ -416,7 +416,6 @@ class Validator(Runnable):
     # chunking function returns empty list or list of 2 chunks
     # first chunk is the chunk to validate
     # second chunk is incomplete chunk that needs further accumulation
-    chunking_function = split_sentence
     accumulated_chunks = []
     run_in_separate_process = False
     override_value_on_pass = False
@@ -476,6 +475,9 @@ def __init__(
             self.rail_alias in validators_registry
         ), f"Validator {self.__class__.__name__} is not registered. "
 
+    def chunking_function(self, chunk: str):
+        return split_sentence(chunk)
+
     def validate(self, value: Any, metadata: Dict[str, Any]) -> ValidationResult:
         """Validates a value and return a validation result."""
         raise NotImplementedError

From a39b5af45c7fa4b20d139bf3f62d04111ebf24b4 Mon Sep 17 00:00:00 2001
From: Nicholas Chen <nichwch@gmail.com>
Date: Fri, 17 May 2024 19:30:51 -0400
Subject: [PATCH 24/85] incomplete tests for streaming chunk accumulation

---
 guardrails/run/stream_runner.py           |  1 +
 tests/integration_tests/test_streaming.py | 94 ++++++++++++++++++-----
 2 files changed, 75 insertions(+), 20 deletions(-)

diff --git a/guardrails/run/stream_runner.py b/guardrails/run/stream_runner.py
index 9e91bbc4a..c7e045914 100644
--- a/guardrails/run/stream_runner.py
+++ b/guardrails/run/stream_runner.py
@@ -157,6 +157,7 @@ def step(
 
         if isinstance(output_schema, StringSchema):
             for chunk in stream:
+                print('chunk', chunk)
                 # 1. Get the text from the chunk and append to fragment
                 chunk_text = self.get_chunk_text(chunk, api)
                 fragment += chunk_text
diff --git a/tests/integration_tests/test_streaming.py b/tests/integration_tests/test_streaming.py
index 4551f8f4e..ae98778df 100644
--- a/tests/integration_tests/test_streaming.py
+++ b/tests/integration_tests/test_streaming.py
@@ -3,15 +3,17 @@
 # 2. Test streaming with OpenAIChatCallable (mock openai.ChatCompletion.create)
 # Using the LowerCase Validator
 import json
-from typing import Iterable
+from typing import Any, Callable, Dict, Iterable, List, Optional, Union
 
 import openai
 import pytest
+import nltk
 from pydantic import BaseModel, Field
 
 import guardrails as gd
+from guardrails.utils.casting_utils import to_int
 from guardrails.utils.openai_utils import OPENAI_VERSION
-from guardrails.validator_base import OnFailAction
+from guardrails.validator_base import FailResult, OnFailAction, PassResult, ValidationResult, Validator, register_validator
 from guardrails.validators import LowerCase
 
 expected_raw_output = {"statement": "I am DOING well, and I HOPE you aRe too."}
@@ -19,6 +21,53 @@
 expected_noop_output = {"statement": "I am DOING well, and I HOPE you aRe too."}
 expected_filter_refrain_output = {}
 
+@register_validator(name="minsentencelength", data_type=["string","list"])
+class MinSentenceLengthValidator(Validator):
+    def __init__(
+        self,
+        min: Optional[int] = None,
+        max: Optional[int] = None,
+        on_fail: Optional[Callable] = None,
+    ):
+        super().__init__(
+            on_fail=on_fail,
+            min=min,
+            max=max,
+        )
+        self._min = to_int(min)
+        self._max = to_int(max)
+    
+    def sentence_split(self, value):
+        if '.' not in value:
+            return [value]
+        sentences = nltk.sent_tokenize(value)
+        if len(sentences) == 0:
+            return [value]
+        return sentences
+        
+
+    def validate(self, value: Union[str, List], metadata: Dict) -> ValidationResult:
+        # return PassResult()
+        sentences = self.sentence_split(value)
+        for sentence in sentences:
+            if len(sentence) < self._min:
+                return FailResult(
+                    error_message=f"Sentence has length less than {self._min}. "
+                    f"Please return a longer output, "
+                    f"that is shorter than {self._max} characters.",
+                )
+            if len(sentence) > self._max:
+                return FailResult(
+                    error_message=f"Sentence has length greater than {self._max}. "
+                    f"Please return a shorter output, "
+                    f"that is shorter than {self._max} characters.",
+                )
+        return PassResult()
+    
+    def validate_stream(self, chunk: Any, metadata: Dict, **kwargs) -> ValidationResult:
+        print(chunk, 'here!!!!')
+        return super().validate_stream(chunk, metadata, **kwargs)
+
 
 class Delta:
     content: str
@@ -145,6 +194,15 @@ class LowerCaseRefrain(BaseModel):
         validators=[LowerCase(on_fail=OnFailAction.REFRAIN)],
     )
 
+expected_minsentence_noop_output = ''
+class MinSentenceLengthNoOp(BaseModel):
+    statement: str = Field(
+        description="Validates whether the text is in lower case.",
+        validators=[MinSentenceLengthValidator(on_fail=OnFailAction.NOOP)],
+    )
+
+
+STR_PROMPT = 'Say something nice to me.'
 
 PROMPT = """
 Say something nice to me.
@@ -154,17 +212,18 @@ class LowerCaseRefrain(BaseModel):
 
 
 @pytest.mark.parametrize(
-    "op_class, expected_validated_output",
+    "guard, expected_validated_output",
     [
-        (LowerCaseNoop, expected_noop_output),
-        (LowerCaseFix, expected_fix_output),
-        (LowerCaseFilter, expected_filter_refrain_output),
-        (LowerCaseRefrain, expected_filter_refrain_output),
+        (gd.Guard.from_pydantic(output_class=LowerCaseNoop, prompt=PROMPT), expected_noop_output, ),
+        (gd.Guard.from_pydantic(output_class=LowerCaseFix, prompt=PROMPT), expected_fix_output, ),
+        (gd.Guard.from_pydantic(output_class=LowerCaseFilter, prompt=PROMPT), expected_filter_refrain_output, ),
+        (gd.Guard.from_pydantic(output_class=LowerCaseRefrain, prompt=PROMPT), expected_filter_refrain_output, ),
+        (gd.Guard.from_string(validators=[MinSentenceLengthValidator(5, 10)], prompt=STR_PROMPT), '' , ),
     ],
 )
 def test_streaming_with_openai_callable(
     mocker,
-    op_class,
+    guard,
     expected_validated_output,
 ):
     """Test streaming with OpenAICallable.
@@ -181,9 +240,6 @@ def test_streaming_with_openai_callable(
             return_value=mock_openai_completion_create(),
         )
 
-    # Create a guard object
-    guard = gd.Guard.from_pydantic(output_class=op_class, prompt=PROMPT)
-
     method = (
         openai.Completion.create
         if OPENAI_VERSION.startswith("0")
@@ -210,17 +266,18 @@ def test_streaming_with_openai_callable(
 
 
 @pytest.mark.parametrize(
-    "op_class, expected_validated_output",
+    "guard, expected_validated_output",
     [
-        (LowerCaseNoop, expected_noop_output),
-        (LowerCaseFix, expected_fix_output),
-        (LowerCaseFilter, expected_filter_refrain_output),
-        (LowerCaseRefrain, expected_filter_refrain_output),
+        (gd.Guard.from_pydantic(output_class=LowerCaseNoop, prompt=PROMPT), expected_noop_output, ),
+        (gd.Guard.from_pydantic(output_class=LowerCaseFix, prompt=PROMPT), expected_fix_output, ),
+        (gd.Guard.from_pydantic(output_class=LowerCaseFilter, prompt=PROMPT), expected_filter_refrain_output, ),
+        (gd.Guard.from_pydantic(output_class=LowerCaseRefrain, prompt=PROMPT), expected_filter_refrain_output, ),
+        (gd.Guard.from_string(validators=[MinSentenceLengthValidator(5, 10)], prompt=STR_PROMPT), '' , ),
     ],
 )
 def test_streaming_with_openai_chat_callable(
     mocker,
-    op_class,
+    guard,
     expected_validated_output,
 ):
     """Test streaming with OpenAIChatCallable.
@@ -238,9 +295,6 @@ def test_streaming_with_openai_chat_callable(
             return_value=mock_openai_chat_completion_create(),
         )
 
-    # Create a guard object
-    guard = gd.Guard.from_pydantic(output_class=op_class, prompt=PROMPT)
-
     method = (
         openai.ChatCompletion.create
         if OPENAI_VERSION.startswith("0")

From 0ae850e872a02e3427d6986a88ad2466985de433 Mon Sep 17 00:00:00 2001
From: Nicholas Chen <nichwch@gmail.com>
Date: Mon, 20 May 2024 11:24:02 -0400
Subject: [PATCH 25/85] format

---
 guardrails/run/stream_runner.py           |  2 +-
 tests/integration_tests/test_streaming.py | 82 +++++++++++++++++------
 2 files changed, 64 insertions(+), 20 deletions(-)

diff --git a/guardrails/run/stream_runner.py b/guardrails/run/stream_runner.py
index c7e045914..25402b380 100644
--- a/guardrails/run/stream_runner.py
+++ b/guardrails/run/stream_runner.py
@@ -157,7 +157,7 @@ def step(
 
         if isinstance(output_schema, StringSchema):
             for chunk in stream:
-                print('chunk', chunk)
+                print("chunk", chunk)
                 # 1. Get the text from the chunk and append to fragment
                 chunk_text = self.get_chunk_text(chunk, api)
                 fragment += chunk_text
diff --git a/tests/integration_tests/test_streaming.py b/tests/integration_tests/test_streaming.py
index ae98778df..7a27ed107 100644
--- a/tests/integration_tests/test_streaming.py
+++ b/tests/integration_tests/test_streaming.py
@@ -13,7 +13,14 @@
 import guardrails as gd
 from guardrails.utils.casting_utils import to_int
 from guardrails.utils.openai_utils import OPENAI_VERSION
-from guardrails.validator_base import FailResult, OnFailAction, PassResult, ValidationResult, Validator, register_validator
+from guardrails.validator_base import (
+    FailResult,
+    OnFailAction,
+    PassResult,
+    ValidationResult,
+    Validator,
+    register_validator,
+)
 from guardrails.validators import LowerCase
 
 expected_raw_output = {"statement": "I am DOING well, and I HOPE you aRe too."}
@@ -21,7 +28,8 @@
 expected_noop_output = {"statement": "I am DOING well, and I HOPE you aRe too."}
 expected_filter_refrain_output = {}
 
-@register_validator(name="minsentencelength", data_type=["string","list"])
+
+@register_validator(name="minsentencelength", data_type=["string", "list"])
 class MinSentenceLengthValidator(Validator):
     def __init__(
         self,
@@ -36,15 +44,14 @@ def __init__(
         )
         self._min = to_int(min)
         self._max = to_int(max)
-    
+
     def sentence_split(self, value):
-        if '.' not in value:
+        if "." not in value:
             return [value]
         sentences = nltk.sent_tokenize(value)
         if len(sentences) == 0:
             return [value]
         return sentences
-        
 
     def validate(self, value: Union[str, List], metadata: Dict) -> ValidationResult:
         # return PassResult()
@@ -63,9 +70,9 @@ def validate(self, value: Union[str, List], metadata: Dict) -> ValidationResult:
                     f"that is shorter than {self._max} characters.",
                 )
         return PassResult()
-    
+
     def validate_stream(self, chunk: Any, metadata: Dict, **kwargs) -> ValidationResult:
-        print(chunk, 'here!!!!')
+        print(chunk, "here!!!!")
         return super().validate_stream(chunk, metadata, **kwargs)
 
 
@@ -194,7 +201,10 @@ class LowerCaseRefrain(BaseModel):
         validators=[LowerCase(on_fail=OnFailAction.REFRAIN)],
     )
 
-expected_minsentence_noop_output = ''
+
+expected_minsentence_noop_output = ""
+
+
 class MinSentenceLengthNoOp(BaseModel):
     statement: str = Field(
         description="Validates whether the text is in lower case.",
@@ -202,7 +212,7 @@ class MinSentenceLengthNoOp(BaseModel):
     )
 
 
-STR_PROMPT = 'Say something nice to me.'
+STR_PROMPT = "Say something nice to me."
 
 PROMPT = """
 Say something nice to me.
@@ -214,11 +224,28 @@ class MinSentenceLengthNoOp(BaseModel):
 @pytest.mark.parametrize(
     "guard, expected_validated_output",
     [
-        (gd.Guard.from_pydantic(output_class=LowerCaseNoop, prompt=PROMPT), expected_noop_output, ),
-        (gd.Guard.from_pydantic(output_class=LowerCaseFix, prompt=PROMPT), expected_fix_output, ),
-        (gd.Guard.from_pydantic(output_class=LowerCaseFilter, prompt=PROMPT), expected_filter_refrain_output, ),
-        (gd.Guard.from_pydantic(output_class=LowerCaseRefrain, prompt=PROMPT), expected_filter_refrain_output, ),
-        (gd.Guard.from_string(validators=[MinSentenceLengthValidator(5, 10)], prompt=STR_PROMPT), '' , ),
+        (
+            gd.Guard.from_pydantic(output_class=LowerCaseNoop, prompt=PROMPT),
+            expected_noop_output,
+        ),
+        (
+            gd.Guard.from_pydantic(output_class=LowerCaseFix, prompt=PROMPT),
+            expected_fix_output,
+        ),
+        (
+            gd.Guard.from_pydantic(output_class=LowerCaseFilter, prompt=PROMPT),
+            expected_filter_refrain_output,
+        ),
+        (
+            gd.Guard.from_pydantic(output_class=LowerCaseRefrain, prompt=PROMPT),
+            expected_filter_refrain_output,
+        ),
+        (
+            gd.Guard.from_string(
+                validators=[MinSentenceLengthValidator(5, 10)], prompt=STR_PROMPT
+            ),
+            "",
+        ),
     ],
 )
 def test_streaming_with_openai_callable(
@@ -268,11 +295,28 @@ def test_streaming_with_openai_callable(
 @pytest.mark.parametrize(
     "guard, expected_validated_output",
     [
-        (gd.Guard.from_pydantic(output_class=LowerCaseNoop, prompt=PROMPT), expected_noop_output, ),
-        (gd.Guard.from_pydantic(output_class=LowerCaseFix, prompt=PROMPT), expected_fix_output, ),
-        (gd.Guard.from_pydantic(output_class=LowerCaseFilter, prompt=PROMPT), expected_filter_refrain_output, ),
-        (gd.Guard.from_pydantic(output_class=LowerCaseRefrain, prompt=PROMPT), expected_filter_refrain_output, ),
-        (gd.Guard.from_string(validators=[MinSentenceLengthValidator(5, 10)], prompt=STR_PROMPT), '' , ),
+        (
+            gd.Guard.from_pydantic(output_class=LowerCaseNoop, prompt=PROMPT),
+            expected_noop_output,
+        ),
+        (
+            gd.Guard.from_pydantic(output_class=LowerCaseFix, prompt=PROMPT),
+            expected_fix_output,
+        ),
+        (
+            gd.Guard.from_pydantic(output_class=LowerCaseFilter, prompt=PROMPT),
+            expected_filter_refrain_output,
+        ),
+        (
+            gd.Guard.from_pydantic(output_class=LowerCaseRefrain, prompt=PROMPT),
+            expected_filter_refrain_output,
+        ),
+        (
+            gd.Guard.from_string(
+                validators=[MinSentenceLengthValidator(5, 10)], prompt=STR_PROMPT
+            ),
+            "",
+        ),
     ],
 )
 def test_streaming_with_openai_chat_callable(

From 847dd0a216e73ad42ecb76b60a5efe1e701aad4d Mon Sep 17 00:00:00 2001
From: Nicholas Chen <nichwch@gmail.com>
Date: Mon, 20 May 2024 12:30:26 -0400
Subject: [PATCH 26/85] remove print

---
 guardrails/run/stream_runner.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/guardrails/run/stream_runner.py b/guardrails/run/stream_runner.py
index 25402b380..9e91bbc4a 100644
--- a/guardrails/run/stream_runner.py
+++ b/guardrails/run/stream_runner.py
@@ -157,7 +157,6 @@ def step(
 
         if isinstance(output_schema, StringSchema):
             for chunk in stream:
-                print("chunk", chunk)
                 # 1. Get the text from the chunk and append to fragment
                 chunk_text = self.get_chunk_text(chunk, api)
                 fragment += chunk_text

From f0b3030c3ceb24949c215c23fb41cf5cc52b2842 Mon Sep 17 00:00:00 2001
From: Nicholas Chen <nichwch@gmail.com>
Date: Mon, 20 May 2024 12:58:38 -0400
Subject: [PATCH 27/85] fix a few bugs uncovered by testing

---
 guardrails/run/stream_runner.py |  8 ++++++--
 guardrails/validator_base.py    |  4 ++--
 guardrails/validator_service.py | 10 +++++-----
 3 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/guardrails/run/stream_runner.py b/guardrails/run/stream_runner.py
index 9e91bbc4a..65f88aab7 100644
--- a/guardrails/run/stream_runner.py
+++ b/guardrails/run/stream_runner.py
@@ -191,10 +191,14 @@ def step(
                         "Reasks are not yet supported with streaming. Please "
                         "remove reasks from schema or disable streaming."
                     )
+                print("valoutcome args")
+                print(chunk_text)
+                print(validated_result)
+                print(validated_fragment)
                 # 5. Convert validated fragment to a pretty JSON string
                 yield ValidationOutcome(
                     #  The chunk or the whole output?
-                    raw_llm_output=chunk,
+                    raw_llm_output=chunk_text,
                     validated_output=validated_result,
                     validation_passed=validated_fragment is not None,
                 )
@@ -228,7 +232,7 @@ def step(
             # 5. Convert validated fragment to a pretty JSON string
             yield ValidationOutcome(
                 #  The chunk or the whole output?
-                raw_llm_output=chunk,
+                raw_llm_output=chunk_text,
                 validated_output=remainder_validation,
                 validation_passed=remainder_validation is not None,
             )
diff --git a/guardrails/validator_base.py b/guardrails/validator_base.py
index c5661d4ce..8bbe401bf 100644
--- a/guardrails/validator_base.py
+++ b/guardrails/validator_base.py
@@ -497,6 +497,7 @@ def validate_stream(
         Otherwise, the validator will validate the chunk and return the result.
         """
         # combine accumulated chunks and new chunk
+        print("acc chunk", self.accumulated_chunks)
         self.accumulated_chunks.append(chunk)
         accumulated_text = "".join(self.accumulated_chunks)
         # check if enough chunks have accumulated for validation
@@ -509,11 +510,10 @@ def validate_stream(
         if len(splitcontents) == 0:
             return None
         [chunk_to_validate, new_accumulated_chunks] = splitcontents
-        self.accumulated_chunks = new_accumulated_chunks
+        self.accumulated_chunks = [new_accumulated_chunks]
         # exclude last chunk, because it may not be a complete chunk
         validation_result = self.validate(chunk_to_validate, metadata)
         # include the chunk that we've validated in the metadata
-        validation_result.metadata["validated_chunk"] = chunk_to_validate
         return validation_result
 
     def to_prompt(self, with_keywords: bool = True) -> str:
diff --git a/guardrails/validator_service.py b/guardrails/validator_service.py
index b13d659c7..0c502b802 100644
--- a/guardrails/validator_service.py
+++ b/guardrails/validator_service.py
@@ -179,11 +179,11 @@ def run_validators(
                 iteration, validator, value, metadata, property_path, stream, **kwargs
             )
             result = validator_logs.validation_result
-            # I assume we would want to wait until the end to do reasks for streams
-            if isinstance(result, FailResult) and not stream:
-                value = self.perform_correction(
-                    [result], value, validator, validator.on_fail_descriptor
-                )
+            if isinstance(result, FailResult):
+                if not stream:
+                    value = self.perform_correction(
+                        [result], value, validator, validator.on_fail_descriptor
+                    )
             elif isinstance(result, PassResult):
                 if (
                     validator.override_value_on_pass

From e8b6069b03b5259d1e31625d510b45520d4f46fc Mon Sep 17 00:00:00 2001
From: Nicholas Chen <nichwch@gmail.com>
Date: Mon, 20 May 2024 18:38:51 -0400
Subject: [PATCH 28/85] tests (WIP) for streaming

---
 guardrails/run/stream_runner.py           |   6 +-
 guardrails/validator_base.py              |   2 +-
 guardrails/validator_service.py           |   4 +-
 tests/integration_tests/test_streaming.py | 129 +++++++++++++++-------
 4 files changed, 97 insertions(+), 44 deletions(-)

diff --git a/guardrails/run/stream_runner.py b/guardrails/run/stream_runner.py
index 65f88aab7..99b2c11cc 100644
--- a/guardrails/run/stream_runner.py
+++ b/guardrails/run/stream_runner.py
@@ -191,16 +191,12 @@ def step(
                         "Reasks are not yet supported with streaming. Please "
                         "remove reasks from schema or disable streaming."
                     )
-                print("valoutcome args")
-                print(chunk_text)
-                print(validated_result)
-                print(validated_fragment)
                 # 5. Convert validated fragment to a pretty JSON string
                 yield ValidationOutcome(
                     #  The chunk or the whole output?
                     raw_llm_output=chunk_text,
                     validated_output=validated_result,
-                    validation_passed=validated_fragment is not None,
+                    validation_passed=validated_result is not None,
                 )
             ######################################
             # need to validate remainder of chunks
diff --git a/guardrails/validator_base.py b/guardrails/validator_base.py
index 8bbe401bf..05846c1a4 100644
--- a/guardrails/validator_base.py
+++ b/guardrails/validator_base.py
@@ -497,7 +497,6 @@ def validate_stream(
         Otherwise, the validator will validate the chunk and return the result.
         """
         # combine accumulated chunks and new chunk
-        print("acc chunk", self.accumulated_chunks)
         self.accumulated_chunks.append(chunk)
         accumulated_text = "".join(self.accumulated_chunks)
         # check if enough chunks have accumulated for validation
@@ -505,6 +504,7 @@ def validate_stream(
 
         # if remainder kwargs is passed, validate remainder regardless
         remainder = kwargs.get("remainder", False)
+        print("split contents:", splitcontents)
         if remainder:
             splitcontents = [accumulated_text, []]
         if len(splitcontents) == 0:
diff --git a/guardrails/validator_service.py b/guardrails/validator_service.py
index 0c502b802..86da6aabe 100644
--- a/guardrails/validator_service.py
+++ b/guardrails/validator_service.py
@@ -179,11 +179,14 @@ def run_validators(
                 iteration, validator, value, metadata, property_path, stream, **kwargs
             )
             result = validator_logs.validation_result
+            print("result log", validator_logs)
             if isinstance(result, FailResult):
                 if not stream:
                     value = self.perform_correction(
                         [result], value, validator, validator.on_fail_descriptor
                     )
+                else:
+                    return None, None
             elif isinstance(result, PassResult):
                 if (
                     validator.override_value_on_pass
@@ -239,7 +242,6 @@ def validate(
         value, metadata = self.run_validators(
             iteration, validator_setup, value, metadata, property_path
         )
-
         return value, metadata
 
     def validate_stream(
diff --git a/tests/integration_tests/test_streaming.py b/tests/integration_tests/test_streaming.py
index 7a27ed107..3fa09dc6a 100644
--- a/tests/integration_tests/test_streaming.py
+++ b/tests/integration_tests/test_streaming.py
@@ -1,7 +1,8 @@
-# 2 tests
+# 3 tests
 # 1. Test streaming with OpenAICallable (mock openai.Completion.create)
 # 2. Test streaming with OpenAIChatCallable (mock openai.ChatCompletion.create)
-# Using the LowerCase Validator
+# 3. Test string schema streaming
+# Using the LowerCase Validator, and a custom validator to show new streaming behavior
 import json
 from typing import Any, Callable, Dict, Iterable, List, Optional, Union
 
@@ -56,6 +57,7 @@ def sentence_split(self, value):
     def validate(self, value: Union[str, List], metadata: Dict) -> ValidationResult:
         # return PassResult()
         sentences = self.sentence_split(value)
+        print("validating sentence:", sentences)
         for sentence in sentences:
             if len(sentence) < self._min:
                 return FailResult(
@@ -72,7 +74,6 @@ def validate(self, value: Union[str, List], metadata: Dict) -> ValidationResult:
         return PassResult()
 
     def validate_stream(self, chunk: Any, metadata: Dict, **kwargs) -> ValidationResult:
-        print(chunk, "here!!!!")
         return super().validate_stream(chunk, metadata, **kwargs)
 
 
@@ -105,16 +106,8 @@ def __init__(self, choices, model):
         self.model = model
 
 
-def mock_openai_completion_create():
+def mock_openai_completion_create(chunks):
     # Returns a generator
-    chunks = [
-        '{"statement":',
-        ' "I am DOING',
-        " well, and I",
-        " HOPE you aRe",
-        ' too."}',
-    ]
-
     def gen():
         for chunk in chunks:
             if OPENAI_VERSION.startswith("0"):
@@ -137,16 +130,8 @@ def gen():
     return gen()
 
 
-def mock_openai_chat_completion_create():
+def mock_openai_chat_completion_create(chunks):
     # Returns a generator
-    chunks = [
-        '{"statement":',
-        ' "I am DOING',
-        " well, and I",
-        " HOPE you aRe",
-        ' too."}',
-    ]
-
     def gen():
         for chunk in chunks:
             if OPENAI_VERSION.startswith("0"):
@@ -220,6 +205,14 @@ class MinSentenceLengthNoOp(BaseModel):
 ${gr.complete_json_suffix}
 """
 
+JSON_LLM_CHUNKS = [
+    '{"statement":',
+    ' "I am DOING',
+    " well, and I",
+    " HOPE you aRe",
+    ' too."}',
+]
+
 
 @pytest.mark.parametrize(
     "guard, expected_validated_output",
@@ -240,12 +233,6 @@ class MinSentenceLengthNoOp(BaseModel):
             gd.Guard.from_pydantic(output_class=LowerCaseRefrain, prompt=PROMPT),
             expected_filter_refrain_output,
         ),
-        (
-            gd.Guard.from_string(
-                validators=[MinSentenceLengthValidator(5, 10)], prompt=STR_PROMPT
-            ),
-            "",
-        ),
     ],
 )
 def test_streaming_with_openai_callable(
@@ -259,12 +246,13 @@ def test_streaming_with_openai_callable(
     """
     if OPENAI_VERSION.startswith("0"):
         mocker.patch(
-            "openai.Completion.create", return_value=mock_openai_completion_create()
+            "openai.Completion.create",
+            return_value=mock_openai_completion_create(JSON_LLM_CHUNKS),
         )
     else:
         mocker.patch(
             "openai.resources.Completions.create",
-            return_value=mock_openai_completion_create(),
+            return_value=mock_openai_completion_create(JSON_LLM_CHUNKS),
         )
 
     method = (
@@ -311,12 +299,6 @@ def test_streaming_with_openai_callable(
             gd.Guard.from_pydantic(output_class=LowerCaseRefrain, prompt=PROMPT),
             expected_filter_refrain_output,
         ),
-        (
-            gd.Guard.from_string(
-                validators=[MinSentenceLengthValidator(5, 10)], prompt=STR_PROMPT
-            ),
-            "",
-        ),
     ],
 )
 def test_streaming_with_openai_chat_callable(
@@ -331,12 +313,12 @@ def test_streaming_with_openai_chat_callable(
     if OPENAI_VERSION.startswith("0"):
         mocker.patch(
             "openai.ChatCompletion.create",
-            return_value=mock_openai_chat_completion_create(),
+            return_value=mock_openai_chat_completion_create(JSON_LLM_CHUNKS),
         )
     else:
         mocker.patch(
             "openai.resources.chat.completions.Completions.create",
-            return_value=mock_openai_chat_completion_create(),
+            return_value=mock_openai_chat_completion_create(JSON_LLM_CHUNKS),
         )
 
     method = (
@@ -363,3 +345,76 @@ def test_streaming_with_openai_chat_callable(
 
     assert actual_output.raw_llm_output == json.dumps(expected_raw_output)
     assert actual_output.validated_output == expected_validated_output
+
+
+STR_LLM_CHUNKS = [
+    # 38 characters
+    "This sentence is simply just ",
+    "too long."
+    # 25 characters long
+    "This ",
+    "sentence ",
+    "is 2 ",
+    "short."
+    # 29 characters long
+    "This sentence is just ",
+    "right.",
+]
+
+
+@pytest.mark.parametrize(
+    "guard, expected_validated_output",
+    [
+        (
+            gd.Guard.from_string(
+                # only the middle sentence should pass
+                validators=[
+                    MinSentenceLengthValidator(26, 30, on_fail=OnFailAction.NOOP)
+                ],
+                prompt=STR_PROMPT,
+            ),
+            # For now these should be correct.
+            # This will be different pending validation outcome
+            # schema changes.
+            [True, False, True, True, False, False],
+        )
+    ],
+)
+def test_string_schema_streaming_with_openai_chat(
+    mocker, guard, expected_validated_output
+):
+    """Test string schema streaming with OpenAIChatCallable.
+
+    Mocks openai.ChatCompletion.create.
+    """
+    if OPENAI_VERSION.startswith("0"):
+        mocker.patch(
+            "openai.ChatCompletion.create",
+            return_value=mock_openai_chat_completion_create(STR_LLM_CHUNKS),
+        )
+    else:
+        mocker.patch(
+            "openai.resources.chat.completions.Completions.create",
+            return_value=mock_openai_chat_completion_create(STR_LLM_CHUNKS),
+        )
+
+    method = (
+        openai.ChatCompletion.create
+        if OPENAI_VERSION.startswith("0")
+        else openai.chat.completions.create
+    )
+
+    method.__name__ = "mock_openai_chat_completion_create"
+    generator = guard(
+        method,
+        model="gpt-3.5-turbo",
+        max_tokens=10,
+        temperature=0,
+        stream=True,
+    )
+
+    assert isinstance(generator, Iterable)
+
+    for op, desired_result in zip(generator, expected_validated_output):
+        assert op.validation_passed == desired_result
+        print("op", op)

From eec8e19b25d4b8ec2e0ddc63cf9e725b1dc4894a Mon Sep 17 00:00:00 2001
From: Nicholas Chen <nichwch@gmail.com>
Date: Tue, 21 May 2024 11:38:50 -0400
Subject: [PATCH 29/85] base model

---
 guardrails/validator_base.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/guardrails/validator_base.py b/guardrails/validator_base.py
index d2a576085..4109e7644 100644
--- a/guardrails/validator_base.py
+++ b/guardrails/validator_base.py
@@ -394,7 +394,7 @@ class ValueOverrideSentinel:
 
 
 # specifies the start and end of segment of validate_chunk
-class ErrorSpan:
+class ErrorSpan(BaseModel):
     start: int
     end: int
     # reason validation failed, specific to this chunk

From 8726a28e514cfba9fedb0ae76095dbb698b91d56 Mon Sep 17 00:00:00 2001
From: Nicholas Chen <nichwch@gmail.com>
Date: Tue, 21 May 2024 12:04:39 -0400
Subject: [PATCH 30/85] optional typing to avoid breaking existing validators

---
 guardrails/validator_base.py              |  4 +--
 tests/integration_tests/test_streaming.py | 37 +++++++++++++++++------
 2 files changed, 29 insertions(+), 12 deletions(-)

diff --git a/guardrails/validator_base.py b/guardrails/validator_base.py
index 4109e7644..238f59c5b 100644
--- a/guardrails/validator_base.py
+++ b/guardrails/validator_base.py
@@ -380,7 +380,7 @@ class ValidationResult(BaseModel):
     metadata: Optional[Dict[str, Any]] = None
     # value argument passed to validator.validate
     # or validator.validate_stream
-    validated_chunk: Any
+    validated_chunk: Optional[Any] = None
 
 
 class PassResult(ValidationResult):
@@ -407,7 +407,7 @@ class FailResult(ValidationResult):
     error_message: str
     fix_value: Optional[Any] = None
     # segments that caused validation to fail
-    error_spans: List[ErrorSpan]
+    error_spans: Optional[List[ErrorSpan]] = []
 
 
 class OnFailAction(str, Enum):
diff --git a/tests/integration_tests/test_streaming.py b/tests/integration_tests/test_streaming.py
index 3fa09dc6a..90940207b 100644
--- a/tests/integration_tests/test_streaming.py
+++ b/tests/integration_tests/test_streaming.py
@@ -15,6 +15,7 @@
 from guardrails.utils.casting_utils import to_int
 from guardrails.utils.openai_utils import OPENAI_VERSION
 from guardrails.validator_base import (
+    ErrorSpan,
     FailResult,
     OnFailAction,
     PassResult,
@@ -55,22 +56,38 @@ def sentence_split(self, value):
         return sentences
 
     def validate(self, value: Union[str, List], metadata: Dict) -> ValidationResult:
-        # return PassResult()
         sentences = self.sentence_split(value)
-        print("validating sentence:", sentences)
+        error_spans = []
+        index = 0
         for sentence in sentences:
             if len(sentence) < self._min:
-                return FailResult(
-                    error_message=f"Sentence has length less than {self._min}. "
-                    f"Please return a longer output, "
-                    f"that is shorter than {self._max} characters.",
+                error_spans.append(
+                    ErrorSpan(
+                        start=index,
+                        end=index + len(sentence),
+                        reason=f"Sentence has length less than {self._min}. "
+                        f"Please return a longer output, "
+                        f"that is shorter than {self._max} characters.",
+                    )
                 )
             if len(sentence) > self._max:
-                return FailResult(
-                    error_message=f"Sentence has length greater than {self._max}. "
-                    f"Please return a shorter output, "
-                    f"that is shorter than {self._max} characters.",
+                error_spans.append(
+                    ErrorSpan(
+                        start=index,
+                        end=index + len(sentence),
+                        reason=f"Sentence has length greater than {self._max}. "
+                        f"Please return a shorter output, "
+                        f"that is shorter than {self._max} characters.",
+                    )
                 )
+        if len(error_spans) > 0:
+            return FailResult(
+                validated_chunk="".join(self.accumulated_chunks),
+                error_spans=error_spans,
+                error_message=f"Sentence has length less than {self._min}. "
+                f"Please return a longer output, "
+                f"that is shorter than {self._max} characters.",
+            )
         return PassResult()
 
     def validate_stream(self, chunk: Any, metadata: Dict, **kwargs) -> ValidationResult:

From ba68eb6d7399a01347f6b710406378ce9a47e9df Mon Sep 17 00:00:00 2001
From: Nicholas Chen <nichwch@gmail.com>
Date: Tue, 21 May 2024 13:58:39 -0400
Subject: [PATCH 31/85] top level helper function for spans on guard, patch
 validated_chunk in validate_stream if not provided

---
 guardrails/classes/history/outputs.py     |  2 +-
 guardrails/guard.py                       | 10 ++++++++++
 guardrails/validator_base.py              |  9 ++++-----
 tests/integration_tests/test_streaming.py |  6 ++++--
 4 files changed, 19 insertions(+), 8 deletions(-)

diff --git a/guardrails/classes/history/outputs.py b/guardrails/classes/history/outputs.py
index ec9e34a92..db6d183ac 100644
--- a/guardrails/classes/history/outputs.py
+++ b/guardrails/classes/history/outputs.py
@@ -90,7 +90,7 @@ def error_spans_in_output(self) -> List[ErrorSpan]:
                         ErrorSpan(
                             start=error_span.start + total_len,
                             end=error_span.end + total_len,
-                            message=error_span.message,
+                            reason=error_span.reason,
                         )
                     )
             total_len += len(log.value_before_validation)
diff --git a/guardrails/guard.py b/guardrails/guard.py
index 38fae77d0..cf691081d 100644
--- a/guardrails/guard.py
+++ b/guardrails/guard.py
@@ -1130,6 +1130,16 @@ async def _async_parse(
 
         return ValidationOutcome[OT].from_guard_history(call)
 
+    def error_spans_in_output(self):
+        try:
+            history = self.history
+            call = history[0]
+            iter = call.iterations[0]
+            llm_spans = iter.error_spans_in_output
+            return llm_spans
+        except (AttributeError, TypeError):
+            return []
+
     @deprecated(
         """The `with_prompt_validation` method is deprecated,
         and will be removed in 0.5.x. Instead, please use
diff --git a/guardrails/validator_base.py b/guardrails/validator_base.py
index 238f59c5b..a26a98046 100644
--- a/guardrails/validator_base.py
+++ b/guardrails/validator_base.py
@@ -517,18 +517,17 @@ def validate_stream(
 
         # if remainder kwargs is passed, validate remainder regardless
         remainder = kwargs.get("remainder", False)
-        print("split contents:", splitcontents)
         if remainder:
-            splitcontents = [accumulated_text, []]
+            splitcontents = [accumulated_text, ""]
         if len(splitcontents) == 0:
             return None
         [chunk_to_validate, new_accumulated_chunks] = splitcontents
         self.accumulated_chunks = [new_accumulated_chunks]
         # exclude last chunk, because it may not be a complete chunk
         validation_result = self.validate(chunk_to_validate, metadata)
-        # include the chunk that we've validated in the metadata
-        # TODO: Can I count on the validator to do this?
-        validation_result.validated_chunk = chunk_to_validate
+        # if validate doesn't set validated chunk, we set it
+        if validation_result.validated_chunk is None:
+            validation_result.validated_chunk = chunk_to_validate
         return validation_result
 
     def to_prompt(self, with_keywords: bool = True) -> str:
diff --git a/tests/integration_tests/test_streaming.py b/tests/integration_tests/test_streaming.py
index 90940207b..4c2c0af94 100644
--- a/tests/integration_tests/test_streaming.py
+++ b/tests/integration_tests/test_streaming.py
@@ -432,6 +432,8 @@ def test_string_schema_streaming_with_openai_chat(
 
     assert isinstance(generator, Iterable)
 
+    final_outcome = None
     for op, desired_result in zip(generator, expected_validated_output):
-        assert op.validation_passed == desired_result
-        print("op", op)
+        final_outcome = op
+    error_spans = guard.error_spans_in_output()
+    # TODO assert something about these error spans

From 2607423f898fab546fccc67fea4acda11c866809 Mon Sep 17 00:00:00 2001
From: Nicholas Chen <nichwch@gmail.com>
Date: Tue, 21 May 2024 18:44:41 -0400
Subject: [PATCH 32/85] attempt to use openai finish_reason field

---
 guardrails/classes/history/outputs.py     |  1 -
 guardrails/run/stream_runner.py           | 66 +++++++++++------------
 guardrails/validator_base.py              | 22 ++++++--
 guardrails/validator_service.py           |  1 -
 tests/integration_tests/test_streaming.py | 36 +++++++++----
 5 files changed, 76 insertions(+), 50 deletions(-)

diff --git a/guardrails/classes/history/outputs.py b/guardrails/classes/history/outputs.py
index db6d183ac..eb12f05c6 100644
--- a/guardrails/classes/history/outputs.py
+++ b/guardrails/classes/history/outputs.py
@@ -75,7 +75,6 @@ def failed_validations(self) -> List[ValidatorLogs]:
             ]
         )
 
-    # TODO: Is it possible the user wants the entire ValidationResult here too?
     @property
     def error_spans_in_output(self) -> List[ErrorSpan]:
         """The error spans from the LLM response.
diff --git a/guardrails/run/stream_runner.py b/guardrails/run/stream_runner.py
index 99b2c11cc..d6bd03fc9 100644
--- a/guardrails/run/stream_runner.py
+++ b/guardrails/run/stream_runner.py
@@ -156,9 +156,13 @@ def step(
         # for now, handle string and json schema differently
 
         if isinstance(output_schema, StringSchema):
+            print("stream l, stre", stream)
             for chunk in stream:
+                print("ccc l", chunk)
                 # 1. Get the text from the chunk and append to fragment
                 chunk_text = self.get_chunk_text(chunk, api)
+                finished = self.is_last_chunk(chunk, api)
+                print("finished", finished)
                 fragment += chunk_text
 
                 # 2. Parse the chunk
@@ -175,6 +179,8 @@ def step(
                     output_schema,
                     True,
                     validate_subschema=True,
+                    # if it is the last chunk, validate everything that's left
+                    remainder=finished,
                 )
                 if isinstance(validated_result, SkeletonReAsk):
                     raise ValueError(
@@ -198,40 +204,6 @@ def step(
                     validated_output=validated_result,
                     validation_passed=validated_result is not None,
                 )
-            ######################################
-            # need to validate remainder of chunks
-            ######################################
-            remainder_validation = self.validate(
-                iteration,
-                index,
-                "",
-                output_schema,
-                True,
-                validate_subschema=True,
-                remainder=True,
-            )
-            if isinstance(remainder_validation, SkeletonReAsk):
-                raise ValueError(
-                    "Received fragment schema is an invalid sub-schema "
-                    "of the expected output JSON schema."
-                )
-
-            # 4. Introspect: inspect the validated fragment for reasks
-            reasks, valid_op = self.introspect(
-                index, remainder_validation, output_schema
-            )
-            if reasks:
-                raise ValueError(
-                    "Reasks are not yet supported with streaming. Please "
-                    "remove reasks from schema or disable streaming."
-                )
-            # 5. Convert validated fragment to a pretty JSON string
-            yield ValidationOutcome(
-                #  The chunk or the whole output?
-                raw_llm_output=chunk_text,
-                validated_output=remainder_validation,
-                validation_passed=remainder_validation is not None,
-            )
         # handle non string schema
         else:
             for chunk in stream:
@@ -281,6 +253,32 @@ def step(
         iteration.outputs.validation_response = validated_fragment
         iteration.outputs.guarded_output = valid_op
 
+    def is_last_chunk(self, chunk: Any, api: Union[PromptCallableBase, None]) -> bool:
+        """Detect if chunk is final chunk"""
+        if isinstance(api, OpenAICallable):
+            if OPENAI_VERSION.startswith("0"):
+                finished = chunk["choices"][0]["finish_reason"]
+                return finished is not None
+            else:
+                finished = chunk.choices[0].finish_reason
+                return finished is not None
+        elif isinstance(api, OpenAIChatCallable):
+            if OPENAI_VERSION.startswith("0"):
+                finished = chunk["choices"][0]["finish_reason"]
+                return finished is not None
+            else:
+                finished = chunk.choices[0].finish_reason
+                return finished is not None
+        elif isinstance(api, LiteLLMCallable):
+            finished = chunk.choices[0].finish_reason
+            return finished is not None
+        else:
+            try:
+                finished = chunk.choices[0].finish_reason
+                return finished is not None
+            except (AttributeError, TypeError):
+                return False
+
     def get_chunk_text(self, chunk: Any, api: Union[PromptCallableBase, None]) -> str:
         """Get the text from a chunk."""
         chunk_text = ""
diff --git a/guardrails/validator_base.py b/guardrails/validator_base.py
index a26a98046..3e4cc131a 100644
--- a/guardrails/validator_base.py
+++ b/guardrails/validator_base.py
@@ -178,10 +178,21 @@ class Refrain:
 
 # functions to get chunks
 def split_word(chunk: str):
-    return list(map(lambda x: x + " ", chunk.split(" ")))[:-1]
+    fragments = list(map(lambda x: x + " ", chunk.split(" ")))[:-1]
+    if len(fragments) == 0:
+        return []
+    return [fragments[0], "".join(fragments[1:])]
+
 
+def split_sentence_str(chunk: str):
+    fragments = list(map(lambda x: x + " ", chunk.split(".")))[:-1]
+    print("frags", fragments)
+    if len(fragments) == 0:
+        return []
+    return [fragments[0], "".join(fragments[1:])]
 
-def split_sentence(chunk: str):
+
+def split_sentence_nltk(chunk: str):
     # using the sentence tokenizer is expensive
     # we check for a . to avoid wastefully calling the tokenizer
     if "." not in chunk:
@@ -195,7 +206,10 @@ def split_sentence(chunk: str):
 
 
 def split_paragraph(chunk: str):
-    return list(map(lambda x: x + "\n", chunk.split("\n")))[:-1]
+    fragments = list(map(lambda x: x + "\n", chunk.split("\n")))[:-1]
+    if len(fragments) == 0:
+        return []
+    return [fragments[0], "".join(fragments[1:])]
 
 
 def check_refrain_in_list(schema: List) -> bool:
@@ -489,7 +503,7 @@ def __init__(
         ), f"Validator {self.__class__.__name__} is not registered. "
 
     def chunking_function(self, chunk: str):
-        return split_sentence(chunk)
+        return split_sentence_str(chunk)
 
     def validate(self, value: Any, metadata: Dict[str, Any]) -> ValidationResult:
         """Validates a value and return a validation result."""
diff --git a/guardrails/validator_service.py b/guardrails/validator_service.py
index 86da6aabe..1d71fdbbf 100644
--- a/guardrails/validator_service.py
+++ b/guardrails/validator_service.py
@@ -179,7 +179,6 @@ def run_validators(
                 iteration, validator, value, metadata, property_path, stream, **kwargs
             )
             result = validator_logs.validation_result
-            print("result log", validator_logs)
             if isinstance(result, FailResult):
                 if not stream:
                     value = self.perform_correction(
diff --git a/tests/integration_tests/test_streaming.py b/tests/integration_tests/test_streaming.py
index 4c2c0af94..1fb8ec4e6 100644
--- a/tests/integration_tests/test_streaming.py
+++ b/tests/integration_tests/test_streaming.py
@@ -8,7 +8,6 @@
 
 import openai
 import pytest
-import nltk
 from pydantic import BaseModel, Field
 
 import guardrails as gd
@@ -48,19 +47,16 @@ def __init__(
         self._max = to_int(max)
 
     def sentence_split(self, value):
-        if "." not in value:
-            return [value]
-        sentences = nltk.sent_tokenize(value)
-        if len(sentences) == 0:
-            return [value]
-        return sentences
+        return list(map(lambda x: x + ".", value.split(".")))
 
     def validate(self, value: Union[str, List], metadata: Dict) -> ValidationResult:
         sentences = self.sentence_split(value)
         error_spans = []
         index = 0
+        print("sentences", sentences)
         for sentence in sentences:
             if len(sentence) < self._min:
+                print("error in:", sentence)
                 error_spans.append(
                     ErrorSpan(
                         start=index,
@@ -71,6 +67,7 @@ def validate(self, value: Union[str, List], metadata: Dict) -> ValidationResult:
                     )
                 )
             if len(sentence) > self._max:
+                print("error in:", sentence)
                 error_spans.append(
                     ErrorSpan(
                         start=index,
@@ -126,7 +123,12 @@ def __init__(self, choices, model):
 def mock_openai_completion_create(chunks):
     # Returns a generator
     def gen():
+        index = 0
         for chunk in chunks:
+            index = index + 1
+            finished = index == len(chunks)
+            finish_reason = "stop" if finished else None
+            print("FINISH REASON", finish_reason)
             if OPENAI_VERSION.startswith("0"):
                 yield {
                     "choices": [{"text": chunk, "finish_reason": None}],
@@ -150,7 +152,12 @@ def gen():
 def mock_openai_chat_completion_create(chunks):
     # Returns a generator
     def gen():
+        index = 0
         for chunk in chunks:
+            index = index + 1
+            finished = index == len(chunks)
+            finish_reason = "stop" if finished else None
+            print("FINISH REASON", finish_reason)
             if OPENAI_VERSION.startswith("0"):
                 yield {
                     "choices": [
@@ -358,8 +365,10 @@ def test_streaming_with_openai_chat_callable(
 
     actual_output = ""
     for op in generator:
+        print("op", op)
         actual_output = op
 
+    print("actual_output", actual_output)
     assert actual_output.raw_llm_output == json.dumps(expected_raw_output)
     assert actual_output.validated_output == expected_validated_output
 
@@ -432,8 +441,15 @@ def test_string_schema_streaming_with_openai_chat(
 
     assert isinstance(generator, Iterable)
 
-    final_outcome = None
-    for op, desired_result in zip(generator, expected_validated_output):
-        final_outcome = op
+    accumulated_output = ""
+    for op in generator:
+        accumulated_output += op.raw_llm_output
     error_spans = guard.error_spans_in_output()
+    # print spans
+    print("llmoutput", accumulated_output)
+    for error_span in error_spans:
+        print("-------span--------")
+        print("content: ", accumulated_output[error_span.start : error_span.end])
+        print("reason: ", error_span.reason)
+        print("-------span--------")
     # TODO assert something about these error spans

From da720c3b71705e38548307fe76ae637291f31f58 Mon Sep 17 00:00:00 2001
From: Nicholas Chen <nichwch@gmail.com>
Date: Tue, 21 May 2024 18:49:00 -0400
Subject: [PATCH 33/85] add comment explaining problem with using openai
 finish_message

---
 guardrails/run/stream_runner.py           | 3 ---
 tests/integration_tests/test_streaming.py | 4 ++++
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/guardrails/run/stream_runner.py b/guardrails/run/stream_runner.py
index d6bd03fc9..e23687f66 100644
--- a/guardrails/run/stream_runner.py
+++ b/guardrails/run/stream_runner.py
@@ -156,13 +156,10 @@ def step(
         # for now, handle string and json schema differently
 
         if isinstance(output_schema, StringSchema):
-            print("stream l, stre", stream)
             for chunk in stream:
-                print("ccc l", chunk)
                 # 1. Get the text from the chunk and append to fragment
                 chunk_text = self.get_chunk_text(chunk, api)
                 finished = self.is_last_chunk(chunk, api)
-                print("finished", finished)
                 fragment += chunk_text
 
                 # 2. Parse the chunk
diff --git a/tests/integration_tests/test_streaming.py b/tests/integration_tests/test_streaming.py
index 1fb8ec4e6..7c79bbd51 100644
--- a/tests/integration_tests/test_streaming.py
+++ b/tests/integration_tests/test_streaming.py
@@ -131,6 +131,7 @@ def gen():
             print("FINISH REASON", finish_reason)
             if OPENAI_VERSION.startswith("0"):
                 yield {
+                    # TODO: for some reason using finish_reason here breaks everything
                     "choices": [{"text": chunk, "finish_reason": None}],
                     "model": "OpenAI model name",
                 }
@@ -140,6 +141,7 @@ def gen():
                         Choice(
                             text=chunk,
                             delta=Delta(content=""),
+                            # TODO: for some reason using finish_reason here breaks everything
                             finish_reason=None,
                         )
                     ],
@@ -164,6 +166,7 @@ def gen():
                         {
                             "index": 0,
                             "delta": {"content": chunk},
+                            # TODO: for some reason using finish_reason here breaks everything
                             "finish_reason": None,
                         }
                     ]
@@ -174,6 +177,7 @@ def gen():
                         Choice(
                             text="",
                             delta=Delta(content=chunk),
+                            # TODO: for some reason using finish_reason here breaks everything
                             finish_reason=None,
                         )
                     ],

From 8bdb292316e3f5bc2f5f808d69a9c093543adf6f Mon Sep 17 00:00:00 2001
From: Nicholas Chen <nichwch@gmail.com>
Date: Wed, 22 May 2024 11:24:11 -0400
Subject: [PATCH 34/85] test error span behavior

---
 guardrails/classes/history/outputs.py     |  4 +-
 guardrails/validator_base.py              | 21 ++--------
 tests/integration_tests/test_streaming.py | 48 ++++++++++++-----------
 3 files changed, 31 insertions(+), 42 deletions(-)

diff --git a/guardrails/classes/history/outputs.py b/guardrails/classes/history/outputs.py
index eb12f05c6..d073cbb20 100644
--- a/guardrails/classes/history/outputs.py
+++ b/guardrails/classes/history/outputs.py
@@ -1,5 +1,4 @@
 from typing import Dict, List, Optional, Sequence, Union
-
 from pydantic import Field
 from typing_extensions import deprecated
 
@@ -92,7 +91,8 @@ def error_spans_in_output(self) -> List[ErrorSpan]:
                             reason=error_span.reason,
                         )
                     )
-            total_len += len(log.value_before_validation)
+            if result.validated_chunk is not None:
+                total_len += len(result.validated_chunk)
         return spans_in_output
 
     @property
diff --git a/guardrails/validator_base.py b/guardrails/validator_base.py
index 3e4cc131a..185f1bc30 100644
--- a/guardrails/validator_base.py
+++ b/guardrails/validator_base.py
@@ -177,19 +177,13 @@ class Refrain:
 
 
 # functions to get chunks
-def split_word(chunk: str):
-    fragments = list(map(lambda x: x + " ", chunk.split(" ")))[:-1]
-    if len(fragments) == 0:
-        return []
-    return [fragments[0], "".join(fragments[1:])]
 
 
 def split_sentence_str(chunk: str):
-    fragments = list(map(lambda x: x + " ", chunk.split(".")))[:-1]
-    print("frags", fragments)
-    if len(fragments) == 0:
+    if "." not in chunk:
         return []
-    return [fragments[0], "".join(fragments[1:])]
+    fragments = chunk.split(".")
+    return [fragments[0] + ".", ".".join(fragments[1:])]
 
 
 def split_sentence_nltk(chunk: str):
@@ -205,13 +199,6 @@ def split_sentence_nltk(chunk: str):
     return [sentences[0], "".join(sentences[1:])]
 
 
-def split_paragraph(chunk: str):
-    fragments = list(map(lambda x: x + "\n", chunk.split("\n")))[:-1]
-    if len(fragments) == 0:
-        return []
-    return [fragments[0], "".join(fragments[1:])]
-
-
 def check_refrain_in_list(schema: List) -> bool:
     """Checks if a Refrain object exists in a list.
 
@@ -523,7 +510,7 @@ def validate_stream(
 
         Otherwise, the validator will validate the chunk and return the result.
         """
-        # combine accumulated chunks and new chunk
+        # combine accumulated chunks and new [:-1]chunk
         self.accumulated_chunks.append(chunk)
         accumulated_text = "".join(self.accumulated_chunks)
         # check if enough chunks have accumulated for validation
diff --git a/tests/integration_tests/test_streaming.py b/tests/integration_tests/test_streaming.py
index 7c79bbd51..cdcc6ab23 100644
--- a/tests/integration_tests/test_streaming.py
+++ b/tests/integration_tests/test_streaming.py
@@ -47,16 +47,14 @@ def __init__(
         self._max = to_int(max)
 
     def sentence_split(self, value):
-        return list(map(lambda x: x + ".", value.split(".")))
+        return list(map(lambda x: x + ".", value.split(".")[:-1]))
 
     def validate(self, value: Union[str, List], metadata: Dict) -> ValidationResult:
         sentences = self.sentence_split(value)
         error_spans = []
         index = 0
-        print("sentences", sentences)
         for sentence in sentences:
             if len(sentence) < self._min:
-                print("error in:", sentence)
                 error_spans.append(
                     ErrorSpan(
                         start=index,
@@ -67,7 +65,6 @@ def validate(self, value: Union[str, List], metadata: Dict) -> ValidationResult:
                     )
                 )
             if len(sentence) > self._max:
-                print("error in:", sentence)
                 error_spans.append(
                     ErrorSpan(
                         start=index,
@@ -77,9 +74,10 @@ def validate(self, value: Union[str, List], metadata: Dict) -> ValidationResult:
                         f"that is shorter than {self._max} characters.",
                     )
                 )
+            index = index + len(sentence)
         if len(error_spans) > 0:
             return FailResult(
-                validated_chunk="".join(self.accumulated_chunks),
+                validated_chunk=value,
                 error_spans=error_spans,
                 error_message=f"Sentence has length less than {self._min}. "
                 f"Please return a longer output, "
@@ -128,7 +126,7 @@ def gen():
             index = index + 1
             finished = index == len(chunks)
             finish_reason = "stop" if finished else None
-            print("FINISH REASON", finish_reason)
+            # print("FINISH REASON", finish_reason)
             if OPENAI_VERSION.startswith("0"):
                 yield {
                     # TODO: for some reason using finish_reason here breaks everything
@@ -159,7 +157,7 @@ def gen():
             index = index + 1
             finished = index == len(chunks)
             finish_reason = "stop" if finished else None
-            print("FINISH REASON", finish_reason)
+            # print("FINISH REASON", finish_reason)
             if OPENAI_VERSION.startswith("0"):
                 yield {
                     "choices": [
@@ -369,10 +367,8 @@ def test_streaming_with_openai_chat_callable(
 
     actual_output = ""
     for op in generator:
-        print("op", op)
         actual_output = op
 
-    print("actual_output", actual_output)
     assert actual_output.raw_llm_output == json.dumps(expected_raw_output)
     assert actual_output.validated_output == expected_validated_output
 
@@ -393,7 +389,7 @@ def test_streaming_with_openai_chat_callable(
 
 
 @pytest.mark.parametrize(
-    "guard, expected_validated_output",
+    "guard, expected_error_spans",
     [
         (
             gd.Guard.from_string(
@@ -403,16 +399,23 @@ def test_streaming_with_openai_chat_callable(
                 ],
                 prompt=STR_PROMPT,
             ),
-            # For now these should be correct.
-            # This will be different pending validation outcome
-            # schema changes.
-            [True, False, True, True, False, False],
+            # each value is a tuple
+            # first is expected text inside span
+            # second is the reason for failure
+            [
+                [
+                    "This sentence is simply just too long.",
+                    "Sentence has length greater than 30. Please return a shorter output, that is shorter than 30 characters.",
+                ],
+                [
+                    "This sentence is 2 short.",
+                    "Sentence has length less than 26. Please return a longer output, that is shorter than 30 characters.",
+                ],
+            ],
         )
     ],
 )
-def test_string_schema_streaming_with_openai_chat(
-    mocker, guard, expected_validated_output
-):
+def test_string_schema_streaming_with_openai_chat(mocker, guard, expected_error_spans):
     """Test string schema streaming with OpenAIChatCallable.
 
     Mocks openai.ChatCompletion.create.
@@ -449,11 +452,10 @@ def test_string_schema_streaming_with_openai_chat(
     for op in generator:
         accumulated_output += op.raw_llm_output
     error_spans = guard.error_spans_in_output()
+
     # print spans
-    print("llmoutput", accumulated_output)
-    for error_span in error_spans:
-        print("-------span--------")
-        print("content: ", accumulated_output[error_span.start : error_span.end])
-        print("reason: ", error_span.reason)
-        print("-------span--------")
+    assert len(error_spans) == len(expected_error_spans)
+    for error_span, expected in zip(error_spans, expected_error_spans):
+        assert accumulated_output[error_span.start : error_span.end] == expected[0]
+        assert error_span.reason == expected[1]
     # TODO assert something about these error spans

From 37b7014be0078b78dee5de58407e459bac6c3ffa Mon Sep 17 00:00:00 2001
From: Wyatt Lansford <22553069+wylansford@users.noreply.github.com>
Date: Thu, 23 May 2024 17:19:33 -0700
Subject: [PATCH 35/85] fixes, async_step is not iterating correctly

---
 guardrails/async_guard.py             | 178 +++++++----
 guardrails/llm_providers.py           |   9 +
 guardrails/run/__init__.py            |   2 +
 guardrails/run/async_runner.py        |   1 -
 guardrails/run/async_stream_runner.py | 420 ++++++++++++++++++++++++++
 5 files changed, 543 insertions(+), 67 deletions(-)
 create mode 100644 guardrails/run/async_stream_runner.py

diff --git a/guardrails/async_guard.py b/guardrails/async_guard.py
index 77d5e5ccf..6b35e4df1 100644
--- a/guardrails/async_guard.py
+++ b/guardrails/async_guard.py
@@ -1,11 +1,11 @@
 import contextvars
-import inspect
 from typing import (
     Any,
     Awaitable,
     Callable,
     Dict,
     Iterable,
+    AsyncIterable,
     List,
     Optional,
     Union,
@@ -17,7 +17,7 @@
 from guardrails.classes.history.call_inputs import CallInputs
 from guardrails.llm_providers import get_async_llm_ask, model_is_supported_server_side
 from guardrails.logger import set_scope
-from guardrails.run import AsyncRunner
+from guardrails.run import AsyncRunner, AsyncStreamRunner
 from guardrails.stores.context import set_call_kwargs, set_tracer, set_tracer_context
 
 
@@ -159,13 +159,13 @@ async def __call(
 
             # If the LLM API is not async, fail
             # FIXME: it seems like this check isn't actually working?
-            if not inspect.isawaitable(llm_api) and not inspect.iscoroutinefunction(
-                llm_api
-            ):
-                raise RuntimeError(
-                    f"The LLM API `{llm_api.__name__}` is not a coroutine. "
-                    "Please use an async LLM API."
-                )
+            # if not inspect.isawaitable(llm_api) and not inspect.iscoroutinefunction(
+            #     llm_api
+            # ):
+            #     raise RuntimeError(
+            #         f"The LLM API `{llm_api.__name__}` is not a coroutine. "
+            #         "Please use an async LLM API."
+            #     )
             # Otherwise, call the LLM
             return await self._call_async(
                 llm_api,
@@ -210,7 +210,7 @@ async def _call_async(
         call_log: Call,
         *args,
         **kwargs,
-    ) -> ValidationOutcome[OT]:
+    ) -> Union[ValidationOutcome[OT], AsyncIterable[ValidationOutcome[OT]]]:
         """Call the LLM asynchronously and validate the output.
 
         Args:
@@ -238,24 +238,48 @@ async def _call_async(
                     "You must provide a prompt if msg_history is empty. "
                     "Alternatively, you can provide a prompt in the RAIL spec."
                 )
-
-        runner = AsyncRunner(
-            instructions=instructions_obj,
-            prompt=prompt_obj,
-            msg_history=msg_history_obj,
-            api=get_async_llm_ask(llm_api, *args, **kwargs),
-            prompt_schema=self.rail.prompt_schema,
-            instructions_schema=self.rail.instructions_schema,
-            msg_history_schema=self.rail.msg_history_schema,
-            output_schema=self.rail.output_schema,
-            num_reasks=num_reasks,
-            metadata=metadata,
-            base_model=self.base_model,
-            full_schema_reask=full_schema_reask,
-            disable_tracer=self._disable_tracer,
-        )
-        call = await runner.async_run(call_log=call_log, prompt_params=prompt_params)
-        return ValidationOutcome[OT].from_guard_history(call)
+        if kwargs.get("stream", False):
+            runner = AsyncStreamRunner(
+                instructions=instructions_obj,
+                prompt=prompt_obj,
+                msg_history=msg_history_obj,
+                api=get_async_llm_ask(llm_api, *args, **kwargs),
+                prompt_schema=self.rail.prompt_schema,
+                instructions_schema=self.rail.instructions_schema,
+                msg_history_schema=self.rail.msg_history_schema,
+                output_schema=self.rail.output_schema,
+                num_reasks=num_reasks,
+                metadata=metadata,
+                base_model=self.base_model,
+                full_schema_reask=full_schema_reask,
+                disable_tracer=self._disable_tracer,
+            )
+            # Here we have an async generator
+            async_generator = runner.async_run(
+                call_log=call_log, prompt_params=prompt_params
+            )
+            return async_generator
+
+        else:
+            runner = AsyncRunner(
+                instructions=instructions_obj,
+                prompt=prompt_obj,
+                msg_history=msg_history_obj,
+                api=get_async_llm_ask(llm_api, *args, **kwargs),
+                prompt_schema=self.rail.prompt_schema,
+                instructions_schema=self.rail.instructions_schema,
+                msg_history_schema=self.rail.msg_history_schema,
+                output_schema=self.rail.output_schema,
+                num_reasks=num_reasks,
+                metadata=metadata,
+                base_model=self.base_model,
+                full_schema_reask=full_schema_reask,
+                disable_tracer=self._disable_tracer,
+            )
+            call = await runner.async_run(
+                call_log=call_log, prompt_params=prompt_params
+            )
+            return ValidationOutcome[OT].from_guard_history(call)
 
     async def parse(
         self,
@@ -368,29 +392,30 @@ async def __parse(
 
             # FIXME: checking not llm_api because it can still fall back on defaults and
             # function as expected. We should handle this better.
-            if (
-                not llm_api
-                or inspect.iscoroutinefunction(llm_api)
-                or inspect.isasyncgenfunction(llm_api)
-            ):
-                return await self._async_parse(
-                    llm_output,
-                    metadata,
-                    llm_api=llm_api,
-                    num_reasks=self.num_reasks,
-                    prompt_params=prompt_params,
-                    full_schema_reask=full_schema_reask,
-                    call_log=call_log,
-                    *args,
-                    **kwargs,
-                )
+            # if (
+            #     not llm_api
+            #     or inspect.iscoroutinefunction(llm_api)
+            #     or inspect.isasyncgenfunction(llm_api)
+            # ):
+            print("calling async parse...\n\n\n")
+            return await self._async_parse(
+                llm_output,
+                metadata,
+                llm_api=llm_api,
+                num_reasks=self.num_reasks,
+                prompt_params=prompt_params,
+                full_schema_reask=full_schema_reask,
+                call_log=call_log,
+                *args,
+                **kwargs,
+            )
 
-            else:
-                raise NotImplementedError(
-                    "AsyncGuard does not support non-async LLM APIs. "
-                    "Please use the synchronous API Guard or supply an asynchronous "
-                    "LLM API."
-                )
+            # else:
+            #     raise NotImplementedError(
+            #         "AsyncGuard does not support non-async LLM APIs. "
+            #         "Please use the synchronous API Guard or supply an asynchronous "
+            #         "LLM API."
+            #     )
 
         guard_context = contextvars.Context()
         return await guard_context.run(
@@ -428,22 +453,43 @@ async def _async_parse(
         Returns:
             The validated response.
         """
-        runner = AsyncRunner(
-            instructions=kwargs.pop("instructions", None),
-            prompt=kwargs.pop("prompt", None),
-            msg_history=kwargs.pop("msg_history", None),
-            api=get_async_llm_ask(llm_api, *args, **kwargs) if llm_api else None,
-            prompt_schema=self.rail.prompt_schema,
-            instructions_schema=self.rail.instructions_schema,
-            msg_history_schema=self.rail.msg_history_schema,
-            output_schema=self.rail.output_schema,
-            num_reasks=num_reasks,
-            metadata=metadata,
-            output=llm_output,
-            base_model=self.base_model,
-            full_schema_reask=full_schema_reask,
-            disable_tracer=self._disable_tracer,
-        )
+        print("\n\n\n\n\n\n")
+        print(kwargs)
+        print("\n\n\n\n\n\n")
+        if kwargs["stream"]:
+            runner = AsyncStreamRunner(
+                instructions=kwargs.pop("instructions", None),
+                prompt=kwargs.pop("prompt", None),
+                msg_history=kwargs.pop("msg_history", None),
+                api=get_async_llm_ask(llm_api, *args, **kwargs) if llm_api else None,
+                prompt_schema=self.rail.prompt_schema,
+                instructions_schema=self.rail.instructions_schema,
+                msg_history_schema=self.rail.msg_history_schema,
+                output_schema=self.rail.output_schema,
+                num_reasks=num_reasks,
+                metadata=metadata,
+                output=llm_output,
+                base_model=self.base_model,
+                full_schema_reask=full_schema_reask,
+                disable_tracer=self._disable_tracer,
+            )
+        else:
+            runner = AsyncRunner(
+                instructions=kwargs.pop("instructions", None),
+                prompt=kwargs.pop("prompt", None),
+                msg_history=kwargs.pop("msg_history", None),
+                api=get_async_llm_ask(llm_api, *args, **kwargs) if llm_api else None,
+                prompt_schema=self.rail.prompt_schema,
+                instructions_schema=self.rail.instructions_schema,
+                msg_history_schema=self.rail.msg_history_schema,
+                output_schema=self.rail.output_schema,
+                num_reasks=num_reasks,
+                metadata=metadata,
+                output=llm_output,
+                base_model=self.base_model,
+                full_schema_reask=full_schema_reask,
+                disable_tracer=self._disable_tracer,
+            )
         call = await runner.async_run(call_log=call_log, prompt_params=prompt_params)
 
         return ValidationOutcome[OT].from_guard_history(call)
diff --git a/guardrails/llm_providers.py b/guardrails/llm_providers.py
index 9bea2d850..19e4a4dc7 100644
--- a/guardrails/llm_providers.py
+++ b/guardrails/llm_providers.py
@@ -5,6 +5,7 @@
     Callable,
     Dict,
     Iterable,
+    AsyncIterable,
     List,
     Optional,
     Type,
@@ -831,6 +832,14 @@ async def invoke_llm(
             *args,
             **kwargs,
         )
+        if kwargs.get("stream", False):
+            # If stream is defined and set to True,
+            # the callable returns a generator object
+            llm_response = cast(AsyncIterable[str], response)
+            return LLMResponse(
+                output="",
+                stream_output=llm_response,
+            )
 
         return LLMResponse(
             output=response.choices[0].message.content,  # type: ignore
diff --git a/guardrails/run/__init__.py b/guardrails/run/__init__.py
index e777106f7..9893f2124 100644
--- a/guardrails/run/__init__.py
+++ b/guardrails/run/__init__.py
@@ -1,12 +1,14 @@
 from guardrails.run.async_runner import AsyncRunner
 from guardrails.run.runner import Runner
 from guardrails.run.stream_runner import StreamRunner
+from guardrails.run.async_stream_runner import AsyncStreamRunner
 from guardrails.run.utils import msg_history_source, msg_history_string
 
 __all__ = [
     "Runner",
     "AsyncRunner",
     "StreamRunner",
+    "AsyncStreamRunner",
     "msg_history_source",
     "msg_history_string",
 ]
diff --git a/guardrails/run/async_runner.py b/guardrails/run/async_runner.py
index 95b00d9c2..c5854eaa2 100644
--- a/guardrails/run/async_runner.py
+++ b/guardrails/run/async_runner.py
@@ -281,7 +281,6 @@ async def async_call(
             llm_response = await api_fn(prompt.source)
         else:
             raise ValueError("'output', 'prompt' or 'msg_history' must be provided.")
-
         return llm_response
 
     async def async_validate(
diff --git a/guardrails/run/async_stream_runner.py b/guardrails/run/async_stream_runner.py
new file mode 100644
index 000000000..17cdb6e14
--- /dev/null
+++ b/guardrails/run/async_stream_runner.py
@@ -0,0 +1,420 @@
+import copy
+from functools import partial
+from typing import Any, AsyncGenerator, Dict, List, Optional, Tuple, Type, Union
+
+from pydantic import BaseModel
+
+from guardrails.classes.history import Call, Inputs, Iteration, Outputs
+from guardrails.classes.output_type import OT
+from guardrails.classes.validation_outcome import ValidationOutcome
+from guardrails.datatypes import verify_metadata_requirements
+from guardrails.errors import ValidationError
+from guardrails.llm_providers import AsyncPromptCallableBase, PromptCallableBase
+from guardrails.prompt import Instructions, Prompt
+from guardrails.run.runner import Runner
+from guardrails.run.utils import msg_history_source, msg_history_string
+from guardrails.schema import Schema, StringSchema
+from guardrails.utils.llm_response import LLMResponse
+from guardrails.utils.reask_utils import ReAsk, SkeletonReAsk
+from guardrails.utils.telemetry_utils import async_trace
+
+
+class AsyncStreamRunner(Runner):
+    def __init__(
+        self,
+        output_schema: Schema,
+        num_reasks: int,
+        prompt: Optional[Union[str, Prompt]] = None,
+        instructions: Optional[Union[str, Instructions]] = None,
+        msg_history: Optional[List[Dict]] = None,
+        api: Optional[AsyncPromptCallableBase] = None,
+        prompt_schema: Optional[StringSchema] = None,
+        instructions_schema: Optional[StringSchema] = None,
+        msg_history_schema: Optional[StringSchema] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+        output: Optional[str] = None,
+        base_model: Optional[
+            Union[Type[BaseModel], Type[List[Type[BaseModel]]]]
+        ] = None,
+        full_schema_reask: bool = False,
+        disable_tracer: Optional[bool] = True,
+    ):
+        super().__init__(
+            output_schema=output_schema,
+            num_reasks=num_reasks,
+            prompt=prompt,
+            instructions=instructions,
+            msg_history=msg_history,
+            api=api,
+            prompt_schema=prompt_schema,
+            instructions_schema=instructions_schema,
+            msg_history_schema=msg_history_schema,
+            metadata=metadata,
+            output=output,
+            base_model=base_model,
+            full_schema_reask=full_schema_reask,
+            disable_tracer=disable_tracer,
+        )
+        self.api: Optional[AsyncPromptCallableBase] = api
+
+    async def async_run(
+        self, call_log: Call, prompt_params: Optional[Dict] = None
+    ) -> Call:
+        if prompt_params is None:
+            prompt_params = {}
+
+        missing_keys = verify_metadata_requirements(
+            self.metadata, self.output_schema.root_datatype
+        )
+
+        if missing_keys:
+            raise ValueError(
+                f"Missing required metadata keys: {', '.join(missing_keys)}"
+            )
+
+        (
+            instructions,
+            prompt,
+            msg_history,
+            prompt_schema,
+            instructions_schema,
+            msg_history_schema,
+            output_schema,
+        ) = (
+            self.instructions,
+            self.prompt,
+            self.msg_history,
+            self.prompt_schema,
+            self.instructions_schema,
+            self.msg_history_schema,
+            self.output_schema,
+        )
+
+        result = self.async_step(
+            index=0,
+            api=self.api,
+            instructions=instructions,
+            prompt=prompt,
+            msg_history=msg_history,
+            prompt_params=prompt_params,
+            prompt_schema=prompt_schema,
+            instructions_schema=instructions_schema,
+            msg_history_schema=msg_history_schema,
+            output_schema=output_schema,
+            output=self.output,
+            call_log=call_log,
+        )
+        async for call in result:
+            yield ValidationOutcome[OT].from_guard_history(call)
+
+    # @async_trace(name="step")
+    async def async_step(
+        self,
+        index: int,
+        api: Optional[AsyncPromptCallableBase],
+        instructions: Optional[Instructions],
+        prompt: Optional[Prompt],
+        msg_history: Optional[List[Dict]],
+        prompt_params: Dict,
+        prompt_schema: Optional[StringSchema],
+        instructions_schema: Optional[StringSchema],
+        msg_history_schema: Optional[StringSchema],
+        output_schema: Schema,
+        call_log: Call,
+        output: Optional[str] = None,
+    ) -> AsyncGenerator[ValidationOutcome[OT], None]:
+        inputs = Inputs(
+            llm_api=api,
+            llm_output=output,
+            instructions=instructions,
+            prompt=prompt,
+            msg_history=msg_history,
+            prompt_params=prompt_params,
+            num_reasks=self.num_reasks,
+            metadata=self.metadata,
+            full_schema_reask=self.full_schema_reask,
+        )
+        outputs = Outputs()
+        iteration = Iteration(inputs=inputs, outputs=outputs)
+        call_log.iterations.push(iteration)
+        if output:
+            instructions = None
+            prompt = None
+            msg_history = None
+        else:
+            instructions, prompt, msg_history = await self.async_prepare(
+                call_log,
+                index,
+                instructions,
+                prompt,
+                msg_history,
+                prompt_params,
+                api,
+                prompt_schema,
+                instructions_schema,
+                msg_history_schema,
+                output_schema,
+            )
+
+        iteration.inputs.prompt = prompt
+        iteration.inputs.instructions = instructions
+        iteration.inputs.msg_history = msg_history
+
+        llm_response = await self.async_call(
+            index, instructions, prompt, msg_history, api, output
+        )
+        try:
+            stream = llm_response.completion_stream
+        except AttributeError:
+            stream = llm_response.stream_output
+        if stream is None:
+            raise ValueError(
+                "No stream was returned from the API. Please check that "
+                "the API is returning an async generator."
+            )
+
+        fragment = ""
+        parsed_fragment, validated_fragment, valid_op = None, None, None
+        verified = set()
+
+        if isinstance(output_schema, StringSchema):
+            async for chunk in stream:
+                chunk_text = self.get_chunk_text(chunk, api)
+                finished = self.is_last_chunk(chunk, api)
+                fragment += chunk_text
+
+                parsed_chunk, move_to_next = self.parse(
+                    index, chunk_text, output_schema, verified
+                )
+                if move_to_next:
+                    continue
+                validated_result = await self.async_validate(
+                    iteration,
+                    index,
+                    parsed_chunk,
+                    output_schema,
+                    True,
+                    validate_subschema=True,
+                    remainder=finished,
+                )
+                if isinstance(validated_result, SkeletonReAsk):
+                    raise ValueError(
+                        "Received fragment schema is an invalid sub-schema "
+                        "of the expected output JSON schema."
+                    )
+
+                reasks, valid_op = await self.introspect(
+                    index, validated_result, output_schema
+                )
+                if reasks:
+                    raise ValueError(
+                        "Reasks are not yet supported with streaming. Please "
+                        "remove reasks from schema or disable streaming."
+                    )
+
+                yield ValidationOutcome(
+                    raw_llm_output=chunk_text,
+                    validated_output=validated_result,
+                    validation_passed=validated_result is not None,
+                )
+        else:
+            async for chunk in stream:
+                chunk_text = self.get_chunk_text(chunk, api)
+                fragment += chunk_text
+
+                parsed_fragment, move_to_next = self.parse(
+                    index, fragment, output_schema, verified
+                )
+                if move_to_next:
+                    continue
+                validated_fragment = await self.async_validate(
+                    iteration,
+                    index,
+                    parsed_fragment,
+                    output_schema,
+                    validate_subschema=True,
+                )
+                if isinstance(validated_fragment, SkeletonReAsk):
+                    raise ValueError(
+                        "Received fragment schema is an invalid sub-schema "
+                        "of the expected output JSON schema."
+                    )
+
+                reasks, valid_op = await self.introspect(
+                    index, validated_fragment, output_schema
+                )
+                if reasks:
+                    raise ValueError(
+                        "Reasks are not yet supported with streaming. Please "
+                        "remove reasks from schema or disable streaming."
+                    )
+
+                yield ValidationOutcome(
+                    raw_llm_output=fragment,
+                    validated_output=validated_fragment,
+                    validation_passed=validated_fragment is not None,
+                )
+
+        iteration.outputs.raw_output = fragment
+        iteration.outputs.parsed_output = parsed_fragment
+        iteration.outputs.validation_response = validated_fragment
+        iteration.outputs.guarded_output = valid_op
+
+    @async_trace(name="call")
+    async def async_call(
+        self,
+        index: int,
+        instructions: Optional[Instructions],
+        prompt: Optional[Prompt],
+        msg_history: Optional[List[Dict]],
+        api: Optional[AsyncPromptCallableBase],
+        output: Optional[str] = None,
+    ) -> LLMResponse:
+        api_fn = api
+        if api is not None:
+            supports_base_model = getattr(api, "supports_base_model", False)
+            if supports_base_model:
+                api_fn = partial(api, base_model=self.base_model)
+        if output is not None:
+            llm_response = LLMResponse(
+                output=output,
+            )
+        elif api_fn is None:
+            raise ValueError("Either API or output must be provided.")
+        elif msg_history:
+            llm_response = await api_fn(msg_history=msg_history_source(msg_history))
+        elif prompt and instructions:
+            llm_response = await api_fn(prompt.source, instructions=instructions.source)
+        elif prompt:
+            llm_response = await api_fn(prompt.source)
+        else:
+            raise ValueError("'output', 'prompt' or 'msg_history' must be provided.")
+        return llm_response
+
+    async def async_validate(
+        self,
+        iteration: Iteration,
+        index: int,
+        parsed_output: Any,
+        output_schema: Schema,
+        validate_subschema: bool = False,
+    ):
+        if validate_subschema:
+            validated_output = await output_schema.async_validate_subschema(
+                iteration, parsed_output, self.metadata, attempt_number=index
+            )
+        else:
+            validated_output = await output_schema.async_validate(
+                iteration, parsed_output, self.metadata, attempt_number=index
+            )
+
+        return validated_output
+
+    async def introspect(
+        self,
+        index: int,
+        validated_output: Any,
+        output_schema: Schema,
+    ) -> Tuple[List[ReAsk], Any]:
+        # Introspect: inspect validated output for reasks.
+        reasks, valid_output = await output_schema.async_introspect(
+            validated_output, self.metadata, attempt_number=index + 1
+        )
+        return reasks, valid_output
+
+    async def async_prepare(
+        self,
+        call_log: Call,
+        index: int,
+        instructions: Optional[Instructions],
+        prompt: Optional[Prompt],
+        msg_history: Optional[List[Dict]],
+        prompt_params: Dict,
+        api: Optional[Union[PromptCallableBase, AsyncPromptCallableBase]],
+        prompt_schema: Optional[StringSchema],
+        instructions_schema: Optional[StringSchema],
+        msg_history_schema: Optional[StringSchema],
+        output_schema: Schema,
+    ) -> Tuple[Optional[Instructions], Optional[Prompt], Optional[List[Dict]]]:
+        if api is None:
+            raise ValueError("API must be provided.")
+
+        if prompt_params is None:
+            prompt_params = {}
+
+        if msg_history:
+            msg_history = copy.deepcopy(msg_history)
+            for msg in msg_history:
+                msg["content"] = msg["content"].format(**prompt_params)
+
+            prompt, instructions = None, None
+
+            if msg_history_schema is not None:
+                msg_str = msg_history_string(msg_history)
+                inputs = Inputs(
+                    llm_output=msg_str,
+                )
+                iteration = Iteration(inputs=inputs)
+                call_log.iterations.insert(0, iteration)
+                validated_msg_history = await msg_history_schema.async_validate(
+                    iteration, msg_str, self.metadata
+                )
+                if isinstance(validated_msg_history, ReAsk):
+                    raise ValidationError(
+                        f"Message history validation failed: "
+                        f"{validated_msg_history}"
+                    )
+                if validated_msg_history != msg_str:
+                    raise ValidationError("Message history validation failed")
+        elif prompt is not None:
+            if isinstance(prompt, str):
+                prompt = Prompt(prompt)
+
+            prompt = prompt.format(**prompt_params)
+
+            if instructions is not None and isinstance(instructions, Instructions):
+                instructions = instructions.format(**prompt_params)
+
+            instructions, prompt = output_schema.preprocess_prompt(
+                api, instructions, prompt
+            )
+
+            if prompt_schema is not None and prompt is not None:
+                inputs = Inputs(
+                    llm_output=prompt.source,
+                )
+                iteration = Iteration(inputs=inputs)
+                call_log.iterations.insert(0, iteration)
+                validated_prompt = await prompt_schema.async_validate(
+                    iteration, prompt.source, self.metadata
+                )
+                iteration.outputs.validation_response = validated_prompt
+                if validated_prompt is None:
+                    raise ValidationError("Prompt validation failed")
+                if isinstance(validated_prompt, ReAsk):
+                    raise ValidationError(
+                        f"Prompt validation failed: {validated_prompt}"
+                    )
+                prompt = Prompt(validated_prompt)
+
+            if instructions_schema is not None and instructions is not None:
+                inputs = Inputs(
+                    llm_output=instructions.source,
+                )
+                iteration = Iteration(inputs=inputs)
+                call_log.iterations.insert(0, iteration)
+                validated_instructions = await instructions_schema.async_validate(
+                    iteration, instructions.source, self.metadata
+                )
+                iteration.outputs.validation_response = validated_instructions
+                if validated_instructions is None:
+                    raise ValidationError("Instructions validation failed")
+                if isinstance(validated_instructions, ReAsk):
+                    raise ValidationError(
+                        f"Instructions validation failed: {validated_instructions}"
+                    )
+                instructions = Instructions(validated_instructions)
+        else:
+            raise ValueError("Prompt or message history must be provided.")
+
+        return instructions, prompt, msg_history

From dedfcfc10fb07a7044576303421b313b7f2b199f Mon Sep 17 00:00:00 2001
From: Wyatt Lansford <22553069+wylansford@users.noreply.github.com>
Date: Fri, 24 May 2024 16:09:20 -0700
Subject: [PATCH 36/85] updated async streamrunner to work. Still need to fix
 post introspect call

---
 guardrails/async_guard.py             |   7 +-
 guardrails/llm_providers.py           |   5 +-
 guardrails/run/async_stream_runner.py | 123 ++++++++++++++++++++++----
 guardrails/utils/llm_response.py      |   3 +-
 4 files changed, 115 insertions(+), 23 deletions(-)

diff --git a/guardrails/async_guard.py b/guardrails/async_guard.py
index 6b35e4df1..fd979097c 100644
--- a/guardrails/async_guard.py
+++ b/guardrails/async_guard.py
@@ -19,6 +19,7 @@
 from guardrails.logger import set_scope
 from guardrails.run import AsyncRunner, AsyncStreamRunner
 from guardrails.stores.context import set_call_kwargs, set_tracer, set_tracer_context
+import inspect
 
 
 class AsyncGuard(Guard):
@@ -167,7 +168,7 @@ async def __call(
             #         "Please use an async LLM API."
             #     )
             # Otherwise, call the LLM
-            return await self._call_async(
+            result = self._call_async(
                 llm_api,
                 prompt_params=prompt_params,
                 num_reasks=self.num_reasks,
@@ -181,6 +182,10 @@ async def __call(
                 **kwargs,
             )
 
+            if inspect.isawaitable(result):
+                result = await result
+            return result
+
         guard_context = contextvars.Context()
         return await guard_context.run(
             __call,
diff --git a/guardrails/llm_providers.py b/guardrails/llm_providers.py
index 19e4a4dc7..34e2a423c 100644
--- a/guardrails/llm_providers.py
+++ b/guardrails/llm_providers.py
@@ -5,7 +5,6 @@
     Callable,
     Dict,
     Iterable,
-    AsyncIterable,
     List,
     Optional,
     Type,
@@ -835,10 +834,10 @@ async def invoke_llm(
         if kwargs.get("stream", False):
             # If stream is defined and set to True,
             # the callable returns a generator object
-            llm_response = cast(AsyncIterable[str], response)
+            # response = cast(AsyncIterable[str], response)
             return LLMResponse(
                 output="",
-                stream_output=llm_response,
+                async_stream_output=response.completion_stream,
             )
 
         return LLMResponse(
diff --git a/guardrails/run/async_stream_runner.py b/guardrails/run/async_stream_runner.py
index 17cdb6e14..88ccb1fc8 100644
--- a/guardrails/run/async_stream_runner.py
+++ b/guardrails/run/async_stream_runner.py
@@ -9,17 +9,25 @@
 from guardrails.classes.validation_outcome import ValidationOutcome
 from guardrails.datatypes import verify_metadata_requirements
 from guardrails.errors import ValidationError
-from guardrails.llm_providers import AsyncPromptCallableBase, PromptCallableBase
+from guardrails.llm_providers import (
+    AsyncLiteLLMCallable,
+    AsyncPromptCallableBase,
+    LiteLLMCallable,
+    OpenAICallable,
+    OpenAIChatCallable,
+    PromptCallableBase,
+)
 from guardrails.prompt import Instructions, Prompt
-from guardrails.run.runner import Runner
+from guardrails.run import StreamRunner
 from guardrails.run.utils import msg_history_source, msg_history_string
 from guardrails.schema import Schema, StringSchema
 from guardrails.utils.llm_response import LLMResponse
+from guardrails.utils.openai_utils import OPENAI_VERSION
 from guardrails.utils.reask_utils import ReAsk, SkeletonReAsk
 from guardrails.utils.telemetry_utils import async_trace
 
 
-class AsyncStreamRunner(Runner):
+class AsyncStreamRunner(StreamRunner):
     def __init__(
         self,
         output_schema: Schema,
@@ -164,9 +172,13 @@ async def async_step(
             index, instructions, prompt, msg_history, api, output
         )
         try:
-            stream = llm_response.completion_stream
+            stream = llm_response.async_stream_output
         except AttributeError:
-            stream = llm_response.stream_output
+            try:
+                stream = llm_response.stream_output
+
+            except AttributeError:
+                stream = llm_response.stream_output
         if stream is None:
             raise ValueError(
                 "No stream was returned from the API. Please check that "
@@ -193,10 +205,9 @@ async def async_step(
                     index,
                     parsed_chunk,
                     output_schema,
-                    True,
                     validate_subschema=True,
-                    remainder=finished,
                 )
+
                 if isinstance(validated_result, SkeletonReAsk):
                     raise ValueError(
                         "Received fragment schema is an invalid sub-schema "
@@ -299,14 +310,15 @@ async def async_validate(
         output_schema: Schema,
         validate_subschema: bool = False,
     ):
-        if validate_subschema:
-            validated_output = await output_schema.async_validate_subschema(
-                iteration, parsed_output, self.metadata, attempt_number=index
-            )
-        else:
-            validated_output = await output_schema.async_validate(
-                iteration, parsed_output, self.metadata, attempt_number=index
-            )
+        # if validate_subschema:
+        #     validated_output = await output_schema.async_validate_subschema(
+        #         iteration, parsed_output, self.metadata, attempt_number=index
+        #     )
+        # else:
+        # TODO: What is supposed to happen here with subschema?
+        validated_output = await output_schema.async_validate(
+            iteration, parsed_output, self.metadata, attempt_number=index
+        )
 
         return validated_output
 
@@ -317,9 +329,10 @@ async def introspect(
         output_schema: Schema,
     ) -> Tuple[List[ReAsk], Any]:
         # Introspect: inspect validated output for reasks.
-        reasks, valid_output = await output_schema.async_introspect(
-            validated_output, self.metadata, attempt_number=index + 1
-        )
+        if validated_output is None:
+            return [], None
+        reasks, valid_output = output_schema.introspect(validated_output)
+
         return reasks, valid_output
 
     async def async_prepare(
@@ -418,3 +431,77 @@ async def async_prepare(
             raise ValueError("Prompt or message history must be provided.")
 
         return instructions, prompt, msg_history
+
+    def get_chunk_text(self, chunk: Any, api: Union[PromptCallableBase, None]) -> str:
+        """Get the text from a chunk."""
+        chunk_text = ""
+        if isinstance(api, OpenAICallable):
+            if OPENAI_VERSION.startswith("0"):
+                finished = chunk["choices"][0]["finish_reason"]
+                if "text" in chunk["choices"][0]:
+                    content = chunk["choices"][0]["text"]
+                    if not finished and content:
+                        chunk_text = content
+            else:
+                finished = chunk.choices[0].finish_reason
+                content = chunk.choices[0].text
+                if not finished and content:
+                    chunk_text = content
+        elif isinstance(api, OpenAIChatCallable):
+            if OPENAI_VERSION.startswith("0"):
+                finished = chunk["choices"][0]["finish_reason"]
+                if "content" in chunk["choices"][0]["delta"]:
+                    content = chunk["choices"][0]["delta"]["content"]
+                    if not finished and content:
+                        chunk_text = content
+            else:
+                finished = chunk.choices[0].finish_reason
+                content = chunk.choices[0].delta.content
+                if not finished and content:
+                    chunk_text = content
+        elif isinstance(api, LiteLLMCallable):
+            finished = chunk.choices[0].finish_reason
+            content = chunk.choices[0].delta.content
+            if not finished and content:
+                chunk_text = content
+        elif isinstance(api, AsyncLiteLLMCallable):
+            finished = chunk.choices[0].finish_reason
+            content = chunk.choices[0].delta.content
+            if not finished and content:
+                chunk_text = content
+        else:
+            try:
+                chunk_text = chunk
+            except Exception as e:
+                raise ValueError(
+                    f"Error getting chunk from stream: {e}. "
+                    "Non-OpenAI API callables expected to return "
+                    "a generator of strings."
+                ) from e
+        return chunk_text
+
+    def is_last_chunk(self, chunk: Any, api: Union[PromptCallableBase, None]) -> bool:
+        """Detect if chunk is final chunk"""
+        if isinstance(api, OpenAICallable):
+            if OPENAI_VERSION.startswith("0"):
+                finished = chunk["choices"][0]["finish_reason"]
+                return finished is not None
+            else:
+                finished = chunk.choices[0].finish_reason
+                return finished is not None
+        elif isinstance(api, OpenAIChatCallable):
+            if OPENAI_VERSION.startswith("0"):
+                finished = chunk["choices"][0]["finish_reason"]
+                return finished is not None
+            else:
+                finished = chunk.choices[0].finish_reason
+                return finished is not None
+        elif isinstance(api, LiteLLMCallable):
+            finished = chunk.choices[0].finish_reason
+            return finished is not None
+        else:
+            try:
+                finished = chunk.choices[0].finish_reason
+                return finished is not None
+            except (AttributeError, TypeError):
+                return False
diff --git a/guardrails/utils/llm_response.py b/guardrails/utils/llm_response.py
index f0248fe0d..b797f5b5a 100644
--- a/guardrails/utils/llm_response.py
+++ b/guardrails/utils/llm_response.py
@@ -1,4 +1,4 @@
-from typing import Iterable, Optional
+from typing import Iterable, Optional, AsyncIterable
 
 from guardrails.utils.pydantic_utils import ArbitraryModel
 
@@ -8,3 +8,4 @@ class LLMResponse(ArbitraryModel):
     response_token_count: Optional[int] = None
     output: str
     stream_output: Optional[Iterable] = None
+    async_stream_output: Optional[AsyncIterable] = None

From 0abac83766fa032bf983149bdc5a4d98fb429626 Mon Sep 17 00:00:00 2001
From: Nicholas Chen <nichwch@gmail.com>
Date: Tue, 28 May 2024 18:03:38 -0400
Subject: [PATCH 37/85] address some changes

---
 guardrails/classes/history/outputs.py | 15 ++++++++-------
 guardrails/guard.py                   |  3 +--
 guardrails/validator_base.py          |  2 +-
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/guardrails/classes/history/outputs.py b/guardrails/classes/history/outputs.py
index d073cbb20..0085d00c2 100644
--- a/guardrails/classes/history/outputs.py
+++ b/guardrails/classes/history/outputs.py
@@ -83,14 +83,15 @@ def error_spans_in_output(self) -> List[ErrorSpan]:
         for log in self.validator_logs:
             result = log.validation_result
             if isinstance(result, FailResult):
-                for error_span in result.error_spans:
-                    spans_in_output.append(
-                        ErrorSpan(
-                            start=error_span.start + total_len,
-                            end=error_span.end + total_len,
-                            reason=error_span.reason,
+                if result.error_spans is not None:
+                    for error_span in result.error_spans:
+                        spans_in_output.append(
+                            ErrorSpan(
+                                start=error_span.start + total_len,
+                                end=error_span.end + total_len,
+                                reason=error_span.reason,
+                            )
                         )
-                    )
             if result.validated_chunk is not None:
                 total_len += len(result.validated_chunk)
         return spans_in_output
diff --git a/guardrails/guard.py b/guardrails/guard.py
index cf691081d..b3751895d 100644
--- a/guardrails/guard.py
+++ b/guardrails/guard.py
@@ -1132,8 +1132,7 @@ async def _async_parse(
 
     def error_spans_in_output(self):
         try:
-            history = self.history
-            call = history[0]
+            call = self.history[0]
             iter = call.iterations[0]
             llm_spans = iter.error_spans_in_output
             return llm_spans
diff --git a/guardrails/validator_base.py b/guardrails/validator_base.py
index 185f1bc30..5751b6e74 100644
--- a/guardrails/validator_base.py
+++ b/guardrails/validator_base.py
@@ -408,7 +408,7 @@ class FailResult(ValidationResult):
     error_message: str
     fix_value: Optional[Any] = None
     # segments that caused validation to fail
-    error_spans: Optional[List[ErrorSpan]] = []
+    error_spans: Optional[List[ErrorSpan]] = None
 
 
 class OnFailAction(str, Enum):

From 8f45a0ace822d8fcaedbd3d613b6edf5c7872c27 Mon Sep 17 00:00:00 2001
From: Nicholas Chen <nichwch@gmail.com>
Date: Tue, 28 May 2024 18:22:03 -0400
Subject: [PATCH 38/85] handle case where llm callable doesnt provide finished
 flag

---
 guardrails/run/stream_runner.py | 34 ++++++++++++++++++++++++++-------
 1 file changed, 27 insertions(+), 7 deletions(-)

diff --git a/guardrails/run/stream_runner.py b/guardrails/run/stream_runner.py
index e23687f66..bfe9196cb 100644
--- a/guardrails/run/stream_runner.py
+++ b/guardrails/run/stream_runner.py
@@ -156,10 +156,15 @@ def step(
         # for now, handle string and json schema differently
 
         if isinstance(output_schema, StringSchema):
+            stream_finished = False
+            last_chunk_text = ""
             for chunk in stream:
                 # 1. Get the text from the chunk and append to fragment
                 chunk_text = self.get_chunk_text(chunk, api)
+                last_chunk_text = chunk_text
                 finished = self.is_last_chunk(chunk, api)
+                if finished:
+                    stream_finished = True
                 fragment += chunk_text
 
                 # 2. Parse the chunk
@@ -169,7 +174,7 @@ def step(
                 if move_to_next:
                     # Continue to next chunk
                     continue
-                validated_result = self.validate(
+                validated_text = self.validate(
                     iteration,
                     index,
                     parsed_chunk,
@@ -179,16 +184,14 @@ def step(
                     # if it is the last chunk, validate everything that's left
                     remainder=finished,
                 )
-                if isinstance(validated_result, SkeletonReAsk):
+                if isinstance(validated_text, SkeletonReAsk):
                     raise ValueError(
                         "Received fragment schema is an invalid sub-schema "
                         "of the expected output JSON schema."
                     )
 
                 # 4. Introspect: inspect the validated fragment for reasks
-                reasks, valid_op = self.introspect(
-                    index, validated_result, output_schema
-                )
+                reasks, valid_op = self.introspect(index, validated_text, output_schema)
                 if reasks:
                     raise ValueError(
                         "Reasks are not yet supported with streaming. Please "
@@ -198,9 +201,26 @@ def step(
                 yield ValidationOutcome(
                     #  The chunk or the whole output?
                     raw_llm_output=chunk_text,
-                    validated_output=validated_result,
-                    validation_passed=validated_result is not None,
+                    validated_output=validated_text,
+                    validation_passed=validated_text is not None,
                 )
+            # handle case where generator doesn't give finished status
+            if not stream_finished:
+                last_result = self.validate(
+                    iteration,
+                    index,
+                    "",
+                    output_schema,
+                    True,
+                    validate_subschema=True,
+                    remainder=True,
+                )
+                if len(last_result) > 0:
+                    yield ValidationOutcome(
+                        raw_llm_output=last_chunk_text,
+                        validated_output=last_result,
+                        validation_passed=last_result is not None,
+                    )
         # handle non string schema
         else:
             for chunk in stream:

From fdf3c8ba9c7b9d761fe9e1d003e3c0f3ed489442 Mon Sep 17 00:00:00 2001
From: Wyatt Lansford <22553069+wylansford@users.noreply.github.com>
Date: Wed, 29 May 2024 09:14:04 -0700
Subject: [PATCH 39/85] adding latest

---
 guardrails/run/async_stream_runner.py | 76 ++++++++++++++++++++++++---
 guardrails/schema/schema.py           |  8 ++-
 guardrails/schema/string_schema.py    |  2 +
 guardrails/validator_service.py       | 23 ++++----
 4 files changed, 91 insertions(+), 18 deletions(-)

diff --git a/guardrails/run/async_stream_runner.py b/guardrails/run/async_stream_runner.py
index 88ccb1fc8..2cb612c07 100644
--- a/guardrails/run/async_stream_runner.py
+++ b/guardrails/run/async_stream_runner.py
@@ -65,6 +65,63 @@ def __init__(
         )
         self.api: Optional[AsyncPromptCallableBase] = api
 
+    def __call__(
+        self, call_log: Call, prompt_params: Optional[Dict] = None
+    ) -> AsyncGenerator[ValidationOutcome[OT], None]:
+        """Execute the StreamRunner.
+
+        Args:
+            prompt_params: Parameters to pass to the prompt in order to
+                generate the prompt string.
+
+        Returns:
+            The Call log for this run.
+        """
+        if prompt_params is None:
+            prompt_params = {}
+
+        # check if validator requirements are fulfilled
+        missing_keys = verify_metadata_requirements(
+            self.metadata, self.output_schema.root_datatype
+        )
+        if missing_keys:
+            raise ValueError(
+                f"Missing required metadata keys: {', '.join(missing_keys)}"
+            )
+
+        (
+            instructions,
+            prompt,
+            msg_history,
+            prompt_schema,
+            instructions_schema,
+            msg_history_schema,
+            output_schema,
+        ) = (
+            self.instructions,
+            self.prompt,
+            self.msg_history,
+            self.prompt_schema,
+            self.instructions_schema,
+            self.msg_history_schema,
+            self.output_schema,
+        )
+
+        return self.async_step(
+            index=0,
+            api=self.api,
+            instructions=instructions,
+            prompt=prompt,
+            msg_history=msg_history,
+            prompt_params=prompt_params,
+            prompt_schema=prompt_schema,
+            instructions_schema=instructions_schema,
+            msg_history_schema=msg_history_schema,
+            output_schema=output_schema,
+            output=self.output,
+            call_log=call_log,
+        )
+
     async def async_run(
         self, call_log: Call, prompt_params: Optional[Dict] = None
     ) -> Call:
@@ -113,7 +170,8 @@ async def async_run(
             call_log=call_log,
         )
         async for call in result:
-            yield ValidationOutcome[OT].from_guard_history(call)
+            # yield ValidationOutcome[OT].from_guard_history(call)
+            yield call
 
     # @async_trace(name="step")
     async def async_step(
@@ -172,14 +230,14 @@ async def async_step(
             index, instructions, prompt, msg_history, api, output
         )
         try:
-            stream = llm_response.async_stream_output
+            stream_output = llm_response.async_stream_output
         except AttributeError:
             try:
-                stream = llm_response.stream_output
+                stream_output = llm_response.stream_output
 
             except AttributeError:
-                stream = llm_response.stream_output
-        if stream is None:
+                stream_output = llm_response.stream_output
+        if stream_output is None:
             raise ValueError(
                 "No stream was returned from the API. Please check that "
                 "the API is returning an async generator."
@@ -190,7 +248,7 @@ async def async_step(
         verified = set()
 
         if isinstance(output_schema, StringSchema):
-            async for chunk in stream:
+            async for chunk in stream_output:
                 chunk_text = self.get_chunk_text(chunk, api)
                 finished = self.is_last_chunk(chunk, api)
                 fragment += chunk_text
@@ -206,6 +264,7 @@ async def async_step(
                     parsed_chunk,
                     output_schema,
                     validate_subschema=True,
+                    stream=True
                 )
 
                 if isinstance(validated_result, SkeletonReAsk):
@@ -229,7 +288,7 @@ async def async_step(
                     validation_passed=validated_result is not None,
                 )
         else:
-            async for chunk in stream:
+            async for chunk in stream_output:
                 chunk_text = self.get_chunk_text(chunk, api)
                 fragment += chunk_text
 
@@ -309,6 +368,7 @@ async def async_validate(
         parsed_output: Any,
         output_schema: Schema,
         validate_subschema: bool = False,
+        stream: bool = False,
     ):
         # if validate_subschema:
         #     validated_output = await output_schema.async_validate_subschema(
@@ -317,7 +377,7 @@ async def async_validate(
         # else:
         # TODO: What is supposed to happen here with subschema?
         validated_output = await output_schema.async_validate(
-            iteration, parsed_output, self.metadata, attempt_number=index
+            iteration, parsed_output, self.metadata, attempt_number=index, stream=stream
         )
 
         return validated_output
diff --git a/guardrails/schema/schema.py b/guardrails/schema/schema.py
index 7d8dad8c0..6b6eb33cf 100644
--- a/guardrails/schema/schema.py
+++ b/guardrails/schema/schema.py
@@ -87,7 +87,13 @@ def validate(
         raise NotImplementedError
 
     async def async_validate(
-        self, iteration: Iteration, data: Any, metadata: Dict, attempt_number: int = 0
+        self,
+        iteration: Iteration,
+        data: Any,
+        metadata: Dict,
+        attempt_number: int = 0,
+        stream: Optional[bool] = False,
+        **kwargs,
     ) -> Any:
         """Asynchronously validate a dictionary of data against the schema.
 
diff --git a/guardrails/schema/string_schema.py b/guardrails/schema/string_schema.py
index 200666041..db8a2b903 100644
--- a/guardrails/schema/string_schema.py
+++ b/guardrails/schema/string_schema.py
@@ -196,6 +196,7 @@ async def async_validate(
         data: Any,
         metadata: Dict,
         attempt_number: int = 0,
+        stream: Optional[bool] = False,
     ) -> Any:
         """Validate a dictionary of data against the schema.
 
@@ -225,6 +226,7 @@ async def async_validate(
             metadata=metadata,
             validator_setup=validation,
             iteration=iteration,
+            stream=stream
         )
 
         validated_response = {dummy_key: validated_response}
diff --git a/guardrails/validator_service.py b/guardrails/validator_service.py
index 1d71fdbbf..804e13884 100644
--- a/guardrails/validator_service.py
+++ b/guardrails/validator_service.py
@@ -4,7 +4,7 @@
 from concurrent.futures import ProcessPoolExecutor
 from datetime import datetime
 from typing import Any, Dict, List, Optional, Tuple, Union
-
+import inspect 
 from guardrails.classes.history import Iteration
 from guardrails.datatypes import FieldValidation
 from guardrails.errors import ValidationError
@@ -302,6 +302,7 @@ async def run_validators(
         value: Any,
         metadata: Dict,
         property_path: str,
+        stream: bool = False,
     ):
         loop = asyncio.get_running_loop()
         for on_fail, validator_group in self.group_validators(
@@ -321,12 +322,13 @@ async def run_validators(
                             value,
                             metadata,
                             property_path,
+                            stream=stream
                         )
                     )
                 else:
                     # run the validators in the current process
                     result = self.run_validator(
-                        iteration, validator, value, metadata, property_path
+                        iteration, validator, value, metadata, property_path, stream=stream
                     )
                     validators_logs.append(result)
 
@@ -398,6 +400,7 @@ async def async_validate(
         validator_setup: FieldValidation,
         iteration: Iteration,
         path: str = "$",
+        stream: bool = False,
     ) -> Tuple[Any, dict]:
         property_path = (
             f"{path}.{validator_setup.key}"
@@ -412,7 +415,7 @@ async def async_validate(
 
         # Validate the field
         value, metadata = await self.run_validators(
-            iteration, validator_setup, value, metadata, property_path
+            iteration, validator_setup, value, metadata, property_path, stream=stream
         )
 
         return value, metadata
@@ -431,7 +434,7 @@ def validate(
                 "Async event loop found, please call `validate_async` instead."
             )
         value, metadata = loop.run_until_complete(
-            self.async_validate(
+            self.x(
                 value,
                 metadata,
                 validator_setup,
@@ -459,11 +462,6 @@ def validate(
             "To run asynchronously, specify a process count"
             "greater than 1 or unset this environment variable."
         )
-    if stream:
-        sequential_validator_service = SequentialValidatorService(disable_tracer)
-        return sequential_validator_service.validate_stream(
-            value, metadata, validator_setup, iteration, **kwargs
-        )
     try:
         loop = asyncio.get_event_loop()
     except RuntimeError:
@@ -475,6 +473,11 @@ def validate(
         validator_service = AsyncValidatorService(disable_tracer)
     else:
         validator_service = SequentialValidatorService(disable_tracer)
+
+    if stream:
+        return validator_service.validate_stream(
+            value, metadata, validator_setup, iteration, **kwargs
+        )
     return validator_service.validate(
         value,
         metadata,
@@ -489,6 +492,7 @@ async def async_validate(
     validator_setup: FieldValidation,
     iteration: Iteration,
     disable_tracer: Optional[bool] = True,
+    stream: Optional[bool] = False,
 ):
     validator_service = AsyncValidatorService(disable_tracer)
     return await validator_service.async_validate(
@@ -496,4 +500,5 @@ async def async_validate(
         metadata,
         validator_setup,
         iteration,
+        stream=stream
     )

From 6798b68d40af47ab1a424eebb7d6e03059a42eb7 Mon Sep 17 00:00:00 2001
From: Caleb Courier <13314870+CalebCourier@users.noreply.github.com>
Date: Thu, 30 May 2024 12:46:14 -0500
Subject: [PATCH 40/85] API Client Updates (#688)

* remove unnecessary warning

* escape validator names in on-fail attributes

* lint fixes

* fix stubbed document stores

* add required_model_auth prop to manifest class

* add display name to model auth

* enable passing serializeable metadata to api

* lint fixes

* enable server support for litellm

* fix litellm checks

* update guardrails api client

* pass prompt, instr, msg history as kwargs to api

* api streaming support

* autoformat

* lint fix

* type fixes

* refactor guard save, enable guard.use syntax for api

* autoformat

* error handling

* remove hardcoded name

* poetry lock

* lint fix

* type fix

* Fix Notebook Runs (#793)

* Try to fix hub installs for notebooks

* remove poetry references

* install huggingface cli

* reduce to one notebook for debugging

* use bin processs

* install nltk

* show nltk

* which pip

* alter path, debug

* which python

* source in every step

* don't use poetry in script

* run all notebooks

* exit 1 on notebook failure

* remove double bar

* fix precommit

* loosen otel requirements

* use semver in checks

* fix bad merge

* fix excludes

* type ignore for python 3.8
---
 .github/workflows/ci.yml                   |  16 +-
 .github/workflows/examples_check.yml       |  55 +++-
 .github/workflows/install_from_hub.yml     |   2 +-
 .github/workflows/scripts/run_notebooks.sh |   8 +-
 Makefile                                   |   2 +-
 guardrails/api_client.py                   |  37 ++-
 guardrails/cli/server/module_manifest.py   |   8 +
 guardrails/document_store.py               |   8 +-
 guardrails/guard.py                        | 334 ++++++++++++++-------
 guardrails/llm_providers.py                |  15 +-
 guardrails/utils/api_utils.py              |  16 +
 guardrails/utils/hub_telemetry_utils.py    |   2 +-
 guardrails/utils/telemetry_utils.py        |   7 +-
 guardrails/validator_service.py            |   8 -
 guardrails/validatorsattr.py               |   5 +-
 poetry.lock                                | 242 ++++++++-------
 pyproject.toml                             |   8 +-
 tests/unit_tests/utils/test_api_utils.py   |  19 ++
 18 files changed, 516 insertions(+), 276 deletions(-)
 create mode 100644 guardrails/utils/api_utils.py
 create mode 100644 tests/unit_tests/utils/test_api_utils.py

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index f6ed06922..a9f8ca96c 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -49,8 +49,8 @@ jobs:
     strategy:
       matrix:
         python-version: ["3.8", "3.9", "3.10", "3.11"]
-        pydantic-version: ["1.10.9", "2.4.2"]
-        openai-version: ["1.30.1"]
+        pydantic-version: ["==1.10.9", ">=2.x"]
+        openai-version: [">=1.x"]
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
@@ -71,8 +71,8 @@ jobs:
         # TODO: fix errors so that we can run `make dev` instead
         run: |
           make full
-          poetry run pip install pydantic==${{ matrix.pydantic-version }}
-          poetry run pip install openai==${{ matrix.openai-version }}
+          poetry run pip install "pydantic${{ matrix.pydantic-version }}"
+          poetry run pip install "openai${{ matrix.openai-version }}"
 
       - name: Static analysis with pyright
         run: |
@@ -86,8 +86,8 @@ jobs:
         # TODO: fix errors so that we can run both `make dev` and `make full`
         # dependencies: ['dev', 'full']
         dependencies: ["full"]
-        pydantic-version: ["1.10.9", "2.4.2"]
-        openai-version: ["1.30.1"]
+        pydantic-version: ["==1.10.9", ">=2.x"]
+        openai-version: [">=1.x"]
     steps:
       - uses: actions/checkout@v4
 
@@ -113,8 +113,8 @@ jobs:
       - name: Install Dependencies
         run: |
           make ${{ matrix.dependencies }}
-          poetry run pip install pydantic==${{ matrix.pydantic-version }}
-          poetry run pip install openai==${{ matrix.openai-version }}
+          poetry run pip install "pydantic${{ matrix.pydantic-version }}"
+          poetry run pip install "openai${{ matrix.openai-version }}"
 
       - name: Run Pytests
         run: |
diff --git a/.github/workflows/examples_check.yml b/.github/workflows/examples_check.yml
index 9be015077..b330a6a36 100644
--- a/.github/workflows/examples_check.yml
+++ b/.github/workflows/examples_check.yml
@@ -15,7 +15,7 @@ jobs:
     strategy:
       matrix:
         # this line is automatically generated by the script in .github/workflows/scripts/update_notebook_matrix.sh
-        notebook: ["bug_free_python_code.ipynb","check_for_pii.ipynb","competitors_check.ipynb","extracting_entities.ipynb","generate_structured_data.ipynb","generate_structured_data_cohere.ipynb","guardrails_with_chat_models.ipynb","input_validation.ipynb","llamaindex-output-parsing.ipynb","no_secrets_in_generated_text.ipynb","provenance.ipynb","recipe_generation.ipynb","regex_validation.ipynb","response_is_on_topic.ipynb","secrets_detection.ipynb","select_choice_based_on_action.ipynb","streaming.ipynb","syntax_error_free_sql.ipynb","text_summarization_quality.ipynb","toxic_language.ipynb","translation_to_specific_language.ipynb","translation_with_quality_check.ipynb","valid_chess_moves.ipynb","value_within_distribution.ipynb"]
+        notebook: ["bug_free_python_code.ipynb","check_for_pii.ipynb","competitors_check.ipynb","extracting_entities.ipynb","generate_structured_data.ipynb","generate_structured_data_cohere.ipynb","guardrails_with_chat_models.ipynb","input_validation.ipynb","llamaindex-output-parsing.ipynb","no_secrets_in_generated_text.ipynb","provenance.ipynb","recipe_generation.ipynb","regex_validation.ipynb","response_is_on_topic.ipynb","secrets_detection.ipynb","select_choice_based_on_action.ipynb","syntax_error_free_sql.ipynb","text_summarization_quality.ipynb","toxic_language.ipynb","translation_to_specific_language.ipynb","translation_with_quality_check.ipynb","valid_chess_moves.ipynb","value_within_distribution.ipynb"]
     env:
       COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
       OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
@@ -37,26 +37,49 @@ jobs:
     #   with:
     #     path: ~/.cache/pypoetry
     #     key: poetry-cache-${{ runner.os }}-${{ steps.setup_python.outputs.python-version }}-${{ env.POETRY_VERSION }}
-    - name: Install Poetry
-      uses: snok/install-poetry@v1
-      with:
-        virtualenvs-create: true
-        virtualenvs-in-project: true
-        installer-parallel: true
+    # - name: Install Poetry
+    #   uses: snok/install-poetry@v1
+    #   with:
+    #     virtualenvs-create: true
+    #     virtualenvs-in-project: true
+    #     installer-parallel: true
     - name: Install dependencies
       run: |
-        make full;
-        poetry add "openai>=1.2.4" jupyter nbconvert cohere==5.3.2;
-    - name: Check for pypdfium2
-      run: poetry run pip show pypdfium2
+        # Legacy run script
+        # make full;
+        # poetry add "openai>=1.2.4" jupyter nbconvert cohere==5.3.2;
+        
+        # Setup Virtual Environment
+        python3 -m venv ./.venv
+        source .venv/bin/activate
+
+        # Install the current branch
+        pip install .
+
+        # Install extra stuff for notebook runs
+        pip install "huggingface_hub[cli]" jupyter nbconvert cohere==5.3.2
+        pip install nltk
+        echo "which python $(which python)"
+        echo "which pip $(which pip)"
+        echo "pip show nltk: $(pip show nltk)"
     - name: Huggingface Hub Login
-      run: poetry run huggingface-cli login --token $HUGGINGFACE_API_KEY
+      run: |
+        source .venv/bin/activate
+        echo "which python: $(which python)"
+        echo "which pip $(which pip)"
+        huggingface-cli login --token $HUGGINGFACE_API_KEY
     - name: download nltk data
       run: |
+        source .venv/bin/activate
+        echo "which pip $(which pip)"
+        echo "pip show nltk: $(pip show nltk)"
         mkdir /tmp/nltk_data;
-        poetry run python -m nltk.downloader -d /tmp/nltk_data punkt;
-    - name: Use venv
-      run: source .venv/bin/activate
+        python -m nltk.downloader -d /tmp/nltk_data punkt;
+    # - name: Use venv
+    #   run: source .venv/bin/activate
     - name: Execute notebooks and check for errors
-      run: bash ./.github/workflows/scripts/run_notebooks.sh ${{ matrix.notebook }}
+      # run: bash ./.github/workflows/scripts/run_notebooks.sh ${{ matrix.notebook }}
+      run: |
+        source .venv/bin/activate
+        bash ./.github/workflows/scripts/run_notebooks.sh ${{ matrix.notebook }}
 
diff --git a/.github/workflows/install_from_hub.yml b/.github/workflows/install_from_hub.yml
index c39cc8502..1e6483547 100644
--- a/.github/workflows/install_from_hub.yml
+++ b/.github/workflows/install_from_hub.yml
@@ -1,4 +1,4 @@
-name: Notebook Execution and Error Check
+name: Install from Hub
 
 on:
   push:
diff --git a/.github/workflows/scripts/run_notebooks.sh b/.github/workflows/scripts/run_notebooks.sh
index e32932948..cd854ceb6 100755
--- a/.github/workflows/scripts/run_notebooks.sh
+++ b/.github/workflows/scripts/run_notebooks.sh
@@ -1,6 +1,10 @@
 #!/bin/bash
 export NLTK_DATA=/tmp/nltk_data;
 
+# Remove the local guardrails directory and use the installed version
+rm -rf ./guardrails
+
+# Navigate to notebooks
 cd docs/examples
 
 # Get the notebook name from the matrix variable
@@ -10,10 +14,12 @@ notebook="$1"
 invalid_notebooks=("valid_chess_moves.ipynb" "llamaindex-output-parsing.ipynb" "competitors_check.ipynb")
 if [[ ! " ${invalid_notebooks[@]} " =~ " ${notebook} " ]]; then
   echo "Processing $notebook..."
-  poetry run jupyter nbconvert --to notebook --execute "$notebook"
+  # poetry run jupyter nbconvert --to notebook --execute "$notebook"
+  jupyter nbconvert --to notebook --execute "$notebook"
   if [ $? -ne 0 ]; then
     echo "Error found in $notebook"
     echo "Error in $notebook. See logs for details." >> errors.txt
+    exit 1
   fi
 fi
 
diff --git a/Makefile b/Makefile
index ee919a60b..59f3d0fc5 100644
--- a/Makefile
+++ b/Makefile
@@ -84,4 +84,4 @@ precommit:
 	# pytest -x -q --no-summary
 	pyright guardrails/
 	make lint
-	./github/workflows/scripts/update_notebook_matrix.sh
+	./.github/workflows/scripts/update_notebook_matrix.sh
diff --git a/guardrails/api_client.py b/guardrails/api_client.py
index bf8b7323b..55af0fdaf 100644
--- a/guardrails/api_client.py
+++ b/guardrails/api_client.py
@@ -1,9 +1,11 @@
+import json
 import os
-from typing import Optional
+from typing import Generator, Optional
 
+import requests
 from guardrails_api_client import AuthenticatedClient
 from guardrails_api_client.api.guard import update_guard, validate
-from guardrails_api_client.models import Guard, ValidatePayload
+from guardrails_api_client.models import Guard, ValidatePayload, ValidationOutput
 from guardrails_api_client.types import UNSET
 from httpx import Timeout
 
@@ -49,3 +51,34 @@ def validate(
             body=payload,
             x_openai_api_key=_openai_api_key,
         )
+
+    def stream_validate(
+        self,
+        guard: Guard,
+        payload: ValidatePayload,
+        openai_api_key: Optional[str] = None,
+    ) -> Generator[ValidationOutput, None, None]:
+        _openai_api_key = (
+            openai_api_key
+            if openai_api_key is not None
+            else os.environ.get("OPENAI_API_KEY", UNSET)
+        )
+
+        url = f"{self.base_url}/guards/{guard.name}/validate"
+        headers = {
+            "Content-Type": "application/json",
+            "x-openai-api-key": _openai_api_key,
+        }
+
+        s = requests.Session()
+
+        with s.post(url, json=payload.to_dict(), headers=headers, stream=True) as resp:
+            for line in resp.iter_lines():
+                if not resp.ok:
+                    raise ValueError(
+                        f"status_code: {resp.status_code}"
+                        " reason: {resp.reason} text: {resp.text}"
+                    )
+                if line:
+                    json_output = json.loads(line)
+                    yield ValidationOutput.from_dict(json_output)
diff --git a/guardrails/cli/server/module_manifest.py b/guardrails/cli/server/module_manifest.py
index 454d718a8..61a5c7ccf 100644
--- a/guardrails/cli/server/module_manifest.py
+++ b/guardrails/cli/server/module_manifest.py
@@ -28,6 +28,13 @@ class ModuleTags(Serializeable):
     process_requirements: Optional[List[str]] = field(default_factory=list)
 
 
+@dataclass
+class ModelAuth(Serializeable):
+    type: str
+    name: str
+    displayName: Optional[str] = None
+
+
 @dataclass
 class ModuleManifest(Serializeable):
     id: str
@@ -43,6 +50,7 @@ class ModuleManifest(Serializeable):
     requires_auth: Optional[bool] = True
     post_install: Optional[str] = None
     index: Optional[str] = None
+    required_model_auth: Optional[List[ModelAuth]] = field(default_factory=list)
 
     # @override
     @classmethod
diff --git a/guardrails/document_store.py b/guardrails/document_store.py
index 4a9a31298..8e6ad7c25 100644
--- a/guardrails/document_store.py
+++ b/guardrails/document_store.py
@@ -238,21 +238,23 @@ def get_pages_for_for_indexes(self, indexes: List[int]) -> List[Page]:
 except ImportError:
 
     class FallbackEphemeralDocumentStore:
-        def __init__(self):
+        def __init__(self, *args, **kwargs):
+            # Why don't we just raise this when the import
+            # error occurs instead of at runtime?
             raise ImportError(
                 "SQLAlchemy is required for EphemeralDocumentStore"
                 "Please install it using `poetry add SqlAlchemy`"
             )
 
     class FallbackSQLDocument:
-        def __init__(self):
+        def __init__(self, *args, **kwargs):
             raise ImportError(
                 "SQLAlchemy is required for SQLDocument"
                 "Please install it using `poetry add SqlAlchemy`"
             )
 
     class FallbackSQLMetadataStore:
-        def __init__(self):
+        def __init__(self, *args, **kwargs):
             raise ImportError(
                 "SQLAlchemy is required for SQLMetadataStore"
                 "Please install it using `poetry add SqlAlchemy`"
diff --git a/guardrails/guard.py b/guardrails/guard.py
index 335c832af..8940be790 100644
--- a/guardrails/guard.py
+++ b/guardrails/guard.py
@@ -10,6 +10,7 @@
     Awaitable,
     Callable,
     Dict,
+    Generator,
     Generic,
     Iterable,
     List,
@@ -66,6 +67,7 @@
     set_tracer,
     set_tracer_context,
 )
+from guardrails.utils.api_utils import extract_serializeable_metadata
 from guardrails.utils.hub_telemetry_utils import HubTelemetry
 from guardrails.utils.llm_response import LLMResponse
 from guardrails.utils.reask_utils import FieldReAsk
@@ -146,18 +148,7 @@ def __init__(
         self.description = description
         self.name = name
 
-        api_key = os.environ.get("GUARDRAILS_API_KEY")
-        if api_key is not None:
-            if self.name is None:
-                self.name = f"gr-{str(self._guard_id)}"
-                logger.warn("Warning: No name passed to guard!")
-                logger.warn(
-                    "Use this auto-generated name to re-use this guard: {name}".format(
-                        name=self.name
-                    )
-                )
-            self._api_client = GuardrailsApiClient(api_key=api_key)
-            self.upsert_guard()
+        self._save()
 
     @property
     @deprecated(
@@ -653,8 +644,12 @@ def __call(
             ):
                 return self._call_server(
                     llm_api=llm_api,
-                    num_reasks=self.num_reasks,
                     prompt_params=prompt_params,
+                    num_reasks=self.num_reasks,
+                    prompt=prompt,
+                    instructions=instructions,
+                    msg_history=msg_history,
+                    metadata=metadata,
                     full_schema_reask=full_schema_reask,
                     call_log=call_log,
                     *args,
@@ -1009,6 +1004,7 @@ def __parse(
             ):
                 return self._call_server(
                     llm_output=llm_output,
+                    metadata=metadata,
                     llm_api=llm_api,
                     num_reasks=self.num_reasks,
                     prompt_params=prompt_params,
@@ -1295,6 +1291,7 @@ def use(
         """
         hydrated_validator = get_validator(validator, *args, **kwargs)
         self.__add_validator(hydrated_validator, on=on)
+        self._save()
         return self
 
     @overload
@@ -1336,6 +1333,7 @@ def use_many(
         for v in validators:
             hydrated_validator = get_validator(v)
             self.__add_validator(hydrated_validator, on=on)
+        self._save()
         return self
 
     def validate(self, llm_output: str, *args, **kwargs) -> ValidationOutcome[str]:
@@ -1358,6 +1356,7 @@ def validate(self, llm_output: str, *args, **kwargs) -> ValidationOutcome[str]:
                 if self.rail.output_schema.reask_instructions_template
                 else None,
             )
+            self._save()
 
         return self.parse(llm_output=llm_output, *args, **kwargs)
 
@@ -1413,39 +1412,111 @@ def upsert_guard(self):
         else:
             raise ValueError("Guard does not have an api client!")
 
-    def _call_server(
+    def _construct_history_from_server_response(
         self,
-        *args,
-        llm_output: Optional[str] = None,
+        *,
+        validation_output: Optional[ValidationOutput] = None,
         llm_api: Optional[Callable] = None,
+        llm_output: Optional[str] = None,
         num_reasks: Optional[int] = None,
         prompt_params: Optional[Dict] = None,
         metadata: Optional[Dict] = {},
         full_schema_reask: Optional[bool] = True,
         call_log: Optional[Call],
-        # prompt: Optional[str],
-        # instructions: Optional[str],
-        # msg_history: Optional[List[Dict]],
-        **kwargs,
     ):
+        call_log = call_log or Call()
+        if llm_api is not None:
+            llm_api = get_llm_ask(llm_api)
+            if asyncio.iscoroutinefunction(llm_api):
+                llm_api = get_async_llm_ask(llm_api)
+        session_history = (
+            validation_output.session_history
+            if validation_output is not None and validation_output.session_history
+            else []
+        )
+        history: History
+        for history in session_history:
+            history_events: Optional[List[HistoryEvent]] = (  # type: ignore
+                history.history if history.history != UNSET else None
+            )
+            if history_events is None:
+                continue
+
+            iterations = [
+                Iteration(
+                    inputs=Inputs(
+                        llm_api=llm_api,
+                        llm_output=llm_output,
+                        instructions=(
+                            Instructions(h.instructions) if h.instructions else None
+                        ),
+                        prompt=(
+                            Prompt(h.prompt.source)  # type: ignore
+                            if h.prompt is not None and h.prompt != UNSET
+                            else None
+                        ),
+                        prompt_params=prompt_params,
+                        num_reasks=(num_reasks or 0),
+                        metadata=metadata,
+                        full_schema_reask=full_schema_reask,
+                    ),
+                    outputs=Outputs(
+                        llm_response_info=LLMResponse(output=h.output),  # type: ignore
+                        raw_output=h.output,
+                        parsed_output=(
+                            h.parsed_output.to_dict()
+                            if isinstance(h.parsed_output, AnyObject)
+                            else h.parsed_output
+                        ),
+                        validation_output=(
+                            h.validated_output.to_dict()
+                            if isinstance(h.validated_output, AnyObject)
+                            else h.validated_output
+                        ),
+                        reasks=list(
+                            [
+                                FieldReAsk(
+                                    incorrect_value=r.to_dict().get("incorrect_value"),
+                                    path=r.to_dict().get("path"),
+                                    fail_results=[
+                                        FailResult(
+                                            error_message=r.to_dict().get(
+                                                "error_message"
+                                            ),
+                                            fix_value=r.to_dict().get("fix_value"),
+                                        )
+                                    ],
+                                )
+                                for r in h.reasks  # type: ignore
+                            ]
+                            if h.reasks != UNSET
+                            else []
+                        ),
+                    ),
+                )
+                for h in history_events
+            ]
+            call_log.iterations.extend(iterations)
+            if self.history.length == 0:
+                self.history.push(call_log)
+
+    def _single_server_call(
+        self,
+        *,
+        payload: Dict[str, Any],
+        llm_output: Optional[str] = None,
+        num_reasks: Optional[int] = None,
+        prompt_params: Optional[Dict] = None,
+        metadata: Optional[Dict] = {},
+        full_schema_reask: Optional[bool] = True,
+        call_log: Optional[Call],
+    ) -> ValidationOutcome[OT]:
         if self._api_client:
-            payload: Dict[str, Any] = {"args": list(args)}
-            payload.update(**kwargs)
-            if llm_output is not None:
-                payload["llmOutput"] = llm_output
-            if num_reasks is not None:
-                payload["numReasks"] = num_reasks
-            if prompt_params is not None:
-                payload["promptParams"] = prompt_params
-            if llm_api is not None:
-                payload["llmApi"] = get_llm_api_enum(llm_api)
-            # TODO: get enum for llm_api
-            validation_output: Optional[ValidationOutput] = self._api_client.validate(
+            validation_output: ValidationOutput = self._api_client.validate(
                 guard=self,  # type: ignore
                 payload=ValidatePayload.from_dict(payload),
                 openai_api_key=get_call_kwarg("api_key"),
             )
-
             if not validation_output:
                 return ValidationOutcome[OT](
                     raw_llm_output=None,
@@ -1453,86 +1524,15 @@ def _call_server(
                     validation_passed=False,
                     error="The response from the server was empty!",
                 )
-
-            call_log = call_log or Call()
-            if llm_api is not None:
-                llm_api = get_llm_ask(llm_api)
-                if asyncio.iscoroutinefunction(llm_api):
-                    llm_api = get_async_llm_ask(llm_api)
-            session_history = (
-                validation_output.session_history
-                if validation_output is not None and validation_output.session_history
-                else []
+            self._construct_history_from_server_response(
+                validation_output=validation_output,
+                llm_output=llm_output,
+                num_reasks=num_reasks,
+                prompt_params=prompt_params,
+                metadata=metadata,
+                full_schema_reask=full_schema_reask,
+                call_log=call_log,
             )
-            history: History
-            for history in session_history:
-                history_events: Optional[List[HistoryEvent]] = (  # type: ignore
-                    history.history if history.history != UNSET else None
-                )
-                if history_events is None:
-                    continue
-
-                iterations = [
-                    Iteration(
-                        inputs=Inputs(
-                            llm_api=llm_api,
-                            llm_output=llm_output,
-                            instructions=(
-                                Instructions(h.instructions) if h.instructions else None
-                            ),
-                            prompt=(
-                                Prompt(h.prompt.source)  # type: ignore
-                                if h.prompt is not None and h.prompt != UNSET
-                                else None
-                            ),
-                            prompt_params=prompt_params,
-                            num_reasks=(num_reasks or 0),
-                            metadata=metadata,
-                            full_schema_reask=full_schema_reask,
-                        ),
-                        outputs=Outputs(
-                            llm_response_info=LLMResponse(
-                                output=h.output  # type: ignore
-                            ),
-                            raw_output=h.output,
-                            parsed_output=(
-                                h.parsed_output.to_dict()
-                                if isinstance(h.parsed_output, AnyObject)
-                                else h.parsed_output
-                            ),
-                            validation_output=(
-                                h.validated_output.to_dict()
-                                if isinstance(h.validated_output, AnyObject)
-                                else h.validated_output
-                            ),
-                            reasks=list(
-                                [
-                                    FieldReAsk(
-                                        incorrect_value=r.to_dict().get(
-                                            "incorrect_value"
-                                        ),
-                                        path=r.to_dict().get("path"),
-                                        fail_results=[
-                                            FailResult(
-                                                error_message=r.to_dict().get(
-                                                    "error_message"
-                                                ),
-                                                fix_value=r.to_dict().get("fix_value"),
-                                            )
-                                        ],
-                                    )
-                                    for r in h.reasks  # type: ignore
-                                ]
-                                if h.reasks != UNSET
-                                else []
-                            ),
-                        ),
-                    )
-                    for h in history_events
-                ]
-                call_log.iterations.extend(iterations)
-                if self.history.length == 0:
-                    self.history.push(call_log)
 
             # Our interfaces are too different for this to work right now.
             # Once we move towards shared interfaces for both the open source
@@ -1545,3 +1545,113 @@ def _call_server(
             )
         else:
             raise ValueError("Guard does not have an api client!")
+
+    def _stream_server_call(
+        self,
+        *,
+        payload: Dict[str, Any],
+        llm_output: Optional[str] = None,
+        num_reasks: Optional[int] = None,
+        prompt_params: Optional[Dict] = None,
+        metadata: Optional[Dict] = {},
+        full_schema_reask: Optional[bool] = True,
+        call_log: Optional[Call],
+    ) -> Generator[ValidationOutcome[OT], None, None]:
+        if self._api_client:
+            validation_output: Optional[ValidationOutput] = None
+            response = self._api_client.stream_validate(
+                guard=self,  # type: ignore
+                payload=ValidatePayload.from_dict(payload),
+                openai_api_key=get_call_kwarg("api_key"),
+            )
+            for fragment in response:
+                validation_output = fragment
+                if not validation_output:
+                    yield ValidationOutcome[OT](
+                        raw_llm_output=None,
+                        validated_output=None,
+                        validation_passed=False,
+                        error="The response from the server was empty!",
+                    )
+                yield ValidationOutcome[OT](
+                    raw_llm_output=validation_output.raw_llm_response,  # type: ignore
+                    validated_output=cast(OT, validation_output.validated_output),
+                    validation_passed=validation_output.result,
+                )
+            if validation_output:
+                self._construct_history_from_server_response(
+                    validation_output=validation_output,
+                    llm_output=llm_output,
+                    num_reasks=num_reasks,
+                    prompt_params=prompt_params,
+                    metadata=metadata,
+                    full_schema_reask=full_schema_reask,
+                    call_log=call_log,
+                )
+        else:
+            raise ValueError("Guard does not have an api client!")
+
+    def _call_server(
+        self,
+        *args,
+        llm_output: Optional[str] = None,
+        llm_api: Optional[Callable] = None,
+        num_reasks: Optional[int] = None,
+        prompt_params: Optional[Dict] = None,
+        metadata: Optional[Dict] = {},
+        full_schema_reask: Optional[bool] = True,
+        call_log: Optional[Call],
+        **kwargs,
+    ) -> Union[ValidationOutcome[OT], Generator[ValidationOutcome[OT], None, None]]:
+        if self._api_client:
+            payload: Dict[str, Any] = {"args": list(args)}
+            payload.update(**kwargs)
+            if metadata:
+                payload["metadata"] = extract_serializeable_metadata(metadata)
+            if llm_output is not None:
+                payload["llmOutput"] = llm_output
+            if num_reasks is not None:
+                payload["numReasks"] = num_reasks
+            if prompt_params is not None:
+                payload["promptParams"] = prompt_params
+            if llm_api is not None:
+                payload["llmApi"] = get_llm_api_enum(llm_api, *args, **kwargs)
+
+            should_stream = kwargs.get("stream", False)
+            if should_stream:
+                return self._stream_server_call(
+                    payload=payload,
+                    llm_output=llm_output,
+                    num_reasks=num_reasks,
+                    prompt_params=prompt_params,
+                    metadata=metadata,
+                    full_schema_reask=full_schema_reask,
+                    call_log=call_log,
+                )
+            else:
+                return self._single_server_call(
+                    payload=payload,
+                    llm_output=llm_output,
+                    num_reasks=num_reasks,
+                    prompt_params=prompt_params,
+                    metadata=metadata,
+                    full_schema_reask=full_schema_reask,
+                    call_log=call_log,
+                )
+        else:
+            raise ValueError("Guard does not have an api client!")
+
+    def _save(self):
+        api_key = os.environ.get("GUARDRAILS_API_KEY")
+        if api_key is not None:
+            if self.name is None:
+                self.name = f"gr-{str(self._guard_id)}"
+                logger.warn("Warning: No name passed to guard!")
+                logger.warn(
+                    "Use this auto-generated name to re-use this guard: {name}".format(
+                        name=self.name
+                    )
+                )
+            if not self._api_client:
+                self._api_client = GuardrailsApiClient(api_key=api_key)
+            self.upsert_guard()
diff --git a/guardrails/llm_providers.py b/guardrails/llm_providers.py
index 9bea2d850..fd000cca2 100644
--- a/guardrails/llm_providers.py
+++ b/guardrails/llm_providers.py
@@ -939,16 +939,20 @@ def model_is_supported_server_side(
     model = get_llm_ask(llm_api, *args, **kwargs)
     if asyncio.iscoroutinefunction(llm_api):
         model = get_async_llm_ask(llm_api, *args, **kwargs)
-    return issubclass(type(model), OpenAIModel) or issubclass(
-        type(model), AsyncOpenAIModel
+    return (
+        issubclass(type(model), OpenAIModel)
+        or issubclass(type(model), AsyncOpenAIModel)
+        or isinstance(model, LiteLLMCallable)
+        or isinstance(model, AsyncLiteLLMCallable)
     )
 
 
 # FIXME: Update with newly supported LLMs
 def get_llm_api_enum(
-    llm_api: Callable[[Any], Awaitable[Any]],
+    llm_api: Callable[[Any], Awaitable[Any]], *args, **kwargs
 ) -> Optional[ValidatePayloadLlmApi]:
     # TODO: Distinguish between v1 and v2
+    model = get_llm_ask(llm_api, *args, **kwargs)
     if llm_api == get_static_openai_create_func():
         return ValidatePayloadLlmApi.OPENAI_COMPLETION_CREATE
     elif llm_api == get_static_openai_chat_create_func():
@@ -957,5 +961,10 @@ def get_llm_api_enum(
         return ValidatePayloadLlmApi.OPENAI_COMPLETION_ACREATE
     elif llm_api == get_static_openai_chat_acreate_func():
         return ValidatePayloadLlmApi.OPENAI_CHATCOMPLETION_ACREATE
+    elif isinstance(model, LiteLLMCallable):
+        return ValidatePayloadLlmApi.LITELLM_COMPLETION
+    elif isinstance(model, AsyncLiteLLMCallable):
+        return ValidatePayloadLlmApi.LITELLM_ACOMPLETION
+
     else:
         return None
diff --git a/guardrails/utils/api_utils.py b/guardrails/utils/api_utils.py
new file mode 100644
index 000000000..8d91f64f2
--- /dev/null
+++ b/guardrails/utils/api_utils.py
@@ -0,0 +1,16 @@
+import json
+from typing import Any, Dict
+
+
+def try_to_json(value: Any):
+    try:
+        json.dumps(value)
+        return True
+    except ValueError:
+        return False
+    except TypeError:
+        return False
+
+
+def extract_serializeable_metadata(metadata: Dict[str, Any]) -> Dict[str, Any]:
+    return {k: metadata[k] for k in metadata if try_to_json(metadata[k])}
diff --git a/guardrails/utils/hub_telemetry_utils.py b/guardrails/utils/hub_telemetry_utils.py
index 719f80050..6d9517130 100644
--- a/guardrails/utils/hub_telemetry_utils.py
+++ b/guardrails/utils/hub_telemetry_utils.py
@@ -115,7 +115,7 @@ def create_new_span(
         if self._tracer is None:
             return
         with self._tracer.start_as_current_span(
-            span_name,
+            span_name,  # type: ignore (Fails in Python 3.8 for invalid reason)
             context=self.extract_current_context() if has_parent else None,
         ) as span:
             if is_parent:
diff --git a/guardrails/utils/telemetry_utils.py b/guardrails/utils/telemetry_utils.py
index 599d013b6..35c453491 100644
--- a/guardrails/utils/telemetry_utils.py
+++ b/guardrails/utils/telemetry_utils.py
@@ -156,7 +156,8 @@ def with_trace(*args, **kwargs):
             if _tracer is None:
                 return fn(*args, **kwargs)
             with _tracer.start_as_current_span(
-                span_name, trace_context
+                span_name,  # type: ignore (Fails in Python 3.8 for invalid reason)
+                trace_context,
             ) as validator_span:
                 try:
                     validator_span.set_attribute(
@@ -200,7 +201,7 @@ def to_trace_or_not_to_trace(*args, **kwargs):
 
             if _tracer is not None and hasattr(_tracer, "start_as_current_span"):
                 trace_context = get_current_context()
-                with _tracer.start_as_current_span(name, trace_context) as trace_span:
+                with _tracer.start_as_current_span(name, trace_context) as trace_span:  # type: ignore (Fails in Python 3.8 for invalid reason)
                     try:
                         # TODO: Capture args and kwargs as attributes?
                         response = fn(*args, **kwargs)
@@ -226,7 +227,7 @@ async def to_trace_or_not_to_trace(*args, **kwargs):
 
             if _tracer is not None and hasattr(_tracer, "start_as_current_span"):
                 trace_context = get_current_context()
-                with _tracer.start_as_current_span(name, trace_context) as trace_span:
+                with _tracer.start_as_current_span(name, trace_context) as trace_span:  # type: ignore (Fails in Python 3.8 for invalid reason)
                     try:
                         # TODO: Capture args and kwargs as attributes?
                         response = await fn(*args, **kwargs)
diff --git a/guardrails/validator_service.py b/guardrails/validator_service.py
index 92b7879a2..14f610846 100644
--- a/guardrails/validator_service.py
+++ b/guardrails/validator_service.py
@@ -8,7 +8,6 @@
 from guardrails.classes.history import Iteration
 from guardrails.datatypes import FieldValidation
 from guardrails.errors import ValidationError
-from guardrails.logger import logger
 from guardrails.utils.hub_telemetry_utils import HubTelemetry
 from guardrails.utils.logs_utils import ValidatorLogs
 from guardrails.utils.reask_utils import FieldReAsk, ReAsk
@@ -421,13 +420,6 @@ def validate(
         loop = None
 
     if process_count == 1:
-        logger.warning(
-            "Process count was set to 1 via the GUARDRAILS_PROCESS_COUNT"
-            "environment variable."
-            "This will cause all validations to run synchronously."
-            "To run asynchronously, specify a process count"
-            "greater than 1 or unset this environment variable."
-        )
         validator_service = SequentialValidatorService(disable_tracer)
     elif loop is not None and not loop.is_running():
         validator_service = AsyncValidatorService(disable_tracer)
diff --git a/guardrails/validatorsattr.py b/guardrails/validatorsattr.py
index fbf5e3f99..9d5468ebf 100644
--- a/guardrails/validatorsattr.py
+++ b/guardrails/validatorsattr.py
@@ -327,7 +327,10 @@ def get_validators(
                 continue
 
             # See if the formatter has an associated on_fail method.
-            on_fail = on_fail_handlers.get(validator_name, None)
+            escaped_validator_name = validator_name.replace("/", "_")
+            on_fail = on_fail_handlers.get(
+                validator_name, on_fail_handlers.get(escaped_validator_name)
+            )
             # TODO(shreya): Load the on_fail method.
             # This method should be loaded from an optional script given at the
             # beginning of a rail file.
diff --git a/poetry.lock b/poetry.lock
index 5b3139c96..cf76aefb1 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand.
 
 [[package]]
 name = "aiohttp"
@@ -368,17 +368,6 @@ files = [
     {file = "backcall-0.2.0.tar.gz", hash = "sha256:5cbdbf27be5e7cfadb448baf0aa95508f91f2bbc6c6437cd9cd06e2a4c215e1e"},
 ]
 
-[[package]]
-name = "backoff"
-version = "2.2.1"
-description = "Function decoration for backoff and retry"
-optional = false
-python-versions = ">=3.7,<4.0"
-files = [
-    {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"},
-    {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"},
-]
-
 [[package]]
 name = "backports-tarfile"
 version = "1.1.1"
@@ -1662,29 +1651,62 @@ optional = true
 python-versions = ">=3.7"
 files = [
     {file = "greenlet-3.0.3-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:9da2bd29ed9e4f15955dd1595ad7bc9320308a3b766ef7f837e23ad4b4aac31a"},
+    {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d353cadd6083fdb056bb46ed07e4340b0869c305c8ca54ef9da3421acbdf6881"},
+    {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dca1e2f3ca00b84a396bc1bce13dd21f680f035314d2379c4160c98153b2059b"},
+    {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3ed7fb269f15dc662787f4119ec300ad0702fa1b19d2135a37c2c4de6fadfd4a"},
     {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd4f49ae60e10adbc94b45c0b5e6a179acc1736cf7a90160b404076ee283cf83"},
     {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:73a411ef564e0e097dbe7e866bb2dda0f027e072b04da387282b02c308807405"},
+    {file = "greenlet-3.0.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:7f362975f2d179f9e26928c5b517524e89dd48530a0202570d55ad6ca5d8a56f"},
     {file = "greenlet-3.0.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:649dde7de1a5eceb258f9cb00bdf50e978c9db1b996964cd80703614c86495eb"},
+    {file = "greenlet-3.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:68834da854554926fbedd38c76e60c4a2e3198c6fbed520b106a8986445caaf9"},
     {file = "greenlet-3.0.3-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:b1b5667cced97081bf57b8fa1d6bfca67814b0afd38208d52538316e9422fc61"},
+    {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:52f59dd9c96ad2fc0d5724107444f76eb20aaccb675bf825df6435acb7703559"},
+    {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:afaff6cf5200befd5cec055b07d1c0a5a06c040fe5ad148abcd11ba6ab9b114e"},
+    {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fe754d231288e1e64323cfad462fcee8f0288654c10bdf4f603a39ed923bef33"},
     {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2797aa5aedac23af156bbb5a6aa2cd3427ada2972c828244eb7d1b9255846379"},
     {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b7f009caad047246ed379e1c4dbcb8b020f0a390667ea74d2387be2998f58a22"},
+    {file = "greenlet-3.0.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c5e1536de2aad7bf62e27baf79225d0d64360d4168cf2e6becb91baf1ed074f3"},
     {file = "greenlet-3.0.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:894393ce10ceac937e56ec00bb71c4c2f8209ad516e96033e4b3b1de270e200d"},
+    {file = "greenlet-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:1ea188d4f49089fc6fb283845ab18a2518d279c7cd9da1065d7a84e991748728"},
     {file = "greenlet-3.0.3-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:70fb482fdf2c707765ab5f0b6655e9cfcf3780d8d87355a063547b41177599be"},
+    {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4d1ac74f5c0c0524e4a24335350edad7e5f03b9532da7ea4d3c54d527784f2e"},
+    {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:149e94a2dd82d19838fe4b2259f1b6b9957d5ba1b25640d2380bea9c5df37676"},
+    {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:15d79dd26056573940fcb8c7413d84118086f2ec1a8acdfa854631084393efcc"},
     {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:881b7db1ebff4ba09aaaeae6aa491daeb226c8150fc20e836ad00041bcb11230"},
     {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fcd2469d6a2cf298f198f0487e0a5b1a47a42ca0fa4dfd1b6862c999f018ebbf"},
+    {file = "greenlet-3.0.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:1f672519db1796ca0d8753f9e78ec02355e862d0998193038c7073045899f305"},
     {file = "greenlet-3.0.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2516a9957eed41dd8f1ec0c604f1cdc86758b587d964668b5b196a9db5bfcde6"},
+    {file = "greenlet-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:bba5387a6975598857d86de9eac14210a49d554a77eb8261cc68b7d082f78ce2"},
     {file = "greenlet-3.0.3-cp37-cp37m-macosx_11_0_universal2.whl", hash = "sha256:5b51e85cb5ceda94e79d019ed36b35386e8c37d22f07d6a751cb659b180d5274"},
+    {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:daf3cb43b7cf2ba96d614252ce1684c1bccee6b2183a01328c98d36fcd7d5cb0"},
+    {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:99bf650dc5d69546e076f413a87481ee1d2d09aaaaaca058c9251b6d8c14783f"},
+    {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2dd6e660effd852586b6a8478a1d244b8dc90ab5b1321751d2ea15deb49ed414"},
     {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e3391d1e16e2a5a1507d83e4a8b100f4ee626e8eca43cf2cadb543de69827c4c"},
     {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e1f145462f1fa6e4a4ae3c0f782e580ce44d57c8f2c7aae1b6fa88c0b2efdb41"},
+    {file = "greenlet-3.0.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:1a7191e42732df52cb5f39d3527217e7ab73cae2cb3694d241e18f53d84ea9a7"},
     {file = "greenlet-3.0.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0448abc479fab28b00cb472d278828b3ccca164531daab4e970a0458786055d6"},
+    {file = "greenlet-3.0.3-cp37-cp37m-win32.whl", hash = "sha256:b542be2440edc2d48547b5923c408cbe0fc94afb9f18741faa6ae970dbcb9b6d"},
+    {file = "greenlet-3.0.3-cp37-cp37m-win_amd64.whl", hash = "sha256:01bc7ea167cf943b4c802068e178bbf70ae2e8c080467070d01bfa02f337ee67"},
     {file = "greenlet-3.0.3-cp38-cp38-macosx_11_0_universal2.whl", hash = "sha256:1996cb9306c8595335bb157d133daf5cf9f693ef413e7673cb07e3e5871379ca"},
+    {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ddc0f794e6ad661e321caa8d2f0a55ce01213c74722587256fb6566049a8b04"},
+    {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c9db1c18f0eaad2f804728c67d6c610778456e3e1cc4ab4bbd5eeb8e6053c6fc"},
+    {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7170375bcc99f1a2fbd9c306f5be8764eaf3ac6b5cb968862cad4c7057756506"},
     {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b66c9c1e7ccabad3a7d037b2bcb740122a7b17a53734b7d72a344ce39882a1b"},
     {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:098d86f528c855ead3479afe84b49242e174ed262456c342d70fc7f972bc13c4"},
+    {file = "greenlet-3.0.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:81bb9c6d52e8321f09c3d165b2a78c680506d9af285bfccbad9fb7ad5a5da3e5"},
     {file = "greenlet-3.0.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:fd096eb7ffef17c456cfa587523c5f92321ae02427ff955bebe9e3c63bc9f0da"},
+    {file = "greenlet-3.0.3-cp38-cp38-win32.whl", hash = "sha256:d46677c85c5ba00a9cb6f7a00b2bfa6f812192d2c9f7d9c4f6a55b60216712f3"},
+    {file = "greenlet-3.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:419b386f84949bf0e7c73e6032e3457b82a787c1ab4a0e43732898a761cc9dbf"},
     {file = "greenlet-3.0.3-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:da70d4d51c8b306bb7a031d5cff6cc25ad253affe89b70352af5f1cb68e74b53"},
+    {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:086152f8fbc5955df88382e8a75984e2bb1c892ad2e3c80a2508954e52295257"},
+    {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d73a9fe764d77f87f8ec26a0c85144d6a951a6c438dfe50487df5595c6373eac"},
+    {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7dcbe92cc99f08c8dd11f930de4d99ef756c3591a5377d1d9cd7dd5e896da71"},
     {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1551a8195c0d4a68fac7a4325efac0d541b48def35feb49d803674ac32582f61"},
     {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:64d7675ad83578e3fc149b617a444fab8efdafc9385471f868eb5ff83e446b8b"},
+    {file = "greenlet-3.0.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b37eef18ea55f2ffd8f00ff8fe7c8d3818abd3e25fb73fae2ca3b672e333a7a6"},
     {file = "greenlet-3.0.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:77457465d89b8263bca14759d7c1684df840b6811b2499838cc5b040a8b5b113"},
+    {file = "greenlet-3.0.3-cp39-cp39-win32.whl", hash = "sha256:57e8974f23e47dac22b83436bdcf23080ade568ce77df33159e019d161ce1d1e"},
+    {file = "greenlet-3.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:c5ee858cfe08f34712f548c3c363e807e7186f03ad7a5039ebadb29e8c6be067"},
     {file = "greenlet-3.0.3.tar.gz", hash = "sha256:43374442353259554ce33599da8b692d5aa96f8976d567d4badf263371fbe491"},
 ]
 
@@ -1708,71 +1730,71 @@ colorama = ">=0.4"
 
 [[package]]
 name = "grpcio"
-version = "1.63.0"
+version = "1.64.0"
 description = "HTTP/2-based RPC framework"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "grpcio-1.63.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:2e93aca840c29d4ab5db93f94ed0a0ca899e241f2e8aec6334ab3575dc46125c"},
-    {file = "grpcio-1.63.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:91b73d3f1340fefa1e1716c8c1ec9930c676d6b10a3513ab6c26004cb02d8b3f"},
-    {file = "grpcio-1.63.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:b3afbd9d6827fa6f475a4f91db55e441113f6d3eb9b7ebb8fb806e5bb6d6bd0d"},
-    {file = "grpcio-1.63.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8f3f6883ce54a7a5f47db43289a0a4c776487912de1a0e2cc83fdaec9685cc9f"},
-    {file = "grpcio-1.63.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cf8dae9cc0412cb86c8de5a8f3be395c5119a370f3ce2e69c8b7d46bb9872c8d"},
-    {file = "grpcio-1.63.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:08e1559fd3b3b4468486b26b0af64a3904a8dbc78d8d936af9c1cf9636eb3e8b"},
-    {file = "grpcio-1.63.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5c039ef01516039fa39da8a8a43a95b64e288f79f42a17e6c2904a02a319b357"},
-    {file = "grpcio-1.63.0-cp310-cp310-win32.whl", hash = "sha256:ad2ac8903b2eae071055a927ef74121ed52d69468e91d9bcbd028bd0e554be6d"},
-    {file = "grpcio-1.63.0-cp310-cp310-win_amd64.whl", hash = "sha256:b2e44f59316716532a993ca2966636df6fbe7be4ab6f099de6815570ebe4383a"},
-    {file = "grpcio-1.63.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:f28f8b2db7b86c77916829d64ab21ff49a9d8289ea1564a2b2a3a8ed9ffcccd3"},
-    {file = "grpcio-1.63.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:65bf975639a1f93bee63ca60d2e4951f1b543f498d581869922910a476ead2f5"},
-    {file = "grpcio-1.63.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:b5194775fec7dc3dbd6a935102bb156cd2c35efe1685b0a46c67b927c74f0cfb"},
-    {file = "grpcio-1.63.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e4cbb2100ee46d024c45920d16e888ee5d3cf47c66e316210bc236d5bebc42b3"},
-    {file = "grpcio-1.63.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ff737cf29b5b801619f10e59b581869e32f400159e8b12d7a97e7e3bdeee6a2"},
-    {file = "grpcio-1.63.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:cd1e68776262dd44dedd7381b1a0ad09d9930ffb405f737d64f505eb7f77d6c7"},
-    {file = "grpcio-1.63.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:93f45f27f516548e23e4ec3fbab21b060416007dbe768a111fc4611464cc773f"},
-    {file = "grpcio-1.63.0-cp311-cp311-win32.whl", hash = "sha256:878b1d88d0137df60e6b09b74cdb73db123f9579232c8456f53e9abc4f62eb3c"},
-    {file = "grpcio-1.63.0-cp311-cp311-win_amd64.whl", hash = "sha256:756fed02dacd24e8f488f295a913f250b56b98fb793f41d5b2de6c44fb762434"},
-    {file = "grpcio-1.63.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:93a46794cc96c3a674cdfb59ef9ce84d46185fe9421baf2268ccb556f8f81f57"},
-    {file = "grpcio-1.63.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:a7b19dfc74d0be7032ca1eda0ed545e582ee46cd65c162f9e9fc6b26ef827dc6"},
-    {file = "grpcio-1.63.0-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:8064d986d3a64ba21e498b9a376cbc5d6ab2e8ab0e288d39f266f0fca169b90d"},
-    {file = "grpcio-1.63.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:219bb1848cd2c90348c79ed0a6b0ea51866bc7e72fa6e205e459fedab5770172"},
-    {file = "grpcio-1.63.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a2d60cd1d58817bc5985fae6168d8b5655c4981d448d0f5b6194bbcc038090d2"},
-    {file = "grpcio-1.63.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:9e350cb096e5c67832e9b6e018cf8a0d2a53b2a958f6251615173165269a91b0"},
-    {file = "grpcio-1.63.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:56cdf96ff82e3cc90dbe8bac260352993f23e8e256e063c327b6cf9c88daf7a9"},
-    {file = "grpcio-1.63.0-cp312-cp312-win32.whl", hash = "sha256:3a6d1f9ea965e750db7b4ee6f9fdef5fdf135abe8a249e75d84b0a3e0c668a1b"},
-    {file = "grpcio-1.63.0-cp312-cp312-win_amd64.whl", hash = "sha256:d2497769895bb03efe3187fb1888fc20e98a5f18b3d14b606167dacda5789434"},
-    {file = "grpcio-1.63.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:fdf348ae69c6ff484402cfdb14e18c1b0054ac2420079d575c53a60b9b2853ae"},
-    {file = "grpcio-1.63.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a3abfe0b0f6798dedd2e9e92e881d9acd0fdb62ae27dcbbfa7654a57e24060c0"},
-    {file = "grpcio-1.63.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:6ef0ad92873672a2a3767cb827b64741c363ebaa27e7f21659e4e31f4d750280"},
-    {file = "grpcio-1.63.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b416252ac5588d9dfb8a30a191451adbf534e9ce5f56bb02cd193f12d8845b7f"},
-    {file = "grpcio-1.63.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e3b77eaefc74d7eb861d3ffbdf91b50a1bb1639514ebe764c47773b833fa2d91"},
-    {file = "grpcio-1.63.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:b005292369d9c1f80bf70c1db1c17c6c342da7576f1c689e8eee4fb0c256af85"},
-    {file = "grpcio-1.63.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:cdcda1156dcc41e042d1e899ba1f5c2e9f3cd7625b3d6ebfa619806a4c1aadda"},
-    {file = "grpcio-1.63.0-cp38-cp38-win32.whl", hash = "sha256:01799e8649f9e94ba7db1aeb3452188048b0019dc37696b0f5ce212c87c560c3"},
-    {file = "grpcio-1.63.0-cp38-cp38-win_amd64.whl", hash = "sha256:6a1a3642d76f887aa4009d92f71eb37809abceb3b7b5a1eec9c554a246f20e3a"},
-    {file = "grpcio-1.63.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:75f701ff645858a2b16bc8c9fc68af215a8bb2d5a9b647448129de6e85d52bce"},
-    {file = "grpcio-1.63.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:cacdef0348a08e475a721967f48206a2254a1b26ee7637638d9e081761a5ba86"},
-    {file = "grpcio-1.63.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:0697563d1d84d6985e40ec5ec596ff41b52abb3fd91ec240e8cb44a63b895094"},
-    {file = "grpcio-1.63.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6426e1fb92d006e47476d42b8f240c1d916a6d4423c5258ccc5b105e43438f61"},
-    {file = "grpcio-1.63.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e48cee31bc5f5a31fb2f3b573764bd563aaa5472342860edcc7039525b53e46a"},
-    {file = "grpcio-1.63.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:50344663068041b34a992c19c600236e7abb42d6ec32567916b87b4c8b8833b3"},
-    {file = "grpcio-1.63.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:259e11932230d70ef24a21b9fb5bb947eb4703f57865a404054400ee92f42f5d"},
-    {file = "grpcio-1.63.0-cp39-cp39-win32.whl", hash = "sha256:a44624aad77bf8ca198c55af811fd28f2b3eaf0a50ec5b57b06c034416ef2d0a"},
-    {file = "grpcio-1.63.0-cp39-cp39-win_amd64.whl", hash = "sha256:166e5c460e5d7d4656ff9e63b13e1f6029b122104c1633d5f37eaea348d7356d"},
-    {file = "grpcio-1.63.0.tar.gz", hash = "sha256:f3023e14805c61bc439fb40ca545ac3d5740ce66120a678a3c6c2c55b70343d1"},
-]
-
-[package.extras]
-protobuf = ["grpcio-tools (>=1.63.0)"]
+    {file = "grpcio-1.64.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:3b09c3d9de95461214a11d82cc0e6a46a6f4e1f91834b50782f932895215e5db"},
+    {file = "grpcio-1.64.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:7e013428ab472892830287dd082b7d129f4d8afef49227a28223a77337555eaa"},
+    {file = "grpcio-1.64.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:02cc9cc3f816d30f7993d0d408043b4a7d6a02346d251694d8ab1f78cc723e7e"},
+    {file = "grpcio-1.64.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f5de082d936e0208ce8db9095821361dfa97af8767a6607ae71425ac8ace15c"},
+    {file = "grpcio-1.64.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d7b7bf346391dffa182fba42506adf3a84f4a718a05e445b37824136047686a1"},
+    {file = "grpcio-1.64.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:b2cbdfba18408389a1371f8c2af1659119e1831e5ed24c240cae9e27b4abc38d"},
+    {file = "grpcio-1.64.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:aca4f15427d2df592e0c8f3d38847e25135e4092d7f70f02452c0e90d6a02d6d"},
+    {file = "grpcio-1.64.0-cp310-cp310-win32.whl", hash = "sha256:7c1f5b2298244472bcda49b599be04579f26425af0fd80d3f2eb5fd8bc84d106"},
+    {file = "grpcio-1.64.0-cp310-cp310-win_amd64.whl", hash = "sha256:73f84f9e5985a532e47880b3924867de16fa1aa513fff9b26106220c253c70c5"},
+    {file = "grpcio-1.64.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:2a18090371d138a57714ee9bffd6c9c9cb2e02ce42c681aac093ae1e7189ed21"},
+    {file = "grpcio-1.64.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:59c68df3a934a586c3473d15956d23a618b8f05b5e7a3a904d40300e9c69cbf0"},
+    {file = "grpcio-1.64.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:b52e1ec7185512103dd47d41cf34ea78e7a7361ba460187ddd2416b480e0938c"},
+    {file = "grpcio-1.64.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8d598b5d5e2c9115d7fb7e2cb5508d14286af506a75950762aa1372d60e41851"},
+    {file = "grpcio-1.64.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:01615bbcae6875eee8091e6b9414072f4e4b00d8b7e141f89635bdae7cf784e5"},
+    {file = "grpcio-1.64.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:0b2dfe6dcace264807d9123d483d4c43274e3f8c39f90ff51de538245d7a4145"},
+    {file = "grpcio-1.64.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:7f17572dc9acd5e6dfd3014d10c0b533e9f79cd9517fc10b0225746f4c24b58e"},
+    {file = "grpcio-1.64.0-cp311-cp311-win32.whl", hash = "sha256:6ec5ed15b4ffe56e2c6bc76af45e6b591c9be0224b3fb090adfb205c9012367d"},
+    {file = "grpcio-1.64.0-cp311-cp311-win_amd64.whl", hash = "sha256:597191370951b477b7a1441e1aaa5cacebeb46a3b0bd240ec3bb2f28298c7553"},
+    {file = "grpcio-1.64.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:1ce4cd5a61d4532651079e7aae0fedf9a80e613eed895d5b9743e66b52d15812"},
+    {file = "grpcio-1.64.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:650a8150a9b288f40d5b7c1d5400cc11724eae50bd1f501a66e1ea949173649b"},
+    {file = "grpcio-1.64.0-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:8de0399b983f8676a7ccfdd45e5b2caec74a7e3cc576c6b1eecf3b3680deda5e"},
+    {file = "grpcio-1.64.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:46b8b43ba6a2a8f3103f103f97996cad507bcfd72359af6516363c48793d5a7b"},
+    {file = "grpcio-1.64.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a54362f03d4dcfae63be455d0a7d4c1403673498b92c6bfe22157d935b57c7a9"},
+    {file = "grpcio-1.64.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:1f8ea18b928e539046bb5f9c124d717fbf00cc4b2d960ae0b8468562846f5aa1"},
+    {file = "grpcio-1.64.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:c56c91bd2923ddb6e7ed28ebb66d15633b03e0df22206f22dfcdde08047e0a48"},
+    {file = "grpcio-1.64.0-cp312-cp312-win32.whl", hash = "sha256:874c741c8a66f0834f653a69e7e64b4e67fcd4a8d40296919b93bab2ccc780ba"},
+    {file = "grpcio-1.64.0-cp312-cp312-win_amd64.whl", hash = "sha256:0da1d921f8e4bcee307aeef6c7095eb26e617c471f8cb1c454fd389c5c296d1e"},
+    {file = "grpcio-1.64.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:c46fb6bfca17bfc49f011eb53416e61472fa96caa0979b4329176bdd38cbbf2a"},
+    {file = "grpcio-1.64.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:3d2004e85cf5213995d09408501f82c8534700d2babeb81dfdba2a3bff0bb396"},
+    {file = "grpcio-1.64.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:6d5541eb460d73a07418524fb64dcfe0adfbcd32e2dac0f8f90ce5b9dd6c046c"},
+    {file = "grpcio-1.64.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f279ad72dd7d64412e10f2443f9f34872a938c67387863c4cd2fb837f53e7d2"},
+    {file = "grpcio-1.64.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85fda90b81da25993aa47fae66cae747b921f8f6777550895fb62375b776a231"},
+    {file = "grpcio-1.64.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a053584079b793a54bece4a7d1d1b5c0645bdbee729215cd433703dc2532f72b"},
+    {file = "grpcio-1.64.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:579dd9fb11bc73f0de061cab5f8b2def21480fd99eb3743ed041ad6a1913ee2f"},
+    {file = "grpcio-1.64.0-cp38-cp38-win32.whl", hash = "sha256:23b6887bb21d77649d022fa1859e05853fdc2e60682fd86c3db652a555a282e0"},
+    {file = "grpcio-1.64.0-cp38-cp38-win_amd64.whl", hash = "sha256:753cb58683ba0c545306f4e17dabf468d29cb6f6b11832e1e432160bb3f8403c"},
+    {file = "grpcio-1.64.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:2186d76a7e383e1466e0ea2b0febc343ffeae13928c63c6ec6826533c2d69590"},
+    {file = "grpcio-1.64.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:0f30596cdcbed3c98024fb4f1d91745146385b3f9fd10c9f2270cbfe2ed7ed91"},
+    {file = "grpcio-1.64.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:d9171f025a196f5bcfec7e8e7ffb7c3535f7d60aecd3503f9e250296c7cfc150"},
+    {file = "grpcio-1.64.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cf4c8daed18ae2be2f1fc7d613a76ee2a2e28fdf2412d5c128be23144d28283d"},
+    {file = "grpcio-1.64.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3550493ac1d23198d46dc9c9b24b411cef613798dc31160c7138568ec26bc9b4"},
+    {file = "grpcio-1.64.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:3161a8f8bb38077a6470508c1a7301cd54301c53b8a34bb83e3c9764874ecabd"},
+    {file = "grpcio-1.64.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2e8fabe2cc57a369638ab1ad8e6043721014fdf9a13baa7c0e35995d3a4a7618"},
+    {file = "grpcio-1.64.0-cp39-cp39-win32.whl", hash = "sha256:31890b24d47b62cc27da49a462efe3d02f3c120edb0e6c46dcc0025506acf004"},
+    {file = "grpcio-1.64.0-cp39-cp39-win_amd64.whl", hash = "sha256:5a56797dea8c02e7d3a85dfea879f286175cf4d14fbd9ab3ef2477277b927baa"},
+    {file = "grpcio-1.64.0.tar.gz", hash = "sha256:257baf07f53a571c215eebe9679c3058a313fd1d1f7c4eede5a8660108c52d9c"},
+]
+
+[package.extras]
+protobuf = ["grpcio-tools (>=1.64.0)"]
 
 [[package]]
 name = "guardrails-api-client"
-version = "0.1.1"
+version = "0.2.1"
 description = "A client library for accessing Guardrails API"
 optional = false
 python-versions = "<4,>=3.8"
 files = [
-    {file = "guardrails-api-client-0.1.1.tar.gz", hash = "sha256:d707661b0e63269c9ab257c4ba6eed20af18a734607cab8bb3bee0d2883bc50f"},
-    {file = "guardrails_api_client-0.1.1-py3-none-any.whl", hash = "sha256:99baf4a11fcc61b420197c019fa24a479a44afb44b858e43d874cc95926295fd"},
+    {file = "guardrails_api_client-0.2.1-py3-none-any.whl", hash = "sha256:e6f70304b498a79c621149ae433041bf298c2268acf622442f2356e0b371e903"},
+    {file = "guardrails_api_client-0.2.1.tar.gz", hash = "sha256:68c5e31abffe227c2ec9e2f46e00ab94c7af5692e1c075331145280e0addde99"},
 ]
 
 [package.dependencies]
@@ -2807,6 +2829,7 @@ files = [
     {file = "lxml-4.9.4-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:e8f9f93a23634cfafbad6e46ad7d09e0f4a25a2400e4a64b1b7b7c0fbaa06d9d"},
     {file = "lxml-4.9.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:3f3f00a9061605725df1816f5713d10cd94636347ed651abdbc75828df302b20"},
     {file = "lxml-4.9.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:953dd5481bd6252bd480d6ec431f61d7d87fdcbbb71b0d2bdcfc6ae00bb6fb10"},
+    {file = "lxml-4.9.4-cp312-cp312-win32.whl", hash = "sha256:266f655d1baff9c47b52f529b5f6bec33f66042f65f7c56adde3fcf2ed62ae8b"},
     {file = "lxml-4.9.4-cp312-cp312-win_amd64.whl", hash = "sha256:f1faee2a831fe249e1bae9cbc68d3cd8a30f7e37851deee4d7962b17c410dd56"},
     {file = "lxml-4.9.4-cp35-cp35m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:23d891e5bdc12e2e506e7d225d6aa929e0a0368c9916c1fddefab88166e98b20"},
     {file = "lxml-4.9.4-cp35-cp35m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:e96a1788f24d03e8d61679f9881a883ecdf9c445a38f9ae3f3f193ab6c591c66"},
@@ -4108,91 +4131,85 @@ datalib = ["numpy (>=1)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)"]
 
 [[package]]
 name = "opentelemetry-api"
-version = "1.20.0"
+version = "1.24.0"
 description = "OpenTelemetry Python API"
 optional = false
-python-versions = ">=3.7"
+python-versions = ">=3.8"
 files = [
-    {file = "opentelemetry_api-1.20.0-py3-none-any.whl", hash = "sha256:982b76036fec0fdaf490ae3dfd9f28c81442a33414f737abc687a32758cdcba5"},
-    {file = "opentelemetry_api-1.20.0.tar.gz", hash = "sha256:06abe351db7572f8afdd0fb889ce53f3c992dbf6f6262507b385cc1963e06983"},
+    {file = "opentelemetry_api-1.24.0-py3-none-any.whl", hash = "sha256:0f2c363d98d10d1ce93330015ca7fd3a65f60be64e05e30f557c61de52c80ca2"},
+    {file = "opentelemetry_api-1.24.0.tar.gz", hash = "sha256:42719f10ce7b5a9a73b10a4baf620574fb8ad495a9cbe5c18d76b75d8689c67e"},
 ]
 
 [package.dependencies]
 deprecated = ">=1.2.6"
-importlib-metadata = ">=6.0,<7.0"
+importlib-metadata = ">=6.0,<=7.0"
 
 [[package]]
 name = "opentelemetry-exporter-otlp-proto-common"
-version = "1.20.0"
+version = "1.24.0"
 description = "OpenTelemetry Protobuf encoding"
 optional = false
-python-versions = ">=3.7"
+python-versions = ">=3.8"
 files = [
-    {file = "opentelemetry_exporter_otlp_proto_common-1.20.0-py3-none-any.whl", hash = "sha256:dd63209b40702636ab6ae76a06b401b646ad7b008a906ecb41222d4af24fbdef"},
-    {file = "opentelemetry_exporter_otlp_proto_common-1.20.0.tar.gz", hash = "sha256:df60c681bd61812e50b3a39a7a1afeeb6d4066117583249fcc262269374e7a49"},
+    {file = "opentelemetry_exporter_otlp_proto_common-1.24.0-py3-none-any.whl", hash = "sha256:e51f2c9735054d598ad2df5d3eca830fecfb5b0bda0a2fa742c9c7718e12f641"},
+    {file = "opentelemetry_exporter_otlp_proto_common-1.24.0.tar.gz", hash = "sha256:5d31fa1ff976cacc38be1ec4e3279a3f88435c75b38b1f7a099a1faffc302461"},
 ]
 
 [package.dependencies]
-backoff = {version = ">=1.10.0,<3.0.0", markers = "python_version >= \"3.7\""}
-opentelemetry-proto = "1.20.0"
+opentelemetry-proto = "1.24.0"
 
 [[package]]
 name = "opentelemetry-exporter-otlp-proto-grpc"
-version = "1.20.0"
+version = "1.24.0"
 description = "OpenTelemetry Collector Protobuf over gRPC Exporter"
 optional = false
-python-versions = ">=3.7"
+python-versions = ">=3.8"
 files = [
-    {file = "opentelemetry_exporter_otlp_proto_grpc-1.20.0-py3-none-any.whl", hash = "sha256:7c3f066065891b56348ba2c7f9df6ec635a712841cae0a36f2f6a81642ae7dec"},
-    {file = "opentelemetry_exporter_otlp_proto_grpc-1.20.0.tar.gz", hash = "sha256:6c06d43c3771bda1795226e327722b4b980fa1ca1ec9e985f2ef3e29795bdd52"},
+    {file = "opentelemetry_exporter_otlp_proto_grpc-1.24.0-py3-none-any.whl", hash = "sha256:f40d62aa30a0a43cc1657428e59fcf82ad5f7ea8fff75de0f9d9cb6f739e0a3b"},
+    {file = "opentelemetry_exporter_otlp_proto_grpc-1.24.0.tar.gz", hash = "sha256:217c6e30634f2c9797999ea9da29f7300479a94a610139b9df17433f915e7baa"},
 ]
 
 [package.dependencies]
-backoff = {version = ">=1.10.0,<3.0.0", markers = "python_version >= \"3.7\""}
 deprecated = ">=1.2.6"
 googleapis-common-protos = ">=1.52,<2.0"
 grpcio = ">=1.0.0,<2.0.0"
 opentelemetry-api = ">=1.15,<2.0"
-opentelemetry-exporter-otlp-proto-common = "1.20.0"
-opentelemetry-proto = "1.20.0"
-opentelemetry-sdk = ">=1.20.0,<1.21.0"
+opentelemetry-exporter-otlp-proto-common = "1.24.0"
+opentelemetry-proto = "1.24.0"
+opentelemetry-sdk = ">=1.24.0,<1.25.0"
 
 [package.extras]
 test = ["pytest-grpc"]
 
 [[package]]
 name = "opentelemetry-exporter-otlp-proto-http"
-version = "1.20.0"
+version = "1.24.0"
 description = "OpenTelemetry Collector Protobuf over HTTP Exporter"
 optional = false
-python-versions = ">=3.7"
+python-versions = ">=3.8"
 files = [
-    {file = "opentelemetry_exporter_otlp_proto_http-1.20.0-py3-none-any.whl", hash = "sha256:03f6e768ad25f1c3a9586e8c695db4a4adf978f8546a1285fa962e16bfbb0bd6"},
-    {file = "opentelemetry_exporter_otlp_proto_http-1.20.0.tar.gz", hash = "sha256:500f42821420fdf0759193d6438edc0f4e984a83e14c08a23023c06a188861b4"},
+    {file = "opentelemetry_exporter_otlp_proto_http-1.24.0-py3-none-any.whl", hash = "sha256:25af10e46fdf4cd3833175e42f4879a1255fc01655fe14c876183a2903949836"},
+    {file = "opentelemetry_exporter_otlp_proto_http-1.24.0.tar.gz", hash = "sha256:704c066cc96f5131881b75c0eac286cd73fc735c490b054838b4513254bd7850"},
 ]
 
 [package.dependencies]
-backoff = {version = ">=1.10.0,<3.0.0", markers = "python_version >= \"3.7\""}
 deprecated = ">=1.2.6"
 googleapis-common-protos = ">=1.52,<2.0"
 opentelemetry-api = ">=1.15,<2.0"
-opentelemetry-exporter-otlp-proto-common = "1.20.0"
-opentelemetry-proto = "1.20.0"
-opentelemetry-sdk = ">=1.20.0,<1.21.0"
+opentelemetry-exporter-otlp-proto-common = "1.24.0"
+opentelemetry-proto = "1.24.0"
+opentelemetry-sdk = ">=1.24.0,<1.25.0"
 requests = ">=2.7,<3.0"
 
-[package.extras]
-test = ["responses (==0.22.0)"]
-
 [[package]]
 name = "opentelemetry-proto"
-version = "1.20.0"
+version = "1.24.0"
 description = "OpenTelemetry Python Proto"
 optional = false
-python-versions = ">=3.7"
+python-versions = ">=3.8"
 files = [
-    {file = "opentelemetry_proto-1.20.0-py3-none-any.whl", hash = "sha256:512c3d2c6864fb7547a69577c3907348e6c985b7a204533563cb4c4c5046203b"},
-    {file = "opentelemetry_proto-1.20.0.tar.gz", hash = "sha256:cf01f49b3072ee57468bccb1a4f93bdb55411f4512d0ac3f97c5c04c0040b5a2"},
+    {file = "opentelemetry_proto-1.24.0-py3-none-any.whl", hash = "sha256:bcb80e1e78a003040db71ccf83f2ad2019273d1e0828089d183b18a1476527ce"},
+    {file = "opentelemetry_proto-1.24.0.tar.gz", hash = "sha256:ff551b8ad63c6cabb1845ce217a6709358dfaba0f75ea1fa21a61ceddc78cab8"},
 ]
 
 [package.dependencies]
@@ -4200,29 +4217,29 @@ protobuf = ">=3.19,<5.0"
 
 [[package]]
 name = "opentelemetry-sdk"
-version = "1.20.0"
+version = "1.24.0"
 description = "OpenTelemetry Python SDK"
 optional = false
-python-versions = ">=3.7"
+python-versions = ">=3.8"
 files = [
-    {file = "opentelemetry_sdk-1.20.0-py3-none-any.whl", hash = "sha256:f2230c276ff4c63ea09b3cb2e2ac6b1265f90af64e8d16bbf275c81a9ce8e804"},
-    {file = "opentelemetry_sdk-1.20.0.tar.gz", hash = "sha256:702e432a457fa717fd2ddfd30640180e69938f85bb7fec3e479f85f61c1843f8"},
+    {file = "opentelemetry_sdk-1.24.0-py3-none-any.whl", hash = "sha256:fa731e24efe832e98bcd90902085b359dcfef7d9c9c00eb5b9a18587dae3eb59"},
+    {file = "opentelemetry_sdk-1.24.0.tar.gz", hash = "sha256:75bc0563affffa827700e0f4f4a68e1e257db0df13372344aebc6f8a64cde2e5"},
 ]
 
 [package.dependencies]
-opentelemetry-api = "1.20.0"
-opentelemetry-semantic-conventions = "0.41b0"
+opentelemetry-api = "1.24.0"
+opentelemetry-semantic-conventions = "0.45b0"
 typing-extensions = ">=3.7.4"
 
 [[package]]
 name = "opentelemetry-semantic-conventions"
-version = "0.41b0"
+version = "0.45b0"
 description = "OpenTelemetry Semantic Conventions"
 optional = false
-python-versions = ">=3.7"
+python-versions = ">=3.8"
 files = [
-    {file = "opentelemetry_semantic_conventions-0.41b0-py3-none-any.whl", hash = "sha256:45404391ed9e50998183a4925ad1b497c01c143f06500c3b9c3d0013492bb0f2"},
-    {file = "opentelemetry_semantic_conventions-0.41b0.tar.gz", hash = "sha256:0ce5b040b8a3fc816ea5879a743b3d6fe5db61f6485e4def94c6ee4d402e1eb7"},
+    {file = "opentelemetry_semantic_conventions-0.45b0-py3-none-any.whl", hash = "sha256:a4a6fb9a7bacd9167c082aa4681009e9acdbfa28ffb2387af50c2fef3d30c864"},
+    {file = "opentelemetry_semantic_conventions-0.45b0.tar.gz", hash = "sha256:7c84215a44ac846bc4b8e32d5e78935c5c43482e491812a0bb8aaf87e4d92118"},
 ]
 
 [[package]]
@@ -4349,8 +4366,8 @@ files = [
 [package.dependencies]
 numpy = [
     {version = ">=1.20.3", markers = "python_version < \"3.10\""},
-    {version = ">=1.21.0", markers = "python_version >= \"3.10\" and python_version < \"3.11\""},
     {version = ">=1.23.2", markers = "python_version >= \"3.11\""},
+    {version = ">=1.21.0", markers = "python_version >= \"3.10\" and python_version < \"3.11\""},
 ]
 python-dateutil = ">=2.8.2"
 pytz = ">=2020.1"
@@ -5343,6 +5360,7 @@ files = [
     {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"},
     {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"},
     {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"},
+    {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"},
     {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"},
     {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"},
     {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"},
@@ -8321,4 +8339,4 @@ vectordb = ["faiss-cpu", "numpy"]
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.8.1"
-content-hash = "ebae16aa66c7e7789668d5f7ec3779ee60055f427df9b4ede98596a3a6bc5d2f"
+content-hash = "6e5bf0528e7e0cb74b8bf8b9c544e162c09bd85aaad2add60c3395f1118d7196"
diff --git a/pyproject.toml b/pyproject.toml
index f8413f823..11bc11409 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -50,15 +50,15 @@ huggingface_hub = {version = "^0.19.3", optional = true}
 pydash = "^7.0.6"
 docspec_python = "2.2.1"
 pydoc-markdown = "4.8.2"
-opentelemetry-sdk = "1.20.0"
-opentelemetry-exporter-otlp-proto-grpc = "1.20.0"
-opentelemetry-exporter-otlp-proto-http = "1.20.0"
 langchain-core = "^0.1.18"
 coloredlogs = "^15.0.1"
 requests = "^2.31.0"
-guardrails-api-client = "^0.1.1"
+guardrails-api-client = "^0.2.1"
 jwt = "^1.3.1"
 pip = ">=22"
+opentelemetry-sdk = "^1.24.0"
+opentelemetry-exporter-otlp-proto-grpc = "^1.24.0"
+opentelemetry-exporter-otlp-proto-http = "^1.24.0"
 
 [tool.poetry.extras]
 sql = ["sqlvalidator", "sqlalchemy", "sqlglot"]
diff --git a/tests/unit_tests/utils/test_api_utils.py b/tests/unit_tests/utils/test_api_utils.py
new file mode 100644
index 000000000..4a74cea67
--- /dev/null
+++ b/tests/unit_tests/utils/test_api_utils.py
@@ -0,0 +1,19 @@
+from guardrails.utils.api_utils import extract_serializeable_metadata
+
+
+def test_extract_serializeable_metadata():
+    def baz():
+        print("baz")
+
+    class NonMeta:
+        data = "data"
+
+    metadata = {
+        "foo": "bar",
+        "baz": baz,
+        "non_meta": NonMeta(),
+    }
+
+    extracted_metadata = extract_serializeable_metadata(metadata)
+
+    assert extracted_metadata == {"foo": "bar"}

From b05c241d84c2efde99649c824dae9254c18dae1b Mon Sep 17 00:00:00 2001
From: Caleb Courier <caleb.courier@gmail.com>
Date: Thu, 30 May 2024 12:55:01 -0500
Subject: [PATCH 41/85] try to run PR checks on PRs against feature branches

---
 .github/workflows/ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a9f8ca96c..45891ea37 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -9,6 +9,7 @@ on:
     branches:
       - main
       - dev
+      - feat/*
 
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:

From b52b8cbc2737eea26ed475feb8e7f92c98e26da7 Mon Sep 17 00:00:00 2001
From: Caleb Courier <caleb.courier@gmail.com>
Date: Thu, 30 May 2024 13:54:38 -0500
Subject: [PATCH 42/85] lint, type, and test fixes

---
 guardrails/classes/history/call.py        |  4 ++--
 guardrails/classes/history/iteration.py   |  4 +++-
 guardrails/classes/history/outputs.py     |  6 ++++--
 guardrails/run/stream_runner.py           |  2 +-
 guardrails/validator_base.py              | 23 ++++++++++----------
 guardrails/validator_service.py           | 24 ++++++++++-----------
 tests/integration_tests/test_streaming.py | 26 +++++++++++++++--------
 tests/unit_tests/test_validators.py       | 20 ++++++++---------
 8 files changed, 60 insertions(+), 49 deletions(-)

diff --git a/guardrails/classes/history/call.py b/guardrails/classes/history/call.py
index 287d344cc..503cefb9a 100644
--- a/guardrails/classes/history/call.py
+++ b/guardrails/classes/history/call.py
@@ -99,8 +99,8 @@ def reask_prompts(self) -> Stack[Optional[str]]:
 
     @property
     def instructions(self) -> Optional[str]:
-        """The instructions as provided by the user when initializing or calling
-        the Guard."""
+        """The instructions as provided by the user when initializing or
+        calling the Guard."""
         return self.inputs.instructions
 
     @property
diff --git a/guardrails/classes/history/iteration.py b/guardrails/classes/history/iteration.py
index 12120d914..ef913bafd 100644
--- a/guardrails/classes/history/iteration.py
+++ b/guardrails/classes/history/iteration.py
@@ -159,7 +159,9 @@ def failed_validations(self) -> List[ValidatorLogs]:
     @property
     def error_spans_in_output(self) -> List[ErrorSpan]:
         """The error spans from the LLM response.
-        These indices are relative to the complete LLM output."""
+
+        These indices are relative to the complete LLM output.
+        """
         return self.outputs.error_spans_in_output
 
     @property
diff --git a/guardrails/classes/history/outputs.py b/guardrails/classes/history/outputs.py
index 0085d00c2..953ed6f4b 100644
--- a/guardrails/classes/history/outputs.py
+++ b/guardrails/classes/history/outputs.py
@@ -77,7 +77,9 @@ def failed_validations(self) -> List[ValidatorLogs]:
     @property
     def error_spans_in_output(self) -> List[ErrorSpan]:
         """The error spans from the LLM response.
-        These indices are relative to the complete LLM output."""
+
+        These indices are relative to the complete LLM output.
+        """
         total_len = 0
         spans_in_output = []
         for log in self.validator_logs:
@@ -92,7 +94,7 @@ def error_spans_in_output(self) -> List[ErrorSpan]:
                                 reason=error_span.reason,
                             )
                         )
-            if result.validated_chunk is not None:
+            if result and result.validated_chunk is not None:
                 total_len += len(result.validated_chunk)
         return spans_in_output
 
diff --git a/guardrails/run/stream_runner.py b/guardrails/run/stream_runner.py
index bfe9196cb..223151804 100644
--- a/guardrails/run/stream_runner.py
+++ b/guardrails/run/stream_runner.py
@@ -271,7 +271,7 @@ def step(
         iteration.outputs.guarded_output = valid_op
 
     def is_last_chunk(self, chunk: Any, api: Union[PromptCallableBase, None]) -> bool:
-        """Detect if chunk is final chunk"""
+        """Detect if chunk is final chunk."""
         if isinstance(api, OpenAICallable):
             if OPENAI_VERSION.startswith("0"):
                 finished = chunk["choices"][0]["finish_reason"]
diff --git a/guardrails/validator_base.py b/guardrails/validator_base.py
index 5751b6e74..6fb1cc9df 100644
--- a/guardrails/validator_base.py
+++ b/guardrails/validator_base.py
@@ -498,17 +498,18 @@ def validate(self, value: Any, metadata: Dict[str, Any]) -> ValidationResult:
 
     def validate_stream(
         self, chunk: Any, metadata: Dict[str, Any], **kwargs
-    ) -> ValidationResult:
-        """Validates a chunk emitted by an LLM.
-        If the LLM chunk is smaller than the validator's chunking strategy,
-        it will be accumulated until it reaches the desired size. In the meantime,
-        the validator will return None.
-
-        If the LLM chunk is larger than the validator's chunking strategy,
-        it will split it into validator-sized chunks and validate each one,
-        returning an array of validation results.
-
-        Otherwise, the validator will validate the chunk and return the result.
+    ) -> Optional[ValidationResult]:
+        """Validates a chunk emitted by an LLM. If the LLM chunk is smaller
+        than the validator's chunking strategy, it will be accumulated until it
+        reaches the desired size. In the meantime, the validator will return
+        None.
+
+        If the LLM chunk is larger than the validator's chunking
+        strategy, it will split it into validator-sized chunks and
+        validate each one, returning an array of validation results.
+
+        Otherwise, the validator will validate the chunk and return the
+        result.
         """
         # combine accumulated chunks and new [:-1]chunk
         self.accumulated_chunks.append(chunk)
diff --git a/guardrails/validator_service.py b/guardrails/validator_service.py
index 75c72b0ca..132bbf6f3 100644
--- a/guardrails/validator_service.py
+++ b/guardrails/validator_service.py
@@ -47,7 +47,7 @@ def execute_validator(
         metadata: Optional[Dict],
         stream: Optional[bool] = False,
         **kwargs,
-    ) -> ValidationResult:
+    ) -> Optional[ValidationResult]:
         validate_func = validator.validate_stream if stream else validator.validate
         traced_validator = trace_validator(
             validator_name=validator.rail_alias,
@@ -178,24 +178,22 @@ def run_validators(
                 iteration, validator, value, metadata, property_path, stream, **kwargs
             )
             result = validator_logs.validation_result
-            if isinstance(result, FailResult):
-                if not stream:
+            if not stream:
+                if isinstance(result, FailResult):
                     value = self.perform_correction(
                         [result], value, validator, validator.on_fail_descriptor
                     )
+                elif isinstance(result, PassResult):
+                    if (
+                        validator.override_value_on_pass
+                        and result.value_override is not result.ValueOverrideSentinel
+                    ):
+                        value = result.value_override
                 else:
-                    return None, None
-            elif isinstance(result, PassResult):
-                if (
-                    validator.override_value_on_pass
-                    and result.value_override is not result.ValueOverrideSentinel
-                ):
-                    value = result.value_override
-            else:
-                raise RuntimeError(f"Unexpected result type {type(result)}")
+                    raise RuntimeError(f"Unexpected result type {type(result)}")
 
             validator_logs.value_after_validation = value
-            if result.metadata is not None:
+            if result and result.metadata is not None:
                 metadata = result.metadata
 
             if isinstance(value, (Refrain, Filter, ReAsk)):
diff --git a/tests/integration_tests/test_streaming.py b/tests/integration_tests/test_streaming.py
index cdcc6ab23..623ce4a9f 100644
--- a/tests/integration_tests/test_streaming.py
+++ b/tests/integration_tests/test_streaming.py
@@ -124,8 +124,8 @@ def gen():
         index = 0
         for chunk in chunks:
             index = index + 1
-            finished = index == len(chunks)
-            finish_reason = "stop" if finished else None
+            # finished = index == len(chunks)
+            # finish_reason = "stop" if finished else None
             # print("FINISH REASON", finish_reason)
             if OPENAI_VERSION.startswith("0"):
                 yield {
@@ -139,7 +139,7 @@ def gen():
                         Choice(
                             text=chunk,
                             delta=Delta(content=""),
-                            # TODO: for some reason using finish_reason here breaks everything
+                            # TODO: for some reason using finish_reason here breaks everything  # noqa
                             finish_reason=None,
                         )
                     ],
@@ -155,8 +155,8 @@ def gen():
         index = 0
         for chunk in chunks:
             index = index + 1
-            finished = index == len(chunks)
-            finish_reason = "stop" if finished else None
+            # finished = index == len(chunks)
+            # finish_reason = "stop" if finished else None
             # print("FINISH REASON", finish_reason)
             if OPENAI_VERSION.startswith("0"):
                 yield {
@@ -164,7 +164,7 @@ def gen():
                         {
                             "index": 0,
                             "delta": {"content": chunk},
-                            # TODO: for some reason using finish_reason here breaks everything
+                            # TODO: for some reason using finish_reason here breaks everything  # noqa
                             "finish_reason": None,
                         }
                     ]
@@ -175,7 +175,7 @@ def gen():
                         Choice(
                             text="",
                             delta=Delta(content=chunk),
-                            # TODO: for some reason using finish_reason here breaks everything
+                            # TODO: for some reason using finish_reason here breaks everything  # noqa
                             finish_reason=None,
                         )
                     ],
@@ -405,11 +405,19 @@ def test_streaming_with_openai_chat_callable(
             [
                 [
                     "This sentence is simply just too long.",
-                    "Sentence has length greater than 30. Please return a shorter output, that is shorter than 30 characters.",
+                    (
+                        "Sentence has length greater than 30. "
+                        "Please return a shorter output, "
+                        "that is shorter than 30 characters."
+                    ),
                 ],
                 [
                     "This sentence is 2 short.",
-                    "Sentence has length less than 26. Please return a longer output, that is shorter than 30 characters.",
+                    (
+                        "Sentence has length less than 26. "
+                        "Please return a longer output, "
+                        "that is shorter than 30 characters."
+                    ),
                 ],
             ],
         )
diff --git a/tests/unit_tests/test_validators.py b/tests/unit_tests/test_validators.py
index 3f6f2c8c9..e8cf1ea73 100644
--- a/tests/unit_tests/test_validators.py
+++ b/tests/unit_tests/test_validators.py
@@ -899,11 +899,11 @@ async def mock_llm_api(*args, **kwargs):
     [
         (
             OnFailAction.REASK,
-            "Prompt validation failed: incorrect_value='What kind of pet should I get?\\n\\nJson Output:\\n\\n' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='What kind')] path=None",  # noqa
-            "Instructions validation failed: incorrect_value='What kind of pet should I get?' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='What kind')] path=None",  # noqa
-            "Message history validation failed: incorrect_value='What kind of pet should I get?' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='What kind')] path=None",  # noqa
-            "Prompt validation failed: incorrect_value='\\nThis is not two words\\n\\n\\nString Output:\\n\\n' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='This is')] path=None",  # noqa
-            "Instructions validation failed: incorrect_value='\\nThis also is not two words\\n' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='This also')] path=None",  # noqa
+            "Prompt validation failed: incorrect_value='What kind of pet should I get?\\n\\nJson Output:\\n\\n' fail_results=[FailResult(outcome='fail', metadata=None, validated_chunk=None, error_message='must be exactly two words', fix_value='What kind', error_spans=None)] path=None",  # noqa
+            "Instructions validation failed: incorrect_value='What kind of pet should I get?' fail_results=[FailResult(outcome='fail', metadata=None, validated_chunk=None, error_message='must be exactly two words', fix_value='What kind', error_spans=None)] path=None",  # noqa
+            "Message history validation failed: incorrect_value='What kind of pet should I get?' fail_results=[FailResult(outcome='fail', metadata=None, validated_chunk=None, error_message='must be exactly two words', fix_value='What kind', error_spans=None)] path=None",  # noqa
+            "Prompt validation failed: incorrect_value='\\nThis is not two words\\n\\n\\nString Output:\\n\\n' fail_results=[FailResult(outcome='fail', metadata=None, validated_chunk=None, error_message='must be exactly two words', fix_value='This is', error_spans=None)] path=None",  # noqa
+            "Instructions validation failed: incorrect_value='\\nThis also is not two words\\n' fail_results=[FailResult(outcome='fail', metadata=None, validated_chunk=None, error_message='must be exactly two words', fix_value='This also', error_spans=None)] path=None",  # noqa
         ),
         (
             OnFailAction.FILTER,
@@ -1044,11 +1044,11 @@ def test_input_validation_fail(
     [
         (
             OnFailAction.REASK,
-            "Prompt validation failed: incorrect_value='What kind of pet should I get?\\n\\nJson Output:\\n\\n' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='What kind')] path=None",  # noqa
-            "Instructions validation failed: incorrect_value='What kind of pet should I get?' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='What kind')] path=None",  # noqa
-            "Message history validation failed: incorrect_value='What kind of pet should I get?' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='What kind')] path=None",  # noqa
-            "Prompt validation failed: incorrect_value='\\nThis is not two words\\n\\n\\nString Output:\\n\\n' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='This is')] path=None",  # noqa
-            "Instructions validation failed: incorrect_value='\\nThis also is not two words\\n' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='This also')] path=None",  # noqa
+            "Prompt validation failed: incorrect_value='What kind of pet should I get?\\n\\nJson Output:\\n\\n' fail_results=[FailResult(outcome='fail', metadata=None, validated_chunk=None, error_message='must be exactly two words', fix_value='What kind', error_spans=None)] path=None",  # noqa
+            "Instructions validation failed: incorrect_value='What kind of pet should I get?' fail_results=[FailResult(outcome='fail', metadata=None, validated_chunk=None, error_message='must be exactly two words', fix_value='What kind', error_spans=None)] path=None",  # noqa
+            "Message history validation failed: incorrect_value='What kind of pet should I get?' fail_results=[FailResult(outcome='fail', metadata=None, validated_chunk=None, error_message='must be exactly two words', fix_value='What kind', error_spans=None)] path=None",  # noqa
+            "Prompt validation failed: incorrect_value='\\nThis is not two words\\n\\n\\nString Output:\\n\\n' fail_results=[FailResult(outcome='fail', metadata=None, validated_chunk=None, error_message='must be exactly two words', fix_value='This is', error_spans=None)] path=None",  # noqa
+            "Instructions validation failed: incorrect_value='\\nThis also is not two words\\n' fail_results=[FailResult(outcome='fail', metadata=None, validated_chunk=None, error_message='must be exactly two words', fix_value='This also', error_spans=None)] path=None",  # noqa
         ),
         (
             OnFailAction.FILTER,

From da468bf9b5747a18f15896f3dd385b1a21c6f791 Mon Sep 17 00:00:00 2001
From: Wyatt Lansford <22553069+wylansford@users.noreply.github.com>
Date: Thu, 30 May 2024 12:11:12 -0700
Subject: [PATCH 43/85] hacky fix for returning result correctly

---
 guardrails/run/async_stream_runner.py |  4 ++--
 guardrails/validator_service.py       | 24 +++++++++++++++---------
 2 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/guardrails/run/async_stream_runner.py b/guardrails/run/async_stream_runner.py
index 2cb612c07..44ab4d680 100644
--- a/guardrails/run/async_stream_runner.py
+++ b/guardrails/run/async_stream_runner.py
@@ -264,7 +264,7 @@ async def async_step(
                     parsed_chunk,
                     output_schema,
                     validate_subschema=True,
-                    stream=True
+                    stream=True,
                 )
 
                 if isinstance(validated_result, SkeletonReAsk):
@@ -380,7 +380,7 @@ async def async_validate(
             iteration, parsed_output, self.metadata, attempt_number=index, stream=stream
         )
 
-        return validated_output
+        return iteration.validator_logs[-1].validation_result
 
     async def introspect(
         self,
diff --git a/guardrails/validator_service.py b/guardrails/validator_service.py
index 804e13884..7f9bce3b1 100644
--- a/guardrails/validator_service.py
+++ b/guardrails/validator_service.py
@@ -4,7 +4,6 @@
 from concurrent.futures import ProcessPoolExecutor
 from datetime import datetime
 from typing import Any, Dict, List, Optional, Tuple, Union
-import inspect 
 from guardrails.classes.history import Iteration
 from guardrails.datatypes import FieldValidation
 from guardrails.errors import ValidationError
@@ -133,6 +132,7 @@ def run_validator(
 
         start_time = datetime.now()
         result = self.execute_validator(validator, value, metadata, stream, **kwargs)
+
         end_time = datetime.now()
         if result is None:
             result = PassResult()
@@ -322,16 +322,26 @@ async def run_validators(
                             value,
                             metadata,
                             property_path,
-                            stream=stream
+                            stream=stream,
                         )
                     )
                 else:
                     # run the validators in the current process
                     result = self.run_validator(
-                        iteration, validator, value, metadata, property_path, stream=stream
+                        iteration,
+                        validator,
+                        value,
+                        metadata,
+                        property_path,
+                        stream=stream,
                     )
+                    try:
+                        result.validation_result = await result.validation_result
+                    except TypeError:
+                        pass
                     validators_logs.append(result)
 
+            # TODO iterate over async generator in result?
             # wait for the parallel tasks to finish
             if parallel_tasks:
                 parallel_results = await asyncio.gather(*parallel_tasks)
@@ -434,7 +444,7 @@ def validate(
                 "Async event loop found, please call `validate_async` instead."
             )
         value, metadata = loop.run_until_complete(
-            self.x(
+            self.async_validate(
                 value,
                 metadata,
                 validator_setup,
@@ -496,9 +506,5 @@ async def async_validate(
 ):
     validator_service = AsyncValidatorService(disable_tracer)
     return await validator_service.async_validate(
-        value,
-        metadata,
-        validator_setup,
-        iteration,
-        stream=stream
+        value, metadata, validator_setup, iteration, stream=stream
     )

From e985120fd62a3f4888b9dba312c957163bf70c15 Mon Sep 17 00:00:00 2001
From: Wyatt Lansford <22553069+wylansford@users.noreply.github.com>
Date: Thu, 30 May 2024 14:31:41 -0700
Subject: [PATCH 44/85] cleanup

---
 guardrails/run/async_runner.py        |  4 ++--
 guardrails/run/async_stream_runner.py | 16 +++++++---------
 guardrails/validator_service.py       |  5 +----
 3 files changed, 10 insertions(+), 15 deletions(-)

diff --git a/guardrails/run/async_runner.py b/guardrails/run/async_runner.py
index c5854eaa2..d3346e824 100644
--- a/guardrails/run/async_runner.py
+++ b/guardrails/run/async_runner.py
@@ -144,7 +144,7 @@ async def async_run(
 
         return call_log
 
-    @async_trace(name="step")
+    @async_trace(name="async_step")
     async def async_step(
         self,
         index: int,
@@ -244,7 +244,7 @@ async def async_step(
             raise e
         return iteration
 
-    @async_trace(name="call")
+    @async_trace(name="async_call")
     async def async_call(
         self,
         index: int,
diff --git a/guardrails/run/async_stream_runner.py b/guardrails/run/async_stream_runner.py
index 44ab4d680..df4a4f755 100644
--- a/guardrails/run/async_stream_runner.py
+++ b/guardrails/run/async_stream_runner.py
@@ -169,8 +169,9 @@ async def async_run(
             output=self.output,
             call_log=call_log,
         )
+        # FIXME: Where can this be moved to be less verbose? This is an await call on
+        # the async generator.
         async for call in result:
-            # yield ValidationOutcome[OT].from_guard_history(call)
             yield call
 
     # @async_trace(name="step")
@@ -250,7 +251,7 @@ async def async_step(
         if isinstance(output_schema, StringSchema):
             async for chunk in stream_output:
                 chunk_text = self.get_chunk_text(chunk, api)
-                finished = self.is_last_chunk(chunk, api)
+                _ = self.is_last_chunk(chunk, api)
                 fragment += chunk_text
 
                 parsed_chunk, move_to_next = self.parse(
@@ -370,13 +371,10 @@ async def async_validate(
         validate_subschema: bool = False,
         stream: bool = False,
     ):
-        # if validate_subschema:
-        #     validated_output = await output_schema.async_validate_subschema(
-        #         iteration, parsed_output, self.metadata, attempt_number=index
-        #     )
-        # else:
-        # TODO: What is supposed to happen here with subschema?
-        validated_output = await output_schema.async_validate(
+        # FIXME: Subschema is currently broken, it always returns a string from async
+        # streaming.
+        # Should return None/empty if fail result?
+        _ = await output_schema.async_validate(
             iteration, parsed_output, self.metadata, attempt_number=index, stream=stream
         )
 
diff --git a/guardrails/validator_service.py b/guardrails/validator_service.py
index b6051927d..6cd039359 100644
--- a/guardrails/validator_service.py
+++ b/guardrails/validator_service.py
@@ -4,6 +4,7 @@
 from concurrent.futures import ProcessPoolExecutor
 from datetime import datetime
 from typing import Any, Dict, List, Optional, Tuple, Union
+
 from guardrails.classes.history import Iteration
 from guardrails.datatypes import FieldValidation
 from guardrails.errors import ValidationError
@@ -480,10 +481,6 @@ def validate(
     else:
         validator_service = SequentialValidatorService(disable_tracer)
 
-    if stream:
-        return validator_service.validate_stream(
-            value, metadata, validator_setup, iteration, **kwargs
-        )
     return validator_service.validate(
         value,
         metadata,

From e8929d9dfc4bbb5ab33cd17ed93150788f04b2aa Mon Sep 17 00:00:00 2001
From: Wyatt Lansford <22553069+wylansford@users.noreply.github.com>
Date: Thu, 30 May 2024 14:38:32 -0700
Subject: [PATCH 45/85] fixing some invoke_llm streaming problems, adding todo
 and fixme

---
 guardrails/llm_providers.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/guardrails/llm_providers.py b/guardrails/llm_providers.py
index b92787d30..ed7755bc3 100644
--- a/guardrails/llm_providers.py
+++ b/guardrails/llm_providers.py
@@ -781,6 +781,7 @@ async def invoke_llm(
             api_key = None
 
         aclient = AsyncOpenAIClient(api_key=api_key)
+        # FIXME: OpenAI async streaming seems to be broken
         return await aclient.create_chat_completion(
             model=model,
             messages=chat_prompt(
@@ -880,6 +881,14 @@ async def invoke_llm(
             *args,
             **kwargs,
         )
+        # TODO: Check if async streaming is working here
+        if kwargs.get("stream", False):
+            # If stream is defined and set to True,
+            # the callable returns a generator object
+            return LLMResponse(
+                output="",
+                async_stream_output=manifest_response.completion_stream,
+            )
         return LLMResponse(
             output=manifest_response[0],
         )
@@ -903,6 +912,13 @@ async def invoke_llm(self, *args, **kwargs) -> LLMResponse:
         ```
         """
         output = await self.llm_api(*args, **kwargs)
+        if kwargs.get("stream", False):
+            # If stream is defined and set to True,
+            # the callable returns a generator object
+            return LLMResponse(
+                output="",
+                async_stream_output=output.completion_stream,
+            )
         return LLMResponse(
             output=output,
         )

From 0aede77e1b34fc56ee108ff3145bb829028528ac Mon Sep 17 00:00:00 2001
From: Caleb Courier <caleb.courier@gmail.com>
Date: Thu, 30 May 2024 18:01:02 -0500
Subject: [PATCH 46/85] use status for validation_passed in streaming

---
 guardrails/run/stream_runner.py |  7 +++++--
 guardrails/validator_service.py | 25 ++++++++++++-------------
 2 files changed, 17 insertions(+), 15 deletions(-)

diff --git a/guardrails/run/stream_runner.py b/guardrails/run/stream_runner.py
index 223151804..5dc842eda 100644
--- a/guardrails/run/stream_runner.py
+++ b/guardrails/run/stream_runner.py
@@ -15,6 +15,7 @@
 from guardrails.schema import Schema, StringSchema
 from guardrails.utils.openai_utils import OPENAI_VERSION
 from guardrails.utils.reask_utils import SkeletonReAsk
+from guardrails.constants import pass_status
 
 
 class StreamRunner(Runner):
@@ -198,11 +199,12 @@ def step(
                         "remove reasks from schema or disable streaming."
                     )
                 # 5. Convert validated fragment to a pretty JSON string
+                passed = call_log.status == pass_status
                 yield ValidationOutcome(
                     #  The chunk or the whole output?
                     raw_llm_output=chunk_text,
                     validated_output=validated_text,
-                    validation_passed=validated_text is not None,
+                    validation_passed=passed,
                 )
             # handle case where generator doesn't give finished status
             if not stream_finished:
@@ -216,10 +218,11 @@ def step(
                     remainder=True,
                 )
                 if len(last_result) > 0:
+                    passed = call_log.status == pass_status
                     yield ValidationOutcome(
                         raw_llm_output=last_chunk_text,
                         validated_output=last_result,
-                        validation_passed=last_result is not None,
+                        validation_passed=passed,
                     )
         # handle non string schema
         else:
diff --git a/guardrails/validator_service.py b/guardrails/validator_service.py
index 132bbf6f3..fa5e60c2e 100644
--- a/guardrails/validator_service.py
+++ b/guardrails/validator_service.py
@@ -178,19 +178,18 @@ def run_validators(
                 iteration, validator, value, metadata, property_path, stream, **kwargs
             )
             result = validator_logs.validation_result
-            if not stream:
-                if isinstance(result, FailResult):
-                    value = self.perform_correction(
-                        [result], value, validator, validator.on_fail_descriptor
-                    )
-                elif isinstance(result, PassResult):
-                    if (
-                        validator.override_value_on_pass
-                        and result.value_override is not result.ValueOverrideSentinel
-                    ):
-                        value = result.value_override
-                else:
-                    raise RuntimeError(f"Unexpected result type {type(result)}")
+            if isinstance(result, FailResult):
+                value = self.perform_correction(
+                    [result], value, validator, validator.on_fail_descriptor
+                )
+            elif isinstance(result, PassResult):
+                if (
+                    validator.override_value_on_pass
+                    and result.value_override is not result.ValueOverrideSentinel
+                ):
+                    value = result.value_override
+            elif not stream:
+                raise RuntimeError(f"Unexpected result type {type(result)}")
 
             validator_logs.value_after_validation = value
             if result and result.metadata is not None:

From 77a7eee157fef650c8360d180bffaf4b19e1028a Mon Sep 17 00:00:00 2001
From: Nicholas Chen <nichwch@gmail.com>
Date: Thu, 30 May 2024 20:50:58 -0400
Subject: [PATCH 47/85] fix error_spans_in_output

---
 guardrails/guard.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/guardrails/guard.py b/guardrails/guard.py
index 948514eda..27ac1bfd2 100644
--- a/guardrails/guard.py
+++ b/guardrails/guard.py
@@ -1146,8 +1146,8 @@ async def _async_parse(
 
     def error_spans_in_output(self):
         try:
-            call = self.history[0]
-            iter = call.iterations[0]
+            call = self.history.last
+            iter = call.iterations.last
             llm_spans = iter.error_spans_in_output
             return llm_spans
         except (AttributeError, TypeError):

From 54ccee0b639b294ee747caf20d1ecebe2bfe1d7f Mon Sep 17 00:00:00 2001
From: Nicholas Chen <nichwch@gmail.com>
Date: Thu, 30 May 2024 20:51:19 -0400
Subject: [PATCH 48/85] fix status in validationoutcomes

---
 guardrails/run/async_stream_runner.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/guardrails/run/async_stream_runner.py b/guardrails/run/async_stream_runner.py
index df4a4f755..c0803855f 100644
--- a/guardrails/run/async_stream_runner.py
+++ b/guardrails/run/async_stream_runner.py
@@ -25,6 +25,7 @@
 from guardrails.utils.openai_utils import OPENAI_VERSION
 from guardrails.utils.reask_utils import ReAsk, SkeletonReAsk
 from guardrails.utils.telemetry_utils import async_trace
+from guardrails.constants import pass_status
 
 
 class AsyncStreamRunner(StreamRunner):
@@ -282,11 +283,11 @@ async def async_step(
                         "Reasks are not yet supported with streaming. Please "
                         "remove reasks from schema or disable streaming."
                     )
-
+                passed = call_log.status == pass_status
                 yield ValidationOutcome(
                     raw_llm_output=chunk_text,
                     validated_output=validated_result,
-                    validation_passed=validated_result is not None,
+                    validation_passed=passed,
                 )
         else:
             async for chunk in stream_output:

From 1b44d99d7d0322f9d128b09ad6dc396ef271a264 Mon Sep 17 00:00:00 2001
From: Wyatt Lansford <22553069+wylansford@users.noreply.github.com>
Date: Mon, 3 Jun 2024 13:52:56 -0700
Subject: [PATCH 49/85] removing streaming support from parse call, doesn't
 conceptually make sense

---
 guardrails/async_guard.py | 65 +++++++++++----------------------------
 1 file changed, 18 insertions(+), 47 deletions(-)

diff --git a/guardrails/async_guard.py b/guardrails/async_guard.py
index fd979097c..f2dda04b3 100644
--- a/guardrails/async_guard.py
+++ b/guardrails/async_guard.py
@@ -1,11 +1,12 @@
 import contextvars
+import inspect
 from typing import (
     Any,
+    AsyncIterable,
     Awaitable,
     Callable,
     Dict,
     Iterable,
-    AsyncIterable,
     List,
     Optional,
     Union,
@@ -19,7 +20,6 @@
 from guardrails.logger import set_scope
 from guardrails.run import AsyncRunner, AsyncStreamRunner
 from guardrails.stores.context import set_call_kwargs, set_tracer, set_tracer_context
-import inspect
 
 
 class AsyncGuard(Guard):
@@ -395,14 +395,6 @@ async def __parse(
                     **kwargs,
                 )
 
-            # FIXME: checking not llm_api because it can still fall back on defaults and
-            # function as expected. We should handle this better.
-            # if (
-            #     not llm_api
-            #     or inspect.iscoroutinefunction(llm_api)
-            #     or inspect.isasyncgenfunction(llm_api)
-            # ):
-            print("calling async parse...\n\n\n")
             return await self._async_parse(
                 llm_output,
                 metadata,
@@ -458,43 +450,22 @@ async def _async_parse(
         Returns:
             The validated response.
         """
-        print("\n\n\n\n\n\n")
-        print(kwargs)
-        print("\n\n\n\n\n\n")
-        if kwargs["stream"]:
-            runner = AsyncStreamRunner(
-                instructions=kwargs.pop("instructions", None),
-                prompt=kwargs.pop("prompt", None),
-                msg_history=kwargs.pop("msg_history", None),
-                api=get_async_llm_ask(llm_api, *args, **kwargs) if llm_api else None,
-                prompt_schema=self.rail.prompt_schema,
-                instructions_schema=self.rail.instructions_schema,
-                msg_history_schema=self.rail.msg_history_schema,
-                output_schema=self.rail.output_schema,
-                num_reasks=num_reasks,
-                metadata=metadata,
-                output=llm_output,
-                base_model=self.base_model,
-                full_schema_reask=full_schema_reask,
-                disable_tracer=self._disable_tracer,
-            )
-        else:
-            runner = AsyncRunner(
-                instructions=kwargs.pop("instructions", None),
-                prompt=kwargs.pop("prompt", None),
-                msg_history=kwargs.pop("msg_history", None),
-                api=get_async_llm_ask(llm_api, *args, **kwargs) if llm_api else None,
-                prompt_schema=self.rail.prompt_schema,
-                instructions_schema=self.rail.instructions_schema,
-                msg_history_schema=self.rail.msg_history_schema,
-                output_schema=self.rail.output_schema,
-                num_reasks=num_reasks,
-                metadata=metadata,
-                output=llm_output,
-                base_model=self.base_model,
-                full_schema_reask=full_schema_reask,
-                disable_tracer=self._disable_tracer,
-            )
+        runner = AsyncRunner(
+            instructions=kwargs.pop("instructions", None),
+            prompt=kwargs.pop("prompt", None),
+            msg_history=kwargs.pop("msg_history", None),
+            api=get_async_llm_ask(llm_api, *args, **kwargs) if llm_api else None,
+            prompt_schema=self.rail.prompt_schema,
+            instructions_schema=self.rail.instructions_schema,
+            msg_history_schema=self.rail.msg_history_schema,
+            output_schema=self.rail.output_schema,
+            num_reasks=num_reasks,
+            metadata=metadata,
+            output=llm_output,
+            base_model=self.base_model,
+            full_schema_reask=full_schema_reask,
+            disable_tracer=self._disable_tracer,
+        )
         call = await runner.async_run(call_log=call_log, prompt_params=prompt_params)
 
         return ValidationOutcome[OT].from_guard_history(call)

From b7186cb4435b744b716183ee39ca835b637efa67 Mon Sep 17 00:00:00 2001
From: Wyatt Lansford <22553069+wylansford@users.noreply.github.com>
Date: Mon, 3 Jun 2024 13:57:26 -0700
Subject: [PATCH 50/85] removing unused __call__

---
 guardrails/run/async_stream_runner.py | 59 +--------------------------
 1 file changed, 1 insertion(+), 58 deletions(-)

diff --git a/guardrails/run/async_stream_runner.py b/guardrails/run/async_stream_runner.py
index c0803855f..d30ad7375 100644
--- a/guardrails/run/async_stream_runner.py
+++ b/guardrails/run/async_stream_runner.py
@@ -7,6 +7,7 @@
 from guardrails.classes.history import Call, Inputs, Iteration, Outputs
 from guardrails.classes.output_type import OT
 from guardrails.classes.validation_outcome import ValidationOutcome
+from guardrails.constants import pass_status
 from guardrails.datatypes import verify_metadata_requirements
 from guardrails.errors import ValidationError
 from guardrails.llm_providers import (
@@ -25,7 +26,6 @@
 from guardrails.utils.openai_utils import OPENAI_VERSION
 from guardrails.utils.reask_utils import ReAsk, SkeletonReAsk
 from guardrails.utils.telemetry_utils import async_trace
-from guardrails.constants import pass_status
 
 
 class AsyncStreamRunner(StreamRunner):
@@ -66,63 +66,6 @@ def __init__(
         )
         self.api: Optional[AsyncPromptCallableBase] = api
 
-    def __call__(
-        self, call_log: Call, prompt_params: Optional[Dict] = None
-    ) -> AsyncGenerator[ValidationOutcome[OT], None]:
-        """Execute the StreamRunner.
-
-        Args:
-            prompt_params: Parameters to pass to the prompt in order to
-                generate the prompt string.
-
-        Returns:
-            The Call log for this run.
-        """
-        if prompt_params is None:
-            prompt_params = {}
-
-        # check if validator requirements are fulfilled
-        missing_keys = verify_metadata_requirements(
-            self.metadata, self.output_schema.root_datatype
-        )
-        if missing_keys:
-            raise ValueError(
-                f"Missing required metadata keys: {', '.join(missing_keys)}"
-            )
-
-        (
-            instructions,
-            prompt,
-            msg_history,
-            prompt_schema,
-            instructions_schema,
-            msg_history_schema,
-            output_schema,
-        ) = (
-            self.instructions,
-            self.prompt,
-            self.msg_history,
-            self.prompt_schema,
-            self.instructions_schema,
-            self.msg_history_schema,
-            self.output_schema,
-        )
-
-        return self.async_step(
-            index=0,
-            api=self.api,
-            instructions=instructions,
-            prompt=prompt,
-            msg_history=msg_history,
-            prompt_params=prompt_params,
-            prompt_schema=prompt_schema,
-            instructions_schema=instructions_schema,
-            msg_history_schema=msg_history_schema,
-            output_schema=output_schema,
-            output=self.output,
-            call_log=call_log,
-        )
-
     async def async_run(
         self, call_log: Call, prompt_params: Optional[Dict] = None
     ) -> Call:

From 51e3b88891fd4310bee24cd74b4e1405fd94fecf Mon Sep 17 00:00:00 2001
From: Caleb Courier <caleb.courier@gmail.com>
Date: Tue, 4 Jun 2024 11:26:56 -0500
Subject: [PATCH 51/85] make nltk a required dependency

---
 poetry.lock    | 8 ++++----
 pyproject.toml | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/poetry.lock b/poetry.lock
index cf76aefb1..014178930 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -2219,7 +2219,7 @@ i18n = ["Babel (>=2.7)"]
 name = "joblib"
 version = "1.4.2"
 description = "Lightweight pipelining with Python functions"
-optional = true
+optional = false
 python-versions = ">=3.8"
 files = [
     {file = "joblib-1.4.2-py3-none-any.whl", hash = "sha256:06d478d5674cbc267e7496a410ee875abd68e4340feff4490bcb7afb88060ae6"},
@@ -3797,7 +3797,7 @@ files = [
 name = "nltk"
 version = "3.8.1"
 description = "Natural Language Toolkit"
-optional = true
+optional = false
 python-versions = ">=3.7"
 files = [
     {file = "nltk-3.8.1-py3-none-any.whl", hash = "sha256:fd5c9109f976fa86bcadba8f91e47f5e9293bd034474752e92a520f81c93dda5"},
@@ -8332,11 +8332,11 @@ manifest = ["manifest-ml"]
 pii = ["presidio_analyzer", "presidio_anonymizer"]
 profanity = ["alt-profanity-check"]
 sql = ["sqlalchemy", "sqlglot", "sqlvalidator"]
-summary = ["nltk", "thefuzz"]
+summary = ["thefuzz"]
 toxic-language = ["torch", "transformers"]
 vectordb = ["faiss-cpu", "numpy"]
 
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.8.1"
-content-hash = "6e5bf0528e7e0cb74b8bf8b9c544e162c09bd85aaad2add60c3395f1118d7196"
+content-hash = "e6ecb1b00ae6f7a00ecd840f41f4652cc54b446e3fbf36a77b3d614f7e8b0014"
diff --git a/pyproject.toml b/pyproject.toml
index 11bc11409..f174be6cf 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -26,12 +26,12 @@ rstr = "^3.2.2"
 typing-extensions = "^4.8.0"
 python-dateutil = "^2.8.2"
 tiktoken = ">=0.5.1"
+nltk = "^3.8.1"
 
 sqlvalidator = {version = "^0.0.20", optional = true}
 sqlalchemy = {version = ">=2.0.9", optional = true}
 sqlglot = {version = "^19.0.3", optional = true}
 thefuzz = {version = "^0.20.0", optional = true}
-nltk = {version = "^3.8.1", optional = true}
 faiss-cpu = {version = "^1.7.4", optional = true}
 numpy = {version = ">=1.24", optional = true}
 alt-profanity-check = {version = "^1.3.1", optional = true}
@@ -62,7 +62,7 @@ opentelemetry-exporter-otlp-proto-http = "^1.24.0"
 
 [tool.poetry.extras]
 sql = ["sqlvalidator", "sqlalchemy", "sqlglot"]
-summary = ["thefuzz", "nltk"]
+summary = ["thefuzz"]
 vectordb = ["faiss-cpu", "numpy"]
 profanity = ["alt-profanity-check"]
 detect-secrets = ["detect-secrets"]

From 5a260dd87b2f05031492d5c14ddd12dbb1011c2e Mon Sep 17 00:00:00 2001
From: Wyatt Lansford <22553069+wylansford@users.noreply.github.com>
Date: Tue, 4 Jun 2024 14:48:11 -0700
Subject: [PATCH 52/85] adding todo for async streaming from server

---
 guardrails/async_guard.py | 106 ++++++++++++++++++++++++++------------
 1 file changed, 72 insertions(+), 34 deletions(-)

diff --git a/guardrails/async_guard.py b/guardrails/async_guard.py
index f2dda04b3..0fab18bc1 100644
--- a/guardrails/async_guard.py
+++ b/guardrails/async_guard.py
@@ -10,8 +10,11 @@
     List,
     Optional,
     Union,
+    cast,
 )
 
+from guardrails_api_client.models import ValidatePayload, ValidationOutput
+
 from guardrails import Guard
 from guardrails.classes import OT, ValidationOutcome
 from guardrails.classes.history import Call
@@ -19,7 +22,12 @@
 from guardrails.llm_providers import get_async_llm_ask, model_is_supported_server_side
 from guardrails.logger import set_scope
 from guardrails.run import AsyncRunner, AsyncStreamRunner
-from guardrails.stores.context import set_call_kwargs, set_tracer, set_tracer_context
+from guardrails.stores.context import (
+    get_call_kwarg,
+    set_call_kwargs,
+    set_tracer,
+    set_tracer_context,
+)
 
 
 class AsyncGuard(Guard):
@@ -148,7 +156,7 @@ async def __call(
             if self._api_client is not None and model_is_supported_server_side(
                 llm_api, *args, **kwargs
             ):
-                return self._call_server(
+                result = self._call_server(
                     llm_api=llm_api,
                     num_reasks=self.num_reasks,
                     prompt_params=prompt_params,
@@ -157,33 +165,23 @@ async def __call(
                     *args,
                     **kwargs,
                 )
-
-            # If the LLM API is not async, fail
-            # FIXME: it seems like this check isn't actually working?
-            # if not inspect.isawaitable(llm_api) and not inspect.iscoroutinefunction(
-            #     llm_api
-            # ):
-            #     raise RuntimeError(
-            #         f"The LLM API `{llm_api.__name__}` is not a coroutine. "
-            #         "Please use an async LLM API."
-            #     )
-            # Otherwise, call the LLM
-            result = self._call_async(
-                llm_api,
-                prompt_params=prompt_params,
-                num_reasks=self.num_reasks,
-                prompt=prompt,
-                instructions=instructions,
-                msg_history=msg_history,
-                metadata=metadata,
-                full_schema_reask=full_schema_reask,
-                call_log=call_log,
-                *args,
-                **kwargs,
-            )
+            else:
+                result = self._call_async(
+                    llm_api,
+                    prompt_params=prompt_params,
+                    num_reasks=self.num_reasks,
+                    prompt=prompt,
+                    instructions=instructions,
+                    msg_history=msg_history,
+                    metadata=metadata,
+                    full_schema_reask=full_schema_reask,
+                    call_log=call_log,
+                    *args,
+                    **kwargs,
+                )
 
             if inspect.isawaitable(result):
-                result = await result
+                return await result
             return result
 
         guard_context = contextvars.Context()
@@ -407,13 +405,6 @@ async def __parse(
                 **kwargs,
             )
 
-            # else:
-            #     raise NotImplementedError(
-            #         "AsyncGuard does not support non-async LLM APIs. "
-            #         "Please use the synchronous API Guard or supply an asynchronous "
-            #         "LLM API."
-            #     )
-
         guard_context = contextvars.Context()
         return await guard_context.run(
             __parse,
@@ -469,3 +460,50 @@ async def _async_parse(
         call = await runner.async_run(call_log=call_log, prompt_params=prompt_params)
 
         return ValidationOutcome[OT].from_guard_history(call)
+
+    async def _stream_server_call(
+        self,
+        *,
+        payload: Dict[str, Any],
+        llm_output: Optional[str] = None,
+        num_reasks: Optional[int] = None,
+        prompt_params: Optional[Dict] = None,
+        metadata: Optional[Dict] = {},
+        full_schema_reask: Optional[bool] = True,
+        call_log: Optional[Call],
+    ) -> AsyncIterable[ValidationOutcome[OT]]:
+        # TODO: Once server side supports async streaming, this function will need to
+        # yield async generators, not generators
+        if self._api_client:
+            validation_output: Optional[ValidationOutput] = None
+            response = self._api_client.stream_validate(
+                guard=self,  # type: ignore
+                payload=ValidatePayload.from_dict(payload),
+                openai_api_key=get_call_kwarg("api_key"),
+            )
+            for fragment in response:
+                validation_output = fragment
+                if not validation_output:
+                    yield ValidationOutcome[OT](
+                        raw_llm_output=None,
+                        validated_output=None,
+                        validation_passed=False,
+                        error="The response from the server was empty!",
+                    )
+                yield ValidationOutcome[OT](
+                    raw_llm_output=validation_output.raw_llm_response,  # type: ignore
+                    validated_output=cast(OT, validation_output.validated_output),
+                    validation_passed=validation_output.result,
+                )
+            if validation_output:
+                self._construct_history_from_server_response(
+                    validation_output=validation_output,
+                    llm_output=llm_output,
+                    num_reasks=num_reasks,
+                    prompt_params=prompt_params,
+                    metadata=metadata,
+                    full_schema_reask=full_schema_reask,
+                    call_log=call_log,
+                )
+        else:
+            raise ValueError("Guard does not have an api client!")

From 6c1904b05aa01c8fdd960f24f559515195297d20 Mon Sep 17 00:00:00 2001
From: Wyatt Lansford <22553069+wylansford@users.noreply.github.com>
Date: Tue, 4 Jun 2024 14:51:52 -0700
Subject: [PATCH 53/85] comment

---
 guardrails/validator_service.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/guardrails/validator_service.py b/guardrails/validator_service.py
index 18f0da635..1f0026e49 100644
--- a/guardrails/validator_service.py
+++ b/guardrails/validator_service.py
@@ -333,12 +333,12 @@ async def run_validators(
                         stream=stream,
                     )
                     try:
+                        # If the result from the validator is a future, await it
                         result.validation_result = await result.validation_result
                     except TypeError:
                         pass
                     validators_logs.append(result)
 
-            # TODO iterate over async generator in result?
             # wait for the parallel tasks to finish
             if parallel_tasks:
                 parallel_results = await asyncio.gather(*parallel_tasks)

From 37fc698ae3a0356d27e26fc909006b7c3fe7f1b7 Mon Sep 17 00:00:00 2001
From: Wyatt Lansford <22553069+wylansford@users.noreply.github.com>
Date: Tue, 4 Jun 2024 14:57:14 -0700
Subject: [PATCH 54/85] ruff

---
 guardrails/schema/string_schema.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/guardrails/schema/string_schema.py b/guardrails/schema/string_schema.py
index db8a2b903..4b9e76634 100644
--- a/guardrails/schema/string_schema.py
+++ b/guardrails/schema/string_schema.py
@@ -226,7 +226,7 @@ async def async_validate(
             metadata=metadata,
             validator_setup=validation,
             iteration=iteration,
-            stream=stream
+            stream=stream,
         )
 
         validated_response = {dummy_key: validated_response}

From 7b3b2a86a63a70596b032b2ee012325dbcd6eae0 Mon Sep 17 00:00:00 2001
From: Wyatt Lansford <22553069+wylansford@users.noreply.github.com>
Date: Tue, 4 Jun 2024 15:09:28 -0700
Subject: [PATCH 55/85] typing cleanup

---
 guardrails/run/async_stream_runner.py | 16 +++++++++++++---
 guardrails/validator_service.py       |  7 ++++---
 2 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/guardrails/run/async_stream_runner.py b/guardrails/run/async_stream_runner.py
index d30ad7375..51834b25b 100644
--- a/guardrails/run/async_stream_runner.py
+++ b/guardrails/run/async_stream_runner.py
@@ -1,6 +1,16 @@
 import copy
 from functools import partial
-from typing import Any, AsyncGenerator, Dict, List, Optional, Tuple, Type, Union
+from typing import (
+    Any,
+    AsyncGenerator,
+    Dict,
+    List,
+    Optional,
+    Sequence,
+    Tuple,
+    Type,
+    Union,
+)
 
 from pydantic import BaseModel
 
@@ -68,7 +78,7 @@ def __init__(
 
     async def async_run(
         self, call_log: Call, prompt_params: Optional[Dict] = None
-    ) -> Call:
+    ) -> AsyncGenerator[ValidationOutcome]:
         if prompt_params is None:
             prompt_params = {}
 
@@ -329,7 +339,7 @@ async def introspect(
         index: int,
         validated_output: Any,
         output_schema: Schema,
-    ) -> Tuple[List[ReAsk], Any]:
+    ) -> Tuple[Sequence[ReAsk], Any]:
         # Introspect: inspect validated output for reasks.
         if validated_output is None:
             return [], None
diff --git a/guardrails/validator_service.py b/guardrails/validator_service.py
index 1f0026e49..536c1e189 100644
--- a/guardrails/validator_service.py
+++ b/guardrails/validator_service.py
@@ -319,7 +319,7 @@ async def run_validators(
                             value,
                             metadata,
                             property_path,
-                            stream=stream,
+                            stream,
                         )
                     )
                 else:
@@ -334,7 +334,8 @@ async def run_validators(
                     )
                     try:
                         # If the result from the validator is a future, await it
-                        result.validation_result = await result.validation_result
+                        if result and result.validation_result:
+                            result.validation_result = await result.validation_result
                     except TypeError:
                         pass
                     validators_logs.append(result)
@@ -493,7 +494,7 @@ async def async_validate(
     iteration: Iteration,
     disable_tracer: Optional[bool] = True,
     stream: Optional[bool] = False,
-):
+) -> Tuple[Any, dict]:
     validator_service = AsyncValidatorService(disable_tracer)
     return await validator_service.async_validate(
         value, metadata, validator_setup, iteration, stream=stream

From 11024e260cd7f80fff182f0fd0dc139d90a0f96a Mon Sep 17 00:00:00 2001
From: Wyatt Lansford <22553069+wylansford@users.noreply.github.com>
Date: Tue, 4 Jun 2024 15:38:14 -0700
Subject: [PATCH 56/85] cleanup, addressing TODO

---
 guardrails/llm_providers.py           | 8 ++------
 guardrails/run/async_stream_runner.py | 6 +++---
 2 files changed, 5 insertions(+), 9 deletions(-)

diff --git a/guardrails/llm_providers.py b/guardrails/llm_providers.py
index ed7755bc3..1d489d90b 100644
--- a/guardrails/llm_providers.py
+++ b/guardrails/llm_providers.py
@@ -881,13 +881,9 @@ async def invoke_llm(
             *args,
             **kwargs,
         )
-        # TODO: Check if async streaming is working here
         if kwargs.get("stream", False):
-            # If stream is defined and set to True,
-            # the callable returns a generator object
-            return LLMResponse(
-                output="",
-                async_stream_output=manifest_response.completion_stream,
+            raise NotImplementedError(
+                "Manifest async streaming is not yet supported by manifest."
             )
         return LLMResponse(
             output=manifest_response[0],
diff --git a/guardrails/run/async_stream_runner.py b/guardrails/run/async_stream_runner.py
index 51834b25b..27e47bfd3 100644
--- a/guardrails/run/async_stream_runner.py
+++ b/guardrails/run/async_stream_runner.py
@@ -2,7 +2,7 @@
 from functools import partial
 from typing import (
     Any,
-    AsyncGenerator,
+    AsyncIterable,
     Dict,
     List,
     Optional,
@@ -78,7 +78,7 @@ def __init__(
 
     async def async_run(
         self, call_log: Call, prompt_params: Optional[Dict] = None
-    ) -> AsyncGenerator[ValidationOutcome]:
+    ) -> AsyncIterable[ValidationOutcome]:
         if prompt_params is None:
             prompt_params = {}
 
@@ -143,7 +143,7 @@ async def async_step(
         output_schema: Schema,
         call_log: Call,
         output: Optional[str] = None,
-    ) -> AsyncGenerator[ValidationOutcome[OT], None]:
+    ) -> AsyncIterable[ValidationOutcome[OT]]:
         inputs = Inputs(
             llm_api=api,
             llm_output=output,

From 97da9af5ebe5541feb9692b516e636e0defed0eb Mon Sep 17 00:00:00 2001
From: Wyatt Lansford <22553069+wylansford@users.noreply.github.com>
Date: Tue, 4 Jun 2024 15:57:47 -0700
Subject: [PATCH 57/85] typing cleanup

---
 guardrails/run/async_stream_runner.py | 10 ++--------
 guardrails/validator_service.py       |  4 ++--
 2 files changed, 4 insertions(+), 10 deletions(-)

diff --git a/guardrails/run/async_stream_runner.py b/guardrails/run/async_stream_runner.py
index 27e47bfd3..5ef4d0150 100644
--- a/guardrails/run/async_stream_runner.py
+++ b/guardrails/run/async_stream_runner.py
@@ -187,14 +187,8 @@ async def async_step(
         try:
             stream_output = llm_response.async_stream_output
         except AttributeError:
-            try:
-                stream_output = llm_response.stream_output
-
-            except AttributeError:
-                stream_output = llm_response.stream_output
-        if stream_output is None:
             raise ValueError(
-                "No stream was returned from the API. Please check that "
+                "No async stream was returned from the API. Please check that "
                 "the API is returning an async generator."
             )
 
@@ -323,7 +317,7 @@ async def async_validate(
         parsed_output: Any,
         output_schema: Schema,
         validate_subschema: bool = False,
-        stream: bool = False,
+        stream: Optional[bool] = False,
     ):
         # FIXME: Subschema is currently broken, it always returns a string from async
         # streaming.
diff --git a/guardrails/validator_service.py b/guardrails/validator_service.py
index 536c1e189..c44d2e19d 100644
--- a/guardrails/validator_service.py
+++ b/guardrails/validator_service.py
@@ -299,7 +299,7 @@ async def run_validators(
         value: Any,
         metadata: Dict,
         property_path: str,
-        stream: bool = False,
+        stream: Optional[bool] = False,
     ):
         loop = asyncio.get_running_loop()
         for on_fail, validator_group in self.group_validators(
@@ -408,7 +408,7 @@ async def async_validate(
         validator_setup: FieldValidation,
         iteration: Iteration,
         path: str = "$",
-        stream: bool = False,
+        stream: Optional[bool] = False,
     ) -> Tuple[Any, dict]:
         property_path = (
             f"{path}.{validator_setup.key}"

From 984325f593a4ea4a835116afe8c439a56455b290 Mon Sep 17 00:00:00 2001
From: Wyatt Lansford <22553069+wylansford@users.noreply.github.com>
Date: Tue, 4 Jun 2024 16:35:16 -0700
Subject: [PATCH 58/85] more typing cleanup

---
 guardrails/guard.py                   | 9 ++++++---
 guardrails/llm_providers.py           | 2 +-
 guardrails/run/async_stream_runner.py | 7 +++----
 guardrails/validator_service.py       | 5 ++++-
 4 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/guardrails/guard.py b/guardrails/guard.py
index 27ac1bfd2..94d139eef 100644
--- a/guardrails/guard.py
+++ b/guardrails/guard.py
@@ -1147,9 +1147,12 @@ async def _async_parse(
     def error_spans_in_output(self):
         try:
             call = self.history.last
-            iter = call.iterations.last
-            llm_spans = iter.error_spans_in_output
-            return llm_spans
+            if call:
+                iter = call.iterations.last
+                llm_spans = iter.error_spans_in_output
+                return llm_spans
+            else:
+                return []
         except (AttributeError, TypeError):
             return []
 
diff --git a/guardrails/llm_providers.py b/guardrails/llm_providers.py
index 1d489d90b..a5b224330 100644
--- a/guardrails/llm_providers.py
+++ b/guardrails/llm_providers.py
@@ -838,7 +838,7 @@ async def invoke_llm(
             # response = cast(AsyncIterable[str], response)
             return LLMResponse(
                 output="",
-                async_stream_output=response.completion_stream,
+                async_stream_output=response.completion_stream,  # pyright: ignore[reportGeneralTypeIssues]
             )
 
         return LLMResponse(
diff --git a/guardrails/run/async_stream_runner.py b/guardrails/run/async_stream_runner.py
index 5ef4d0150..e73e692e0 100644
--- a/guardrails/run/async_stream_runner.py
+++ b/guardrails/run/async_stream_runner.py
@@ -184,9 +184,8 @@ async def async_step(
         llm_response = await self.async_call(
             index, instructions, prompt, msg_history, api, output
         )
-        try:
-            stream_output = llm_response.async_stream_output
-        except AttributeError:
+        stream_output = llm_response.async_stream_output
+        if not stream_output:
             raise ValueError(
                 "No async stream was returned from the API. Please check that "
                 "the API is returning an async generator."
@@ -318,7 +317,7 @@ async def async_validate(
         output_schema: Schema,
         validate_subschema: bool = False,
         stream: Optional[bool] = False,
-    ):
+    ) -> ValidationOutcome[OT]:
         # FIXME: Subschema is currently broken, it always returns a string from async
         # streaming.
         # Should return None/empty if fail result?
diff --git a/guardrails/validator_service.py b/guardrails/validator_service.py
index c44d2e19d..5382d33ea 100644
--- a/guardrails/validator_service.py
+++ b/guardrails/validator_service.py
@@ -3,7 +3,7 @@
 import os
 from concurrent.futures import ProcessPoolExecutor
 from datetime import datetime
-from typing import Any, Dict, List, Optional, Tuple, Union
+from typing import Any, Awaitable, Dict, List, Optional, Tuple, Union, cast
 
 from guardrails.classes.history import Iteration
 from guardrails.datatypes import FieldValidation
@@ -335,6 +335,9 @@ async def run_validators(
                     try:
                         # If the result from the validator is a future, await it
                         if result and result.validation_result:
+                            result.validation_result = cast(
+                                Awaitable, result.validation_result
+                            )
                             result.validation_result = await result.validation_result
                     except TypeError:
                         pass

From 1adead15793b427f332cc179ffb63a685078ce43 Mon Sep 17 00:00:00 2001
From: Wyatt Lansford <22553069+wylansford@users.noreply.github.com>
Date: Tue, 4 Jun 2024 16:45:53 -0700
Subject: [PATCH 59/85] attempt to fix typing on validation result call

---
 guardrails/validator_service.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/guardrails/validator_service.py b/guardrails/validator_service.py
index 5382d33ea..192aecfa3 100644
--- a/guardrails/validator_service.py
+++ b/guardrails/validator_service.py
@@ -3,7 +3,7 @@
 import os
 from concurrent.futures import ProcessPoolExecutor
 from datetime import datetime
-from typing import Any, Awaitable, Dict, List, Optional, Tuple, Union, cast
+from typing import Any, Awaitable, Dict, List, Optional, Tuple, Union
 
 from guardrails.classes.history import Iteration
 from guardrails.datatypes import FieldValidation
@@ -47,7 +47,7 @@ def execute_validator(
         metadata: Optional[Dict],
         stream: Optional[bool] = False,
         **kwargs,
-    ) -> Optional[ValidationResult]:
+    ) -> Optional[Union[ValidationResult, Awaitable[ValidationResult]]]:
         validate_func = validator.validate_stream if stream else validator.validate
         traced_validator = trace_validator(
             validator_name=validator.rail_alias,
@@ -120,7 +120,7 @@ def run_validator(
         property_path: str,
         stream: Optional[bool] = False,
         **kwargs,
-    ) -> ValidatorLogs:
+    ) -> Union[Awaitable, ValidatorLogs]:
         validator_class_name = validator.__class__.__name__
         validator_logs = ValidatorLogs(
             validator_name=validator_class_name,
@@ -335,9 +335,6 @@ async def run_validators(
                     try:
                         # If the result from the validator is a future, await it
                         if result and result.validation_result:
-                            result.validation_result = cast(
-                                Awaitable, result.validation_result
-                            )
                             result.validation_result = await result.validation_result
                     except TypeError:
                         pass

From fef060d16fa70d284bb14c06bdcd0da2bbe10395 Mon Sep 17 00:00:00 2001
From: Wyatt Lansford <22553069+wylansford@users.noreply.github.com>
Date: Tue, 4 Jun 2024 17:01:07 -0700
Subject: [PATCH 60/85] cleaning typing

---
 guardrails/guard.py                   | 8 ++++----
 guardrails/run/async_stream_runner.py | 8 +++-----
 2 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/guardrails/guard.py b/guardrails/guard.py
index 94d139eef..1bca53944 100644
--- a/guardrails/guard.py
+++ b/guardrails/guard.py
@@ -1149,10 +1149,10 @@ def error_spans_in_output(self):
             call = self.history.last
             if call:
                 iter = call.iterations.last
-                llm_spans = iter.error_spans_in_output
-                return llm_spans
-            else:
-                return []
+                if iter:
+                    llm_spans = iter.error_spans_in_output
+                    return llm_spans
+            return []
         except (AttributeError, TypeError):
             return []
 
diff --git a/guardrails/run/async_stream_runner.py b/guardrails/run/async_stream_runner.py
index e73e692e0..eacf0b954 100644
--- a/guardrails/run/async_stream_runner.py
+++ b/guardrails/run/async_stream_runner.py
@@ -15,7 +15,6 @@
 from pydantic import BaseModel
 
 from guardrails.classes.history import Call, Inputs, Iteration, Outputs
-from guardrails.classes.output_type import OT
 from guardrails.classes.validation_outcome import ValidationOutcome
 from guardrails.constants import pass_status
 from guardrails.datatypes import verify_metadata_requirements
@@ -143,7 +142,7 @@ async def async_step(
         output_schema: Schema,
         call_log: Call,
         output: Optional[str] = None,
-    ) -> AsyncIterable[ValidationOutcome[OT]]:
+    ) -> AsyncIterable[ValidationOutcome]:
         inputs = Inputs(
             llm_api=api,
             llm_output=output,
@@ -214,7 +213,6 @@ async def async_step(
                     validate_subschema=True,
                     stream=True,
                 )
-
                 if isinstance(validated_result, SkeletonReAsk):
                     raise ValueError(
                         "Received fragment schema is an invalid sub-schema "
@@ -269,7 +267,7 @@ async def async_step(
 
                 yield ValidationOutcome(
                     raw_llm_output=fragment,
-                    validated_output=validated_fragment,
+                    validated_output=chunk_text,
                     validation_passed=validated_fragment is not None,
                 )
 
@@ -317,7 +315,7 @@ async def async_validate(
         output_schema: Schema,
         validate_subschema: bool = False,
         stream: Optional[bool] = False,
-    ) -> ValidationOutcome[OT]:
+    ) -> ValidationOutcome:
         # FIXME: Subschema is currently broken, it always returns a string from async
         # streaming.
         # Should return None/empty if fail result?

From 39607ad6f986aa57ccc9b9e39db297b85a05168f Mon Sep 17 00:00:00 2001
From: Wyatt Lansford <22553069+wylansford@users.noreply.github.com>
Date: Tue, 4 Jun 2024 17:28:58 -0700
Subject: [PATCH 61/85] cleanup related to awaitable in base validator service

---
 guardrails/run/async_stream_runner.py | 18 +++++++++++-------
 guardrails/utils/logs_utils.py        |  6 ++++--
 guardrails/validator_service.py       | 15 ++++++++++++---
 3 files changed, 27 insertions(+), 12 deletions(-)

diff --git a/guardrails/run/async_stream_runner.py b/guardrails/run/async_stream_runner.py
index eacf0b954..de1f3ddaa 100644
--- a/guardrails/run/async_stream_runner.py
+++ b/guardrails/run/async_stream_runner.py
@@ -10,12 +10,13 @@
     Tuple,
     Type,
     Union,
+    cast,
 )
 
 from pydantic import BaseModel
 
+from guardrails.classes import ValidationOutcome
 from guardrails.classes.history import Call, Inputs, Iteration, Outputs
-from guardrails.classes.validation_outcome import ValidationOutcome
 from guardrails.constants import pass_status
 from guardrails.datatypes import verify_metadata_requirements
 from guardrails.errors import ValidationError
@@ -35,6 +36,7 @@
 from guardrails.utils.openai_utils import OPENAI_VERSION
 from guardrails.utils.reask_utils import ReAsk, SkeletonReAsk
 from guardrails.utils.telemetry_utils import async_trace
+from guardrails.validator_base import ValidationResult
 
 
 class AsyncStreamRunner(StreamRunner):
@@ -205,7 +207,7 @@ async def async_step(
                 )
                 if move_to_next:
                     continue
-                validated_result = await self.async_validate(
+                validated_fragment = await self.async_validate(
                     iteration,
                     index,
                     parsed_chunk,
@@ -213,14 +215,14 @@ async def async_step(
                     validate_subschema=True,
                     stream=True,
                 )
-                if isinstance(validated_result, SkeletonReAsk):
+                if isinstance(validated_fragment, SkeletonReAsk):
                     raise ValueError(
                         "Received fragment schema is an invalid sub-schema "
                         "of the expected output JSON schema."
                     )
 
                 reasks, valid_op = await self.introspect(
-                    index, validated_result, output_schema
+                    index, validated_fragment, output_schema
                 )
                 if reasks:
                     raise ValueError(
@@ -230,7 +232,7 @@ async def async_step(
                 passed = call_log.status == pass_status
                 yield ValidationOutcome(
                     raw_llm_output=chunk_text,
-                    validated_output=validated_result,
+                    validated_output=validated_fragment,
                     validation_passed=passed,
                 )
         else:
@@ -273,8 +275,10 @@ async def async_step(
 
         iteration.outputs.raw_output = fragment
         iteration.outputs.parsed_output = parsed_fragment
-        iteration.outputs.validation_response = validated_fragment
         iteration.outputs.guarded_output = valid_op
+        iteration.outputs.validation_response = (
+            cast(str, validated_fragment) if validated_fragment else None
+        )
 
     @async_trace(name="call")
     async def async_call(
@@ -315,7 +319,7 @@ async def async_validate(
         output_schema: Schema,
         validate_subschema: bool = False,
         stream: Optional[bool] = False,
-    ) -> ValidationOutcome:
+    ) -> Optional[ValidationResult]:
         # FIXME: Subschema is currently broken, it always returns a string from async
         # streaming.
         # Should return None/empty if fail result?
diff --git a/guardrails/utils/logs_utils.py b/guardrails/utils/logs_utils.py
index 9ffe3e6e3..07267f330 100644
--- a/guardrails/utils/logs_utils.py
+++ b/guardrails/utils/logs_utils.py
@@ -1,6 +1,6 @@
 from copy import deepcopy
 from datetime import datetime
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Union, Awaitable
 
 from guardrails.utils.pydantic_utils import ArbitraryModel
 from guardrails.utils.reask_utils import FieldReAsk, ReAsk, prune_obj_for_reasking
@@ -13,7 +13,9 @@ class ValidatorLogs(ArbitraryModel):
     validator_name: str
     registered_name: str
     value_before_validation: Any
-    validation_result: Optional[ValidationResult] = None
+    validation_result: Optional[
+        Union[Awaitable[ValidationResult], ValidationResult]
+    ] = None
     value_after_validation: Optional[Any] = None
     start_time: Optional[datetime] = None
     end_time: Optional[datetime] = None
diff --git a/guardrails/validator_service.py b/guardrails/validator_service.py
index 192aecfa3..046c36cc1 100644
--- a/guardrails/validator_service.py
+++ b/guardrails/validator_service.py
@@ -3,7 +3,7 @@
 import os
 from concurrent.futures import ProcessPoolExecutor
 from datetime import datetime
-from typing import Any, Awaitable, Dict, List, Optional, Tuple, Union
+from typing import Any, Awaitable, Dict, List, Optional, Tuple, Union, cast
 
 from guardrails.classes.history import Iteration
 from guardrails.datatypes import FieldValidation
@@ -120,7 +120,7 @@ def run_validator(
         property_path: str,
         stream: Optional[bool] = False,
         **kwargs,
-    ) -> Union[Awaitable, ValidatorLogs]:
+    ) -> ValidatorLogs:
         validator_class_name = validator.__class__.__name__
         validator_logs = ValidatorLogs(
             validator_name=validator_class_name,
@@ -153,7 +153,12 @@ def run_validator(
                 attributes=[
                     ("validator_name", validator.rail_alias),
                     ("validator_on_fail", validator.on_fail_descriptor),
-                    ("validator_result", result.outcome),
+                    (
+                        "validator_result",
+                        result.outcome
+                        if isinstance(result, ValidationResult)
+                        else None,
+                    ),
                 ],
                 is_parent=False,  # This span will have no children
                 has_parent=True,  # This span has a parent
@@ -179,6 +184,7 @@ def run_validators(
                 iteration, validator, value, metadata, property_path, stream, **kwargs
             )
             result = validator_logs.validation_result
+            result = cast(ValidationResult, result)
             if isinstance(result, FailResult):
                 value = self.perform_correction(
                     [result], value, validator, validator.on_fail_descriptor
@@ -335,6 +341,9 @@ async def run_validators(
                     try:
                         # If the result from the validator is a future, await it
                         if result and result.validation_result:
+                            result.validation_result = cast(
+                                Awaitable[ValidationResult], result.validation_result
+                            )
                             result.validation_result = await result.validation_result
                     except TypeError:
                         pass

From f81ed6c62a402eedcfa5d7dd87cae364458ab1b2 Mon Sep 17 00:00:00 2001
From: Wyatt Lansford <22553069+wylansford@users.noreply.github.com>
Date: Tue, 4 Jun 2024 17:38:11 -0700
Subject: [PATCH 62/85] final cleanup

---
 guardrails/classes/history/outputs.py | 9 ++++++---
 guardrails/run/async_stream_runner.py | 3 ++-
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/guardrails/classes/history/outputs.py b/guardrails/classes/history/outputs.py
index 953ed6f4b..ead39de20 100644
--- a/guardrails/classes/history/outputs.py
+++ b/guardrails/classes/history/outputs.py
@@ -1,4 +1,5 @@
 from typing import Dict, List, Optional, Sequence, Union
+
 from pydantic import Field
 from typing_extensions import deprecated
 
@@ -7,7 +8,7 @@
 from guardrails.utils.logs_utils import ValidatorLogs
 from guardrails.utils.pydantic_utils import ArbitraryModel
 from guardrails.utils.reask_utils import ReAsk
-from guardrails.validator_base import ErrorSpan, FailResult
+from guardrails.validator_base import ErrorSpan, FailResult, ValidationResult
 
 
 class Outputs(ArbitraryModel):
@@ -70,6 +71,7 @@ def failed_validations(self) -> List[ValidatorLogs]:
                 log
                 for log in self.validator_logs
                 if log.validation_result is not None
+                and isinstance(log.validation_result, ValidationResult)
                 and log.validation_result.outcome == "fail"
             ]
         )
@@ -94,8 +96,9 @@ def error_spans_in_output(self) -> List[ErrorSpan]:
                                 reason=error_span.reason,
                             )
                         )
-            if result and result.validated_chunk is not None:
-                total_len += len(result.validated_chunk)
+            if isinstance(result, ValidationResult):
+                if result and result.validated_chunk is not None:
+                    total_len += len(result.validated_chunk)
         return spans_in_output
 
     @property
diff --git a/guardrails/run/async_stream_runner.py b/guardrails/run/async_stream_runner.py
index de1f3ddaa..b2c0c7135 100644
--- a/guardrails/run/async_stream_runner.py
+++ b/guardrails/run/async_stream_runner.py
@@ -11,6 +11,7 @@
     Type,
     Union,
     cast,
+    Awaitable,
 )
 
 from pydantic import BaseModel
@@ -319,7 +320,7 @@ async def async_validate(
         output_schema: Schema,
         validate_subschema: bool = False,
         stream: Optional[bool] = False,
-    ) -> Optional[ValidationResult]:
+    ) -> Optional[Union[Awaitable[ValidationResult], ValidationResult]]:
         # FIXME: Subschema is currently broken, it always returns a string from async
         # streaming.
         # Should return None/empty if fail result?

From 6fa82f1529b47adf35d45031d1c3828933a3bd73 Mon Sep 17 00:00:00 2001
From: Wyatt Lansford <22553069+wylansford@users.noreply.github.com>
Date: Tue, 4 Jun 2024 17:41:52 -0700
Subject: [PATCH 63/85] final cleanup, again

---
 guardrails/classes/history/call.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/guardrails/classes/history/call.py b/guardrails/classes/history/call.py
index 503cefb9a..6598e3c12 100644
--- a/guardrails/classes/history/call.py
+++ b/guardrails/classes/history/call.py
@@ -20,7 +20,7 @@
     sub_reasks_with_fixed_values,
 )
 from guardrails.utils.safe_get import get_value_from_path
-from guardrails.validator_base import Filter, Refrain
+from guardrails.validator_base import Filter, Refrain, ValidationResult
 
 
 # We can't inherit from Iteration because python
@@ -353,6 +353,7 @@ def failed_validations(self) -> Stack[ValidatorLogs]:
                 log
                 for log in self.validator_logs
                 if log.validation_result is not None
+                and isinstance(log.validation_result, ValidationResult)
                 and log.validation_result.outcome == "fail"
             ]
         )

From cbfc5698d96fb82ee2b58a17ad8dc17e20d66a53 Mon Sep 17 00:00:00 2001
From: Wyatt Lansford <22553069+wylansford@users.noreply.github.com>
Date: Wed, 5 Jun 2024 13:23:17 -0700
Subject: [PATCH 64/85] async_call to call in trace

---
 guardrails/run/async_runner.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/guardrails/run/async_runner.py b/guardrails/run/async_runner.py
index d3346e824..45bb2bd74 100644
--- a/guardrails/run/async_runner.py
+++ b/guardrails/run/async_runner.py
@@ -244,7 +244,7 @@ async def async_step(
             raise e
         return iteration
 
-    @async_trace(name="async_call")
+    @async_trace(name="call")
     async def async_call(
         self,
         index: int,

From 8d275e74417b6ab7987d2d933a23a9eb47efaa30 Mon Sep 17 00:00:00 2001
From: Wyatt Lansford <22553069+wylansford@users.noreply.github.com>
Date: Wed, 5 Jun 2024 13:41:02 -0700
Subject: [PATCH 65/85] fix missing arg in mock tests

---
 tests/unit_tests/test_async_validator_service.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/tests/unit_tests/test_async_validator_service.py b/tests/unit_tests/test_async_validator_service.py
index 34431fe14..8e314c095 100644
--- a/tests/unit_tests/test_async_validator_service.py
+++ b/tests/unit_tests/test_async_validator_service.py
@@ -91,7 +91,7 @@ async def test_async_validate_with_children(mocker):
 
     assert run_validators_mock.call_count == 1
     run_validators_mock.assert_called_once_with(
-        iteration, field_validation, True, {}, "$.mock-parent-key"
+        iteration, field_validation, True, {}, "$.mock-parent-key", stream=False
     )
 
     assert validated_value == "run_validators_mock"
@@ -118,7 +118,7 @@ async def test_async_validate_without_children(mocker):
 
     assert run_validators_mock.call_count == 1
     run_validators_mock.assert_called_once_with(
-        iteration, empty_field_validation, True, {}, "$.mock-key"
+        iteration, empty_field_validation, True, {}, "$.mock-key", stream=False
     )
 
     assert validated_value == "run_validators_mock"
@@ -186,7 +186,9 @@ async def test_run_validators(mocker):
         (OnFailAction.NOOP, [noop_validator_1, noop_validator_2]),
     ]
 
-    def mock_run_validator(iteration, validator, value, metadata, property_path):
+    def mock_run_validator(
+        iteration, validator, value, metadata, property_path, stream
+    ):
         return ValidatorLogs(
             registered_name=validator.name,
             validator_name=validator.name,
@@ -234,6 +236,7 @@ async def mock_gather(*args):
         empty_field_validation.value,
         {},
         "$",
+        False,
     )
 
     assert run_validator_mock.call_count == 3

From 9166830695f011103b9e6bc03b92ed37c6f7342d Mon Sep 17 00:00:00 2001
From: Nicholas Chen <nichwch@gmail.com>
Date: Wed, 5 Jun 2024 16:53:50 -0400
Subject: [PATCH 66/85] directly emit PassResult when not finished accumulating

---
 guardrails/validator_base.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/guardrails/validator_base.py b/guardrails/validator_base.py
index 6fb1cc9df..2287c558f 100644
--- a/guardrails/validator_base.py
+++ b/guardrails/validator_base.py
@@ -522,7 +522,7 @@ def validate_stream(
         if remainder:
             splitcontents = [accumulated_text, ""]
         if len(splitcontents) == 0:
-            return None
+            return PassResult()
         [chunk_to_validate, new_accumulated_chunks] = splitcontents
         self.accumulated_chunks = [new_accumulated_chunks]
         # exclude last chunk, because it may not be a complete chunk

From e1cc48e72cd5c0ef2c8333b5ed1c783f1fb9ad6f Mon Sep 17 00:00:00 2001
From: Wyatt Lansford <22553069+wylansford@users.noreply.github.com>
Date: Wed, 5 Jun 2024 13:58:08 -0700
Subject: [PATCH 67/85] removing kwarg for stream, replacing with positional
 arg

---
 guardrails/validator_service.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/guardrails/validator_service.py b/guardrails/validator_service.py
index 046c36cc1..71955e8b5 100644
--- a/guardrails/validator_service.py
+++ b/guardrails/validator_service.py
@@ -506,5 +506,5 @@ async def async_validate(
 ) -> Tuple[Any, dict]:
     validator_service = AsyncValidatorService(disable_tracer)
     return await validator_service.async_validate(
-        value, metadata, validator_setup, iteration, stream=stream
+        value, metadata, validator_setup, iteration, None, stream
     )

From a71645cc100867591dc1ebdaea47a0707a0fd688 Mon Sep 17 00:00:00 2001
From: Wyatt Lansford <22553069+wylansford@users.noreply.github.com>
Date: Wed, 5 Jun 2024 14:02:38 -0700
Subject: [PATCH 68/85] moving positional args

---
 guardrails/validator_service.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/guardrails/validator_service.py b/guardrails/validator_service.py
index 71955e8b5..6d73d54ab 100644
--- a/guardrails/validator_service.py
+++ b/guardrails/validator_service.py
@@ -416,8 +416,8 @@ async def async_validate(
         metadata: dict,
         validator_setup: FieldValidation,
         iteration: Iteration,
-        path: str = "$",
         stream: Optional[bool] = False,
+        path: str = "$",
     ) -> Tuple[Any, dict]:
         property_path = (
             f"{path}.{validator_setup.key}"
@@ -506,5 +506,5 @@ async def async_validate(
 ) -> Tuple[Any, dict]:
     validator_service = AsyncValidatorService(disable_tracer)
     return await validator_service.async_validate(
-        value, metadata, validator_setup, iteration, None, stream
+        value, metadata, validator_setup, iteration, stream
     )

From 5b823142e65f42138690e1b7e0d0c4584792b38d Mon Sep 17 00:00:00 2001
From: Wyatt Lansford <22553069+wylansford@users.noreply.github.com>
Date: Wed, 5 Jun 2024 14:11:39 -0700
Subject: [PATCH 69/85] reverting, adding arg to async_validate call

---
 guardrails/validator_service.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/guardrails/validator_service.py b/guardrails/validator_service.py
index 6d73d54ab..01e20de72 100644
--- a/guardrails/validator_service.py
+++ b/guardrails/validator_service.py
@@ -416,8 +416,8 @@ async def async_validate(
         metadata: dict,
         validator_setup: FieldValidation,
         iteration: Iteration,
-        stream: Optional[bool] = False,
         path: str = "$",
+        stream: Optional[bool] = False,
     ) -> Tuple[Any, dict]:
         property_path = (
             f"{path}.{validator_setup.key}"
@@ -506,5 +506,5 @@ async def async_validate(
 ) -> Tuple[Any, dict]:
     validator_service = AsyncValidatorService(disable_tracer)
     return await validator_service.async_validate(
-        value, metadata, validator_setup, iteration, stream
+        value, metadata, validator_setup, iteration, "$", stream
     )

From d3c716a540092567d9d00a1fd5ed9b6d0c6c3427 Mon Sep 17 00:00:00 2001
From: Nicholas Chen <nichwch@gmail.com>
Date: Wed, 5 Jun 2024 17:27:23 -0400
Subject: [PATCH 70/85] filter logs in iteration.py

---
 guardrails/classes/history/iteration.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/guardrails/classes/history/iteration.py b/guardrails/classes/history/iteration.py
index ef913bafd..2c666e2bf 100644
--- a/guardrails/classes/history/iteration.py
+++ b/guardrails/classes/history/iteration.py
@@ -137,7 +137,12 @@ def reasks(self) -> Sequence[ReAsk]:
     def validator_logs(self) -> List[ValidatorLogs]:
         """The results of each individual validation performed on the LLM
         response during this iteration."""
-        return self.outputs.validator_logs
+        filtered_logs = [
+            log
+            for log in self.outputs.validator_logs
+            if log.validation_result.validated_chunk
+        ]
+        return filtered_logs
 
     @property
     def error(self) -> Optional[str]:

From 0a443eda56508ee1b923573af7f904542e9958fb Mon Sep 17 00:00:00 2001
From: Nicholas Chen <nichwch@gmail.com>
Date: Wed, 5 Jun 2024 17:29:47 -0400
Subject: [PATCH 71/85] add docstrings for sentence splitters

---
 guardrails/validator_base.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/guardrails/validator_base.py b/guardrails/validator_base.py
index 2287c558f..dec45ed05 100644
--- a/guardrails/validator_base.py
+++ b/guardrails/validator_base.py
@@ -180,6 +180,9 @@ class Refrain:
 
 
 def split_sentence_str(chunk: str):
+    """
+    A naive sentence splitter that splits on periods.
+    """
     if "." not in chunk:
         return []
     fragments = chunk.split(".")
@@ -187,6 +190,13 @@ def split_sentence_str(chunk: str):
 
 
 def split_sentence_nltk(chunk: str):
+    """
+    NOTE: this approach currently does not work
+    Use a sentence tokenizer to split the chunk into sentences.
+
+    Because using the tokenizer is expensive, we only use it if there
+    is a period present in the chunk.
+    """
     # using the sentence tokenizer is expensive
     # we check for a . to avoid wastefully calling the tokenizer
     if "." not in chunk:

From 79ddc1705afef03800d0550a907216646cad0e48 Mon Sep 17 00:00:00 2001
From: Caleb Courier <caleb.courier@gmail.com>
Date: Wed, 5 Jun 2024 18:10:18 -0500
Subject: [PATCH 72/85] poorly handle async validators

---
 guardrails/classes/history/iteration.py |   2 +-
 guardrails/run/async_stream_runner.py   |   2 +-
 guardrails/utils/logs_utils.py          |   6 +-
 guardrails/validator_base.py            |   4 +-
 guardrails/validator_service.py         | 180 ++++++++++++++++++++----
 5 files changed, 155 insertions(+), 39 deletions(-)

diff --git a/guardrails/classes/history/iteration.py b/guardrails/classes/history/iteration.py
index 2c666e2bf..1f6e71657 100644
--- a/guardrails/classes/history/iteration.py
+++ b/guardrails/classes/history/iteration.py
@@ -140,7 +140,7 @@ def validator_logs(self) -> List[ValidatorLogs]:
         filtered_logs = [
             log
             for log in self.outputs.validator_logs
-            if log.validation_result.validated_chunk
+            if log.validation_result and log.validation_result.validated_chunk
         ]
         return filtered_logs
 
diff --git a/guardrails/run/async_stream_runner.py b/guardrails/run/async_stream_runner.py
index b2c0c7135..ac721e41b 100644
--- a/guardrails/run/async_stream_runner.py
+++ b/guardrails/run/async_stream_runner.py
@@ -489,7 +489,7 @@ def get_chunk_text(self, chunk: Any, api: Union[PromptCallableBase, None]) -> st
         return chunk_text
 
     def is_last_chunk(self, chunk: Any, api: Union[PromptCallableBase, None]) -> bool:
-        """Detect if chunk is final chunk"""
+        """Detect if chunk is final chunk."""
         if isinstance(api, OpenAICallable):
             if OPENAI_VERSION.startswith("0"):
                 finished = chunk["choices"][0]["finish_reason"]
diff --git a/guardrails/utils/logs_utils.py b/guardrails/utils/logs_utils.py
index 07267f330..9ffe3e6e3 100644
--- a/guardrails/utils/logs_utils.py
+++ b/guardrails/utils/logs_utils.py
@@ -1,6 +1,6 @@
 from copy import deepcopy
 from datetime import datetime
-from typing import Any, Dict, List, Optional, Union, Awaitable
+from typing import Any, Dict, List, Optional
 
 from guardrails.utils.pydantic_utils import ArbitraryModel
 from guardrails.utils.reask_utils import FieldReAsk, ReAsk, prune_obj_for_reasking
@@ -13,9 +13,7 @@ class ValidatorLogs(ArbitraryModel):
     validator_name: str
     registered_name: str
     value_before_validation: Any
-    validation_result: Optional[
-        Union[Awaitable[ValidationResult], ValidationResult]
-    ] = None
+    validation_result: Optional[ValidationResult] = None
     value_after_validation: Optional[Any] = None
     start_time: Optional[datetime] = None
     end_time: Optional[datetime] = None
diff --git a/guardrails/validator_base.py b/guardrails/validator_base.py
index dec45ed05..7347b6111 100644
--- a/guardrails/validator_base.py
+++ b/guardrails/validator_base.py
@@ -180,9 +180,7 @@ class Refrain:
 
 
 def split_sentence_str(chunk: str):
-    """
-    A naive sentence splitter that splits on periods.
-    """
+    """A naive sentence splitter that splits on periods."""
     if "." not in chunk:
         return []
     fragments = chunk.split(".")
diff --git a/guardrails/validator_service.py b/guardrails/validator_service.py
index 01e20de72..91cdbb93e 100644
--- a/guardrails/validator_service.py
+++ b/guardrails/validator_service.py
@@ -8,6 +8,7 @@
 from guardrails.classes.history import Iteration
 from guardrails.datatypes import FieldValidation
 from guardrails.errors import ValidationError
+from guardrails.utils.exception_utils import UserFacingException
 from guardrails.utils.hub_telemetry_utils import HubTelemetry
 from guardrails.utils.logs_utils import ValidatorLogs
 from guardrails.utils.reask_utils import FieldReAsk, ReAsk
@@ -23,6 +24,8 @@
     Validator,
 )
 
+ValidatorResult = Optional[Union[ValidationResult, Awaitable[ValidationResult]]]
+
 
 def key_not_empty(key: str) -> bool:
     return key is not None and len(str(key)) > 0
@@ -47,7 +50,7 @@ def execute_validator(
         metadata: Optional[Dict],
         stream: Optional[bool] = False,
         **kwargs,
-    ) -> Optional[Union[ValidationResult, Awaitable[ValidationResult]]]:
+    ) -> ValidatorResult:
         validate_func = validator.validate_stream if stream else validator.validate
         traced_validator = trace_validator(
             validator_name=validator.rail_alias,
@@ -66,6 +69,7 @@ def perform_correction(
         value: Any,
         validator: Validator,
         on_fail_descriptor: Union[OnFailAction, str],
+        rechecked_value: Optional[ValidationResult] = None,
     ):
         if on_fail_descriptor == OnFailAction.FIX:
             # FIXME: Should we still return fix_value if it is None?
@@ -74,11 +78,11 @@ def perform_correction(
         elif on_fail_descriptor == OnFailAction.FIX_REASK:
             # FIXME: Same thing here
             fixed_value = results[0].fix_value
-            result = self.execute_validator(
-                validator, fixed_value, results[0].metadata or {}
-            )
+            # result = self.execute_validator(
+            #     validator, fixed_value, results[0].metadata or {}
+            # )
 
-            if isinstance(result, FailResult):
+            if isinstance(rechecked_value, FailResult):
                 return FieldReAsk(
                     incorrect_value=fixed_value,
                     fail_results=results,
@@ -111,15 +115,12 @@ def perform_correction(
                 f"expected 'fix' or 'exception'."
             )
 
-    def run_validator(
+    def before_run_validator(
         self,
         iteration: Iteration,
         validator: Validator,
         value: Any,
-        metadata: Dict,
         property_path: str,
-        stream: Optional[bool] = False,
-        **kwargs,
     ) -> ValidatorLogs:
         validator_class_name = validator.__class__.__name__
         validator_logs = ValidatorLogs(
@@ -127,22 +128,26 @@ def run_validator(
             value_before_validation=value,
             registered_name=validator.rail_alias,
             property_path=property_path,
+            # If we ever re-use validator instances across multiple properties,
+            #   this will have to change.
+            instance_id=id(validator),
         )
         iteration.outputs.validator_logs.append(validator_logs)
 
         start_time = datetime.now()
-        result = self.execute_validator(validator, value, metadata, stream, **kwargs)
+        validator_logs.start_time = start_time
 
-        end_time = datetime.now()
-        if result is None:
-            result = PassResult()
+        return validator_logs
 
+    def after_run_validator(
+        self,
+        validator: Validator,
+        validator_logs: ValidatorLogs,
+        result: ValidationResult,
+    ):
+        end_time = datetime.now()
         validator_logs.validation_result = result
-        validator_logs.start_time = start_time
         validator_logs.end_time = end_time
-        # If we ever re-use validator instances across multiple properties,
-        #   this will have to change.
-        validator_logs.instance_id = id(validator)
 
         if not self._disable_tracer:
             # Get HubTelemetry singleton and create a new span to
@@ -166,8 +171,61 @@ def run_validator(
 
         return validator_logs
 
+    def run_validator(
+        self,
+        iteration: Iteration,
+        validator: Validator,
+        value: Any,
+        metadata: Dict,
+        property_path: str,
+        stream: Optional[bool] = False,
+        **kwargs,
+    ) -> ValidatorLogs:
+        raise NotImplementedError
+
 
 class SequentialValidatorService(ValidatorServiceBase):
+    def run_validator_sync(
+        self,
+        validator: Validator,
+        value: Any,
+        metadata: Dict,
+        validator_logs: ValidatorLogs,
+        stream: Optional[bool] = False,
+        **kwargs,
+    ) -> ValidationResult:
+        result = self.execute_validator(validator, value, metadata, stream, **kwargs)
+        if asyncio.iscoroutine(result):
+            raise UserFacingException(
+                ValueError(
+                    "Cannot use async validators with a synchronous Guard! "
+                    f"Either use AsyncGuard or remove {validator_logs.validator_name}."
+                )
+            )
+        elif result is None:
+            result = PassResult()
+        return cast(ValidationResult, result)
+
+    def run_validator(
+        self,
+        iteration: Iteration,
+        validator: Validator,
+        value: Any,
+        metadata: Dict,
+        property_path: str,
+        stream: Optional[bool] = False,
+        **kwargs,
+    ) -> ValidatorLogs:
+        validator_logs = self.before_run_validator(
+            iteration, validator, value, property_path
+        )
+
+        result = self.run_validator_sync(
+            validator, value, metadata, validator_logs, stream, **kwargs
+        )
+
+        return self.after_run_validator(validator, validator_logs, result)
+
     def run_validators(
         self,
         iteration: Iteration,
@@ -186,8 +244,23 @@ def run_validators(
             result = validator_logs.validation_result
             result = cast(ValidationResult, result)
             if isinstance(result, FailResult):
+                rechecked_value = None
+                if validator.on_fail_descriptor == OnFailAction.FIX_REASK:
+                    fixed_value = result.fix_value
+                    rechecked_value = self.run_validator_sync(
+                        validator,
+                        fixed_value,
+                        metadata,
+                        validator_logs,
+                        stream,
+                        **kwargs,
+                    )
                 value = self.perform_correction(
-                    [result], value, validator, validator.on_fail_descriptor
+                    [result],
+                    value,
+                    validator,
+                    validator.on_fail_descriptor,
+                    rechecked_value=rechecked_value,
                 )
             elif isinstance(result, PassResult):
                 if (
@@ -283,6 +356,46 @@ def __init__(self):
 
 
 class AsyncValidatorService(ValidatorServiceBase, MultiprocMixin):
+    async def run_validator_async(
+        self,
+        validator: Validator,
+        value: Any,
+        metadata: Dict,
+        stream: Optional[bool] = False,
+        **kwargs,
+    ) -> ValidationResult:
+        result: ValidatorResult = self.execute_validator(
+            validator, value, metadata, stream, **kwargs
+        )
+        if asyncio.iscoroutine(result):
+            result = await result
+
+        if result is None:
+            result = PassResult()
+        else:
+            result = cast(ValidationResult, result)
+        return result
+
+    async def run_validator(
+        self,
+        iteration: Iteration,
+        validator: Validator,
+        value: Any,
+        metadata: Dict,
+        property_path: str,
+        stream: Optional[bool] = False,
+        **kwargs,
+    ) -> ValidatorLogs:
+        validator_logs = self.before_run_validator(
+            iteration, validator, value, property_path
+        )
+
+        result = await self.run_validator_async(
+            validator, value, metadata, stream, **kwargs
+        )
+
+        return self.after_run_validator(validator, validator_logs, result)
+
     def group_validators(self, validators):
         groups = itertools.groupby(
             validators, key=lambda v: (v.on_fail_descriptor, v.override_value_on_pass)
@@ -330,7 +443,7 @@ async def run_validators(
                     )
                 else:
                     # run the validators in the current process
-                    result = self.run_validator(
+                    result = await self.run_validator(
                         iteration,
                         validator,
                         value,
@@ -338,21 +451,17 @@ async def run_validators(
                         property_path,
                         stream=stream,
                     )
-                    try:
-                        # If the result from the validator is a future, await it
-                        if result and result.validation_result:
-                            result.validation_result = cast(
-                                Awaitable[ValidationResult], result.validation_result
-                            )
-                            result.validation_result = await result.validation_result
-                    except TypeError:
-                        pass
                     validators_logs.append(result)
 
             # wait for the parallel tasks to finish
             if parallel_tasks:
                 parallel_results = await asyncio.gather(*parallel_tasks)
-                validators_logs.extend(parallel_results)
+                awaited_results = []
+                for res in parallel_results:
+                    if asyncio.iscoroutine(res):
+                        res = await res
+                    awaited_results.append(res)
+                validators_logs.extend(awaited_results)
 
             # process the results, handle failures
             fails = [
@@ -362,8 +471,19 @@ async def run_validators(
             ]
             if fails:
                 fail_results = [logs.validation_result for logs in fails]
+                rechecked_value = None
+                validator: Validator = validator_group[0]
+                if validator.on_fail_descriptor == OnFailAction.FIX_REASK:
+                    fixed_value = fail_results[0].fix_value
+                    rechecked_value = await self.run_validator_async(
+                        validator, fixed_value, fail_results[0].metadata or {}, stream
+                    )
                 value = self.perform_correction(
-                    fail_results, value, validator_group[0], on_fail
+                    fail_results,
+                    value,
+                    validator_group[0],
+                    on_fail,
+                    rechecked_value=rechecked_value,
                 )
 
             # handle overrides

From bb6ca031859710b5f5045e5db76163719e71794e Mon Sep 17 00:00:00 2001
From: Caleb Courier <caleb.courier@gmail.com>
Date: Wed, 5 Jun 2024 18:11:26 -0500
Subject: [PATCH 73/85] clean up commented code

---
 guardrails/validator_service.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/guardrails/validator_service.py b/guardrails/validator_service.py
index 91cdbb93e..070dbe230 100644
--- a/guardrails/validator_service.py
+++ b/guardrails/validator_service.py
@@ -78,9 +78,6 @@ def perform_correction(
         elif on_fail_descriptor == OnFailAction.FIX_REASK:
             # FIXME: Same thing here
             fixed_value = results[0].fix_value
-            # result = self.execute_validator(
-            #     validator, fixed_value, results[0].metadata or {}
-            # )
 
             if isinstance(rechecked_value, FailResult):
                 return FieldReAsk(

From df7d71a9c370737a541711fcd3e78d0c6cffb080 Mon Sep 17 00:00:00 2001
From: Wyatt Lansford <22553069+wylansford@users.noreply.github.com>
Date: Wed, 5 Jun 2024 16:28:49 -0700
Subject: [PATCH 74/85] async validate now checks failed logs if validator logs
 is empty

---
 guardrails/run/async_stream_runner.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/guardrails/run/async_stream_runner.py b/guardrails/run/async_stream_runner.py
index ac721e41b..073eac50b 100644
--- a/guardrails/run/async_stream_runner.py
+++ b/guardrails/run/async_stream_runner.py
@@ -327,8 +327,16 @@ async def async_validate(
         _ = await output_schema.async_validate(
             iteration, parsed_output, self.metadata, attempt_number=index, stream=stream
         )
-
-        return iteration.validator_logs[-1].validation_result
+        try:
+            return iteration.validator_logs[-1].validation_result
+        except IndexError:
+            pass
+
+        try:
+            return iteration.failed_validations[-1].validation_result
+        except IndexError:
+            pass
+        return None
 
     async def introspect(
         self,

From ef1af2ce5bbbe4627068c9c7553a3f1dfc501038 Mon Sep 17 00:00:00 2001
From: Wyatt Lansford <22553069+wylansford@users.noreply.github.com>
Date: Wed, 5 Jun 2024 16:35:52 -0700
Subject: [PATCH 75/85] reverting to unfiltered logs

---
 guardrails/run/async_stream_runner.py | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/guardrails/run/async_stream_runner.py b/guardrails/run/async_stream_runner.py
index 073eac50b..62f98c656 100644
--- a/guardrails/run/async_stream_runner.py
+++ b/guardrails/run/async_stream_runner.py
@@ -328,16 +328,10 @@ async def async_validate(
             iteration, parsed_output, self.metadata, attempt_number=index, stream=stream
         )
         try:
-            return iteration.validator_logs[-1].validation_result
+            return iteration.outputs.validator_logs[-1].validation_result
         except IndexError:
             pass
 
-        try:
-            return iteration.failed_validations[-1].validation_result
-        except IndexError:
-            pass
-        return None
-
     async def introspect(
         self,
         index: int,

From 0fc240372910d6e42cd7bc061067e603791a0399 Mon Sep 17 00:00:00 2001
From: Wyatt Lansford <22553069+wylansford@users.noreply.github.com>
Date: Wed, 5 Jun 2024 16:37:48 -0700
Subject: [PATCH 76/85] return none instead of no return

---
 guardrails/run/async_stream_runner.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/guardrails/run/async_stream_runner.py b/guardrails/run/async_stream_runner.py
index 62f98c656..713eda367 100644
--- a/guardrails/run/async_stream_runner.py
+++ b/guardrails/run/async_stream_runner.py
@@ -330,7 +330,7 @@ async def async_validate(
         try:
             return iteration.outputs.validator_logs[-1].validation_result
         except IndexError:
-            pass
+            return None
 
     async def introspect(
         self,

From 184d0dd8bcd4ee2c7fafc98d322c30b195968e5c Mon Sep 17 00:00:00 2001
From: Caleb Courier <caleb.courier@gmail.com>
Date: Thu, 6 Jun 2024 08:19:54 -0500
Subject: [PATCH 77/85] only filter logs when streaming

---
 guardrails/async_guard.py                 |  2 ++
 guardrails/classes/history/inputs.py      |  4 ++++
 guardrails/classes/history/iteration.py   | 14 ++++++++------
 guardrails/guard.py                       | 10 ++++++++++
 guardrails/run/async_stream_runner.py     |  1 +
 guardrails/run/stream_runner.py           |  1 +
 tests/integration_tests/test_streaming.py |  2 +-
 7 files changed, 27 insertions(+), 7 deletions(-)

diff --git a/guardrails/async_guard.py b/guardrails/async_guard.py
index 0fab18bc1..561edd2d3 100644
--- a/guardrails/async_guard.py
+++ b/guardrails/async_guard.py
@@ -148,6 +148,7 @@ async def __call(
                 full_schema_reask=full_schema_reask,
                 args=list(args),
                 kwargs=kwargs,
+                stream=kwargs.get("stream"),
             )
             call_log = Call(inputs=call_inputs)
             set_scope(str(id(call_log)))
@@ -374,6 +375,7 @@ async def __parse(
                 full_schema_reask=full_schema_reask,
                 args=list(args),
                 kwargs=kwargs,
+                stream=kwargs.get("stream"),
             )
             call_log = Call(inputs=call_inputs)
             set_scope(str(id(call_log)))
diff --git a/guardrails/classes/history/inputs.py b/guardrails/classes/history/inputs.py
index fd9e6762a..07a34920e 100644
--- a/guardrails/classes/history/inputs.py
+++ b/guardrails/classes/history/inputs.py
@@ -46,3 +46,7 @@ class Inputs(ArbitraryModel):
         "or at the field level.",
         default=None,
     )
+    stream: Optional[bool] = Field(
+        description="Whether to use streaming.",
+        default=False,
+    )
diff --git a/guardrails/classes/history/iteration.py b/guardrails/classes/history/iteration.py
index 1f6e71657..a8fb67ada 100644
--- a/guardrails/classes/history/iteration.py
+++ b/guardrails/classes/history/iteration.py
@@ -137,12 +137,14 @@ def reasks(self) -> Sequence[ReAsk]:
     def validator_logs(self) -> List[ValidatorLogs]:
         """The results of each individual validation performed on the LLM
         response during this iteration."""
-        filtered_logs = [
-            log
-            for log in self.outputs.validator_logs
-            if log.validation_result and log.validation_result.validated_chunk
-        ]
-        return filtered_logs
+        if self.inputs.stream:
+            filtered_logs = [
+                log
+                for log in self.outputs.validator_logs
+                if log.validation_result and log.validation_result.validated_chunk
+            ]
+            return filtered_logs
+        return self.outputs.validator_logs
 
     @property
     def error(self) -> Optional[str]:
diff --git a/guardrails/guard.py b/guardrails/guard.py
index 1bca53944..73877731f 100644
--- a/guardrails/guard.py
+++ b/guardrails/guard.py
@@ -634,6 +634,7 @@ def __call(
                 full_schema_reask=full_schema_reask,
                 args=list(args),
                 kwargs=kwargs,
+                stream=kwargs.get("stream"),
             )
             call_log = Call(inputs=call_inputs)
             set_scope(str(id(call_log)))
@@ -994,6 +995,7 @@ def __parse(
                 full_schema_reask=full_schema_reask,
                 args=list(args),
                 kwargs=kwargs,
+                stream=kwargs.get("stream"),
             )
             call_log = Call(inputs=call_inputs)
             set_scope(str(id(call_log)))
@@ -1435,6 +1437,7 @@ def _construct_history_from_server_response(
         metadata: Optional[Dict] = {},
         full_schema_reask: Optional[bool] = True,
         call_log: Optional[Call],
+        stream: Optional[bool] = False,
     ):
         call_log = call_log or Call()
         if llm_api is not None:
@@ -1471,6 +1474,7 @@ def _construct_history_from_server_response(
                         num_reasks=(num_reasks or 0),
                         metadata=metadata,
                         full_schema_reask=full_schema_reask,
+                        stream=stream,
                     ),
                     outputs=Outputs(
                         llm_response_info=LLMResponse(output=h.output),  # type: ignore
@@ -1522,6 +1526,7 @@ def _single_server_call(
         metadata: Optional[Dict] = {},
         full_schema_reask: Optional[bool] = True,
         call_log: Optional[Call],
+        stream: Optional[bool] = False,
     ) -> ValidationOutcome[OT]:
         if self._api_client:
             validation_output: ValidationOutput = self._api_client.validate(
@@ -1544,6 +1549,7 @@ def _single_server_call(
                 metadata=metadata,
                 full_schema_reask=full_schema_reask,
                 call_log=call_log,
+                stream=stream,
             )
 
             # Our interfaces are too different for this to work right now.
@@ -1568,6 +1574,7 @@ def _stream_server_call(
         metadata: Optional[Dict] = {},
         full_schema_reask: Optional[bool] = True,
         call_log: Optional[Call],
+        stream: Optional[bool] = False,
     ) -> Generator[ValidationOutcome[OT], None, None]:
         if self._api_client:
             validation_output: Optional[ValidationOutput] = None
@@ -1599,6 +1606,7 @@ def _stream_server_call(
                     metadata=metadata,
                     full_schema_reask=full_schema_reask,
                     call_log=call_log,
+                    stream=stream,
                 )
         else:
             raise ValueError("Guard does not have an api client!")
@@ -1639,6 +1647,7 @@ def _call_server(
                     metadata=metadata,
                     full_schema_reask=full_schema_reask,
                     call_log=call_log,
+                    stream=should_stream,
                 )
             else:
                 return self._single_server_call(
@@ -1649,6 +1658,7 @@ def _call_server(
                     metadata=metadata,
                     full_schema_reask=full_schema_reask,
                     call_log=call_log,
+                    stream=should_stream,
                 )
         else:
             raise ValueError("Guard does not have an api client!")
diff --git a/guardrails/run/async_stream_runner.py b/guardrails/run/async_stream_runner.py
index 713eda367..fc1fd4018 100644
--- a/guardrails/run/async_stream_runner.py
+++ b/guardrails/run/async_stream_runner.py
@@ -156,6 +156,7 @@ async def async_step(
             num_reasks=self.num_reasks,
             metadata=self.metadata,
             full_schema_reask=self.full_schema_reask,
+            stream=True,
         )
         outputs = Outputs()
         iteration = Iteration(inputs=inputs, outputs=outputs)
diff --git a/guardrails/run/stream_runner.py b/guardrails/run/stream_runner.py
index 5dc842eda..56e7873eb 100644
--- a/guardrails/run/stream_runner.py
+++ b/guardrails/run/stream_runner.py
@@ -109,6 +109,7 @@ def step(
             num_reasks=self.num_reasks,
             metadata=self.metadata,
             full_schema_reask=self.full_schema_reask,
+            stream=True,
         )
         outputs = Outputs()
         iteration = Iteration(inputs=inputs, outputs=outputs)
diff --git a/tests/integration_tests/test_streaming.py b/tests/integration_tests/test_streaming.py
index 623ce4a9f..149ddd62d 100644
--- a/tests/integration_tests/test_streaming.py
+++ b/tests/integration_tests/test_streaming.py
@@ -83,7 +83,7 @@ def validate(self, value: Union[str, List], metadata: Dict) -> ValidationResult:
                 f"Please return a longer output, "
                 f"that is shorter than {self._max} characters.",
             )
-        return PassResult()
+        return PassResult(validated_chunk=value)
 
     def validate_stream(self, chunk: Any, metadata: Dict, **kwargs) -> ValidationResult:
         return super().validate_stream(chunk, metadata, **kwargs)

From de28c7b97bf9de42794659b0c24606bd52f5dc42 Mon Sep 17 00:00:00 2001
From: Caleb Courier <caleb.courier@gmail.com>
Date: Thu, 6 Jun 2024 08:32:54 -0500
Subject: [PATCH 78/85] update poetry lock

---
 poetry.lock | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/poetry.lock b/poetry.lock
index c7085b049..a4cd0ea91 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -8339,4 +8339,4 @@ vectordb = ["faiss-cpu", "numpy"]
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.8.1"
-content-hash = "fe6b6c42df209d16b013d3b8cd070399bb5804d4c10a28868a92d513c7fc520a"
+content-hash = "f2c26b1ad671f41fb695448f5f007d9f29609581db2a23301169a4d0a5ab11f1"

From 8d73f58421b03488bd41ee525bcfe597b27ed38c Mon Sep 17 00:00:00 2001
From: Caleb Courier <caleb.courier@gmail.com>
Date: Thu, 6 Jun 2024 11:53:47 -0500
Subject: [PATCH 79/85] fix notebook installs, fix a couple notebooks

---
 docs/examples/bug_free_python_code.ipynb      |  2 +-
 .../no_secrets_in_generated_text.ipynb        | 98 +++++++++++--------
 docs/examples/response_is_on_topic.ipynb      | 61 +++++++-----
 guardrails/cli/hub/install.py                 |  3 +-
 guardrails/cli/hub/utils.py                   | 28 ++++--
 tests/unit_tests/cli/hub/test_install.py      |  9 +-
 6 files changed, 121 insertions(+), 80 deletions(-)

diff --git a/docs/examples/bug_free_python_code.ipynb b/docs/examples/bug_free_python_code.ipynb
index 340ca3bda..401448d98 100644
--- a/docs/examples/bug_free_python_code.ipynb
+++ b/docs/examples/bug_free_python_code.ipynb
@@ -398,7 +398,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.0"
+   "version": "3.11.9"
   },
   "orig_nbformat": 4,
   "vscode": {
diff --git a/docs/examples/no_secrets_in_generated_text.ipynb b/docs/examples/no_secrets_in_generated_text.ipynb
index f90372c0e..05f0ecc16 100644
--- a/docs/examples/no_secrets_in_generated_text.ipynb
+++ b/docs/examples/no_secrets_in_generated_text.ipynb
@@ -50,8 +50,18 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "/home/zayd/workspace/guardrails/.venv/lib/python3.9/site-packages/torch/cuda/__init__.py:611: UserWarning: Can't initialize NVML\n",
-      "  warnings.warn(\"Can't initialize NVML\")\n"
+      "/Users/calebcourier/Projects/gr-mono/guardrails/docs/examples/.venv/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n",
+      "/Users/calebcourier/Projects/gr-mono/guardrails/docs/examples/.venv/lib/python3.11/site-packages/guardrails/validators/__init__.py:51: FutureWarning: \n",
+      "    Importing validators from `guardrails.validators` is deprecated.\n",
+      "    All validators are now available in the Guardrails Hub. Please install\n",
+      "    and import them from the hub instead. All validators will be\n",
+      "    removed from this module in the next major release.\n",
+      "\n",
+      "    Install with: `guardrails hub install hub://<namespace>/<validator_name>`\n",
+      "    Import as: from guardrails.hub import `ValidatorName`\n",
+      "    \n",
+      "  warn(\n"
      ]
     }
    ],
@@ -177,7 +187,16 @@
    "cell_type": "code",
    "execution_count": 5,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/calebcourier/Projects/gr-mono/guardrails/docs/examples/.venv/lib/python3.11/site-packages/guardrails/prompt/base_prompt.py:59: FutureWarning: Prompt Primitives are moving! To keep the same behaviour, switch from `json` constants to `xml` constants. Example: ${gr.complete_json_suffix} -> ${gr.complete_xml_suffix}\n",
+      "  warn(\n"
+     ]
+    }
+   ],
    "source": [
     "guard = gd.Guard.from_rail_string(rail_str)"
    ]
@@ -193,7 +212,16 @@
    "cell_type": "code",
    "execution_count": 6,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/calebcourier/Projects/gr-mono/guardrails/docs/examples/.venv/lib/python3.11/site-packages/guardrails/prompt/base_prompt.py:59: FutureWarning: Prompt Primitives are moving! To keep the same behaviour, switch from `json` constants to `xml` constants. Example: ${gr.complete_json_suffix} -> ${gr.complete_xml_suffix}\n",
+      "  warn(\n"
+     ]
+    }
+   ],
    "source": [
     "guard = gd.Guard.from_pydantic(output_class=ScrubbedCode, prompt=prompt)"
    ]
@@ -211,6 +239,14 @@
    "execution_count": 7,
    "metadata": {},
    "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/var/folders/w2/ssf16z690zd7_4dggw0y5s_m0000gn/T/ipykernel_82075/3983563700.py:1: DeprecationWarning: 'Guard.base_prompt' is deprecated and will be removed in versions 0.5.x and beyond. Use 'Guard.history.last.prompt' instead.\n",
+      "  print(guard.base_prompt)\n"
+     ]
+    },
     {
      "data": {
       "text/html": [
@@ -295,14 +331,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [],
    "source": [
     "import openai\n",
     "\n",
     "raw_llm_response, validated_response, *rest = guard(\n",
-    "   openai.completions.create, model=\"text-davinci-003\", max_tokens=2048, temperature=0\n",
+    "   openai.completions.create, model=\"gpt-3.5-turbo-instruct\", max_tokens=2048, temperature=0\n",
     ")"
    ]
   },
@@ -314,19 +350,11 @@
     {
      "data": {
       "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">{</span>\n",
-       "    <span style=\"color: #008000; text-decoration-color: #008000\">'api_help'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'curl -X POST -H \"Content-Type: application/json\" -H \"Authorization: Bearer YOUR_API_KEY\" -d </span>\n",
-       "<span style=\"color: #008000; text-decoration-color: #008000\">\\'{\"prompt\": \"Once upon a time\", \"max_tokens\": 100}\\' </span>\n",
-       "<span style=\"color: #008000; text-decoration-color: #008000\">\"https://api.openai.com/v1/engines/davinci-codex/completions\"'</span>\n",
-       "<span style=\"font-weight: bold\">}</span>\n",
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'api_help'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'Show an example curl command for using openai Completion API'</span><span style=\"font-weight: bold\">}</span>\n",
        "</pre>\n"
       ],
       "text/plain": [
-       "\u001b[1m{\u001b[0m\n",
-       "    \u001b[32m'api_help'\u001b[0m: \u001b[32m'curl -X POST -H \"Content-Type: application/json\" -H \"Authorization: Bearer YOUR_API_KEY\" -d \u001b[0m\n",
-       "\u001b[32m\\'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"prompt\": \"Once upon a time\", \"max_tokens\": 100\u001b[0m\u001b[32m}\u001b[0m\u001b[32m\\' \u001b[0m\n",
-       "\u001b[32m\"https://api.openai.com/v1/engines/davinci-codex/completions\"'\u001b[0m\n",
-       "\u001b[1m}\u001b[0m\n"
+       "\u001b[1m{\u001b[0m\u001b[32m'api_help'\u001b[0m: \u001b[32m'Show an example curl command for using openai Completion API'\u001b[0m\u001b[1m}\u001b[0m\n"
       ]
      },
      "metadata": {},
@@ -376,27 +404,21 @@
        "    │ <span style=\"background-color: #f0f8ff\">│ format=\"1-indexed\" /&gt;&lt;/object&gt;` =&gt; `{'baz': {'foo': 'Some String', 'index': 1}}`                        │</span> │\n",
        "    │ <span style=\"background-color: #f0f8ff\">│                                                                                                         │</span> │\n",
        "    │ <span style=\"background-color: #f0f8ff\">│                                                                                                         │</span> │\n",
+       "    │ <span style=\"background-color: #f0f8ff\">│                                                                                                         │</span> │\n",
+       "    │ <span style=\"background-color: #f0f8ff\">│ Json Output:                                                                                            │</span> │\n",
+       "    │ <span style=\"background-color: #f0f8ff\">│                                                                                                         │</span> │\n",
+       "    │ <span style=\"background-color: #f0f8ff\">│                                                                                                         │</span> │\n",
        "    │ <span style=\"background-color: #f0f8ff\">╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span> │\n",
-       "    │ <span style=\"background-color: #fff0f2\">╭───────────────────────────────────────────── Instructions ──────────────────────────────────────────────╮</span> │\n",
-       "    │ <span style=\"background-color: #fff0f2\">│ You are a helpful assistant, able to express yourself purely through JSON, strictly and precisely       │</span> │\n",
-       "    │ <span style=\"background-color: #fff0f2\">│ adhering to the provided XML schemas.                                                                   │</span> │\n",
-       "    │ <span style=\"background-color: #fff0f2\">╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span> │\n",
        "    │ <span style=\"background-color: #e7dfeb\">╭──────────────────────────────────────────── Message History ────────────────────────────────────────────╮</span> │\n",
        "    │ <span style=\"background-color: #e7dfeb\">│ No message history.                                                                                     │</span> │\n",
        "    │ <span style=\"background-color: #e7dfeb\">╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span> │\n",
        "    │ <span style=\"background-color: #f5f5dc\">╭──────────────────────────────────────────── Raw LLM Output ─────────────────────────────────────────────╮</span> │\n",
        "    │ <span style=\"background-color: #f5f5dc\">│ {                                                                                                       │</span> │\n",
-       "    │ <span style=\"background-color: #f5f5dc\">│   \"api_help\": \"curl -X POST -H \\\"Content-Type: application/json\\\" -H \\\"Authorization: Bearer            │</span> │\n",
-       "    │ <span style=\"background-color: #f5f5dc\">│ YOUR_API_KEY\\\" -d '{\\\"prompt\\\": \\\"Once upon a time\\\", \\\"max_tokens\\\": 100}'                             │</span> │\n",
-       "    │ <span style=\"background-color: #f5f5dc\">│ \\\"https://api.openai.com/v1/engines/davinci-codex/completions\\\"\"                                        │</span> │\n",
+       "    │ <span style=\"background-color: #f5f5dc\">│     \"api_help\": \"Show an example curl command for using openai Completion API\"                          │</span> │\n",
        "    │ <span style=\"background-color: #f5f5dc\">│ }                                                                                                       │</span> │\n",
        "    │ <span style=\"background-color: #f5f5dc\">╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span> │\n",
        "    │ <span style=\"background-color: #f0fff0\">╭─────────────────────────────────────────── Validated Output ────────────────────────────────────────────╮</span> │\n",
-       "    │ <span style=\"background-color: #f0fff0\">│ {                                                                                                       │</span> │\n",
-       "    │ <span style=\"background-color: #f0fff0\">│     'api_help': 'curl -X POST -H \"Content-Type: application/json\" -H \"Authorization: Bearer             │</span> │\n",
-       "    │ <span style=\"background-color: #f0fff0\">│ YOUR_API_KEY\" -d \\'{\"prompt\": \"Once upon a time\", \"max_tokens\": 100}\\'                                  │</span> │\n",
-       "    │ <span style=\"background-color: #f0fff0\">│ \"https://api.openai.com/v1/engines/davinci-codex/completions\"'                                          │</span> │\n",
-       "    │ <span style=\"background-color: #f0fff0\">│ }                                                                                                       │</span> │\n",
+       "    │ <span style=\"background-color: #f0fff0\">│ {'api_help': 'Show an example curl command for using openai Completion API'}                            │</span> │\n",
        "    │ <span style=\"background-color: #f0fff0\">╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span> │\n",
        "    ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
        "</pre>\n"
@@ -433,27 +455,21 @@
        "    │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255mformat=\"1-indexed\" /></object>` => `{'baz': {'foo': 'Some String', 'index': 1}}`\u001b[0m\u001b[48;2;240;248;255m                       \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m                                                                                                       \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m                                                                                                       \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m                                                                                                       \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255mJson Output:\u001b[0m\u001b[48;2;240;248;255m                                                                                           \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m                                                                                                       \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m                                                                                                       \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;248;255m╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m │\n",
-       "    │ \u001b[48;2;255;240;242m╭─\u001b[0m\u001b[48;2;255;240;242m────────────────────────────────────────────\u001b[0m\u001b[48;2;255;240;242m Instructions \u001b[0m\u001b[48;2;255;240;242m─────────────────────────────────────────────\u001b[0m\u001b[48;2;255;240;242m─╮\u001b[0m │\n",
-       "    │ \u001b[48;2;255;240;242m│\u001b[0m\u001b[48;2;255;240;242m \u001b[0m\u001b[48;2;255;240;242mYou are a helpful assistant, able to express yourself purely through JSON, strictly and precisely \u001b[0m\u001b[48;2;255;240;242m     \u001b[0m\u001b[48;2;255;240;242m \u001b[0m\u001b[48;2;255;240;242m│\u001b[0m │\n",
-       "    │ \u001b[48;2;255;240;242m│\u001b[0m\u001b[48;2;255;240;242m \u001b[0m\u001b[48;2;255;240;242madhering to the provided XML schemas.\u001b[0m\u001b[48;2;255;240;242m                                                                  \u001b[0m\u001b[48;2;255;240;242m \u001b[0m\u001b[48;2;255;240;242m│\u001b[0m │\n",
-       "    │ \u001b[48;2;255;240;242m╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m │\n",
        "    │ \u001b[48;2;231;223;235m╭─\u001b[0m\u001b[48;2;231;223;235m───────────────────────────────────────────\u001b[0m\u001b[48;2;231;223;235m Message History \u001b[0m\u001b[48;2;231;223;235m───────────────────────────────────────────\u001b[0m\u001b[48;2;231;223;235m─╮\u001b[0m │\n",
        "    │ \u001b[48;2;231;223;235m│\u001b[0m\u001b[48;2;231;223;235m \u001b[0m\u001b[48;2;231;223;235mNo message history.\u001b[0m\u001b[48;2;231;223;235m                                                                                    \u001b[0m\u001b[48;2;231;223;235m \u001b[0m\u001b[48;2;231;223;235m│\u001b[0m │\n",
        "    │ \u001b[48;2;231;223;235m╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m╭─\u001b[0m\u001b[48;2;245;245;220m───────────────────────────────────────────\u001b[0m\u001b[48;2;245;245;220m Raw LLM Output \u001b[0m\u001b[48;2;245;245;220m────────────────────────────────────────────\u001b[0m\u001b[48;2;245;245;220m─╮\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m{\u001b[0m\u001b[48;2;245;245;220m                                                                                                      \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
-       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m  \"api_help\": \"curl -X POST -H \\\"Content-Type: application/json\\\" -H \\\"Authorization: Bearer \u001b[0m\u001b[48;2;245;245;220m          \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
-       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220mYOUR_API_KEY\\\" -d '{\\\"prompt\\\": \\\"Once upon a time\\\", \\\"max_tokens\\\": 100}' \u001b[0m\u001b[48;2;245;245;220m                           \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
-       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m\\\"https://api.openai.com/v1/engines/davinci-codex/completions\\\"\"\u001b[0m\u001b[48;2;245;245;220m                                       \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
+       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m    \"api_help\": \"Show an example curl command for using openai Completion API\"\u001b[0m\u001b[48;2;245;245;220m                         \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m}\u001b[0m\u001b[48;2;245;245;220m                                                                                                      \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m╭─\u001b[0m\u001b[48;2;240;255;240m──────────────────────────────────────────\u001b[0m\u001b[48;2;240;255;240m Validated Output \u001b[0m\u001b[48;2;240;255;240m───────────────────────────────────────────\u001b[0m\u001b[48;2;240;255;240m─╮\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m{\u001b[0m\u001b[48;2;240;255;240m                                                                                                      \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m    'api_help': 'curl -X POST -H \"Content-Type: application/json\" -H \"Authorization: Bearer \u001b[0m\u001b[48;2;240;255;240m           \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240mYOUR_API_KEY\" -d \\'{\"prompt\": \"Once upon a time\", \"max_tokens\": 100}\\' \u001b[0m\u001b[48;2;240;255;240m                                \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m\"https://api.openai.com/v1/engines/davinci-codex/completions\"'\u001b[0m\u001b[48;2;240;255;240m                                         \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m}\u001b[0m\u001b[48;2;240;255;240m                                                                                                      \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m{'api_help': 'Show an example curl command for using openai Completion API'}\u001b[0m\u001b[48;2;240;255;240m                           \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m │\n",
        "    ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────╯\n"
       ]
@@ -483,7 +499,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.17"
+   "version": "3.11.9"
   },
   "orig_nbformat": 4,
   "vscode": {
diff --git a/docs/examples/response_is_on_topic.ipynb b/docs/examples/response_is_on_topic.ipynb
index 2eefbbf54..58692a146 100644
--- a/docs/examples/response_is_on_topic.ipynb
+++ b/docs/examples/response_is_on_topic.ipynb
@@ -2,10 +2,32 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 8,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Installing hub:\u001b[35m/\u001b[0m\u001b[35m/tryolabs/\u001b[0m\u001b[95mrestricttotopic...\u001b[0m\n",
+      "\u001b[2K\u001b[32m[ ===]\u001b[0m Fetching manifestst\n",
+      "\u001b[2K\u001b[32m[==  ]\u001b[0m Downloading dependencies  Running command git clone --filter=blob:none --quiet https://github.com/tryolabs/restricttotopic.git /private/var/folders/w2/ssf16z690zd7_4dggw0y5s_m0000gn/T/pip-req-build-advwvzw9\n",
+      "\u001b[2K\u001b[32m[=== ]\u001b[0m Downloading dependencies\n",
+      "\u001b[1A\u001b[2K\u001b[?25l\u001b[32m[    ]\u001b[0m Running post-install setup\n",
+      "\u001b[1A\u001b[2K✅Successfully installed tryolabs/restricttotopic!\n",
+      "\n",
+      "\n",
+      "\u001b[1mImport validator:\u001b[0m\n",
+      "from guardrails.hub import RestrictToTopic\n",
+      "\n",
+      "\u001b[1mGet more info:\u001b[0m\n",
+      "\u001b[4;94mhttps://hub.guardrailsai.com/validator/tryolabs/restricttotopic\u001b[0m\n",
+      "\n"
+     ]
+    }
+   ],
    "source": [
+    "\n",
     "!guardrails hub install hub://tryolabs/restricttotopic"
    ]
   },
@@ -46,7 +68,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -63,7 +85,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -85,7 +107,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 11,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -124,22 +146,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 12,
    "metadata": {},
    "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/home/zayd/workspace/guardrails/.venv/lib/python3.9/site-packages/torch/cuda/__init__.py:611: UserWarning: Can't initialize NVML\n",
-      "  warnings.warn(\"Can't initialize NVML\")\n"
-     ]
-    },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Validation failed for field with errors: Most relevant topic is tablet.\n"
+      "Validation failed for field with errors: Invalid topics found: ['tablet', 'computer', 'phone']\n"
      ]
     }
    ],
@@ -183,14 +197,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 13,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Validation failed for field with errors: Most relevant topic is tablet.\n"
+      "Validation failed for field with errors: Invalid topics found: ['tablet', 'computer', 'phone']\n"
      ]
     }
    ],
@@ -229,21 +243,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 14,
    "metadata": {},
    "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n"
-     ]
-    },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Validation failed for field with errors: Most relevant topic is tablet.\n"
+      "Validation failed for field with errors: Invalid topics found: ['tablet']\n"
      ]
     }
    ],
@@ -288,7 +295,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.7"
+   "version": "3.11.9"
   }
  },
  "nbformat": 4,
diff --git a/guardrails/cli/hub/install.py b/guardrails/cli/hub/install.py
index d4d94ad6f..931b2383f 100644
--- a/guardrails/cli/hub/install.py
+++ b/guardrails/cli/hub/install.py
@@ -156,6 +156,7 @@ def install_hub_module(
         flags=[f"--path={install_directory}"],
         format=json_format,
         quiet=quiet,
+        no_color=True,
     )
 
     # throw if inspect_output is a string. Mostly for pyright
@@ -169,7 +170,7 @@ def install_hub_module(
         .get("metadata", {})  # type: ignore
         .get("requires_dist", [])  # type: ignore
     )
-    requirements = filter(lambda dep: "extra" not in dep, dependencies)
+    requirements = list(filter(lambda dep: "extra" not in dep, dependencies))
     for req in requirements:
         if "git+" in req:
             install_spec = req.replace(" ", "")
diff --git a/guardrails/cli/hub/utils.py b/guardrails/cli/hub/utils.py
index 384e76ee2..6c0b2d968 100644
--- a/guardrails/cli/hub/utils.py
+++ b/guardrails/cli/hub/utils.py
@@ -1,5 +1,6 @@
 import json
 import os
+import re
 import subprocess
 import sys
 from email.parser import BytesHeaderParser
@@ -20,33 +21,44 @@ def pip_process(
     flags: List[str] = [],
     format: Union[Literal["string"], Literal["json"]] = string_format,
     quiet: bool = False,
+    no_color: bool = False,
 ) -> Union[str, dict]:
     try:
         if not quiet:
             logger.debug(f"running pip {action} {' '.join(flags)} {package}")
-        command = [sys.executable, "-m", "pip", action] + flags
+        command = [sys.executable, "-m", "pip", action]
+        command.extend(flags)
         if package:
             command.append(package)
 
+        env = dict(os.environ)
+        if no_color:
+            env["NO_COLOR"] = "true"
         if not quiet:
             logger.debug(f"decoding output from pip {action} {package}")
-            output = subprocess.check_output(command)
+            output = subprocess.check_output(command, env=env)
         else:
-            output = subprocess.check_output(command, stderr=subprocess.DEVNULL)
+            output = subprocess.check_output(
+                command, stderr=subprocess.DEVNULL, env=env
+            )
 
         if format == json_format:
             parsed = BytesHeaderParser().parsebytes(output)
             try:
-                return json.loads(str(parsed))
+                remove_color_codes = re.compile(r"\x1b\[[0-9;]*m")
+                parsed_as_string = re.sub(
+                    remove_color_codes, "", parsed.as_string().strip()
+                )
+                return json.loads(parsed_as_string)
             except Exception:
                 logger.debug(
                     f"JSON parse exception in decoding output from pip \
 {action} {package}. Falling back to accumulating the byte stream",
                 )
-                accumulator = {}
-                for key, value in parsed.items():
-                    accumulator[key] = value
-                return accumulator
+            accumulator = {}
+            for key, value in parsed.items():
+                accumulator[key] = value
+            return accumulator
         return str(output.decode())
     except subprocess.CalledProcessError as exc:
         logger.error(
diff --git a/tests/unit_tests/cli/hub/test_install.py b/tests/unit_tests/cli/hub/test_install.py
index 89865567c..a5bf4bf8f 100644
--- a/tests/unit_tests/cli/hub/test_install.py
+++ b/tests/unit_tests/cli/hub/test_install.py
@@ -88,6 +88,7 @@ def test_happy_path(self, mocker):
 
 class TestPipProcess:
     def test_no_package_string_format(self, mocker):
+        mocker.patch("guardrails.cli.hub.install.os.environ", return_value={})
         mock_logger_debug = mocker.patch("guardrails.cli.hub.utils.logger.debug")
 
         mock_sys_executable = mocker.patch("guardrails.cli.hub.install.sys.executable")
@@ -109,12 +110,14 @@ def test_no_package_string_format(self, mocker):
         mock_logger_debug.assert_has_calls(debug_calls)
 
         mock_subprocess_check_output.assert_called_once_with(
-            [mock_sys_executable, "-m", "pip", "inspect", "--path=./install-here"]
+            [mock_sys_executable, "-m", "pip", "inspect", "--path=./install-here"],
+            env={},
         )
 
         assert response == "string output"
 
     def test_json_format(self, mocker):
+        mocker.patch("guardrails.cli.hub.install.os.environ", return_value={})
         mock_logger_debug = mocker.patch("guardrails.cli.hub.install.logger.debug")
 
         mock_sys_executable = mocker.patch("guardrails.cli.hub.install.sys.executable")
@@ -147,7 +150,7 @@ def parsebytes(self, *args):
         mock_logger_debug.assert_has_calls(debug_calls)
 
         mock_subprocess_check_output.assert_called_once_with(
-            [mock_sys_executable, "-m", "pip", "show", "pip"]
+            [mock_sys_executable, "-m", "pip", "show", "pip"], env={}
         )
 
         assert response == {"output": "json"}
@@ -708,6 +711,7 @@ def test_install_hub_module(mocker):
             flags=["--path=mock/install/directory"],
             format="json",
             quiet=False,
+            no_color=True,
         ),
         call("install", "rstr", quiet=False),
         call("install", "openai<2", quiet=False),
@@ -783,6 +787,7 @@ def test_quiet_install(mocker):
             flags=["--path=mock/install/directory"],
             format="json",
             quiet=True,
+            no_color=True,
         ),
         call("install", "rstr", quiet=True),
         call("install", "openai<2", quiet=True),

From fd2ad02c9ebd76bca176aa997c370eb83fc6b2dd Mon Sep 17 00:00:00 2001
From: Caleb Courier <caleb.courier@gmail.com>
Date: Thu, 6 Jun 2024 12:00:23 -0500
Subject: [PATCH 80/85] use version of notebooks from the PR

---
 .github/workflows/examples_check.yml | 29 ++++------------------------
 1 file changed, 4 insertions(+), 25 deletions(-)

diff --git a/.github/workflows/examples_check.yml b/.github/workflows/examples_check.yml
index b330a6a36..a7eab18b0 100644
--- a/.github/workflows/examples_check.yml
+++ b/.github/workflows/examples_check.yml
@@ -24,6 +24,9 @@ jobs:
     steps:
     - name: Checkout repository
       uses: actions/checkout@v4
+      with: 
+        ref: ${{ github.head_ref }}
+        fetch-depth: 0
     - name: Create .guardrailsrc
       run: |
         echo 'id="SYSTEM TESTING"' > ~/.guardrailsrc
@@ -32,23 +35,9 @@ jobs:
       uses: actions/setup-python@v5
       with: 
         python-version: 3.11.x
-    # - name: Poetry cache
-    #   uses: actions/cache@v3
-    #   with:
-    #     path: ~/.cache/pypoetry
-    #     key: poetry-cache-${{ runner.os }}-${{ steps.setup_python.outputs.python-version }}-${{ env.POETRY_VERSION }}
-    # - name: Install Poetry
-    #   uses: snok/install-poetry@v1
-    #   with:
-    #     virtualenvs-create: true
-    #     virtualenvs-in-project: true
-    #     installer-parallel: true
     - name: Install dependencies
       run: |
-        # Legacy run script
-        # make full;
-        # poetry add "openai>=1.2.4" jupyter nbconvert cohere==5.3.2;
-        
+
         # Setup Virtual Environment
         python3 -m venv ./.venv
         source .venv/bin/activate
@@ -59,26 +48,16 @@ jobs:
         # Install extra stuff for notebook runs
         pip install "huggingface_hub[cli]" jupyter nbconvert cohere==5.3.2
         pip install nltk
-        echo "which python $(which python)"
-        echo "which pip $(which pip)"
-        echo "pip show nltk: $(pip show nltk)"
     - name: Huggingface Hub Login
       run: |
         source .venv/bin/activate
-        echo "which python: $(which python)"
-        echo "which pip $(which pip)"
         huggingface-cli login --token $HUGGINGFACE_API_KEY
     - name: download nltk data
       run: |
         source .venv/bin/activate
-        echo "which pip $(which pip)"
-        echo "pip show nltk: $(pip show nltk)"
         mkdir /tmp/nltk_data;
         python -m nltk.downloader -d /tmp/nltk_data punkt;
-    # - name: Use venv
-    #   run: source .venv/bin/activate
     - name: Execute notebooks and check for errors
-      # run: bash ./.github/workflows/scripts/run_notebooks.sh ${{ matrix.notebook }}
       run: |
         source .venv/bin/activate
         bash ./.github/workflows/scripts/run_notebooks.sh ${{ matrix.notebook }}

From 38d4597110cc4f799310b50c3f6c758bbc704052 Mon Sep 17 00:00:00 2001
From: Caleb Courier <caleb.courier@gmail.com>
Date: Thu, 6 Jun 2024 13:09:05 -0500
Subject: [PATCH 81/85] notebook fixes

---
 docs/examples/bug_free_python_code.ipynb      |  23 +-
 .../generate_structured_data_cohere.ipynb     | 221 ++++++------------
 2 files changed, 74 insertions(+), 170 deletions(-)

diff --git a/docs/examples/bug_free_python_code.ipynb b/docs/examples/bug_free_python_code.ipynb
index 401448d98..d377b4eb5 100644
--- a/docs/examples/bug_free_python_code.ipynb
+++ b/docs/examples/bug_free_python_code.ipynb
@@ -30,15 +30,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Note: you may need to restart the kernel to use updated packages.\n",
-      "Note: you may need to restart the kernel to use updated packages.\n",
       "Installing hub:\u001b[35m/\u001b[0m\u001b[35m/reflex/\u001b[0m\u001b[95mvalid_python...\u001b[0m\n",
       "✅Successfully installed reflex/valid_python!\n",
       "\n",
@@ -47,15 +45,12 @@
     }
    ],
    "source": [
-    "%pip install guardrails-ai -q\n",
-    "%pip install pydantic -q\n",
-    "\n",
     "!guardrails hub install hub://reflex/valid_python --quiet"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -93,7 +88,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -111,7 +106,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -128,7 +123,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [
     {
@@ -209,7 +204,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -239,7 +234,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [
     {
@@ -284,7 +279,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [
     {
@@ -356,7 +351,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": 11,
    "metadata": {},
    "outputs": [
     {
diff --git a/docs/examples/generate_structured_data_cohere.ipynb b/docs/examples/generate_structured_data_cohere.ipynb
index 35286463b..e22624ea8 100644
--- a/docs/examples/generate_structured_data_cohere.ipynb
+++ b/docs/examples/generate_structured_data_cohere.ipynb
@@ -10,14 +10,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 1,
    "id": "346e1b5c",
    "metadata": {},
    "outputs": [],
    "source": [
     "prompt = \"\"\"\n",
     "Generate a dataset of fake user orders. Each row of the dataset should be valid. The format should not be a list, it should be a JSON object.\n",
-    "${gr.complete_json_suffix}\n",
+    "${gr.complete_xml_suffix}\n",
     "\n",
     "an example of output may look like this:\n",
     "{\n",
@@ -41,7 +41,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 2,
    "id": "6a7c7d4a",
    "metadata": {},
    "outputs": [
@@ -49,103 +49,26 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "\u001b[32m2024-03-25 16:13:45\u001b[0m \u001b[35mzmac\u001b[0m \u001b[34mguardrails-cli[96111]\u001b[0m \u001b[1;30mNOTICE\u001b[0m \u001b[1;36mInstalling hub://guardrails/valid_length...\u001b[0m\n",
-      "  Running command git clone --filter=blob:none --quiet https://github.com/guardrails-ai/valid_length.git /private/var/folders/c8/jqt82fpx785dpwpp36ljkgm40000gn/T/pip-req-build-y3acxka5\n",
-      "\u001b[33mWARNING: Target directory /Users/zaydsimjee/workspace/guardrails/.venv/lib/python3.11/site-packages/guardrails/hub/guardrails/valid_length/validator already exists. Specify --upgrade to force replacement.\u001b[0m\u001b[33m\n",
-      "\u001b[0m\u001b[33mWARNING: Target directory /Users/zaydsimjee/workspace/guardrails/.venv/lib/python3.11/site-packages/guardrails/hub/guardrails/valid_length/valid_length-0.0.0.dist-info already exists. Specify --upgrade to force replacement.\u001b[0m\u001b[33m\n",
-      "\u001b[0m\u001b[32m2024-03-25 16:13:52\u001b[0m \u001b[35mzmac\u001b[0m \u001b[34mguardrails-cli[96111]\u001b[0m \u001b[1;30mINFO\u001b[0m Collecting git+https://github.com/guardrails-ai/valid_length.git\n",
-      "  Cloning https://github.com/guardrails-ai/valid_length.git to /private/var/folders/c8/jqt82fpx785dpwpp36ljkgm40000gn/T/pip-req-build-y3acxka5\n",
-      "  Resolved https://github.com/guardrails-ai/valid_length.git to commit 4b59a5fc1ae2106a585881784e3c2086f1fe8b9b\n",
-      "  Installing build dependencies: started\n",
-      "  Installing build dependencies: finished with status 'done'\n",
-      "  Getting requirements to build wheel: started\n",
-      "  Getting requirements to build wheel: finished with status 'done'\n",
-      "  Installing backend dependencies: started\n",
-      "  Installing backend dependencies: finished with status 'done'\n",
-      "  Preparing metadata (pyproject.toml): started\n",
-      "  Preparing metadata (pyproject.toml): finished with status 'done'\n",
-      "Building wheels for collected packages: valid_length\n",
-      "  Building wheel for valid_length (pyproject.toml): started\n",
-      "  Building wheel for valid_length (pyproject.toml): finished with status 'done'\n",
-      "  Created wheel for valid_length: filename=valid_length-0.0.0-py3-none-any.whl size=12348 sha256=98e297c72fa6bc34b9c52e2ce3b87365ce925225b0792d1a72096baf11b4e792\n",
-      "  Stored in directory: /private/var/folders/c8/jqt82fpx785dpwpp36ljkgm40000gn/T/pip-ephem-wheel-cache-odhptidc/wheels/48/9f/75/34a76a1e575dafaf9df180a2074f698d77193d5d3670823f69\n",
-      "Successfully built valid_length\n",
-      "Installing collected packages: valid_length\n",
-      "Successfully installed valid_length-0.0.0\n",
-      "\n",
-      "\u001b[32m2024-03-25 16:13:52\u001b[0m \u001b[35mzmac\u001b[0m \u001b[34mguardrails-cli[96111]\u001b[0m \u001b[1;30mSUCCESS\u001b[0m \u001b[1;32m\n",
-      "\n",
-      "    Successfully installed guardrails/valid_length!\n",
-      "\n",
-      "    See how to use it here: https://hub.guardrailsai.com/validator/guardrails/valid_length\n",
-      "    \u001b[0m\n",
-      "\u001b[32m2024-03-25 16:13:53\u001b[0m \u001b[35mzmac\u001b[0m \u001b[34mguardrails-cli[96229]\u001b[0m \u001b[1;30mNOTICE\u001b[0m \u001b[1;36mInstalling hub://guardrails/two_words...\u001b[0m\n",
-      "  Running command git clone --filter=blob:none --quiet https://github.com/guardrails-ai/two_words.git /private/var/folders/c8/jqt82fpx785dpwpp36ljkgm40000gn/T/pip-req-build-c9bdjnpk\n",
-      "\u001b[33mWARNING: Target directory /Users/zaydsimjee/workspace/guardrails/.venv/lib/python3.11/site-packages/guardrails/hub/guardrails/two_words/validator already exists. Specify --upgrade to force replacement.\u001b[0m\u001b[33m\n",
-      "\u001b[0m\u001b[33mWARNING: Target directory /Users/zaydsimjee/workspace/guardrails/.venv/lib/python3.11/site-packages/guardrails/hub/guardrails/two_words/two_words-0.0.0.dist-info already exists. Specify --upgrade to force replacement.\u001b[0m\u001b[33m\n",
-      "\u001b[0m\u001b[32m2024-03-25 16:13:59\u001b[0m \u001b[35mzmac\u001b[0m \u001b[34mguardrails-cli[96229]\u001b[0m \u001b[1;30mINFO\u001b[0m Collecting git+https://github.com/guardrails-ai/two_words.git\n",
-      "  Cloning https://github.com/guardrails-ai/two_words.git to /private/var/folders/c8/jqt82fpx785dpwpp36ljkgm40000gn/T/pip-req-build-c9bdjnpk\n",
-      "  Resolved https://github.com/guardrails-ai/two_words.git to commit e7f682c0b8d45a9407e966028d72682cd909601e\n",
-      "  Installing build dependencies: started\n",
-      "  Installing build dependencies: finished with status 'done'\n",
-      "  Getting requirements to build wheel: started\n",
-      "  Getting requirements to build wheel: finished with status 'done'\n",
-      "  Installing backend dependencies: started\n",
-      "  Installing backend dependencies: finished with status 'done'\n",
-      "  Preparing metadata (pyproject.toml): started\n",
-      "  Preparing metadata (pyproject.toml): finished with status 'done'\n",
-      "Building wheels for collected packages: two_words\n",
-      "  Building wheel for two_words (pyproject.toml): started\n",
-      "  Building wheel for two_words (pyproject.toml): finished with status 'done'\n",
-      "  Created wheel for two_words: filename=two_words-0.0.0-py3-none-any.whl size=11227 sha256=a10ae6f93738a3223ec28db3712fdc288547689235606516c589caff1f84889c\n",
-      "  Stored in directory: /private/var/folders/c8/jqt82fpx785dpwpp36ljkgm40000gn/T/pip-ephem-wheel-cache-b_l3rvkp/wheels/36/68/76/f184dbc7d9cea0daec56ec1394537018f2ddeb660f9ad79ce6\n",
-      "Successfully built two_words\n",
-      "Installing collected packages: two_words\n",
-      "Successfully installed two_words-0.0.0\n",
-      "\n",
-      "\u001b[32m2024-03-25 16:14:00\u001b[0m \u001b[35mzmac\u001b[0m \u001b[34mguardrails-cli[96229]\u001b[0m \u001b[1;30mSUCCESS\u001b[0m \u001b[1;32m\n",
+      "Installing hub:\u001b[35m/\u001b[0m\u001b[35m/guardrails/\u001b[0m\u001b[95mvalid_length...\u001b[0m\n",
+      "✅Successfully installed guardrails/valid_length!\n",
       "\n",
-      "    Successfully installed guardrails/two_words!\n",
       "\n",
-      "    See how to use it here: https://hub.guardrailsai.com/validator/guardrails/two_words\n",
-      "    \u001b[0m\n",
-      "\u001b[32m2024-03-25 16:14:01\u001b[0m \u001b[35mzmac\u001b[0m \u001b[34mguardrails-cli[96313]\u001b[0m \u001b[1;30mNOTICE\u001b[0m \u001b[1;36mInstalling hub://guardrails/valid_range...\u001b[0m\n",
-      "  Running command git clone --filter=blob:none --quiet https://github.com/guardrails-ai/valid_range.git /private/var/folders/c8/jqt82fpx785dpwpp36ljkgm40000gn/T/pip-req-build-_ndib8nc\n",
-      "\u001b[33mWARNING: Target directory /Users/zaydsimjee/workspace/guardrails/.venv/lib/python3.11/site-packages/guardrails/hub/guardrails/valid_range/validator already exists. Specify --upgrade to force replacement.\u001b[0m\u001b[33m\n",
-      "\u001b[0m\u001b[33mWARNING: Target directory /Users/zaydsimjee/workspace/guardrails/.venv/lib/python3.11/site-packages/guardrails/hub/guardrails/valid_range/valid_range-0.0.0.dist-info already exists. Specify --upgrade to force replacement.\u001b[0m\u001b[33m\n",
-      "\u001b[0m\u001b[32m2024-03-25 16:14:07\u001b[0m \u001b[35mzmac\u001b[0m \u001b[34mguardrails-cli[96313]\u001b[0m \u001b[1;30mINFO\u001b[0m Collecting git+https://github.com/guardrails-ai/valid_range.git\n",
-      "  Cloning https://github.com/guardrails-ai/valid_range.git to /private/var/folders/c8/jqt82fpx785dpwpp36ljkgm40000gn/T/pip-req-build-_ndib8nc\n",
-      "  Resolved https://github.com/guardrails-ai/valid_range.git to commit d01ad21d73d753ad224fd395bce18428196951c5\n",
-      "  Installing build dependencies: started\n",
-      "  Installing build dependencies: finished with status 'done'\n",
-      "  Getting requirements to build wheel: started\n",
-      "  Getting requirements to build wheel: finished with status 'done'\n",
-      "  Installing backend dependencies: started\n",
-      "  Installing backend dependencies: finished with status 'done'\n",
-      "  Preparing metadata (pyproject.toml): started\n",
-      "  Preparing metadata (pyproject.toml): finished with status 'done'\n",
-      "Building wheels for collected packages: valid_range\n",
-      "  Building wheel for valid_range (pyproject.toml): started\n",
-      "  Building wheel for valid_range (pyproject.toml): finished with status 'done'\n",
-      "  Created wheel for valid_range: filename=valid_range-0.0.0-py3-none-any.whl size=11575 sha256=ca6ffe0537e7a64c74332398596451c01395a6fa8086f98e514d0a1cfad5fff9\n",
-      "  Stored in directory: /private/var/folders/c8/jqt82fpx785dpwpp36ljkgm40000gn/T/pip-ephem-wheel-cache-7zrcqsov/wheels/0c/66/c0/f9ea25da535775c4ffca5bbd385863945a6397fd1863f6abe8\n",
-      "Successfully built valid_range\n",
-      "Installing collected packages: valid_range\n",
-      "Successfully installed valid_range-0.0.0\n",
+      "Installing hub:\u001b[35m/\u001b[0m\u001b[35m/guardrails/\u001b[0m\u001b[95mtwo_words...\u001b[0m\n",
+      "✅Successfully installed guardrails/two_words!\n",
       "\n",
-      "\u001b[32m2024-03-25 16:14:07\u001b[0m \u001b[35mzmac\u001b[0m \u001b[34mguardrails-cli[96313]\u001b[0m \u001b[1;30mSUCCESS\u001b[0m \u001b[1;32m\n",
       "\n",
-      "    Successfully installed guardrails/valid_range!\n",
+      "Installing hub:\u001b[35m/\u001b[0m\u001b[35m/guardrails/\u001b[0m\u001b[95mvalid_range...\u001b[0m\n",
+      "✅Successfully installed guardrails/valid_range!\n",
       "\n",
-      "    See how to use it here: https://hub.guardrailsai.com/validator/guardrails/valid_range\n",
-      "    \u001b[0m\n"
+      "\n"
      ]
     }
    ],
    "source": [
-    "!guardrails hub install hub://guardrails/valid_length\n",
-    "!guardrails hub install hub://guardrails/two_words\n",
-    "!guardrails hub install hub://guardrails/valid_range"
+    "!guardrails hub install hub://guardrails/valid_length --quiet\n",
+    "!guardrails hub install hub://guardrails/two_words --quiet\n",
+    "!guardrails hub install hub://guardrails/valid_range --quiet\n",
+    "!pip install cohere==5.3.2 --quiet"
    ]
   },
   {
@@ -158,7 +81,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
    "id": "3088fd99",
    "metadata": {},
    "outputs": [],
@@ -198,7 +121,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 4,
    "id": "840006ca-21ca-4f76-9ce1-e406d5d68412",
    "metadata": {},
    "outputs": [],
@@ -221,7 +144,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 5,
    "id": "42766922-14d0-4b5e-853a-23f05b896a09",
    "metadata": {},
    "outputs": [
@@ -229,22 +152,8 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "/Users/zaydsimjee/workspace/guardrails/.venv/lib/python3.11/site-packages/guardrails/validatorsattr.py:307: UserWarning: Validator 1-indexed is not installed!\n",
-      "  warnings.warn(f\"Validator {validator_name} is not installed!\")\n",
-      "/Users/zaydsimjee/workspace/guardrails/.venv/lib/python3.11/site-packages/guardrails/validators/__init__.py:50: FutureWarning: \n",
-      "    Importing validators from `guardrails.validators` is deprecated.\n",
-      "    All validators are now available in the Guardrails Hub. Please install\n",
-      "    and import them from the hub instead. All validators will be\n",
-      "    removed from this module in the next major release.\n",
-      "\n",
-      "    Install with: `guardrails hub install hub://<namespace>/<validator_name>`\n",
-      "    Import as: from guardrails.hub import `ValidatorName`\n",
-      "    \n",
-      "  warn(\n",
-      "\n",
-      "HTTP Request: POST https://api.cohere.ai/v1/chat \"HTTP/1.1 200 OK\"\n",
-      "Diffusion not supported. Skipping import.\n",
-      "HTTP Request: POST https://api.cohere.ai/v1/chat \"HTTP/1.1 200 OK\"\n"
+      "/Users/calebcourier/Projects/gr-mono/guardrails/docs/examples/.venv/lib/python3.11/site-packages/guardrails/validatorsattr.py:307: UserWarning: Validator 1-indexed is not installed!\n",
+      "  warnings.warn(f\"Validator {validator_name} is not installed!\")\n"
      ]
     }
    ],
@@ -272,7 +181,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 6,
    "id": "0e910d87",
    "metadata": {},
    "outputs": [
@@ -343,47 +252,47 @@
        "    │ <span style=\"background-color: #f5f5dc\">│         {                                                                                               │</span> │\n",
        "    │ <span style=\"background-color: #f5f5dc\">│             \"user_id\": 2,                                                                               │</span> │\n",
        "    │ <span style=\"background-color: #f5f5dc\">│             \"user_name\": \"Jane Smith\",                                                                  │</span> │\n",
-       "    │ <span style=\"background-color: #f5f5dc\">│             \"num_orders\": 10                                                                            │</span> │\n",
+       "    │ <span style=\"background-color: #f5f5dc\">│             \"num_orders\": 4                                                                             │</span> │\n",
        "    │ <span style=\"background-color: #f5f5dc\">│         },                                                                                              │</span> │\n",
        "    │ <span style=\"background-color: #f5f5dc\">│         {                                                                                               │</span> │\n",
        "    │ <span style=\"background-color: #f5f5dc\">│             \"user_id\": 3,                                                                               │</span> │\n",
        "    │ <span style=\"background-color: #f5f5dc\">│             \"user_name\": \"David Lee\",                                                                   │</span> │\n",
-       "    │ <span style=\"background-color: #f5f5dc\">│             \"num_orders\": 4                                                                             │</span> │\n",
+       "    │ <span style=\"background-color: #f5f5dc\">│             \"num_orders\": 2                                                                             │</span> │\n",
        "    │ <span style=\"background-color: #f5f5dc\">│         },                                                                                              │</span> │\n",
        "    │ <span style=\"background-color: #f5f5dc\">│         {                                                                                               │</span> │\n",
        "    │ <span style=\"background-color: #f5f5dc\">│             \"user_id\": 4,                                                                               │</span> │\n",
        "    │ <span style=\"background-color: #f5f5dc\">│             \"user_name\": \"Rachelle Gonzalez\",                                                           │</span> │\n",
-       "    │ <span style=\"background-color: #f5f5dc\">│             \"num_orders\": 2                                                                             │</span> │\n",
+       "    │ <span style=\"background-color: #f5f5dc\">│             \"num_orders\": 1                                                                             │</span> │\n",
        "    │ <span style=\"background-color: #f5f5dc\">│         },                                                                                              │</span> │\n",
        "    │ <span style=\"background-color: #f5f5dc\">│         {                                                                                               │</span> │\n",
        "    │ <span style=\"background-color: #f5f5dc\">│             \"user_id\": 5,                                                                               │</span> │\n",
-       "    │ <span style=\"background-color: #f5f5dc\">│             \"user_name\": \"Peter Brown\",                                                                 │</span> │\n",
+       "    │ <span style=\"background-color: #f5f5dc\">│             \"user_name\": \"Frank Anderson\",                                                              │</span> │\n",
        "    │ <span style=\"background-color: #f5f5dc\">│             \"num_orders\": 3                                                                             │</span> │\n",
        "    │ <span style=\"background-color: #f5f5dc\">│         },                                                                                              │</span> │\n",
        "    │ <span style=\"background-color: #f5f5dc\">│         {                                                                                               │</span> │\n",
        "    │ <span style=\"background-color: #f5f5dc\">│             \"user_id\": 6,                                                                               │</span> │\n",
-       "    │ <span style=\"background-color: #f5f5dc\">│             \"user_name\": \"Micheal Wilson\",                                                              │</span> │\n",
+       "    │ <span style=\"background-color: #f5f5dc\">│             \"user_name\": \"Lisa Taylor\",                                                                 │</span> │\n",
        "    │ <span style=\"background-color: #f5f5dc\">│             \"num_orders\": 5                                                                             │</span> │\n",
        "    │ <span style=\"background-color: #f5f5dc\">│         },                                                                                              │</span> │\n",
        "    │ <span style=\"background-color: #f5f5dc\">│         {                                                                                               │</span> │\n",
        "    │ <span style=\"background-color: #f5f5dc\">│             \"user_id\": 7,                                                                               │</span> │\n",
-       "    │ <span style=\"background-color: #f5f5dc\">│             \"user_name\": \"Sarah Jones\",                                                                 │</span> │\n",
-       "    │ <span style=\"background-color: #f5f5dc\">│             \"num_orders\": 0                                                                             │</span> │\n",
+       "    │ <span style=\"background-color: #f5f5dc\">│             \"user_name\": \"Peter Wilson\",                                                                │</span> │\n",
+       "    │ <span style=\"background-color: #f5f5dc\">│             \"num_orders\": 7                                                                             │</span> │\n",
        "    │ <span style=\"background-color: #f5f5dc\">│         },                                                                                              │</span> │\n",
        "    │ <span style=\"background-color: #f5f5dc\">│         {                                                                                               │</span> │\n",
        "    │ <span style=\"background-color: #f5f5dc\">│             \"user_id\": 8,                                                                               │</span> │\n",
-       "    │ <span style=\"background-color: #f5f5dc\">│             \"user_name\": \"Rachelle Perez\",                                                              │</span> │\n",
-       "    │ <span style=\"background-color: #f5f5dc\">│             \"num_orders\": 8                                                                             │</span> │\n",
+       "    │ <span style=\"background-color: #f5f5dc\">│             \"user_name\": \"Micheal Harris\",                                                              │</span> │\n",
+       "    │ <span style=\"background-color: #f5f5dc\">│             \"num_orders\": 4                                                                             │</span> │\n",
        "    │ <span style=\"background-color: #f5f5dc\">│         },                                                                                              │</span> │\n",
        "    │ <span style=\"background-color: #f5f5dc\">│         {                                                                                               │</span> │\n",
        "    │ <span style=\"background-color: #f5f5dc\">│             \"user_id\": 9,                                                                               │</span> │\n",
-       "    │ <span style=\"background-color: #f5f5dc\">│             \"user_name\": \"John Garcia\",                                                                 │</span> │\n",
-       "    │ <span style=\"background-color: #f5f5dc\">│             \"num_orders\": 1                                                                             │</span> │\n",
+       "    │ <span style=\"background-color: #f5f5dc\">│             \"user_name\": \"Sarah Anderson\",                                                              │</span> │\n",
+       "    │ <span style=\"background-color: #f5f5dc\">│             \"num_orders\": 2                                                                             │</span> │\n",
        "    │ <span style=\"background-color: #f5f5dc\">│         },                                                                                              │</span> │\n",
        "    │ <span style=\"background-color: #f5f5dc\">│         {                                                                                               │</span> │\n",
        "    │ <span style=\"background-color: #f5f5dc\">│             \"user_id\": 10,                                                                              │</span> │\n",
-       "    │ <span style=\"background-color: #f5f5dc\">│             \"user_name\": \"Jane Martinez\",                                                               │</span> │\n",
-       "    │ <span style=\"background-color: #f5f5dc\">│             \"num_orders\": 7                                                                             │</span> │\n",
+       "    │ <span style=\"background-color: #f5f5dc\">│             \"user_name\": \"Jessica Taylor\",                                                              │</span> │\n",
+       "    │ <span style=\"background-color: #f5f5dc\">│             \"num_orders\": 1                                                                             │</span> │\n",
        "    │ <span style=\"background-color: #f5f5dc\">│         }                                                                                               │</span> │\n",
        "    │ <span style=\"background-color: #f5f5dc\">│     ]                                                                                                   │</span> │\n",
        "    │ <span style=\"background-color: #f5f5dc\">│ }                                                                                                       │</span> │\n",
@@ -392,15 +301,15 @@
        "    │ <span style=\"background-color: #f0fff0\">│ {                                                                                                       │</span> │\n",
        "    │ <span style=\"background-color: #f0fff0\">│     'user_orders': [                                                                                    │</span> │\n",
        "    │ <span style=\"background-color: #f0fff0\">│         {'user_id': 1, 'user_name': 'John Mcdonald', 'num_orders': 6},                                  │</span> │\n",
-       "    │ <span style=\"background-color: #f0fff0\">│         {'user_id': 2, 'user_name': 'Jane Smith', 'num_orders': 10},                                    │</span> │\n",
-       "    │ <span style=\"background-color: #f0fff0\">│         {'user_id': 3, 'user_name': 'David Lee', 'num_orders': 4},                                      │</span> │\n",
-       "    │ <span style=\"background-color: #f0fff0\">│         {'user_id': 4, 'user_name': 'Rachelle Gonzalez', 'num_orders': 2},                              │</span> │\n",
-       "    │ <span style=\"background-color: #f0fff0\">│         {'user_id': 5, 'user_name': 'Peter Brown', 'num_orders': 3},                                    │</span> │\n",
-       "    │ <span style=\"background-color: #f0fff0\">│         {'user_id': 6, 'user_name': 'Micheal Wilson', 'num_orders': 5},                                 │</span> │\n",
-       "    │ <span style=\"background-color: #f0fff0\">│         {'user_id': 7, 'user_name': 'Sarah Jones', 'num_orders': 0},                                    │</span> │\n",
-       "    │ <span style=\"background-color: #f0fff0\">│         {'user_id': 8, 'user_name': 'Rachelle Perez', 'num_orders': 8},                                 │</span> │\n",
-       "    │ <span style=\"background-color: #f0fff0\">│         {'user_id': 9, 'user_name': 'John Garcia', 'num_orders': 1},                                    │</span> │\n",
-       "    │ <span style=\"background-color: #f0fff0\">│         {'user_id': 10, 'user_name': 'Jane Martinez', 'num_orders': 7}                                  │</span> │\n",
+       "    │ <span style=\"background-color: #f0fff0\">│         {'user_id': 2, 'user_name': 'Jane Smith', 'num_orders': 4},                                     │</span> │\n",
+       "    │ <span style=\"background-color: #f0fff0\">│         {'user_id': 3, 'user_name': 'David Lee', 'num_orders': 2},                                      │</span> │\n",
+       "    │ <span style=\"background-color: #f0fff0\">│         {'user_id': 4, 'user_name': 'Rachelle Gonzalez', 'num_orders': 1},                              │</span> │\n",
+       "    │ <span style=\"background-color: #f0fff0\">│         {'user_id': 5, 'user_name': 'Frank Anderson', 'num_orders': 3},                                 │</span> │\n",
+       "    │ <span style=\"background-color: #f0fff0\">│         {'user_id': 6, 'user_name': 'Lisa Taylor', 'num_orders': 5},                                    │</span> │\n",
+       "    │ <span style=\"background-color: #f0fff0\">│         {'user_id': 7, 'user_name': 'Peter Wilson', 'num_orders': 7},                                   │</span> │\n",
+       "    │ <span style=\"background-color: #f0fff0\">│         {'user_id': 8, 'user_name': 'Micheal Harris', 'num_orders': 4},                                 │</span> │\n",
+       "    │ <span style=\"background-color: #f0fff0\">│         {'user_id': 9, 'user_name': 'Sarah Anderson', 'num_orders': 2},                                 │</span> │\n",
+       "    │ <span style=\"background-color: #f0fff0\">│         {'user_id': 10, 'user_name': 'Jessica Taylor', 'num_orders': 1}                                 │</span> │\n",
        "    │ <span style=\"background-color: #f0fff0\">│     ]                                                                                                   │</span> │\n",
        "    │ <span style=\"background-color: #f0fff0\">│ }                                                                                                       │</span> │\n",
        "    │ <span style=\"background-color: #f0fff0\">╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span> │\n",
@@ -472,47 +381,47 @@
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m        {\u001b[0m\u001b[48;2;245;245;220m                                                                                              \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m            \"user_id\": 2,\u001b[0m\u001b[48;2;245;245;220m                                                                              \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m            \"user_name\": \"Jane Smith\",\u001b[0m\u001b[48;2;245;245;220m                                                                 \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
-       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m            \"num_orders\": 10\u001b[0m\u001b[48;2;245;245;220m                                                                           \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
+       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m            \"num_orders\": 4\u001b[0m\u001b[48;2;245;245;220m                                                                            \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m        },\u001b[0m\u001b[48;2;245;245;220m                                                                                             \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m        {\u001b[0m\u001b[48;2;245;245;220m                                                                                              \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m            \"user_id\": 3,\u001b[0m\u001b[48;2;245;245;220m                                                                              \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m            \"user_name\": \"David Lee\",\u001b[0m\u001b[48;2;245;245;220m                                                                  \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
-       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m            \"num_orders\": 4\u001b[0m\u001b[48;2;245;245;220m                                                                            \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
+       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m            \"num_orders\": 2\u001b[0m\u001b[48;2;245;245;220m                                                                            \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m        },\u001b[0m\u001b[48;2;245;245;220m                                                                                             \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m        {\u001b[0m\u001b[48;2;245;245;220m                                                                                              \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m            \"user_id\": 4,\u001b[0m\u001b[48;2;245;245;220m                                                                              \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m            \"user_name\": \"Rachelle Gonzalez\",\u001b[0m\u001b[48;2;245;245;220m                                                          \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
-       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m            \"num_orders\": 2\u001b[0m\u001b[48;2;245;245;220m                                                                            \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
+       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m            \"num_orders\": 1\u001b[0m\u001b[48;2;245;245;220m                                                                            \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m        },\u001b[0m\u001b[48;2;245;245;220m                                                                                             \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m        {\u001b[0m\u001b[48;2;245;245;220m                                                                                              \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m            \"user_id\": 5,\u001b[0m\u001b[48;2;245;245;220m                                                                              \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
-       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m            \"user_name\": \"Peter Brown\",\u001b[0m\u001b[48;2;245;245;220m                                                                \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
+       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m            \"user_name\": \"Frank Anderson\",\u001b[0m\u001b[48;2;245;245;220m                                                             \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m            \"num_orders\": 3\u001b[0m\u001b[48;2;245;245;220m                                                                            \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m        },\u001b[0m\u001b[48;2;245;245;220m                                                                                             \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m        {\u001b[0m\u001b[48;2;245;245;220m                                                                                              \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m            \"user_id\": 6,\u001b[0m\u001b[48;2;245;245;220m                                                                              \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
-       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m            \"user_name\": \"Micheal Wilson\",\u001b[0m\u001b[48;2;245;245;220m                                                             \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
+       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m            \"user_name\": \"Lisa Taylor\",\u001b[0m\u001b[48;2;245;245;220m                                                                \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m            \"num_orders\": 5\u001b[0m\u001b[48;2;245;245;220m                                                                            \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m        },\u001b[0m\u001b[48;2;245;245;220m                                                                                             \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m        {\u001b[0m\u001b[48;2;245;245;220m                                                                                              \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m            \"user_id\": 7,\u001b[0m\u001b[48;2;245;245;220m                                                                              \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
-       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m            \"user_name\": \"Sarah Jones\",\u001b[0m\u001b[48;2;245;245;220m                                                                \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
-       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m            \"num_orders\": 0\u001b[0m\u001b[48;2;245;245;220m                                                                            \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
+       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m            \"user_name\": \"Peter Wilson\",\u001b[0m\u001b[48;2;245;245;220m                                                               \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
+       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m            \"num_orders\": 7\u001b[0m\u001b[48;2;245;245;220m                                                                            \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m        },\u001b[0m\u001b[48;2;245;245;220m                                                                                             \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m        {\u001b[0m\u001b[48;2;245;245;220m                                                                                              \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m            \"user_id\": 8,\u001b[0m\u001b[48;2;245;245;220m                                                                              \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
-       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m            \"user_name\": \"Rachelle Perez\",\u001b[0m\u001b[48;2;245;245;220m                                                             \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
-       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m            \"num_orders\": 8\u001b[0m\u001b[48;2;245;245;220m                                                                            \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
+       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m            \"user_name\": \"Micheal Harris\",\u001b[0m\u001b[48;2;245;245;220m                                                             \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
+       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m            \"num_orders\": 4\u001b[0m\u001b[48;2;245;245;220m                                                                            \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m        },\u001b[0m\u001b[48;2;245;245;220m                                                                                             \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m        {\u001b[0m\u001b[48;2;245;245;220m                                                                                              \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m            \"user_id\": 9,\u001b[0m\u001b[48;2;245;245;220m                                                                              \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
-       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m            \"user_name\": \"John Garcia\",\u001b[0m\u001b[48;2;245;245;220m                                                                \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
-       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m            \"num_orders\": 1\u001b[0m\u001b[48;2;245;245;220m                                                                            \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
+       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m            \"user_name\": \"Sarah Anderson\",\u001b[0m\u001b[48;2;245;245;220m                                                             \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
+       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m            \"num_orders\": 2\u001b[0m\u001b[48;2;245;245;220m                                                                            \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m        },\u001b[0m\u001b[48;2;245;245;220m                                                                                             \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m        {\u001b[0m\u001b[48;2;245;245;220m                                                                                              \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m            \"user_id\": 10,\u001b[0m\u001b[48;2;245;245;220m                                                                             \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
-       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m            \"user_name\": \"Jane Martinez\",\u001b[0m\u001b[48;2;245;245;220m                                                              \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
-       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m            \"num_orders\": 7\u001b[0m\u001b[48;2;245;245;220m                                                                            \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
+       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m            \"user_name\": \"Jessica Taylor\",\u001b[0m\u001b[48;2;245;245;220m                                                             \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
+       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m            \"num_orders\": 1\u001b[0m\u001b[48;2;245;245;220m                                                                            \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m        }\u001b[0m\u001b[48;2;245;245;220m                                                                                              \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m    ]\u001b[0m\u001b[48;2;245;245;220m                                                                                                  \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m}\u001b[0m\u001b[48;2;245;245;220m                                                                                                      \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
@@ -521,15 +430,15 @@
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m{\u001b[0m\u001b[48;2;240;255;240m                                                                                                      \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m    'user_orders': [\u001b[0m\u001b[48;2;240;255;240m                                                                                   \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m        {'user_id': 1, 'user_name': 'John Mcdonald', 'num_orders': 6},\u001b[0m\u001b[48;2;240;255;240m                                 \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m        {'user_id': 2, 'user_name': 'Jane Smith', 'num_orders': 10},\u001b[0m\u001b[48;2;240;255;240m                                   \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m        {'user_id': 3, 'user_name': 'David Lee', 'num_orders': 4},\u001b[0m\u001b[48;2;240;255;240m                                     \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m        {'user_id': 4, 'user_name': 'Rachelle Gonzalez', 'num_orders': 2},\u001b[0m\u001b[48;2;240;255;240m                             \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m        {'user_id': 5, 'user_name': 'Peter Brown', 'num_orders': 3},\u001b[0m\u001b[48;2;240;255;240m                                   \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m        {'user_id': 6, 'user_name': 'Micheal Wilson', 'num_orders': 5},\u001b[0m\u001b[48;2;240;255;240m                                \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m        {'user_id': 7, 'user_name': 'Sarah Jones', 'num_orders': 0},\u001b[0m\u001b[48;2;240;255;240m                                   \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m        {'user_id': 8, 'user_name': 'Rachelle Perez', 'num_orders': 8},\u001b[0m\u001b[48;2;240;255;240m                                \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m        {'user_id': 9, 'user_name': 'John Garcia', 'num_orders': 1},\u001b[0m\u001b[48;2;240;255;240m                                   \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m        {'user_id': 10, 'user_name': 'Jane Martinez', 'num_orders': 7}\u001b[0m\u001b[48;2;240;255;240m                                 \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m        {'user_id': 2, 'user_name': 'Jane Smith', 'num_orders': 4},\u001b[0m\u001b[48;2;240;255;240m                                    \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m        {'user_id': 3, 'user_name': 'David Lee', 'num_orders': 2},\u001b[0m\u001b[48;2;240;255;240m                                     \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m        {'user_id': 4, 'user_name': 'Rachelle Gonzalez', 'num_orders': 1},\u001b[0m\u001b[48;2;240;255;240m                             \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m        {'user_id': 5, 'user_name': 'Frank Anderson', 'num_orders': 3},\u001b[0m\u001b[48;2;240;255;240m                                \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m        {'user_id': 6, 'user_name': 'Lisa Taylor', 'num_orders': 5},\u001b[0m\u001b[48;2;240;255;240m                                   \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m        {'user_id': 7, 'user_name': 'Peter Wilson', 'num_orders': 7},\u001b[0m\u001b[48;2;240;255;240m                                  \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m        {'user_id': 8, 'user_name': 'Micheal Harris', 'num_orders': 4},\u001b[0m\u001b[48;2;240;255;240m                                \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m        {'user_id': 9, 'user_name': 'Sarah Anderson', 'num_orders': 2},\u001b[0m\u001b[48;2;240;255;240m                                \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m        {'user_id': 10, 'user_name': 'Jessica Taylor', 'num_orders': 1}\u001b[0m\u001b[48;2;240;255;240m                                \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m    ]\u001b[0m\u001b[48;2;240;255;240m                                                                                                  \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m}\u001b[0m\u001b[48;2;240;255;240m                                                                                                      \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m │\n",
@@ -563,7 +472,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.7"
+   "version": "3.11.9"
   }
  },
  "nbformat": 4,

From 91f4bda33a4870ebd91d1d6c43b4370bda8b1cc3 Mon Sep 17 00:00:00 2001
From: Wyatt Lansford <22553069+wylansford@users.noreply.github.com>
Date: Thu, 6 Jun 2024 11:23:00 -0700
Subject: [PATCH 82/85] updating old model and api

---
 .../translation_to_specific_language.ipynb    | 202 ++++++++++++------
 1 file changed, 131 insertions(+), 71 deletions(-)

diff --git a/docs/examples/translation_to_specific_language.ipynb b/docs/examples/translation_to_specific_language.ipynb
index 2c03601f8..c8dacdc39 100644
--- a/docs/examples/translation_to_specific_language.ipynb
+++ b/docs/examples/translation_to_specific_language.ipynb
@@ -23,7 +23,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 5,
    "metadata": {
     "tags": []
    },
@@ -32,15 +32,25 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Requirement already satisfied: alt-profanity-check in /home/zayd/workspace/guardrails/.venv/lib/python3.9/site-packages (1.3.2)\n",
-      "Requirement already satisfied: joblib>=1.3.2 in /home/zayd/workspace/guardrails/.venv/lib/python3.9/site-packages (from alt-profanity-check) (1.3.2)\n",
-      "Requirement already satisfied: scikit-learn==1.3.2 in /home/zayd/workspace/guardrails/.venv/lib/python3.9/site-packages (from alt-profanity-check) (1.3.2)\n",
-      "Requirement already satisfied: scipy>=1.5.0 in /home/zayd/workspace/guardrails/.venv/lib/python3.9/site-packages (from scikit-learn==1.3.2->alt-profanity-check) (1.9.3)\n",
-      "Requirement already satisfied: numpy<2.0,>=1.17.3 in /home/zayd/workspace/guardrails/.venv/lib/python3.9/site-packages (from scikit-learn==1.3.2->alt-profanity-check) (1.24.4)\n",
-      "Requirement already satisfied: threadpoolctl>=2.0.0 in /home/zayd/workspace/guardrails/.venv/lib/python3.9/site-packages (from scikit-learn==1.3.2->alt-profanity-check) (3.2.0)\n",
-      "\n",
-      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.3.1\u001b[0m\n",
-      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
+      "Collecting alt-profanity-check\n",
+      "  Downloading alt_profanity_check-1.5.0.tar.gz (758 kB)\n",
+      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m759.0/759.0 kB\u001b[0m \u001b[31m7.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
+      "\u001b[?25h  Installing build dependencies ... \u001b[?25ldone\n",
+      "\u001b[?25h  Getting requirements to build wheel ... \u001b[?25ldone\n",
+      "\u001b[?25h  Installing backend dependencies ... \u001b[?25ldone\n",
+      "\u001b[?25h  Preparing metadata (pyproject.toml) ... \u001b[?25ldone\n",
+      "\u001b[?25hRequirement already satisfied: scikit-learn==1.5.0 in ./.venv/lib/python3.10/site-packages (from alt-profanity-check) (1.5.0)\n",
+      "Requirement already satisfied: joblib>=1.4.0 in ./.venv/lib/python3.10/site-packages (from alt-profanity-check) (1.4.2)\n",
+      "Requirement already satisfied: numpy>=1.19.5 in ./.venv/lib/python3.10/site-packages (from scikit-learn==1.5.0->alt-profanity-check) (1.26.4)\n",
+      "Requirement already satisfied: scipy>=1.6.0 in ./.venv/lib/python3.10/site-packages (from scikit-learn==1.5.0->alt-profanity-check) (1.13.1)\n",
+      "Requirement already satisfied: threadpoolctl>=3.1.0 in ./.venv/lib/python3.10/site-packages (from scikit-learn==1.5.0->alt-profanity-check) (3.5.0)\n",
+      "Building wheels for collected packages: alt-profanity-check\n",
+      "  Building wheel for alt-profanity-check (pyproject.toml) ... \u001b[?25ldone\n",
+      "\u001b[?25h  Created wheel for alt-profanity-check: filename=alt_profanity_check-1.5.0-py3-none-any.whl size=758311 sha256=e0f54f82189ad2c90aeb27cb9239175c71d38606836be9e4762fb64b2e2de0a0\n",
+      "  Stored in directory: /Users/wyatt/Library/Caches/pip/wheels/18/c3/20/637574a9badb43cace85202ca31f49f47e3fe65e076459f3ed\n",
+      "Successfully built alt-profanity-check\n",
+      "Installing collected packages: alt-profanity-check\n",
+      "Successfully installed alt-profanity-check-1.5.0\n"
      ]
     }
    ],
@@ -70,7 +80,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 6,
    "metadata": {
     "tags": []
    },
@@ -79,14 +89,30 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "/home/zayd/workspace/guardrails/.venv/lib/python3.9/site-packages/torch/cuda/__init__.py:611: UserWarning: Can't initialize NVML\n",
-      "  warnings.warn(\"Can't initialize NVML\")\n"
+      "/Users/wyatt/Projects/guardrails/docs/examples/.venv/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n",
+      "/Users/wyatt/Projects/guardrails/guardrails/validators/__init__.py:51: FutureWarning: \n",
+      "    Importing validators from `guardrails.validators` is deprecated.\n",
+      "    All validators are now available in the Guardrails Hub. Please install\n",
+      "    and import them from the hub instead. All validators will be\n",
+      "    removed from this module in the next major release.\n",
+      "\n",
+      "    Install with: `guardrails hub install hub://<namespace>/<validator_name>`\n",
+      "    Import as: from guardrails.hub import `ValidatorName`\n",
+      "    \n",
+      "  warn(\n"
      ]
     }
    ],
    "source": [
     "from profanity_check import predict\n",
-    "from guardrails.validators import Validator, register_validator, ValidationResult, PassResult, FailResult\n",
+    "from guardrails.validators import (\n",
+    "    Validator,\n",
+    "    register_validator,\n",
+    "    ValidationResult,\n",
+    "    PassResult,\n",
+    "    FailResult,\n",
+    ")\n",
     "\n",
     "\n",
     "from typing import Dict, Any\n",
@@ -113,7 +139,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 7,
    "metadata": {
     "tags": []
    },
@@ -153,9 +179,23 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 8,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/wyatt/Projects/guardrails/guardrails/validator_base.py:460: FutureWarning: Accessing `IsProfanityFree` using\n",
+      "`from guardrails.validators import IsProfanityFree` is deprecated and\n",
+      "support will be removed after version 0.5.x. Please switch to the Guardrails Hub syntax:\n",
+      "`from guardrails.hub import ProfanityFree` for future updates and support.\n",
+      "For additional details, please visit: https://hub.guardrailsai.com/validator/guardrails/profanity_free.\n",
+      "\n",
+      "  warn(\n"
+     ]
+    }
+   ],
    "source": [
     "from pydantic import BaseModel, Field\n",
     "\n",
@@ -167,11 +207,12 @@
     "${gr.complete_json_suffix}\n",
     "\"\"\"\n",
     "\n",
+    "\n",
     "class Translation(BaseModel):\n",
     "    translated_statement: str = Field(\n",
     "        description=\"Translate the given statement into english language\",\n",
-    "        validators=[IsProfanityFree(on_fail=\"fix\")]\n",
-    "        )"
+    "        validators=[IsProfanityFree(on_fail=\"fix\")],\n",
+    "    )"
    ]
   },
   {
@@ -198,7 +239,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -216,11 +257,20 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 10,
    "metadata": {
     "tags": []
    },
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/wyatt/Projects/guardrails/guardrails/prompt/base_prompt.py:59: FutureWarning: Prompt Primitives are moving! To keep the same behaviour, switch from `json` constants to `xml` constants. Example: ${gr.complete_json_suffix} -> ${gr.complete_xml_suffix}\n",
+      "  warn(\n"
+     ]
+    }
+   ],
    "source": [
     "guard = gd.Guard.from_rail_string(rail_str)"
    ]
@@ -234,9 +284,25 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 11,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/wyatt/Projects/guardrails/guardrails/validator_base.py:460: FutureWarning: Accessing `IsProfanityFree` using\n",
+      "`from guardrails.validators import IsProfanityFree` is deprecated and\n",
+      "support will be removed after version 0.5.x. Please switch to the Guardrails Hub syntax:\n",
+      "`from guardrails.hub import ProfanityFree` for future updates and support.\n",
+      "For additional details, please visit: https://hub.guardrailsai.com/validator/guardrails/profanity_free.\n",
+      "\n",
+      "  warn(\n",
+      "/Users/wyatt/Projects/guardrails/guardrails/prompt/base_prompt.py:59: FutureWarning: Prompt Primitives are moving! To keep the same behaviour, switch from `json` constants to `xml` constants. Example: ${gr.complete_json_suffix} -> ${gr.complete_xml_suffix}\n",
+      "  warn(\n"
+     ]
+    }
+   ],
    "source": [
     "guard = gd.Guard.from_pydantic(output_class=Translation, prompt=prompt)"
    ]
@@ -250,11 +316,19 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 12,
    "metadata": {
     "tags": []
    },
    "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/var/folders/8n/8qwytjb11kj_46_w3n2v4jzw0000gn/T/ipykernel_6330/3983563700.py:1: DeprecationWarning: 'Guard.base_prompt' is deprecated and will be removed in versions 0.5.x and beyond. Use 'Guard.history.last.prompt' instead.\n",
+      "  print(guard.base_prompt)\n"
+     ]
+    },
     {
      "data": {
       "text/html": [
@@ -350,21 +424,14 @@
    "execution_count": 13,
    "metadata": {},
    "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "HTTP Request: POST https://api.openai.com/v1/completions \"HTTP/1.1 200 OK\"\n"
-     ]
-    },
     {
      "data": {
       "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Validated Output: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'translated_statement'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'Chicken Quesadilla'</span><span style=\"font-weight: bold\">}</span>\n",
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Validated Output: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'translated_statement'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'Chicken quesadilla'</span><span style=\"font-weight: bold\">}</span>\n",
        "</pre>\n"
       ],
       "text/plain": [
-       "Validated Output: \u001b[1m{\u001b[0m\u001b[32m'translated_statement'\u001b[0m: \u001b[32m'Chicken Quesadilla'\u001b[0m\u001b[1m}\u001b[0m\n"
+       "Validated Output: \u001b[1m{\u001b[0m\u001b[32m'translated_statement'\u001b[0m: \u001b[32m'Chicken quesadilla'\u001b[0m\u001b[1m}\u001b[0m\n"
       ]
      },
      "metadata": {},
@@ -375,9 +442,9 @@
     "import openai\n",
     "\n",
     "raw_llm_response, validated_response, *rest = guard(\n",
-    "    openai.completions.create,\n",
+    "    openai.chat.completions.create,\n",
     "    prompt_params={\"statement_to_be_translated\": \"quesadilla de pollo\"},\n",
-    "    model=\"text-davinci-003\",\n",
+    "    model=\"gpt-3.5-turbo\",\n",
     "    max_tokens=2048,\n",
     "    temperature=0,\n",
     ")\n",
@@ -394,7 +461,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": 14,
    "metadata": {},
    "outputs": [
     {
@@ -432,19 +499,19 @@
        "    │ <span style=\"background-color: #f0f8ff\">│ format=\"1-indexed\" /&gt;&lt;/object&gt;` =&gt; `{'baz': {'foo': 'Some String', 'index': 1}}`                        │</span> │\n",
        "    │ <span style=\"background-color: #f0f8ff\">│                                                                                                         │</span> │\n",
        "    │ <span style=\"background-color: #f0f8ff\">│                                                                                                         │</span> │\n",
-       "    │ <span style=\"background-color: #f0f8ff\">│                                                                                                         │</span> │\n",
-       "    │ <span style=\"background-color: #f0f8ff\">│ Json Output:                                                                                            │</span> │\n",
-       "    │ <span style=\"background-color: #f0f8ff\">│                                                                                                         │</span> │\n",
-       "    │ <span style=\"background-color: #f0f8ff\">│                                                                                                         │</span> │\n",
        "    │ <span style=\"background-color: #f0f8ff\">╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span> │\n",
+       "    │ <span style=\"background-color: #fff0f2\">╭───────────────────────────────────────────── Instructions ──────────────────────────────────────────────╮</span> │\n",
+       "    │ <span style=\"background-color: #fff0f2\">│ You are a helpful assistant, able to express yourself purely through JSON, strictly and precisely       │</span> │\n",
+       "    │ <span style=\"background-color: #fff0f2\">│ adhering to the provided XML schemas.                                                                   │</span> │\n",
+       "    │ <span style=\"background-color: #fff0f2\">╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span> │\n",
        "    │ <span style=\"background-color: #e7dfeb\">╭──────────────────────────────────────────── Message History ────────────────────────────────────────────╮</span> │\n",
        "    │ <span style=\"background-color: #e7dfeb\">│ No message history.                                                                                     │</span> │\n",
        "    │ <span style=\"background-color: #e7dfeb\">╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span> │\n",
        "    │ <span style=\"background-color: #f5f5dc\">╭──────────────────────────────────────────── Raw LLM Output ─────────────────────────────────────────────╮</span> │\n",
-       "    │ <span style=\"background-color: #f5f5dc\">│ {\"translated_statement\": \"Chicken Quesadilla\"}                                                          │</span> │\n",
+       "    │ <span style=\"background-color: #f5f5dc\">│ {\"translated_statement\":\"Chicken quesadilla\"}                                                           │</span> │\n",
        "    │ <span style=\"background-color: #f5f5dc\">╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span> │\n",
        "    │ <span style=\"background-color: #f0fff0\">╭─────────────────────────────────────────── Validated Output ────────────────────────────────────────────╮</span> │\n",
-       "    │ <span style=\"background-color: #f0fff0\">│ {'translated_statement': 'Chicken Quesadilla'}                                                          │</span> │\n",
+       "    │ <span style=\"background-color: #f0fff0\">│ {'translated_statement': 'Chicken quesadilla'}                                                          │</span> │\n",
        "    │ <span style=\"background-color: #f0fff0\">╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span> │\n",
        "    ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
        "</pre>\n"
@@ -482,19 +549,19 @@
        "    │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255mformat=\"1-indexed\" /></object>` => `{'baz': {'foo': 'Some String', 'index': 1}}`\u001b[0m\u001b[48;2;240;248;255m                       \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m                                                                                                       \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m                                                                                                       \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m                                                                                                       \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255mJson Output:\u001b[0m\u001b[48;2;240;248;255m                                                                                           \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m                                                                                                       \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m                                                                                                       \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;248;255m╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m │\n",
+       "    │ \u001b[48;2;255;240;242m╭─\u001b[0m\u001b[48;2;255;240;242m────────────────────────────────────────────\u001b[0m\u001b[48;2;255;240;242m Instructions \u001b[0m\u001b[48;2;255;240;242m─────────────────────────────────────────────\u001b[0m\u001b[48;2;255;240;242m─╮\u001b[0m │\n",
+       "    │ \u001b[48;2;255;240;242m│\u001b[0m\u001b[48;2;255;240;242m \u001b[0m\u001b[48;2;255;240;242mYou are a helpful assistant, able to express yourself purely through JSON, strictly and precisely \u001b[0m\u001b[48;2;255;240;242m     \u001b[0m\u001b[48;2;255;240;242m \u001b[0m\u001b[48;2;255;240;242m│\u001b[0m │\n",
+       "    │ \u001b[48;2;255;240;242m│\u001b[0m\u001b[48;2;255;240;242m \u001b[0m\u001b[48;2;255;240;242madhering to the provided XML schemas.\u001b[0m\u001b[48;2;255;240;242m                                                                  \u001b[0m\u001b[48;2;255;240;242m \u001b[0m\u001b[48;2;255;240;242m│\u001b[0m │\n",
+       "    │ \u001b[48;2;255;240;242m╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m │\n",
        "    │ \u001b[48;2;231;223;235m╭─\u001b[0m\u001b[48;2;231;223;235m───────────────────────────────────────────\u001b[0m\u001b[48;2;231;223;235m Message History \u001b[0m\u001b[48;2;231;223;235m───────────────────────────────────────────\u001b[0m\u001b[48;2;231;223;235m─╮\u001b[0m │\n",
        "    │ \u001b[48;2;231;223;235m│\u001b[0m\u001b[48;2;231;223;235m \u001b[0m\u001b[48;2;231;223;235mNo message history.\u001b[0m\u001b[48;2;231;223;235m                                                                                    \u001b[0m\u001b[48;2;231;223;235m \u001b[0m\u001b[48;2;231;223;235m│\u001b[0m │\n",
        "    │ \u001b[48;2;231;223;235m╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m╭─\u001b[0m\u001b[48;2;245;245;220m───────────────────────────────────────────\u001b[0m\u001b[48;2;245;245;220m Raw LLM Output \u001b[0m\u001b[48;2;245;245;220m────────────────────────────────────────────\u001b[0m\u001b[48;2;245;245;220m─╮\u001b[0m │\n",
-       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m{\"translated_statement\": \"Chicken Quesadilla\"}\u001b[0m\u001b[48;2;245;245;220m                                                         \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
+       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m{\"translated_statement\":\"Chicken quesadilla\"}\u001b[0m\u001b[48;2;245;245;220m                                                          \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m╭─\u001b[0m\u001b[48;2;240;255;240m──────────────────────────────────────────\u001b[0m\u001b[48;2;240;255;240m Validated Output \u001b[0m\u001b[48;2;240;255;240m───────────────────────────────────────────\u001b[0m\u001b[48;2;240;255;240m─╮\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m{'translated_statement': 'Chicken Quesadilla'}\u001b[0m\u001b[48;2;240;255;240m                                                         \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m{'translated_statement': 'Chicken quesadilla'}\u001b[0m\u001b[48;2;240;255;240m                                                         \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m │\n",
        "    ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────╯\n"
       ]
@@ -519,18 +586,11 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 20,
+   "execution_count": 15,
    "metadata": {
     "tags": []
    },
    "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "HTTP Request: POST https://api.openai.com/v1/completions \"HTTP/1.1 200 OK\"\n"
-     ]
-    },
     {
      "data": {
       "text/html": [
@@ -547,9 +607,9 @@
    ],
    "source": [
     "raw_llm_response, validated_response, *rest = guard(\n",
-    "    openai.completions.create,\n",
+    "    openai.chat.completions.create,\n",
     "    prompt_params={\"statement_to_be_translated\": \"убей себя\"},\n",
-    "    model=\"text-davinci-003\",\n",
+    "    model=\"gpt-3.5-turbo\",\n",
     "    max_tokens=2048,\n",
     "    temperature=0,\n",
     ")\n",
@@ -567,7 +627,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 21,
+   "execution_count": 16,
    "metadata": {},
    "outputs": [
     {
@@ -605,16 +665,16 @@
        "    │ <span style=\"background-color: #f0f8ff\">│ format=\"1-indexed\" /&gt;&lt;/object&gt;` =&gt; `{'baz': {'foo': 'Some String', 'index': 1}}`                        │</span> │\n",
        "    │ <span style=\"background-color: #f0f8ff\">│                                                                                                         │</span> │\n",
        "    │ <span style=\"background-color: #f0f8ff\">│                                                                                                         │</span> │\n",
-       "    │ <span style=\"background-color: #f0f8ff\">│                                                                                                         │</span> │\n",
-       "    │ <span style=\"background-color: #f0f8ff\">│ Json Output:                                                                                            │</span> │\n",
-       "    │ <span style=\"background-color: #f0f8ff\">│                                                                                                         │</span> │\n",
-       "    │ <span style=\"background-color: #f0f8ff\">│                                                                                                         │</span> │\n",
        "    │ <span style=\"background-color: #f0f8ff\">╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span> │\n",
+       "    │ <span style=\"background-color: #fff0f2\">╭───────────────────────────────────────────── Instructions ──────────────────────────────────────────────╮</span> │\n",
+       "    │ <span style=\"background-color: #fff0f2\">│ You are a helpful assistant, able to express yourself purely through JSON, strictly and precisely       │</span> │\n",
+       "    │ <span style=\"background-color: #fff0f2\">│ adhering to the provided XML schemas.                                                                   │</span> │\n",
+       "    │ <span style=\"background-color: #fff0f2\">╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span> │\n",
        "    │ <span style=\"background-color: #e7dfeb\">╭──────────────────────────────────────────── Message History ────────────────────────────────────────────╮</span> │\n",
        "    │ <span style=\"background-color: #e7dfeb\">│ No message history.                                                                                     │</span> │\n",
        "    │ <span style=\"background-color: #e7dfeb\">╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span> │\n",
        "    │ <span style=\"background-color: #f5f5dc\">╭──────────────────────────────────────────── Raw LLM Output ─────────────────────────────────────────────╮</span> │\n",
-       "    │ <span style=\"background-color: #f5f5dc\">│ {\"translated_statement\": \"Kill yourself\"}                                                               │</span> │\n",
+       "    │ <span style=\"background-color: #f5f5dc\">│ {\"translated_statement\":\"Kill yourself\"}                                                                │</span> │\n",
        "    │ <span style=\"background-color: #f5f5dc\">╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span> │\n",
        "    │ <span style=\"background-color: #f0fff0\">╭─────────────────────────────────────────── Validated Output ────────────────────────────────────────────╮</span> │\n",
        "    │ <span style=\"background-color: #f0fff0\">│ {'translated_statement': ''}                                                                            │</span> │\n",
@@ -655,16 +715,16 @@
        "    │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255mformat=\"1-indexed\" /></object>` => `{'baz': {'foo': 'Some String', 'index': 1}}`\u001b[0m\u001b[48;2;240;248;255m                       \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m                                                                                                       \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m                                                                                                       \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m                                                                                                       \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255mJson Output:\u001b[0m\u001b[48;2;240;248;255m                                                                                           \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m                                                                                                       \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m                                                                                                       \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;248;255m╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m │\n",
+       "    │ \u001b[48;2;255;240;242m╭─\u001b[0m\u001b[48;2;255;240;242m────────────────────────────────────────────\u001b[0m\u001b[48;2;255;240;242m Instructions \u001b[0m\u001b[48;2;255;240;242m─────────────────────────────────────────────\u001b[0m\u001b[48;2;255;240;242m─╮\u001b[0m │\n",
+       "    │ \u001b[48;2;255;240;242m│\u001b[0m\u001b[48;2;255;240;242m \u001b[0m\u001b[48;2;255;240;242mYou are a helpful assistant, able to express yourself purely through JSON, strictly and precisely \u001b[0m\u001b[48;2;255;240;242m     \u001b[0m\u001b[48;2;255;240;242m \u001b[0m\u001b[48;2;255;240;242m│\u001b[0m │\n",
+       "    │ \u001b[48;2;255;240;242m│\u001b[0m\u001b[48;2;255;240;242m \u001b[0m\u001b[48;2;255;240;242madhering to the provided XML schemas.\u001b[0m\u001b[48;2;255;240;242m                                                                  \u001b[0m\u001b[48;2;255;240;242m \u001b[0m\u001b[48;2;255;240;242m│\u001b[0m │\n",
+       "    │ \u001b[48;2;255;240;242m╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m │\n",
        "    │ \u001b[48;2;231;223;235m╭─\u001b[0m\u001b[48;2;231;223;235m───────────────────────────────────────────\u001b[0m\u001b[48;2;231;223;235m Message History \u001b[0m\u001b[48;2;231;223;235m───────────────────────────────────────────\u001b[0m\u001b[48;2;231;223;235m─╮\u001b[0m │\n",
        "    │ \u001b[48;2;231;223;235m│\u001b[0m\u001b[48;2;231;223;235m \u001b[0m\u001b[48;2;231;223;235mNo message history.\u001b[0m\u001b[48;2;231;223;235m                                                                                    \u001b[0m\u001b[48;2;231;223;235m \u001b[0m\u001b[48;2;231;223;235m│\u001b[0m │\n",
        "    │ \u001b[48;2;231;223;235m╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m╭─\u001b[0m\u001b[48;2;245;245;220m───────────────────────────────────────────\u001b[0m\u001b[48;2;245;245;220m Raw LLM Output \u001b[0m\u001b[48;2;245;245;220m────────────────────────────────────────────\u001b[0m\u001b[48;2;245;245;220m─╮\u001b[0m │\n",
-       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m{\"translated_statement\": \"Kill yourself\"}\u001b[0m\u001b[48;2;245;245;220m                                                              \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
+       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m{\"translated_statement\":\"Kill yourself\"}\u001b[0m\u001b[48;2;245;245;220m                                                               \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m╭─\u001b[0m\u001b[48;2;240;255;240m──────────────────────────────────────────\u001b[0m\u001b[48;2;240;255;240m Validated Output \u001b[0m\u001b[48;2;240;255;240m───────────────────────────────────────────\u001b[0m\u001b[48;2;240;255;240m─╮\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m{'translated_statement': ''}\u001b[0m\u001b[48;2;240;255;240m                                                                           \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
@@ -697,7 +757,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.17"
+   "version": "3.10.3"
   }
  },
  "nbformat": 4,

From ed58b7a80f4e9b4217daa47d7169b1a13bfab38d Mon Sep 17 00:00:00 2001
From: Wyatt Lansford <22553069+wylansford@users.noreply.github.com>
Date: Thu, 6 Jun 2024 11:24:01 -0700
Subject: [PATCH 83/85] updating old model and api

---
 docs/examples/translation_with_quality_check.ipynb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/examples/translation_with_quality_check.ipynb b/docs/examples/translation_with_quality_check.ipynb
index 3429514fb..eb627fdba 100644
--- a/docs/examples/translation_with_quality_check.ipynb
+++ b/docs/examples/translation_with_quality_check.ipynb
@@ -360,10 +360,10 @@
     "statement = \"Ich habe keine Ahnung, was ich hier schreiben soll.\"\n",
     "\n",
     "raw_llm_response, validated_response, *rest = guard(\n",
-    "    openai.completions.create,\n",
+    "    openai.chat.completions.create,\n",
     "    prompt_params={\"statement_to_be_translated\": statement},\n",
     "    metadata={\"translation_source\": statement},\n",
-    "    model=\"text-davinci-003\",\n",
+    "    model=\"gpt-3.5-turbo\",\n",
     "    max_tokens=1024,\n",
     "    temperature=0,\n",
     ")\n",

From cea762318149ff4ae14e70ea8d2a6883addca5b6 Mon Sep 17 00:00:00 2001
From: Wyatt Lansford <22553069+wylansford@users.noreply.github.com>
Date: Thu, 6 Jun 2024 11:25:13 -0700
Subject: [PATCH 84/85] updating old model and api

---
 .../examples/text_summarization_quality.ipynb | 34 +++++++++++--------
 docs/examples/valid_chess_moves.ipynb         |  8 ++---
 2 files changed, 24 insertions(+), 18 deletions(-)

diff --git a/docs/examples/text_summarization_quality.ipynb b/docs/examples/text_summarization_quality.ipynb
index cca6538eb..d40e8610b 100644
--- a/docs/examples/text_summarization_quality.ipynb
+++ b/docs/examples/text_summarization_quality.ipynb
@@ -74,10 +74,10 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "with open('data/article1.txt', 'r') as file:\n",
+    "with open(\"data/article1.txt\", \"r\") as file:\n",
     "    document = file.read()\n",
     "    file.seek(0)\n",
-    "    content = ''.join(line.strip() for line in file.readlines())"
+    "    content = \"\".join(line.strip() for line in file.readlines())"
    ]
   },
   {
@@ -95,7 +95,8 @@
    "source": [
     "from string import Template\n",
     "\n",
-    "rail_str = Template(\"\"\"\n",
+    "rail_str = Template(\n",
+    "    \"\"\"\n",
     "<rail version=\"0.1\">\n",
     "\n",
     "<output>\n",
@@ -115,7 +116,8 @@
     "${gr.complete_json_suffix}\n",
     "</prompt>\n",
     "</rail>\n",
-    "\"\"\").safe_substitute(document=document)"
+    "\"\"\"\n",
+    ").safe_substitute(document=document)"
    ]
   },
   {
@@ -132,6 +134,7 @@
    "outputs": [],
    "source": [
     "from pydantic import BaseModel, Field\n",
+    "\n",
     "from guardrails.hub import SimilarToDocument\n",
     "\n",
     "prompt = \"\"\"\n",
@@ -142,10 +145,13 @@
     "${gr.complete_json_suffix}\n",
     "\"\"\"\n",
     "\n",
+    "\n",
     "class DocumentSummary(BaseModel):\n",
     "    summary: str = Field(\n",
     "        description=\"Summarize the given document faithfully.\",\n",
-    "        validators=[SimilarToDocument(document=f\"'{content}'\", threshold=0.60, on_fail=\"filter\")]\n",
+    "        validators=[\n",
+    "            SimilarToDocument(document=f\"'{content}'\", threshold=0.60, on_fail=\"filter\")\n",
+    "        ],\n",
     "    )"
    ]
   },
@@ -178,9 +184,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import guardrails as gd\n",
+    "from rich import print\n",
     "\n",
-    "from rich import print"
+    "import guardrails as gd"
    ]
   },
   {
@@ -354,11 +360,11 @@
     "import openai\n",
     "\n",
     "raw_llm_response, validated_response, *rest = guard(\n",
-    "    openai.completions.create,\n",
-    "    prompt_params={'document': document},\n",
-    "    model='text-davinci-003',\n",
+    "    openai.chat.completions.create,\n",
+    "    prompt_params={\"document\": document},\n",
+    "    model=\"gpt-3.5-turbo\",\n",
     "    max_tokens=2048,\n",
-    "    temperature=0\n",
+    "    temperature=0,\n",
     ")\n",
     "\n",
     "print(f\"Validated Output: {validated_response}\")"
@@ -609,10 +615,10 @@
    "source": [
     "raw_llm_response, validated_response, *rest = guard(\n",
     "    openai.completions.create,\n",
-    "    prompt_params={'document': open(\"data/article1.txt\", \"r\").read()},\n",
-    "    model='text-ada-001',\n",
+    "    prompt_params={\"document\": open(\"data/article1.txt\", \"r\").read()},\n",
+    "    model=\"text-ada-001\",\n",
     "    max_tokens=512,\n",
-    "    temperature=0\n",
+    "    temperature=0,\n",
     ")\n",
     "\n",
     "print(f\"Validated Output: {validated_response}\")"
diff --git a/docs/examples/valid_chess_moves.ipynb b/docs/examples/valid_chess_moves.ipynb
index 46b5a5c44..57980d31d 100644
--- a/docs/examples/valid_chess_moves.ipynb
+++ b/docs/examples/valid_chess_moves.ipynb
@@ -339,13 +339,13 @@
     "import openai\n",
     "\n",
     "raw_llm_response, validated_response, *rest = guard(\n",
-    "    openai.completions.create,\n",
+    "    openai.chat.completions.create,\n",
     "    prompt_params={\n",
     "        \"board_state\": str(board.move_stack)\n",
     "        if board.move_stack\n",
     "        else \"Starting position.\"\n",
     "    },\n",
-    "    model=\"text-davinci-003\",\n",
+    "    model=\"gpt-3.5-turbo\",\n",
     "    max_tokens=2048,\n",
     "    temperature=0.3,\n",
     ")"
@@ -476,13 +476,13 @@
    ],
    "source": [
     "raw_llm_response, validated_response, *rest = guard(\n",
-    "    openai.completions.create,\n",
+    "    openai.chat.completions.create,\n",
     "    prompt_params={\n",
     "        \"board_state\": str(board.move_stack)\n",
     "        if board.move_stack\n",
     "        else \"Starting position.\"\n",
     "    },\n",
-    "    model=\"text-davinci-003\",\n",
+    "    model=\"gpt-3.5-turbo\",\n",
     "    max_tokens=2048,\n",
     "    temperature=0.3,\n",
     ")"

From fad055198446b1aa6d4214fc701bf543514f90fb Mon Sep 17 00:00:00 2001
From: Wyatt Lansford <22553069+wylansford@users.noreply.github.com>
Date: Thu, 6 Jun 2024 11:35:31 -0700
Subject: [PATCH 85/85] adding todo

---
 guardrails/run/async_runner.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/guardrails/run/async_runner.py b/guardrails/run/async_runner.py
index 45bb2bd74..762cfa0d7 100644
--- a/guardrails/run/async_runner.py
+++ b/guardrails/run/async_runner.py
@@ -144,6 +144,7 @@ async def async_run(
 
         return call_log
 
+    # TODO: Do we want to revert this name to step?
     @async_trace(name="async_step")
     async def async_step(
         self,