Add ability to specify the evaluation name for all provided Evaluators (#1725)

Author: dmontagu

pydantic_evals/pydantic_evals/dataset.py

@@ -1036,14 +1036,14 @@ def _get_registry(
             raise ValueError(
                 f'All custom evaluator classes must be decorated with `@dataclass`, but {evaluator_class} is not'
             )
-        name = evaluator_class.name()
+        name = evaluator_class.get_serialization_name()
         if name in registry:
             raise ValueError(f'Duplicate evaluator class name: {name!r}')
         registry[name] = evaluator_class
 
     for evaluator_class in DEFAULT_EVALUATORS:
         # Allow overriding the default evaluators with custom evaluators raising an error
-        registry.setdefault(evaluator_class.name(), evaluator_class)
+        registry.setdefault(evaluator_class.get_serialization_name(), evaluator_class)
 
     return registry
 

pydantic_evals/pydantic_evals/evaluators/__init__.py

@@ -1,4 +1,14 @@
-from .common import Contains, Equals, EqualsExpected, HasMatchingSpan, IsInstance, LLMJudge, MaxDuration, Python
+from .common import (
+    Contains,
+    Equals,
+    EqualsExpected,
+    HasMatchingSpan,
+    IsInstance,
+    LLMJudge,
+    MaxDuration,
+    OutputConfig,
+    Python,
+)
 from .context import EvaluatorContext
 from .evaluator import EvaluationReason, EvaluationResult, Evaluator, EvaluatorOutput
 
@@ -11,6 +21,7 @@
     'MaxDuration',
     'LLMJudge',
     'HasMatchingSpan',
+    'OutputConfig',
     'Python',
     # context
     'EvaluatorContext',

pydantic_evals/pydantic_evals/evaluators/_run_evaluator.py

@@ -42,7 +42,7 @@ async def run_evaluator(
     except ValidationError as e:
         raise ValueError(f'{evaluator!r}.evaluate returned a value of an invalid type: {raw_results!r}.') from e
 
-    results = _convert_to_mapping(results, scalar_name=evaluator.name())
+    results = _convert_to_mapping(results, scalar_name=evaluator.get_default_evaluation_name())
 
     details: list[EvaluationResult] = []
     for name, result in results.items():

pydantic_evals/pydantic_evals/evaluators/_spec.py

@@ -32,18 +32,19 @@ class EvaluatorSpec(BaseModel):
     * `{'MyEvaluator': {k1: v1, k2: v2}}` - Multiple kwargs are passed to `MyEvaluator.__init__`
 
     Args:
-        name: The name of the evaluator to use. Unless overridden, this is the snake_case version of the class name.
+        name: The serialization name of the evaluator class returned by `EvaluatorClass.get_serialization_name()`;
+            this is usually just the class name itself.
         arguments: The arguments to pass to the evaluator's constructor. Can be None (for no arguments),
             a tuple (for a single positional argument), or a dict (for multiple keyword arguments).
     """
 
     name: str
-    """The name of the evaluator class; should be the value returned by EvaluatorClass.name()"""
+    """The name of the evaluator class; should be the value returned by `EvaluatorClass.get_serialization_name()`"""
 
     arguments: None | tuple[Any] | dict[str, Any]
     """The arguments to pass to the evaluator's constructor.
 
-    Can be None (no arguments), a tuple (positional arguments), or a dict (keyword arguments).
+    Can be None (no arguments), a tuple (a single positional argument), or a dict (keyword arguments).
     """
 
     @property

pydantic_evals/pydantic_evals/evaluators/common.py

@@ -1,15 +1,17 @@
 from __future__ import annotations as _annotations
 
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from datetime import timedelta
-from typing import Any, cast
+from typing import Any, Literal, cast
+
+from typing_extensions import TypedDict
 
 from pydantic_ai import models
 from pydantic_ai.settings import ModelSettings
 
 from ..otel.span_tree import SpanQuery
 from .context import EvaluatorContext
-from .evaluator import EvaluationReason, Evaluator, EvaluatorOutput
+from .evaluator import EvaluationReason, EvaluationScalar, Evaluator, EvaluatorOutput
 
 __all__ = (
     'Equals',
@@ -20,6 +22,7 @@
     'LLMJudge',
     'HasMatchingSpan',
     'Python',
+    'OutputConfig',
 )
 
 
@@ -28,6 +31,7 @@ class Equals(Evaluator[object, object, object]):
     """Check if the output exactly equals the provided value."""
 
     value: Any
+    evaluation_name: str | None = field(default=None, repr=False)
 
     def evaluate(self, ctx: EvaluatorContext[object, object, object]) -> bool:
         return ctx.output == self.value
@@ -37,6 +41,8 @@ def evaluate(self, ctx: EvaluatorContext[object, object, object]) -> bool:
 class EqualsExpected(Evaluator[object, object, object]):
     """Check if the output exactly equals the expected output."""
 
+    evaluation_name: str | None = field(default=None, repr=False)
+
     def evaluate(self, ctx: EvaluatorContext[object, object, object]) -> bool | dict[str, bool]:
         if ctx.expected_output is None:
             return {}  # Only compare if expected output is provided
@@ -68,6 +74,7 @@ class Contains(Evaluator[object, object, object]):
     value: Any
     case_sensitive: bool = True
     as_strings: bool = False
+    evaluation_name: str | None = field(default=None, repr=False)
 
     def evaluate(
         self,
@@ -127,6 +134,7 @@ class IsInstance(Evaluator[object, object, object]):
     """Check if the output is an instance of a type with the given name."""
 
     type_name: str
+    evaluation_name: str | None = field(default=None, repr=False)
 
     def evaluate(self, ctx: EvaluatorContext[object, object, object]) -> EvaluationReason:
         output = ctx.output
@@ -154,6 +162,27 @@ def evaluate(self, ctx: EvaluatorContext[object, object, object]) -> bool:
         return duration <= seconds
 
 
+class OutputConfig(TypedDict, total=False):
+    """Configuration for the score and assertion outputs of the LLMJudge evaluator."""
+
+    evaluation_name: str
+    include_reason: bool
+
+
+def _update_combined_output(
+    combined_output: dict[str, EvaluationScalar | EvaluationReason],
+    value: EvaluationScalar,
+    reason: str | None,
+    config: OutputConfig,
+    default_name: str,
+) -> None:
+    name = config.get('evaluation_name') or default_name
+    if config.get('include_reason') and reason is not None:
+        combined_output[name] = EvaluationReason(value=value, reason=reason)
+    else:
+        combined_output[name] = value
+
+
 @dataclass
 class LLMJudge(Evaluator[object, object, object]):
     """Judge whether the output of a language model meets the criteria of a provided rubric.
@@ -166,11 +195,13 @@ class LLMJudge(Evaluator[object, object, object]):
     model: models.Model | models.KnownModelName | None = None
     include_input: bool = False
     model_settings: ModelSettings | None = None
+    score: OutputConfig | Literal[False] = False
+    assertion: OutputConfig | Literal[False] = field(default_factory=lambda: OutputConfig(include_reason=True))
 
     async def evaluate(
         self,
         ctx: EvaluatorContext[object, object, object],
-    ) -> EvaluationReason:
+    ) -> EvaluatorOutput:
         if self.include_input:
             from .llm_as_a_judge import judge_input_output
 
@@ -181,7 +212,20 @@ async def evaluate(
             from .llm_as_a_judge import judge_output
 
             grading_output = await judge_output(ctx.output, self.rubric, self.model, self.model_settings)
-        return EvaluationReason(value=grading_output.pass_, reason=grading_output.reason)
+
+        output: dict[str, EvaluationScalar | EvaluationReason] = {}
+        include_both = self.score is not False and self.assertion is not False
+        evaluation_name = self.get_default_evaluation_name()
+
+        if self.score is not False:
+            default_name = f'{evaluation_name}_score' if include_both else evaluation_name
+            _update_combined_output(output, grading_output.score, grading_output.reason, self.score, default_name)
+
+        if self.assertion is not False:
+            default_name = f'{evaluation_name}_pass' if include_both else evaluation_name
+            _update_combined_output(output, grading_output.pass_, grading_output.reason, self.assertion, default_name)
+
+        return output
 
     def build_serialization_arguments(self):
         result = super().build_serialization_arguments()
@@ -200,6 +244,7 @@ class HasMatchingSpan(Evaluator[object, object, object]):
     """Check if the span tree contains a span that matches the specified query."""
 
     query: SpanQuery
+    evaluation_name: str | None = field(default=None, repr=False)
 
     def evaluate(
         self,
@@ -217,6 +262,7 @@ class Python(Evaluator[object, object, object]):
     """
 
     expression: str
+    evaluation_name: str | None = field(default=None, repr=False)
 
     def evaluate(self, ctx: EvaluatorContext[object, object, object]) -> EvaluatorOutput:
         # Evaluate the condition, exposing access to the evaluator context as `ctx`.

pydantic_evals/pydantic_evals/evaluators/evaluator.py

@@ -12,7 +12,7 @@
 )
 from pydantic_core import to_jsonable_python
 from pydantic_core.core_schema import SerializationInfo
-from typing_extensions import TypeVar
+from typing_extensions import TypeVar, deprecated
 
 from .._utils import get_event_loop
 from ._spec import EvaluatorSpec
@@ -146,17 +146,38 @@ def evaluate(self, ctx: EvaluatorContext) -> bool:
     __pydantic_config__ = ConfigDict(arbitrary_types_allowed=True)
 
     @classmethod
-    def name(cls) -> str:
+    def get_serialization_name(cls) -> str:
         """Return the 'name' of this Evaluator to use during serialization.
 
         Returns:
             The name of the Evaluator, which is typically the class name.
         """
-        # Note: if we wanted to prefer snake_case, we could use:
-        # from pydantic.alias_generators import to_snake
-        # return to_snake(cls.__name__)
         return cls.__name__
 
+    @classmethod
+    @deprecated('`name` has been renamed, use `get_serialization_name` instead.')
+    def name(cls) -> str:
+        """`name` has been renamed, use `get_serialization_name` instead."""
+        return cls.get_serialization_name()
+
+    def get_default_evaluation_name(self) -> str:
+        """Return the default name to use in reports for the output of this evaluator.
+
+        By default, if the evaluator has an attribute called `evaluation_name` of type string, that will be used.
+        Otherwise, the serialization name of the evaluator (which is usually the class name) will be used.
+
+        This can be overridden to get a more descriptive name in evaluation reports, e.g. using instance information.
+
+        Note that evaluators that return a mapping of results will always use the keys of that mapping as the names
+        of the associated evaluation results.
+        """
+        evaluation_name = getattr(self, 'evaluation_name', None)
+        if isinstance(evaluation_name, str):
+            # If the evaluator has an attribute `name` of type string, use that
+            return evaluation_name
+
+        return self.get_serialization_name()
+
     @abstractmethod
     def evaluate(
         self, ctx: EvaluatorContext[InputsT, OutputT, MetadataT]
@@ -233,7 +254,9 @@ def serialize(self, info: SerializationInfo) -> Any:
         else:
             arguments = raw_arguments
         return to_jsonable_python(
-            EvaluatorSpec(name=self.name(), arguments=arguments), context=info.context, serialize_unknown=True
+            EvaluatorSpec(name=self.get_serialization_name(), arguments=arguments),
+            context=info.context,
+            serialize_unknown=True,
         )
 
     def build_serialization_arguments(self) -> dict[str, Any]:

tests/evals/test_evaluator_base.py

@@ -172,10 +172,11 @@ async def test_evaluator_async():
     assert result is True
 
 
-async def test_evaluator_name():
+async def test_evaluation_name():
     """Test evaluator name method."""
     evaluator = SimpleEvaluator()
-    assert evaluator.name() == 'SimpleEvaluator'
+    assert evaluator.get_serialization_name() == 'SimpleEvaluator'
+    assert evaluator.get_default_evaluation_name() == 'SimpleEvaluator'
 
 
 async def test_evaluator_serialization():

tests/evals/test_evaluator_common.py

@@ -5,6 +5,7 @@
 
 import pytest
 from inline_snapshot import snapshot
+from pydantic_core import to_jsonable_python
 from pytest_mock import MockerFixture
 
 from pydantic_ai.settings import ModelSettings
@@ -25,6 +26,7 @@
         IsInstance,
         LLMJudge,
         MaxDuration,
+        OutputConfig,
         Python,
     )
     from pydantic_evals.otel._context_in_memory_span_exporter import context_subtree
@@ -194,6 +196,7 @@ async def test_llm_judge_evaluator(mocker: MockerFixture):
     """Test LLMJudge evaluator."""
     # Create a mock GradingOutput
     mock_grading_output = mocker.MagicMock()
+    mock_grading_output.score = 1.0
     mock_grading_output.pass_ = True
     mock_grading_output.reason = 'Test passed'
 
@@ -219,31 +222,42 @@ async def test_llm_judge_evaluator(mocker: MockerFixture):
 
     # Test without input
     evaluator = LLMJudge(rubric='Content contains a greeting')
-    result = await evaluator.evaluate(ctx)
-    assert isinstance(result, EvaluationReason)
-    assert result.value is True
-    assert result.reason == 'Test passed'
+    assert to_jsonable_python(await evaluator.evaluate(ctx)) == snapshot(
+        {'LLMJudge': {'value': True, 'reason': 'Test passed'}}
+    )
 
     mock_judge_output.assert_called_once_with('Hello world', 'Content contains a greeting', None, None)
 
     # Test with input
     evaluator = LLMJudge(rubric='Output contains input', include_input=True, model='openai:gpt-4o')
-    result = await evaluator.evaluate(ctx)
-    assert isinstance(result, EvaluationReason)
-    assert result.value is True
-    assert result.reason == 'Test passed'
+    assert to_jsonable_python(await evaluator.evaluate(ctx)) == snapshot(
+        {'LLMJudge': {'value': True, 'reason': 'Test passed'}}
+    )
 
     mock_judge_input_output.assert_called_once_with(
         {'prompt': 'Hello'}, 'Hello world', 'Output contains input', 'openai:gpt-4o', None
     )
 
     # Test with failing result
+    mock_grading_output.score = 0.0
     mock_grading_output.pass_ = False
     mock_grading_output.reason = 'Test failed'
-    result = await evaluator.evaluate(ctx)
-    assert isinstance(result, EvaluationReason)
-    assert result.value is False
-    assert result.reason == 'Test failed'
+    assert to_jsonable_python(await evaluator.evaluate(ctx)) == snapshot(
+        {'LLMJudge': {'value': False, 'reason': 'Test failed'}}
+    )
+
+    # Test with overridden configs
+    evaluator = LLMJudge(rubric='Mock rubric', assertion=False)
+    assert to_jsonable_python(await evaluator.evaluate(ctx)) == snapshot({})
+
+    evaluator = LLMJudge(
+        rubric='Mock rubric',
+        score=OutputConfig(evaluation_name='my_score', include_reason=True),
+        assertion=OutputConfig(evaluation_name='my_assertion'),
+    )
+    assert to_jsonable_python(await evaluator.evaluate(ctx)) == snapshot(
+        {'my_assertion': False, 'my_score': {'reason': 'Test failed', 'value': 0.0}}
+    )
 
 
 @pytest.mark.anyio
@@ -275,9 +289,9 @@ async def test_llm_judge_evaluator_with_model_settings(mocker: MockerFixture):
 
     # Test without input, with custom model_settings
     evaluator_no_input = LLMJudge(rubric='Greeting with custom settings', model_settings=custom_model_settings)
-    result_no_input = await evaluator_no_input.evaluate(ctx)
-    assert result_no_input.value is True
-    assert result_no_input.reason == 'Test passed with settings'
+    assert to_jsonable_python(await evaluator_no_input.evaluate(ctx)) == snapshot(
+        {'LLMJudge': {'value': True, 'reason': 'Test passed with settings'}}
+    )
     mock_judge_output.assert_called_once_with(
         'Hello world custom settings', 'Greeting with custom settings', None, custom_model_settings
     )
@@ -289,9 +303,9 @@ async def test_llm_judge_evaluator_with_model_settings(mocker: MockerFixture):
         model='openai:gpt-3.5-turbo',
         model_settings=custom_model_settings,
     )
-    result_with_input = await evaluator_with_input.evaluate(ctx)
-    assert result_with_input.value is True
-    assert result_with_input.reason == 'Test passed with settings'
+    assert to_jsonable_python(await evaluator_with_input.evaluate(ctx)) == snapshot(
+        {'LLMJudge': {'value': True, 'reason': 'Test passed with settings'}}
+    )
     mock_judge_input_output.assert_called_once_with(
         {'prompt': 'Hello Custom'},
         'Hello world custom settings',