diff --git a/libs/labelbox/src/labelbox/schema/tool_building/step_reasoning_tool.py b/libs/labelbox/src/labelbox/schema/tool_building/step_reasoning_tool.py index 7b0536cec..236061cfa 100644 --- a/libs/labelbox/src/labelbox/schema/tool_building/step_reasoning_tool.py +++ b/libs/labelbox/src/labelbox/schema/tool_building/step_reasoning_tool.py @@ -19,7 +19,7 @@ class IncorrectStepReasoningVariant: id: int name: str regenerate_conversations_after_incorrect_step: Optional[bool] = True - rate_alternative_responses: Optional[bool] = False + rate_alternative_responses: Optional[bool] = True def asdict(self) -> Dict[str, Any]: actions = [] @@ -169,12 +169,12 @@ def reset_regenerate_conversations_after_incorrect_step(self): """ self.definition.variants.incorrect_step.regenerate_conversations_after_incorrect_step = False - def set_rate_alternative_responses(self): + def reset_rate_alternative_responses(self): """ For live models, will require labelers to rate the alternatives generated by the model """ self.definition.variants.incorrect_step.rate_alternative_responses = ( - True + False ) def asdict(self) -> Dict[str, Any]: diff --git a/libs/labelbox/tests/integration/test_ontology.py b/libs/labelbox/tests/integration/test_ontology.py index acb4e7bb1..84cd2b853 100644 --- a/libs/labelbox/tests/integration/test_ontology.py +++ b/libs/labelbox/tests/integration/test_ontology.py @@ -339,7 +339,7 @@ def test_step_reasoning_ontology(chat_evaluation_ontology): { "id": 2, "name": "Incorrect", - "actions": ["regenerateSteps"], + "actions": ["regenerateSteps", "generateAndRateAlternativeSteps"], }, ] assert step_reasoning_tool["definition"]["version"] == 1 @@ -364,6 +364,6 @@ def test_step_reasoning_ontology(chat_evaluation_ontology): { "id": 2, "name": "Incorrect", - "actions": ["regenerateSteps"], + "actions": ["regenerateSteps", "generateAndRateAlternativeSteps"], }, ] diff --git a/libs/labelbox/tests/unit/test_unit_step_reasoning_tool.py b/libs/labelbox/tests/unit/test_unit_step_reasoning_tool.py index 4d6986b86..a950f375c 100644 --- a/libs/labelbox/tests/unit/test_unit_step_reasoning_tool.py +++ b/libs/labelbox/tests/unit/test_unit_step_reasoning_tool.py @@ -16,7 +16,10 @@ def test_step_reasoning_as_dict_default(): { "id": 2, "name": "Incorrect", - "actions": ["regenerateSteps"], + "actions": [ + "regenerateSteps", + "generateAndRateAlternativeSteps", + ], }, ], "version": 1, @@ -26,7 +29,7 @@ def test_step_reasoning_as_dict_default(): def test_step_reasoning_as_dict_with_actions(): tool = StepReasoningTool(name="step reasoning") - tool.set_rate_alternative_responses() + tool.reset_rate_alternative_responses() tool.reset_regenerate_conversations_after_incorrect_step() assert tool.asdict() == { "tool": "step-reasoning", @@ -41,9 +44,7 @@ def test_step_reasoning_as_dict_with_actions(): { "id": 2, "name": "Incorrect", - "actions": [ - "generateAndRateAlternativeSteps", - ], + "actions": [], }, ], "version": 1,