Skip to content

Commit 54062ab

Browse files
author
Val Brodsky
committed
Cleanup actions
1 parent df64dc2 commit 54062ab

File tree

6 files changed

+26
-24
lines changed

6 files changed

+26
-24
lines changed

libs/labelbox/src/labelbox/schema/tool_building/step_reasoning_tool.py

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,16 @@
11
from dataclasses import dataclass, field
2+
from enum import Enum
23
from typing import Any, Dict, List, Optional
34

45
from labelbox.schema.tool_building.tool_type import ToolType
56
from labelbox.schema.tool_building.variant import Variant, VariantWithActions
67

78

9+
class IncorrectStepActions(Enum):
10+
REGENERATE_STEPS = "regenerateSteps"
11+
GENERATE_AND_RATE_ALTERNATIVE_STEPS = "generateAndRateAlternativeSteps"
12+
13+
814
@dataclass
915
class StepReasoningVariants:
1016
"""
@@ -24,8 +30,7 @@ class StepReasoningVariants:
2430
id=2,
2531
name="Incorrect",
2632
_available_actions={
27-
"regenerateSteps",
28-
"generateAndRateAlternativeSteps",
33+
action.value for action in IncorrectStepActions
2934
},
3035
actions=["regenerateSteps"], # regenerateSteps is on by default
3136
)
@@ -104,19 +109,12 @@ class StepReasoningTool:
104109
default_factory=StepReasoningDefinition
105110
)
106111

107-
def set_regenerate_conversations_after_incorrect_step(self):
112+
def set_incorrect_step_actions(self, actions: List[IncorrectStepActions]):
108113
"""
109114
For live models, will invoke the model to generate alternatives if a step is marked as incorrect
110115
"""
111-
self.definition.variants.incorrect_step.regenerate_conversations_after_incorrect_step = True
112-
113-
def set_rate_alternative_responses(self):
114-
"""
115-
For live models, will require labelers to rate the alternatives generated by the model
116-
"""
117-
self.definition.variants.incorrect_step.rate_alternative_responses = (
118-
True
119-
)
116+
actions_values = [action.value for action in actions]
117+
self.definition.variants.incorrect_step.set_actions(actions_values)
120118

121119
def asdict(self) -> Dict[str, Any]:
122120
return {

libs/labelbox/src/labelbox/schema/tool_building/tool_type_mapping.py

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,10 @@
55

66
def map_tool_type_to_tool_cls(tool_type_str: str):
77
if not ToolType.valid(tool_type_str):
8-
raise ValueError(f"Invalid tool type {tool_type_str}")
8+
return None
99

1010
tool_type = ToolType(tool_type_str.lower())
1111
if tool_type == ToolType.STEP_REASONING:
1212
return StepReasoningTool
1313
elif tool_type == ToolType.FACT_CHECKING:
1414
return FactCheckingTool
15-
16-
return None

libs/labelbox/src/labelbox/schema/tool_building/variant.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,8 @@ class VariantWithActions:
2222
actions: List[str] = field(default_factory=list)
2323
_available_actions: Set[str] = field(default_factory=set)
2424

25-
def set_actions(self, actions: Set[str]) -> None:
25+
def set_actions(self, actions: List[str]) -> None:
26+
self.actions = []
2627
for action in actions:
2728
if action in self._available_actions:
2829
self.actions.append(action)

libs/labelbox/tests/integration/conftest.py

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
)
2222
from labelbox.schema.data_row import DataRowMetadataField
2323
from labelbox.schema.ontology_kind import OntologyKind
24-
from labelbox.schema.tool_building.fact_checking_tool import FactCheckingTool
2524
from labelbox.schema.tool_building.step_reasoning_tool import StepReasoningTool
2625
from labelbox.schema.user import User
2726

@@ -580,7 +579,6 @@ def chat_evaluation_ontology(client, rand_gen):
580579
name="model output multi ranking",
581580
),
582581
StepReasoningTool(name="step reasoning"),
583-
FactCheckingTool(name="fact checking"),
584582
],
585583
classifications=[
586584
Classification(

libs/labelbox/tests/integration/test_ontology.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -339,7 +339,7 @@ def test_step_reasoning_ontology(chat_evaluation_ontology):
339339
{
340340
"id": 2,
341341
"name": "Incorrect",
342-
"actions": [],
342+
"actions": ["regenerateSteps"],
343343
},
344344
]
345345
assert step_reasoning_tool["definition"]["version"] == 1
@@ -364,6 +364,6 @@ def test_step_reasoning_ontology(chat_evaluation_ontology):
364364
{
365365
"id": 2,
366366
"name": "Incorrect",
367-
"actions": [],
367+
"actions": ["regenerateSteps"],
368368
},
369369
]

libs/labelbox/tests/unit/test_unit_step_reasoning_tool.py

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
1-
from labelbox.schema.tool_building.step_reasoning_tool import StepReasoningTool
1+
from labelbox.schema.tool_building.step_reasoning_tool import (
2+
IncorrectStepActions,
3+
StepReasoningTool,
4+
)
25

36

47
def test_step_reasoning_as_dict_default():
@@ -16,7 +19,7 @@ def test_step_reasoning_as_dict_default():
1619
{
1720
"id": 2,
1821
"name": "Incorrect",
19-
"actions": [],
22+
"actions": ["regenerateSteps"],
2023
},
2124
],
2225
"version": 1,
@@ -26,8 +29,12 @@ def test_step_reasoning_as_dict_default():
2629

2730
def test_step_reasoning_as_dict_with_actions():
2831
tool = StepReasoningTool(name="step reasoning")
29-
tool.set_rate_alternative_responses()
30-
tool.set_regenerate_conversations_after_incorrect_step()
32+
tool.set_incorrect_step_actions(
33+
[
34+
IncorrectStepActions.REGENERATE_STEPS,
35+
IncorrectStepActions.GENERATE_AND_RATE_ALTERNATIVE_STEPS,
36+
]
37+
)
3138
assert tool.asdict() == {
3239
"tool": "step-reasoning",
3340
"name": "step reasoning",

0 commit comments

Comments
 (0)