Skip to content

Commit 64156f6

Browse files
author
Val Brodsky
committed
Cleanup actions
1 parent 073191b commit 64156f6

File tree

6 files changed

+27
-26
lines changed

6 files changed

+27
-26
lines changed

libs/labelbox/src/labelbox/schema/tool_building/step_reasoning_tool.py

Lines changed: 11 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,17 @@
11
import warnings
22
from dataclasses import dataclass, field
3+
from enum import Enum
34
from typing import Any, Dict, List, Optional
45

56
from labelbox.schema.tool_building.tool_type import ToolType
67
from labelbox.schema.tool_building.variant import Variant, VariantWithActions
78

89

10+
class IncorrectStepActions(Enum):
11+
REGENERATE_STEPS = "regenerateSteps"
12+
GENERATE_AND_RATE_ALTERNATIVE_STEPS = "generateAndRateAlternativeSteps"
13+
14+
915
@dataclass
1016
class StepReasoningVariants:
1117
"""
@@ -25,8 +31,7 @@ class StepReasoningVariants:
2531
id=2,
2632
name="Incorrect",
2733
_available_actions={
28-
"regenerateSteps",
29-
"generateAndRateAlternativeSteps",
34+
action.value for action in IncorrectStepActions
3035
},
3136
actions=["regenerateSteps"], # regenerateSteps is on by default
3237
)
@@ -110,20 +115,12 @@ def __post_init__(self):
110115
"This feature is experimental and subject to change.",
111116
)
112117

113-
def reset_regenerate_conversations_after_incorrect_step(self):
114-
"""
115-
For live models, the default acation will invoke the model to generate alternatives if a step is marked as incorrect
116-
This method will reset the action to not regenerate the conversation
117-
"""
118-
self.definition.variants.incorrect_step.regenerate_conversations_after_incorrect_step = False
119-
120-
def reset_rate_alternative_responses(self):
118+
def set_incorrect_step_actions(self, actions: List[IncorrectStepActions]):
121119
"""
122-
For live models, will require labelers to rate the alternatives generated by the model
120+
For live models, will invoke the model to generate alternatives if a step is marked as incorrect
123121
"""
124-
self.definition.variants.incorrect_step.rate_alternative_responses = (
125-
False
126-
)
122+
actions_values = [action.value for action in actions]
123+
self.definition.variants.incorrect_step.set_actions(actions_values)
127124

128125
def asdict(self) -> Dict[str, Any]:
129126
return {

libs/labelbox/src/labelbox/schema/tool_building/tool_type_mapping.py

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,10 @@
55

66
def map_tool_type_to_tool_cls(tool_type_str: str):
77
if not ToolType.valid(tool_type_str):
8-
raise ValueError(f"Invalid tool type {tool_type_str}")
8+
return None
99

1010
tool_type = ToolType(tool_type_str.lower())
1111
if tool_type == ToolType.STEP_REASONING:
1212
return StepReasoningTool
1313
elif tool_type == ToolType.FACT_CHECKING:
1414
return FactCheckingTool
15-
16-
return None

libs/labelbox/src/labelbox/schema/tool_building/variant.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,8 @@ class VariantWithActions:
2222
actions: List[str] = field(default_factory=list)
2323
_available_actions: Set[str] = field(default_factory=set)
2424

25-
def set_actions(self, actions: Set[str]) -> None:
25+
def set_actions(self, actions: List[str]) -> None:
26+
self.actions = []
2627
for action in actions:
2728
if action in self._available_actions:
2829
self.actions.append(action)

libs/labelbox/tests/integration/conftest.py

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
)
2222
from labelbox.schema.data_row import DataRowMetadataField
2323
from labelbox.schema.ontology_kind import OntologyKind
24-
from labelbox.schema.tool_building.fact_checking_tool import FactCheckingTool
2524
from labelbox.schema.tool_building.step_reasoning_tool import StepReasoningTool
2625
from labelbox.schema.user import User
2726

@@ -580,7 +579,6 @@ def chat_evaluation_ontology(client, rand_gen):
580579
name="model output multi ranking",
581580
),
582581
StepReasoningTool(name="step reasoning"),
583-
FactCheckingTool(name="fact checking"),
584582
],
585583
classifications=[
586584
Classification(

libs/labelbox/tests/integration/test_ontology.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -339,7 +339,7 @@ def test_step_reasoning_ontology(chat_evaluation_ontology):
339339
{
340340
"id": 2,
341341
"name": "Incorrect",
342-
"actions": [],
342+
"actions": ["regenerateSteps"],
343343
},
344344
]
345345
assert step_reasoning_tool["definition"]["version"] == 1
@@ -364,6 +364,6 @@ def test_step_reasoning_ontology(chat_evaluation_ontology):
364364
{
365365
"id": 2,
366366
"name": "Incorrect",
367-
"actions": [],
367+
"actions": ["regenerateSteps"],
368368
},
369369
]

libs/labelbox/tests/unit/test_unit_step_reasoning_tool.py

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
1-
from labelbox.schema.tool_building.step_reasoning_tool import StepReasoningTool
1+
from labelbox.schema.tool_building.step_reasoning_tool import (
2+
IncorrectStepActions,
3+
StepReasoningTool,
4+
)
25

36

47
def test_step_reasoning_as_dict_default():
@@ -16,7 +19,7 @@ def test_step_reasoning_as_dict_default():
1619
{
1720
"id": 2,
1821
"name": "Incorrect",
19-
"actions": [],
22+
"actions": ["regenerateSteps"],
2023
},
2124
],
2225
"version": 1,
@@ -26,8 +29,12 @@ def test_step_reasoning_as_dict_default():
2629

2730
def test_step_reasoning_as_dict_with_actions():
2831
tool = StepReasoningTool(name="step reasoning")
29-
tool.reset_rate_alternative_responses()
30-
tool.reset_regenerate_conversations_after_incorrect_step()
32+
tool.set_incorrect_step_actions(
33+
[
34+
IncorrectStepActions.REGENERATE_STEPS,
35+
IncorrectStepActions.GENERATE_AND_RATE_ALTERNATIVE_STEPS,
36+
]
37+
)
3138
assert tool.asdict() == {
3239
"tool": "step-reasoning",
3340
"name": "step reasoning",

0 commit comments

Comments
 (0)