Skip to content

Commit e045479

Browse files
author
Val Brodsky
committed
Add FactCheckingTool
1 parent 5cdbfb4 commit e045479

File tree

4 files changed

+217
-0
lines changed

4 files changed

+217
-0
lines changed
Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,4 @@
11
import labelbox.schema.tool_building.tool_type
2+
import labelbox.schema.tool_building.variant
23
import labelbox.schema.tool_building.step_reasoning_tool
4+
import labelbox.schema.tool_building.fact_checking_tool
Lines changed: 180 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,180 @@
1+
from dataclasses import dataclass, field
2+
from typing import Any, Dict, List, Optional, Set
3+
4+
from labelbox.schema.tool_building.tool_type import ToolType
5+
from labelbox.schema.tool_building.variant import (
6+
Variant,
7+
VariantWithActions,
8+
)
9+
10+
11+
@dataclass
12+
class FactCheckingVariants:
13+
"""
14+
This class is used to define the possible options for evaluating a step
15+
Currently the options are correct, neutral, and incorrect
16+
"""
17+
18+
accurate_step: Variant = field(
19+
default_factory=lambda: Variant(id=0, name="Accurate")
20+
)
21+
inaccurate_step: Variant = field(
22+
default_factory=lambda: Variant(id=1, name="Inaccurate")
23+
)
24+
disputed_step: Variant = field(
25+
default_factory=lambda: Variant(id=2, name="Disputed")
26+
)
27+
unsupported_step: VariantWithActions = field(
28+
default_factory=lambda: VariantWithActions(
29+
id=3, name="Unsupported", _available_actions={"writeJustification"}
30+
)
31+
)
32+
cant_confidently_assess_step: VariantWithActions = field(
33+
default_factory=lambda: VariantWithActions(
34+
id=4,
35+
name="Can't confidently assess",
36+
_available_actions={"writeJustification"},
37+
)
38+
)
39+
no_factual_information_step: VariantWithActions = field(
40+
default_factory=lambda: VariantWithActions(
41+
id=5,
42+
name="No factual information",
43+
_available_actions={"writeJustification"},
44+
)
45+
)
46+
47+
def asdict(self):
48+
return [
49+
self.accurate_step.asdict(),
50+
self.inaccurate_step.asdict(),
51+
self.disputed_step.asdict(),
52+
self.unsupported_step.asdict(),
53+
self.cant_confidently_assess_step.asdict(),
54+
self.no_factual_information_step.asdict(),
55+
]
56+
57+
@classmethod
58+
def from_dict(cls, dictionary: List[Dict[str, Any]]):
59+
accurate_step = None
60+
inaccurate_step = None
61+
disputed_step = None
62+
unsupported_step = None
63+
cant_confidently_assess_step = None
64+
no_factual_information_step = None
65+
66+
for variant in dictionary:
67+
if variant["id"] == 0:
68+
accurate_step = Variant(**variant)
69+
elif variant["id"] == 1:
70+
inaccurate_step = Variant(**variant)
71+
elif variant["id"] == 2:
72+
disputed_step = Variant(**variant)
73+
elif variant["id"] == 3:
74+
unsupported_step = VariantWithActions(**variant)
75+
elif variant["id"] == 4:
76+
cant_confidently_assess_step = VariantWithActions(**variant)
77+
elif variant["id"] == 5:
78+
no_factual_information_step = VariantWithActions(**variant)
79+
80+
if not all(
81+
[
82+
accurate_step,
83+
inaccurate_step,
84+
disputed_step,
85+
unsupported_step,
86+
cant_confidently_assess_step,
87+
no_factual_information_step,
88+
]
89+
):
90+
raise ValueError("Missing variant")
91+
92+
return cls(
93+
accurate_step=accurate_step,
94+
inaccurate_step=inaccurate_step,
95+
disputed_step=disputed_step,
96+
unsupported_step=unsupported_step,
97+
cant_confidently_assess_step=cant_confidently_assess_step,
98+
no_factual_information_step=no_factual_information_step,
99+
) # type: ignore
100+
101+
102+
@dataclass
103+
class FactCheckingDefinition:
104+
variants: FactCheckingVariants = field(default_factory=FactCheckingVariants)
105+
version: int = field(default=1)
106+
title: Optional[str] = None
107+
value: Optional[str] = None
108+
109+
def asdict(self) -> Dict[str, Any]:
110+
result = {"variants": self.variants.asdict(), "version": self.version}
111+
if self.title is not None:
112+
result["title"] = self.title
113+
if self.value is not None:
114+
result["value"] = self.value
115+
return result
116+
117+
@classmethod
118+
def from_dict(cls, dictionary: Dict[str, Any]) -> "FactCheckingDefinition":
119+
variants = FactCheckingVariants.from_dict(dictionary["variants"])
120+
title = dictionary.get("title", None)
121+
value = dictionary.get("value", None)
122+
return cls(variants=variants, title=title, value=value)
123+
124+
125+
@dataclass
126+
class FactCheckingTool:
127+
"""
128+
Use this class in OntologyBuilder to create a tool for step fact checking
129+
"""
130+
131+
name: str
132+
type: ToolType = field(default=ToolType.FACT_CHECKING, init=False)
133+
required: bool = False
134+
schema_id: Optional[str] = None
135+
feature_schema_id: Optional[str] = None
136+
color: Optional[str] = None
137+
definition: FactCheckingDefinition = field(
138+
default_factory=FactCheckingDefinition
139+
)
140+
141+
def set_unsupported_step_actions(
142+
self, actions: Set[str] = {"writeJustification"}
143+
) -> None:
144+
self.definition.variants.unsupported_step.set_actions(actions)
145+
146+
def set_cant_confidently_assess_step_actions(
147+
self, actions: Set[str] = {"writeJustification"}
148+
) -> None:
149+
self.definition.variants.cant_confidently_assess_step.set_actions(
150+
actions
151+
)
152+
153+
def set_no_factual_information_step_actions(
154+
self, actions: Set[str] = {"writeJustification"}
155+
) -> None:
156+
self.definition.variants.no_factual_information_step.set_actions(
157+
actions
158+
)
159+
160+
def asdict(self) -> Dict[str, Any]:
161+
return {
162+
"tool": self.type.value,
163+
"name": self.name,
164+
"required": self.required,
165+
"schemaNodeId": self.schema_id,
166+
"featureSchemaId": self.feature_schema_id,
167+
"definition": self.definition.asdict(),
168+
}
169+
170+
@classmethod
171+
def from_dict(cls, dictionary: Dict[str, Any]) -> "FactCheckingTool":
172+
return cls(
173+
name=dictionary["name"],
174+
schema_id=dictionary.get("schemaNodeId", None),
175+
feature_schema_id=dictionary.get("featureSchemaId", None),
176+
required=dictionary.get("required", False),
177+
definition=FactCheckingDefinition.from_dict(
178+
dictionary["definition"]
179+
),
180+
)

libs/labelbox/src/labelbox/schema/tool_building/tool_type.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,4 @@
33

44
class ToolType(Enum):
55
STEP_REASONING = "step-reasoning"
6+
FACT_CHECKING = "fact-checking"
Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
from dataclasses import dataclass, field
2+
from typing import Any, Dict, List, Optional, Set, Tuple, Union
3+
4+
5+
@dataclass
6+
class Variant:
7+
"""
8+
A variant is a single option in step-by-step reasoning or fact-checking tool.
9+
"""
10+
11+
id: int
12+
name: str
13+
14+
def asdict(self) -> Dict[str, Any]:
15+
return {"id": self.id, "name": self.name}
16+
17+
18+
@dataclass
19+
class VariantWithActions:
20+
id: int
21+
name: str
22+
actions: Set[str] = field(default_factory=set)
23+
_available_actions: Set[str] = field(default_factory=set)
24+
25+
def set_actions(self, actions: Set[str]) -> None:
26+
for action in actions:
27+
if action in self._available_actions:
28+
self.actions.add(action)
29+
30+
def reset_actions(self) -> None:
31+
self.actions = set()
32+
33+
def asdict(self) -> Dict[str, Any]:
34+
return {"id": self.id, "name": self.name, "actions": self.actions}

0 commit comments

Comments
 (0)