Add FactCheckingTool

Author: Val Brodsky

libs/labelbox/src/labelbox/schema/tool_building/__init__.py

@@ -1,2 +1,4 @@
 import labelbox.schema.tool_building.tool_type
+import labelbox.schema.tool_building.variant
 import labelbox.schema.tool_building.step_reasoning_tool
+import labelbox.schema.tool_building.fact_checking_tool

libs/labelbox/src/labelbox/schema/tool_building/fact_checking_tool.py

@@ -0,0 +1,180 @@
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Set
+
+from labelbox.schema.tool_building.tool_type import ToolType
+from labelbox.schema.tool_building.variant import (
+    Variant,
+    VariantWithActions,
+)
+
+
+@dataclass
+class FactCheckingVariants:
+    """
+    This class is used to define the possible options for evaluating a step
+    Currently the options are correct, neutral, and incorrect
+    """
+
+    accurate_step: Variant = field(
+        default_factory=lambda: Variant(id=0, name="Accurate")
+    )
+    inaccurate_step: Variant = field(
+        default_factory=lambda: Variant(id=1, name="Inaccurate")
+    )
+    disputed_step: Variant = field(
+        default_factory=lambda: Variant(id=2, name="Disputed")
+    )
+    unsupported_step: VariantWithActions = field(
+        default_factory=lambda: VariantWithActions(
+            id=3, name="Unsupported", _available_actions={"writeJustification"}
+        )
+    )
+    cant_confidently_assess_step: VariantWithActions = field(
+        default_factory=lambda: VariantWithActions(
+            id=4,
+            name="Can't confidently assess",
+            _available_actions={"writeJustification"},
+        )
+    )
+    no_factual_information_step: VariantWithActions = field(
+        default_factory=lambda: VariantWithActions(
+            id=5,
+            name="No factual information",
+            _available_actions={"writeJustification"},
+        )
+    )
+
+    def asdict(self):
+        return [
+            self.accurate_step.asdict(),
+            self.inaccurate_step.asdict(),
+            self.disputed_step.asdict(),
+            self.unsupported_step.asdict(),
+            self.cant_confidently_assess_step.asdict(),
+            self.no_factual_information_step.asdict(),
+        ]
+
+    @classmethod
+    def from_dict(cls, dictionary: List[Dict[str, Any]]):
+        accurate_step = None
+        inaccurate_step = None
+        disputed_step = None
+        unsupported_step = None
+        cant_confidently_assess_step = None
+        no_factual_information_step = None
+
+        for variant in dictionary:
+            if variant["id"] == 0:
+                accurate_step = Variant(**variant)
+            elif variant["id"] == 1:
+                inaccurate_step = Variant(**variant)
+            elif variant["id"] == 2:
+                disputed_step = Variant(**variant)
+            elif variant["id"] == 3:
+                unsupported_step = VariantWithActions(**variant)
+            elif variant["id"] == 4:
+                cant_confidently_assess_step = VariantWithActions(**variant)
+            elif variant["id"] == 5:
+                no_factual_information_step = VariantWithActions(**variant)
+
+        if not all(
+            [
+                accurate_step,
+                inaccurate_step,
+                disputed_step,
+                unsupported_step,
+                cant_confidently_assess_step,
+                no_factual_information_step,
+            ]
+        ):
+            raise ValueError("Missing variant")
+
+        return cls(
+            accurate_step=accurate_step,
+            inaccurate_step=inaccurate_step,
+            disputed_step=disputed_step,
+            unsupported_step=unsupported_step,
+            cant_confidently_assess_step=cant_confidently_assess_step,
+            no_factual_information_step=no_factual_information_step,
+        )  # type: ignore
+
+
+@dataclass
+class FactCheckingDefinition:
+    variants: FactCheckingVariants = field(default_factory=FactCheckingVariants)
+    version: int = field(default=1)
+    title: Optional[str] = None
+    value: Optional[str] = None
+
+    def asdict(self) -> Dict[str, Any]:
+        result = {"variants": self.variants.asdict(), "version": self.version}
+        if self.title is not None:
+            result["title"] = self.title
+        if self.value is not None:
+            result["value"] = self.value
+        return result
+
+    @classmethod
+    def from_dict(cls, dictionary: Dict[str, Any]) -> "FactCheckingDefinition":
+        variants = FactCheckingVariants.from_dict(dictionary["variants"])
+        title = dictionary.get("title", None)
+        value = dictionary.get("value", None)
+        return cls(variants=variants, title=title, value=value)
+
+
+@dataclass
+class FactCheckingTool:
+    """
+    Use this class in OntologyBuilder to create a tool for step fact checking
+    """
+
+    name: str
+    type: ToolType = field(default=ToolType.FACT_CHECKING, init=False)
+    required: bool = False
+    schema_id: Optional[str] = None
+    feature_schema_id: Optional[str] = None
+    color: Optional[str] = None
+    definition: FactCheckingDefinition = field(
+        default_factory=FactCheckingDefinition
+    )
+
+    def set_unsupported_step_actions(
+        self, actions: Set[str] = {"writeJustification"}
+    ) -> None:
+        self.definition.variants.unsupported_step.set_actions(actions)
+
+    def set_cant_confidently_assess_step_actions(
+        self, actions: Set[str] = {"writeJustification"}
+    ) -> None:
+        self.definition.variants.cant_confidently_assess_step.set_actions(
+            actions
+        )
+
+    def set_no_factual_information_step_actions(
+        self, actions: Set[str] = {"writeJustification"}
+    ) -> None:
+        self.definition.variants.no_factual_information_step.set_actions(
+            actions
+        )
+
+    def asdict(self) -> Dict[str, Any]:
+        return {
+            "tool": self.type.value,
+            "name": self.name,
+            "required": self.required,
+            "schemaNodeId": self.schema_id,
+            "featureSchemaId": self.feature_schema_id,
+            "definition": self.definition.asdict(),
+        }
+
+    @classmethod
+    def from_dict(cls, dictionary: Dict[str, Any]) -> "FactCheckingTool":
+        return cls(
+            name=dictionary["name"],
+            schema_id=dictionary.get("schemaNodeId", None),
+            feature_schema_id=dictionary.get("featureSchemaId", None),
+            required=dictionary.get("required", False),
+            definition=FactCheckingDefinition.from_dict(
+                dictionary["definition"]
+            ),
+        )

libs/labelbox/src/labelbox/schema/tool_building/tool_type.py

@@ -3,3 +3,4 @@
 
 class ToolType(Enum):
     STEP_REASONING = "step-reasoning"
+    FACT_CHECKING = "fact-checking"

libs/labelbox/src/labelbox/schema/tool_building/variant.py

@@ -0,0 +1,34 @@
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Set, Tuple, Union
+
+
+@dataclass
+class Variant:
+    """
+    A variant is a single option in step-by-step reasoning or fact-checking tool.
+    """
+
+    id: int
+    name: str
+
+    def asdict(self) -> Dict[str, Any]:
+        return {"id": self.id, "name": self.name}
+
+
+@dataclass
+class VariantWithActions:
+    id: int
+    name: str
+    actions: Set[str] = field(default_factory=set)
+    _available_actions: Set[str] = field(default_factory=set)
+
+    def set_actions(self, actions: Set[str]) -> None:
+        for action in actions:
+            if action in self._available_actions:
+                self.actions.add(action)
+
+    def reset_actions(self) -> None:
+        self.actions = set()
+
+    def asdict(self) -> Dict[str, Any]:
+        return {"id": self.id, "name": self.name, "actions": self.actions}