First pass at the prompt issue tool

Author: Val Brodsky

libs/labelbox/src/labelbox/schema/tool_building/prompt_issue_tool.py

@@ -0,0 +1,74 @@
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+from labelbox.schema.ontology import Classification, Option
+from labelbox.schema.tool_building.tool_type import ToolType
+
+
+def _supported_classifications() -> List[Classification]:
+    option_1_text = "This prompt cannot be rated (eg. contains PII, a nonsense prompt, a foreign language, or other scenario that makes the responses impossible to assess reliably). If you simply do not have expertise to tackle this prompt, please skip the task; do not mark it as not rateable"
+    option_2_text = 'This prompt contains a false, offensive, or controversial premise (eg. "why does 1+1=3"?)'
+    option_3_text = "This prompt is not self-contained (i.e. the prompt cannot be understood without additional context about previous turns, account information or images)."
+    options = [
+        Option(label=option_1_text, value="not_rateable"),
+        Option(label=option_2_text, value="false_offensive_controversial"),
+        Option(label=option_3_text, value="not_self_contained"),
+    ]
+    return [
+        Classification(
+            class_type=Classification.Type.CHECKLIST,
+            name="prompt_issue",
+            options=options,
+        ),
+    ]
+
+
+@dataclass
+class PromptIssueTool:
+    """
+    Use this class in OntologyBuilder to create a tool for prompt rating
+    It comes with a prebuild checklist of options, which a user can modify or override
+    So essentially this is a tool with a prebuilt checklist classification
+    """
+
+    name: str
+    type: ToolType = field(default=ToolType.PROMPT_ISSUE, init=False)
+    required: bool = False
+    schema_id: Optional[str] = None
+    feature_schema_id: Optional[str] = None
+    color: Optional[str] = None
+    classifications: List[Classification] = field(
+        default_factory=_supported_classifications
+    )
+
+    def __post_init__(self):
+        if self.name.strip() == "":
+            raise ValueError("Name cannot be empty")
+
+    def asdict(self) -> Dict[str, Any]:
+        return {
+            "tool": self.type.value,
+            "name": self.name,
+            "required": self.required,
+            "schemaNodeId": self.schema_id,
+            "featureSchemaId": self.feature_schema_id,
+            "classifications": [
+                classification.asdict()
+                for classification in self.classifications
+            ],
+            "color": self.color,
+        }
+
+    @classmethod
+    def from_dict(cls, dictionary: Dict[str, Any]) -> "FactCheckingTool":
+        return cls(
+            name=dictionary["name"],
+            schema_id=dictionary.get("schemaNodeId", None),
+            feature_schema_id=dictionary.get("featureSchemaId", None),
+            required=dictionary.get("required", False),
+            classifications=[
+                Classification.from_dict(classification)
+                for classification in dictionary["classifications"]
+            ],
+            color=dictionary.get("color", None),
+        )

libs/labelbox/src/labelbox/schema/tool_building/tool_type.py

@@ -4,6 +4,7 @@
 class ToolType(Enum):
     STEP_REASONING = "step-reasoning"
     FACT_CHECKING = "fact-checking"
+    PROMPT_ISSUE = "prompt-issue"
 
     @classmethod
     def valid(cls, tool_type: str) -> bool: