From 320ba965ffdbe0b49030925dfa32b43ed69a4227 Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Fri, 18 Oct 2024 16:27:23 -0700 Subject: [PATCH 01/12] Add step by step reasoning ontology tool --- libs/labelbox/src/labelbox/schema/ontology.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libs/labelbox/src/labelbox/schema/ontology.py b/libs/labelbox/src/labelbox/schema/ontology.py index 607bf8fec..ff886aecc 100644 --- a/libs/labelbox/src/labelbox/schema/ontology.py +++ b/libs/labelbox/src/labelbox/schema/ontology.py @@ -545,6 +545,12 @@ def __init__(self, *args, **kwargs) -> None: Union[List[Classification], List[PromptResponseClassification]] ] = None + def _tool_deserializer_cls(self, tool: Dict[str, Any]) -> Tool: + import pdb + + pdb.set_trace() + return Tool + def tools(self) -> List[Tool]: """Get list of tools (AKA objects) in an Ontology.""" if self._tools is None: From 664fe4d65ced45a5f72c94b9f7615d60e9c21916 Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Tue, 22 Oct 2024 15:51:46 -0700 Subject: [PATCH 02/12] Update to support create_ontology_from_feature_schemas --- libs/labelbox/src/labelbox/schema/ontology.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/libs/labelbox/src/labelbox/schema/ontology.py b/libs/labelbox/src/labelbox/schema/ontology.py index ff886aecc..607bf8fec 100644 --- a/libs/labelbox/src/labelbox/schema/ontology.py +++ b/libs/labelbox/src/labelbox/schema/ontology.py @@ -545,12 +545,6 @@ def __init__(self, *args, **kwargs) -> None: Union[List[Classification], List[PromptResponseClassification]] ] = None - def _tool_deserializer_cls(self, tool: Dict[str, Any]) -> Tool: - import pdb - - pdb.set_trace() - return Tool - def tools(self) -> List[Tool]: """Get list of tools (AKA objects) in an Ontology.""" if self._tools is None: From e2e2d91d257074ede4f645d419632f9cba1d8ced Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Fri, 25 Oct 2024 15:21:38 -0700 Subject: [PATCH 03/12] Add FactCheckingTool --- .../labelbox/schema/tool_building/__init__.py | 1 + .../labelbox/schema/tool_building/variant.py | 38 +++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 libs/labelbox/src/labelbox/schema/tool_building/variant.py diff --git a/libs/labelbox/src/labelbox/schema/tool_building/__init__.py b/libs/labelbox/src/labelbox/schema/tool_building/__init__.py index dab325388..b5327d643 100644 --- a/libs/labelbox/src/labelbox/schema/tool_building/__init__.py +++ b/libs/labelbox/src/labelbox/schema/tool_building/__init__.py @@ -1,4 +1,5 @@ import labelbox.schema.tool_building.tool_type +import labelbox.schema.tool_building.variant import labelbox.schema.tool_building.step_reasoning_tool import labelbox.schema.tool_building.fact_checking_tool import labelbox.schema.tool_building.tool_type_mapping diff --git a/libs/labelbox/src/labelbox/schema/tool_building/variant.py b/libs/labelbox/src/labelbox/schema/tool_building/variant.py new file mode 100644 index 000000000..a101ee66a --- /dev/null +++ b/libs/labelbox/src/labelbox/schema/tool_building/variant.py @@ -0,0 +1,38 @@ +from dataclasses import dataclass, field +from typing import Any, Dict, List, Set + + +@dataclass +class Variant: + """ + A variant is a single option in step-by-step reasoning or fact-checking tool. + """ + + id: int + name: str + + def asdict(self) -> Dict[str, Any]: + return {"id": self.id, "name": self.name} + + +@dataclass +class VariantWithActions: + id: int + name: str + actions: List[str] = field(default_factory=list) + _available_actions: Set[str] = field(default_factory=set) + + def set_actions(self, actions: Set[str]) -> None: + for action in actions: + if action in self._available_actions: + self.actions.append(action) + + def reset_actions(self) -> None: + self.actions = [] + + def asdict(self) -> Dict[str, Any]: + return { + "id": self.id, + "name": self.name, + "actions": list(set(self.actions)), + } From 0b9e931ee6f8042600ffc85369b3a3898eeead3e Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Mon, 28 Oct 2024 14:12:32 -0700 Subject: [PATCH 04/12] Refactor StepReasoning to also reuse Variants --- .../schema/tool_building/fact_checking_tool.py | 12 ++++++++++++ .../src/labelbox/schema/tool_building/variant.py | 10 +++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/libs/labelbox/src/labelbox/schema/tool_building/fact_checking_tool.py b/libs/labelbox/src/labelbox/schema/tool_building/fact_checking_tool.py index 4a02a482c..7084088d8 100644 --- a/libs/labelbox/src/labelbox/schema/tool_building/fact_checking_tool.py +++ b/libs/labelbox/src/labelbox/schema/tool_building/fact_checking_tool.py @@ -61,6 +61,18 @@ def build_fact_checking_definition(): return _Definition(variants=variants) +class UnsupportedStepActions(Enum): + WRITE_JUSTIFICATION = "writeJustification" + + +class CanConfidentlyAssessStepActions(Enum): + WRITE_JUSTIFICATION = "writeJustification" + + +class NoFactualInformationStepActions(Enum): + WRITE_JUSTIFICATION = "writeJustification" + + @dataclass class FactCheckingTool(_BaseStepReasoningTool): """ diff --git a/libs/labelbox/src/labelbox/schema/tool_building/variant.py b/libs/labelbox/src/labelbox/schema/tool_building/variant.py index a101ee66a..131d76709 100644 --- a/libs/labelbox/src/labelbox/schema/tool_building/variant.py +++ b/libs/labelbox/src/labelbox/schema/tool_building/variant.py @@ -22,7 +22,8 @@ class VariantWithActions: actions: List[str] = field(default_factory=list) _available_actions: Set[str] = field(default_factory=set) - def set_actions(self, actions: Set[str]) -> None: + def set_actions(self, actions: List[str]) -> None: + self.actions = [] for action in actions: if action in self._available_actions: self.actions.append(action) @@ -31,8 +32,11 @@ def reset_actions(self) -> None: self.actions = [] def asdict(self) -> Dict[str, Any]: - return { + data = { "id": self.id, "name": self.name, - "actions": list(set(self.actions)), } + if len(self.actions) > 0: + data["actions"] = self.actions + + return data From 97be3988d9bf83faae83e95a99141c644c4cb229 Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Tue, 29 Oct 2024 10:05:16 -0700 Subject: [PATCH 05/12] Add some more documentation --- .../src/labelbox/schema/tool_building/fact_checking_tool.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libs/labelbox/src/labelbox/schema/tool_building/fact_checking_tool.py b/libs/labelbox/src/labelbox/schema/tool_building/fact_checking_tool.py index 7084088d8..cddb2aef0 100644 --- a/libs/labelbox/src/labelbox/schema/tool_building/fact_checking_tool.py +++ b/libs/labelbox/src/labelbox/schema/tool_building/fact_checking_tool.py @@ -77,6 +77,8 @@ class NoFactualInformationStepActions(Enum): class FactCheckingTool(_BaseStepReasoningTool): """ Use this class in OntologyBuilder to create a tool for fact checking + + Note variant kinds can not be changed """ type: ToolType = field(default=ToolType.FACT_CHECKING, init=False) From a358455d00b1958464cb0f5a6abd1a75b720f1be Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Tue, 29 Oct 2024 11:40:39 -0700 Subject: [PATCH 06/12] Make variants internal --- .../labelbox/schema/tool_building/variant.py | 12 ++--- .../unit/test_unit_fact_checking_tool.py | 44 +++++++++++++++++++ 2 files changed, 47 insertions(+), 9 deletions(-) diff --git a/libs/labelbox/src/labelbox/schema/tool_building/variant.py b/libs/labelbox/src/labelbox/schema/tool_building/variant.py index 131d76709..66df4eb94 100644 --- a/libs/labelbox/src/labelbox/schema/tool_building/variant.py +++ b/libs/labelbox/src/labelbox/schema/tool_building/variant.py @@ -16,9 +16,7 @@ def asdict(self) -> Dict[str, Any]: @dataclass -class VariantWithActions: - id: int - name: str +class VariantWithActions(Variant): actions: List[str] = field(default_factory=list) _available_actions: Set[str] = field(default_factory=set) @@ -32,11 +30,7 @@ def reset_actions(self) -> None: self.actions = [] def asdict(self) -> Dict[str, Any]: - data = { - "id": self.id, - "name": self.name, - } - if len(self.actions) > 0: - data["actions"] = self.actions + data = super().asdict() + data["actions"] = self.actions return data diff --git a/libs/labelbox/tests/unit/test_unit_fact_checking_tool.py b/libs/labelbox/tests/unit/test_unit_fact_checking_tool.py index 019f1355b..72a2d4d8e 100644 --- a/libs/labelbox/tests/unit/test_unit_fact_checking_tool.py +++ b/libs/labelbox/tests/unit/test_unit_fact_checking_tool.py @@ -41,3 +41,47 @@ def test_fact_checking_as_dict_default(): } assert tool_dict == expected_dict + + +def test_step_reasoning_as_dict_with_actions(): + tool = FactCheckingTool(name="Fact Checking Tool") + tool.set_unsupported_step_actions([]) + tool.set_cant_confidently_assess_step_actions([]) + tool.set_no_factual_information_step_actions([]) + + # Get the dictionary representation + tool_dict = tool.asdict() + + # Expected dictionary structure + expected_dict = { + "tool": "fact-checking", + "name": "Fact Checking Tool", + "required": False, + "schemaNodeId": None, + "featureSchemaId": None, + "definition": { + "variants": [ + {"id": 0, "name": "Accurate"}, + {"id": 1, "name": "Inaccurate"}, + {"id": 2, "name": "Disputed"}, + { + "id": 3, + "name": "Unsupported", + "actions": [], + }, + { + "id": 4, + "name": "Can't confidently assess", + "actions": [], + }, + { + "id": 5, + "name": "No factual information", + "actions": [], + }, + ], + "version": 1, + }, + } + + assert tool_dict == expected_dict From cd59a7638d18fb4c6f0a94142f16b9035d1e195c Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Fri, 1 Nov 2024 12:44:47 -0700 Subject: [PATCH 07/12] Add prompt issue tool --- libs/labelbox/src/labelbox/__init__.py | 7 +- .../src/labelbox/schema/data_row_metadata.py | 2 +- libs/labelbox/src/labelbox/schema/ontology.py | 395 +----------------- .../schema/ontology_building/__init__.py | 7 + .../ontology_building/classification.py | 367 ++++++++++++++++ .../fact_checking_tool.py | 0 .../ontology_building/prompt_issue_tool.py | 77 ++++ .../step_reasoning_tool.py | 0 .../tool_type.py | 1 + .../tool_type_mapping.py | 10 +- .../schema/ontology_building/types.py | 10 + .../variant.py | 0 .../labelbox/schema/tool_building/__init__.py | 5 - .../schema/tool_building/prompt_issue_tool.py | 76 ++++ libs/labelbox/tests/integration/conftest.py | 8 + .../tests/integration/test_ontology.py | 8 +- 16 files changed, 582 insertions(+), 391 deletions(-) create mode 100644 libs/labelbox/src/labelbox/schema/ontology_building/__init__.py create mode 100644 libs/labelbox/src/labelbox/schema/ontology_building/classification.py rename libs/labelbox/src/labelbox/schema/{tool_building => ontology_building}/fact_checking_tool.py (100%) create mode 100644 libs/labelbox/src/labelbox/schema/ontology_building/prompt_issue_tool.py rename libs/labelbox/src/labelbox/schema/{tool_building => ontology_building}/step_reasoning_tool.py (100%) rename libs/labelbox/src/labelbox/schema/{tool_building => ontology_building}/tool_type.py (90%) rename libs/labelbox/src/labelbox/schema/{tool_building => ontology_building}/tool_type_mapping.py (56%) create mode 100644 libs/labelbox/src/labelbox/schema/ontology_building/types.py rename libs/labelbox/src/labelbox/schema/{tool_building => ontology_building}/variant.py (100%) delete mode 100644 libs/labelbox/src/labelbox/schema/tool_building/__init__.py create mode 100644 libs/labelbox/src/labelbox/schema/tool_building/prompt_issue_tool.py diff --git a/libs/labelbox/src/labelbox/__init__.py b/libs/labelbox/src/labelbox/__init__.py index 74fc047ed..61b22d697 100644 --- a/libs/labelbox/src/labelbox/__init__.py +++ b/libs/labelbox/src/labelbox/__init__.py @@ -50,9 +50,7 @@ FeatureSchema, Ontology, OntologyBuilder, - Option, PromptResponseClassification, - ResponseOption, Tool, ) from labelbox.schema.tool_building.fact_checking_tool import FactCheckingTool @@ -94,3 +92,8 @@ from labelbox.schema.task_queue import TaskQueue from labelbox.schema.user import User from labelbox.schema.webhook import Webhook +from labelbox.schema.ontology_building.classification import ( + Classification, + Option, + ResponseOption, +) diff --git a/libs/labelbox/src/labelbox/schema/data_row_metadata.py b/libs/labelbox/src/labelbox/schema/data_row_metadata.py index 58111828f..7872ed55c 100644 --- a/libs/labelbox/src/labelbox/schema/data_row_metadata.py +++ b/libs/labelbox/src/labelbox/schema/data_row_metadata.py @@ -29,7 +29,7 @@ from labelbox.schema.identifiable import GlobalKey, UniqueId from labelbox.schema.identifiables import DataRowIdentifiers, UniqueIds -from labelbox.schema.ontology import SchemaId +from labelbox.schema.ontology_building.types import SchemaId from labelbox.utils import ( _CamelCaseMixin, format_iso_datetime, diff --git a/libs/labelbox/src/labelbox/schema/ontology.py b/libs/labelbox/src/labelbox/schema/ontology.py index 607bf8fec..24cdf56b9 100644 --- a/libs/labelbox/src/labelbox/schema/ontology.py +++ b/libs/labelbox/src/labelbox/schema/ontology.py @@ -2,30 +2,30 @@ import colorsys import json -import warnings from dataclasses import dataclass, field from enum import Enum -from typing import Annotated, Any, Dict, List, Optional, Type, Union +from typing import Any, Dict, List, Optional, Union from lbox.exceptions import InconsistentOntologyException -from pydantic import StringConstraints from labelbox.orm.db_object import DbObject from labelbox.orm.model import Field, Relationship -from labelbox.schema.tool_building.fact_checking_tool import FactCheckingTool -from labelbox.schema.tool_building.step_reasoning_tool import StepReasoningTool -from labelbox.schema.tool_building.tool_type import ToolType -from labelbox.schema.tool_building.tool_type_mapping import ( +from labelbox.schema.ontology_building.classification import ( + Classification, + PromptResponseClassification, +) +from labelbox.schema.ontology_building.fact_checking_tool import ( + FactCheckingTool, +) +from labelbox.schema.ontology_building.prompt_issue_tool import PromptIssueTool +from labelbox.schema.ontology_building.step_reasoning_tool import ( + StepReasoningTool, +) +from labelbox.schema.ontology_building.tool_type import ToolType +from labelbox.schema.ontology_building.tool_type_mapping import ( map_tool_type_to_tool_cls, ) -FeatureSchemaId: Type[str] = Annotated[ - str, StringConstraints(min_length=25, max_length=25) -] -SchemaId: Type[str] = Annotated[ - str, StringConstraints(min_length=25, max_length=25) -] - class DeleteFeatureFromOntologyResult: archived: bool @@ -44,367 +44,6 @@ class FeatureSchema(DbObject): normalized = Field.Json("normalized") -@dataclass -class Option: - """ - An option is a possible answer within a Classification object in - a Project's ontology. - - To instantiate, only the "value" parameter needs to be passed in. - - Example(s): - option = Option(value = "Option Example") - - Attributes: - value: (str) - schema_id: (str) - feature_schema_id: (str) - options: (list) - """ - - value: Union[str, int] - label: Optional[Union[str, int]] = None - schema_id: Optional[str] = None - feature_schema_id: Optional[FeatureSchemaId] = None - options: Union[ - List["Classification"], List["PromptResponseClassification"] - ] = field(default_factory=list) - - def __post_init__(self): - if self.label is None: - self.label = self.value - - @classmethod - def from_dict( - cls, dictionary: Dict[str, Any] - ) -> Dict[Union[str, int], Union[str, int]]: - return cls( - value=dictionary["value"], - label=dictionary["label"], - schema_id=dictionary.get("schemaNodeId", None), - feature_schema_id=dictionary.get("featureSchemaId", None), - options=[ - Classification.from_dict(o) - for o in dictionary.get("options", []) - ], - ) - - def asdict(self) -> Dict[str, Any]: - return { - "schemaNodeId": self.schema_id, - "featureSchemaId": self.feature_schema_id, - "label": self.label, - "value": self.value, - "options": [o.asdict(is_subclass=True) for o in self.options], - } - - def add_option( - self, option: Union["Classification", "PromptResponseClassification"] - ) -> None: - if option.name in (o.name for o in self.options): - raise InconsistentOntologyException( - f"Duplicate nested classification '{option.name}' " - f"for option '{self.label}'" - ) - self.options.append(option) - - -@dataclass -class Classification: - """ - A classification to be added to a Project's ontology. The - classification is dependent on the Classification Type. - - To instantiate, the "class_type" and "name" parameters must - be passed in. - - The "options" parameter holds a list of Option objects. This is not - necessary for some Classification types, such as TEXT. To see which - types require options, look at the "_REQUIRES_OPTIONS" class variable. - - Example(s): - classification = Classification( - class_type = Classification.Type.TEXT, - name = "Classification Example") - - classification_two = Classification( - class_type = Classification.Type.RADIO, - name = "Second Example") - classification_two.add_option(Option( - value = "Option Example")) - - Attributes: - class_type: (Classification.Type) - name: (str) - instructions: (str) - required: (bool) - options: (list) - ui_mode: (str) - schema_id: (str) - feature_schema_id: (str) - scope: (str) - """ - - class Type(Enum): - TEXT = "text" - CHECKLIST = "checklist" - RADIO = "radio" - - class Scope(Enum): - GLOBAL = "global" - INDEX = "index" - - class UIMode(Enum): - HOTKEY = "hotkey" - SEARCHABLE = "searchable" - - _REQUIRES_OPTIONS = {Type.CHECKLIST, Type.RADIO} - - class_type: Type - name: Optional[str] = None - instructions: Optional[str] = None - required: bool = False - options: List[Option] = field(default_factory=list) - schema_id: Optional[str] = None - feature_schema_id: Optional[str] = None - scope: Scope = None - ui_mode: Optional[UIMode] = ( - None # How this classification should be answered (e.g. hotkeys / autocomplete, etc) - ) - - def __post_init__(self): - if self.name is None: - msg = ( - "When creating the Classification feature, please use “name” " - "for the classification schema name, which will be used when " - "creating annotation payload for Model-Assisted Labeling " - "Import and Label Import. “instructions” is no longer " - "supported to specify classification schema name." - ) - if self.instructions is not None: - self.name = self.instructions - warnings.warn(msg) - else: - raise ValueError(msg) - else: - if self.instructions is None: - self.instructions = self.name - - @classmethod - def from_dict(cls, dictionary: Dict[str, Any]) -> Dict[str, Any]: - return cls( - class_type=Classification.Type(dictionary["type"]), - name=dictionary["name"], - instructions=dictionary["instructions"], - required=dictionary.get("required", False), - options=[Option.from_dict(o) for o in dictionary["options"]], - ui_mode=cls.UIMode(dictionary["uiMode"]) - if "uiMode" in dictionary - else None, - schema_id=dictionary.get("schemaNodeId", None), - feature_schema_id=dictionary.get("featureSchemaId", None), - scope=cls.Scope(dictionary.get("scope", cls.Scope.GLOBAL)), - ) - - def asdict(self, is_subclass: bool = False) -> Dict[str, Any]: - if self.class_type in self._REQUIRES_OPTIONS and len(self.options) < 1: - raise InconsistentOntologyException( - f"Classification '{self.name}' requires options." - ) - classification = { - "type": self.class_type.value, - "instructions": self.instructions, - "name": self.name, - "required": self.required, - "options": [o.asdict() for o in self.options], - "schemaNodeId": self.schema_id, - "featureSchemaId": self.feature_schema_id, - } - if ( - self.class_type == self.Type.RADIO - or self.class_type == self.Type.CHECKLIST - ) and self.ui_mode: - # added because this key does nothing for text so no point of including - classification["uiMode"] = self.ui_mode.value - if is_subclass: - return classification - classification["scope"] = ( - self.scope.value - if self.scope is not None - else self.Scope.GLOBAL.value - ) - return classification - - def add_option(self, option: Option) -> None: - if option.value in (o.value for o in self.options): - raise InconsistentOntologyException( - f"Duplicate option '{option.value}' " - f"for classification '{self.name}'." - ) - self.options.append(option) - - -@dataclass -class ResponseOption(Option): - """ - An option is a possible answer within a PromptResponseClassification response object in - a Project's ontology. - - To instantiate, only the "value" parameter needs to be passed in. - - Example(s): - option = ResponseOption(value = "Response Option Example") - - Attributes: - value: (str) - schema_id: (str) - feature_schema_id: (str) - options: (list) - """ - - @classmethod - def from_dict( - cls, dictionary: Dict[str, Any] - ) -> Dict[Union[str, int], Union[str, int]]: - return cls( - value=dictionary["value"], - label=dictionary["label"], - schema_id=dictionary.get("schemaNodeId", None), - feature_schema_id=dictionary.get("featureSchemaId", None), - options=[ - PromptResponseClassification.from_dict(o) - for o in dictionary.get("options", []) - ], - ) - - -@dataclass -class PromptResponseClassification: - """ - - A PromptResponseClassification to be added to a Project's ontology. The - classification is dependent on the PromptResponseClassification Type. - - To instantiate, the "class_type" and "name" parameters must - be passed in. - - The "options" parameter holds a list of Response Option objects. This is not - necessary for some Classification types, such as RESPONSE_TEXT or PROMPT. To see which - types require options, look at the "_REQUIRES_OPTIONS" class variable. - - Example(s): - >>> classification = PromptResponseClassification( - >>> class_type = PromptResponseClassification.Type.Prompt, - >>> character_min = 1, - >>> character_max = 1 - >>> name = "Prompt Classification Example") - - >>> classification_two = PromptResponseClassification( - >>> class_type = PromptResponseClassification.Type.RESPONSE_RADIO, - >>> name = "Second Example") - - >>> classification_two.add_option(ResponseOption( - >>> value = "Option Example")) - - Attributes: - class_type: (Classification.Type) - name: (str) - instructions: (str) - required: (bool) - options: (list) - character_min: (int) - character_max: (int) - schema_id: (str) - feature_schema_id: (str) - """ - - def __post_init__(self): - if self.name is None: - msg = ( - "When creating the Classification feature, please use “name” " - "for the classification schema name, which will be used when " - "creating annotation payload for Model-Assisted Labeling " - "Import and Label Import. “instructions” is no longer " - "supported to specify classification schema name." - ) - if self.instructions is not None: - self.name = self.instructions - warnings.warn(msg) - else: - raise ValueError(msg) - else: - if self.instructions is None: - self.instructions = self.name - - class Type(Enum): - PROMPT = "prompt" - RESPONSE_TEXT = "response-text" - RESPONSE_CHECKLIST = "response-checklist" - RESPONSE_RADIO = "response-radio" - - _REQUIRES_OPTIONS = {Type.RESPONSE_CHECKLIST, Type.RESPONSE_RADIO} - - class_type: Type - name: Optional[str] = None - instructions: Optional[str] = None - required: bool = True - options: List[ResponseOption] = field(default_factory=list) - character_min: Optional[int] = None - character_max: Optional[int] = None - schema_id: Optional[str] = None - feature_schema_id: Optional[str] = None - - @classmethod - def from_dict(cls, dictionary: Dict[str, Any]) -> Dict[str, Any]: - return cls( - class_type=PromptResponseClassification.Type(dictionary["type"]), - name=dictionary["name"], - instructions=dictionary["instructions"], - required=True, # always required - options=[ - ResponseOption.from_dict(o) for o in dictionary["options"] - ], - character_min=dictionary.get("minCharacters", None), - character_max=dictionary.get("maxCharacters", None), - schema_id=dictionary.get("schemaNodeId", None), - feature_schema_id=dictionary.get("featureSchemaId", None), - ) - - def asdict(self, is_subclass: bool = False) -> Dict[str, Any]: - if self.class_type in self._REQUIRES_OPTIONS and len(self.options) < 1: - raise InconsistentOntologyException( - f"Response Classification '{self.name}' requires options." - ) - classification = { - "type": self.class_type.value, - "instructions": self.instructions, - "name": self.name, - "required": True, # always required - "options": [o.asdict() for o in self.options], - "schemaNodeId": self.schema_id, - "featureSchemaId": self.feature_schema_id, - } - if ( - self.class_type == self.Type.PROMPT - or self.class_type == self.Type.RESPONSE_TEXT - ): - if self.character_min: - classification["minCharacters"] = self.character_min - if self.character_max: - classification["maxCharacters"] = self.character_max - if is_subclass: - return classification - return classification - - def add_option(self, option: ResponseOption) -> None: - if option.value in (o.value for o in self.options): - raise InconsistentOntologyException( - f"Duplicate option '{option.value}' " - f"for response classification '{self.name}'." - ) - self.options.append(option) - - @dataclass class Tool: """ @@ -606,9 +245,9 @@ class OntologyBuilder: """ - tools: List[Union[Tool, StepReasoningTool, FactCheckingTool]] = field( - default_factory=list - ) + tools: List[ + Union[Tool, StepReasoningTool, FactCheckingTool, PromptIssueTool] + ] = field(default_factory=list) classifications: List[ Union[Classification, PromptResponseClassification] ] = field(default_factory=list) diff --git a/libs/labelbox/src/labelbox/schema/ontology_building/__init__.py b/libs/labelbox/src/labelbox/schema/ontology_building/__init__.py new file mode 100644 index 000000000..242566659 --- /dev/null +++ b/libs/labelbox/src/labelbox/schema/ontology_building/__init__.py @@ -0,0 +1,7 @@ +import labelbox.schema.ontology_building.tool_type +import labelbox.schema.ontology_building.variant +import labelbox.schema.ontology_building.step_reasoning_tool +import labelbox.schema.ontology_building.fact_checking_tool +import labelbox.schema.ontology_building.tool_type_mapping +import labelbox.schema.ontology_building.types +import labelbox.schema.ontology_building.classification diff --git a/libs/labelbox/src/labelbox/schema/ontology_building/classification.py b/libs/labelbox/src/labelbox/schema/ontology_building/classification.py new file mode 100644 index 000000000..dcf7787a1 --- /dev/null +++ b/libs/labelbox/src/labelbox/schema/ontology_building/classification.py @@ -0,0 +1,367 @@ +import warnings +from dataclasses import dataclass, field +from enum import Enum +from typing import Any, Dict, List, Optional, Union + +from lbox.exceptions import InconsistentOntologyException + +from labelbox.schema.ontology_building.types import FeatureSchemaId + + +@dataclass +class Classification: + """ + A classification to be added to a Project's ontology. The + classification is dependent on the Classification Type. + + To instantiate, the "class_type" and "name" parameters must + be passed in. + + The "options" parameter holds a list of Option objects. This is not + necessary for some Classification types, such as TEXT. To see which + types require options, look at the "_REQUIRES_OPTIONS" class variable. + + Example(s): + classification = Classification( + class_type = Classification.Type.TEXT, + name = "Classification Example") + + classification_two = Classification( + class_type = Classification.Type.RADIO, + name = "Second Example") + classification_two.add_option(Option( + value = "Option Example")) + + Attributes: + class_type: (Classification.Type) + name: (str) + instructions: (str) + required: (bool) + options: (list) + ui_mode: (str) + schema_id: (str) + feature_schema_id: (str) + scope: (str) + """ + + class Type(Enum): + TEXT = "text" + CHECKLIST = "checklist" + RADIO = "radio" + + class Scope(Enum): + GLOBAL = "global" + INDEX = "index" + + class UIMode(Enum): + HOTKEY = "hotkey" + SEARCHABLE = "searchable" + + _REQUIRES_OPTIONS = {Type.CHECKLIST, Type.RADIO} + + class_type: Type + name: Optional[str] = None + instructions: Optional[str] = None + required: bool = False + options: List["Option"] = field(default_factory=list) + schema_id: Optional[str] = None + feature_schema_id: Optional[str] = None + scope: Scope = None + ui_mode: Optional[UIMode] = ( + None # How this classification should be answered (e.g. hotkeys / autocomplete, etc) + ) + + def __post_init__(self): + if self.name is None: + msg = ( + "When creating the Classification feature, please use “name” " + "for the classification schema name, which will be used when " + "creating annotation payload for Model-Assisted Labeling " + "Import and Label Import. “instructions” is no longer " + "supported to specify classification schema name." + ) + if self.instructions is not None: + self.name = self.instructions + warnings.warn(msg) + else: + raise ValueError(msg) + else: + if self.instructions is None: + self.instructions = self.name + + @classmethod + def from_dict(cls, dictionary: Dict[str, Any]) -> "Classification": + return cls( + class_type=Classification.Type(dictionary["type"]), + name=dictionary["name"], + instructions=dictionary["instructions"], + required=dictionary.get("required", False), + options=[Option.from_dict(o) for o in dictionary["options"]], + ui_mode=cls.UIMode(dictionary["uiMode"]) + if "uiMode" in dictionary + else None, + schema_id=dictionary.get("schemaNodeId", None), + feature_schema_id=dictionary.get("featureSchemaId", None), + scope=cls.Scope(dictionary.get("scope", cls.Scope.GLOBAL)), + ) + + def asdict(self, is_subclass: bool = False) -> Dict[str, Any]: + if self.class_type in self._REQUIRES_OPTIONS and len(self.options) < 1: + raise InconsistentOntologyException( + f"Classification '{self.name}' requires options." + ) + classification = { + "type": self.class_type.value, + "instructions": self.instructions, + "name": self.name, + "required": self.required, + "options": [o.asdict() for o in self.options], + "schemaNodeId": self.schema_id, + "featureSchemaId": self.feature_schema_id, + } + if ( + self.class_type == self.Type.RADIO + or self.class_type == self.Type.CHECKLIST + ) and self.ui_mode: + # added because this key does nothing for text so no point of including + classification["uiMode"] = self.ui_mode.value + if is_subclass: + return classification + classification["scope"] = ( + self.scope.value + if self.scope is not None + else self.Scope.GLOBAL.value + ) + return classification + + def add_option(self, option: "Option") -> None: + if option.value in (o.value for o in self.options): + raise InconsistentOntologyException( + f"Duplicate option '{option.value}' " + f"for classification '{self.name}'." + ) + self.options.append(option) + + +@dataclass +class Option: + """ + An option is a possible answer within a Classification object in + a Project's ontology. + + To instantiate, only the "value" parameter needs to be passed in. + + Example(s): + option = Option(value = "Option Example") + + Attributes: + value: (str) + schema_id: (str) + feature_schema_id: (str) + options: (list) + """ + + value: Union[str, int] + label: Optional[Union[str, int]] = None + schema_id: Optional[str] = None + feature_schema_id: Optional[FeatureSchemaId] = None # type: ignore + options: Union[ + List["Classification"], List["PromptResponseClassification"] + ] = field(default_factory=list) + + def __post_init__(self): + if self.label is None: + self.label = self.value + + @classmethod + def from_dict(cls, dictionary: Dict[str, Any]) -> "Option": + return cls( + value=dictionary["value"], + label=dictionary["label"], + schema_id=dictionary.get("schemaNodeId", None), + feature_schema_id=dictionary.get("featureSchemaId", None), + options=[ + Classification.from_dict(o) + for o in dictionary.get("options", []) + ], + ) + + def asdict(self) -> Dict[str, Any]: + return { + "schemaNodeId": self.schema_id, + "featureSchemaId": self.feature_schema_id, + "label": self.label, + "value": self.value, + "options": [o.asdict(is_subclass=True) for o in self.options], + } + + def add_option( + self, option: Union["Classification", "PromptResponseClassification"] + ) -> None: + if option.name in (o.name for o in self.options): + raise InconsistentOntologyException( + f"Duplicate nested classification '{option.name}' " + f"for option '{self.label}'" + ) + self.options.append(option) + + +@dataclass +class PromptResponseClassification: + """ + + A PromptResponseClassification to be added to a Project's ontology. The + classification is dependent on the PromptResponseClassification Type. + + To instantiate, the "class_type" and "name" parameters must + be passed in. + + The "options" parameter holds a list of Response Option objects. This is not + necessary for some Classification types, such as RESPONSE_TEXT or PROMPT. To see which + types require options, look at the "_REQUIRES_OPTIONS" class variable. + + Example(s): + >>> classification = PromptResponseClassification( + >>> class_type = PromptResponseClassification.Type.Prompt, + >>> character_min = 1, + >>> character_max = 1 + >>> name = "Prompt Classification Example") + + >>> classification_two = PromptResponseClassification( + >>> class_type = PromptResponseClassification.Type.RESPONSE_RADIO, + >>> name = "Second Example") + + >>> classification_two.add_option(ResponseOption( + >>> value = "Option Example")) + + Attributes: + class_type: (Classification.Type) + name: (str) + instructions: (str) + required: (bool) + options: (list) + character_min: (int) + character_max: (int) + schema_id: (str) + feature_schema_id: (str) + """ + + def __post_init__(self): + if self.name is None: + msg = ( + "When creating the Classification feature, please use “name” " + "for the classification schema name, which will be used when " + "creating annotation payload for Model-Assisted Labeling " + "Import and Label Import. “instructions” is no longer " + "supported to specify classification schema name." + ) + if self.instructions is not None: + self.name = self.instructions + warnings.warn(msg) + else: + raise ValueError(msg) + else: + if self.instructions is None: + self.instructions = self.name + + class Type(Enum): + PROMPT = "prompt" + RESPONSE_TEXT = "response-text" + RESPONSE_CHECKLIST = "response-checklist" + RESPONSE_RADIO = "response-radio" + + _REQUIRES_OPTIONS = {Type.RESPONSE_CHECKLIST, Type.RESPONSE_RADIO} + + class_type: Type + name: Optional[str] = None + instructions: Optional[str] = None + required: bool = True + options: List["ResponseOption"] = field(default_factory=list) + character_min: Optional[int] = None + character_max: Optional[int] = None + schema_id: Optional[str] = None + feature_schema_id: Optional[str] = None + + @classmethod + def from_dict( + cls, dictionary: Dict[str, Any] + ) -> "PromptResponseClassification": + return cls( + class_type=PromptResponseClassification.Type(dictionary["type"]), + name=dictionary["name"], + instructions=dictionary["instructions"], + required=True, # always required + options=[ + ResponseOption.from_dict(o) for o in dictionary["options"] + ], + character_min=dictionary.get("minCharacters", None), + character_max=dictionary.get("maxCharacters", None), + schema_id=dictionary.get("schemaNodeId", None), + feature_schema_id=dictionary.get("featureSchemaId", None), + ) + + def asdict(self, is_subclass: bool = False) -> Dict[str, Any]: + if self.class_type in self._REQUIRES_OPTIONS and len(self.options) < 1: + raise InconsistentOntologyException( + f"Response Classification '{self.name}' requires options." + ) + classification = { + "type": self.class_type.value, + "instructions": self.instructions, + "name": self.name, + "required": True, # always required + "options": [o.asdict() for o in self.options], + "schemaNodeId": self.schema_id, + "featureSchemaId": self.feature_schema_id, + } + if ( + self.class_type == self.Type.PROMPT + or self.class_type == self.Type.RESPONSE_TEXT + ): + if self.character_min: + classification["minCharacters"] = self.character_min + if self.character_max: + classification["maxCharacters"] = self.character_max + if is_subclass: + return classification + return classification + + def add_option(self, option: "ResponseOption") -> None: + if option.value in (o.value for o in self.options): + raise InconsistentOntologyException( + f"Duplicate option '{option.value}' " + f"for response classification '{self.name}'." + ) + self.options.append(option) + + +@dataclass +class ResponseOption(Option): + """ + An option is a possible answer within a PromptResponseClassification response object in + a Project's ontology. + + To instantiate, only the "value" parameter needs to be passed in. + + Example(s): + option = ResponseOption(value = "Response Option Example") + + Attributes: + value: (str) + schema_id: (str) + feature_schema_id: (str) + options: (list) + """ + + @classmethod + def from_dict(cls, dictionary: Dict[str, Any]) -> "ResponseOption": + return cls( + value=dictionary["value"], + label=dictionary["label"], + schema_id=dictionary.get("schemaNodeId", None), + feature_schema_id=dictionary.get("featureSchemaId", None), + options=[ + PromptResponseClassification.from_dict(o) + for o in dictionary.get("options", []) + ], + ) diff --git a/libs/labelbox/src/labelbox/schema/tool_building/fact_checking_tool.py b/libs/labelbox/src/labelbox/schema/ontology_building/fact_checking_tool.py similarity index 100% rename from libs/labelbox/src/labelbox/schema/tool_building/fact_checking_tool.py rename to libs/labelbox/src/labelbox/schema/ontology_building/fact_checking_tool.py diff --git a/libs/labelbox/src/labelbox/schema/ontology_building/prompt_issue_tool.py b/libs/labelbox/src/labelbox/schema/ontology_building/prompt_issue_tool.py new file mode 100644 index 000000000..025db6c1f --- /dev/null +++ b/libs/labelbox/src/labelbox/schema/ontology_building/prompt_issue_tool.py @@ -0,0 +1,77 @@ +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + +from labelbox.schema.ontology_building.classification import ( + Classification, + Option, +) +from labelbox.schema.ontology_building.tool_type import ToolType + + +def _supported_classifications() -> List[Classification]: + option_1_text = "This prompt cannot be rated (eg. contains PII, a nonsense prompt, a foreign language, or other scenario that makes the responses impossible to assess reliably). If you simply do not have expertise to tackle this prompt, please skip the task; do not mark it as not rateable" + option_2_text = 'This prompt contains a false, offensive, or controversial premise (eg. "why does 1+1=3"?)' + option_3_text = "This prompt is not self-contained (i.e. the prompt cannot be understood without additional context about previous turns, account information or images)." + options = [ + Option(label=option_1_text, value="not_rateable"), + Option(label=option_2_text, value="false_offensive_controversial"), + Option(label=option_3_text, value="not_self_contained"), + ] + return [ + Classification( + class_type=Classification.Type.CHECKLIST, + name="prompt_issue", + options=options, + ), + ] + + +@dataclass +class PromptIssueTool: + """ + Use this class in OntologyBuilder to create a tool for prompt rating + It comes with a prebuild checklist of options, which a user can modify or override + So essentially this is a tool with a prebuilt checklist classification + """ + + name: str + type: ToolType = field(default=ToolType.PROMPT_ISSUE, init=False) + required: bool = False + schema_id: Optional[str] = None + feature_schema_id: Optional[str] = None + color: Optional[str] = None + classifications: List[Classification] = field( + default_factory=_supported_classifications + ) + + def __post_init__(self): + if self.name.strip() == "": + raise ValueError("Name cannot be empty") + + def asdict(self) -> Dict[str, Any]: + return { + "tool": self.type.value, + "name": self.name, + "required": self.required, + "schemaNodeId": self.schema_id, + "featureSchemaId": self.feature_schema_id, + "classifications": [ + classification.asdict() + for classification in self.classifications + ], + "color": self.color, + } + + @classmethod + def from_dict(cls, dictionary: Dict[str, Any]) -> "PromptIssueTool": + return cls( + name=dictionary["name"], + schema_id=dictionary.get("schemaNodeId", None), + feature_schema_id=dictionary.get("featureSchemaId", None), + required=dictionary.get("required", False), + classifications=[ + Classification.from_dict(classification) + for classification in dictionary["classifications"] + ], + color=dictionary.get("color", None), + ) diff --git a/libs/labelbox/src/labelbox/schema/tool_building/step_reasoning_tool.py b/libs/labelbox/src/labelbox/schema/ontology_building/step_reasoning_tool.py similarity index 100% rename from libs/labelbox/src/labelbox/schema/tool_building/step_reasoning_tool.py rename to libs/labelbox/src/labelbox/schema/ontology_building/step_reasoning_tool.py diff --git a/libs/labelbox/src/labelbox/schema/tool_building/tool_type.py b/libs/labelbox/src/labelbox/schema/ontology_building/tool_type.py similarity index 90% rename from libs/labelbox/src/labelbox/schema/tool_building/tool_type.py rename to libs/labelbox/src/labelbox/schema/ontology_building/tool_type.py index faab626ff..9fc0ba892 100644 --- a/libs/labelbox/src/labelbox/schema/tool_building/tool_type.py +++ b/libs/labelbox/src/labelbox/schema/ontology_building/tool_type.py @@ -4,6 +4,7 @@ class ToolType(Enum): STEP_REASONING = "step-reasoning" FACT_CHECKING = "fact-checking" + PROMPT_ISSUE = "prompt-issue" @classmethod def valid(cls, tool_type: str) -> bool: diff --git a/libs/labelbox/src/labelbox/schema/tool_building/tool_type_mapping.py b/libs/labelbox/src/labelbox/schema/ontology_building/tool_type_mapping.py similarity index 56% rename from libs/labelbox/src/labelbox/schema/tool_building/tool_type_mapping.py rename to libs/labelbox/src/labelbox/schema/ontology_building/tool_type_mapping.py index 68bfb4890..4a98f421a 100644 --- a/libs/labelbox/src/labelbox/schema/tool_building/tool_type_mapping.py +++ b/libs/labelbox/src/labelbox/schema/ontology_building/tool_type_mapping.py @@ -1,6 +1,10 @@ -from labelbox.schema.tool_building.fact_checking_tool import FactCheckingTool -from labelbox.schema.tool_building.step_reasoning_tool import StepReasoningTool -from labelbox.schema.tool_building.tool_type import ToolType +from labelbox.schema.ontology_building.fact_checking_tool import ( + FactCheckingTool, +) +from labelbox.schema.ontology_building.step_reasoning_tool import ( + StepReasoningTool, +) +from labelbox.schema.ontology_building.tool_type import ToolType def map_tool_type_to_tool_cls(tool_type_str: str): diff --git a/libs/labelbox/src/labelbox/schema/ontology_building/types.py b/libs/labelbox/src/labelbox/schema/ontology_building/types.py new file mode 100644 index 000000000..564a52c35 --- /dev/null +++ b/libs/labelbox/src/labelbox/schema/ontology_building/types.py @@ -0,0 +1,10 @@ +from typing import Annotated, Type + +from pydantic import StringConstraints + +FeatureSchemaId: Type[str] = Annotated[ + str, StringConstraints(min_length=25, max_length=25) +] +SchemaId: Type[str] = Annotated[ + str, StringConstraints(min_length=25, max_length=25) +] diff --git a/libs/labelbox/src/labelbox/schema/tool_building/variant.py b/libs/labelbox/src/labelbox/schema/ontology_building/variant.py similarity index 100% rename from libs/labelbox/src/labelbox/schema/tool_building/variant.py rename to libs/labelbox/src/labelbox/schema/ontology_building/variant.py diff --git a/libs/labelbox/src/labelbox/schema/tool_building/__init__.py b/libs/labelbox/src/labelbox/schema/tool_building/__init__.py deleted file mode 100644 index b5327d643..000000000 --- a/libs/labelbox/src/labelbox/schema/tool_building/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -import labelbox.schema.tool_building.tool_type -import labelbox.schema.tool_building.variant -import labelbox.schema.tool_building.step_reasoning_tool -import labelbox.schema.tool_building.fact_checking_tool -import labelbox.schema.tool_building.tool_type_mapping diff --git a/libs/labelbox/src/labelbox/schema/tool_building/prompt_issue_tool.py b/libs/labelbox/src/labelbox/schema/tool_building/prompt_issue_tool.py new file mode 100644 index 000000000..6fe613d7f --- /dev/null +++ b/libs/labelbox/src/labelbox/schema/tool_building/prompt_issue_tool.py @@ -0,0 +1,76 @@ +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + +from labelbox.schema.ontology_building.classification import ( + Classification, +) +from labelbox.schema.ontology_building.tool_type import ToolType + + +def _supported_classifications() -> List[Classification]: + option_1_text = "This prompt cannot be rated (eg. contains PII, a nonsense prompt, a foreign language, or other scenario that makes the responses impossible to assess reliably). If you simply do not have expertise to tackle this prompt, please skip the task; do not mark it as not rateable" + option_2_text = 'This prompt contains a false, offensive, or controversial premise (eg. "why does 1+1=3"?)' + option_3_text = "This prompt is not self-contained (i.e. the prompt cannot be understood without additional context about previous turns, account information or images)." + options = [ + Option(label=option_1_text, value="not_rateable"), + Option(label=option_2_text, value="false_offensive_controversial"), + Option(label=option_3_text, value="not_self_contained"), + ] + return [ + Classification( + class_type=Classification.Type.CHECKLIST, + name="prompt_issue", + options=options, + ), + ] + + +@dataclass +class PromptIssueTool: + """ + Use this class in OntologyBuilder to create a tool for prompt rating + It comes with a prebuild checklist of options, which a user can modify or override + So essentially this is a tool with a prebuilt checklist classification + """ + + name: str + type: ToolType = field(default=ToolType.PROMPT_ISSUE, init=False) + required: bool = False + schema_id: Optional[str] = None + feature_schema_id: Optional[str] = None + color: Optional[str] = None + classifications: List[Classification] = field( + default_factory=_supported_classifications + ) + + def __post_init__(self): + if self.name.strip() == "": + raise ValueError("Name cannot be empty") + + def asdict(self) -> Dict[str, Any]: + return { + "tool": self.type.value, + "name": self.name, + "required": self.required, + "schemaNodeId": self.schema_id, + "featureSchemaId": self.feature_schema_id, + "classifications": [ + classification.asdict() + for classification in self.classifications + ], + "color": self.color, + } + + @classmethod + def from_dict(cls, dictionary: Dict[str, Any]) -> "PromptIssueTool": + return cls( + name=dictionary["name"], + schema_id=dictionary.get("schemaNodeId", None), + feature_schema_id=dictionary.get("featureSchemaId", None), + required=dictionary.get("required", False), + classifications=[ + Classification.from_dict(classification) + for classification in dictionary["classifications"] + ], + color=dictionary.get("color", None), + ) diff --git a/libs/labelbox/tests/integration/conftest.py b/libs/labelbox/tests/integration/conftest.py index acc400c21..9c443a2d3 100644 --- a/libs/labelbox/tests/integration/conftest.py +++ b/libs/labelbox/tests/integration/conftest.py @@ -22,6 +22,13 @@ Tool, ) from labelbox.schema.data_row import DataRowMetadataField +from labelbox.schema.ontology_building.fact_checking_tool import ( + FactCheckingTool, +) +from labelbox.schema.ontology_building.prompt_issue_tool import PromptIssueTool +from labelbox.schema.ontology_building.step_reasoning_tool import ( + StepReasoningTool, +) from labelbox.schema.ontology_kind import OntologyKind from labelbox.schema.user import User @@ -581,6 +588,7 @@ def chat_evaluation_ontology(client, rand_gen): ), StepReasoningTool(name="step reasoning"), FactCheckingTool(name="fact checking"), + PromptIssueTool(name="prompt issue"), ], classifications=[ Classification( diff --git a/libs/labelbox/tests/integration/test_ontology.py b/libs/labelbox/tests/integration/test_ontology.py index f87197b62..034cae135 100644 --- a/libs/labelbox/tests/integration/test_ontology.py +++ b/libs/labelbox/tests/integration/test_ontology.py @@ -5,8 +5,12 @@ from labelbox import MediaType, OntologyBuilder, Tool from labelbox.orm.model import Entity -from labelbox.schema.tool_building.fact_checking_tool import FactCheckingTool -from labelbox.schema.tool_building.step_reasoning_tool import StepReasoningTool +from labelbox.schema.ontology_building.fact_checking_tool import ( + FactCheckingTool, +) +from labelbox.schema.ontology_building.step_reasoning_tool import ( + StepReasoningTool, +) def test_feature_schema_is_not_archived(client, ontology): From 9ea76fdd158198e2e404285ef0f43f3a4baf29f5 Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Wed, 20 Nov 2024 14:20:10 -0800 Subject: [PATCH 08/12] Move all files from ontology_building to tool_building --- libs/labelbox/src/labelbox/__init__.py | 3 +- .../src/labelbox/schema/data_row_metadata.py | 2 +- libs/labelbox/src/labelbox/schema/ontology.py | 12 +-- .../schema/ontology_building/__init__.py | 7 -- .../ontology_building/prompt_issue_tool.py | 77 ------------------- .../schema/ontology_building/variant.py | 36 --------- .../labelbox/schema/tool_building/__init__.py | 8 ++ .../classification.py | 0 .../fact_checking_tool.py | 0 .../schema/tool_building/prompt_issue_tool.py | 4 +- .../step_reasoning_tool.py | 0 .../tool_type.py | 0 .../tool_type_mapping.py | 2 + .../types.py | 0 libs/labelbox/tests/integration/conftest.py | 8 +- .../tests/integration/test_ontology.py | 4 +- 16 files changed, 27 insertions(+), 136 deletions(-) delete mode 100644 libs/labelbox/src/labelbox/schema/ontology_building/__init__.py delete mode 100644 libs/labelbox/src/labelbox/schema/ontology_building/prompt_issue_tool.py delete mode 100644 libs/labelbox/src/labelbox/schema/ontology_building/variant.py create mode 100644 libs/labelbox/src/labelbox/schema/tool_building/__init__.py rename libs/labelbox/src/labelbox/schema/{ontology_building => tool_building}/classification.py (100%) rename libs/labelbox/src/labelbox/schema/{ontology_building => tool_building}/fact_checking_tool.py (100%) rename libs/labelbox/src/labelbox/schema/{ontology_building => tool_building}/step_reasoning_tool.py (100%) rename libs/labelbox/src/labelbox/schema/{ontology_building => tool_building}/tool_type.py (100%) rename libs/labelbox/src/labelbox/schema/{ontology_building => tool_building}/tool_type_mapping.py (88%) rename libs/labelbox/src/labelbox/schema/{ontology_building => tool_building}/types.py (100%) diff --git a/libs/labelbox/src/labelbox/__init__.py b/libs/labelbox/src/labelbox/__init__.py index 61b22d697..109b766a3 100644 --- a/libs/labelbox/src/labelbox/__init__.py +++ b/libs/labelbox/src/labelbox/__init__.py @@ -55,6 +55,7 @@ ) from labelbox.schema.tool_building.fact_checking_tool import FactCheckingTool from labelbox.schema.tool_building.step_reasoning_tool import StepReasoningTool +from labelbox.schema.tool_building.prompt_issue_tool import PromptIssueTool from labelbox.schema.role import Role, ProjectRole from labelbox.schema.invite import Invite, InviteLimit from labelbox.schema.data_row_metadata import ( @@ -92,7 +93,7 @@ from labelbox.schema.task_queue import TaskQueue from labelbox.schema.user import User from labelbox.schema.webhook import Webhook -from labelbox.schema.ontology_building.classification import ( +from labelbox.schema.tool_building.classification import ( Classification, Option, ResponseOption, diff --git a/libs/labelbox/src/labelbox/schema/data_row_metadata.py b/libs/labelbox/src/labelbox/schema/data_row_metadata.py index 7872ed55c..2140fa5c3 100644 --- a/libs/labelbox/src/labelbox/schema/data_row_metadata.py +++ b/libs/labelbox/src/labelbox/schema/data_row_metadata.py @@ -29,7 +29,7 @@ from labelbox.schema.identifiable import GlobalKey, UniqueId from labelbox.schema.identifiables import DataRowIdentifiers, UniqueIds -from labelbox.schema.ontology_building.types import SchemaId +from labelbox.schema.tool_building.types import SchemaId from labelbox.utils import ( _CamelCaseMixin, format_iso_datetime, diff --git a/libs/labelbox/src/labelbox/schema/ontology.py b/libs/labelbox/src/labelbox/schema/ontology.py index 24cdf56b9..ba3996d8d 100644 --- a/libs/labelbox/src/labelbox/schema/ontology.py +++ b/libs/labelbox/src/labelbox/schema/ontology.py @@ -10,19 +10,19 @@ from labelbox.orm.db_object import DbObject from labelbox.orm.model import Field, Relationship -from labelbox.schema.ontology_building.classification import ( +from labelbox.schema.ontology_building.prompt_issue_tool import PromptIssueTool +from labelbox.schema.tool_building.classification import ( Classification, PromptResponseClassification, ) -from labelbox.schema.ontology_building.fact_checking_tool import ( +from labelbox.schema.tool_building.fact_checking_tool import ( FactCheckingTool, ) -from labelbox.schema.ontology_building.prompt_issue_tool import PromptIssueTool -from labelbox.schema.ontology_building.step_reasoning_tool import ( +from labelbox.schema.tool_building.step_reasoning_tool import ( StepReasoningTool, ) -from labelbox.schema.ontology_building.tool_type import ToolType -from labelbox.schema.ontology_building.tool_type_mapping import ( +from labelbox.schema.tool_building.tool_type import ToolType +from labelbox.schema.tool_building.tool_type_mapping import ( map_tool_type_to_tool_cls, ) diff --git a/libs/labelbox/src/labelbox/schema/ontology_building/__init__.py b/libs/labelbox/src/labelbox/schema/ontology_building/__init__.py deleted file mode 100644 index 242566659..000000000 --- a/libs/labelbox/src/labelbox/schema/ontology_building/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -import labelbox.schema.ontology_building.tool_type -import labelbox.schema.ontology_building.variant -import labelbox.schema.ontology_building.step_reasoning_tool -import labelbox.schema.ontology_building.fact_checking_tool -import labelbox.schema.ontology_building.tool_type_mapping -import labelbox.schema.ontology_building.types -import labelbox.schema.ontology_building.classification diff --git a/libs/labelbox/src/labelbox/schema/ontology_building/prompt_issue_tool.py b/libs/labelbox/src/labelbox/schema/ontology_building/prompt_issue_tool.py deleted file mode 100644 index 025db6c1f..000000000 --- a/libs/labelbox/src/labelbox/schema/ontology_building/prompt_issue_tool.py +++ /dev/null @@ -1,77 +0,0 @@ -from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional - -from labelbox.schema.ontology_building.classification import ( - Classification, - Option, -) -from labelbox.schema.ontology_building.tool_type import ToolType - - -def _supported_classifications() -> List[Classification]: - option_1_text = "This prompt cannot be rated (eg. contains PII, a nonsense prompt, a foreign language, or other scenario that makes the responses impossible to assess reliably). If you simply do not have expertise to tackle this prompt, please skip the task; do not mark it as not rateable" - option_2_text = 'This prompt contains a false, offensive, or controversial premise (eg. "why does 1+1=3"?)' - option_3_text = "This prompt is not self-contained (i.e. the prompt cannot be understood without additional context about previous turns, account information or images)." - options = [ - Option(label=option_1_text, value="not_rateable"), - Option(label=option_2_text, value="false_offensive_controversial"), - Option(label=option_3_text, value="not_self_contained"), - ] - return [ - Classification( - class_type=Classification.Type.CHECKLIST, - name="prompt_issue", - options=options, - ), - ] - - -@dataclass -class PromptIssueTool: - """ - Use this class in OntologyBuilder to create a tool for prompt rating - It comes with a prebuild checklist of options, which a user can modify or override - So essentially this is a tool with a prebuilt checklist classification - """ - - name: str - type: ToolType = field(default=ToolType.PROMPT_ISSUE, init=False) - required: bool = False - schema_id: Optional[str] = None - feature_schema_id: Optional[str] = None - color: Optional[str] = None - classifications: List[Classification] = field( - default_factory=_supported_classifications - ) - - def __post_init__(self): - if self.name.strip() == "": - raise ValueError("Name cannot be empty") - - def asdict(self) -> Dict[str, Any]: - return { - "tool": self.type.value, - "name": self.name, - "required": self.required, - "schemaNodeId": self.schema_id, - "featureSchemaId": self.feature_schema_id, - "classifications": [ - classification.asdict() - for classification in self.classifications - ], - "color": self.color, - } - - @classmethod - def from_dict(cls, dictionary: Dict[str, Any]) -> "PromptIssueTool": - return cls( - name=dictionary["name"], - schema_id=dictionary.get("schemaNodeId", None), - feature_schema_id=dictionary.get("featureSchemaId", None), - required=dictionary.get("required", False), - classifications=[ - Classification.from_dict(classification) - for classification in dictionary["classifications"] - ], - color=dictionary.get("color", None), - ) diff --git a/libs/labelbox/src/labelbox/schema/ontology_building/variant.py b/libs/labelbox/src/labelbox/schema/ontology_building/variant.py deleted file mode 100644 index 66df4eb94..000000000 --- a/libs/labelbox/src/labelbox/schema/ontology_building/variant.py +++ /dev/null @@ -1,36 +0,0 @@ -from dataclasses import dataclass, field -from typing import Any, Dict, List, Set - - -@dataclass -class Variant: - """ - A variant is a single option in step-by-step reasoning or fact-checking tool. - """ - - id: int - name: str - - def asdict(self) -> Dict[str, Any]: - return {"id": self.id, "name": self.name} - - -@dataclass -class VariantWithActions(Variant): - actions: List[str] = field(default_factory=list) - _available_actions: Set[str] = field(default_factory=set) - - def set_actions(self, actions: List[str]) -> None: - self.actions = [] - for action in actions: - if action in self._available_actions: - self.actions.append(action) - - def reset_actions(self) -> None: - self.actions = [] - - def asdict(self) -> Dict[str, Any]: - data = super().asdict() - data["actions"] = self.actions - - return data diff --git a/libs/labelbox/src/labelbox/schema/tool_building/__init__.py b/libs/labelbox/src/labelbox/schema/tool_building/__init__.py new file mode 100644 index 000000000..8341fbe50 --- /dev/null +++ b/libs/labelbox/src/labelbox/schema/tool_building/__init__.py @@ -0,0 +1,8 @@ +import labelbox.schema.tool_building.tool_type +import labelbox.schema.ontology_building.variant +import labelbox.schema.tool_building.step_reasoning_tool +import labelbox.schema.tool_building.fact_checking_tool +import labelbox.schema.tool_building.prompt_issue_tool +import labelbox.schema.tool_building.tool_type_mapping +import labelbox.schema.tool_building.types +import labelbox.schema.tool_building.classification diff --git a/libs/labelbox/src/labelbox/schema/ontology_building/classification.py b/libs/labelbox/src/labelbox/schema/tool_building/classification.py similarity index 100% rename from libs/labelbox/src/labelbox/schema/ontology_building/classification.py rename to libs/labelbox/src/labelbox/schema/tool_building/classification.py diff --git a/libs/labelbox/src/labelbox/schema/ontology_building/fact_checking_tool.py b/libs/labelbox/src/labelbox/schema/tool_building/fact_checking_tool.py similarity index 100% rename from libs/labelbox/src/labelbox/schema/ontology_building/fact_checking_tool.py rename to libs/labelbox/src/labelbox/schema/tool_building/fact_checking_tool.py diff --git a/libs/labelbox/src/labelbox/schema/tool_building/prompt_issue_tool.py b/libs/labelbox/src/labelbox/schema/tool_building/prompt_issue_tool.py index 6fe613d7f..5fc2c6069 100644 --- a/libs/labelbox/src/labelbox/schema/tool_building/prompt_issue_tool.py +++ b/libs/labelbox/src/labelbox/schema/tool_building/prompt_issue_tool.py @@ -1,10 +1,10 @@ from dataclasses import dataclass, field from typing import Any, Dict, List, Optional -from labelbox.schema.ontology_building.classification import ( +from labelbox.schema.tool_building.classification import ( Classification, ) -from labelbox.schema.ontology_building.tool_type import ToolType +from labelbox.schema.tool_building.tool_type import ToolType def _supported_classifications() -> List[Classification]: diff --git a/libs/labelbox/src/labelbox/schema/ontology_building/step_reasoning_tool.py b/libs/labelbox/src/labelbox/schema/tool_building/step_reasoning_tool.py similarity index 100% rename from libs/labelbox/src/labelbox/schema/ontology_building/step_reasoning_tool.py rename to libs/labelbox/src/labelbox/schema/tool_building/step_reasoning_tool.py diff --git a/libs/labelbox/src/labelbox/schema/ontology_building/tool_type.py b/libs/labelbox/src/labelbox/schema/tool_building/tool_type.py similarity index 100% rename from libs/labelbox/src/labelbox/schema/ontology_building/tool_type.py rename to libs/labelbox/src/labelbox/schema/tool_building/tool_type.py diff --git a/libs/labelbox/src/labelbox/schema/ontology_building/tool_type_mapping.py b/libs/labelbox/src/labelbox/schema/tool_building/tool_type_mapping.py similarity index 88% rename from libs/labelbox/src/labelbox/schema/ontology_building/tool_type_mapping.py rename to libs/labelbox/src/labelbox/schema/tool_building/tool_type_mapping.py index 4a98f421a..c022fe334 100644 --- a/libs/labelbox/src/labelbox/schema/ontology_building/tool_type_mapping.py +++ b/libs/labelbox/src/labelbox/schema/tool_building/tool_type_mapping.py @@ -16,3 +16,5 @@ def map_tool_type_to_tool_cls(tool_type_str: str): return StepReasoningTool elif tool_type == ToolType.FACT_CHECKING: return FactCheckingTool + elif tool_type == ToolType.PROMPT_ISSUE: + return PromptIssueTool diff --git a/libs/labelbox/src/labelbox/schema/ontology_building/types.py b/libs/labelbox/src/labelbox/schema/tool_building/types.py similarity index 100% rename from libs/labelbox/src/labelbox/schema/ontology_building/types.py rename to libs/labelbox/src/labelbox/schema/tool_building/types.py diff --git a/libs/labelbox/tests/integration/conftest.py b/libs/labelbox/tests/integration/conftest.py index 9c443a2d3..671603503 100644 --- a/libs/labelbox/tests/integration/conftest.py +++ b/libs/labelbox/tests/integration/conftest.py @@ -22,14 +22,14 @@ Tool, ) from labelbox.schema.data_row import DataRowMetadataField -from labelbox.schema.ontology_building.fact_checking_tool import ( +from labelbox.schema.ontology_building.prompt_issue_tool import PromptIssueTool +from labelbox.schema.ontology_kind import OntologyKind +from labelbox.schema.tool_building.fact_checking_tool import ( FactCheckingTool, ) -from labelbox.schema.ontology_building.prompt_issue_tool import PromptIssueTool -from labelbox.schema.ontology_building.step_reasoning_tool import ( +from labelbox.schema.tool_building.step_reasoning_tool import ( StepReasoningTool, ) -from labelbox.schema.ontology_kind import OntologyKind from labelbox.schema.user import User diff --git a/libs/labelbox/tests/integration/test_ontology.py b/libs/labelbox/tests/integration/test_ontology.py index 034cae135..df1c935c9 100644 --- a/libs/labelbox/tests/integration/test_ontology.py +++ b/libs/labelbox/tests/integration/test_ontology.py @@ -5,10 +5,10 @@ from labelbox import MediaType, OntologyBuilder, Tool from labelbox.orm.model import Entity -from labelbox.schema.ontology_building.fact_checking_tool import ( +from labelbox.schema.tool_building.fact_checking_tool import ( FactCheckingTool, ) -from labelbox.schema.ontology_building.step_reasoning_tool import ( +from labelbox.schema.tool_building.step_reasoning_tool import ( StepReasoningTool, ) From 6f3a5260d99a37a909f78339a274291bb705ca05 Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Wed, 20 Nov 2024 15:48:30 -0800 Subject: [PATCH 09/12] Update for the latest api --- libs/labelbox/src/labelbox/schema/ontology.py | 2 +- .../labelbox/schema/tool_building/__init__.py | 1 - .../schema/tool_building/classification.py | 2 +- .../schema/tool_building/prompt_issue_tool.py | 7 ++-- .../schema/tool_building/tool_type_mapping.py | 7 ++-- libs/labelbox/tests/integration/conftest.py | 8 +--- .../tests/integration/test_ontology.py | 40 +++++++++++++++++++ 7 files changed, 51 insertions(+), 16 deletions(-) diff --git a/libs/labelbox/src/labelbox/schema/ontology.py b/libs/labelbox/src/labelbox/schema/ontology.py index ba3996d8d..0032aaad1 100644 --- a/libs/labelbox/src/labelbox/schema/ontology.py +++ b/libs/labelbox/src/labelbox/schema/ontology.py @@ -10,7 +10,6 @@ from labelbox.orm.db_object import DbObject from labelbox.orm.model import Field, Relationship -from labelbox.schema.ontology_building.prompt_issue_tool import PromptIssueTool from labelbox.schema.tool_building.classification import ( Classification, PromptResponseClassification, @@ -18,6 +17,7 @@ from labelbox.schema.tool_building.fact_checking_tool import ( FactCheckingTool, ) +from labelbox.schema.tool_building.prompt_issue_tool import PromptIssueTool from labelbox.schema.tool_building.step_reasoning_tool import ( StepReasoningTool, ) diff --git a/libs/labelbox/src/labelbox/schema/tool_building/__init__.py b/libs/labelbox/src/labelbox/schema/tool_building/__init__.py index 8341fbe50..5a555fdff 100644 --- a/libs/labelbox/src/labelbox/schema/tool_building/__init__.py +++ b/libs/labelbox/src/labelbox/schema/tool_building/__init__.py @@ -1,5 +1,4 @@ import labelbox.schema.tool_building.tool_type -import labelbox.schema.ontology_building.variant import labelbox.schema.tool_building.step_reasoning_tool import labelbox.schema.tool_building.fact_checking_tool import labelbox.schema.tool_building.prompt_issue_tool diff --git a/libs/labelbox/src/labelbox/schema/tool_building/classification.py b/libs/labelbox/src/labelbox/schema/tool_building/classification.py index dcf7787a1..55f740de2 100644 --- a/libs/labelbox/src/labelbox/schema/tool_building/classification.py +++ b/libs/labelbox/src/labelbox/schema/tool_building/classification.py @@ -5,7 +5,7 @@ from lbox.exceptions import InconsistentOntologyException -from labelbox.schema.ontology_building.types import FeatureSchemaId +from labelbox.schema.tool_building.types import FeatureSchemaId @dataclass diff --git a/libs/labelbox/src/labelbox/schema/tool_building/prompt_issue_tool.py b/libs/labelbox/src/labelbox/schema/tool_building/prompt_issue_tool.py index 5fc2c6069..12b4d338a 100644 --- a/libs/labelbox/src/labelbox/schema/tool_building/prompt_issue_tool.py +++ b/libs/labelbox/src/labelbox/schema/tool_building/prompt_issue_tool.py @@ -3,14 +3,15 @@ from labelbox.schema.tool_building.classification import ( Classification, + Option, ) from labelbox.schema.tool_building.tool_type import ToolType def _supported_classifications() -> List[Classification]: - option_1_text = "This prompt cannot be rated (eg. contains PII, a nonsense prompt, a foreign language, or other scenario that makes the responses impossible to assess reliably). If you simply do not have expertise to tackle this prompt, please skip the task; do not mark it as not rateable" - option_2_text = 'This prompt contains a false, offensive, or controversial premise (eg. "why does 1+1=3"?)' - option_3_text = "This prompt is not self-contained (i.e. the prompt cannot be understood without additional context about previous turns, account information or images)." + option_1_text = "This prompt cannot be rated (eg. contains PII, a nonsense prompt, a foreign language, or other scenario that makes the responses impossible to assess reliably). However, if you simply do not have expertise to tackle this prompt, please skip the task; do not mark it as not rateable." + option_2_text = "This prompt contains a false, offensive, or controversial premise (eg. “why does 1+1=3”?)" + option_3_text = "This prompt is not self-contained, i.e. the prompt cannot be understood without additional context about previous turns, account information or images." options = [ Option(label=option_1_text, value="not_rateable"), Option(label=option_2_text, value="false_offensive_controversial"), diff --git a/libs/labelbox/src/labelbox/schema/tool_building/tool_type_mapping.py b/libs/labelbox/src/labelbox/schema/tool_building/tool_type_mapping.py index c022fe334..6de255cc3 100644 --- a/libs/labelbox/src/labelbox/schema/tool_building/tool_type_mapping.py +++ b/libs/labelbox/src/labelbox/schema/tool_building/tool_type_mapping.py @@ -1,10 +1,11 @@ -from labelbox.schema.ontology_building.fact_checking_tool import ( +from labelbox.schema.tool_building.fact_checking_tool import ( FactCheckingTool, ) -from labelbox.schema.ontology_building.step_reasoning_tool import ( +from labelbox.schema.tool_building.prompt_issue_tool import PromptIssueTool +from labelbox.schema.tool_building.step_reasoning_tool import ( StepReasoningTool, ) -from labelbox.schema.ontology_building.tool_type import ToolType +from labelbox.schema.tool_building.tool_type import ToolType def map_tool_type_to_tool_cls(tool_type_str: str): diff --git a/libs/labelbox/tests/integration/conftest.py b/libs/labelbox/tests/integration/conftest.py index 671603503..11984cbd7 100644 --- a/libs/labelbox/tests/integration/conftest.py +++ b/libs/labelbox/tests/integration/conftest.py @@ -16,20 +16,14 @@ MediaType, OntologyBuilder, Option, + PromptIssueTool, PromptResponseClassification, ResponseOption, StepReasoningTool, Tool, ) from labelbox.schema.data_row import DataRowMetadataField -from labelbox.schema.ontology_building.prompt_issue_tool import PromptIssueTool from labelbox.schema.ontology_kind import OntologyKind -from labelbox.schema.tool_building.fact_checking_tool import ( - FactCheckingTool, -) -from labelbox.schema.tool_building.step_reasoning_tool import ( - StepReasoningTool, -) from labelbox.schema.user import User diff --git a/libs/labelbox/tests/integration/test_ontology.py b/libs/labelbox/tests/integration/test_ontology.py index df1c935c9..febfdacfd 100644 --- a/libs/labelbox/tests/integration/test_ontology.py +++ b/libs/labelbox/tests/integration/test_ontology.py @@ -5,12 +5,15 @@ from labelbox import MediaType, OntologyBuilder, Tool from labelbox.orm.model import Entity +from labelbox.schema.tool_building.classification import Classification from labelbox.schema.tool_building.fact_checking_tool import ( FactCheckingTool, ) +from labelbox.schema.tool_building.prompt_issue_tool import PromptIssueTool from labelbox.schema.tool_building.step_reasoning_tool import ( StepReasoningTool, ) +from labelbox.schema.tool_building.tool_type import ToolType def test_feature_schema_is_not_archived(client, ontology): @@ -448,3 +451,40 @@ def test_fact_checking_ontology(chat_evaluation_ontology): ], "version": 1, } + + +def test_prompt_issue_ontology(chat_evaluation_ontology): + ontology = chat_evaluation_ontology + prompt_issue = None + for tool in ontology.normalized["tools"]: + if tool["tool"] == "prompt-issue": + prompt_issue = tool + break + assert prompt_issue is not None + + assert prompt_issue["definition"] == { + "title": "prompt issue", + "value": "prompt_issue", + "color": "#ff00ff", + } + assert prompt_issue["schemaNodeId"] is not None + assert prompt_issue["featureSchemaId"] is not None + assert len(prompt_issue["classifications"]) == 1 + + prompt_issue_tool = None + for tool in ontology.tools(): + if isinstance(tool, PromptIssueTool): + prompt_issue_tool = tool + break + assert prompt_issue_tool is not None + # Assertions + assert prompt_issue_tool.name == "prompt issue" + assert prompt_issue_tool.type == ToolType.PROMPT_ISSUE + assert prompt_issue_tool.schema_id is not None + assert prompt_issue_tool.feature_schema_id is not None + + # Check classifications + assert len(prompt_issue_tool.classifications) == 1 + classification = prompt_issue_tool.classifications[0] + assert classification.class_type == Classification.Type.CHECKLIST + assert len(classification.options) == 3 # Check number of options From bd18ac1ceff55007274ff4de859a94e3f5b8fbad Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Wed, 20 Nov 2024 17:19:11 -0800 Subject: [PATCH 10/12] Validate classifications --- .../schema/tool_building/prompt_issue_tool.py | 20 ++++++ .../tests/unit/test_unit_prompt_issue_tool.py | 70 +++++++++++++++++++ 2 files changed, 90 insertions(+) create mode 100644 libs/labelbox/tests/unit/test_unit_prompt_issue_tool.py diff --git a/libs/labelbox/src/labelbox/schema/tool_building/prompt_issue_tool.py b/libs/labelbox/src/labelbox/schema/tool_building/prompt_issue_tool.py index 12b4d338a..5b9b55d9c 100644 --- a/libs/labelbox/src/labelbox/schema/tool_building/prompt_issue_tool.py +++ b/libs/labelbox/src/labelbox/schema/tool_building/prompt_issue_tool.py @@ -48,6 +48,26 @@ def __post_init__(self): if self.name.strip() == "": raise ValueError("Name cannot be empty") + if not self._validate_classifications(self.classifications): + raise ValueError("Only one checklist classification is supported") + + def __setattr__(self, name, value): + if name == "classifications" and not self._validate_classifications( + value + ): + raise ValueError("Classifications are immutable") + object.__setattr__(self, name, value) + + def _validate_classifications( + self, classifications: List[Classification] + ) -> bool: + if ( + len(classifications) != 1 + or classifications[0].class_type != Classification.Type.CHECKLIST + ): + return False + return True + def asdict(self) -> Dict[str, Any]: return { "tool": self.type.value, diff --git a/libs/labelbox/tests/unit/test_unit_prompt_issue_tool.py b/libs/labelbox/tests/unit/test_unit_prompt_issue_tool.py new file mode 100644 index 000000000..04dc89668 --- /dev/null +++ b/libs/labelbox/tests/unit/test_unit_prompt_issue_tool.py @@ -0,0 +1,70 @@ +import pytest + +from labelbox.schema.tool_building.classification import Classification +from labelbox.schema.tool_building.prompt_issue_tool import PromptIssueTool + + +def test_as_dict(): + tool = PromptIssueTool(name="Prompt Issue Tool") + + # Get the dictionary representation + tool_dict = tool.asdict() + expected_dict = { + "tool": "prompt-issue", + "name": "Prompt Issue Tool", + "required": False, + "schemaNodeId": None, + "featureSchemaId": None, + "classifications": [ + { + "type": "checklist", + "instructions": "prompt_issue", + "name": "prompt_issue", + "required": False, + "options": [ + { + "schemaNodeId": None, + "featureSchemaId": None, + "label": "This prompt cannot be rated (eg. contains PII, a nonsense prompt, a foreign language, or other scenario that makes the responses impossible to assess reliably). However, if you simply do not have expertise to tackle this prompt, please skip the task; do not mark it as not rateable.", + "value": "not_rateable", + "options": [], + }, + { + "schemaNodeId": None, + "featureSchemaId": None, + "label": "This prompt contains a false, offensive, or controversial premise (eg. “why does 1+1=3”?)", + "value": "false_offensive_controversial", + "options": [], + }, + { + "schemaNodeId": None, + "featureSchemaId": None, + "label": "This prompt is not self-contained, i.e. the prompt cannot be understood without additional context about previous turns, account information or images.", + "value": "not_self_contained", + "options": [], + }, + ], + "schemaNodeId": None, + "featureSchemaId": None, + "scope": "global", + } + ], + "color": None, + } + assert tool_dict == expected_dict + + with pytest.raises(ValueError): + tool.classifications = [ + Classification(Classification.Type.TEXT, "prompt_issue") + ] + + with pytest.raises(ValueError): + tool.classifications = "abcd" + + # Test that we can modify the classification options + classification = tool.classifications[0] + classification.options.pop() + classification.options[0].label = "test" + classification.options[0].value == "test_value" + tool.classifications = [classification] + assert tool.classifications == [classification] From 0a407e66c8eb9972b4e16039ad30ca394c71c4c0 Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Wed, 20 Nov 2024 17:30:04 -0800 Subject: [PATCH 11/12] Removed experimental and updated doc files --- docs/labelbox/index.rst | 3 ++- docs/labelbox/prompt-issue-tool.rst | 6 ++++++ ...ep_reasoning_tool.rst => step-reasoning-tool.rst} | 0 libs/labelbox/src/labelbox/__init__.py | 3 +-- .../schema/tool_building/base_step_reasoning_tool.py | 4 ---- .../schema/tool_building/fact_checking_tool.py | 12 ------------ 6 files changed, 9 insertions(+), 19 deletions(-) create mode 100644 docs/labelbox/prompt-issue-tool.rst rename docs/labelbox/{step_reasoning_tool.rst => step-reasoning-tool.rst} (100%) diff --git a/docs/labelbox/index.rst b/docs/labelbox/index.rst index f28de02fe..41207c78f 100644 --- a/docs/labelbox/index.rst +++ b/docs/labelbox/index.rst @@ -40,6 +40,7 @@ Labelbox Python SDK Documentation pagination project project-model-config + prompt-issue-tool quality-mode request-client resource-tag @@ -47,7 +48,7 @@ Labelbox Python SDK Documentation search-filters send-to-annotate-params slice - step_reasoning_tool + step-reasoning-tool task task-queue user diff --git a/docs/labelbox/prompt-issue-tool.rst b/docs/labelbox/prompt-issue-tool.rst new file mode 100644 index 000000000..7f1842e4b --- /dev/null +++ b/docs/labelbox/prompt-issue-tool.rst @@ -0,0 +1,6 @@ +Step Reasoning Tool +=============================================================================================== + +.. automodule:: labelbox.schema.tool_building.prompt_issue_tool + :members: + :show-inheritance: \ No newline at end of file diff --git a/docs/labelbox/step_reasoning_tool.rst b/docs/labelbox/step-reasoning-tool.rst similarity index 100% rename from docs/labelbox/step_reasoning_tool.rst rename to docs/labelbox/step-reasoning-tool.rst diff --git a/libs/labelbox/src/labelbox/__init__.py b/libs/labelbox/src/labelbox/__init__.py index 109b766a3..9d43ab241 100644 --- a/libs/labelbox/src/labelbox/__init__.py +++ b/libs/labelbox/src/labelbox/__init__.py @@ -46,11 +46,9 @@ from labelbox.schema.model_config import ModelConfig from labelbox.schema.model_run import DataSplit, ModelRun from labelbox.schema.ontology import ( - Classification, FeatureSchema, Ontology, OntologyBuilder, - PromptResponseClassification, Tool, ) from labelbox.schema.tool_building.fact_checking_tool import FactCheckingTool @@ -97,4 +95,5 @@ Classification, Option, ResponseOption, + PromptResponseClassification, ) diff --git a/libs/labelbox/src/labelbox/schema/tool_building/base_step_reasoning_tool.py b/libs/labelbox/src/labelbox/schema/tool_building/base_step_reasoning_tool.py index c9c9809d0..624e951a7 100644 --- a/libs/labelbox/src/labelbox/schema/tool_building/base_step_reasoning_tool.py +++ b/libs/labelbox/src/labelbox/schema/tool_building/base_step_reasoning_tool.py @@ -79,10 +79,6 @@ class _BaseStepReasoningTool(ABC): required: bool = False def __post_init__(self): - warnings.warn( - "This feature is experimental and subject to change.", - ) - if not self.name.strip(): raise ValueError("Name is required") diff --git a/libs/labelbox/src/labelbox/schema/tool_building/fact_checking_tool.py b/libs/labelbox/src/labelbox/schema/tool_building/fact_checking_tool.py index cddb2aef0..440f343cd 100644 --- a/libs/labelbox/src/labelbox/schema/tool_building/fact_checking_tool.py +++ b/libs/labelbox/src/labelbox/schema/tool_building/fact_checking_tool.py @@ -61,18 +61,6 @@ def build_fact_checking_definition(): return _Definition(variants=variants) -class UnsupportedStepActions(Enum): - WRITE_JUSTIFICATION = "writeJustification" - - -class CanConfidentlyAssessStepActions(Enum): - WRITE_JUSTIFICATION = "writeJustification" - - -class NoFactualInformationStepActions(Enum): - WRITE_JUSTIFICATION = "writeJustification" - - @dataclass class FactCheckingTool(_BaseStepReasoningTool): """ From bee392418a1b18fbc85345605f9e860eb66d33e3 Mon Sep 17 00:00:00 2001 From: Val Brodsky Date: Thu, 21 Nov 2024 10:21:07 -0800 Subject: [PATCH 12/12] Fix lint issues --- .../labelbox/schema/labeling_service_dashboard.py | 10 +++++++--- .../labelbox/schema/tool_building/classification.py | 6 +++--- .../src/labelbox/schema/tool_building/types.py | 12 ++++-------- .../test_chat_evaluation_ontology_project.py | 2 +- .../tests/unit/test_unit_fact_checking_tool.py | 12 ++++++------ 5 files changed, 21 insertions(+), 21 deletions(-) diff --git a/libs/labelbox/src/labelbox/schema/labeling_service_dashboard.py b/libs/labelbox/src/labelbox/schema/labeling_service_dashboard.py index 9179a4665..9899212e5 100644 --- a/libs/labelbox/src/labelbox/schema/labeling_service_dashboard.py +++ b/libs/labelbox/src/labelbox/schema/labeling_service_dashboard.py @@ -71,13 +71,17 @@ class LabelingServiceDashboard(_CamelCaseMixin): created_at: Optional[datetime] = Field(frozen=True, default=None) updated_at: Optional[datetime] = Field(frozen=True, default=None) created_by_id: Optional[str] = Field(frozen=True, default=None) - status: LabelingServiceStatus = Field(frozen=True, default=None) + status: Optional[LabelingServiceStatus] = Field(frozen=True, default=None) data_rows_count: int = Field(frozen=True) tasks_completed_count: int = Field(frozen=True) tasks_remaining_count: Optional[int] = Field(frozen=True, default=None) media_type: Optional[MediaType] = Field(frozen=True, default=None) - editor_task_type: EditorTaskType = Field(frozen=True, default=None) - tags: List[LabelingServiceDashboardTags] = Field(frozen=True, default=None) + editor_task_type: Optional[EditorTaskType] = Field( + frozen=True, default=None + ) + tags: Optional[List[LabelingServiceDashboardTags]] = Field( + frozen=True, default=None + ) client: Any # type Any to avoid circular import from client diff --git a/libs/labelbox/src/labelbox/schema/tool_building/classification.py b/libs/labelbox/src/labelbox/schema/tool_building/classification.py index 55f740de2..9c0c69bea 100644 --- a/libs/labelbox/src/labelbox/schema/tool_building/classification.py +++ b/libs/labelbox/src/labelbox/schema/tool_building/classification.py @@ -66,7 +66,7 @@ class UIMode(Enum): options: List["Option"] = field(default_factory=list) schema_id: Optional[str] = None feature_schema_id: Optional[str] = None - scope: Scope = None + scope: Optional[Scope] = None ui_mode: Optional[UIMode] = ( None # How this classification should be answered (e.g. hotkeys / autocomplete, etc) ) @@ -203,7 +203,7 @@ def add_option( f"Duplicate nested classification '{option.name}' " f"for option '{self.label}'" ) - self.options.append(option) + self.options.append(option) # type: ignore @dataclass @@ -305,7 +305,7 @@ def asdict(self, is_subclass: bool = False) -> Dict[str, Any]: raise InconsistentOntologyException( f"Response Classification '{self.name}' requires options." ) - classification = { + classification: Dict[str, Any] = { "type": self.class_type.value, "instructions": self.instructions, "name": self.name, diff --git a/libs/labelbox/src/labelbox/schema/tool_building/types.py b/libs/labelbox/src/labelbox/schema/tool_building/types.py index 564a52c35..0d6e34717 100644 --- a/libs/labelbox/src/labelbox/schema/tool_building/types.py +++ b/libs/labelbox/src/labelbox/schema/tool_building/types.py @@ -1,10 +1,6 @@ -from typing import Annotated, Type +from typing import Annotated -from pydantic import StringConstraints +from pydantic import Field -FeatureSchemaId: Type[str] = Annotated[ - str, StringConstraints(min_length=25, max_length=25) -] -SchemaId: Type[str] = Annotated[ - str, StringConstraints(min_length=25, max_length=25) -] +FeatureSchemaId = Annotated[str, Field(min_length=25, max_length=25)] +SchemaId = Annotated[str, Field(min_length=25, max_length=25)] diff --git a/libs/labelbox/tests/integration/test_chat_evaluation_ontology_project.py b/libs/labelbox/tests/integration/test_chat_evaluation_ontology_project.py index c5db9760c..8d94bdba2 100644 --- a/libs/labelbox/tests/integration/test_chat_evaluation_ontology_project.py +++ b/libs/labelbox/tests/integration/test_chat_evaluation_ontology_project.py @@ -15,7 +15,7 @@ def test_create_chat_evaluation_ontology_project( # here we are essentially testing the ontology creation which is a fixture assert ontology assert ontology.name - assert len(ontology.tools()) == 5 + assert len(ontology.tools()) == 6 for tool in ontology.tools(): assert tool.schema_id assert tool.feature_schema_id diff --git a/libs/labelbox/tests/unit/test_unit_fact_checking_tool.py b/libs/labelbox/tests/unit/test_unit_fact_checking_tool.py index 72a2d4d8e..6f0739d30 100644 --- a/libs/labelbox/tests/unit/test_unit_fact_checking_tool.py +++ b/libs/labelbox/tests/unit/test_unit_fact_checking_tool.py @@ -45,9 +45,8 @@ def test_fact_checking_as_dict_default(): def test_step_reasoning_as_dict_with_actions(): tool = FactCheckingTool(name="Fact Checking Tool") - tool.set_unsupported_step_actions([]) - tool.set_cant_confidently_assess_step_actions([]) - tool.set_no_factual_information_step_actions([]) + for variant in tool.definition.variants: + variant.set_actions([]) # Get the dictionary representation tool_dict = tool.asdict() @@ -59,11 +58,12 @@ def test_step_reasoning_as_dict_with_actions(): "required": False, "schemaNodeId": None, "featureSchemaId": None, + "color": None, "definition": { "variants": [ - {"id": 0, "name": "Accurate"}, - {"id": 1, "name": "Inaccurate"}, - {"id": 2, "name": "Disputed"}, + {"id": 0, "name": "Accurate", "actions": []}, + {"id": 1, "name": "Inaccurate", "actions": []}, + {"id": 2, "name": "Disputed", "actions": []}, { "id": 3, "name": "Unsupported",