Removed transformers dependency (#52)

* made transformers dep optional

* resolved issue with versioning

Author: soldni

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "smashed"
-version = "0.17.1"
+version = "0.18.0"
 description = """\
 SMASHED is a toolkit designed to apply transformations to samples in \
 datasets, such as fields extraction, tokenization, prompting, batching, \
@@ -12,8 +12,7 @@ readme = "README.md"
 requires-python = ">=3.8"
 dependencies = [
     "torch>=1.9",
-    "transformers>=4.5",
-    "necessary>=0.3.3",
+    "necessary>=0.4.1",
     "trouting>=0.3.3",
     "ftfy>=6.1.1",
     "platformdirs>=2.5.0",
@@ -104,10 +103,12 @@ remote = [
     "boto3>=1.25.5",
 ]
 datasets = [
-  "datasets>=2.8.0",
-  "dill>=0.3.0",
+    "transformers>=4.5",
+    "datasets>=2.8.0",
+    "dill>=0.3.0",
 ]
 prompting = [
+    "transformers>=4.5",
     "promptsource>=0.2.3",
     "blingfire>=0.1.8",
 ]

src/smashed/contrib/squad.py

@@ -1,7 +1,7 @@
 from bisect import bisect_left, bisect_right
 from typing import Any, Literal, Optional, Sequence, Tuple, TypeVar, Union
 
-from transformers.tokenization_utils_base import PreTrainedTokenizerBase
+from necessary import necessary, Necessary
 
 from smashed.base import BaseRecipe, SingleBaseMapper, TransformElementType
 from smashed.base.mappers import ChainableMapperMixIn
@@ -16,6 +16,10 @@
 )
 from smashed.recipes.prompting import PromptingRecipe
 
+with necessary("transformers", soft=True):
+    from transformers.tokenization_utils_base import PreTrainedTokenizerBase
+
+
 __all__ = [
     "AddEvidencesLocationMapper",
     "ConcatenateContextMapper",
@@ -317,7 +321,7 @@ def strider_mapper(self, **kwargs) -> SingleSequenceStriderMapper:
     def __init__(
         self,
         *args,
-        tokenizer: PreTrainedTokenizerBase,
+        tokenizer: 'PreTrainedTokenizerBase',
         context_field: str = "context",
         location_field: str = "locations",
         **kwargs,
@@ -331,13 +335,14 @@ def __init__(
 C = TypeVar("C", bound=ChainableMapperMixIn)
 
 
+@Necessary("transformers")
 class SquadPromptTrainRecipe(BaseRecipe):
     def unpacking(self, pipeline: C, **kwargs: Any) -> C:
         return pipeline >> UnpackingMapper(**kwargs)
 
     def __init__(
         self,
-        tokenizer: PreTrainedTokenizerBase,
+        tokenizer: 'PreTrainedTokenizerBase',
         source_template: str,
         context_length: int,
         context_stride: int,
@@ -439,11 +444,12 @@ def __init__(
         self.chain(pipeline)
 
 
+@Necessary("transformers")
 class SquadPromptValidRecipe(SquadPromptTrainRecipe):
     def __init__(
         self,
         *args,
-        tokenizer: PreTrainedTokenizerBase,
+        tokenizer: 'PreTrainedTokenizerBase',
         target_output_name: Optional[str] = None,
         answer_field: str = "answers",
         **kwargs,

src/smashed/mappers/collators.py

@@ -4,11 +4,15 @@
 from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple, Union
 
 import torch
-from transformers.tokenization_utils_base import PreTrainedTokenizerBase
+from necessary import necessary
 
 from ..base import SingleBaseMapper, TransformElementType
 from ..base.abstract import AbstractBaseMapper
 
+with necessary("transformers", soft=True):
+    from transformers.tokenization_utils_base import PreTrainedTokenizerBase
+
+
 __all__ = [
     "ListCollatorMapper",
     "TensorCollatorMapper",
@@ -87,7 +91,7 @@ def collate(
 class FromTokenizerMixIn(BaseCollator):
     def __init__(
         self,
-        tokenizer: PreTrainedTokenizerBase,
+        tokenizer: 'PreTrainedTokenizerBase',
         pad_to_length: Optional[Union[int, Sequence[int]]] = None,
         fields_pad_ids: Optional[Mapping[str, int]] = None,
         unk_fields_pad_id: Optional[int] = None,

src/smashed/mappers/decoding.py

@@ -6,17 +6,22 @@
 
 from typing import Any, Dict, Optional, Sequence, Union
 
-from transformers.tokenization_utils_base import PreTrainedTokenizerBase
+from necessary import necessary
 
 from ..base import SingleBaseMapper, TransformElementType
 
+
+with necessary("transformers", soft=True):
+    from transformers.tokenization_utils_base import PreTrainedTokenizerBase
+
+
 __all__ = ["DecodingMapper"]
 
 
 class DecodingMapper(SingleBaseMapper):
     def __init__(
         self,
-        tokenizer: PreTrainedTokenizerBase,
+        tokenizer: 'PreTrainedTokenizerBase',
         fields: Union[str, Sequence[str]],
         decode_batch: bool = False,
         skip_special_tokens: bool = False,

src/smashed/mappers/multiseq.py

@@ -12,10 +12,13 @@
     Union,
 )
 
-from transformers.tokenization_utils_base import PreTrainedTokenizerBase
+from necessary import necessary
 
 from ..base import BatchedBaseMapper, SingleBaseMapper, TransformElementType
 
+with necessary('transformers', soft=True):
+    from transformers.tokenization_utils_base import PreTrainedTokenizerBase
+
 
 class TokensSequencesPaddingMapper(SingleBaseMapper):
     bos: List[int]
@@ -24,7 +27,7 @@ class TokensSequencesPaddingMapper(SingleBaseMapper):
 
     def __init__(
         self,
-        tokenizer: PreTrainedTokenizerBase,
+        tokenizer: 'PreTrainedTokenizerBase',
         input_field: str = "input_ids",
     ) -> None:
         """Mapper that add BOS/SEP/EOS sequences of tokens.
@@ -42,7 +45,7 @@ def __init__(
 
     @staticmethod
     def _find_special_token_ids(
-        tokenizer: PreTrainedTokenizerBase,
+        tokenizer: 'PreTrainedTokenizerBase',
     ) -> Tuple[List[int], List[int], List[int]]:
         """By default, tokenizers only know how to concatenate 2 fields
         as input; However, for our purposes, we might care about more than
@@ -99,7 +102,7 @@ def transform(self, data: TransformElementType) -> TransformElementType:
 class AttentionMaskSequencePaddingMapper(TokensSequencesPaddingMapper):
     def __init__(
         self,
-        tokenizer: PreTrainedTokenizerBase,
+        tokenizer: 'PreTrainedTokenizerBase',
         input_field: str = "attention_mask",
     ) -> None:
         """Mapper to add BOS/SEP/EOS tokens to an attention mask sequence.
@@ -121,7 +124,7 @@ def __init__(
 class TokenTypeIdsSequencePaddingMapper(TokensSequencesPaddingMapper):
     def __init__(
         self,
-        tokenizer: PreTrainedTokenizerBase,
+        tokenizer: 'PreTrainedTokenizerBase',
         input_field: str = "token_type_ids",
     ) -> None:
         """Mapper to add BOS/SEP/EOS tokens to a token type ids sequence.
@@ -295,7 +298,7 @@ def __init__(
         fields_to_stride: Optional[List[str]] = None,
         max_length: Optional[int] = None,
         extra_length_per_seq: Optional[int] = None,
-        tokenizer: Optional[PreTrainedTokenizerBase] = None,
+        tokenizer: Optional['PreTrainedTokenizerBase'] = None,
         max_step: Optional[int] = None,
     ) -> None:
         """Mapper to create multiple subset sequences from a single sequence

src/smashed/mappers/prompting.py

@@ -4,13 +4,15 @@
 from string import Formatter
 from typing import Dict, List, Literal, Optional, Sequence, Union
 
-from transformers.tokenization_utils_base import PreTrainedTokenizerBase
-from transformers.tokenization_utils_fast import PreTrainedTokenizerFast
+from necessary import necessary
 
 from ..base import SingleBaseMapper, TransformElementType
 from ..utils.shape_utils import flatten_with_indices, reconstruct_from_indices
 from .tokenize import GetTokenizerOutputFieldsAndNamesMixIn
 
+with necessary("transformers", soft=True):
+    from transformers.tokenization_utils_base import PreTrainedTokenizerBase
+
 __all__ = [
     "EncodeFieldsMapper",
     "FillEncodedPromptMapper",
@@ -23,7 +25,7 @@
 class EncodeFieldsMapper(SingleBaseMapper):
     """Simply encodes the fields in the input data using the tokenizer."""
 
-    tokenizer: PreTrainedTokenizerBase
+    tokenizer: 'PreTrainedTokenizerBase'
     is_split_into_words: bool
     fields_to_encode: Dict[str, None]
 
@@ -35,7 +37,7 @@ class EncodeFieldsMapper(SingleBaseMapper):
     def __init__(
         self,
         fields_to_encode: Sequence[str],
-        tokenizer: PreTrainedTokenizerBase,
+        tokenizer: 'PreTrainedTokenizerBase',
         is_split_into_words: bool = False,
         fields_to_return_offset_mapping: Union[Sequence[str], bool] = False,
         offset_prefix: str = "offset",
@@ -61,13 +63,16 @@ def __init__(
                 new field with offsets. Defaults to "pos_start".
         """
 
-        if fields_to_return_offset_mapping and not isinstance(
-            tokenizer, PreTrainedTokenizerFast
-        ):
-            raise TypeError(
-                "return_offsets_mapping is only supported for fast tokenizers,"
-                " i.e. those that inherit from PreTrainedTokenizerFast."
-            )
+        if fields_to_return_offset_mapping and necessary("transformers"):
+            from transformers.tokenization_utils_fast \
+                import PreTrainedTokenizerFast
+
+            if not isinstance(tokenizer, PreTrainedTokenizerFast):
+                raise TypeError(
+                    "return_offsets_mapping is only supported for fast "
+                    "tokenizers, i.e., those that inherit from "
+                    "PreTrainedTokenizerFast."
+                )
 
         if isinstance(fields_to_return_offset_mapping, bool):
             # if user provides true, it means they want to return the
@@ -139,7 +144,7 @@ def __init__(
         self,
         fields_to_truncate: List[str],
         fields_to_preserve: Optional[List[str]] = None,
-        tokenizer: Optional[PreTrainedTokenizerBase] = None,
+        tokenizer: Optional['PreTrainedTokenizerBase'] = None,
         max_length: Optional[int] = None,
         length_penalty: int = 0,
         strategy: Union[Literal["longest"], Literal["uniform"]] = "longest",

src/smashed/mappers/tokenize.py

@@ -7,10 +7,13 @@
 import unicodedata
 from typing import Any, Dict, List, Optional
 
-from transformers.tokenization_utils_base import PreTrainedTokenizerBase
+from necessary import necessary
 
 from ..base import SingleBaseMapper, TransformElementType
 
+with necessary("transformers", soft=True):
+    from transformers.tokenization_utils_base import PreTrainedTokenizerBase
+
 __all__ = [
     "PaddingMapper",
     "TokenizerMapper",
@@ -23,7 +26,7 @@ class GetTokenizerOutputFieldsAndNamesMixIn:
     """A mixin class that figures out the output fields based on the arguments
     that will be passed a to tokenizer.__call__ method."""
 
-    tokenizer: PreTrainedTokenizerBase
+    tokenizer: 'PreTrainedTokenizerBase'
     _prefix: Optional[str]
 
     def __init__(
@@ -81,7 +84,7 @@ class TokenizerMapper(SingleBaseMapper, GetTokenizerOutputFieldsAndNamesMixIn):
 
     def __init__(
         self,
-        tokenizer: PreTrainedTokenizerBase,
+        tokenizer: 'PreTrainedTokenizerBase',
         input_field: str,
         output_prefix: Optional[str] = None,
         output_rename_map: Optional[Dict[str, str]] = None,

src/smashed/recipes/prompting.py

@@ -1,6 +1,6 @@
 from typing import Dict, Literal, Optional, Sequence, TypeVar, Union
 
-from transformers.tokenization_utils_base import PreTrainedTokenizerBase
+from necessary import necessary
 
 from ..base.mappers import ChainableMapperMixIn
 from ..base.recipes import BaseRecipe
@@ -12,6 +12,9 @@
 )
 from ..mappers.shape import SingleSequenceStriderMapper
 
+with necessary("transformers", soft=True):
+    from transformers.tokenization_utils_base import PreTrainedTokenizerBase
+
 C = TypeVar("C", bound=ChainableMapperMixIn)
 
 
@@ -34,7 +37,7 @@ def strider_mapper(self, **kwargs) -> SingleSequenceStriderMapper:
 
     def __init__(
         self,
-        tokenizer: PreTrainedTokenizerBase,
+        tokenizer: 'PreTrainedTokenizerBase',
         source_template: str,
         source_add_bos_token: bool = True,
         source_add_eos_token: bool = False,
@@ -229,7 +232,7 @@ def _add_truncation_and_striding(
         self,
         pipeline: C,
         prompt_mapper: FillEncodedPromptMapper,
-        tokenizer: PreTrainedTokenizerBase,
+        tokenizer: 'PreTrainedTokenizerBase',
         all_fields_to_truncate: Sequence[str],
         all_fields_to_stride: Sequence[str],
         strategy: Union[Literal["longest"], Literal["uniform"]],

src/smashed/utils/version.py

@@ -4,7 +4,6 @@
 
 def get_version() -> str:
     """Get the version of the package."""
-
     # This is a workaround for the fact that if the package is installed
     # in editable mode, the version is not reliability available.
     # Therefore, we check for the existence of a file called EDITABLE,
@@ -16,7 +15,7 @@ def get_version() -> str:
     try:
         # package has been installed, so it has a version number
         # from pyproject.toml
-        version = importlib.metadata.version(__package__ or __name__)
+        version = importlib.metadata.version(get_name())
     except importlib.metadata.PackageNotFoundError:
         # package hasn't been installed, so set version to "dev"
         version = "dev"
@@ -26,9 +25,7 @@ def get_version() -> str:
 
 def get_name() -> str:
     """Get the name of the package."""
-    import smashed
-
-    return smashed.__package__ or smashed.__name__
+    return 'smashed'
 
 
 def get_name_and_version() -> str: