New Splitter trail; revamped io_utils (#51)

* new splitter

* formatting

* revamped io utils

* formatting

* removed unused import

Author: soldni

examples/squad.py

@@ -14,7 +14,7 @@
     )
     >> sm.TextToWordsMapper(
         fields=["question", "context", "answers"],
-        splitter="whitespace",
+        splitter="ws",
     )
     >> sm.SingleSequenceStriderMapper(
         field_to_stride=["context"],

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "smashed"
-version = "0.16.0"
+version = "0.17.0"
 description = """\
 SMASHED is a toolkit designed to apply transformations to samples in \
 datasets, such as fields extraction, tokenization, prompting, batching, \

src/smashed/mappers/text.py

@@ -9,6 +9,7 @@
     BlingFireSplitter,
     WhitespacePlusSplitter,
     WhitespaceSplitter,
+    WhitespaceTrailSplitter,
 )
 
 
@@ -69,15 +70,15 @@ class TextToWordsMapper(SingleBaseMapper):
     def __init__(
         self,
         fields: Union[str, Sequence[str]],
-        splitter: Literal[
-            "blingfire", "whitespace", "whitespace_plus"
-        ] = "whitespace_plus",
+        splitter: Literal["blingfire", "ws", "plus", "trail"] = "plus",
     ):
         if splitter == "blingfire":
             self.splitter = BlingFireSplitter()
-        elif splitter == "whitespace_plus":
+        elif splitter == "plus":
             self.splitter = WhitespacePlusSplitter()
-        elif splitter == "whitespace":
+        elif splitter == "trail":
+            self.splitter = WhitespaceTrailSplitter()
+        elif splitter == "ws":
             self.splitter = WhitespaceSplitter()
         else:
             raise ValueError(f"Unknown splitter: {splitter}")