List files in local mode (#60)

* recurse_fix

* added option to follow links

Author: soldni

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "smashed"
-version = "0.21.0"
+version = "0.21.1"
 description = """\
 SMASHED is a toolkit designed to apply transformations to samples in \
 datasets, such as fields extraction, tokenization, prompting, batching, \

src/smashed/utils/__init__.py

@@ -2,9 +2,19 @@
 from .convert import bytes_from_int, int_from_bytes
 from .io_utils import (
     MultiPath,
+    compress_stream,
+    copy_directory,
+    decompress_stream,
+    exists,
+    is_dir,
+    is_file,
     open_file_for_read,
     open_file_for_write,
     recursively_list_files,
+    remove_directory,
+    remove_file,
+    stream_file_for_read,
+    upload_on_success,
 )
 from .version import get_name, get_name_and_version, get_version
 from .warnings import SmashedWarnings
@@ -13,15 +23,25 @@
 __all__ = [
     "BlingFireSplitter",
     "bytes_from_int",
+    "compress_stream",
+    "copy_directory",
+    "decompress_stream",
+    "exists",
     "get_cache_dir",
     "get_name_and_version",
     "get_name",
     "get_version",
     "int_from_bytes",
+    "is_dir",
+    "is_file",
     "MultiPath",
     "open_file_for_read",
     "open_file_for_write",
     "recursively_list_files",
+    "remove_directory",
+    "remove_file",
     "SmashedWarnings",
+    "stream_file_for_read",
+    "upload_on_success",
     "WhitespaceSplitter",
 ]

src/smashed/utils/io_utils/operations.py

@@ -329,6 +329,7 @@ def recursively_list_files(
     include_dirs: bool = False,
     include_files: bool = True,
     client: Optional[ClientType] = None,
+    local_follow_links: bool = False,
 ) -> Iterable[str]:
     """Recursively list all files in the given directory for a given
     path, local or remote.
@@ -343,6 +344,8 @@ def recursively_list_files(
             listing. Defaults to True.
         client (boto3.client, optional): The boto3 client to use. If not
             provided, one will be created if necessary.
+        local_follow_links (bool, optional): Whether to follow symlinks when
+            listing local files. Defaults to False.
     """
 
     path = MultiPath.parse(path)
@@ -373,7 +376,14 @@ def recursively_list_files(
                             yield str(path)
 
     if path.is_local:
-        for _root, dirnames, filenames in local_walk(path.as_str):
+        if not path.as_path.is_dir():
+            # yield the path itself if it's not a directory; this matches
+            # the behavior that we get for S3.
+            yield path.as_str
+
+        for _root, dirnames, filenames in local_walk(
+            top=path.as_str, followlinks=local_follow_links
+        ):
             root = Path(_root)
             to_list = [
                 *(dirnames if include_dirs else []),