Merge pull request #123 from MeltanoLabs/fail_when_no_files_found

feat: Sometimes we don't want to fail when there's no files found

Author: visch

README.md

@@ -23,6 +23,7 @@ pipx install git+https://github.com/MeltanoLabs/tap-universal-file.git
 | file_path                    | True     | None    | The path to obtain files from. Example: `/foo/bar`. Or, for `protocol==s3`, use `s3-bucket-name` instead. |
 | file_regex                  | False    | None    | A regex pattern to only include certain files. Example: `.*\.csv`. |
 | file_type                   | False    | delimited | Must be one of `delimited`, `jsonl`, or `avro`. Indicates the type of file to sync, where `delimited` is for CSV/TSV files and similar. Note that *all* files will be read as that type, regardless of file extension. To only read from files with a matching file extension, appropriately configure `file_regex`. |
+| fail_when_no_files_found | True |     true | Fail when no files are found |
 | compression                 | False    | detect  | The encoding used to decompress data. Must be one of `none`, `zip`, `bz2`, `gzip`, `lzma`, `xz`, or `detect`. If set to `none` or any encoding, that setting will be applied to *all* files, regardless of file extension. If set to `detect`, encodings will be applied based on file extension. |
 | additional_info             | False    |       1 | If `True`, each row in tap's output will have three additional columns: `_sdc_file_name`, `_sdc_line_number`, and `_sdc_last_modified`. If `False`, these columns will not be present. Incremental replication requires `additional_info==True`. |
 | start_date                  | False    | None    | Used in place of state. Files that were last modified before the `start_date` wwill not be synced. |

tap_universal_file/files.py

@@ -133,7 +133,7 @@ def get_files(
                 yield file_dict
                 continue
 
-        if none_found:
+        if self.config["fail_when_no_files_found"] and none_found:
             msg = (
                 "No files found. Choose a different `file_path` or try a more lenient "
                 "`file_regex`."

tap_universal_file/tap.py

@@ -90,6 +90,13 @@ class TapUniversalFile(Tap):
                 "matching file extension, appropriately configure `file_regex`."
             ),
         ),
+        th.Property(
+            "fail_when_no_files_found",
+            th.BooleanType,
+            default=True,
+            required=True,
+            description="Fail when no files are found",
+        ),
         th.Property(
             "compression",
             th.StringType,