Handle WARC filename conflicts with wb-manager add (#902)

Append -index to end of filename prior to extension until there is no conflict

Also makes sure this behavior is documented in tests

Author: tw4l

pywb/manager/manager.py

@@ -7,6 +7,7 @@
 import re
 import gzip
 import six
+import pathlib
 
 from distutils.util import strtobool
 from pkg_resources import resource_string, get_distribution
@@ -147,18 +148,32 @@ def add_archives(self, archives, unpack_wacz=False):
         if invalid_archives:
             logging.warning(f'Invalid archives weren\'t added: {", ".join(invalid_archives)}')
 
+    def _rename_warc(self, warc_basename):
+        dupe_idx = 1
+        ext = ''.join(pathlib.Path(warc_basename).suffixes)
+        pre_ext_name = warc_basename.split(ext)[0]
+
+        while True:
+            new_basename = f'{pre_ext_name}-{dupe_idx}{ext}'
+            if not os.path.exists(os.path.join(self.archive_dir, new_basename)):
+                break
+            dupe_idx += 1
+
+        return new_basename
+
     def _add_warc(self, warc):
-        filename = os.path.abspath(warc)
+        warc_source = os.path.abspath(warc)
+        source_dir, warc_basename = os.path.split(warc_source)
 
         # don't overwrite existing warcs with duplicate names
-        if os.path.exists(os.path.join(self.archive_dir, os.path.basename(filename))):
-            logging.warning(f'Warc {filename} wasn\'t added because of duplicate name.')
-            return None
+        if os.path.exists(os.path.join(self.archive_dir, warc_basename)):
+            warc_basename = self._rename_warc(warc_basename)
+            logging.info(f'Warc {os.path.basename(warc)} already exists - renamed to {warc_basename}.')
 
-        shutil.copy2(filename, self.archive_dir)
-        full_path = os.path.join(self.archive_dir, filename)
-        logging.info('Copied ' + filename + ' to ' + self.archive_dir)
-        return full_path
+        warc_dest = os.path.join(self.archive_dir, warc_basename)
+        shutil.copy2(warc_source, warc_dest)
+        logging.info(f'Copied {warc} to {self.archive_dir} as {warc_basename}')
+        return warc_dest
 
     def _add_wacz_unpacked(self, wacz):
         wacz = os.path.abspath(wacz)
@@ -198,8 +213,9 @@ def _add_wacz_unpacked(self, wacz):
             warc_destination_path = os.path.join(self.archive_dir, warc_filename)
 
             if os.path.exists(warc_destination_path):
-                logging.warning(f'Warc {warc_filename} wasn\'t added because of duplicate name.')
-                continue
+                warc_filename = self._rename_warc(warc_filename)
+                logging.info(f'Warc {warc_destination_path} already exists - renamed to {warc_filename}.')
+                warc_destination_path = os.path.join(self.archive_dir, warc_filename)
 
             warc_filename_mapping[os.path.basename(extracted_warc_file)] = warc_filename
             shutil.copy2(os.path.join(temp_dir, extracted_warc_file), warc_destination_path)

tests/test_manager.py

@@ -20,6 +20,20 @@ def test_add_valid_wacz_unpacked(self, tmp_path):
         with open(os.path.join(manager.indexes_dir, manager.DEF_INDEX_FILE), 'r') as f:
             assert '"filename": "valid_example_1-0.warc"' in f.read()
 
+    def test_add_valid_wacz_unpacked_dupe_name(self, tmp_path):
+        """Test if warc that already exists is renamed with -index suffix"""
+        manager = self.get_test_collections_manager(tmp_path)
+        manager._add_wacz_unpacked(VALID_WACZ_PATH)
+        # Add it again to see if there are name conflicts
+        manager._add_wacz_unpacked(VALID_WACZ_PATH)
+        assert 'valid_example_1-0.warc' in os.listdir(manager.archive_dir)
+        assert 'valid_example_1-0-1.warc' in os.listdir(manager.archive_dir)
+        assert manager.DEF_INDEX_FILE in os.listdir(manager.indexes_dir)
+        with open(os.path.join(manager.indexes_dir, manager.DEF_INDEX_FILE), 'r') as f:
+            data = f.read()
+            assert '"filename": "valid_example_1-0.warc"' in data
+            assert '"filename": "valid_example_1-0-1.warc"' in data
+
     def test_add_invalid_wacz_unpacked(self, tmp_path, caplog):
         """Test if adding an invalid wacz file to a collection fails"""
         manager = self.get_test_collections_manager(tmp_path)
@@ -51,6 +65,20 @@ def test_add_valid_archives_unpack_wacz(self, tmp_path):
             assert archive in os.listdir(manager.archive_dir)
             assert archive in index_text
 
+    def test_add_valid_archives_dupe_name(self, tmp_path):
+        manager = self.get_test_collections_manager(tmp_path)
+        warc_filename = 'sample_archive/warcs/example.warc.gz'
+        manager.add_archives([warc_filename, warc_filename])
+
+        with open(os.path.join(manager.indexes_dir, manager.DEF_INDEX_FILE), 'r') as f:
+            index_text = f.read()
+
+        expected_archives = ('example.warc.gz', 'example-1.warc.gz')
+
+        for archive in expected_archives:
+            assert archive in os.listdir(manager.archive_dir)
+            assert archive in index_text
+
     def test_add_valid_archives_dont_unpack_wacz(self, tmp_path):
         manager = self.get_test_collections_manager(tmp_path)
         archives = ['sample_archive/warcs/example.arc', 'sample_archive/warcs/example.arc.gz',