Merge branch 'r/1.3.0'

Author: bitdruid

README.md

@@ -11,6 +11,10 @@ Internet-archive is a nice source for several OSINT-information. This tool is a
 
 This tool allows you to download content from the Wayback Machine (archive.org). You can use it to download either the latest version or all versions of web page snapshots within a specified range.
 
+## Info
+
+Linux recommended: On windows machines, the path length is limited. It can only be overcome by editing the registry. Files which exceed the path length will not be downloaded.
+
 ## Installation
 
 ### Pip
@@ -82,6 +86,10 @@ Path defaults to output-dir. Saves the result of CDX query as a file. Useful for
 - **`--cdxinject`** `<filepath>`:<br>
 Injects a CDX query file to download snapshots. Ensure the query matches the previous `--url` for correct folder structure.
 
+**Auto:**
+- **`--auto`**:<br>
+If set, csv, skip and cdxbackup/cdxinject are handled automatically. Keep the files and folders as they are. Otherwise they will not be recognized when restarting a download.
+
 ### Debug
 
 - `--debug`: If set, full traceback will be printed in case of an error. The full exception will be written into `waybackup_error.log`.

dev/venv_create.sh

@@ -11,7 +11,7 @@ fi
 
 # update pip
 "$TARGET_PATH/.venv/bin/python" -m pip install --upgrade pip
-"$TARGET_PATH/.venv/bin/python" -m pip install twine wheel
+"$TARGET_PATH/.venv/bin/python" -m pip install twine wheel setuptools
 
 # install requirements
 "$TARGET_PATH/.venv/bin/python" -m pip install -r "$TARGET_PATH/requirements.txt"

pywaybackup/Exception.py

@@ -5,6 +5,10 @@
 import linecache
 import traceback
 
+import re
+
+from pywaybackup.__version__ import __version__
+
 class Exception:
 
     new_debug = True
@@ -22,19 +26,20 @@ def init(cls, debug=False, output=None, command=None):
     @classmethod
     def exception(cls, message: str, e: Exception, tb=None):
         custom_tb = sys.exc_info()[-1] if tb is None else tb
-        original_tb = "".join(traceback.format_exception(type(e), e, e.__traceback__))
+        original_tb = cls.relativate_path("".join(traceback.format_exception(type(e), e, e.__traceback__)))
         exception_message = (
             "-------------------------\n" 
             f"!-- Exception: {message}\n"
         )
         if custom_tb is not None:
-            while custom_tb.tb_next: # loop to last traceback frame
-                custom_tb = custom_tb.tb_next 
+            while custom_tb.tb_next:  # loop to last traceback frame
+                custom_tb = custom_tb.tb_next
             tb_frame = custom_tb.tb_frame
             tb_line = custom_tb.tb_lineno
             func_name = tb_frame.f_code.co_name
-            filename = tb_frame.f_code.co_filename
+            filename = cls.relativate_path(tb_frame.f_code.co_filename)
             codeline = linecache.getline(filename, tb_line).strip()
+            local_vars = tb_frame.f_locals
             exception_message += (
                 f"!-- File: {filename}\n"
                 f"!-- Function: {func_name}\n"
@@ -45,24 +50,57 @@ def exception(cls, message: str, e: Exception, tb=None):
             exception_message += "!-- Traceback is None\n"
         exception_message += (
             f"!-- Description: {e}\n"
-            "-------------------------")
+            "-------------------------"
+        )
         print(exception_message)
         if cls.debug:
             debug_file = os.path.join(cls.output, "waybackup_error.log")
             print(f"Exception log: {debug_file}")
             print("-------------------------")
             print(f"Full traceback:\n{original_tb}")
-            if cls.new_debug: # new run, overwrite file
+            if cls.new_debug:  # new run, overwrite file
                 cls.new_debug = False
                 f = open(debug_file, "w")
                 f.write("-------------------------\n")
+                f.write(f"Version: {__version__}\n")
+                f.write("-------------------------\n")
                 f.write(f"Command: {cls.command}\n")
                 f.write("-------------------------\n\n")
-            else: # current run, append to file
+            else:  # current run, append to file
                 f = open(debug_file, "a")
             f.write(datetime.now().strftime("%Y-%m-%d %H:%M:%S") + "\n")
             f.write(exception_message + "\n")
+            f.write("!-- Local Variables:\n")
+            for var_name, value in local_vars.items():
+                if var_name in ["status_message", "headers"]:
+                    continue
+                value = cls.relativate_path(str(value))
+                value = value[:666] + " ... " if len(value) > 666 else value
+                f.write(f"    -- {var_name} = {value}\n")
+            f.write("-------------------------\n")
             f.write(original_tb + "\n")
+            f.close()
+
+    @classmethod
+    def relativate_path(cls, input: str) -> str:
+        try:
+            path_pattern = re.compile(r'File "([^"]+)"')
+            if os.path.isfile(input):  # case single path
+                return os.path.relpath(input, os.getcwd())
+            input_modified = ""
+            input_lines = input.split('\n')
+            if len(input_lines) == 1: # case single line
+                return input
+            for line in input.split('\n'): # case multiple lines
+                match = path_pattern.search(line)
+                if match:
+                    original_path = match.group(1)
+                    relative_path = os.path.relpath(original_path, os.getcwd())
+                    line = line.replace(original_path, relative_path)
+                input_modified += line + "\n"
+            return input_modified
+        except ValueError:
+            return input
 
     @staticmethod
     def exception_handler(exception_type, exception, traceback):

pywaybackup/SnapshotCollection.py

@@ -31,7 +31,13 @@ def create_list(cls, cdxResult, mode):
 
 
     @classmethod
-    def count_list(cls):
+    def count(cls, collection=False, success=False, fail=False):
+        if collection:
+            return len(cls.SNAPSHOT_COLLECTION)
+        if success:
+            return len([entry for entry in cls.SNAPSHOT_COLLECTION if entry["file"]])
+        if fail:
+            return len([entry for entry in cls.SNAPSHOT_COLLECTION if not entry["file"]])
         return len(cls.SNAPSHOT_COLLECTION)
 
 

pywaybackup/Verbosity.py

@@ -38,7 +38,7 @@ def fini(cls):
     def write(cls, message: str = None, progress: int = None):
         if cls.mode == "progress":
             if cls.pbar is None and progress == 0:
-                maxval = sc.count_list()
+                maxval = sc.count(collection=True)
                 cls.pbar = tqdm.tqdm(total=maxval, desc="Downloading", unit=" snapshot", ascii="░▒█")
             if cls.pbar is not None and progress is not None and progress > 0 :
                 cls.pbar.update(progress)

pywaybackup/__version__.py

@@ -1 +1 @@
-__version__ = "1.2.0"
+__version__ = "1.3.0"