Merge branch 'r/beta'

Author: bitdruid

README.md

@@ -2,7 +2,7 @@
 
 [![PyPI](https://img.shields.io/pypi/v/pywaybackup)](https://pypi.org/project/pywaybackup/)
 [![PyPI - Downloads](https://img.shields.io/pypi/dm/pywaybackup)](https://pypi.org/project/pywaybackup/)
-![Release](https://img.shields.io/badge/Release-alpha-red)
+![Release](https://img.shields.io/badge/Release-beta-orange)
 ![Python Version](https://img.shields.io/badge/Python-3.6-blue)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 
@@ -61,6 +61,8 @@ Specify the range in years or a specific timestamp either start, end or both. If
 
 #### Additional
 
+- `--redirect`: Follow redirects of snapshots. Default is False. If a source has not statuscode 200, archive.org will redirect to the closest snapshot. So when setting this to `true`, parts of a timestamp-folder may not truly belong to the given timestamp.
+<!-- - `--harvest`: The downloaded files are scanned for locations on the same domain. These locations (mostly resources) are then tried to be accessed within the same timestamp. Setting this to `true` may result in identical files in different timestamps but you may get a more complete snapshot of the website. -->
 - `--verbosity [LEVEL]`: Set the verbosity: json (print json response), progress (show progress bar) or standard (default).
 - `--retry [RETRY_FAILED]`: Retry failed downloads. You can specify the number of retry attempts as an integer.
 - `--worker [AMOUNT]`: The number of worker to use for downloading (simultaneous downloads). Default is 1. Beware: Using too many worker will lead into refused connections from the Wayback Machine. Duration about 1.5 minutes.
@@ -73,19 +75,15 @@ Download latest snapshot of all files:<br>
 Download latest snapshot of all files with retries:<br>
 `waybackup -u http://example.com -c --retry 3`
 
-Download all snapshots sorted per timestamp with a specified range:<br>
-`waybackup -u http://example.com -f -r 5`
+Download all snapshots sorted per timestamp with a specified range and follow redirects:<br>
+`waybackup -u http://example.com -f -r 5 --redirect`
 
 Download all snapshots sorted per timestamp with a specified range and save to a specified folder with 3 worker:<br>
 `waybackup -u http://example.com -f -r 5 -o /home/user/Downloads/snapshots --worker 3`
 
 List available snapshots per timestamp without downloading:<br>
 `waybackup -u http://example.com -f -l`
 
-## Info
-
-The script automatically follows redirects of snapshots.
-
 ## Contributing
 
 I'm always happy for some feature requests to improve the usability of this script.

pywaybackup/SnapshotCollection.py

@@ -31,15 +31,13 @@ def create_current(self):
 
     def create_entry(self, cdx_entry: dict, output: str) -> dict:
         timestamp, url = cdx_entry["timestamp"], cdx_entry["url"]
-        url_type = self.__get_url_filetype(url)
-        download_url = f"http://web.archive.org/web/{timestamp}{url_type}/{url}"
-        domain, subdir, filename = self.__split_url(url)
+        domain, subdir, filename = self.split_url(url)
         if self.MODE_CURRENT: download_dir = os.path.join(output, domain, subdir)
         else: download_dir = os.path.join(output, domain, timestamp, subdir)
         download_file = os.path.join(download_dir, filename)
         cdx_entry = {
                 "id": len(self.SNAPSHOT_COLLECTION),
-                "url": download_url, 
+                "url": self.create_archive_url(timestamp, url),
                 "file": download_file,
                 "timestamp": timestamp,
                 "origin_url": url,
@@ -48,6 +46,11 @@ def create_entry(self, cdx_entry: dict, output: str) -> dict:
             }
         return cdx_entry
 
+    @classmethod    
+    def create_archive_url(cls, timestamp: str, url: str) -> str:
+        url_type = cls.__get_url_filetype(url)
+        return f"http://web.archive.org/web/{timestamp}{url_type}/{url}"
+
     def count_list(self):
         return len(self.CDX_LIST)
 
@@ -60,7 +63,8 @@ def snapshot_collection_update(self, id: int, key: str, value: str):
         if index is not None:
             self.SNAPSHOT_COLLECTION[index][key] = value
 
-    def __get_url_filetype(self, url):
+    @classmethod
+    def get_url_filetype(cls, url):
         file_extension = os.path.splitext(url)[1][1:]
         urltype_mapping = {
             "jpg": "im_",
@@ -69,13 +73,14 @@ def __get_url_filetype(self, url):
             "gif": "im_",
             "svg": "im_",
             "ico": "im_",
-            "css": "cs_",
-            "js": "js_"
+            "css": "cs_"
+            #"js": "js_"
         }
         urltype = urltype_mapping.get(file_extension, "id_")
         return urltype
-    
-    def __split_url(self, url):
+
+    @staticmethod    
+    def split_url(url):
         parsed_url = urlparse(url)
         domain = parsed_url.netloc
         subdir = parsed_url.path.strip("/").rsplit("/", 1)[0]

pywaybackup/Verbosity.py

@@ -1,6 +1,5 @@
 import tqdm
 import json
-import time
 import pywaybackup.SnapshotCollection as sc
 
 class Verbosity:
@@ -23,15 +22,22 @@ def open(cls, args: list, snapshots: sc.SnapshotCollection):
 
     @classmethod
     def close(cls):
+        if cls.mode == "progress":
+            cls.pbar.close()
+        if cls.mode == "progress" or cls.mode == "standard":
+            successed = len([snapshot for snapshot in cls.snapshots.SNAPSHOT_COLLECTION if snapshot["success"]])
+            failed = len([snapshot for snapshot in cls.snapshots.SNAPSHOT_COLLECTION if not snapshot["success"]])
+            print(f"\nSuccessed downloads: {successed}")
+            print(f"Failed downloads: {failed}")
+            print("")
         if cls.mode == "json":
             print(json.dumps(cls.snapshots.SNAPSHOT_COLLECTION, indent=4, sort_keys=True))
-        elif cls.mode == "standard":
-            print("")
 
     @classmethod
     def write(cls, message: str = None, progress: int = None):
         if cls.mode == "progress":
             if progress == 0:
+                print("")
                 maxval = cls.snapshots.count_list()
                 cls.pbar = tqdm.tqdm(total=maxval, desc="Downloading", unit=" snapshot", ascii="░▒█")
             elif progress == 1:

pywaybackup/__version__.py

@@ -1 +1 @@
-__version__ = "0.6.3"
+__version__ = "0.6.4"