progress bar for downloader init

Author: stared

bdfr/__main__.py

@@ -39,6 +39,7 @@
     click.option("-L", "--limit", default=None, type=int),
     click.option("-l", "--link", multiple=True, default=None, type=str),
     click.option("-m", "--multireddit", multiple=True, default=None, type=str),
+    click.option("-p", "--progress-bar", is_flag=True, default=None),
     click.option(
         "-S", "--sort", type=click.Choice(("hot", "top", "new", "controversial", "rising", "relevance")), default=None
     ),

bdfr/configuration.py

@@ -34,6 +34,7 @@ def __init__(self):
         self.max_wait_time = None
         self.multireddit: list[str] = []
         self.no_dupes: bool = False
+        self.progress_bar: bool = False
         self.saved: bool = False
         self.search: Optional[str] = None
         self.search_existing: bool = False

bdfr/downloader.py

@@ -19,6 +19,7 @@
 from bdfr import exceptions as errors
 from bdfr.configuration import Configuration
 from bdfr.connector import RedditConnector
+from bdfr.progress_bar import Progress
 from bdfr.site_downloaders.download_factory import DownloadFactory
 
 logger = logging.getLogger(__name__)
@@ -43,17 +44,22 @@ def __init__(self, args: Configuration, logging_handlers: Iterable[logging.Handl
             self.master_hash_list = self.scan_existing_files(self.download_directory)
 
     def download(self):
+        progress = Progress(self.args.progress_bar, len(self.reddit_lists))
         for generator in self.reddit_lists:
+            progress.subreddit_new(generator)
             try:
                 for submission in generator:
                     try:
-                        self._download_submission(submission)
+                        success = self._download_submission(submission)
                     except prawcore.PrawcoreException as e:
                         logger.error(f"Submission {submission.id} failed to download due to a PRAW exception: {e}")
+                        success = False
+                    progress.post_done(submission, success)
             except prawcore.PrawcoreException as e:
                 logger.error(f"The submission after {submission.id} failed to download due to a PRAW exception: {e}")
                 logger.debug("Waiting 60 seconds to continue")
                 sleep(60)
+            progress.subreddit_done()
 
     def _download_submission(self, submission: praw.models.Submission):
         if submission.id in self.excluded_submission_ids:
@@ -152,6 +158,7 @@ def _download_submission(self, submission: praw.models.Submission):
             self.master_hash_list[resource_hash] = destination
             logger.debug(f"Hash added to master list: {resource_hash}")
         logger.info(f"Downloaded submission {submission.id} from {submission.subreddit.display_name}")
+        return True
 
     @staticmethod
     def scan_existing_files(directory: Path) -> dict[str, Path]:

bdfr/progress_bar.py

@@ -0,0 +1,56 @@
+import logging
+from typing import Optional
+
+from tqdm import tqdm
+
+logger = logging.getLogger()
+
+class Progress:
+    def __init__(self, progress_bar: bool, n_subreddits: int):
+        self.progress_bar = progress_bar
+        self.bar_outer: Optional[tqdm] = None
+        self.bar_inner: Optional[tqdm] = None
+
+        if self.progress_bar:
+            logger.setLevel(logging.CRITICAL)
+            self.bar_outer = tqdm(
+                total=n_subreddits,
+                initial=0,
+                desc="Subreddits",
+                unit="subreddit",
+                colour="green")
+
+    def subreddit_new(self, generator):
+        if self.progress_bar:
+            
+            # generator is a ListingGenerator or a (usually empty) list
+            try:
+                desc = generator.url
+            except:
+                desc = "Posts"
+
+            try:
+                total = generator.limit
+            except:
+                total = 1
+
+            self.bar_inner = tqdm(
+                total=total,
+                initial=0,
+                desc=desc,
+                unit="post",
+                colour="green",
+                leave=False)
+
+    def subreddit_done(self):
+        if self.progress_bar:
+            self.bar_outer.update(1)
+            self.bar_inner.close()
+
+    def post_done(self, submission, success: bool):
+        if self.progress_bar:
+            self.bar_inner.update(1)
+            title_short = submission.title[:60] + (submission.title[60:] and "...")
+            log_str = f"{submission.score:5d}🔼 {title_short}"
+            icon = "✅" if success else "❌"
+            self.bar_outer.write(f"{icon} {log_str}")