9
9
from datetime import datetime
10
10
from multiprocessing import Pool
11
11
from pathlib import Path
12
- from tqdm import tqdm
13
- from tqdm .contrib .logging import logging_redirect_tqdm
14
12
15
13
import praw
16
14
import praw .exceptions
17
15
import praw .models
18
16
import prawcore
17
+ from tqdm import tqdm
18
+ from tqdm .contrib .logging import logging_redirect_tqdm
19
19
20
20
from bdfr import exceptions as errors
21
21
from bdfr .configuration import Configuration
22
22
from bdfr .connector import RedditConnector
23
23
from bdfr .site_downloaders .download_factory import DownloadFactory
24
24
25
25
logger = logging .getLogger (__name__ )
26
- logger_progress_bar = logging .getLogger ('progress_bar' )
26
+ logger_progress_bar = logging .getLogger ("progress_bar" )
27
+
27
28
28
29
def _calc_hash (existing_file : Path ):
29
30
chunk_size = 1024 * 1024
@@ -36,6 +37,7 @@ def _calc_hash(existing_file: Path):
36
37
file_hash = md5_hash .hexdigest ()
37
38
return existing_file , file_hash
38
39
40
+
39
41
class RedditDownloader (RedditConnector ):
40
42
def __init__ (self , args : Configuration ):
41
43
super (RedditDownloader , self ).__init__ (args )
@@ -48,23 +50,23 @@ def download(self):
48
50
for generator in tqdm (self .reddit_lists , desc = "Subreddits" , unit = "subreddit" , colour = "red" ):
49
51
desc = generator .url if isinstance (generator , praw .models .ListingGenerator ) else "Posts"
50
52
for submission in tqdm (list (generator ), desc = desc , unit = "post" , colour = "green" , leave = False ):
51
- title_short = submission .title [:60 ] + (submission .title [60 :] and ' ...' )
52
- log_str = f' { submission .score :5d} 🔼 { title_short } '
53
+ title_short = submission .title [:60 ] + (submission .title [60 :] and " ..." )
54
+ log_str = f" { submission .score :5d} 🔼 { title_short } "
53
55
try :
54
56
if self ._download_submission (submission ):
55
- logger_progress_bar .info (f' ✅ { log_str } ' )
57
+ logger_progress_bar .info (f" ✅ { log_str } " )
56
58
else :
57
- logger_progress_bar .info (f' ❌ { log_str } ' )
59
+ logger_progress_bar .info (f" ❌ { log_str } " )
58
60
except prawcore .PrawcoreException as e :
59
- logger .error (f' Submission { submission .id } failed to download due to a PRAW exception: { e } ' )
60
- logger_progress_bar .info (f' ❌ { log_str } ' )
61
+ logger .error (f" Submission { submission .id } failed to download due to a PRAW exception: { e } " )
62
+ logger_progress_bar .info (f" ❌ { log_str } " )
61
63
else :
62
64
for generator in self .reddit_lists :
63
65
for submission in generator :
64
66
try :
65
67
self ._download_submission (submission )
66
68
except prawcore .PrawcoreException as e :
67
- logger .error (f' Submission { submission .id } failed to download due to a PRAW exception: { e } ' )
69
+ logger .error (f" Submission { submission .id } failed to download due to a PRAW exception: { e } " )
68
70
69
71
def _download_submission (self , submission : praw .models .Submission ):
70
72
if submission .id in self .excluded_submission_ids :
@@ -163,7 +165,9 @@ def _download_submission(self, submission: praw.models.Submission):
163
165
self .master_hash_list [resource_hash ] = destination
164
166
logger .debug (f"Hash added to master list: { resource_hash } " )
165
167
logger .info (f"Downloaded submission { submission .id } from { submission .subreddit .display_name } " )
166
- logger_progress_bar .info (f"✓ { submission .subreddit .display_name } ({ submission .score } ) { submission .title } - { submission .author } " )
168
+ logger_progress_bar .info (
169
+ f"✓ { submission .subreddit .display_name } ({ submission .score } ) { submission .title } - { submission .author } "
170
+ )
167
171
return True
168
172
169
173
@staticmethod
0 commit comments