Merge pull request #2554 from sebix/download-log-size

HTTP, Mail URL collector: log downloaded sizes

Author: sebix

CHANGELOG.md

@@ -23,7 +23,10 @@
 - `intelmq.bots.collectors.shadowserver.collector_reports_api.py`:
   - Fixed behaviour if parameter `types` value is empty string, behave the same way as not set, not like no type.
 - `intelmq.bots.collectors.misp`: Use `PyMISP` class instead of deprecated `ExpandedPyMISP` (PR#2532 by Radek Vyhnal)
-- `intelmq.bots.collectors.mail.collector_mail_url`: Fix import for Timeout exception preventing another exception (fixes #2555, PR#2556 by Sebastian Wagner).
+- `intelmq.bots.collectors.http.collector_http`: Log the downloaded size in bytes to ease troubleshooting (PR#2554 by Sebastian Wagner).
+- `intelmq.bots.collectors.mail.collector_mail_url`:
+  - Log the downloaded size in bytes to ease troubleshooting (PR#2554 by Sebastian Wagner).
+  - Fix import for Timeout exception preventing another exception (fixes #2555, PR#2556 by Sebastian Wagner).
 
 #### Parsers
 - `intelmq.bots.parsers.shadowserver._config`:

intelmq/bots/collectors/http/collector_http.py

@@ -90,7 +90,7 @@ def process(self):
             self.logger.debug('Response body: %r.', resp.text)
             raise ValueError('HTTP response status code was %i.' % resp.status_code)
 
-        self.logger.info("Report downloaded.")
+        self.logger.info("Report downloaded (%sB).", len(resp.content))
 
         # PGP verification
         if self.use_gpg:

intelmq/bots/collectors/mail/collector_mail_url.py

@@ -67,7 +67,7 @@ def process_message(self, uid, message):
                 if not resp.content:
                     self.logger.warning('Got empty response from server.')
                 else:
-                    self.logger.info("Report downloaded.")
+                    self.logger.info("Report downloaded (%sB).", len(resp.content))
 
                     template = self.new_report()
                     template["feed.url"] = url

intelmq/tests/bots/collectors/http/test_collector.py

@@ -107,6 +107,7 @@ def test_gzip(self, mocker):
         output['feed.url'] = 'http://localhost/foobar.gz'
         del output['extra.file_name']
         self.assertMessageEqual(0, output)
+        self.assertLogMatches('Report downloaded \(29B\).', 'INFO')
 
     def test_zip_auto(self, mocker):
         """

intelmq/tests/bots/collectors/mail/test_collector_url.py

@@ -56,3 +56,4 @@ def test_fetch(self, mocker):
         with mock.patch('imbox.Imbox', new=MockedTxtImbox):
             self.run_bot()
         self.assertMessageEqual(0, REPORT_FOOBARTXT)
+        self.assertLogMatches('Report downloaded \(9B\).', 'INFO')