Merge branch 'develop' into amgp_timeout

Author: kamil-certat

.github/workflows/debian-package.yml

@@ -34,7 +34,7 @@ jobs:
 
     - name: Upload artifact
       if: ${{ github.event_name == 'push' }}
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: debian-package-${{ matrix.codename }}-${{ github.sha }}
         path: '~/artifacts'

.github/workflows/python/github.py

@@ -31,20 +31,20 @@ def __init__(self):
 
     def get_reviews(self):
         """ Get a list of reviews on a Github pull request as json object """
-        reviews = self.session.get(self.api + 'repos/{}/pulls/{}/reviews'.format(self.github_repository, self.pr_id))
+        reviews = self.session.get(self.api + f'repos/{self.github_repository}/pulls/{self.pr_id}/reviews')
         reviews.raise_for_status()
         return reviews.json()
 
     def update_review(self, review_id, body):
         """ Update a review given by `review_id` and set its body to `body` """
         payload = {'body': body}
-        resp = self.session.put(self.api + 'repos/{}/pulls/{}/reviews/{}'.format(self.github_repository, self.pr_id, review_id), json=payload)
+        resp = self.session.put(self.api + f'repos/{self.github_repository}/pulls/{self.pr_id}/reviews/{review_id}', json=payload)
         resp.raise_for_status()
         return resp.json()
 
     def post_review(self, body):
         """ Post a pull request review containing `body` and requesting changes """
         payload = {'body': body, 'event': "REQUEST_CHANGES"}
-        resp = self.session.post(self.api + 'repos/{}/pulls/{}/reviews'.format(self.github_repository, self.pr_id), json=payload)
+        resp = self.session.post(self.api + f'repos/{self.github_repository}/pulls/{self.pr_id}/reviews', json=payload)
         resp.raise_for_status()
         return resp.json()

.github/workflows/python/pycodestyle_comment.py

@@ -34,7 +34,7 @@ def style_error_format(style_error_list) -> str:
     """ Format the list of pycodestyle errors and return them a one string. """
     ret = ''
     for error in style_error_list:
-        ret += '* {}\n'.format(error)
+        ret += f'* {error}\n'
     return ret
 
 
@@ -45,7 +45,7 @@ def style_error_format(style_error_list) -> str:
         style_errors = list_style_errors()
 
         if style_errors:
-            print("Found {} errors.".format(len(style_errors)))
+            print(f"Found {len(style_errors)} errors.")
 
         gh = github.Github()
 

.github/workflows/scripts/setup-full.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# SPDX-FileCopyrightText: 2020 Birger Schacht
+# SPDX-FileCopyrightText: 2020 Birger Schacht, 2024 Institute for Common Good Technology
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
 set -x
@@ -14,6 +14,14 @@ echo -e '-XX:+DisableExplicitGC\n-Djdk.io.permissionsUseCanonicalPath=true\n-Dlo
 sudo chown -R elasticsearch:elasticsearch /etc/default/elasticsearch
 sudo systemctl start elasticsearch
 
+sudo apt update
+if [ $python_version == '3.8' ]; then
+	# for pymssql there are no wheels for 3.8 https://github.com/certtools/intelmq/issues/2539
+	DEBIAN_FRONTEND="noninteractive" sudo -E apt install -y build-essential freetds-dev libssl-dev libkrb5-dev
+fi
+# for psql (used below)
+DEBIAN_FRONTEND="noninteractive" sudo -E apt install -y postgresql-client
+
 # Install the dependencies of all the bots
 pip install wheel
 for file in intelmq/bots/*/*/REQUIREMENTS.txt; do
@@ -30,7 +38,16 @@ done
 # Setup sudo and install intelmq
 sudo sed -i '/^Defaults\tsecure_path.*$/ d' /etc/sudoers
 sudo pip install .
-sudo intelmqsetup --skip-ownership
+
+intelmq_user_exists=$(getent passwd intelmq ||:)
+if [[ "$UID" -eq '0' && -z "$intelmq_user_exists" ]]; then
+	# create an unprivileged user, if currently running as root. Otherwise dropping privileges won't work
+	groupadd -r intelmq
+	useradd -r -d /var/lib/intelmq/ -c "user running intelmq" -g intelmq -s /bin/bash intelmq
+	sudo intelmqsetup
+else
+	sudo intelmqsetup --skip-ownership
+fi
 
 # Initialize the postgres database
 intelmq_psql_initdb

.github/workflows/unittests.yml

@@ -18,7 +18,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13']
         type: ['full', 'basic']
 
     services:
@@ -59,6 +59,7 @@ jobs:
         PGPORT: 5432
         PGUSER: intelmq
         PGPASSWORD: intelmq
+        python_version: ${{ matrix.python-version }}
       run: bash .github/workflows/scripts/setup-full.sh
 
     - name: Install test dependencies

CHANGELOG.md

@@ -1,5 +1,5 @@
 <!-- comment
-   SPDX-FileCopyrightText: 2015-2024 Sebastian Wagner
+   SPDX-FileCopyrightText: 2015-2025 Sebastian Wagner
    SPDX-License-Identifier: AGPL-3.0-or-later
 -->
 
@@ -13,27 +13,59 @@
 
 ### Core
 - AMQP: Fix maintaining pipeline connection when during interrupted connections (PR#2533 by Kamil Mankowski).
+- Python 3.8 or newer is required (PR#2541 by Sebastian Wagner).
+- `intelmq.lib.utils.list_all_bots`/`intelmqctl check`: Fix check for bot executable in $PATH by using the bot name instead of the import path (fixes #2559, PR#2564 by Sebastian Wagner).
 
 ### Development
 
 ### Data Format
 
 ### Bots
 #### Collectors
+- `intelmq.bots.collectors.shadowserver.collector_reports_api.py`:
+  - Fixed behaviour if parameter `types` value is empty string, behave the same way as not set, not like no type.
+- `intelmq.bots.collectors.misp`: Use `PyMISP` class instead of deprecated `ExpandedPyMISP` (PR#2532 by Radek Vyhnal)
+- `intelmq.bots.collectors.http.collector_http`: Log the downloaded size in bytes to ease troubleshooting (PR#2554 by Sebastian Wagner).
+- `intelmq.bots.collectors.mail.collector_mail_url`:
+  - Log the downloaded size in bytes to ease troubleshooting (PR#2554 by Sebastian Wagner).
+  - Fix import for Timeout exception preventing another exception (fixes #2555, PR#2556 by Sebastian Wagner).
+- Remove `intelmq.bots.collectors.twitter` as it uses an unmaintained library and does not work any more (fixes #2346, #2441, PR#2568 by Sebastian Wagner).
+- Renamed `intelmq.bots.parser.twitter` to `intelmq.bots.parser.ioc_extractor` (PR#2568 by Sebastian Wagner).
+  - Added `intelmq.bots.parser.twitter` as a stub to load the IoC Extractor parser.
 
 #### Parsers
 - `intelmq.bots.parsers.shadowserver._config`:
   - fix error message formatting if schema file is absent (PR#2528 by Sebastian Wagner).
+- `intelmq.bots.parsers.shadowserver.parser`:
+  - Fix to avoid schema download if not configured #2530.
 
 #### Experts
+- `intelmq.bots.experts.securitytxt`:
+  - Added new bot (PR#2538 by Frank Westers and Sebastian Wagner)
+- `intelmq.bots.experts.misp`: Use `PyMISP` class instead of deprecated `ExpandedPyMISP` (PR#2532 by Radek Vyhnal)
 
 #### Outputs
+- `intelmq.bots.outputs.cif3.output`:
+  - The requirement can only be installed on Python version < 3.12.
+  - Add a check on the Python version and exit if incompatible.
+  - Add a deprecation warning (PR#2544 by Sebastian Wagner)
+- `intelmq.bots.outputs.sql.output`:
+  - Treat an empty string `fields` parameter as unset parameter, fixing a crash in default configuration (PR#2548 by Sebastian Wagner, fixes #2548).
 
 ### Documentation
+- `docs/admin/installation/linux-packages`: Add `[signed-by=]` options, add wget command as alternative to curl (PR#2547 by Sebastian Wagner).
+- Add documentation on the Redis pipeline (databases, configuration), fix generic pipeline documentation and add missing information on parameters, add unlinked intelmqctl docs to the index and TOC (PR#2560 by Sebastian Wagner).
+- Remove empty page tutorials/intelmq-manager (PR#2562 by Sebastian Wagner).
 
 ### Packaging
 
 ### Tests
+- Install build dependencies for `pymssql` on Python 3.8 as there are no wheels available for this Python version (PR#2542 by Sebastian Wagner).
+- Install `psql` explicitly for workflow support on other platforms such as act (PR#2542 by Sebastian Wagner).
+- Create intelmq user & group if running privileged to allow dropping privileges (PR#2542 by Sebastian Wagner).
+- `intelmq.tests.lib.test_pipeline.TestAmqp.test_acknowledge`: Also skip on Python 3.11 and 3.12 besides on 3.8 when running on CI (PR#2542 by Sebastian Wagner).
+- Full pytest workflow: Version-independent install of postgres client, for Ubuntu 24.04 (default on GitHub now) test environment compatibility (PR#2557 by Sebastian Wagner).
+- Debian package build workflow: Use artifact upload v4 instead of v3 (PR#2565 by Sebastian Wagner).
 
 ### Tools
 
@@ -340,7 +372,7 @@ This is short list of the most important known issues. The full list can be retr
 - Added an ExpertBot class - it should be used by all expert bots as a parent class
 - Introduced a module for IntelMQ related datatypes `intelmq.lib.datatypes` which for now only contains an Enum listing the four bot types
 - Added a `bottype` attribute to CollectorBot, ParserBot, ExpertBot, OutputBot
-- Introduces a module for IntelMQ processmanagers. The processmanagers were up until now part of the intelmqct script.
+- Introduces a module for IntelMQ processmanagers. The processmanagers were up until now part of the intelmqctl script.
   They now reside in `intelmq.lib.processmanager` which also contains an interface definition the processmanager implementations must adhere to.
   Both the processmanagers and the `intelmqctl` script were cleaned up a bit.
   The `LogLevel` and `ReturnType` Enums were added to `intelmq.lib.datatypes`.

Makefile

@@ -19,3 +19,6 @@ codespell:
 
 test:
 	pytest --no-cov -v intelmq/tests/ && echo "Success!"
+
+codestyle:
+	pycodestyle intelmq/{bots,lib,bin}

NEWS.md

@@ -14,6 +14,12 @@ Please refer to the change log for a full list of changes.
 ---------------------------------
 
 ### Requirements
+Python 3.8 or newer is required.
+
+## Bots
+#### CIF 3 API
+The CIF 3 API Output bot is not compatible with Python version greater or equal to 3.12 and will be removed in the future due to lack of maintenance.
+See https://lists.cert.at/pipermail/intelmq-users/2024-December/000474.html for more information.
 
 ### Tools
 
@@ -36,7 +42,7 @@ No changes are required by administrators.
 ----------------------------------
 
 ### Documentation
-The documentation is now available at [docs.intelmq.org](https://docs.intelmq.org/). Documentation has been updated and restructured into User, Administrator and Developer Guide. It provides modern look with various quality of life improvements. Big thanks to to @gethvi. 
+The documentation is now available at [docs.intelmq.org](https://docs.intelmq.org/). Documentation has been updated and restructured into User, Administrator and Developer Guide. It provides modern look with various quality of life improvements. Big thanks to to @gethvi.
 We now have a slick, modern mkdocs based documentation. Please do check it out!
 
 
@@ -55,7 +61,7 @@ Shadowserver adds new scans on a nearly weekly basis. IntelMQ's release cycle an
 We therefore (thanks to @eslif2) move the shadowserver reports collector and parser to a new, dynamic system. It can:
 
  - fetch the shadowserver schema from shadowserver (https://interchange.shadowserver.org/intelmq/v1/schema)
- - dynamically collect new reports (see also https://docs.intelmq.org/latest/user/bots/?h=shadow#shadowserver-reports-api) 
+ - dynamically collect new reports (see also https://docs.intelmq.org/latest/user/bots/?h=shadow#shadowserver-reports-api)
  - parse the new reports
 
 **Note well**: if your IntelMQ system runs in an airgapped environment or if it may only reach out to specific IPs/sites, you should read the notes here:
@@ -86,7 +92,7 @@ Quite a few changes (thanks to Kamil, @gethvi) on AMQP
 ### General changes and bug fixes
 
 Digital Trust Center fixed a bug where the config was loaded twice in intelmqctl which created quite some speedups. Thanks!
-This speeds up IntelMQ API calls. 
+This speeds up IntelMQ API calls.
 
 ### Data Format
 

contrib/eventdb/README.md

@@ -10,8 +10,8 @@ EventDB Utilities
 - Apply Malware Name Mapping: Applies the malware name mapping to the eventdb. Source and destination columns can be given, also a local file. If no local file is present, the mapping can be downloaded on demand.
   It queries the database for all distinct malware names with the taxonomy "malicious-code" and sets another column to the malware family name.
 - Apply Domain Suffix: Writes the public domain suffix to the `source.domain_suffix` / `destination.domain_suffix` columns, extracted from `source.fqdn` / `destination.fqdn`.
-- PostgreSQL trigger keeping track of the oldest inserted/updated "time.source" data. This can be useful to (re-)generate statistics or aggregation data.
-- SQL queries to set up a separate `raws` table, described in https://docs.intelmq.org/latest/admin/database/postgresql/#separating-raw-values-in-postgresql-using-view-and-trigger
+- `trigger_oldest_time.source.sql`: PostgreSQL trigger keeping track of the oldest inserted/updated "time.source" data. This can be useful to (re-)generate statistics or aggregation data.
+- `to_json.py`: Export EventDB data to JSON, to use it in IntelMQ again.
 
 Usage
 -----
@@ -22,6 +22,16 @@ See `--help` for more information:
 ```
 apply_mapping_eventdb.py -h
 apply_domain_suffix.py -h
+to_json.py -h
 ```
 
 The SQL script can be executed in the database directly.
+
+### `to_json.py`
+
+
+- Get an event by ID: `~intevation/to_json.py --id $id`
+	- You can give multiple IDs
+- Pretty printed: Add `--pretty`
+- Inject the data into an IntelMQ bot (dry run):
+	- `intelmqctl run $botid process --dry-run --show-sent --msg '$jsondata'`

contrib/eventdb/to_json.py

@@ -0,0 +1,52 @@
+#!/usr/bin/python3
+
+# SPDX-FileCopyrightText: 2025 Institute for Common Good Technology
+#
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+from argparse import ArgumentParser
+from datetime import datetime
+import json
+from sys import exit, stderr
+from pprint import pprint
+
+from psycopg2 import connect
+from psycopg2.extras import RealDictCursor
+
+parser = ArgumentParser(
+                    prog='EventDB to JSON',
+                    description='Extract data from the IntelMQ EventDB')
+parser.add_argument('-v', '--verbose', action='store_true')
+parser.add_argument('-i', '--id', help='Get events by ID')
+parser.add_argument('-p', '--pretty', action='store_true', help='Pretty print JSON output')
+parser.add_argument('--dsn', help='A complete libpg conninfo string. If not given, it will be loaded from /etc/intelmq/eventdb-serve.conf')
+args = parser.parse_args()
+
+if args.dsn:
+    conninfo = args.dsn
+else:
+    try:
+        with open('/etc/intelmq/eventdb-serve.conf') as fody_config:
+            conninfo = json.load(fody_config)['libpg conninfo']
+    except FileNotFoundError as exc:
+        print(f'Could not load database configuration. {exc}', file=stderr)
+        exit(2)
+
+if args.verbose:
+    print(f'Using DSN {conninfo!r}.')
+db = connect(dsn=conninfo)
+cur = db.cursor(cursor_factory=RealDictCursor)
+cur.execute ('SELECT * FROM events WHERE id = %s', (args.id, ))
+
+for row in cur.fetchall():
+    del row['id']
+    for key in list(row.keys()):
+        if isinstance(row[key], datetime):
+            # data from the database has TZ information already included
+            row[key] = row[key].isoformat()
+        elif row[key] is None:
+            del row[key]
+    if args.pretty:
+        print(json.dumps(row, indent=2))
+    else:
+        print(json.dumps(row))