diff --git a/CHANGELOG.md b/CHANGELOG.md index fefa2f65f..95ddc970d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,9 @@ Please refer to the [NEWS](NEWS.md) for a list of changes which have an affect o ### Data Format +- added `severity` field to help with triaging received events (PR#2575 by Kamil MaƄkowski). + To allow saving the field in PostgreSQL database in existing installations, the following schema update is necessary: `ALTER TABLE events ADD severity varchar(10);`. + ### Bots #### Collectors diff --git a/NEWS.md b/NEWS.md index 12c225784..9b23a0533 100644 --- a/NEWS.md +++ b/NEWS.md @@ -18,6 +18,11 @@ Please refer to the change log for a full list of changes. ### Tools ### Data Format +To save new fields from IntelMQ Data Format in existing PostgreSQL instances, the following schema +update is necessary: +```sql +ALTER TABLE events ADD severity varchar(10); +``` ### Configuration diff --git a/intelmq/etc/harmonization.conf b/intelmq/etc/harmonization.conf index 027643ac9..871894161 100644 --- a/intelmq/etc/harmonization.conf +++ b/intelmq/etc/harmonization.conf @@ -233,6 +233,12 @@ "description": "Some source may report URLs related to a an image generated of a resource without any metadata. Or an URL pointing to resource, which has been rendered into a webshot, e.g. a PNG image and the relevant metadata related to its retrieval/generation.", "type": "URL" }, + "severity": { + "description": "Severity of the event, based on the information from the source, and eventually modified by IntelMQ during processing. Meaning of the levels may differ based on the event source.", + "length": 10, + "regex": "^(critical|high|medium|low|info|undefined)$", + "type": "LowercaseString" + }, "source.abuse_contact": { "description": "Abuse contact for source address. A comma separated list.", "type": "LowercaseString" diff --git a/intelmq/lib/upgrades.py b/intelmq/lib/upgrades.py index ee22b60a6..9ab53a1da 100644 --- a/intelmq/lib/upgrades.py +++ b/intelmq/lib/upgrades.py @@ -41,7 +41,8 @@ 'v320_update_turris_greylist_url', 'v322_url_replacement', 'v322_removed_feeds_and_bots', - 'v340_deprecations' + 'v340_deprecations', + 'v341_new_fields' ] @@ -974,6 +975,29 @@ def v340_deprecations(configuration, harmonization, dry_run, **kwargs): return message or changed, configuration, harmonization +def v341_new_fields(configuration, harmonization, dry_run, **kwargs): + """ + Add new fields to IntelMQ Data Format + """ + changed = None + if "event" not in harmonization: + return changed, configuration, harmonization + + builtin_harmonisation = load_configuration( + resource_filename("intelmq", "etc/harmonization.conf") + ) + for field in [ + "severity", + ]: + if field not in harmonization["event"]: + if field not in builtin_harmonisation["event"]: + # ensure forward-compatibility if we ever remove something from harmonisation + continue + harmonization["event"][field] = builtin_harmonisation["event"][field] + changed = True + return changed, configuration, harmonization + + UPGRADES = OrderedDict([ ((1, 0, 0, 'dev7'), (v100_dev7_modify_syntax,)), ((1, 1, 0), (v110_shadowserver_feednames, v110_deprecations)), @@ -1004,7 +1028,7 @@ def v340_deprecations(configuration, harmonization, dry_run, **kwargs): ((3, 3, 0), ()), ((3, 3, 1), ()), ((3, 4, 0), (v340_deprecations, )), - ((3, 4, 1), ()), + ((3, 4, 1), (v341_new_fields, )), ]) ALWAYS = (harmonization,) diff --git a/intelmq/tests/bin/initdb.sql b/intelmq/tests/bin/initdb.sql index 5a5f839f5..020af93d2 100644 --- a/intelmq/tests/bin/initdb.sql +++ b/intelmq/tests/bin/initdb.sql @@ -52,6 +52,7 @@ CREATE TABLE events ( "raw" text, "rtir_id" integer, "screenshot_url" text, + "severity" varchar(10), "source.abuse_contact" text, "source.account" text, "source.allocated" timestamp with time zone, diff --git a/intelmq/tests/lib/test_upgrades.py b/intelmq/tests/lib/test_upgrades.py index a30800b9c..d5f386fde 100644 --- a/intelmq/tests/lib/test_upgrades.py +++ b/intelmq/tests/lib/test_upgrades.py @@ -856,6 +856,13 @@ def test_v340_twitter_collector(self): self.assertIn('twitter-collector', result[0]) self.assertEqual(V340_TWITTER_COLLECTOR_IN, result[1]) + def test_v341_new_fields(self): + """ Test adding new harmonisation fields """ + result = upgrades.v341_new_fields({}, {"event": {"old-field": "must stay"}}, False) + self.assertTrue(result[0]) + self.assertIn("old-field", result[2]["event"]) + self.assertIn("severity", result[2]["event"]) + for name in upgrades.__all__: setattr(TestUpgradeLib, 'test_function_%s' % name,