Severity field in IDF

CHANGELOG.md

@@ -21,6 +21,9 @@ Please refer to the [NEWS](NEWS.md) for a list of changes which have an affect o
 
 ### Data Format
 
+- added `severity` field to help with triaging received events (PR#2575 by Kamil Mańkowski).
+  To allow saving the field in PostgreSQL database in existing installations, the following schema update is necessary: `ALTER TABLE events ADD severity varchar(10);`.
+
 ### Bots
 #### Collectors
 

NEWS.md

@@ -18,6 +18,11 @@ Please refer to the change log for a full list of changes.
 ### Tools
 
 ### Data Format
+To save new fields from IntelMQ Data Format in existing PostgreSQL instances, the following schema
+update is necessary:
+```sql
+ALTER TABLE events ADD severity varchar(10);
+```
 
 ### Configuration
 

intelmq/etc/harmonization.conf

@@ -233,6 +233,12 @@
             "description": "Some source may report URLs related to a an image generated of a resource without any metadata. Or an URL pointing to resource, which has been rendered into a webshot, e.g. a PNG image and the relevant metadata related to its retrieval/generation.",
             "type": "URL"
         },
+        "severity": {
+            "description": "Severity of the event, based on the information from the source, and eventually modified by IntelMQ during processing. Meaning of the levels may differ based on the event source.",
+            "length": 10,
+            "regex": "^(critical|high|medium|low|info|undefined)$",
+            "type": "LowercaseString"
+        },
         "source.abuse_contact": {
             "description": "Abuse contact for source address. A comma separated list.",
             "type": "LowercaseString"

intelmq/lib/upgrades.py

@@ -41,7 +41,8 @@
            'v320_update_turris_greylist_url',
            'v322_url_replacement',
            'v322_removed_feeds_and_bots',
-           'v340_deprecations'
+           'v340_deprecations',
+           'v341_new_fields'
            ]
 
 
@@ -974,6 +975,29 @@ def v340_deprecations(configuration, harmonization, dry_run, **kwargs):
     return message or changed, configuration, harmonization
 
 
+def v341_new_fields(configuration, harmonization, dry_run, **kwargs):
+    """
+    Add new fields to IntelMQ Data Format
+    """
+    changed = None
+    if "event" not in harmonization:
+        return changed, configuration, harmonization
+
+    builtin_harmonisation = load_configuration(
+        resource_filename("intelmq", "etc/harmonization.conf")
+    )
+    for field in [
+        "severity",
+    ]:
+        if field not in harmonization["event"]:
+            if field not in builtin_harmonisation["event"]:
+                # ensure forward-compatibility if we ever remove something from harmonisation
+                continue
+            harmonization["event"][field] = builtin_harmonisation["event"][field]
+            changed = True
+    return changed, configuration, harmonization
+
+
 UPGRADES = OrderedDict([
     ((1, 0, 0, 'dev7'), (v100_dev7_modify_syntax,)),
     ((1, 1, 0), (v110_shadowserver_feednames, v110_deprecations)),
@@ -1004,7 +1028,7 @@ def v340_deprecations(configuration, harmonization, dry_run, **kwargs):
     ((3, 3, 0), ()),
     ((3, 3, 1), ()),
     ((3, 4, 0), (v340_deprecations, )),
-    ((3, 4, 1), ()),
+    ((3, 4, 1), (v341_new_fields, )),
 ])
 
 ALWAYS = (harmonization,)

intelmq/tests/bin/initdb.sql

@@ -52,6 +52,7 @@ CREATE TABLE events (
     "raw" text,
     "rtir_id" integer,
     "screenshot_url" text,
+    "severity" varchar(10),
     "source.abuse_contact" text,
     "source.account" text,
     "source.allocated" timestamp with time zone,

intelmq/tests/lib/test_upgrades.py

@@ -856,6 +856,13 @@ def test_v340_twitter_collector(self):
         self.assertIn('twitter-collector', result[0])
         self.assertEqual(V340_TWITTER_COLLECTOR_IN, result[1])
 
+    def test_v341_new_fields(self):
+        """ Test adding new harmonisation fields """
+        result = upgrades.v341_new_fields({}, {"event": {"old-field": "must stay"}}, False)
+        self.assertTrue(result[0])
+        self.assertIn("old-field", result[2]["event"])
+        self.assertIn("severity", result[2]["event"])
+
 
 for name in upgrades.__all__:
     setattr(TestUpgradeLib, 'test_function_%s' % name,