Upgrade harmonisation

Author: kamil-certat

NEWS.md

@@ -18,6 +18,15 @@ Please refer to the change log for a full list of changes.
 ### Tools
 
 ### Data Format
+To save new fields from IntelMQ Data Format in existing PostgreSQL instances, the following schema
+update is necessary:
+```sql
+ALTER TABLE events ADD "product.full_name" text;
+ALTER TABLE events ADD "product.name" text;
+ALTER TABLE events ADD "product.vendor" text;
+ALTER TABLE events ADD "product.version" text;
+ALTER TABLE events ADD "product.vulnerabilities" text;
+```
 
 ### Configuration
 

intelmq/lib/upgrades.py

@@ -41,7 +41,8 @@
            'v320_update_turris_greylist_url',
            'v322_url_replacement',
            'v322_removed_feeds_and_bots',
-           'v340_deprecations'
+           'v340_deprecations',
+           'v341_new_fields'
            ]
 
 
@@ -974,6 +975,30 @@ def v340_deprecations(configuration, harmonization, dry_run, **kwargs):
     return message or changed, configuration, harmonization
 
 
+def v341_new_fields(configuration, harmonization, dry_run, **kwargs):
+    """
+    Add new fields to IntelMQ Data Format
+    """
+    changed = None
+    if "event" not in harmonization:
+        return changed, configuration, harmonization
+
+    builtin_harmonisation = load_configuration(
+        resource_filename("intelmq", "etc/harmonization.conf")
+    )
+    for field in [
+        "product.full_name",
+        "product.name",
+        "product.vendor",
+        "product.version",
+        "product.vulnerabilities",
+    ]:
+        if field not in harmonization["event"]:
+            harmonization["event"][field] = builtin_harmonisation["event"][field]
+            changed = True
+    return changed, configuration, harmonization
+
+
 UPGRADES = OrderedDict([
     ((1, 0, 0, 'dev7'), (v100_dev7_modify_syntax,)),
     ((1, 1, 0), (v110_shadowserver_feednames, v110_deprecations)),
@@ -1004,7 +1029,8 @@ def v340_deprecations(configuration, harmonization, dry_run, **kwargs):
     ((3, 3, 0), ()),
     ((3, 3, 1), ()),
     ((3, 4, 0), (v340_deprecations, )),
-    ((3, 4, 1), ()),
+    ((3, 4, 1), (v341_new_fields, )),
+
 ])
 
 ALWAYS = (harmonization,)

intelmq/tests/bin/initdb.sql

@@ -98,4 +98,4 @@ CREATE INDEX "idx_events_source.asn" ON events USING btree ("source.asn");
 CREATE INDEX "idx_events_source.ip" ON events USING btree ("source.ip");
 CREATE INDEX "idx_events_source.fqdn" ON events USING btree ("source.fqdn");
 CREATE INDEX "idx_events_time.observation" ON events USING btree ("time.observation");
-CREATE INDEX "idx_events_time.source" ON events USING btree ("time.source");
+CREATE INDEX "idx_events_time.source" ON events USING btree ("time.source");