Merge pull request #2500 from certtools/sieve-extra-dict

sieve expert bugfix: for string matching, convert value to string first

Author: sebix

CHANGELOG.md

@@ -32,6 +32,8 @@
 - `intelmq.bots.parsers.dataplane.parser`: Use `  |  ` as field delimiter, fix parsing of AS names including `|` (PR#2488 by DigitalTrustCenter).
 
 #### Experts
+- `intelmq.bots.experts.sieve.expert`:
+  - For `:contains`, `=~` and `!~`, convert the value to string before matching avoiding an exception. If the value is a dict, convert the value to JSON (PR#2500 by Sebastian Wagner).
 
 #### Outputs
 - `intelmq.bots.outputs.misp.output_feed`: Handle failures if saved current event wasn't saved or is incorrect (PR by Kamil Mankowski).

docs/user/bots.md

@@ -3586,10 +3586,12 @@ if :exists source.fqdn { ... }
 if feed.name != 'acme-security' || feed.accuracy == 100 || extra.false_positive == false { ... }
 ```
 
-- `:contains` matches on substrings.
+- `:contains` matches on substrings ([`str.find`](https://docs.python.org/3/library/stdtypes.html#str.find)).
 
 - `=~` matches strings based on the given regular expression. `!~` is the inverse regular expression match.
 
+- For `:contains`, `=~` and `!~`, the value is converted to string before matching. If the value is a dict, convert the value to JSON.
+
 - Numerical comparisons are evaluated with `<`, `<=`, `>`, `>=`.
 
 - `<<` matches if an IP address is contained in the specified network range:

intelmq/bots/experts/sieve/expert.py

@@ -16,6 +16,7 @@
 import operator
 
 from datetime import datetime, timedelta, timezone
+from json import dumps
 from typing import Callable, Dict, Optional, Union
 from enum import Enum, auto
 
@@ -272,7 +273,14 @@ def process_single_string_match(self, key, op, value, event) -> bool:
         if key not in event:
             return op in {'!=', '!~'}
 
-        return self._string_op_map[op](event[key], value.value)
+        lhs = event[key]
+        if not isinstance(lhs, str) and op not in ('==', '!='):
+            if isinstance(lhs, dict):
+                lhs = dumps(lhs)
+            else:
+                lhs = str(lhs)
+
+        return self._string_op_map[op](lhs, value.value)
 
     def process_multi_string_match(self, key, op, value, event) -> bool:
         if key not in event:

intelmq/tests/bots/experts/sieve/test_expert.py

@@ -1738,6 +1738,14 @@ def test_empty_list(self):
         self.run_bot()
         self.assertMessageEqual(0, expected)
 
+    def test_extra_dict(self):
+        self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), 'test_sieve_files/test_extra_dict.sieve')
+        event = EXAMPLE_INPUT.copy()
+        event['extra.some_dict'] = {'key': []}
+        self.input_message = event
+        self.run_bot()
+        self.assertOutputQueueLen(0)
+
 
 if __name__ == '__main__':  # pragma: no cover
     unittest.main()

intelmq/tests/bots/experts/sieve/test_sieve_files/test_extra_dict.sieve

@@ -0,0 +1,11 @@
+// '{"extra.some_dict": { "key": [] }}'
+
+if :notexists extra.some_dict {
+ drop
+}
+if extra.some_dict !~ '"key": ' {
+ drop
+}
+if extra.some_dict =~ '"key": \[\]' {
+ drop
+}

intelmq/tests/bots/experts/sieve/test_sieve_files/test_extra_dict.sieve.license

@@ -0,0 +1,2 @@
+SPDX-FileCopyrightText: 2024 Bundesamt für Sicherheit in der Informationstechnik (BSI), Software engineering by Intevation GmbH
+SPDX-License-Identifier: AGPL-3.0-or-later