From 24fa9b9ba738949e4930796d97ea641317e0f55c Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Wed, 16 Apr 2025 22:56:12 +0200 Subject: [PATCH] fake: add new mode random_single_value --- CHANGELOG.md | 4 +- docs/user/bots.md | 32 ++++++++--- intelmq/bots/experts/fake/expert.py | 53 ++++++++++++++----- intelmq/tests/bots/experts/fake/severity.json | 8 +++ .../bots/experts/fake/severity.json.license | 2 + .../tests/bots/experts/fake/test_expert.py | 7 +++ 6 files changed, 87 insertions(+), 19 deletions(-) create mode 100644 intelmq/tests/bots/experts/fake/severity.json create mode 100644 intelmq/tests/bots/experts/fake/severity.json.license diff --git a/CHANGELOG.md b/CHANGELOG.md index 97a7bdb68..171a71fdb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,7 +33,9 @@ Please refer to the [NEWS](NEWS.md) for a list of changes which have an affect o - Print URLs to stdout only in verbose mode (PR#2591 by Sebastian Wagner). - Check for database file existence and writability (fixes #2566). - Use database path matching to installation type (PR#2606 by Sebastian Wagner). -- `intelmq.bots.experts.fake.expert`: Use database path matching to installation type (PR#2606 by Sebastian Wagner). +- `intelmq.bots.experts.fake.expert`: + - Use database path matching to installation type (PR#2606 by Sebastian Wagner). + - Add new mode `random_single_value` (PR#2601 by Sebastian Wagner). #### Outputs diff --git a/docs/user/bots.md b/docs/user/bots.md index 90e30f242..eb1d2c6eb 100644 --- a/docs/user/bots.md +++ b/docs/user/bots.md @@ -2686,11 +2686,12 @@ is `$portal_url + '/api/1.0/ripe/contact?cidr=%s'`. ### Fake
-Adds fake data to events. Currently supports setting the IP address and network. +Adds fake data to events. It currently supports two operation methods: -For each incoming event, the bots chooses one random IP network range from the configured data file. -It set's the first IP address of the range as `source.ip` and the network itself as `source.network`. -To adapt the `source.asn` field accordingly, use the [ASN Lookup Expert](#asn-lookup). +* Setting the IP address and network +* For any Event field, set the value to a random item of a user-defined list (mode `random_single_value`) + +For a detailed description of the modes, see below. **Module:** `intelmq.bots.experts.fake.expert` @@ -2698,13 +2699,21 @@ To adapt the `source.asn` field accordingly, use the [ASN Lookup Expert](#asn-lo **`database`** -(required, string) Path to a JSON file in the following format: +(required, string) Path to a JSON file in the following format (example): ``` { "ip_network": [ "10.0.0.0/8", + "192.168.0.0/16", ... - ] + ], + "event_fields": { + "extra.severity": { + "mode": "random_single_value", + "values": ["critical", "high", "medium", "low", "info", "undefined"] + }, + ... + } } ``` @@ -2712,9 +2721,20 @@ To adapt the `source.asn` field accordingly, use the [ASN Lookup Expert](#asn-lo (optional, boolean) Whether to overwrite existing fields. Defaults to false. +### Modes + +#### IP Network +For each incoming event, the bots chooses one random IP network range (IPv4 or IPv6) from the configured data file. +It set's the first IP address of the range as `source.ip` and the network itself as `source.network`. +To adapt the `source.asn` field accordingly, use the [ASN Lookup Expert](#asn-lookup). + For data consistency `source.network` will only be set if `source.ip` was set or overridden. If overwrite is false, `source.ip` was did not exist before but `source.network` existed before, `source.network` will still be overridden. +#### Event fields +##### Mode `random_single_value` +For any possible event field, the bot chooses a random value of the values in the `values` property. + --- ### Field Reducer
diff --git a/intelmq/bots/experts/fake/expert.py b/intelmq/bots/experts/fake/expert.py index 3dc8b7d76..0bc25982e 100644 --- a/intelmq/bots/experts/fake/expert.py +++ b/intelmq/bots/experts/fake/expert.py @@ -8,6 +8,7 @@ from intelmq.lib.bot import ExpertBot from intelmq import VAR_STATE_PATH +from intelmq.lib.message import Event class FakeExpertBot(ExpertBot): @@ -18,20 +19,29 @@ class FakeExpertBot(ExpertBot): def init(self): with open(self.database) as database: - self.networks = json_load(database)['ip_network'] + database = json_load(database) + self.ip_networks = database.get('ip_network', []) + self.event_fields = database.get('event_fields', {}) def process(self): event = self.receive_message() - network = choice(self.networks) + if self.ip_networks: + network = choice(self.ip_networks) - updated = False - try: - updated = event.add('source.ip', ip_network(network)[1], overwrite=self.overwrite) - except IndexError: - updated = event.add('source.ip', ip_network(network)[0], overwrite=self.overwrite) - # For consistency, only set the network if the source.ip was set or overwritten, but then always overwrite it - if updated: - event.add('source.network', network, overwrite=True) + updated = False + try: + updated = event.add('source.ip', ip_network(network)[1], overwrite=self.overwrite) + except IndexError: + updated = event.add('source.ip', ip_network(network)[0], overwrite=self.overwrite) + # For consistency, only set the network if the source.ip was set or overwritten, but then always overwrite it + if updated: + event.add('source.network', network, overwrite=True) + + for fieldname, field in self.event_fields.items(): + if field['mode'] == 'random_single_value': + event.add(fieldname, choice(field['values']), overwrite=self.overwrite) + else: + raise ValueError(f"Mode {field['mode']} not supported in field {fieldname}.") self.send_message(event) self.acknowledge_message() @@ -39,9 +49,28 @@ def process(self): def check(parameters: dict): try: with open(parameters['database']) as database: - json_load(database)['ip_network'] + database = json_load(database) except Exception as exc: - return [['error', exc]] + return [['error', f"Could not load database: {exc}"]] + errors = [] + if not isinstance(database.get('ip_network', []), list): + errors.append(['error', 'ip_network is not of type list']) + if not isinstance(database.get('event_fields', {}), dict): + errors.append(['error', 'event_fields is not of type dict']) + else: + test_event = Event() + for fieldname, field in database.get('event_fields', {}).items(): + fieldname_check = test_event._Message__is_valid_key(fieldname) + if not fieldname_check[0]: + errors.append(['error', f"Field name {fieldname} is not valid: {fieldname_check[1]}."]) + mode = field.get('mode') + if mode not in ('random_single_value', ): + errors.append(['error', f"Mode {mode} not supported in field {fieldname}."]) + if 'values' not in field: + errors.append(['error', f"No values defined in field {fieldname}."]) + elif not isinstance(field['values'], list): + errors.append(['error', f"Values is not a list in field {fieldname}."]) + return errors if errors else None BOT = FakeExpertBot diff --git a/intelmq/tests/bots/experts/fake/severity.json b/intelmq/tests/bots/experts/fake/severity.json new file mode 100644 index 000000000..92cb9acb0 --- /dev/null +++ b/intelmq/tests/bots/experts/fake/severity.json @@ -0,0 +1,8 @@ +{ + "event_fields": { + "extra.severity": { + "mode": "random_single_value", + "values": ["critical", "high", "medium", "low", "info", "undefined"] + } + } +} diff --git a/intelmq/tests/bots/experts/fake/severity.json.license b/intelmq/tests/bots/experts/fake/severity.json.license new file mode 100644 index 000000000..ae4f29ebb --- /dev/null +++ b/intelmq/tests/bots/experts/fake/severity.json.license @@ -0,0 +1,2 @@ +SPDX-FileCopyrightText: 2025 Institute for Common Good Technology, Sebastian Wagner +SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/experts/fake/test_expert.py b/intelmq/tests/bots/experts/fake/test_expert.py index ccc678b81..c1d49ec70 100644 --- a/intelmq/tests/bots/experts/fake/test_expert.py +++ b/intelmq/tests/bots/experts/fake/test_expert.py @@ -12,6 +12,7 @@ from intelmq.bots.experts.fake.expert import FakeExpertBot FAKE_DB = pkg_resources.resource_filename('intelmq', 'tests/bots/experts/fake/data.json') +SEVERITY_DB = pkg_resources.resource_filename('intelmq', 'tests/bots/experts/fake/severity.json') EXAMPLE_INPUT = {"__type": "Event", "source.ip": "93.184.216.34", # example.com } @@ -45,6 +46,12 @@ def test_network_exists(self): self.assertIn(ip_address(msg['source.ip']), ip_network("10.0.0.0/8")) self.assertEqual(msg['source.network'], "10.0.0.0/8") + def test_random_single_value(self): + self.input_message = {"__type": "Event"} + self.run_bot(parameters={'database': SEVERITY_DB}) + msg = json_loads(self.get_output_queue()[0]) + self.assertIn(msg['extra.severity'], ["critical", "high", "medium", "low", "info", "undefined"]) + if __name__ == '__main__': # pragma: no cover unittest.main()