From 4b26dede961ea8a7a89fcc1d7041b560c1dfb03a Mon Sep 17 00:00:00 2001 From: Kamil Mankowski Date: Wed, 29 Jan 2025 17:11:24 +0100 Subject: [PATCH 1/4] Prepare Tuency bot for new version In currently developed Tuency version, the quering capabilities are being extended to support identifier and feed.code, and the information about consituency is returned. In addition, the bot gots more customisation capabilities. --- CHANGELOG.md | 3 + docs/user/bots.md | 45 ++- intelmq/bots/experts/tuency/expert.py | 61 +++- .../tests/bots/experts/tuency/test_expert.py | 296 +++++++++++++++--- 4 files changed, 345 insertions(+), 60 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b93eddc4e..dfa8b74b62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,9 @@ - `intelmq.bots.experts.securitytxt`: - Added new bot (PR#2538 by Frank Westers and Sebastian Wagner) - `intelmq.bots.experts.misp`: Use `PyMISP` class instead of deprecated `ExpandedPyMISP` (PR#2532 by Radek Vyhnal) +- `intelmq.bots.experts.tuency`: (PR# by Kamil Mańkowski) + - Support for querying using `feed.code` and `classification.identifier` (requires Tuency 2.6+), + - Support for customizing fields and the TTL value for suspended sending. #### Outputs - `intelmq.bots.outputs.cif3.output`: diff --git a/docs/user/bots.md b/docs/user/bots.md index 4f856e7923..1dc403bc37 100644 --- a/docs/user/bots.md +++ b/docs/user/bots.md @@ -4073,8 +4073,9 @@ addresses and delivery settings for IP objects (addresses, netblocks), Autonomou - `classification.taxonomy` - `classification.type` +- `classification.identifier` - `feed.provider` -- `feed.name` +- `feed.name` or `feed.code` These fields therefore need to exist, otherwise the message is skipped. @@ -4083,17 +4084,20 @@ The API parameter "feed_status" is currently set to "production" constantly, unt The API answer is processed as following. For the notification interval: - If *suppress* is true, then `extra.notify` is set to false. + If explicitly configured, a special TTL value can be set. - Otherwise: -- If the interval is *immediate*, then `extra.ttl` is set to 0. -- Otherwise the interval is converted into seconds and saved in - `extra.ttl`. + - If the interval is *immediate*, then `extra.ttl` is set to 0. + - Otherwise the interval is converted into seconds and saved in + `extra.ttl`. For the contact lookup: For both fields *ip* and *domain*, the *destinations* objects are iterated and its *email* fields concatenated to a comma-separated list in `source.abuse_contact`. -The IntelMQ fields used by this bot may change in the next IntelMQ release, as soon as better suited fields are -available. +For constituency: if provided from Tuency, the list of relvant consitituencies will +be saved comma-separated in the `extra.constituency` field. + +The IntelMQ fields used by this bot may be customized by the parameters. **Module:** `intelmq.bots.experts.tuency.expert` @@ -4111,6 +4115,35 @@ available. (optional, boolean) Whether the existing data in `source.abuse_contact` should be overwritten. Defaults to true. +**`notify_field`** + +(optional, string) Name of the field to save information if the message should not be send +(suspention in Tuency). By default `extra.notify` + +**`ttl_field`** + +(optional, string) Name of the field to save the TTL value (in seconds). By default `extra.ttl`. + +**`constituency_field`** + +(optional, string) Name of the gield to save information about the consitutuency. By default +`extra.constituency`. If set to empty value, this information won't be saved. + +**`ttl_on_suspended`** + +(optional, integer) Custom value to set as TTL when the sending is suspended. By default +not set - no value will be set at all. + +**`query_classification_identifier`** + +(optional, boolean) Whether to add `classification.identifier` to the query. Requires +at least Tuency 2.6. By default `False`. + +**`query_feed_code`** + +(optional, boolean) Whether to query using `feed.code` instead of `feed.name`. Requires +at least Tuency 2.6. By default `False`. + --- ### Truncate By Delimiter
diff --git a/intelmq/bots/experts/tuency/expert.py b/intelmq/bots/experts/tuency/expert.py index 15521cb90b..0dd5ee6411 100644 --- a/intelmq/bots/experts/tuency/expert.py +++ b/intelmq/bots/experts/tuency/expert.py @@ -1,6 +1,6 @@ """ -© 2021 Sebastian Wagner - +SPDX-FileCopyrightText: 2021 Sebastian Wagner +SPDX-FileCopyrightText: 2025 CERT.at GmbH SPDX-License-Identifier: AGPL-3.0-or-later https://gitlab.com/intevation/tuency/tuency/-/blob/master/backend/docs/IntelMQ-API.md @@ -26,6 +26,17 @@ class TuencyExpertBot(ExpertBot): authentication_token: str overwrite: bool = True + notify_field = "extra.notify" + ttl_field = "extra.ttl" + constituency_field = "extra.constituency" + + # Allows setting custom TTL for suspended sending + ttl_on_suspended = None + + # Non-default values require Tuency v2.6+ + query_classification_identifier = False + query_feed_code = False + def init(self): self.set_request_parameters() self.session = create_request_session(self) @@ -44,11 +55,17 @@ def process(self): "classification_taxonomy": event["classification.taxonomy"], "classification_type": event["classification.type"], "feed_provider": event["feed.provider"], - "feed_name": event["feed.name"], "feed_status": "production", } + if self.query_feed_code: + params["feed_code"] = event["feed.code"] + else: + params["feed_name"] = event["feed.name"] + + if self.query_classification_identifier: + params["classification_identifier"] = event["classification.identifier"] except KeyError as exc: - self.logger.debug('Skipping event because of missing field: %s.', exc) + self.logger.debug("Skipping event because of missing field: %s.", exc) self.send_message(event) self.acknowledge_message() return @@ -62,24 +79,42 @@ def process(self): pass response = self.session.get(self.url, params=params).json() - self.logger.debug('Received response %r.', response) + self.logger.debug("Received response %r.", response) if response.get("suppress", False): - event["extra.notify"] = False + event.add(self.notify_field, False) + if self.ttl_on_suspended: + event.add(self.ttl_field, self.ttl_on_suspended) else: - if 'interval' not in response: + if "interval" not in response: # empty response self.send_message(event) self.acknowledge_message() return - elif response['interval']['unit'] == 'immediate': - event["extra.ttl"] = 0 + elif response["interval"]["unit"] == "immediate": + event.add(self.ttl_field, 0) else: - event["extra.ttl"] = parse_relative(f"{response['interval']['length']} {response['interval']['unit']}") * 60 + event.add( + self.ttl_field, + ( + parse_relative( + f"{response['interval']['length']} {response['interval']['unit']}" + ) + * 60 + ), + ) contacts = [] - for destination in response.get('ip', {'destinations': []})['destinations'] + response.get('domain', {'destinations': []})['destinations']: - contacts.extend(contact['email'] for contact in destination["contacts"]) - event.add('source.abuse_contact', ','.join(contacts), overwrite=self.overwrite) + for destination in ( + response.get("ip", {"destinations": []})["destinations"] + + response.get("domain", {"destinations": []})["destinations"] + ): + contacts.extend(contact["email"] for contact in destination["contacts"]) + event.add("source.abuse_contact", ",".join(contacts), overwrite=self.overwrite) + + if self.constituency_field and ( + constituencies := response.get("constituencies", []) + ): + event.add(self.constituency_field, ",".join(constituencies)) self.send_message(event) self.acknowledge_message() diff --git a/intelmq/tests/bots/experts/tuency/test_expert.py b/intelmq/tests/bots/experts/tuency/test_expert.py index dad1bcb10d..4b8d8d0737 100644 --- a/intelmq/tests/bots/experts/tuency/test_expert.py +++ b/intelmq/tests/bots/experts/tuency/test_expert.py @@ -6,6 +6,7 @@ This unittest can test the bot against a read tuency instance as well as using requests mock. The latter is the default while the first is only in use if a tunency instance URL and authentication token is given a environment variable. """ + import os import unittest @@ -15,57 +16,160 @@ import requests_mock -INPUT = {'__type': 'Event', - 'classification.taxonomy': 'availability', - 'classification.type': 'system-compromise', - 'feed.provider': 'Some Provider', - 'feed.name': 'FTP', - 'source.ip': '123.123.123.23', - 'source.fqdn': 'www.example.at' - } +INPUT = { + "__type": "Event", + "classification.taxonomy": "availability", + "classification.type": "system-compromise", + "classification.identifier": "hacked-server", + "feed.provider": "Some Provider", + "feed.name": "FTP", + "feed.code": "ftp", + "source.ip": "123.123.123.23", + "source.fqdn": "www.example.at", +} INPUT_IP = INPUT.copy() -del INPUT_IP['source.fqdn'] -INPUT_IP['source.abuse_contact'] = 'abuse@example.com' +del INPUT_IP["source.fqdn"] +INPUT_IP["source.abuse_contact"] = "abuse@example.com" INPUT_DOMAIN = INPUT.copy() -del INPUT_DOMAIN['source.ip'] +del INPUT_DOMAIN["source.ip"] OUTPUT = INPUT.copy() OUTPUT_IP = INPUT_IP.copy() -OUTPUT_IP['extra.notify'] = False -OUTPUT_IP['source.abuse_contact'] = 'test@ntvtn.de' +OUTPUT_IP["extra.notify"] = False +OUTPUT_IP["source.abuse_contact"] = "test@ntvtn.de" +OUTPUT_IP["extra.constituency"] = "Tenant1,Tenant2" OUTPUT_IP_NO_OVERWRITE = OUTPUT_IP.copy() -OUTPUT_IP_NO_OVERWRITE['source.abuse_contact'] = 'abuse@example.com' +OUTPUT_IP_NO_OVERWRITE["source.abuse_contact"] = "abuse@example.com" OUTPUT_DOMAIN = INPUT_DOMAIN.copy() -OUTPUT_DOMAIN['extra.ttl'] = 24*60*60 # 1 day -OUTPUT_DOMAIN['source.abuse_contact'] = 'abuse+www@example.at' +OUTPUT_DOMAIN["extra.ttl"] = 24 * 60 * 60 # 1 day +OUTPUT_DOMAIN["source.abuse_contact"] = "abuse+www@example.at" OUTPUT_BOTH = OUTPUT.copy() -OUTPUT_BOTH['extra.ttl'] = 24*60*60 # 1 day -OUTPUT_BOTH['source.abuse_contact'] = 'test@ntvtn.de,abuse+www@example.at' -EMPTY = {'__type': 'Event', 'comment': 'foobar'} +OUTPUT_BOTH["extra.ttl"] = 24 * 60 * 60 # 1 day +OUTPUT_BOTH["source.abuse_contact"] = "test@ntvtn.de,abuse+www@example.at" +EMPTY = {"__type": "Event", "comment": "foobar"} UNKNOWN_IP = INPUT_IP.copy() -UNKNOWN_IP['source.ip'] = '10.0.0.1' +UNKNOWN_IP["source.ip"] = "10.0.0.1" -PREFIX = 'http://localhost/intelmq/lookup?classification_taxonomy=availability&classification_type=system-compromise&feed_provider=Some+Provider&feed_name=FTP&feed_status=production' +PREFIX = ( + "http://localhost/intelmq/lookup?classification_taxonomy=availability" + "&classification_type=system-compromise&feed_provider=Some+Provider" + "&feed_status=production" +) def prepare_mocker(mocker): # IP address - mocker.get(f'{PREFIX}&ip=123.123.123.23', - request_headers={'Authorization': 'Bearer Lorem ipsum'}, - json={"ip":{"destinations":[{"source":"portal","name":"Thurner","contacts":[{"email":"test@ntvtn.de"}]}]},"suppress":True,"interval":{"unit":"days","length":1}}) + mocker.get( + f"{PREFIX}&ip=123.123.123.23&feed_name=FTP", + request_headers={"Authorization": "Bearer Lorem ipsum"}, + json={ + "ip": { + "destinations": [ + { + "source": "portal", + "name": "Thurner", + "contacts": [{"email": "test@ntvtn.de"}], + } + ] + }, + "suppress": True, + "interval": {"unit": "days", "length": 1}, + "constituencies": ["Tenant1", "Tenant2"], + }, + ) + # Domain: - mocker.get(f'{PREFIX}&domain=www.example.at', - request_headers={'Authorization': 'Bearer Lorem ipsum'}, - json={"domain":{"destinations":[{"source":"portal","name":"EineOrganisation","contacts":[{"email":"abuse+www@example.at"}]}]},"suppress":False,"interval":{"unit":"days","length":1}}) + mocker.get( + f"{PREFIX}&domain=www.example.at&feed_name=FTP", + request_headers={"Authorization": "Bearer Lorem ipsum"}, + json={ + "domain": { + "destinations": [ + { + "source": "portal", + "name": "EineOrganisation", + "contacts": [{"email": "abuse+www@example.at"}], + } + ] + }, + "suppress": False, + "interval": {"unit": "days", "length": 1}, + }, + ) # Both - mocker.get(f'{PREFIX}&ip=123.123.123.23&domain=www.example.at', - request_headers={'Authorization': 'Bearer Lorem ipsum'}, - json={"ip":{"destinations":[{"source":"portal","name":"Thurner","contacts":[{"email":"test@ntvtn.de"}]}]},"domain":{"destinations":[{"source":"portal","name":"EineOrganisation","contacts":[{"email":"abuse+www@example.at"}]}]},"suppress":False,"interval":{"unit":"day","length":1}}) + mocker.get( + f"{PREFIX}&ip=123.123.123.23&domain=www.example.at&feed_name=FTP", + request_headers={"Authorization": "Bearer Lorem ipsum"}, + json={ + "ip": { + "destinations": [ + { + "source": "portal", + "name": "Thurner", + "contacts": [{"email": "test@ntvtn.de"}], + } + ] + }, + "domain": { + "destinations": [ + { + "source": "portal", + "name": "EineOrganisation", + "contacts": [{"email": "abuse+www@example.at"}], + } + ] + }, + "suppress": False, + "interval": {"unit": "day", "length": 1}, + }, + ) # Unknown IP address - mocker.get(f'{PREFIX}&ip=10.0.0.1', - request_headers={'Authorization': 'Bearer Lorem ipsum'}, - json={'ip': {'destinations': [], 'netobject': None}}) + mocker.get( + f"{PREFIX}&ip=10.0.0.1&feed_name=FTP", + request_headers={"Authorization": "Bearer Lorem ipsum"}, + json={"ip": {"destinations": [], "netobject": None}}, + ) + + # feed_code + mocker.get( + f"{PREFIX}&ip=123.123.123.23&feed_code=ftp", + request_headers={"Authorization": "Bearer Lorem ipsum"}, + json={ + "ip": { + "destinations": [ + { + "source": "portal", + "name": "Thurner", + "contacts": [{"email": "test+code@ntvtn.de"}], + } + ] + }, + "suppress": False, + "interval": {"unit": "days", "length": 1}, + "constituencies": ["Tenant1", "Tenant2"], + }, + ) + + # classification identifier + mocker.get( + f"{PREFIX}&ip=123.123.123.23&feed_name=FTP&classification_identifier=hacked-server", + request_headers={"Authorization": "Bearer Lorem ipsum"}, + json={ + "ip": { + "destinations": [ + { + "source": "portal", + "name": "Thurner", + "contacts": [{"email": "test+identifier@ntvtn.de"}], + } + ] + }, + "suppress": True, + "interval": {"unit": "days", "length": 1}, + "constituencies": ["Tenant1", "Tenant2"], + }, + ) @requests_mock.Mocker() @@ -73,16 +177,20 @@ class TestTuencyExpertBot(BotTestCase, unittest.TestCase): @classmethod def set_bot(cls): cls.bot_reference = TuencyExpertBot - if not os.environ.get("INTELMQ_TEST_TUNECY_URL") or not os.environ.get("INTELMQ_TEST_TUNECY_TOKEN"): + if not os.environ.get("INTELMQ_TEST_TUNECY_URL") or not os.environ.get( + "INTELMQ_TEST_TUNECY_TOKEN" + ): cls.mock = True - cls.sysconfig = {"url": 'http://localhost/', - "authentication_token": 'Lorem ipsum', - } + cls.sysconfig = { + "url": "http://localhost/", + "authentication_token": "Lorem ipsum", + } else: cls.mock = False - cls.sysconfig = {"url": os.environ["INTELMQ_TEST_TUNECY_URL"], - "authentication_token": os.environ["INTELMQ_TEST_TUNECY_TOKEN"], - } + cls.sysconfig = { + "url": os.environ["INTELMQ_TEST_TUNECY_URL"], + "authentication_token": os.environ["INTELMQ_TEST_TUNECY_TOKEN"], + } cls.default_input_message = INPUT def test_both(self, mocker): @@ -114,7 +222,7 @@ def test_ip_no_overwrite(self, mocker): else: mocker.real_http = True self.input_message = INPUT_IP - self.run_bot(parameters={'overwrite': False}) + self.run_bot(parameters={"overwrite": False}) self.assertMessageEqual(0, OUTPUT_IP_NO_OVERWRITE) def test_domain(self, mocker): @@ -126,6 +234,112 @@ def test_domain(self, mocker): self.run_bot() self.assertMessageEqual(0, OUTPUT_DOMAIN) + def test_feed_code(self, mocker): + """Using feed.code to identify feeds""" + if self.mock: + prepare_mocker(mocker) + else: + mocker.real_http = True + + self.input_message = INPUT_IP + self.run_bot(parameters={"query_feed_code": True}) + expected = { + **OUTPUT_IP, + "source.abuse_contact": "test+code@ntvtn.de", + "extra.ttl": 86400, + "extra.notify": None, + } + del expected["extra.notify"] + self.assertMessageEqual( + 0, + expected, + ) + + def test_classification_identifier(self, mocker): + """Using classification.identifier to filter events""" + if self.mock: + prepare_mocker(mocker) + else: + mocker.real_http = True + + self.input_message = INPUT_IP + self.run_bot(parameters={"query_classification_identifier": True}) + self.assertMessageEqual( + 0, + { + **OUTPUT_IP, + "source.abuse_contact": "test+identifier@ntvtn.de", + }, + ) + + def test_custom_fields(self, mocker): + """Allow customize fields that bot sets""" + if self.mock: + prepare_mocker(mocker) + else: + mocker.real_http = True + + self.input_message = INPUT_IP + self.run_bot( + parameters={ + "notify_field": "extra.my_notify", + "constituency_field": "extra.my_constituency", + # Response for feed_code is not suspended - allows testing TTL + # "query_feed_code": True, + } + ) + + output = OUTPUT_IP.copy() + output["extra.my_notify"] = output["extra.notify"] + del output["extra.notify"] + output["extra.my_constituency"] = output["extra.constituency"] + del output["extra.constituency"] + self.assertMessageEqual(0, output) + + def test_custom_fields_ttl(self, mocker): + """Allow customize fields that bot sets""" + if self.mock: + prepare_mocker(mocker) + else: + mocker.real_http = True + + self.input_message = INPUT_IP + self.run_bot( + parameters={ + "ttl_field": "extra.my_ttl", + # Response for feed_code is not suspended - allows testing TTL + "query_feed_code": True, + } + ) + + output = OUTPUT_IP.copy() + del output["extra.notify"] + output["extra.my_ttl"] = 86400 + output["source.abuse_contact"] = "test+code@ntvtn.de" + self.assertMessageEqual(0, output) + + def test_ttl_on_suspended(self, mocker): + """Allow setting custom TTL when Tuency decides on suspending sending""" + if self.mock: + prepare_mocker(mocker) + else: + mocker.real_http = True + + self.input_message = INPUT_IP + self.run_bot( + parameters={ + "ttl_on_suspended": -1, + } + ) + + self.assertMessageEqual( + 0, + { + **OUTPUT_IP, + "extra.ttl": -1, + }, + ) + def test_empty(self, mocker): """ A message with neither an IP address nor a domain, should be ignored and just passed on. From cbb48b5d65e6561851038d7ce8d0308f0d578b17 Mon Sep 17 00:00:00 2001 From: Kamil Mankowski Date: Mon, 3 Mar 2025 14:44:58 +0100 Subject: [PATCH 2/4] Fix code style and spelling --- docs/user/bots.md | 2 +- intelmq/bots/experts/tuency/expert.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/user/bots.md b/docs/user/bots.md index 870605147d..0801291385 100644 --- a/docs/user/bots.md +++ b/docs/user/bots.md @@ -4066,7 +4066,7 @@ The IntelMQ fields used by this bot may be customized by the parameters. **`notify_field`** (optional, string) Name of the field to save information if the message should not be send -(suspention in Tuency). By default `extra.notify` +(suspension in Tuency). By default `extra.notify` **`ttl_field`** diff --git a/intelmq/bots/experts/tuency/expert.py b/intelmq/bots/experts/tuency/expert.py index 0dd5ee6411..2c478fa3ca 100644 --- a/intelmq/bots/experts/tuency/expert.py +++ b/intelmq/bots/experts/tuency/expert.py @@ -99,14 +99,13 @@ def process(self): ( parse_relative( f"{response['interval']['length']} {response['interval']['unit']}" - ) - * 60 + ) * 60 ), ) contacts = [] for destination in ( - response.get("ip", {"destinations": []})["destinations"] - + response.get("domain", {"destinations": []})["destinations"] + response.get("ip", {"destinations": []})["destinations"] + + response.get("domain", {"destinations": []})["destinations"] ): contacts.extend(contact["email"] for contact in destination["contacts"]) event.add("source.abuse_contact", ",".join(contacts), overwrite=self.overwrite) From 4dc6b2c598e0daae102fbc38e0b8ef8c65b8fdcf Mon Sep 17 00:00:00 2001 From: Kamil Mankowski Date: Mon, 10 Mar 2025 16:43:53 +0100 Subject: [PATCH 3/4] Patches from tests --- intelmq/bots/experts/tuency/expert.py | 47 ++++++++++++++++++--------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/intelmq/bots/experts/tuency/expert.py b/intelmq/bots/experts/tuency/expert.py index 2c478fa3ca..7f8443ccb6 100644 --- a/intelmq/bots/experts/tuency/expert.py +++ b/intelmq/bots/experts/tuency/expert.py @@ -30,6 +30,9 @@ class TuencyExpertBot(ExpertBot): ttl_field = "extra.ttl" constituency_field = "extra.constituency" + query_ip = True + query_domain = True + # Allows setting custom TTL for suspended sending ttl_on_suspended = None @@ -69,22 +72,30 @@ def process(self): self.send_message(event) self.acknowledge_message() return - try: - params["ip"] = event["source.ip"] - except KeyError: - pass - try: - params["domain"] = event["source.fqdn"] - except KeyError: - pass + + if self.query_ip: + try: + params["ip"] = event["source.ip"] + except KeyError: + pass + + if self.query_domain: + try: + params["domain"] = event["source.fqdn"] + except KeyError: + pass response = self.session.get(self.url, params=params).json() self.logger.debug("Received response %r.", response) if response.get("suppress", False): - event.add(self.notify_field, False) + event.add(self.notify_field, False, overwrite=self.overwrite) if self.ttl_on_suspended: - event.add(self.ttl_field, self.ttl_on_suspended) + event.add( + self.ttl_field, + self.ttl_on_suspended, + overwrite=self.overwrite, + ) else: if "interval" not in response: # empty response @@ -92,20 +103,22 @@ def process(self): self.acknowledge_message() return elif response["interval"]["unit"] == "immediate": - event.add(self.ttl_field, 0) + event.add(self.ttl_field, 0, overwrite=self.overwrite) else: event.add( self.ttl_field, ( parse_relative( f"{response['interval']['length']} {response['interval']['unit']}" - ) * 60 + ) + * 60 ), + overwrite=self.overwrite, ) contacts = [] for destination in ( - response.get("ip", {"destinations": []})["destinations"] + - response.get("domain", {"destinations": []})["destinations"] + response.get("ip", {"destinations": []})["destinations"] + + response.get("domain", {"destinations": []})["destinations"] ): contacts.extend(contact["email"] for contact in destination["contacts"]) event.add("source.abuse_contact", ",".join(contacts), overwrite=self.overwrite) @@ -113,7 +126,11 @@ def process(self): if self.constituency_field and ( constituencies := response.get("constituencies", []) ): - event.add(self.constituency_field, ",".join(constituencies)) + event.add( + self.constituency_field, + ",".join(constituencies), + overwrite=self.overwrite, + ) self.send_message(event) self.acknowledge_message() From ba0f134db593107020d07f596e576e0daa7a832f Mon Sep 17 00:00:00 2001 From: Kamil Mankowski Date: Wed, 12 Mar 2025 15:07:39 +0100 Subject: [PATCH 4/4] Handle RIPE answers, select querying fields, fixes --- CHANGELOG.md | 4 +- docs/user/bots.md | 8 ++ intelmq/bots/experts/tuency/expert.py | 78 +++++++++++++------ .../tests/bots/experts/tuency/test_expert.py | 46 +++++++++++ 4 files changed, 111 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7535386e9f..b9a97358bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,9 +42,11 @@ - `intelmq.bots.experts.securitytxt`: - Added new bot (PR#2538 by Frank Westers and Sebastian Wagner) - `intelmq.bots.experts.misp`: Use `PyMISP` class instead of deprecated `ExpandedPyMISP` (PR#2532 by Radek Vyhnal) -- `intelmq.bots.experts.tuency`: (PR# by Kamil Mańkowski) +- `intelmq.bots.experts.tuency`: (PR#2561 by Kamil Mańkowski) - Support for querying using `feed.code` and `classification.identifier` (requires Tuency 2.6+), - Support for customizing fields and the TTL value for suspended sending. + - Support selecting if IP and/or FQDN should be used for querying Tuency. + - Various fixes. #### Outputs - `intelmq.bots.outputs.cif3.output`: diff --git a/docs/user/bots.md b/docs/user/bots.md index 0801291385..0da61a2e00 100644 --- a/docs/user/bots.md +++ b/docs/user/bots.md @@ -4077,6 +4077,14 @@ The IntelMQ fields used by this bot may be customized by the parameters. (optional, string) Name of the gield to save information about the consitutuency. By default `extra.constituency`. If set to empty value, this information won't be saved. +**`query_ip`** + +(optional, boolean) Whether the bot should query Tuency based on `source.ip`. By default `true`. + +**`query_domain`** + +(optional, boolean) Whether the bot should query Tuency based on `source.fqdn`. By default `true`. + **`ttl_on_suspended`** (optional, integer) Custom value to set as TTL when the sending is suspended. By default diff --git a/intelmq/bots/experts/tuency/expert.py b/intelmq/bots/experts/tuency/expert.py index 7f8443ccb6..45f21f5391 100644 --- a/intelmq/bots/experts/tuency/expert.py +++ b/intelmq/bots/experts/tuency/expert.py @@ -46,12 +46,30 @@ def init(self): self.session.headers["Authorization"] = f"Bearer {self.authentication_token}" self.url = f"{self.url}intelmq/lookup" + if not self.query_ip and not self.query_domain: + self.logger.warning( + "Neither query_ip nor query_domain is set. " + "Bot won't do anything, please ensure it's intended." + ) + + @staticmethod + def check(parameters): + results = [] + if not parameters.get("query_ip", True) and not parameters.get( + "query_domain", True + ): + results.append( + [ + "warning", + "Neither query_ip nor query_domain is set. " + "Bot won't do anything, please ensure it's intended.", + ] + ) + + return results or None + def process(self): event = self.receive_message() - if not ("source.ip" in event or "source.fqdn" in event): - self.send_message(event) - self.acknowledge_message() - return try: params = { @@ -85,8 +103,20 @@ def process(self): except KeyError: pass - response = self.session.get(self.url, params=params).json() - self.logger.debug("Received response %r.", response) + if "ip" not in params and "domain" not in params: + # Nothing to query - skip + self.send_message(event) + self.acknowledge_message() + return + + response = self.session.get(self.url, params=params) + self.logger.debug("Received response %r.", response.text) + response = response.json() + + destinations = ( + response.get("ip", {"destinations": []})["destinations"] + + response.get("domain", {"destinations": []})["destinations"] + ) if response.get("suppress", False): event.add(self.notify_field, False, overwrite=self.overwrite) @@ -97,29 +127,29 @@ def process(self): overwrite=self.overwrite, ) else: - if "interval" not in response: + if not destinations: # empty response self.send_message(event) self.acknowledge_message() return - elif response["interval"]["unit"] == "immediate": - event.add(self.ttl_field, 0, overwrite=self.overwrite) - else: - event.add( - self.ttl_field, - ( - parse_relative( - f"{response['interval']['length']} {response['interval']['unit']}" - ) - * 60 - ), - overwrite=self.overwrite, - ) + + if "interval" in response: + if response["interval"]["unit"] == "immediate": + event.add(self.ttl_field, 0, overwrite=self.overwrite) + else: + event.add( + self.ttl_field, + ( + parse_relative( + f"{response['interval']['length']} {response['interval']['unit']}" + ) + * 60 + ), + overwrite=self.overwrite, + ) + contacts = [] - for destination in ( - response.get("ip", {"destinations": []})["destinations"] - + response.get("domain", {"destinations": []})["destinations"] - ): + for destination in destinations: contacts.extend(contact["email"] for contact in destination["contacts"]) event.add("source.abuse_contact", ",".join(contacts), overwrite=self.overwrite) diff --git a/intelmq/tests/bots/experts/tuency/test_expert.py b/intelmq/tests/bots/experts/tuency/test_expert.py index 4b8d8d0737..d71aeb00d9 100644 --- a/intelmq/tests/bots/experts/tuency/test_expert.py +++ b/intelmq/tests/bots/experts/tuency/test_expert.py @@ -171,6 +171,25 @@ def prepare_mocker(mocker): }, ) + # IP address directly from RIPE + mocker.get( + f"{PREFIX}&ip=123.123.123.123&feed_name=FTP", + request_headers={"Authorization": "Bearer Lorem ipsum"}, + json={ + "ip": { + "destinations": [ + { + "source": "ripe", + "name": "Thurner", + "contacts": [{"email": "test@ntvtn.de"}], + } + ] + }, + "suppress": True, + "constituencies": ["Tenant1", "Tenant2"], + }, + ) + @requests_mock.Mocker() class TestTuencyExpertBot(BotTestCase, unittest.TestCase): @@ -348,10 +367,18 @@ def test_empty(self, mocker): prepare_mocker(mocker) else: mocker.real_http = True + self.input_message = EMPTY self.run_bot() self.assertMessageEqual(0, EMPTY) + self.input_message = INPUT + self.run_bot( + parameters={"query_ip": False, "query_domain": False}, + allowed_warning_count=1, + ) + self.assertMessageEqual(0, INPUT) + def test_no_result(self, mocker): """ This IP address is not in the database @@ -363,3 +390,22 @@ def test_no_result(self, mocker): self.input_message = UNKNOWN_IP self.run_bot() self.assertMessageEqual(0, UNKNOWN_IP) + + def test_data_from_ripe(self, mocker): + """ + Data sourced from ripe don't get interval + """ + if self.mock: + prepare_mocker(mocker) + else: + mocker.real_http = True + + input_msq = INPUT_IP.copy() + input_msq["source.ip"] = "123.123.123.123" + + self.input_message = input_msq + self.run_bot() + + output_msg = OUTPUT_IP.copy() + output_msg["source.ip"] = "123.123.123.123" + self.assertMessageEqual(0, output_msg)