🐛 fix broken AWS Endpoints (#11902)

* 🐛 fix for AWS Parser endpoints

* ruff

* resolve conflicts

* added upgrade note

---------

Co-authored-by: manuelsommer <47991713+manuel-sommer@users.noreply.github.com>

Author: quirinziessler

docs/content/en/connecting_your_tools/parsers/file/awssecurityhub.md

@@ -15,5 +15,8 @@ AWS Security Hub integrates with multiple AWS Tools. Thus, you can retrieve find
 - AWS Security Hub GuardDuty: <br>`aws securityhub get-findings --filters ProductName="[{Value=GuardDuty,Comparison=EQUALS}]" | jq "." > output.json`
 - AWS Security Hub Inspector: <br>`aws securityhub get-findings --filters ProductName="[{Value=Inspector,Comparison=EQUALS}]" | jq "." > output.json`
 
+### Important note
+AWS Security Hub Parser does import the affected service ARNs as hosts to DefectDojo. However, as ARNs contain invalid digits for hosts, the ARN is changed slightly. ":", " " & "/" are replaced by "_".
+
 ### Sample Scan Data
 Sample scan data for testing purposes can be found [here](https://github.com/DefectDojo/django-DefectDojo/tree/master/unittests/scans/awssecurityhub).

docs/content/en/open_source/upgrading/2.44.1.md

@@ -0,0 +1,17 @@
+---
+title: 'Upgrading to DefectDojo Version 2.44.1'
+toc_hide: true
+weight: -20250203
+description: No special instructions.
+---
+
+### AWS Parser Endpoint Migrations
+
+The structure of AWS Endpoints changed slightly. In the past ARNs of Services were imported with invalid digits leading to broken Endpoints.
+The Endpoints thus changed slightly. ":", " " & "/" are replaced by "_".
+
+An Migration is added as part of this release to avoid duplicates at the next import. It can take some time, depending on the amount of AWS Endpoints in the DB.
+
+---
+
+Check the [Release Notes](https://github.com/DefectDojo/django-DefectDojo/releases/tag/2.44.1) for the contents of the release.

dojo/db_migrations/0223_aws_sechub_update_endpoints.py

@@ -0,0 +1,21 @@
+# Generated by Django 5.0.8 on 2024-09-12 18:22
+
+from django.db import migrations
+
+
+def aws_sechub_update_endpoints(apps, schema_editor):
+    endpoint_model = apps.get_model('dojo', 'Endpoint')
+    endpoints = endpoint_model.objects.filter(finding__test__test_type__name__in=["AWS Security Hub Scan", "AWS Inspector2 Scan"])
+    for endpoint in endpoints:
+        endpoint.host = endpoint.host.replace(':', '_').replace("/", "_").replace(" ", "_")
+        endpoint.save()
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('dojo', '0222_clean_old_sessions'),
+    ]
+
+    operations = [
+        migrations.RunPython(aws_sechub_update_endpoints, reverse_code=migrations.RunPython.noop),
+    ]

dojo/tools/aws_inspector2/parser.py

@@ -188,7 +188,7 @@ def process_endpoints(self, finding: Finding, raw_finding: dict) -> Finding:
             resource_type = resource_info.get("type", None)
             resource_id = resource_info.get("id", "N/A")
             resource_details = resource_info.get("details", {})
-            endpoint_host = f"{resource_type} - {resource_id}"
+            endpoint_host = f"{resource_type}_{resource_id}".replace(":", "_").replace("/", "_")
             if resource_type == "AWS_EC2_INSTANCE":
                 aws_account = raw_finding.get("awsAccountId")
                 resource_region = resource_info.get("region", "N/A")

dojo/tools/awssecurityhub/guardduty.py

@@ -43,7 +43,7 @@ def get_item(self, finding: dict, test):
         for resource in finding.get("Resources", []):
             component_name = resource.get("Type")
             if component_name in {"AwsEcrContainerImage", "AwsEc2Instance"}:
-                hosts.append(Endpoint(host=f"{component_name} {resource.get('Id')}"))
+                hosts.append(Endpoint(host=f"{component_name}_{resource.get('Id')}".replace(":", "_").replace("/", "_")))
             if component_name == "AwsEcrContainerImage":
                 details = resource.get("Details", {}).get("AwsEcrContainerImage")
                 arn = resource.get("Id")

dojo/tools/awssecurityhub/inspector.py

@@ -54,7 +54,7 @@ def get_item(self, finding: dict, test):
         hosts = []
         for resource in finding.get("Resources", []):
             component_name = resource.get("Type")
-            hosts.append(Endpoint(host=f"{component_name} {resource.get('Id')}"))
+            hosts.append(Endpoint(host=f"{component_name}_{resource.get('Id')}".replace(":", "_").replace("/", "_")))
             if component_name == "AwsEcrContainerImage":
                 details = resource.get("Details", {}).get("AwsEcrContainerImage")
                 arn = resource.get("Id")

unittests/tools/test_awssecurityhub_parser.py

@@ -68,7 +68,7 @@ def test_inspector_ec2(self):
             self.assertEqual("CVE-2022-3643", finding.unsaved_vulnerability_ids[0])
             self.assertEqual("- Update kernel-4.14.301\n\t- yum update kernel\n", finding.mitigation)
             endpoint = finding.unsaved_endpoints[0]
-            self.assertEqual("AwsEc2Instance arn:aws:ec2:us-east-1:XXXXXXXXXXXX:i-11111111111111111", endpoint.host)
+            self.assertEqual("AwsEc2Instance_arn_aws_ec2_us-east-1_XXXXXXXXXXXX_i-11111111111111111", endpoint.host)
 
     def test_inspector_ec2_with_no_vulnerabilities(self):
         with open(sample_path("inspector_ec2_cve_no_vulnerabilities.json"), encoding="utf-8") as test_file:
@@ -91,7 +91,7 @@ def test_inspector_ec2_ghsa(self):
             self.assertSetEqual({"CVE-2023-34256", "GHSA-p98r-538v-jgw5"}, set(finding.unsaved_vulnerability_ids))
             self.assertEqual("https://github.com/bottlerocket-os/bottlerocket/security/advisories/GHSA-p98r-538v-jgw5", finding.references)
             endpoint = finding.unsaved_endpoints[0]
-            self.assertEqual("AwsEc2Instance arn:aws:ec2:eu-central-1:012345678912:instance/i-07c11cc535d830123", endpoint.host)
+            self.assertEqual("AwsEc2Instance_arn_aws_ec2_eu-central-1_012345678912_instance_i-07c11cc535d830123", endpoint.host)
 
     def test_inspector_ecr(self):
         with open(sample_path("inspector_ecr.json"), encoding="utf-8") as test_file:
@@ -108,7 +108,7 @@ def test_inspector_ecr(self):
             self.assertIn("Repository: repo-os", finding.impact)
             self.assertEqual(0.0014, finding.epss_score)
             endpoint = finding.unsaved_endpoints[0]
-            self.assertEqual("AwsEcrContainerImage arn:aws:ecr:eu-central-1:123456789012:repository/repo-os/sha256:af965ef68c78374a5f987fce98c0ddfa45801df2395bf012c50b863e65978d74", endpoint.host)
+            self.assertEqual("AwsEcrContainerImage_arn_aws_ecr_eu-central-1_123456789012_repository_repo-os_sha256_af965ef68c78374a5f987fce98c0ddfa45801df2395bf012c50b863e65978d74", endpoint.host)
 
     def test_guardduty(self):
         with open(sample_path("guardduty.json"), encoding="utf-8") as test_file:
@@ -124,7 +124,7 @@ def test_guardduty(self):
             self.assertEqual("User AssumedRole : 123123123 is anomalously invoking APIs commonly used in Discovery tactics. - Resource: 123123123", finding.title)
             self.assertEqual("TTPs/Discovery/IAMUser-AnomalousBehavior\n[https://docs.aws.amazon.com/guardduty/latest/ug/guardduty_finding-types-active.html](https://docs.aws.amazon.com/guardduty/latest/ug/guardduty_finding-types-active.html)", finding.mitigation)
             endpoint = findings[0].unsaved_endpoints[0]
-            self.assertEqual("AwsEc2Instance arn:aws:ec2:us-east-1:123456789012:instance/i-1234567890", endpoint.host)
+            self.assertEqual("AwsEc2Instance_arn_aws_ec2_us-east-1_123456789012_instance_i-1234567890", endpoint.host)
             self.assertEqual("This is a GuardDuty Finding\nAPIs commonly used in Discovery tactics were invoked by user AssumedRole : 123123123, under anomalous circumstances. Such activity is not typically seen from this user.\n**AWS Finding ARN:** arn:aws:guardduty:us-east-1:123456789012:detector/123456789/finding/2123123123123\n**SourceURL:** [https://us-east-1.console.aws.amazon.com/guardduty/home?region=us-east-1#/findings?macros=current&fId=2123123123123](https://us-east-1.console.aws.amazon.com/guardduty/home?region=us-east-1#/findings?macros=current&fId=2123123123123)\n**AwsAccountId:** 123456789012\n**Region:** us-east-1\n**Generator ID:** arn:aws:guardduty:us-east-1:123456789012:detector/123456789\n", finding.description)
 
     def test_issue_10956(self):