Fix d2d pipeline bugs from ecosytem configurations (#1716)

* Fix d2d pipeline bugs from ecosytem configurations

Reference: #1715
Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

* Fix license exception compliance parsing bug

Reference: #1719
Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

* Add tests for leading d2d configurations

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

---------

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

Author: AyanSinhaMahapatra

scanpipe/models.py

@@ -2587,7 +2587,7 @@ def compute_compliance_alert(self):
             return ""
 
         licensing = get_licensing()
-        parsed_symbols = licensing.parse(license_expression, simple=True).symbols
+        parsed_symbols = licensing.license_symbols(license_expression, simple=True)
 
         alerts = [
             self.get_alert_for_symbol(license_policy_index, symbol)

scanpipe/pipelines/deploy_to_develop.py

@@ -158,7 +158,7 @@ def match_archives_to_purldb(self):
 
         d2d.match_purldb_resources(
             project=self.project,
-            extensions=self.matchable_package_extensions,
+            extensions=self.ecosystem_config.matchable_package_extensions,
             matcher_func=d2d.match_purldb_package,
             logger=self.log,
         )
@@ -249,7 +249,7 @@ def match_resources_to_purldb(self):
 
         d2d.match_purldb_resources(
             project=self.project,
-            extensions=self.matchable_resource_extensions,
+            extensions=self.ecosystem_config.matchable_resource_extensions,
             matcher_func=d2d.match_purldb_resource,
             logger=self.log,
         )

scanpipe/pipes/d2d_config.py

@@ -184,6 +184,5 @@ def add_ecosystem_config(pipeline_ecosystem_config, ecosystem_config):
             if not pipeline_config_value:
                 new_config_value = config_value
             else:
-                new_config_value = pipeline_config_value.extend(config_value)
-
+                new_config_value = config_value + pipeline_config_value
             setattr(pipeline_ecosystem_config, config_name, new_config_value)

scanpipe/tests/__init__.py

@@ -333,6 +333,14 @@ def make_mock_response(url, content=b"\x00", status_code=200, headers=None):
         "label": "Prohibited License",
         "compliance_alert": "error",
     },
+    {
+        "license_key": "gpl-2.0-plus",
+        "compliance_alert": "warning",
+    },
+    {
+        "license_key": "font-exception-gpl",
+        "compliance_alert": "warning",
+    },
     {
         "license_key": "OFL-1.1",
         "compliance_alert": "warning",
@@ -368,6 +376,14 @@ def make_mock_response(url, content=b"\x00", status_code=200, headers=None):
         "label": "Prohibited License",
         "compliance_alert": "error",
     },
+    "gpl-2.0-plus": {
+        "license_key": "gpl-2.0-plus",
+        "compliance_alert": "warning",
+    },
+    "font-exception-gpl": {
+        "license_key": "font-exception-gpl",
+        "compliance_alert": "warning",
+    },
     "OFL-1.1": {
         "license_key": "OFL-1.1",
         "compliance_alert": "warning",

scanpipe/tests/data/d2d/config/ecosystem_config.json

@@ -0,0 +1,10 @@
+{
+    "ecosystem_option": "Default",
+    "matchable_package_extensions": [".jar", ".war", ".gem", ".zip", ".tar.gz", ".tar.xz"],
+    "matchable_resource_extensions": [".map", ".js", ".mjs", ".ts", ".d.ts", ".jsx", ".tsx", ".css", ".scss", ".less", ".sass", ".soy",".class", ".rb"],
+    "doc_extensions": [".pdf", ".doc", ".docx", ".ppt", ".pptx", ".tex", ".odt", ".odp"],
+    "deployed_resource_path_exclusions": ["*checksums.yaml.gz*", "*metadata.gz*"],
+    "devel_resource_path_exclusions": ["*/tests/*"],
+    "standard_symbols_to_exclude": [],
+    "source_symbol_extensions": []
+}

scanpipe/tests/data/policies/policies.yml

@@ -12,6 +12,13 @@ license_policies:
     label: Prohibited License
     compliance_alert: error
 
+  - license_key: gpl-2.0-plus
+    compliance_alert: warning
+
+  # AboutCode license execption key
+  - license_key: font-exception-gpl
+    compliance_alert: warning
+
   # SPDX license keys
   - license_key: OFL-1.1
     compliance_alert: warning

scanpipe/tests/pipes/test_d2d.py

@@ -21,9 +21,11 @@
 # Visit https://github.com/nexB/scancode.io for support and download.
 
 import io
+import json
 import sys
 import tempfile
 import uuid
+from dataclasses import asdict
 from pathlib import Path
 from unittest import mock
 from unittest import skipIf
@@ -1864,3 +1866,21 @@ def test_scanpipe_pipes_d2d_map_javascript_strings(self):
                 map_type="javascript_strings",
             ).count(),
         )
+
+    def test_scanpipe_d2d_load_ecosystem_config(self):
+        pipeline_name = "map_deploy_to_develop"
+        selected_groups = ["Ruby", "Java", "JavaScript"]
+
+        run = self.project1.add_pipeline(
+            pipeline_name=pipeline_name, selected_groups=selected_groups
+        )
+        pipeline = run.make_pipeline_instance()
+        d2d_config.load_ecosystem_config(pipeline=pipeline, options=selected_groups)
+
+        expected_ecosystem_config = (
+            self.data / "d2d" / "config" / "ecosystem_config.json"
+        )
+        with open(expected_ecosystem_config) as f:
+            expected_extra_data = json.load(f)
+
+        self.assertEqual(expected_extra_data, asdict(pipeline.ecosystem_config))

scanpipe/tests/test_models.py

@@ -1637,6 +1637,14 @@ def test_scanpipe_codebase_resource_model_compliance_alert_update_fields(self):
         resource.refresh_from_db()
         self.assertEqual("ok", resource.compliance_alert)
 
+    @patch.object(scanpipe_app, "policies", new=global_policies)
+    def test_scanpipe_can_compute_compliance_alert_for_license_exceptions(self):
+        scanpipe_app.license_policies_index = license_policies_index
+        resource = CodebaseResource.objects.create(project=self.project1, path="file")
+        license_expression = "gpl-2.0-plus WITH font-exception-gpl"
+        resource.update(detected_license_expression=license_expression)
+        self.assertEqual("warning", resource.compute_compliance_alert())
+
     def test_scanpipe_scan_fields_model_mixin_methods(self):
         expected = [
             "detected_license_expression",