Add support extra_data value from the JSON input in load_inventory #926 (#1507)

Signed-off-by: tdruez <tdruez@nexb.com>

Author: tdruez

CHANGELOG.rst

@@ -10,6 +10,12 @@ v34.9.4 (unreleased)
   Any paginated view can now be navigated using the left/right keyboard keys.
   https://github.com/aboutcode-org/scancode.io/issues/1200
 
+- Add support for importing the ``extra_data`` value from the JSON input with the
+  ``load_inventory`` pipeline.
+  When multiple JSON files are provided as inputs, the ``extra`` is prefixed with
+  the input filename.
+  https://github.com/aboutcode-org/scancode.io/issues/926
+
 v34.9.3 (2024-12-31)
 --------------------
 

scanpipe/pipelines/load_inventory.py

@@ -54,17 +54,28 @@ def build_inventory_from_scans(self):
         Process JSON scan results files to populate packages, dependencies, and
         resources.
         """
+        self.input_paths = list(self.input_paths)
+        is_single_input = len(self.input_paths) == 1
+
         for input_path in self.input_paths:
+            extra_data_prefix = None if is_single_input else input_path.name
+
             if input_path.suffix.endswith(".xlsx"):
-                input.load_inventory_from_xlsx(self.project, input_path)
+                input.load_inventory_from_xlsx(
+                    self.project, input_path, extra_data_prefix
+                )
                 continue
 
             scan_data = json.loads(input_path.read_text())
             tool_name = input.get_tool_name_from_scan_headers(scan_data)
 
             if tool_name == "scancode-toolkit":
                 input.load_inventory_from_toolkit_scan(self.project, input_path)
+
             elif tool_name == "scanpipe":
-                input.load_inventory_from_scanpipe(self.project, scan_data)
+                input.load_inventory_from_scanpipe(
+                    self.project, scan_data, extra_data_prefix
+                )
+
             else:
                 raise Exception(f"Input not supported: {str(input_path)} ")

scanpipe/pipes/input.py

@@ -82,6 +82,14 @@ def get_tool_name_from_scan_headers(scan_data):
         return tool_name
 
 
+def get_extra_data_from_scan_headers(scan_data):
+    """Return the ``extra_data`` of the first header in the provided ``scan_data``."""
+    if headers := scan_data.get("headers", []):
+        first_header = headers[0]
+        if extra_data := first_header.get("extra_data"):
+            return extra_data
+
+
 def is_archive(location):
     """Return True if the file at ``location`` is an archive."""
     return get_type(location).is_archive
@@ -100,10 +108,13 @@ def load_inventory_from_toolkit_scan(project, input_location):
     )
 
 
-def load_inventory_from_scanpipe(project, scan_data):
+def load_inventory_from_scanpipe(project, scan_data, extra_data_prefix=None):
     """
     Create packages, dependencies, resources, and relations loaded from a ScanCode.io
     JSON output provided as ``scan_data``.
+
+    An ``extra_data_prefix`` can be provided in case multiple input files are loaded
+    into the same project. The prefix is usually the filename of the input.
     """
     for package_data in scan_data.get("packages", []):
         pipes.update_or_create_package(project, package_data)
@@ -117,6 +128,11 @@ def load_inventory_from_scanpipe(project, scan_data):
     for relation_data in scan_data.get("relations", []):
         pipes.get_or_create_relation(project, relation_data)
 
+    if extra_data := get_extra_data_from_scan_headers(scan_data):
+        if extra_data_prefix:
+            extra_data = {extra_data_prefix: extra_data}
+        project.update_extra_data(extra_data)
+
 
 model_to_object_maker_func = {
     DiscoveredPackage: pipes.update_or_create_package,
@@ -186,10 +202,13 @@ def clean_xlsx_data_to_model_data(model_class, xlsx_data):
     return cleaned_data
 
 
-def load_inventory_from_xlsx(project, input_location):
+def load_inventory_from_xlsx(project, input_location, extra_data_prefix=None):
     """
     Create packages, dependencies, resources, and relations loaded from XLSX file
     located at ``input_location``.
+
+    An ``extra_data_prefix`` can be provided in case multiple input files are loaded
+    into the same project. The prefix is usually the filename of the input.
     """
     workbook = openpyxl.load_workbook(input_location, read_only=True, data_only=True)
 
@@ -206,4 +225,7 @@ def load_inventory_from_xlsx(project, input_location):
 
     if "LAYERS" in workbook:
         layers_data = get_worksheet_data(worksheet=workbook["LAYERS"])
-        project.update_extra_data({"layers": layers_data})
+        extra_data = {"layers": layers_data}
+        if extra_data_prefix:
+            extra_data = {extra_data_prefix: extra_data}
+        project.update_extra_data(extra_data)

scanpipe/tests/data/outputs/docker_ghcr.io_kyverno_extra_data_expected.json

@@ -0,0 +1,16 @@
+{
+  "layers": [
+    {
+      "layer_tag": "img-12ebda-layer-01-1a058d",
+      "created_by": "/bin/sh -c #(nop) ADD file:762c899ec0505d1a32930ee804c5b008825f41611161be104076cba33b7e5b2b in / ",
+      "layer_id": "1a058d5342cc722ad5439cacae4b2b4eedde51d8fe8800fcf28444302355c16d",
+      "image_id": "12ebda3111cec73a788b0e802a00de04ebf5e9765043925dd396c2d03a7c1e66",
+      "created": "2021-11-12T17:19:44.795237917Z",
+      "size": "5886464",
+      "author": null,
+      "comment": null,
+      "archive_location": "ghcr_io_kyverno_sbom.tar-extract/1a058d5342cc722ad5439cacae4b2b4eedde51d8fe8800fcf28444302355c16d.tar",
+      "xlsx_errors": null
+    }
+  ]
+}

scanpipe/tests/pipes/test_input.py

@@ -109,6 +109,25 @@ def test_scanpipe_pipes_scancode_load_inventory_from_scanpipe_with_relations(sel
         self.assertEqual(57, project.codebaseresources.count())
         self.assertEqual(18, project.codebaserelations.count())
 
+    def test_scanpipe_pipes_scancode_load_inventory_extra_data(self):
+        project = Project.objects.create(name="1")
+        input_location = self.data / "asgiref" / "asgiref-3.3.0_scanpipe_output.json"
+        scan_data = json.loads(input_location.read_text())
+        extra_data = {"key": "value"}
+        scan_data["headers"][0]["extra_data"] = extra_data
+
+        input.load_inventory_from_scanpipe(project, scan_data)
+        project.refresh_from_db()
+        self.assertEqual(extra_data, project.extra_data)
+
+        project.extra_data = {}
+        project.save()
+        input.load_inventory_from_scanpipe(
+            project, scan_data, extra_data_prefix="file.ext"
+        )
+        project.refresh_from_db()
+        self.assertEqual({"file.ext": extra_data}, project.extra_data)
+
     def test_scanpipe_pipes_input_load_inventory_from_xlsx(self):
         project1 = Project.objects.create(name="Analysis")
         input_location = self.data / "outputs" / "asgiref-3.6.0-output.xlsx"
@@ -129,6 +148,14 @@ def test_scanpipe_pipes_input_load_inventory_from_xlsx_layers_sheet(self):
         expected = json.loads(expected_location.read_text())
         self.assertEqual(expected, project1.extra_data)
 
+        project1.extra_data = {}
+        project1.save()
+        input.load_inventory_from_xlsx(
+            project1, input_location, extra_data_prefix="file.ext"
+        )
+        project1.refresh_from_db()
+        self.assertEqual({"file.ext": expected}, project1.extra_data)
+
     def test_scanpipe_pipes_input_load_inventory_from_project_xlsx_output(self):
         fixtures = self.data / "asgiref" / "asgiref-3.3.0_fixtures.json"
         call_command("loaddata", fixtures, **{"verbosity": 0})