Merge pull request #1170 from nexB/elf_d2d

Add D2D for ELFs and Go binaries #1113 #1114

Author: TG1999

scanpipe/filters.py

@@ -409,6 +409,9 @@ def filter(self, qs, value):
     ("js_path", "js path"),
     ("path", "path"),
     ("sha1", "sha1"),
+    ("dwarf_included_paths", "dwarf_included_paths"),
+    ("dwarf_compiled_paths", "dwarf_compiled_paths"),
+    ("go_file_paths", "go_file_paths"),
 )
 
 

scanpipe/models.py

@@ -1993,8 +1993,7 @@ def has_directory_content_fingerprint(self):
 
     def elfs(self):
         """
-        Resources that are ``files`` and their filetype starts with "ELF" and
-        contains any of these "executable", "relocatable", "shared object".
+        ELF executable and shared object Resources.
         Keep sync with the content type implementation at ``typecode.contenttype``.
         """
         return (
@@ -2009,6 +2008,28 @@ def elfs(self):
             )
         )
 
+    def win_exes(self):
+        """
+        Windows executable and DLL Resources.
+        Keep sync with the content type implementation at ``typecode.contenttype``.
+        """
+        return self.files().filter(
+            Q(file_type__icontains="for ms windows") | Q(file_type__istartswith="pe32")
+        )
+
+    def macho_binaries(self):
+        """
+        Mach-O binary Resources.
+        Keep sync with the content type implementation at ``typecode.contenttype``.
+        """
+        return self.files().filter(
+            models.Q(file_type__icontains="mach-o")
+            | models.Q(mime_type__icontains="application/x-mach-binary")
+        )
+
+    def executable_binaries(self):
+        return self.union(self.win_exes(), self.macho_binaries(), self.elfs())
+
 
 class ScanFieldsModelMixin(models.Model):
     """Fields returned by the ScanCode-toolkit scans."""

scanpipe/pipelines/deploy_to_develop.py

@@ -70,6 +70,8 @@ def steps(cls):
             cls.map_java_to_class,
             cls.map_jar_to_source,
             cls.map_javascript,
+            cls.map_elf,
+            cls.map_go,
             cls.match_directories_to_purldb,
             cls.match_resources_to_purldb,
             cls.map_javascript_post_purldb_match,
@@ -189,6 +191,16 @@ def map_javascript(self):
         """
         d2d.map_javascript(project=self.project, logger=self.log)
 
+    @group("Elf")
+    def map_elf(self):
+        """Map ELF binaries to their sources."""
+        d2d.map_elfs(project=self.project, logger=self.log)
+
+    @group("Go")
+    def map_go(self):
+        """Map Go binaries to their sources."""
+        d2d.map_go_paths(project=self.project, logger=self.log)
+
     def match_directories_to_purldb(self):
         """Match selected directories in PurlDB."""
         if not purldb.is_available():

scanpipe/pipes/d2d.py

@@ -38,7 +38,9 @@
 from django.template.defaultfilters import pluralize
 
 from commoncode.paths import common_prefix
+from elf_inspector.dwarf import get_dwarf_paths
 from extractcode import EXTRACT_SUFFIX
+from go_inspector.plugin import collect_and_parse_symbols
 from packagedcode.npm import NpmPackageJsonHandler
 from summarycode.classify import LEGAL_STARTS_ENDS
 
@@ -1662,3 +1664,224 @@ def _match_purldb_resources_post_process(
             package.add_resources(unmapped_resources)
 
     return interesting_codebase_resources.count()
+
+
+def map_paths_resource(
+    to_resource, from_resources, from_resources_index, map_types, logger=None
+):
+    """
+    Map paths found in the ``to_resource`` extra_data to paths of the ``from_resources``
+    CodebaseResource queryset using the precomputed ``from_resources_index`` path index.
+    """
+    # Accumulate unique relation objects for bulk creation
+    relations_to_create = {}
+
+    for map_type in map_types:
+        # These are of type string
+        paths_in_binary = to_resource.extra_data.get(map_type, [])
+        paths_not_mapped = to_resource.extra_data[f"{map_type}_not_mapped"] = []
+        for item in process_paths_in_binary(
+            to_resource=to_resource,
+            from_resources=from_resources,
+            from_resources_index=from_resources_index,
+            map_type=map_type,
+            paths_in_binary=paths_in_binary,
+        ):
+            if isinstance(item, str):
+                paths_not_mapped.append(item)
+            else:
+                rel_key, relation = item
+                if rel_key not in relations_to_create:
+                    relations_to_create[rel_key] = relation
+        if paths_not_mapped:
+            to_resource.save()
+            logger(
+                f"WARNING: #{len(paths_not_mapped)} {map_type} paths NOT mapped for: "
+                f"{to_resource.path!r}"
+            )
+
+    if relations_to_create:
+        rels = CodebaseRelation.objects.bulk_create(relations_to_create.values())
+        logger(
+            f"Created {len(rels)} mappings using "
+            f"{', '.join(map_types)} for: {to_resource.path!r}"
+        )
+    else:
+        logger(
+            f"No mappings using {', '.join(map_types)} for: " f"{to_resource.path!r}"
+        )
+
+
+def process_paths_in_binary(
+    to_resource,
+    from_resources,
+    from_resources_index,
+    map_type,
+    paths_in_binary,
+):
+    """
+    Process list of paths in binary and Yield either:
+    - a tuple of (unique key for a relationship, ``CodebaseRelation`` object)
+    - Or a path if it was not mapped
+    """
+    for path in paths_in_binary:
+        match = pathmap.find_paths(path, from_resources_index)
+        if not match:
+            yield path
+            continue
+
+        matched_path_length = match.matched_path_length
+        if is_invalid_match(match, matched_path_length):
+            yield path
+            continue
+
+        matched_from_resources = [
+            from_resources.get(id=rid) for rid in match.resource_ids
+        ]
+        matched_from_resources = sort_matched_from_resources(matched_from_resources)
+        winning_from_resource = matched_from_resources[0]
+
+        path_length = count_path_segments(path) - 1
+        extra_data = {
+            "path_score": f"{matched_path_length}/{path_length}",
+            map_type: path,
+        }
+
+        rel_key = (winning_from_resource.path, to_resource.path, map_type)
+        relation = CodebaseRelation(
+            project=winning_from_resource.project,
+            from_resource=winning_from_resource,
+            to_resource=to_resource,
+            map_type=map_type,
+            extra_data=extra_data,
+        )
+        yield rel_key, relation
+
+
+def count_path_segments(path):
+    """Return the number of path segments in POSIX ``path`` string"""
+    return len(path.strip("/").split("/"))
+
+
+def sort_matched_from_resources(matched_from_resources):
+    """
+    Return the sorted list of ``matched_from_resources``
+    based on path length and path.
+    """
+
+    def sorter(res):
+        return count_path_segments(res.path), res.path
+
+    return sorted(matched_from_resources, key=sorter)
+
+
+def is_invalid_match(match, matched_path_length):
+    """
+    Check if the match is invalid based on the ``matched_path_length`` and the number
+    of resource IDs.
+    """
+    return matched_path_length == 1 and len(match.resource_ids) != 1
+
+
+def map_elfs(project, logger=None):
+    """Map ELF binaries to their sources in ``project``."""
+    from_resources = project.codebaseresources.files().from_codebase()
+    to_resources = (
+        project.codebaseresources.files().to_codebase().has_no_relation().elfs()
+    )
+    for resource in to_resources:
+        try:
+            paths = get_elf_file_dwarf_paths(resource.location_path)
+            resource.update_extra_data(paths)
+        except Exception as e:
+            logger(f"Can not parse {resource.location_path!r} {e!r}")
+
+    if logger:
+        logger(
+            f"Mapping {to_resources.count():,d} to/ resources using paths "
+            f"with {from_resources.count():,d} from/ resources."
+        )
+
+    from_resources_index = pathmap.build_index(
+        from_resources.values_list("id", "path"), with_subpaths=True
+    )
+
+    if logger:
+        logger("Done building from/ resources index.")
+
+    resource_iterator = to_resources.iterator(chunk_size=2000)
+    progress = LoopProgress(to_resources.count(), logger)
+    for to_resource in progress.iter(resource_iterator):
+        map_paths_resource(
+            to_resource,
+            from_resources,
+            from_resources_index,
+            map_types=["dwarf_compiled_paths", "dwarf_included_paths"],
+            logger=logger,
+        )
+
+
+def get_elf_file_dwarf_paths(location):
+    """Retrieve dwarf paths for ELF files."""
+    paths = get_dwarf_paths(location)
+    compiled_paths = paths.get("compiled_paths") or []
+    included_paths = paths.get("included_paths") or []
+    dwarf_paths = {}
+    if compiled_paths:
+        dwarf_paths["dwarf_compiled_paths"] = compiled_paths
+    if included_paths:
+        dwarf_paths["dwarf_included_paths"] = included_paths
+    return dwarf_paths
+
+
+def get_go_file_paths(location):
+    """Retrieve Go file paths."""
+    go_symbols = (
+        collect_and_parse_symbols(location, check_type=False).get("go_symbols") or {}
+    )
+    file_paths = {}
+    go_file_paths = go_symbols.get("file_paths") or []
+    if go_file_paths:
+        file_paths["go_file_paths"] = go_file_paths
+    return file_paths
+
+
+def map_go_paths(project, logger=None):
+    """Map Go binaries to their source in ``project``."""
+    from_resources = project.codebaseresources.files().from_codebase()
+    to_resources = (
+        project.codebaseresources.files()
+        .to_codebase()
+        .has_no_relation()
+        .executable_binaries()
+    )
+    for resource in to_resources:
+        try:
+            paths = get_go_file_paths(resource.location_path)
+            resource.update_extra_data(paths)
+        except Exception as e:
+            logger(f"Can not parse {resource.location_path!r} {e!r}")
+
+    if logger:
+        logger(
+            f"Mapping {to_resources.count():,d} to/ resources using paths "
+            f"with {from_resources.count():,d} from/ resources."
+        )
+
+    from_resources_index = pathmap.build_index(
+        from_resources.values_list("id", "path"), with_subpaths=True
+    )
+
+    if logger:
+        logger("Done building from/ resources index.")
+
+    resource_iterator = to_resources.iterator(chunk_size=2000)
+    progress = LoopProgress(to_resources.count(), logger)
+    for to_resource in progress.iter(resource_iterator):
+        map_paths_resource(
+            to_resource,
+            from_resources,
+            from_resources_index,
+            map_types=["go_file_paths"],
+            logger=logger,
+        )

scanpipe/tests/data/d2d-elfs/from-data.zip

@@ -29,6 +29,7 @@
 from django.test import TestCase
 
 from scanpipe import pipes
+from scanpipe.models import CodebaseRelation
 from scanpipe.models import CodebaseResource
 from scanpipe.models import Project
 from scanpipe.pipes import d2d
@@ -1461,3 +1462,63 @@ def test_scanpipe_pipes_d2d_match_purldb_resources_post_process(self):
 
         self.assertEqual(2, package1_resource_count)
         self.assertEqual(0, package2_resource_count)
+
+    def test_scanpipe_pipes_d2d_map_elfs(self):
+        input_dir = self.project1.input_path
+        input_resources = [
+            self.data_location / "d2d-elfs/to-data.zip",
+            self.data_location / "d2d-elfs/from-data.zip",
+        ]
+        copy_inputs(input_resources, input_dir)
+        self.from_files, self.to_files = d2d.get_inputs(self.project1)
+        inputs_with_codebase_path_destination = [
+            (self.from_files, self.project1.codebase_path / d2d.FROM),
+            (self.to_files, self.project1.codebase_path / d2d.TO),
+        ]
+        for input_files, codebase_path in inputs_with_codebase_path_destination:
+            for input_file_path in input_files:
+                scancode.extract_archive(input_file_path, codebase_path)
+
+        scancode.extract_archives(
+            self.project1.codebase_path,
+            recurse=True,
+        )
+        pipes.collect_and_create_codebase_resources(self.project1)
+        buffer = io.StringIO()
+        d2d.map_elfs(project=self.project1, logger=buffer.write)
+        self.assertEqual(
+            1,
+            CodebaseRelation.objects.filter(
+                project=self.project1, map_type="dwarf_included_paths"
+            ).count(),
+        )
+
+    def test_scanpipe_pipes_d2d_map_go_paths(self):
+        input_dir = self.project1.input_path
+        input_resources = [
+            self.data_location / "d2d-go/to-data.zip",
+            self.data_location / "d2d-go/from-data.zip",
+        ]
+        copy_inputs(input_resources, input_dir)
+        self.from_files, self.to_files = d2d.get_inputs(self.project1)
+        inputs_with_codebase_path_destination = [
+            (self.from_files, self.project1.codebase_path / d2d.FROM),
+            (self.to_files, self.project1.codebase_path / d2d.TO),
+        ]
+        for input_files, codebase_path in inputs_with_codebase_path_destination:
+            for input_file_path in input_files:
+                scancode.extract_archive(input_file_path, codebase_path)
+
+        scancode.extract_archives(
+            self.project1.codebase_path,
+            recurse=True,
+        )
+        pipes.collect_and_create_codebase_resources(self.project1)
+        buffer = io.StringIO()
+        d2d.map_go_paths(project=self.project1, logger=buffer.write)
+        self.assertEqual(
+            1,
+            CodebaseRelation.objects.filter(
+                project=self.project1, map_type="go_file_paths"
+            ).count(),
+        )

scanpipe/tests/data/d2d-elfs/to-data.zip

@@ -78,8 +78,9 @@ install_requires =
     # FetchCode
     fetchcode-container==1.2.3.210512; sys_platform == "linux"
     # Inspectors
-    python-inspector==0.12.0
     elf-inspector==0.0.1
+    go-inspector==0.2.2
+    python-inspector==0.12.0
     source-inspector==0.3.0
     aboutcode-toolkit==10.1.0
     # Utilities