Merge pull request #3035 from nexB/assemble-package-adder

Add package_adder argument to assemble() #3034

Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>

Author: pombredanne

CHANGELOG.rst

@@ -167,8 +167,12 @@ Package detection:
     of the new format where there is no root conceptually, just a list of files for each
     package.
 
-  - There is a new resource-level attribute `for_packages` which refers to packages
-    through package_uuids (pURL + uuid string).
+  - There is a new resource-level attribute `for_packages` which refers to
+    packages through package_uids (pURL + uuid string). A `package_adder`
+    function is now used to associate a Package to a Resource that is part of
+    it. This gives us the flexibility to use the packagedcode Package handlers
+    in other contexts where `for_packages` on Resource is not implemented in the
+    same way as scancode-toolkit.
 
   - The package_data attribute `dependencies` (which is a list of DependentPackages),
     now has a new attribute `resolved_package` with a package data mapping.
@@ -336,8 +340,8 @@ Miscellaneous
 
 - Added support for usage of shortcut flags
   - `-A` or `--about`
-  - `-q` or `--quiet` 
-  - `-v` or `--verbose` 
+  - `-q` or `--quiet`
+  - `-v` or `--verbose`
   - `-V` or `--version` can be used.
 
 

src/packagedcode/about.py

@@ -107,7 +107,7 @@ def parse(cls, location):
         )
 
     @classmethod
-    def assemble(cls, package_data, resource, codebase):
+    def assemble(cls, package_data, resource, codebase, package_adder):
         """
         Yield a Package. Note that ABOUT files do not carry dependencies.
         """
@@ -122,8 +122,7 @@ def assemble(cls, package_data, resource, codebase):
 
             # NOTE: we do not attach files to the Package level. Instead we
             # update `for_package` in the file
-            resource.for_packages.append(package_uid)
-            resource.save(codebase)
+            package_adder(package_uid, resource, codebase)
 
             if not package.license_expression:
                 package.license_expression = cls.compute_normalized_license(package)
@@ -151,8 +150,7 @@ def assemble(cls, package_data, resource, codebase):
                         # path is found and processed: remove it, so we can
                         # check if we found all of them
                         del file_references_by_path[res.path]
-                        res.for_packages.append(package_uid)
-                        res.save(codebase)
+                        package_adder(package_uid, res, codebase)
 
                         yield res
 

src/packagedcode/alpine.py

@@ -64,7 +64,7 @@ def compute_normalized_license(cls, package):
         return detected
 
     @classmethod
-    def assemble(cls, package_data, resource, codebase):
+    def assemble(cls, package_data, resource, codebase, package_adder):
         # get the root resource of the rootfs
         levels_up = len('lib/apk/db/installed'.split('/'))
         root_resource = get_ancestor(
@@ -107,8 +107,7 @@ def assemble(cls, package_data, resource, codebase):
             # path is found and processed: remove it, so we can check if we
             # found all of them
             del file_references_by_path[res.path]
-            res.for_packages.append(package_uid)
-            res.save(codebase)
+            package_adder(package_uid, res, codebase)
             resources.append(res)
 
         # if we have left over file references, add these to extra data
@@ -139,11 +138,12 @@ def compute_normalized_license(cls, package):
         return detected
 
     @classmethod
-    def assign_package_to_resources(cls, package, resource, codebase):
+    def assign_package_to_resources(cls, package, resource, codebase, package_adder):
         models.DatafileHandler.assign_package_to_parent_tree(
             package=package,
             resource=resource,
             codebase=codebase,
+            package_adder=package_adder,
         )
 
 

src/packagedcode/build.py

@@ -50,11 +50,12 @@ def parse(cls, location):
         )
 
     @classmethod
-    def assign_package_to_resources(cls, package, resource, codebase):
+    def assign_package_to_resources(cls, package, resource, codebase, package_adder):
         models.DatafileHandler.assign_package_to_parent_tree(
             package=package,
             resource=resource,
             codebase=codebase,
+            package_adder=package_adder,
         )
 
 
@@ -75,7 +76,7 @@ class BaseStarlarkManifestHandler(models.DatafileHandler):
     """
 
     @classmethod
-    def assemble(cls, package_data, resource, codebase):
+    def assemble(cls, package_data, resource, codebase, package_adder):
         """
         Given a ``package_data`` PackageData found in the ``resource`` datafile
         of the ``codebase``, assemble package their files and dependencies
@@ -100,6 +101,7 @@ def assemble(cls, package_data, resource, codebase):
                 package=package,
                 resource=resource,
                 codebase=codebase,
+                package_adder=package_adder
             )
 
             yield package
@@ -175,14 +177,13 @@ def parse(cls, location):
             )
 
     @classmethod
-    def assign_package_to_resources(cls, package, resource, codebase, skip_name=None):
+    def assign_package_to_resources(cls, package, resource, codebase, package_adder, skip_name=None):
         package_uid = package.package_uid
         if not package_uid:
             return
         parent = resource.parent(codebase)
         for res in walk_build(resource=parent, codebase=codebase, skip_name=skip_name):
-            res.for_packages.append(package_uid)
-            res.save(codebase)
+            package_adder(package_uid, res, codebase)
 
 
 def walk_build(resource, codebase, skip_name):
@@ -235,11 +236,12 @@ class BazelBuildHandler(BaseStarlarkManifestHandler):
     documentation_url = 'https://bazel.build/'
 
     @classmethod
-    def assign_package_to_resources(cls, package, resource, codebase, skip_name='BUILD'):
+    def assign_package_to_resources(cls, package, resource, codebase, package_adder, skip_name='BUILD'):
         return super().assign_package_to_resources(
             package=package,
             resource=resource,
             codebase=codebase,
+            package_adder=package_adder,
             skip_name=skip_name,
         )
 
@@ -252,11 +254,12 @@ class BuckPackageHandler(BaseStarlarkManifestHandler):
     documentation_url = 'https://buck.build/'
 
     @classmethod
-    def assign_package_to_resources(cls, package, resource, codebase, skip_name='BUCK'):
+    def assign_package_to_resources(cls, package, resource, codebase, package_adder, skip_name='BUCK'):
         return super().assign_package_to_resources(
             package=package,
             resource=resource,
             codebase=codebase,
+            package_adder=package_adder,
             skip_name=skip_name,
         )
 
@@ -377,9 +380,10 @@ def compute_normalized_license(cls, package):
             return models.compute_normalized_license(declared_license)
 
     @classmethod
-    def assign_package_to_resources(cls, package, resource, codebase):
+    def assign_package_to_resources(cls, package, resource, codebase, package_adder):
         models.DatafileHandler.assign_package_to_parent_tree(
             package_=package,
             resource=resource,
             codebase=codebase,
+            package_adder=package_adder,
         )

src/packagedcode/build_gradle.py

@@ -65,11 +65,12 @@ def parse(cls, location):
 
     # TODO: handle complex cases of nested builds with many packages
     @classmethod
-    def assign_package_to_resources(cls, package, resource, codebase):
+    def assign_package_to_resources(cls, package, resource, codebase, package_adder):
         models.DatafileHandler.assign_package_to_parent_tree(
             package=package,
             resource=resource,
             codebase=codebase,
+            package_adder=package_adder,
         )
 
 

src/packagedcode/cargo.py

@@ -85,14 +85,15 @@ def parse(cls, location):
         )
 
     @classmethod
-    def assemble(cls, package_data, resource, codebase):
+    def assemble(cls, package_data, resource, codebase, package_adder):
         """
         Assemble Cargo.toml and possible Cargo.lock datafiles
         """
         yield from cls.assemble_from_many_datafiles(
             datafile_name_patterns=('Cargo.toml', 'cargo.toml', 'Cargo.lock', 'cargo.lock'),
             directory=resource.parent(codebase),
             codebase=codebase,
+            package_adder=package_adder,
         )
 
 
@@ -144,14 +145,15 @@ def parse(cls, location):
         )
 
     @classmethod
-    def assemble(cls, package_data, resource, codebase):
+    def assemble(cls, package_data, resource, codebase, package_adder):
         """
         Assemble Cargo.toml and possible Cargo.lock datafiles
         """
         yield from cls.assemble_from_many_datafiles(
             datafile_name_patterns=('Cargo.toml', 'Cargo.lock',),
             directory=resource.parent(codebase),
             codebase=codebase,
+            package_adder=package_adder,
         )
 
 

src/packagedcode/chef.py

@@ -142,14 +142,15 @@ def format(self, tokens, outfile):
 class BaseChefMetadataHandler(models.DatafileHandler):
 
     @classmethod
-    def assemble(cls, package_data, resource, codebase):
+    def assemble(cls, package_data, resource, codebase, package_adder):
         """
         Assemble Package from Chef metadata.rb, then from metadata.json files.
         """
         yield from cls.assemble_from_many_datafiles(
             datafile_name_patterns=('metadata.rb', 'metadata.json',),
             directory=resource.parent(codebase),
             codebase=codebase,
+            package_adder=package_adder,
         )
 
 

src/packagedcode/cocoapods.py

@@ -114,7 +114,7 @@ def get_first_three_md5_hash_characters(podname):
 class BasePodHandler(models.DatafileHandler):
 
     @classmethod
-    def assemble(cls, package_data, resource, codebase):
+    def assemble(cls, package_data, resource, codebase, package_adder):
         """
         Assemble pod packages and dependencies and handle the specific cases where
         there are more than one podspec in the same directory.
@@ -147,6 +147,7 @@ def assemble(cls, package_data, resource, codebase):
                     datafile_name_patterns=datafile_name_patterns,
                     directory=parent,
                     codebase=codebase,
+                    package_adder=package_adder,
                 )
 
             elif has_multiple_podspec:
@@ -160,6 +161,7 @@ def assemble(cls, package_data, resource, codebase):
                     datafile_name_patterns=datafile_name_patterns,
                     directory=parent,
                     codebase=codebase,
+                    package_adder=package_adder,
                 )
 
                 for resource in sibling_podspecs:
@@ -170,7 +172,12 @@ def assemble(cls, package_data, resource, codebase):
                             package_data=package_data,
                             datafile_path=datafile_path,
                         )
-                        cls.assign_package_to_resources(package, resource, codebase)
+                        cls.assign_package_to_resources(
+                            package=package,
+                            resource=resource,
+                            codebase=codebase,
+                            package_adder=package_adder,
+                        )
                         yield package
                     yield resource
 
@@ -180,6 +187,7 @@ def assemble(cls, package_data, resource, codebase):
                     datafile_name_patterns=datafile_name_patterns,
                     directory=parent,
                     codebase=codebase,
+                    package_adder=package_adder,
                 )
 
 

src/packagedcode/conda.py

@@ -70,11 +70,12 @@ def get_conda_root(cls, resource, codebase):
         return resource.parent(codebase)
 
     @classmethod
-    def assign_package_to_resources(cls, package, resource, codebase):
+    def assign_package_to_resources(cls, package, resource, codebase, package_adder):
         return models.DatafileHandler.assign_package_to_resources(
             package=package,
             resource=cls.get_conda_root(resource, codebase),
             codebase=codebase,
+            package_adder=package_adder,
         )
 
     @classmethod