Move package parsing code to their own functions

    * We would like to have a way to get PackageData objects from different manifest data without using generator from the DatafileHandler.parse() method
    * Update get_urls for several packages to accept arbitrary keyword arguments

Signed-off-by: Jono Yang <jyang@nexb.com>

Author: JonoYang

src/packagedcode/chef.py

@@ -55,7 +55,7 @@ def chef_api_url(name, version):
     return name and version and f'https://supermarket.chef.io/api/v1/cookbooks/{name}/versions/{version}'
 
 
-def get_urls(name, version):
+def get_urls(name, version, **kwargs):
     """
     Return a mapping of URLs given a name and version.
     """

src/packagedcode/cocoapods.py

@@ -434,7 +434,7 @@ def parse(cls, location):
         )
 
 
-def get_urls(name=None, version=None, homepage_url=None, vcs_url=None):
+def get_urls(name=None, version=None, homepage_url=None, vcs_url=None, **kwargs):
     """
     Return a mapping of podspec URLS.
     """

src/packagedcode/freebsd.py

@@ -42,11 +42,7 @@ class CompactManifestHandler(models.DatafileHandler):
     documentation_url = 'https://www.freebsd.org/cgi/man.cgi?pkg-create(8)#MANIFEST_FILE_DETAILS'
 
     @classmethod
-    def parse(cls, location):
-        """
-        Yield one or more Package manifest objects given a file ``location`` pointing to a
-        package archive, manifest or similar.
-        """
+    def _parse_freebsd_manifest_data(cls, location):
         with io.open(location, encoding='utf-8') as loc:
             freebsd_manifest = saneyaml.load(loc)
 
@@ -97,7 +93,19 @@ def parse(cls, location):
         if package_data.declared_license:
             package_data.license_expression = cls.compute_normalized_license(package_data)
 
-        yield package_data
+        return package_data
+
+    @classmethod
+    def parse(cls, location):
+        """
+        Yield one or more Package manifest objects given a file ``location`` pointing to a
+        package archive, manifest or similar.
+        """
+        yield cls._parse_freebsd_manifest_data(
+            location=location,
+            datasource_id=cls.datasource_id,
+            default_package_type=cls.default_package_type,
+        )
 
     @classmethod
     def compute_normalized_license(cls, package):

src/packagedcode/haxe.py

@@ -45,26 +45,7 @@ class HaxelibJsonHandler(models.DatafileHandler):
     documentation_url = 'https://lib.haxe.org/documentation/creating-a-haxelib-package/'
 
     @classmethod
-    def parse(cls, location):
-        """
-        Yield one or more Package manifest objects given a file ``location`` pointing to a
-        package_data archive, manifest or similar.
-
-        {
-            "name": "haxelib",
-            "url" : "https://lib.haxe.org/documentation/",
-            "license": "GPL",
-            "tags": ["haxelib", "core"],
-            "description": "The haxelib client",
-            "classPath": "src",
-            "version": "3.4.0",
-            "releasenote": " * Fix password input issue in Windows (#421).\n * ....",
-            "contributors": ["back2dos", "ncannasse", "jason", "Simn", "nadako", "andyli"]
-        }
-        """
-        with io.open(location, encoding='utf-8') as loc:
-            json_data = json.load(loc)
-
+    def _parse_haxelib_json_data(cls, json_data):
         name = json_data.get('name')
         version = json_data.get('version')
 
@@ -110,4 +91,27 @@ def parse(cls, location):
             dep = models.DependentPackage(purl=dep_purl, is_resolved=is_resolved,)
             package_data.dependencies.append(dep)
 
-        yield package_data
+        return package_data
+
+    @classmethod
+    def parse(cls, location):
+        """
+        Yield one or more Package manifest objects given a file ``location`` pointing to a
+        package_data archive, manifest or similar.
+
+        {
+            "name": "haxelib",
+            "url" : "https://lib.haxe.org/documentation/",
+            "license": "GPL",
+            "tags": ["haxelib", "core"],
+            "description": "The haxelib client",
+            "classPath": "src",
+            "version": "3.4.0",
+            "releasenote": " * Fix password input issue in Windows (#421).\n * ....",
+            "contributors": ["back2dos", "ncannasse", "jason", "Simn", "nadako", "andyli"]
+        }
+        """
+        with io.open(location, encoding='utf-8') as loc:
+            json_data = json.load(loc)
+
+        yield cls._parse_haxelib_json_data(json_data)

src/packagedcode/maven.py

@@ -891,12 +891,12 @@ def has_basic_pom_attributes(pom):
     return basics
 
 
-def get_maven_pom(location=None):
+def get_maven_pom(location=None, text=None):
     """
     Return a MavenPom object from a POM file at `location` or provided as a
     `text` string.
     """
-    pom = MavenPom(location=location)
+    pom = MavenPom(location=location, text=text)
 
     extra_properties = {}
 
@@ -1043,7 +1043,7 @@ def get_parties(pom):
     return parties
 
 
-def get_urls(namespace, name, version, qualifiers, base_url='https://repo1.maven.org/maven2'):
+def get_urls(namespace, name, version, qualifiers, base_url='https://repo1.maven.org/maven2', **kwargs):
     """
     Return a mapping of URLs.
     """
@@ -1095,17 +1095,40 @@ def get_urls(namespace, name, version, qualifiers, base_url='https://repo1.maven
 
 
 def parse(
-    location,
     datasource_id,
     package_type,
     primary_language,
+    location=None,
+    text=None,
     base_url='https://repo1.maven.org/maven2',
 ):
     """
     Yield Packagedata objects from parsing a Maven pom file at `location` or
     using the provided `text` (one or the other but not both).
     """
-    pom = get_maven_pom(location=location)
+    yield parse_pom(
+        datasource_id=datasource_id,
+        package_type=package_type,
+        primary_language=primary_language,
+        location=location,
+        text=text,
+        base_url=base_url
+    )
+
+
+def parse_pom(
+    datasource_id,
+    package_type,
+    primary_language,
+    location=None,
+    text=None,
+    base_url='https://repo1.maven.org/maven2',
+):
+    """
+    Yield Packagedata objects from parsing a Maven pom file at `location` or
+    using the provided `text` (one or the other but not both).
+    """
+    pom = get_maven_pom(location=location, text=text)
 
     if not pom:
         return
@@ -1192,7 +1215,8 @@ def parse(
     if not package_data.license_expression and package_data.declared_license:
         package_data.license_expression = models.compute_normalized_license(package_data.declared_license)
 
-    yield package_data
+    return package_data
+
 
 def build_vcs_and_code_view_urls(scm):
     """

src/packagedcode/models.py

@@ -307,7 +307,7 @@ def set_purl(self, package_url):
         for key, value in package_url.to_dict().items():
             self_val = getattr(self, key)
             if not self_val and value:
-                setattr(self, attr, value)
+                setattr(self, key, value)
 
     def to_dict(self, **kwargs):
         mapping = super().to_dict(**kwargs)
@@ -413,6 +413,7 @@ class Dependency(DependentPackage):
 
     def __attrs_post_init__(self, *args, **kwargs):
         if not self.dependency_uid:
+            print(self.purl)
             self.dependency_uid = build_package_uid(self.purl)
 
     @classmethod

src/packagedcode/npm.py

@@ -146,7 +146,7 @@ def walk_npm(cls, resource, codebase, depth=0):
                     yield subchild
 
 
-def get_urls(namespace, name, version):
+def get_urls(namespace, name, version, **kwargs):
     return dict(
         repository_homepage_url=npm_homepage_url(namespace, name, registry='https://www.npmjs.com/package'),
         repository_download_url=npm_download_url(namespace, name, version, registry='https://registry.npmjs.org'),
@@ -167,77 +167,91 @@ def parse(cls, location):
         with io.open(location, encoding='utf-8') as loc:
             package_data = json.load(loc)
 
-        name = package_data.get('name')
-        version = package_data.get('version')
-        homepage_url = package_data.get('homepage', '')
-
-        # a package.json without name and version can be a private package
+        yield parse_npm_package_json(
+            package_data,
+            datasource_id=cls.datasource_id,
+            default_package_type=cls.default_package_type,
+            default_primary_language=cls.default_primary_language
+        )
 
-        if homepage_url and isinstance(homepage_url, list):
-            # TODO: should we keep other URLs
-            homepage_url = homepage_url[0]
-        homepage_url = homepage_url.strip() or None
+    @classmethod
+    def compute_normalized_license(cls, package):
+        return compute_normalized_license(package.declared_license)
 
-        namespace, name = split_scoped_package_name(name)
 
-        urls = get_urls(namespace, name, version)
-        package = models.PackageData(
-            datasource_id=cls.datasource_id,
-            type=cls.default_package_type,
-            primary_language=cls.default_primary_language,
-            namespace=namespace or None,
-            name=name,
-            version=version or None,
-            description=package_data.get('description', '').strip() or None,
-            homepage_url=homepage_url,
-            **urls,
-        )
-        vcs_revision = package_data.get('gitHead') or None
-
-        # mapping of top level package.json items to a function accepting as
-        # arguments the package.json element value and returning an iterable of (key,
-        # values) to update on a package
-        field_mappers = [
-            ('author', partial(party_mapper, party_type='author')),
-            ('contributors', partial(party_mapper, party_type='contributor')),
-            ('maintainers', partial(party_mapper, party_type='maintainer')),
-
-            ('dependencies', partial(deps_mapper, field_name='dependencies')),
-            ('devDependencies', partial(deps_mapper, field_name='devDependencies')),
-            ('peerDependencies', partial(deps_mapper, field_name='peerDependencies')),
-            ('optionalDependencies', partial(deps_mapper, field_name='optionalDependencies')),
-            ('bundledDependencies', bundle_deps_mapper),
-            ('repository', partial(vcs_repository_mapper, vcs_revision=vcs_revision)),
-            ('keywords', keywords_mapper,),
-            ('bugs', bugs_mapper),
-            ('dist', dist_mapper),
-        ]
-
-        for source, func in field_mappers:
-            value = package_data.get(source) or None
+def parse_npm_package_json(
+    package_data,
+    datasource_id='npm_package_json',
+    default_package_type='npm',
+    default_primary_language='Javascript'
+):
+    name = package_data.get('name')
+    version = package_data.get('version')
+    homepage_url = package_data.get('homepage', '')
+
+    # a package.json without name and version can be a private package
+
+    if homepage_url and isinstance(homepage_url, list):
+        # TODO: should we keep other URLs
+        homepage_url = homepage_url[0]
+    homepage_url = homepage_url.strip() or None
+
+    namespace, name = split_scoped_package_name(name)
+
+    urls = get_urls(namespace, name, version)
+    package = models.PackageData(
+        datasource_id=datasource_id,
+        type=default_package_type,
+        primary_language=default_primary_language,
+        namespace=namespace or None,
+        name=name,
+        version=version or None,
+        description=package_data.get('description', '').strip() or None,
+        homepage_url=homepage_url,
+        **urls,
+    )
+    vcs_revision = package_data.get('gitHead') or None
+
+    # mapping of top level package.json items to a function accepting as
+    # arguments the package.json element value and returning an iterable of (key,
+    # values) to update on a package
+    field_mappers = [
+        ('author', partial(party_mapper, party_type='author')),
+        ('contributors', partial(party_mapper, party_type='contributor')),
+        ('maintainers', partial(party_mapper, party_type='maintainer')),
+
+        ('dependencies', partial(deps_mapper, field_name='dependencies')),
+        ('devDependencies', partial(deps_mapper, field_name='devDependencies')),
+        ('peerDependencies', partial(deps_mapper, field_name='peerDependencies')),
+        ('optionalDependencies', partial(deps_mapper, field_name='optionalDependencies')),
+        ('bundledDependencies', bundle_deps_mapper),
+        ('repository', partial(vcs_repository_mapper, vcs_revision=vcs_revision)),
+        ('keywords', keywords_mapper,),
+        ('bugs', bugs_mapper),
+        ('dist', dist_mapper),
+    ]
+
+    for source, func in field_mappers:
+        value = package_data.get(source) or None
+        if value:
+            if isinstance(value, str):
+                value = value.strip()
             if value:
-                if isinstance(value, str):
-                    value = value.strip()
-                if value:
-                    func(value, package)
+                func(value, package)
 
-        if not package.download_url:
-            # Only add a synthetic download URL if there is none from the dist mapping.
-            package.download_url = npm_download_url(package.namespace, package.name, package.version)
-
-        # licenses are a tad special with many different data structures
-        lic = package_data.get('license')
-        lics = package_data.get('licenses')
-        package = licenses_mapper(lic, lics, package)
+    if not package.download_url:
+        # Only add a synthetic download URL if there is none from the dist mapping.
+        package.download_url = npm_download_url(package.namespace, package.name, package.version)
 
-        if not package.license_expression and package.declared_license:
-            package.license_expression = compute_normalized_license(package.declared_license)
+    # licenses are a tad special with many different data structures
+    lic = package_data.get('license')
+    lics = package_data.get('licenses')
+    package = licenses_mapper(lic, lics, package)
 
-        yield package
+    if not package.license_expression and package.declared_license:
+        package.license_expression = compute_normalized_license(package.declared_license)
 
-    @classmethod
-    def compute_normalized_license(cls, package):
-        return compute_normalized_license(package.declared_license)
+    return package
 
 
 class BaseNpmLockHandler(BaseNpmHandler):

src/packagedcode/nuget.py

@@ -18,7 +18,7 @@
 # TODO: add dependencies
 
 
-def get_urls(name, version):
+def get_urls(name, version, **kwargs):
     return dict(
         repository_homepage_url=f'https://www.nuget.org/packages/{name}/{version}',
         repository_download_url=f'https://www.nuget.org/api/v2/package/{name}/{version}',

src/packagedcode/pypi.py

@@ -1609,7 +1609,7 @@ def get_setup_py_args(location, include_not_parsable=False):
     return parse_setup_py(location)
 
 
-def get_pypi_urls(name, version):
+def get_pypi_urls(name, version, **kwargs):
     """
     Return a mapping of computed Pypi URLs for this package
     """

src/packagedcode/rubygems.py

@@ -337,7 +337,7 @@ def compute_normalized_license(declared_license):
         return combine_expressions(detected_licenses)
 
 
-def get_urls(name, version=None, platform=None):
+def get_urls(name, version=None, platform=None, **kwargs):
     """
     Return a mapping of standard URLs
     """