Normalize parse function names

Signed-off-by: Jono Yang <jyang@nexb.com>

Author: JonoYang

src/packagedcode/freebsd.py

@@ -42,16 +42,13 @@ class CompactManifestHandler(models.DatafileHandler):
     documentation_url = 'https://www.freebsd.org/cgi/man.cgi?pkg-create(8)#MANIFEST_FILE_DETAILS'
 
     @classmethod
-    def _parse_freebsd_manifest_data(cls, location):
-        with io.open(location, encoding='utf-8') as loc:
-            freebsd_manifest = saneyaml.load(loc)
-
+    def _parse(cls, yaml_data):
         package_data = models.PackageData(
             datasource_id=cls.datasource_id,
             type=cls.default_package_type,
             qualifiers=dict(
-                arch=freebsd_manifest.get('arch'),
-                origin=freebsd_manifest.get('origin'),
+                arch=yaml_data.get('arch'),
+                origin=yaml_data.get('origin'),
             )
         )
 
@@ -65,7 +62,7 @@ def _parse_freebsd_manifest_data(cls, location):
         ]
 
         for source, target in plain_fields:
-            value = freebsd_manifest.get(source)
+            value = yaml_data.get(source)
             if value:
                 if isinstance(value, str):
                     value = value.strip()
@@ -83,12 +80,12 @@ def _parse_freebsd_manifest_data(cls, location):
 
         for source, func in field_mappers:
             logger.debug('parse: %(source)r, %(func)r' % locals())
-            value = freebsd_manifest.get(source) or None
+            value = yaml_data.get(source) or None
             if value:
                 func(value, package_data)
 
         # license_mapper needs multiple fields
-        license_mapper(freebsd_manifest, package_data)
+        license_mapper(yaml_data, package_data)
 
         if package_data.declared_license:
             package_data.license_expression = cls.compute_normalized_license(package_data)
@@ -101,11 +98,10 @@ def parse(cls, location):
         Yield one or more Package manifest objects given a file ``location`` pointing to a
         package archive, manifest or similar.
         """
-        yield cls._parse_freebsd_manifest_data(
-            location=location,
-            datasource_id=cls.datasource_id,
-            default_package_type=cls.default_package_type,
-        )
+        with io.open(location, encoding='utf-8') as loc:
+            yaml_data = saneyaml.load(loc)
+
+        yield cls._parse(yaml_data)
 
     @classmethod
     def compute_normalized_license(cls, package):

src/packagedcode/haxe.py

@@ -45,7 +45,7 @@ class HaxelibJsonHandler(models.DatafileHandler):
     documentation_url = 'https://lib.haxe.org/documentation/creating-a-haxelib-package/'
 
     @classmethod
-    def _parse_haxelib_json_data(cls, json_data):
+    def _parse(cls, json_data):
         name = json_data.get('name')
         version = json_data.get('version')
 
@@ -114,4 +114,4 @@ def parse(cls, location):
         with io.open(location, encoding='utf-8') as loc:
             json_data = json.load(loc)
 
-        yield cls._parse_haxelib_json_data(json_data)
+        yield cls._parse(json_data)

src/packagedcode/maven.py

@@ -1095,28 +1095,26 @@ def get_urls(namespace, name, version, qualifiers, base_url='https://repo1.maven
 
 
 def parse(
+    location,
     datasource_id,
     package_type,
     primary_language,
-    location=None,
-    text=None,
     base_url='https://repo1.maven.org/maven2',
 ):
     """
     Yield Packagedata objects from parsing a Maven pom file at `location` or
     using the provided `text` (one or the other but not both).
     """
-    yield parse_pom(
+    yield _parse(
         datasource_id=datasource_id,
         package_type=package_type,
         primary_language=primary_language,
         location=location,
-        text=text,
         base_url=base_url
     )
 
 
-def parse_pom(
+def _parse(
     datasource_id,
     package_type,
     primary_language,

src/packagedcode/models.py

@@ -413,7 +413,6 @@ class Dependency(DependentPackage):
 
     def __attrs_post_init__(self, *args, **kwargs):
         if not self.dependency_uid:
-            print(self.purl)
             self.dependency_uid = build_package_uid(self.purl)
 
     @classmethod

src/packagedcode/npm.py

@@ -163,95 +163,85 @@ class NpmPackageJsonHandler(BaseNpmHandler):
     documentation_url = 'https://docs.npmjs.com/cli/v8/configuring-npm/package-json'
 
     @classmethod
-    def parse(cls, location):
-        with io.open(location, encoding='utf-8') as loc:
-            package_data = json.load(loc)
+    def _parse(cls, json_data):
+        name = json_data.get('name')
+        version = json_data.get('version')
+        homepage_url = json_data.get('homepage', '')
+
+        # a package.json without name and version can be a private package
+
+        if homepage_url and isinstance(homepage_url, list):
+            # TODO: should we keep other URLs
+            homepage_url = homepage_url[0]
+        homepage_url = homepage_url.strip() or None
+
+        namespace, name = split_scoped_package_name(name)
 
-        yield parse_npm_package_json(
-            package_data,
+        urls = get_urls(namespace, name, version)
+        package = models.PackageData(
             datasource_id=cls.datasource_id,
-            default_package_type=cls.default_package_type,
-            default_primary_language=cls.default_primary_language
+            type=cls.default_package_type,
+            primary_language=cls.default_primary_language,
+            namespace=namespace or None,
+            name=name,
+            version=version or None,
+            description=json_data.get('description', '').strip() or None,
+            homepage_url=homepage_url,
+            **urls,
         )
+        vcs_revision = json_data.get('gitHead') or None
+
+        # mapping of top level package.json items to a function accepting as
+        # arguments the package.json element value and returning an iterable of (key,
+        # values) to update on a package
+        field_mappers = [
+            ('author', partial(party_mapper, party_type='author')),
+            ('contributors', partial(party_mapper, party_type='contributor')),
+            ('maintainers', partial(party_mapper, party_type='maintainer')),
+
+            ('dependencies', partial(deps_mapper, field_name='dependencies')),
+            ('devDependencies', partial(deps_mapper, field_name='devDependencies')),
+            ('peerDependencies', partial(deps_mapper, field_name='peerDependencies')),
+            ('optionalDependencies', partial(deps_mapper, field_name='optionalDependencies')),
+            ('bundledDependencies', bundle_deps_mapper),
+            ('repository', partial(vcs_repository_mapper, vcs_revision=vcs_revision)),
+            ('keywords', keywords_mapper,),
+            ('bugs', bugs_mapper),
+            ('dist', dist_mapper),
+        ]
+
+        for source, func in field_mappers:
+            value = json_data.get(source) or None
+            if value:
+                if isinstance(value, str):
+                    value = value.strip()
+                if value:
+                    func(value, package)
 
-    @classmethod
-    def compute_normalized_license(cls, package):
-        return compute_normalized_license(package.declared_license)
+        if not package.download_url:
+            # Only add a synthetic download URL if there is none from the dist mapping.
+            package.download_url = npm_download_url(package.namespace, package.name, package.version)
 
+        # licenses are a tad special with many different data structures
+        lic = json_data.get('license')
+        lics = json_data.get('licenses')
+        package = licenses_mapper(lic, lics, package)
 
-def parse_npm_package_json(
-    package_data,
-    datasource_id='npm_package_json',
-    default_package_type='npm',
-    default_primary_language='Javascript'
-):
-    name = package_data.get('name')
-    version = package_data.get('version')
-    homepage_url = package_data.get('homepage', '')
-
-    # a package.json without name and version can be a private package
-
-    if homepage_url and isinstance(homepage_url, list):
-        # TODO: should we keep other URLs
-        homepage_url = homepage_url[0]
-    homepage_url = homepage_url.strip() or None
-
-    namespace, name = split_scoped_package_name(name)
-
-    urls = get_urls(namespace, name, version)
-    package = models.PackageData(
-        datasource_id=datasource_id,
-        type=default_package_type,
-        primary_language=default_primary_language,
-        namespace=namespace or None,
-        name=name,
-        version=version or None,
-        description=package_data.get('description', '').strip() or None,
-        homepage_url=homepage_url,
-        **urls,
-    )
-    vcs_revision = package_data.get('gitHead') or None
-
-    # mapping of top level package.json items to a function accepting as
-    # arguments the package.json element value and returning an iterable of (key,
-    # values) to update on a package
-    field_mappers = [
-        ('author', partial(party_mapper, party_type='author')),
-        ('contributors', partial(party_mapper, party_type='contributor')),
-        ('maintainers', partial(party_mapper, party_type='maintainer')),
-
-        ('dependencies', partial(deps_mapper, field_name='dependencies')),
-        ('devDependencies', partial(deps_mapper, field_name='devDependencies')),
-        ('peerDependencies', partial(deps_mapper, field_name='peerDependencies')),
-        ('optionalDependencies', partial(deps_mapper, field_name='optionalDependencies')),
-        ('bundledDependencies', bundle_deps_mapper),
-        ('repository', partial(vcs_repository_mapper, vcs_revision=vcs_revision)),
-        ('keywords', keywords_mapper,),
-        ('bugs', bugs_mapper),
-        ('dist', dist_mapper),
-    ]
-
-    for source, func in field_mappers:
-        value = package_data.get(source) or None
-        if value:
-            if isinstance(value, str):
-                value = value.strip()
-            if value:
-                func(value, package)
+        if not package.license_expression and package.declared_license:
+            package.license_expression = compute_normalized_license(package.declared_license)
 
-    if not package.download_url:
-        # Only add a synthetic download URL if there is none from the dist mapping.
-        package.download_url = npm_download_url(package.namespace, package.name, package.version)
+        return package
 
-    # licenses are a tad special with many different data structures
-    lic = package_data.get('license')
-    lics = package_data.get('licenses')
-    package = licenses_mapper(lic, lics, package)
+    @classmethod
+    def parse(cls, location):
+        with io.open(location, encoding='utf-8') as loc:
+            json_data = json.load(loc)
 
-    if not package.license_expression and package.declared_license:
-        package.license_expression = compute_normalized_license(package.declared_license)
+        yield cls._parse(json_data)
 
-    return package
+    @classmethod
+    def compute_normalized_license(cls, package):
+        return compute_normalized_license(package.declared_license)
 
 
 class BaseNpmLockHandler(BaseNpmHandler):