Update is_resolved from resolved_packages

Apply updates for all npm lockfiles.

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

Author: AyanSinhaMahapatra

src/packagedcode/npm.py

@@ -23,6 +23,7 @@
 from packagedcode.utils import normalize_vcs_url
 from packagedcode.utils import yield_dependencies_from_package_data
 from packagedcode.utils import yield_dependencies_from_package_resource
+from packagedcode.utils import update_dependencies_as_resolved
 import saneyaml
 
 """
@@ -420,8 +421,9 @@ def parse(cls, location, package_only=False):
                 )
             resolved_package.dependencies = sub_deps
             dependency.resolved_package = resolved_package.to_dict()
-            dependencies.append(dependency)
+            dependencies.append(dependency.to_dict())
 
+        update_dependencies_as_resolved(dependencies=dependencies)
         root_package_data.dependencies = dependencies
 
         yield root_package_data
@@ -551,8 +553,9 @@ def parse(cls, location, package_only=False):
                     is_optional=False,
                     is_runtime=True,
                 )
-            top_dependencies.append(dependency)
+            top_dependencies.append(dependency.to_dict())
 
+        update_dependencies_as_resolved(dependencies=top_dependencies)
         package_data = dict(
             datasource_id=cls.datasource_id,
             type=cls.default_package_type,
@@ -714,8 +717,9 @@ def parse(cls, location, package_only=False):
                 is_runtime=True,
                 resolved_package=resolved_package_data.to_dict(),
             )
-            dependencies.append(dep)
+            dependencies.append(dep.to_dict())
 
+        update_dependencies_as_resolved(dependencies=dependencies)
         package_data = dict(
             datasource_id=cls.datasource_id,
             type=cls.default_package_type,
@@ -811,7 +815,11 @@ def parse(cls, location, package_only=False):
             )
             dependencies_by_purl[purl] = dependency_data
 
-        dependencies = list(dependencies_by_purl.values())
+        dependencies = [
+            dep.to_dict()
+            for dep in list(dependencies_by_purl.values())
+        ]
+        update_dependencies_as_resolved(dependencies=dependencies)
         root_package_data = dict(
             datasource_id=cls.datasource_id,
             type=cls.default_package_type,

src/packagedcode/utils.py

@@ -7,6 +7,8 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
+from packageurl import PackageURL
+
 try:
     from license_expression import Licensing
     from license_expression import combine_expressions as le_combine_expressions
@@ -215,3 +217,57 @@ def yield_dependencies_from_package_resource(resource, package_uid=None):
     for pkg_data in resource.package_data:
         pkg_data = models.PackageData.from_dict(pkg_data)
         yield from yield_dependencies_from_package_data(pkg_data, resource.path, package_uid)
+
+
+def update_dependencies_as_resolved(dependencies):
+    """
+    For a list of dependency mappings with their respective
+    resolved packages, update in place the dependencies for those
+    resolved packages as resolved (update `is_resolved` as True),
+    if the requirement is also present as a resolved package.
+    """
+    #TODO: Use vers to mark update `is_resolved` even in the case
+    # of incomplete resolution/partially pinned dependencies
+
+    # These are only type, namespace and name (without version and qualifiers)
+    base_resolved_purls = []
+    base_purl_fields = ["type", "namespace", "name"]
+    try:
+        resolved_packages = [
+            dep.get("resolved_package")
+            for dep in dependencies
+            if dep.get("resolved_package")
+        ]
+    except AttributeError:
+        raise Exception(dependencies)
+
+    # No resolved packages are present for dependencies
+    if not resolved_packages:
+        return
+
+    for pkg in resolved_packages:
+        purl_mapping = PackageURL.from_string(purl=pkg.get("purl")).to_dict()
+        base_purl_mapping = {
+            purl_field: purl_value
+            for purl_field, purl_value in purl_mapping.items()
+            if purl_field in base_purl_fields
+        }
+        base_resolved_purls.append(
+            PackageURL(**base_purl_mapping).to_string()
+        )
+
+    for dependency in dependencies:
+        resolved_package = dependency.get("resolved_package")
+        dependencies_from_resolved = []
+        if resolved_package:
+            dependencies_from_resolved = resolved_package.get("dependencies")
+
+        if not dependencies_from_resolved:
+            continue
+
+        for dep in dependencies_from_resolved:
+            dep_purl = dep.get("purl")
+            if dep_purl in base_resolved_purls:
+                dep["is_resolved"] = True
+
+

tests/packagedcode/data/npm/package-lock-v1/package-lock.json-expected

@@ -625,7 +625,7 @@
               "scope": "dependencies",
               "is_runtime": true,
               "is_optional": false,
-              "is_resolved": false,
+              "is_resolved": true,
               "resolved_package": {},
               "extra_data": {}
             }
@@ -707,7 +707,7 @@
               "scope": "dependencies",
               "is_runtime": true,
               "is_optional": false,
-              "is_resolved": false,
+              "is_resolved": true,
               "resolved_package": {},
               "extra_data": {}
             }
@@ -893,7 +893,7 @@
               "scope": "dependencies",
               "is_runtime": true,
               "is_optional": false,
-              "is_resolved": false,
+              "is_resolved": true,
               "resolved_package": {},
               "extra_data": {}
             },
@@ -903,7 +903,7 @@
               "scope": "dependencies",
               "is_runtime": true,
               "is_optional": false,
-              "is_resolved": false,
+              "is_resolved": true,
               "resolved_package": {},
               "extra_data": {}
             },
@@ -913,7 +913,7 @@
               "scope": "dependencies",
               "is_runtime": true,
               "is_optional": false,
-              "is_resolved": false,
+              "is_resolved": true,
               "resolved_package": {},
               "extra_data": {}
             }
@@ -975,7 +975,7 @@
               "scope": "dependencies",
               "is_runtime": true,
               "is_optional": false,
-              "is_resolved": false,
+              "is_resolved": true,
               "resolved_package": {},
               "extra_data": {}
             },
@@ -985,7 +985,7 @@
               "scope": "dependencies",
               "is_runtime": true,
               "is_optional": false,
-              "is_resolved": false,
+              "is_resolved": true,
               "resolved_package": {},
               "extra_data": {}
             },
@@ -995,7 +995,7 @@
               "scope": "dependencies",
               "is_runtime": true,
               "is_optional": false,
-              "is_resolved": false,
+              "is_resolved": true,
               "resolved_package": {},
               "extra_data": {}
             }

tests/packagedcode/data/npm/yarn-lock/v1-complex2/yarn.lock-expected

@@ -475,7 +475,7 @@
               "scope": "dependencies",
               "is_runtime": true,
               "is_optional": false,
-              "is_resolved": false,
+              "is_resolved": true,
               "resolved_package": {},
               "extra_data": {}
             }