Upgrade the SPDX schema to v2.3.1 #1130 (#1139)

Signed-off-by: tdruez <tdruez@nexb.com>

Author: tdruez

CHANGELOG.rst

@@ -10,6 +10,9 @@ v34.2.0 (unreleased)
 - Add support for CycloneDX XML inputs.
   https://github.com/nexB/scancode.io/issues/1136
 
+- Upgrade the SPDX schema to v2.3.1
+  https://github.com/nexB/scancode.io/issues/1130
+
 v34.1.0 (2024-03-27)
 --------------------
 

scanpipe/pipes/schemas/spdx-schema-2.3.json

@@ -1,9 +1,13 @@
 {
-  "$schema" : "http://json-schema.org/draft-07/schema#",
+  "$schema" : "https://json-schema.org/draft/2019-09/schema#",
   "$id" : "http://spdx.org/rdf/terms/2.3",
   "title" : "SPDX 2.3",
   "type" : "object",
   "properties" : {
+    "$schema": {
+      "type": "string",
+      "description": "Reference the SPDX 2.3 JSON schema."
+    },
     "SPDXID" : {
       "type" : "string",
       "description" : "Uniquely identify any element in an SPDX document which may be referenced by other elements."
@@ -187,6 +191,7 @@
     },
     "revieweds" : {
       "description" : "Reviewed",
+      "deprecated": true,
       "type" : "array",
       "items" : {
         "type" : "object",
@@ -217,7 +222,9 @@
       "description" : "The URI provides an unambiguous mechanism for other SPDX documents to reference SPDX elements within this SPDX document."
     },
     "documentDescribes" : {
-      "description" : "Packages, files and/or Snippets described by this SPDX document",
+      "description" : "DEPRECATED: use relationships instead of this field. Packages, files and/or Snippets described by this SPDX document",
+      "deprecated": true,
+      "$comment": "This field has been deprecated as it is a duplicate of using the SPDXRef-DOCUMENT DESCRIBES relationship",
       "type" : "array",
       "items" : {
         "type" : "string",
@@ -322,14 +329,14 @@
                 "referenceCategory" : {
                   "description" : "Category for the external reference",
                   "type" : "string",
-                  "enum" : [ "OTHER", "PERSISTENT-ID", "SECURITY", "PACKAGE-MANAGER" ]
+                  "enum" : [ "OTHER", "PERSISTENT-ID", "PERSISTENT_ID", "SECURITY", "PACKAGE-MANAGER", "PACKAGE_MANAGER" ]
                 },
                 "referenceLocator" : {
                   "description" : "The unique string with no spaces necessary to access the package-specific information, metadata, or content within the target location. The format of the locator is subject to constraints defined by the <type>.",
                   "type" : "string"
                 },
                 "referenceType" : {
-                  "description" : "Type of the external reference. These are definined in an appendix in the SPDX specification.",
+                  "description" : "Type of the external reference. These are defined in an appendix in the SPDX specification.",
                   "type" : "string"
                 }
               },
@@ -343,7 +350,9 @@
             "type" : "boolean"
           },
           "hasFiles" : {
-            "description" : "Indicates that a particular file belongs to a package.",
+            "description" : "DEPRECATED: use relationships instead of this field. Indicates that a particular file belongs to a package.",
+            "deprecated": true,
+            "$comment": "This field has been deprecated as it is a duplicate of using CONTAINS relationships from a package to files",
             "type" : "array",
             "items" : {
               "description" : "SPDX ID for File.  Indicates that a particular file belongs to a package.",
@@ -366,10 +375,10 @@
             "type" : "string"
           },
           "licenseInfoFromFiles" : {
-            "description" : "The licensing information that was discovered directly within the package. There will be an instance of this property for each distinct value of alllicenseInfoInFile properties of all files contained in the package.\n\nIf the licenseInfoFromFiles field is not present for a package and filesAnalyzed property for that same pacakge is true or omitted, it implies an equivalent meaning to NOASSERTION.",
+            "description" : "The licensing information that was discovered directly within the package. There will be an instance of this property for each distinct value of alllicenseInfoInFile properties of all files contained in the package.\n\nIf the licenseInfoFromFiles field is not present for a package and filesAnalyzed property for that same package is true or omitted, it implies an equivalent meaning to NOASSERTION.",
             "type" : "array",
             "items" : {
-              "description" : "License expression for licenseInfoFromFiles. See SPDX Annex D for the license expression syntax.  The licensing information that was discovered directly within the package. There will be an instance of this property for each distinct value of alllicenseInfoInFile properties of all files contained in the package.\n\nIf the licenseInfoFromFiles field is not present for a package and filesAnalyzed property for that same pacakge is true or omitted, it implies an equivalent meaning to NOASSERTION.",
+              "description" : "License expression for licenseInfoFromFiles. See SPDX Annex D for the license expression syntax.  The licensing information that was discovered directly within the package. There will be an instance of this property for each distinct value of alllicenseInfoInFile properties of all files contained in the package.\n\nIf the licenseInfoFromFiles field is not present for a package and filesAnalyzed property for that same package is true or omitted, it implies an equivalent meaning to NOASSERTION.",
               "type" : "string"
             }
           },
@@ -531,6 +540,7 @@
           },
           "fileDependencies" : {
             "description" : "This field is deprecated since SPDX 2.0 in favor of using Section 7 which provides more granularity about relationships.",
+            "deprecated": true,
             "type" : "array",
             "items" : {
               "description" : "SPDX ID for File.  This field is deprecated since SPDX 2.0 in favor of using Section 7 which provides more granularity about relationships.",
@@ -735,6 +745,6 @@
       }
     }
   },
-  "required" : [ "SPDXID", "creationInfo", "dataLicense", "name", "spdxVersion" ],
+  "required" : [ "SPDXID", "creationInfo", "dataLicense", "name", "spdxVersion", "documentNamespace" ],
   "additionalProperties" : false
-}
+}

scanpipe/pipes/schemas/spdx-schema-2.3.json.ABOUT

@@ -1,17 +1,15 @@
 about_resource: spdx-schema-2.3.json
 name: spdx-spec
 version: 2.3
-download_url: https://github.com/spdx/spdx-spec/archive/refs/tags/v2.3.zip
+download_url: https://github.com/spdx/spdx-spec/raw/development/v2.3.1/schemas/spdx-schema.json
 description: The Software Package Data Exchange® (SPDX®) specification is a standard format
   for communicating the components, licenses and copyrights associated with software packages.
 homepage_url: https://spdx.org
-package_url: pkg:github/spdx/spdx-spec@2.3?version_prefix=v#schemas/spdx-schema.json
+package_url: pkg:github/spdx/spdx-spec@2.3.1?version_prefix=v#schemas/spdx-schema.json
 license_expression: cc-by-3.0
 copyright: Copyright (c) SPDX project contributors
 attribute: yes
 track_changes: yes
-checksum_md5: e5b8b72262e342b63cc3dc888f191dbe
-checksum_sha1: 186f20921d05d6bb8f2fe9b4c63a795abffa2a98
 licenses:
   - key: cc-by-3.0
     name: Creative Commons Attribution License 3.0

scanpipe/pipes/spdx.py

@@ -34,7 +34,7 @@
 SPDX_SCHEMA_NAME = "spdx-schema-2.3.json"
 SPDX_SCHEMA_PATH = Path(__file__).parent / "schemas" / SPDX_SCHEMA_NAME
 SPDX_SCHEMA_URL = (
-    "https://raw.githubusercontent.com/spdx/spdx-spec/v2.3/schemas/spdx-schema.json"
+    "https://github.com/spdx/spdx-spec/raw/development/v2.3.1/schemas/spdx-schema.json"
 )
 
 """

scanpipe/tests/data/spdx/example-2.3.1.json

@@ -0,0 +1,33 @@
+{
+  "spdxVersion": "SPDX-2.3",
+  "dataLicense": "CC0-1.0",
+  "SPDXID": "SPDXRef-DOCUMENT",
+  "name": "scancodeio_asgiref",
+  "documentNamespace": "https://scancode.io/spdxdocs/8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e",
+  "creationInfo": {
+    "created": "2000-01-01T01:02:03Z",
+    "creators": [
+      "Tool: ScanCode.io-31.0.0"
+    ],
+    "licenseListVersion": "3.20"
+  },
+  "packages": [
+    {
+      "name": "asgiref",
+      "SPDXID": "SPDXRef-scancodeio-discoveredpackage-b6ef7c90-e3d4-4008-8b67-63f086cea2da",
+      "downloadLocation": "NOASSERTION",
+      "licenseConcluded": "BSD-3-Clause",
+      "copyrightText": "NOASSERTION",
+      "filesAnalyzed": false,
+      "versionInfo": "3.3.0",
+      "licenseDeclared": "BSD-3-Clause",
+      "externalRefs": [
+        {
+          "referenceCategory": "PACKAGE_MANAGER",
+          "referenceType": "purl",
+          "referenceLocator": "pkg:pypi/asgiref@3.3.0"
+        }
+      ]
+    }
+  ]
+}

scanpipe/tests/pipes/test_spdx.py

@@ -20,12 +20,15 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/nexB/scancode.io for support and download.
 
+from pathlib import Path
 from unittest import TestCase
 
 from scanpipe.pipes import spdx
 
 
 class ScanPipeSPDXPipesTest(TestCase):
+    data_location = Path(__file__).parent.parent / "data"
+
     def setUp(self):
         self.schema = spdx.SPDX_SCHEMA_PATH.read_text()
 
@@ -373,5 +376,9 @@ def test_spdx_validate_document(self):
         document = spdx.Document(**self.document_data)
         spdx.validate_document(document, self.schema)
 
+        # Testing support for "PACKAGE_MANAGER" in place of "PACKAGE-MANAGER"
+        document_location = self.data_location / "spdx" / "example-2.3.1.json"
+        spdx.validate_document(document_location.read_text(), self.schema)
+
         with self.assertRaises(Exception):
             spdx.validate_document({}, self.schema)