diff --git a/scancodeio/static/tree-views/expand-collapse.svg b/scancodeio/static/tree-views/expand-collapse.svg new file mode 100644 index 000000000..f34809c99 --- /dev/null +++ b/scancodeio/static/tree-views/expand-collapse.svg @@ -0,0 +1,7 @@ + + + + + + + diff --git a/scancodeio/static/tree-views/tree.css b/scancodeio/static/tree-views/tree.css new file mode 100644 index 000000000..b14023615 --- /dev/null +++ b/scancodeio/static/tree-views/tree.css @@ -0,0 +1,75 @@ +.tree{ + --spacing : 1.5rem; + --radius : 10px; +} + +.tree li{ + display : block; + position : relative; + padding-left : calc(2 * var(--spacing) - var(--radius) - 2px); +} + +.tree ul{ + margin-left : calc(var(--radius) - var(--spacing)); + padding-left : 0; +} + +.tree ul li{ + border-left : 2px solid #ddd; +} + +.tree ul li:last-child{ + border-color : transparent; +} + +.tree ul li::before{ + content : ''; + display : block; + position : absolute; + top : calc(var(--spacing) / -2); + left : -2px; + width : calc(var(--spacing) + 2px); + height : calc(var(--spacing) + 1px); + border : solid #ddd; + border-width : 0 0 2px 2px; +} + +.tree summary{ + display : block; + cursor : pointer; +} + +.tree summary::marker, +.tree summary::-webkit-details-marker{ + display : none; +} + +.tree summary:focus{ + outline : none; +} + +.tree summary:focus-visible{ + outline : 1px dotted #000; +} + +.tree li::after, +.tree summary::before{ + content : ''; + display : block; + position : absolute; + top : calc(var(--spacing) / 2 - var(--radius)); + left : calc(var(--spacing) - var(--radius) - 1px); + width : calc(2 * var(--radius)); + height : calc(2 * var(--radius)); + border-radius : 50%; + background : #ddd; +} + +.tree summary::before{ + z-index : 1; + background : #696 url('expand-collapse.svg') 0 0; +} + +.tree details[open] > summary::before{ + background-position : calc(-2 * var(--radius)) 0; +} diff --git a/scancodeio/static/tree-views/tree.css.ABOUT b/scancodeio/static/tree-views/tree.css.ABOUT new file mode 100644 index 000000000..13316e526 --- /dev/null +++ b/scancodeio/static/tree-views/tree.css.ABOUT @@ -0,0 +1,10 @@ +about_resource: tree.css +name: css-tree-views +homepage_url: https://iamkate.com/code/tree-views/ +description: A tree view (collapsible list) can be created using only html and css, without + the need for JavaScript. Accessibility software will see the tree view as lists nested inside + disclosure widgets, and the standard keyboard interaction is supported automatically. +license_expression: cc0-1.0 +licenses: + - key: cc0-1.0 + name: cc0-1.0 diff --git a/scancodeio/static/tree-views/tree.css.NOTICE b/scancodeio/static/tree-views/tree.css.NOTICE new file mode 100644 index 000000000..319755b97 --- /dev/null +++ b/scancodeio/static/tree-views/tree.css.NOTICE @@ -0,0 +1,29 @@ +Free content on iamkate.com +The web was still young when I first went online in 1998. It felt like a utopian dream of free culture and free knowledge. Anyone could contribute, and within weeks I had learnt html and created my first site, hosted in the 10mb of webspace my isp included as standard. + +I’ve watched as the dream has become a nightmare of surveillance and monetisation. Companies such as Google and Facebook offer their services for free to the public because their real products are their advertising networks powered by the personal data of their visitors. + +The only concern of these companies and their shareholders is to maximise their income from advertising, regardless of the costs to society. They use dubious schemes to avoid paying tax. They encourage addiction and risk the mental health of their visitors. They threaten democratic institutions. + +I have little influence over the wider web, but I can control my small part of it, creating a haven that remains true to the original dream. This page describes my approach to copyright, my promise to protect the privacy of my visitors, and my commitment to transparency. + +Copyright +Copyright limits creativity and holds back progress by restricting our rights to build upon the works of others. Copyleft licences attempt to use copyright against itself, but “the master’s tools will never dismantle the master’s house”, as Audre Lorde remarked in a different context. + +All content on my site is released under the terms of the Creative Commons CC0 1.0 Universal Legal Code. This means I have waived all copyright and related rights to the extent possible under law, with the intention of dedicating the content to the public domain. You can use and adapt it without attribution. + +Privacy +Every site is hosted on a server, which is usually operated by a third party due to the expertise needed to manage servers securely. Most sites are accessed indirectly through the servers of a content delivery network, which protects the original server from attacks that could disable the site. + +My site is hosted on Cloudflare Pages. Cloudflare is both the host and the content delivery network, avoiding the need to trust two separate third parties. Cloudflare have a strong commitment to privacy and data protection, and frequently write about developing systems to protect visitor privacy. + +Almost every site today includes code that tracks visitors for statistical and advertising purposes. Often the site owner includes code with the deliberate aim of tracking their visitors, but sometimes they just want to include a feature provided by a third party, and that provider includes their own tracking code. + +My site doesn’t include any tracking code, and doesn’t load any code from third parties. It doesn’t have a cookie banner because it doesn’t use cookies. Instead of an invasive analytics system, Cloudflare Web Analytics gives me the most important statistics without tracking individual visitors. + +Transparency +You probably don’t know me, and shouldn’t have to trust me. Instead, you should be able to check security and privacy claims for yourself. Unfortunately most sites today use a process called code minification, which makes them faster but also makes it harder for other people to understand their code. + +The Mozilla Observatory report for my site confirms the presence of various security and privacy features, resulting in a perfect A+ rating. One of these features, the content security policy, prevents browsers from loading code and other resources from third parties. + +My site doesn’t need to use code minification in order to load quickly due to its simple design, efficient implementation, and absence of resources loaded from third parties. As a result, other software developers can easily understand how the layout, styling, and interactive features are created. diff --git a/scanpipe/pipelines/load_sbom.py b/scanpipe/pipelines/load_sbom.py index 955e54dd9..2fbef7f9d 100644 --- a/scanpipe/pipelines/load_sbom.py +++ b/scanpipe/pipelines/load_sbom.py @@ -20,6 +20,7 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. +from scanpipe.models import DiscoveredDependency from scanpipe.pipelines.scan_codebase import ScanCodebase from scanpipe.pipes import resolve @@ -44,7 +45,7 @@ def steps(cls): cls.flag_empty_files, cls.flag_ignored_resources, cls.get_sbom_inputs, - cls.get_packages_from_sboms, + cls.get_data_from_sboms, cls.create_packages_from_sboms, cls.create_dependencies_from_sboms, ) @@ -53,13 +54,13 @@ def get_sbom_inputs(self): """Locate all the SBOMs among the codebase resources.""" self.manifest_resources = resolve.get_manifest_resources(self.project) - def get_packages_from_sboms(self): + def get_data_from_sboms(self): """Get packages data from SBOMs.""" - self.packages = resolve.get_packages( + self.packages, self.dependencies = resolve.get_data_from_manifests( project=self.project, package_registry=resolve.sbom_registry, manifest_resources=self.manifest_resources, - model="get_packages_from_sboms", + model="get_data_from_sboms", ) def create_packages_from_sboms(self): @@ -71,4 +72,11 @@ def create_packages_from_sboms(self): def create_dependencies_from_sboms(self): """Create the dependency relationship declared in the SBOMs.""" + # TODO: Migrate the CycloneDX behavior too, see get_dependencies_from_manifest resolve.create_dependencies_from_packages_extra_data(project=self.project) + + for dependency_data in self.dependencies: + DiscoveredDependency.create_from_data( + project=self.project, + dependency_data=dependency_data, + ) diff --git a/scanpipe/pipes/cyclonedx.py b/scanpipe/pipes/cyclonedx.py index 573566c93..c6bb5b929 100644 --- a/scanpipe/pipes/cyclonedx.py +++ b/scanpipe/pipes/cyclonedx.py @@ -155,12 +155,12 @@ def cyclonedx_component_to_package_data(cdx_component, dependencies=None): dependencies = dependencies or {} extra_data = {} - # Store the original bom_ref and dependencies for future processing. bom_ref = str(cdx_component.bom_ref) - if bom_ref: - extra_data["bom_ref"] = bom_ref - if depends_on := dependencies.get(bom_ref): - extra_data["depends_on"] = depends_on + if depends_on := dependencies.get(bom_ref): + extra_data["depends_on"] = depends_on + + # Store the original "bom_ref" as package_uid for dependencies resolution. + package_uid = bom_ref package_url_dict = {} if cdx_component.purl: @@ -176,6 +176,7 @@ def cyclonedx_component_to_package_data(cdx_component, dependencies=None): extra_data["nestedComponents"] = sorted(nested_purls) package_data = { + "package_uid": package_uid, "name": cdx_component.name, "extracted_license_statement": declared_license, "copyright": cdx_component.copyright, diff --git a/scanpipe/pipes/output.py b/scanpipe/pipes/output.py index d4c5f2632..1cf25e64d 100644 --- a/scanpipe/pipes/output.py +++ b/scanpipe/pipes/output.py @@ -671,6 +671,28 @@ def _get_spdx_extracted_licenses(license_expressions): return extracted_licenses +def get_dependency_as_spdx_relationship(dependency, document_spdx_id, packages_as_spdx): + """Return a spdx.Relationship crafted from the provided ``dependency`` instance.""" + if dependency.is_for_package: # Package dependency + parent_id = dependency.for_package.spdx_id + else: # Project dependency + parent_id = document_spdx_id + + if dependency.is_resolved_to_package: # Resolved to a Package + child_id = dependency.resolved_to_package.spdx_id + else: # Not resolved to a Package (only package_url value is available) + dependency_as_package = dependency.as_spdx_package() + packages_as_spdx.append(dependency_as_package) + child_id = dependency_as_package.spdx_id + + spdx_relationship = spdx.Relationship( + spdx_id=child_id, + related_spdx_id=parent_id, + relationship="DEPENDENCY_OF", + ) + return spdx_relationship + + def to_spdx(project, include_files=False): """ Generate output for the provided ``project`` in SPDX document format. @@ -682,6 +704,7 @@ def to_spdx(project, include_files=False): discoveredpackage_qs = get_queryset(project, "discoveredpackage") discovereddependency_qs = get_queryset(project, "discovereddependency") + document_spdx_id = f"SPDXRef-DOCUMENT-{project.uuid}" packages_as_spdx = [] license_expressions = [] relationships = [] @@ -692,15 +715,12 @@ def to_spdx(project, include_files=False): license_expressions.append(license_expression) for dependency in discovereddependency_qs: - packages_as_spdx.append(dependency.as_spdx_package()) - if dependency.for_package: - relationships.append( - spdx.Relationship( - spdx_id=dependency.spdx_id, - related_spdx_id=dependency.for_package.spdx_id, - relationship="DEPENDENCY_OF", - ) - ) + spdx_relationship = get_dependency_as_spdx_relationship( + dependency, + document_spdx_id, + packages_as_spdx, + ) + relationships.append(spdx_relationship) files_as_spdx = [] if include_files: @@ -710,6 +730,7 @@ def to_spdx(project, include_files=False): ] document = spdx.Document( + spdx_id=document_spdx_id, name=f"scancodeio_{project.name}", namespace=f"https://scancode.io/spdxdocs/{project.uuid}", creation_info=spdx.CreationInfo(tool=f"ScanCode.io-{scancodeio_version}"), diff --git a/scanpipe/pipes/resolve.py b/scanpipe/pipes/resolve.py index 20d6c8b14..5438ade5f 100644 --- a/scanpipe/pipes/resolve.py +++ b/scanpipe/pipes/resolve.py @@ -60,12 +60,27 @@ def resolve_manifest_resources(resource, package_registry): return packages -def get_packages(project, package_registry, manifest_resources, model=None): +def get_dependencies_from_manifest(resource): + """Get dependency data from resource.""" + dependencies = [] + + default_package_type = get_default_package_type(resource.location) + if not default_package_type: + return [] + + if default_package_type == "spdx": + dependencies = resolve_spdx_dependencies(input_location=resource.location) + + return dependencies + + +def get_data_from_manifests(project, package_registry, manifest_resources, model=None): """ - Get package data from package manifests/lockfiles/SBOMs or - get package data for resolved packages from package requirements. + Get package and dependency data from package manifests/lockfiles/SBOMs or + for resolved packages from package requirements. """ resolved_packages = [] + resolved_dependencies = [] sboms_headers = {} if not manifest_resources.exists(): @@ -76,7 +91,8 @@ def get_packages(project, package_registry, manifest_resources, model=None): return [] for resource in manifest_resources: - if packages := resolve_manifest_resources(resource, package_registry): + packages = resolve_manifest_resources(resource, package_registry) + if packages: resolved_packages.extend(packages) if headers := get_manifest_headers(resource): sboms_headers[resource.name] = headers @@ -87,10 +103,14 @@ def get_packages(project, package_registry, manifest_resources, model=None): object_instance=resource, ) + dependencies = get_dependencies_from_manifest(resource) + if dependencies: + resolved_dependencies.extend(dependencies) + if sboms_headers: project.update_extra_data({"sboms_headers": sboms_headers}) - return resolved_packages + return resolved_packages, resolved_dependencies def create_packages_and_dependencies(project, packages, resolved=False): @@ -139,7 +159,7 @@ def create_dependencies_from_packages_extra_data(project): for bom_ref in for_package.extra_data.get("depends_on", []): try: - resolved_to_package = project_packages.get(extra_data__bom_ref=bom_ref) + resolved_to_package = project_packages.get(package_uid=bom_ref) except (ObjectDoesNotExist, MultipleObjectsReturned): project.add_error( description=f"Could not find resolved_to package entry: {bom_ref}.", @@ -284,8 +304,12 @@ def convert_spdx_expression(license_expression_spdx): return get_license_detections_and_expression(license_expression_spdx)[1] -def spdx_package_to_discovered_package_data(spdx_package): +def spdx_package_to_package_data(spdx_package): + """Convert the provided spdx_package into package_data.""" package_url_dict = {} + # Store the original "SPDXID" as package_uid for dependencies resolution. + package_uid = spdx_package.spdx_id + for ref in spdx_package.external_refs: if ref.type == "purl": purl = ref.locator @@ -302,6 +326,7 @@ def spdx_package_to_discovered_package_data(spdx_package): declared_expression = convert_spdx_expression(declared_license_expression_spdx) package_data = { + "package_uid": package_uid, "name": spdx_package.name, "download_url": spdx_package.download_location, "declared_license_expression": declared_expression, @@ -324,8 +349,28 @@ def spdx_package_to_discovered_package_data(spdx_package): } -def resolve_spdx_packages(input_location): - """Resolve the packages from the `input_location` SPDX document file.""" +def spdx_relationship_to_dependency_data(spdx_relationship): + """Convert the provided spdx_relationship into dependency_data.""" + # spdx_id is a dependency of related_spdx_id + if spdx_relationship.is_dependency_relationship: + for_package_uid = spdx_relationship.related_spdx_id + resolve_to_package_uid = spdx_relationship.spdx_id + else: # spdx_id depends on related_spdx_id + for_package_uid = spdx_relationship.spdx_id + resolve_to_package_uid = spdx_relationship.related_spdx_id + + dependency_data = { + "for_package_uid": for_package_uid, + "resolve_to_package_uid": resolve_to_package_uid, + "is_runtime": True, + "is_resolved": True, + "is_direct": True, + } + return dependency_data + + +def get_spdx_document_from_file(input_location): + """Return the loaded SPDX document from the `input_location` file.""" input_path = Path(input_location) spdx_document = json.loads(input_path.read_text()) @@ -334,12 +379,32 @@ def resolve_spdx_packages(input_location): except Exception as e: raise Exception(f'SPDX document "{input_path.name}" is not valid: {e}') + return spdx_document + + +def resolve_spdx_packages(input_location): + """Resolve the packages from the `input_location` SPDX document file.""" + spdx_document = get_spdx_document_from_file(input_location) return [ - spdx_package_to_discovered_package_data(spdx.Package.from_data(spdx_package)) + spdx_package_to_package_data(spdx.Package.from_data(spdx_package)) for spdx_package in spdx_document.get("packages", []) ] +def resolve_spdx_dependencies(input_location): + """Resolve the dependencies from the `input_location` SPDX document file.""" + spdx_document = get_spdx_document_from_file(input_location) + spdx_relationships = [ + spdx.Relationship.from_data(spdx_relationship) + for spdx_relationship in spdx_document.get("relationships", []) + ] + + return [ + spdx_relationship_to_dependency_data(spdx_relationship) + for spdx_relationship in spdx_relationships + ] + + def get_default_package_type(input_location): """ Return the package type associated with the provided `input_location`. diff --git a/scanpipe/pipes/spdx.py b/scanpipe/pipes/spdx.py index 9cd2285e5..6ab474f99 100644 --- a/scanpipe/pipes/spdx.py +++ b/scanpipe/pipes/spdx.py @@ -520,6 +520,18 @@ def from_data(cls, data): comment=data.get("comment"), ) + @property + def is_dependency_relationship(self): + """ + Return True if this relationship type implies that the spdx_id element + is a dependency of related_spdx_id. + """ + reverse_dependency_types = ["ANCESTOR_OF", "CONTAINS", "DEPENDS_ON"] + # Every others types implies that the spdx_id element is a dependency of + # related_spdx_id. Such as: + # "DEPENDENCY_OF", "DESCENDANT_OF", "PACKAGE_OF", "CONTAINED_BY", ... + return self.relationship.upper() not in reverse_dependency_types + @dataclass class Document: diff --git a/scanpipe/templates/scanpipe/tree/children.html b/scanpipe/templates/scanpipe/tree/children.html new file mode 100644 index 000000000..ba513c745 --- /dev/null +++ b/scanpipe/templates/scanpipe/tree/children.html @@ -0,0 +1,19 @@ + \ No newline at end of file diff --git a/scanpipe/templates/scanpipe/tree/node.html b/scanpipe/templates/scanpipe/tree/node.html new file mode 100644 index 000000000..e42f671b7 --- /dev/null +++ b/scanpipe/templates/scanpipe/tree/node.html @@ -0,0 +1,23 @@ +{% if node.name %} + {{ node.name }} +{% else %} + Missing data +{% endif %} + + + {% if node.url %} + + + + {% endif %} + {% if node.compliance_alert == "warning" or node.compliance_alert == "error" %} + + + + {% endif %} + {% if node.is_vulnerable %} + + + + {% endif %} + \ No newline at end of file diff --git a/scanpipe/tests/data/asgiref/asgiref-3.3.0.spdx.json b/scanpipe/tests/data/asgiref/asgiref-3.3.0.spdx.json index 07e04fd06..9d3c895c9 100644 --- a/scanpipe/tests/data/asgiref/asgiref-3.3.0.spdx.json +++ b/scanpipe/tests/data/asgiref/asgiref-3.3.0.spdx.json @@ -1,7 +1,7 @@ { "spdxVersion": "SPDX-2.3", "dataLicense": "CC0-1.0", - "SPDXID": "SPDXRef-DOCUMENT", + "SPDXID": "SPDXRef-DOCUMENT-2f5f5927-2cad-4ecb-9043-fda5337bd501", "name": "scancodeio_asgiref", "documentNamespace": "https://scancode.io/spdxdocs/1cdd3f3a-eea9-4c9c-b78e-9fa6bcde9cfd", "creationInfo": { diff --git a/scanpipe/tests/data/spdx/SPDXJSONExample-v2.3.spdx.json b/scanpipe/tests/data/spdx/SPDXJSONExample-v2.3.spdx.json new file mode 100644 index 000000000..4b2057dfe --- /dev/null +++ b/scanpipe/tests/data/spdx/SPDXJSONExample-v2.3.spdx.json @@ -0,0 +1,289 @@ +{ + "SPDXID" : "SPDXRef-DOCUMENT", + "spdxVersion" : "SPDX-2.3", + "creationInfo" : { + "comment" : "This package has been shipped in source and binary form.\nThe binaries were created with gcc 4.5.1 and expect to link to\ncompatible system run time libraries.", + "created" : "2010-01-29T18:30:22Z", + "creators" : [ "Tool: LicenseFind-1.0", "Organization: ExampleCodeInspect ()", "Person: Jane Doe ()" ], + "licenseListVersion" : "3.17" + }, + "name" : "SPDX-Tools-v2.0", + "dataLicense" : "CC0-1.0", + "comment" : "This document was created using SPDX 2.0 using licenses from the web site.", + "externalDocumentRefs" : [ { + "externalDocumentId" : "DocumentRef-spdx-tool-1.2", + "checksum" : { + "algorithm" : "SHA1", + "checksumValue" : "d6a770ba38583ed4bb4525bd96e50461655d2759" + }, + "spdxDocument" : "http://spdx.org/spdxdocs/spdx-tools-v1.2-3F2504E0-4F89-41D3-9A0C-0305E82C3301" + } ], + "hasExtractedLicensingInfos" : [ { + "licenseId" : "LicenseRef-1", + "extractedText" : "/*\n * (c) Copyright 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 Hewlett-Packard Development Company, LP\n * All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * 1. Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * 2. Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * 3. The name of the author may not be used to endorse or promote products\n * derived from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR\n * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES\n * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.\n * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,\n * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT\n * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF\n * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n*/" + }, { + "licenseId" : "LicenseRef-2", + "extractedText" : "This package includes the GRDDL parser developed by Hewlett Packard under the following license:\n© Copyright 2007 Hewlett-Packard Development Company, LP\n\nRedistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: \n\nRedistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. \nRedistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. \nThe name of the author may not be used to endorse or promote products derived from this software without specific prior written permission. \nTHIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE." + }, { + "licenseId" : "LicenseRef-4", + "extractedText" : "/*\n * (c) Copyright 2009 University of Bristol\n * All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * 1. Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer.\n * 2. Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * 3. The name of the author may not be used to endorse or promote products\n * derived from this software without specific prior written permission.\n *\n * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR\n * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES\n * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.\n * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,\n * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT\n * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF\n * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n*/" + }, { + "licenseId" : "LicenseRef-Beerware-4.2", + "comment" : "The beerware license has a couple of other standard variants.", + "extractedText" : "\"THE BEER-WARE LICENSE\" (Revision 42):\nphk@FreeBSD.ORG wrote this file. As long as you retain this notice you\ncan do whatever you want with this stuff. If we meet some day, and you think this stuff is worth it, you can buy me a beer in return Poul-Henning Kamp", + "name" : "Beer-Ware License (Version 42)", + "seeAlsos" : [ "http://people.freebsd.org/~phk/" ] + }, { + "licenseId" : "LicenseRef-3", + "comment" : "This is tye CyperNeko License", + "extractedText" : "The CyberNeko Software License, Version 1.0\n\n \n(C) Copyright 2002-2005, Andy Clark. All rights reserved.\n \nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions\nare met:\n\n1. Redistributions of source code must retain the above copyright\n notice, this list of conditions and the following disclaimer. \n\n2. Redistributions in binary form must reproduce the above copyright\n notice, this list of conditions and the following disclaimer in\n the documentation and/or other materials provided with the\n distribution.\n\n3. The end-user documentation included with the redistribution,\n if any, must include the following acknowledgment: \n \"This product includes software developed by Andy Clark.\"\n Alternately, this acknowledgment may appear in the software itself,\n if and wherever such third-party acknowledgments normally appear.\n\n4. The names \"CyberNeko\" and \"NekoHTML\" must not be used to endorse\n or promote products derived from this software without prior \n written permission. For written permission, please contact \n andyc@cyberneko.net.\n\n5. Products derived from this software may not be called \"CyberNeko\",\n nor may \"CyberNeko\" appear in their name, without prior written\n permission of the author.\n\nTHIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED\nWARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES\nOF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR OTHER CONTRIBUTORS\nBE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, \nOR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT \nOF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR \nBUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, \nWHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE \nOR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, \nEVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.", + "name" : "CyberNeko License", + "seeAlsos" : [ "http://people.apache.org/~andyc/neko/LICENSE", "http://justasample.url.com" ] + } ], + "annotations" : [ { + "annotationDate" : "2010-01-29T18:30:22Z", + "annotationType" : "OTHER", + "annotator" : "Person: Jane Doe ()", + "comment" : "Document level annotation" + }, { + "annotationDate" : "2010-02-10T00:00:00Z", + "annotationType" : "REVIEW", + "annotator" : "Person: Joe Reviewer", + "comment" : "This is just an example. Some of the non-standard licenses look like they are actually BSD 3 clause licenses" + }, { + "annotationDate" : "2011-03-13T00:00:00Z", + "annotationType" : "REVIEW", + "annotator" : "Person: Suzanne Reviewer", + "comment" : "Another example reviewer." + } ], + "documentDescribes" : [ "SPDXRef-File", "SPDXRef-Package" ], + "documentNamespace" : "http://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301", + "packages" : [ { + "SPDXID" : "SPDXRef-Package", + "annotations" : [ { + "annotationDate" : "2011-01-29T18:30:22Z", + "annotationType" : "OTHER", + "annotator" : "Person: Package Commenter", + "comment" : "Package level annotation" + } ], + "attributionTexts" : [ "The GNU C Library is free software. See the file COPYING.LIB for copying conditions, and LICENSES for notices about a few contributions that require these additional notices to be distributed. License copyright years may be listed using range notation, e.g., 1996-2015, indicating that every year in the range, inclusive, is a copyrightable year that would otherwise be listed individually." ], + "builtDate" : "2011-01-29T18:30:22Z", + "checksums" : [ { + "algorithm" : "MD5", + "checksumValue" : "624c1abb3664f4b35547e7c73864ad24" + }, { + "algorithm" : "SHA1", + "checksumValue" : "85ed0817af83a24ad8da68c2b5094de69833983c" + }, { + "algorithm" : "SHA256", + "checksumValue" : "11b6d3ee554eedf79299905a98f9b9a04e498210b59f15094c916c91d150efcd" + }, { + "algorithm" : "BLAKE2b-384", + "checksumValue" : "aaabd89c926ab525c242e6621f2f5fa73aa4afe3d9e24aed727faaadd6af38b620bdb623dd2b4788b1c8086984af8706" + } ], + "copyrightText" : "Copyright 2008-2010 John Smith", + "description" : "The GNU C Library defines functions that are specified by the ISO C standard, as well as additional features specific to POSIX and other derivatives of the Unix operating system, and extensions specific to GNU systems.", + "downloadLocation" : "http://ftp.gnu.org/gnu/glibc/glibc-ports-2.15.tar.gz", + "externalRefs" : [ { + "referenceCategory" : "SECURITY", + "referenceLocator" : "cpe:2.3:a:pivotal_software:spring_framework:4.1.0:*:*:*:*:*:*:*", + "referenceType" : "cpe23Type" + }, { + "comment" : "This is the external ref for Acme", + "referenceCategory" : "OTHER", + "referenceLocator" : "acmecorp/acmenator/4.1.3-alpha", + "referenceType" : "http://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301#LocationRef-acmeforge" + } ], + "filesAnalyzed" : true, + "homepage" : "http://ftp.gnu.org/gnu/glibc", + "licenseComments" : "The license for this project changed with the release of version x.y. The version of the project included here post-dates the license change.", + "licenseConcluded" : "(LGPL-2.0-only OR LicenseRef-3)", + "licenseDeclared" : "(LGPL-2.0-only AND LicenseRef-3)", + "licenseInfoFromFiles" : [ "GPL-2.0-only", "LicenseRef-2", "LicenseRef-1" ], + "name" : "glibc", + "originator" : "Organization: ExampleCodeInspect (contact@example.com)", + "packageFileName" : "glibc-2.11.1.tar.gz", + "packageVerificationCode" : { + "packageVerificationCodeExcludedFiles" : [ "./package.spdx" ], + "packageVerificationCodeValue" : "d6a770ba38583ed4bb4525bd96e50461655d2758" + }, + "primaryPackagePurpose" : "SOURCE", + "hasFiles" : [ "SPDXRef-Specification", "SPDXRef-Specification", "SPDXRef-CommonsLangSrc", "SPDXRef-Specification", "SPDXRef-CommonsLangSrc", "SPDXRef-JenaLib", "SPDXRef-Specification", "SPDXRef-CommonsLangSrc", "SPDXRef-JenaLib", "SPDXRef-DoapSource", "SPDXRef-Specification", "SPDXRef-CommonsLangSrc", "SPDXRef-JenaLib", "SPDXRef-DoapSource" ], + "releaseDate" : "2012-01-29T18:30:22Z", + "sourceInfo" : "uses glibc-2_11-branch from git://sourceware.org/git/glibc.git.", + "summary" : "GNU C library.", + "supplier" : "Person: Jane Doe (jane.doe@example.com)", + "validUntilDate" : "2014-01-29T18:30:22Z", + "versionInfo" : "2.11.1" + }, { + "SPDXID" : "SPDXRef-fromDoap-1", + "copyrightText" : "NOASSERTION", + "downloadLocation" : "NOASSERTION", + "filesAnalyzed" : false, + "homepage" : "http://commons.apache.org/proper/commons-lang/", + "licenseConcluded" : "NOASSERTION", + "licenseDeclared" : "NOASSERTION", + "name" : "Apache Commons Lang" + }, { + "SPDXID" : "SPDXRef-fromDoap-0", + "downloadLocation" : "https://search.maven.org/remotecontent?filepath=org/apache/jena/apache-jena/3.12.0/apache-jena-3.12.0.tar.gz", + "externalRefs" : [ { + "referenceCategory" : "PACKAGE-MANAGER", + "referenceLocator" : "pkg:maven/org.apache.jena/apache-jena@3.12.0", + "referenceType" : "purl" + } ], + "filesAnalyzed" : false, + "homepage" : "http://www.openjena.org/", + "name" : "Jena", + "versionInfo" : "3.12.0" + }, { + "SPDXID" : "SPDXRef-Saxon", + "checksums" : [ { + "algorithm" : "SHA1", + "checksumValue" : "85ed0817af83a24ad8da68c2b5094de69833983c" + } ], + "copyrightText" : "Copyright Saxonica Ltd", + "description" : "The Saxon package is a collection of tools for processing XML documents.", + "downloadLocation" : "https://sourceforge.net/projects/saxon/files/Saxon-B/8.8.0.7/saxonb8-8-0-7j.zip/download", + "filesAnalyzed" : false, + "homepage" : "http://saxon.sourceforge.net/", + "licenseComments" : "Other versions available for a commercial license", + "licenseConcluded" : "MPL-1.0", + "licenseDeclared" : "MPL-1.0", + "name" : "Saxon", + "packageFileName" : "saxonB-8.8.zip", + "versionInfo" : "8.8" + } ], + "files" : [ { + "SPDXID" : "SPDXRef-DoapSource", + "checksums" : [ { + "algorithm" : "SHA1", + "checksumValue" : "2fd4e1c67a2d28fced849ee1bb76e7391b93eb12" + } ], + "copyrightText" : "Copyright 2010, 2011 Source Auditor Inc.", + "fileContributors" : [ "Protecode Inc.", "SPDX Technical Team Members", "Open Logic Inc.", "Source Auditor Inc.", "Black Duck Software In.c" ], + "fileName" : "./src/org/spdx/parser/DOAPProject.java", + "fileTypes" : [ "SOURCE" ], + "licenseConcluded" : "Apache-2.0", + "licenseInfoInFiles" : [ "Apache-2.0" ] + }, { + "SPDXID" : "SPDXRef-CommonsLangSrc", + "checksums" : [ { + "algorithm" : "SHA1", + "checksumValue" : "c2b4e1c67a2d28fced849ee1bb76e7391b93f125" + } ], + "comment" : "This file is used by Jena", + "copyrightText" : "Copyright 2001-2011 The Apache Software Foundation", + "fileContributors" : [ "Apache Software Foundation" ], + "fileName" : "./lib-source/commons-lang3-3.1-sources.jar", + "fileTypes" : [ "ARCHIVE" ], + "licenseConcluded" : "Apache-2.0", + "licenseInfoInFiles" : [ "Apache-2.0" ], + "noticeText" : "Apache Commons Lang\nCopyright 2001-2011 The Apache Software Foundation\n\nThis product includes software developed by\nThe Apache Software Foundation (http://www.apache.org/).\n\nThis product includes software from the Spring Framework,\nunder the Apache License 2.0 (see: StringUtils.containsWhitespace())" + }, { + "SPDXID" : "SPDXRef-JenaLib", + "checksums" : [ { + "algorithm" : "SHA1", + "checksumValue" : "3ab4e1c67a2d28fced849ee1bb76e7391b93f125" + } ], + "comment" : "This file belongs to Jena", + "copyrightText" : "(c) Copyright 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 Hewlett-Packard Development Company, LP", + "fileContributors" : [ "Apache Software Foundation", "Hewlett Packard Inc." ], + "fileName" : "./lib-source/jena-2.6.3-sources.jar", + "fileTypes" : [ "ARCHIVE" ], + "licenseComments" : "This license is used by Jena", + "licenseConcluded" : "LicenseRef-1", + "licenseInfoInFiles" : [ "LicenseRef-1" ] + }, { + "SPDXID" : "SPDXRef-Specification", + "checksums" : [ { + "algorithm" : "SHA1", + "checksumValue" : "fff4e1c67a2d28fced849ee1bb76e7391b93f125" + } ], + "comment" : "Specification Documentation", + "fileName" : "./docs/myspec.pdf", + "fileTypes" : [ "DOCUMENTATION" ] + }, { + "SPDXID" : "SPDXRef-File", + "annotations" : [ { + "annotationDate" : "2011-01-29T18:30:22Z", + "annotationType" : "OTHER", + "annotator" : "Person: File Commenter", + "comment" : "File level annotation" + } ], + "checksums" : [ { + "algorithm" : "SHA1", + "checksumValue" : "d6a770ba38583ed4bb4525bd96e50461655d2758" + }, { + "algorithm" : "MD5", + "checksumValue" : "624c1abb3664f4b35547e7c73864ad24" + } ], + "comment" : "The concluded license was taken from the package level that the file was included in.\nThis information was found in the COPYING.txt file in the xyz directory.", + "copyrightText" : "Copyright 2008-2010 John Smith", + "fileContributors" : [ "The Regents of the University of California", "Modified by Paul Mundt lethal@linux-sh.org", "IBM Corporation" ], + "fileName" : "./package/foo.c", + "fileTypes" : [ "SOURCE" ], + "licenseComments" : "The concluded license was taken from the package level that the file was included in.", + "licenseConcluded" : "(LGPL-2.0-only OR LicenseRef-2)", + "licenseInfoInFiles" : [ "GPL-2.0-only", "LicenseRef-2" ], + "noticeText" : "Copyright (c) 2001 Aaron Lehmann aaroni@vitelus.com\n\nPermission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \"Software\"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: \nThe above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE." + } ], + "snippets" : [ { + "SPDXID" : "SPDXRef-Snippet", + "comment" : "This snippet was identified as significant and highlighted in this Apache-2.0 file, when a commercial scanner identified it as being derived from file foo.c in package xyz which is licensed under GPL-2.0.", + "copyrightText" : "Copyright 2008-2010 John Smith", + "licenseComments" : "The concluded license was taken from package xyz, from which the snippet was copied into the current file. The concluded license information was found in the COPYING.txt file in package xyz.", + "licenseConcluded" : "GPL-2.0-only", + "licenseInfoInSnippets" : [ "GPL-2.0-only" ], + "name" : "from linux kernel", + "ranges" : [ { + "endPointer" : { + "offset" : 420, + "reference" : "SPDXRef-DoapSource" + }, + "startPointer" : { + "offset" : 310, + "reference" : "SPDXRef-DoapSource" + } + }, { + "endPointer" : { + "lineNumber" : 23, + "reference" : "SPDXRef-DoapSource" + }, + "startPointer" : { + "lineNumber" : 5, + "reference" : "SPDXRef-DoapSource" + } + } ], + "snippetFromFile" : "SPDXRef-DoapSource" + } ], + "relationships" : [ { + "spdxElementId" : "SPDXRef-DOCUMENT", + "relationshipType" : "CONTAINS", + "relatedSpdxElement" : "SPDXRef-Package" + }, { + "spdxElementId" : "SPDXRef-DOCUMENT", + "relationshipType" : "COPY_OF", + "relatedSpdxElement" : "DocumentRef-spdx-tool-1.2:SPDXRef-ToolsElement" + }, { + "spdxElementId" : "SPDXRef-Package", + "relationshipType" : "DYNAMIC_LINK", + "relatedSpdxElement" : "SPDXRef-Saxon" + }, { + "spdxElementId" : "SPDXRef-CommonsLangSrc", + "relationshipType" : "GENERATED_FROM", + "relatedSpdxElement" : "NOASSERTION" + }, { + "spdxElementId" : "SPDXRef-JenaLib", + "relationshipType" : "CONTAINS", + "relatedSpdxElement" : "SPDXRef-Package" + }, { + "spdxElementId" : "SPDXRef-Specification", + "relationshipType" : "SPECIFICATION_FOR", + "relatedSpdxElement" : "SPDXRef-fromDoap-0" + }, { + "spdxElementId" : "SPDXRef-File", + "relationshipType" : "GENERATED_FROM", + "relatedSpdxElement" : "SPDXRef-fromDoap-0" + } ] +} \ No newline at end of file diff --git a/scanpipe/tests/data/spdx/dependencies.spdx.json b/scanpipe/tests/data/spdx/dependencies.spdx.json new file mode 100644 index 000000000..bd20db1a2 --- /dev/null +++ b/scanpipe/tests/data/spdx/dependencies.spdx.json @@ -0,0 +1,112 @@ +{ + "spdxVersion": "SPDX-2.3", + "dataLicense": "CC0-1.0", + "SPDXID": "SPDXRef-DOCUMENT-b74fe5df-e965-415e-ba65-f38421a0695d", + "name": "scancodeio_analysis", + "documentNamespace": "https://scancode.io/spdxdocs/b74fe5df-e965-415e-ba65-f38421a0695d", + "creationInfo": { + "created": "2000-01-01T01:02:03Z", + "creators": [ + "Tool: ScanCode.io" + ], + "licenseListVersion": "3.20" + }, + "packages": [ + { + "name": "a", + "SPDXID": "SPDXRef-scancodeio-discoveredpackage-a83a60de-81bc-4bf4-b48c-dc78e0e658a9", + "downloadLocation": "NOASSERTION", + "licenseConcluded": "NOASSERTION", + "copyrightText": "NOASSERTION", + "filesAnalyzed": false, + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:type/a" + } + ] + }, + { + "name": "b", + "SPDXID": "SPDXRef-scancodeio-discoveredpackage-81147701-285f-485c-ba36-9cd3742790b1", + "downloadLocation": "NOASSERTION", + "licenseConcluded": "NOASSERTION", + "copyrightText": "NOASSERTION", + "filesAnalyzed": false, + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:type/b" + } + ] + }, + { + "name": "z", + "SPDXID": "SPDXRef-scancodeio-discoveredpackage-e391c33e-d7d0-4a97-a3c3-e947375c53d5", + "downloadLocation": "NOASSERTION", + "licenseConcluded": "NOASSERTION", + "copyrightText": "NOASSERTION", + "filesAnalyzed": false, + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:type/z" + } + ] + }, + { + "name": "", + "SPDXID": "SPDXRef-scancodeio-discovereddependency-for_package_b", + "downloadLocation": "NOASSERTION", + "licenseConcluded": "NOASSERTION", + "copyrightText": "NOASSERTION", + "filesAnalyzed": false, + "licenseDeclared": "NOASSERTION" + }, + { + "name": "unresolved", + "SPDXID": "SPDXRef-scancodeio-discovereddependency-unresolved", + "downloadLocation": "NOASSERTION", + "licenseConcluded": "NOASSERTION", + "copyrightText": "NOASSERTION", + "filesAnalyzed": false, + "licenseDeclared": "NOASSERTION", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:type/unresolved" + } + ] + } + ], + "documentDescribes": [ + "SPDXRef-scancodeio-discoveredpackage-a83a60de-81bc-4bf4-b48c-dc78e0e658a9", + "SPDXRef-scancodeio-discoveredpackage-81147701-285f-485c-ba36-9cd3742790b1", + "SPDXRef-scancodeio-discoveredpackage-e391c33e-d7d0-4a97-a3c3-e947375c53d5", + "SPDXRef-scancodeio-discovereddependency-for_package_b", + "SPDXRef-scancodeio-discovereddependency-unresolved" + ], + "relationships": [ + { + "spdxElementId": "SPDXRef-scancodeio-discoveredpackage-81147701-285f-485c-ba36-9cd3742790b1", + "relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-a83a60de-81bc-4bf4-b48c-dc78e0e658a9", + "relationshipType": "DEPENDENCY_OF" + }, + { + "spdxElementId": "SPDXRef-scancodeio-discovereddependency-for_package_b", + "relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-81147701-285f-485c-ba36-9cd3742790b1", + "relationshipType": "DEPENDENCY_OF" + }, + { + "spdxElementId": "SPDXRef-scancodeio-discovereddependency-unresolved", + "relatedSpdxElement": "SPDXRef-DOCUMENT-b74fe5df-e965-415e-ba65-f38421a0695d", + "relationshipType": "DEPENDENCY_OF" + } + ], + "comment": "Generated with ScanCode.io and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied.\nNo content created from ScanCode.io should be considered or used as legal advice.\nConsult an Attorney for any legal advice.\nScanCode.io is a free software code scanning tool from nexB Inc. and others\nlicensed under the Apache License version 2.0.\nScanCode is a trademark of nexB Inc.\nVisit https://github.com/nexB/scancode.io for support and download.\n", + "files": [] +} \ No newline at end of file diff --git a/scanpipe/tests/pipes/test_output.py b/scanpipe/tests/pipes/test_output.py index 5b7b217e6..6013db05b 100644 --- a/scanpipe/tests/pipes/test_output.py +++ b/scanpipe/tests/pipes/test_output.py @@ -465,6 +465,47 @@ def test_scanpipe_pipes_outputs_to_spdx_extracted_licenses(self): self.assertEqual(expected, license_infos["seeAlsos"]) self.assertTrue(license_infos["extractedText"].startswith("License:")) + @mock.patch("uuid.uuid4") + def test_scanpipe_pipes_outputs_to_spdx_dependencies(self, mock_uuid4): + forced_uuid = "b74fe5df-e965-415e-ba65-f38421a0695d" + mock_uuid4.return_value = forced_uuid + project = Project.objects.create(name="Analysis", uuid=forced_uuid) + + a = make_package( + project, "pkg:type/a", uuid="a83a60de-81bc-4bf4-b48c-dc78e0e658a9" + ) + b = make_package( + project, "pkg:type/b", uuid="81147701-285f-485c-ba36-9cd3742790b1" + ) + # 1. Package resolved dependency + make_dependency(project, for_package=a, resolved_to_package=b) + # 2. Package unresolved dependency + make_dependency(project, for_package=b, dependency_uid="for_package_b") + # 3. Project unresolved dependency + unresolved_dependency = make_dependency(project, dependency_uid="unresolved") + unresolved_dependency.set_package_url("pkg:type/unresolved") + unresolved_dependency.save() + # 4. Project package + make_package(project, "pkg:type/z", uuid="e391c33e-d7d0-4a97-a3c3-e947375c53d5") + + self.assertEqual(3, project.discoveredpackages.count()) + self.assertEqual(3, project.discovereddependencies.count()) + + output_file = output.to_spdx(project=project) + results_json = json.loads(output_file.read_text()) + self.assertEqual(5, len(results_json["packages"])) + self.assertEqual(3, len(results_json["relationships"])) + + # Patch the `created` date and tool version + results_json["creationInfo"]["created"] = "2000-01-01T01:02:03Z" + results_json["creationInfo"]["creators"] = ["Tool: ScanCode.io"] + # Files ordering is system dependent, excluded for now + results_json["files"] = [] + results = json.dumps(results_json, indent=2) + + expected_file = self.data / "spdx" / "dependencies.spdx.json" + self.assertResultsEqual(expected_file, results) + def test_scanpipe_pipes_outputs_make_unknown_license_object(self): licensing = get_licensing() parsed_expression = licensing.parse("some-unknown-license") diff --git a/scanpipe/tests/pipes/test_resolve.py b/scanpipe/tests/pipes/test_resolve.py index 239860668..0c2170544 100644 --- a/scanpipe/tests/pipes/test_resolve.py +++ b/scanpipe/tests/pipes/test_resolve.py @@ -29,6 +29,7 @@ from scanpipe import pipes from scanpipe.models import Project from scanpipe.pipes import resolve +from scanpipe.pipes import spdx from scanpipe.pipes.input import copy_inputs from scanpipe.pipes.scancode import extract_archives from scanpipe.tests import make_package @@ -173,12 +174,20 @@ def test_scanpipe_pipes_resolve_resolve_about_packages(self): expected = {"extra_data": {}, "name": "project"} self.assertEqual([expected], package) - def test_scanpipe_pipes_resolve_spdx_package_to_discovered_package_data(self): + def test_scanpipe_pipes_resolve_get_spdx_document_from_file(self): + input_location = self.data / "spdx" / "SPDXJSONExample-v2.3.spdx.json" + spdx_document = resolve.get_spdx_document_from_file(input_location) + self.assertIsInstance(spdx_document, dict) + self.assertEqual("SPDXRef-DOCUMENT", spdx_document["SPDXID"]) + self.assertEqual("SPDX-2.3", spdx_document["spdxVersion"]) + + def test_scanpipe_pipes_resolve_spdx_package_to_package_data(self): p1 = Project.objects.create(name="Analysis") package = pipes.update_or_create_package(p1, package_data1) package_spdx = package.as_spdx() - package_data = resolve.spdx_package_to_discovered_package_data(package_spdx) + package_data = resolve.spdx_package_to_package_data(package_spdx) expected = { + "package_uid": package.spdx_id, "name": "adduser", "download_url": "https://download.url/package.zip", "declared_license_expression": "gpl-2.0 AND gpl-2.0-plus", @@ -202,6 +211,35 @@ def test_scanpipe_pipes_resolve_spdx_package_to_discovered_package_data(self): } self.assertEqual(expected, package_data) + def test_scanpipe_pipes_spdx_relationship_to_dependency_data(self): + spdx_relationship_data = { + "spdxElementId": "SPDXRef-package1", + "relatedSpdxElement": "SPDXRef-package2", + "relationshipType": "CONTAINS", + } + spdx_relationship = spdx.Relationship.from_data(spdx_relationship_data) + dependency_data = resolve.spdx_relationship_to_dependency_data( + spdx_relationship + ) + expected = { + "for_package_uid": "SPDXRef-package1", + "resolve_to_package_uid": "SPDXRef-package2", + "is_runtime": True, + "is_resolved": True, + "is_direct": True, + } + self.assertEqual(expected, dependency_data) + + def test_scanpipe_pipes_resolve_spdx_packages(self): + input_location = self.data / "spdx" / "SPDXJSONExample-v2.3.spdx.json" + packages_data = resolve.resolve_spdx_packages(input_location) + self.assertEqual(4, len(packages_data)) + + def test_scanpipe_pipes_resolve_spdx_dependencies(self): + input_location = self.data / "spdx" / "SPDXJSONExample-v2.3.spdx.json" + dependencies_data = resolve.resolve_spdx_dependencies(input_location) + self.assertEqual(7, len(dependencies_data)) + def test_scanpipe_resolve_get_manifest_resources(self): project1 = Project.objects.create(name="Analysis") input_location = self.data / "manifests" / "python-inspector-0.10.0.zip" diff --git a/scanpipe/tests/pipes/test_spdx.py b/scanpipe/tests/pipes/test_spdx.py index 1c62aeaa9..7bb68fcde 100644 --- a/scanpipe/tests/pipes/test_spdx.py +++ b/scanpipe/tests/pipes/test_spdx.py @@ -351,6 +351,12 @@ def test_spdx_relationship_from_data(self): relationship = spdx.Relationship.from_data(self.relationship_spdx_data) assert self.relationship_spdx_data == relationship.as_dict() + def test_spdx_relationship_is_dependency_relationship_property(self): + relationship = spdx.Relationship.from_data(self.relationship_spdx_data) + assert relationship.is_dependency_relationship is False + relationship.relationship = "DEPENDENCY_OF" + assert relationship.is_dependency_relationship + def test_spdx_document_as_dict(self): document = spdx.Document(**self.document_data) assert self.document_spdx_data == document.as_dict()