support-python-private-classifier

src/packagedcode/pypi.py

@@ -511,6 +511,9 @@ def parse(cls, location, package_only=False):
         description = project_data.get('description') or ''
         description = description.strip()
 
+        classifiers = project_data.get('classifiers', [])
+        is_private = any('Private ::' in classifier for classifier in classifiers)
+
         urls, extra_data = get_urls(metainfo=project_data, name=name, version=version)
 
         extracted_license_statement, license_file = get_declared_license(project_data)
@@ -542,8 +545,10 @@ def parse(cls, location, package_only=False):
             keywords=get_keywords(project_data),
             parties=get_pyproject_toml_parties(project_data),
             dependencies=dependencies,
+            is_private=is_private,
             extra_data=extra_data,
             **urls,
+            download_url=urls.get('download'),
         )
         yield models.PackageData.from_data(package_data, package_only)
 
@@ -2281,7 +2286,7 @@ def get_pypi_urls(name, version, **kwargs):
     )
 
 
-def get_urls(metainfo, name, version, poetry=False):
+def get_urls(metainfo, name, version,is_private, poetry=False):
     """
     Return a mapping of standard URLs and a mapping of extra-data URls for URLs
     of this package:
@@ -2323,7 +2328,10 @@ def get_urls(metainfo, name, version, poetry=False):
     # Project-URL: Say Thanks!
 
     extra_data = {}
-    urls = get_pypi_urls(name, version)
+    if not is_private:
+       urls = get_pypi_urls(name, version)
+    else:
+        urls = {}
 
     def add_url(_url, _utype=None, _attribute=None):
         """

tests/packagedcode/data/pypi/pyproject-toml/standard/private-classifier-pyproject.toml-expected.json

@@ -0,0 +1,114 @@
+[
+  {
+    "type": "pypi",
+    "namespace": null,
+    "name": "titanic_ml",
+    "version": "0.1.0",
+    "qualifiers": {},
+    "subpath": null,
+    "primary_language": "Python",
+    "description": "titanic_ml example package",
+    "release_date": null,
+    "parties": [
+      {
+        "type": "person",
+        "role": "author",
+        "name": "Niels Zeilemaker",
+        "email": "nielszeilemaker@xebia.com",
+        "url": null
+      }
+    ],
+    "keywords": [
+      "Development Status :: 4 - Beta",
+      "Intended Audience :: Developers",
+      "Private :: Do Not Upload"
+    ],
+    "homepage_url": null,
+    "download_url": null,
+    "size": null,
+    "sha1": null,
+    "md5": null,
+    "sha256": null,
+    "sha512": null,
+    "bug_tracking_url": null,
+    "code_view_url": null,
+    "vcs_url": null,
+    "copyright": null,
+    "holder": null,
+    "declared_license_expression": null,
+    "declared_license_expression_spdx": null,
+    "license_detections": [],
+    "other_license_expression": null,
+    "other_license_expression_spdx": null,
+    "other_license_detections": [],
+    "extracted_license_statement": null,
+    "notice_text": null,
+    "source_packages": [],
+    "file_references": [],
+    "is_private": true,
+    "is_virtual": false,
+    "extra_data": {},
+    "dependencies": [
+      {
+        "purl": "pkg:pypi/pyspark",
+        "extracted_requirement": null,
+        "scope": "install",
+        "is_runtime": true,
+        "is_optional": false,
+        "is_pinned": false,
+        "is_direct": true,
+        "resolved_package": {},
+        "extra_data": {}
+      },
+      {
+        "purl": "pkg:pypi/sklearn",
+        "extracted_requirement": null,
+        "scope": "install",
+        "is_runtime": true,
+        "is_optional": false,
+        "is_pinned": false,
+        "is_direct": true,
+        "resolved_package": {},
+        "extra_data": {}
+      },
+      {
+        "purl": "pkg:pypi/tox",
+        "extracted_requirement": null,
+        "scope": "dev",
+        "is_runtime": true,
+        "is_optional": true,
+        "is_pinned": false,
+        "is_direct": true,
+        "resolved_package": {},
+        "extra_data": {}
+      },
+      {
+        "purl": "pkg:pypi/pre-commit",
+        "extracted_requirement": null,
+        "scope": "dev",
+        "is_runtime": true,
+        "is_optional": true,
+        "is_pinned": false,
+        "is_direct": true,
+        "resolved_package": {},
+        "extra_data": {}
+      },
+      {
+        "purl": "pkg:pypi/bump2version",
+        "extracted_requirement": null,
+        "scope": "dev",
+        "is_runtime": true,
+        "is_optional": true,
+        "is_pinned": false,
+        "is_direct": true,
+        "resolved_package": {},
+        "extra_data": {}
+      }
+    ],
+    "repository_homepage_url": null,
+    "repository_download_url": null,
+    "api_data_url": null,
+    "datasource_id": "pypi_pyproject_toml",
+    "purl": "pkg:pypi/titanic-ml@0.1.0"
+  }
+]

tests/packagedcode/data/pypi/pyproject-toml/standard/python-private-classifier/pyproject.toml

@@ -0,0 +1,29 @@
+#Taken from : https://xebia.com/blog/minimal-pyproject-toml-example/
+
+[project]
+name = "titanic_ml"
+description = "titanic_ml example package"
+version = "0.1.0"
+authors = [
+    { name = "Niels Zeilemaker", email = "nielszeilemaker@xebia.com" }
+]
+dependencies = [
+    "pyspark[ml]",
+    "sklearn"
+]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Private :: Do Not Upload"
+]
+
+[project.optional-dependencies]
+dev = [
+    "tox",
+    "pre-commit",
+    "bump2version"
+]
+
+[build-system]
+build-backend = "flit_core.buildapi"
+requires = ["flit_core >=3.2,<4"]

tests/packagedcode/test_pypi.py

@@ -339,6 +339,12 @@ def test_parse_pyproject_toml_standard_lc0(self):
         expected_loc = self.get_test_loc('pypi/pyproject-toml/standard/lc0-pyproject.toml-expected.json')
         self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES)
 
+    def test_parse_pyproject_toml_private_package(self):
+        test_file = self.get_test_loc('pypi/pyproject-toml/standard/python-private-classifier/pyproject.toml') 
+        packages = pypi.PyprojectTomlHandler.parse(test_file)
+        expected_loc = self.get_test_loc('pypi/pyproject-toml/standard/private-classifier-pyproject.toml-expected.json') 
+        self.check_packages_data(packages, expected_loc,regen=REGEN_TEST_FIXTURES)
+
 
 class TestPoetryHandler(PackageTester):