From e03db23f1a9bbe53a28c198a21259983cffc5075 Mon Sep 17 00:00:00 2001 From: Varsha U N Date: Mon, 6 Jan 2025 21:10:55 +0530 Subject: [PATCH 1/9] support-private-classifier Signed-off-by: Varsha U N Signed-off-by: Varsha U N Signed-off-by: Varsha U N --- src/packagedcode/pypi.py | 5 +++ ...te-classifier-pyproject.toml-expected.json | 32 +++++++++++++++++++ .../python-private-classifier/pyproject.toml | 29 +++++++++++++++++ tests/packagedcode/test_pypi.py | 6 ++++ 4 files changed, 72 insertions(+) create mode 100644 tests/packagedcode/data/pypi/pyproject-toml/standard/private-classifier-pyproject.toml-expected.json create mode 100644 tests/packagedcode/data/pypi/pyproject-toml/standard/python-private-classifier/pyproject.toml diff --git a/src/packagedcode/pypi.py b/src/packagedcode/pypi.py index 271e98a69e..7d97dabe18 100644 --- a/src/packagedcode/pypi.py +++ b/src/packagedcode/pypi.py @@ -517,6 +517,10 @@ def parse(cls, location, package_only=False): if license_file: extra_data['license_file'] = license_file + classifiers = project_data.get('classifiers', []) + is_private = any('Private ::' in classifier for classifier in classifiers) + cls.is_private=is_private + dependencies = [] parsed_dependencies = get_requires_dependencies( requires=project_data.get("dependencies", []), @@ -542,6 +546,7 @@ def parse(cls, location, package_only=False): keywords=get_keywords(project_data), parties=get_pyproject_toml_parties(project_data), dependencies=dependencies, + is_private=is_private, extra_data=extra_data, **urls, ) diff --git a/tests/packagedcode/data/pypi/pyproject-toml/standard/private-classifier-pyproject.toml-expected.json b/tests/packagedcode/data/pypi/pyproject-toml/standard/private-classifier-pyproject.toml-expected.json new file mode 100644 index 0000000000..ef7fe126ac --- /dev/null +++ b/tests/packagedcode/data/pypi/pyproject-toml/standard/private-classifier-pyproject.toml-expected.json @@ -0,0 +1,32 @@ +[ + { + "datasource_id": "pypi_pyproject_toml", + "type": "pypi", + "primary_language": "Python", + "name": "titanic_ml", + "version": "0.1.0", + "extracted_license_statement": null, + "description": "titanic_ml example package", + "keywords": [], + "parties": [ + { + "type": "author", + "name": "Niels Zeilemaker", + "email": "nielszeilemaker@xebia.com" + } + ], + "dependencies": [ + { + "dependency": "pyspark[ml]", + "version": "" + }, + { + "dependency": "sklearn", + "version": "" + } + ], + "is_private": true, + "extra_data": {}, + "urls": {} + } +] diff --git a/tests/packagedcode/data/pypi/pyproject-toml/standard/python-private-classifier/pyproject.toml b/tests/packagedcode/data/pypi/pyproject-toml/standard/python-private-classifier/pyproject.toml new file mode 100644 index 0000000000..d03383271f --- /dev/null +++ b/tests/packagedcode/data/pypi/pyproject-toml/standard/python-private-classifier/pyproject.toml @@ -0,0 +1,29 @@ +#Taken from : https://xebia.com/blog/minimal-pyproject-toml-example/ + +[project] +name = "titanic_ml" +description = "titanic_ml example package" +version = "0.1.0" +authors = [ + { name = "Niels Zeilemaker", email = "nielszeilemaker@xebia.com" } +] +dependencies = [ + "pyspark[ml]", + "sklearn" +] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Private :: Do Not Upload" +] + +[project.optional-dependencies] +dev = [ + "tox", + "pre-commit", + "bump2version" +] + +[build-system] +build-backend = "flit_core.buildapi" +requires = ["flit_core >=3.2,<4"] diff --git a/tests/packagedcode/test_pypi.py b/tests/packagedcode/test_pypi.py index 37bd204327..56b5b4f2d2 100644 --- a/tests/packagedcode/test_pypi.py +++ b/tests/packagedcode/test_pypi.py @@ -339,6 +339,12 @@ def test_parse_pyproject_toml_standard_lc0(self): expected_loc = self.get_test_loc('pypi/pyproject-toml/standard/lc0-pyproject.toml-expected.json') self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES) + def test_parse_pyproject_toml_private_package(self): + test_file = self.get_test_loc('pypi/pyproject-toml/standard/python-private-classifier/pyproject.toml') + packages = pypi.PyprojectTomlHandler.parse(test_file) + expected_loc = self.get_test_loc('pypi/pyproject-toml/standard/private-classifier-pyproject.toml-expected.json') + self.check_packages_data(packages, expected_loc, regen=REGEN_TEST_FIXTURES) + class TestPoetryHandler(PackageTester): From c941e59793fbba2fe0602785daf2dde3d8a5d93e Mon Sep 17 00:00:00 2001 From: Varsha U N Date: Wed, 26 Feb 2025 20:22:59 +0530 Subject: [PATCH 2/9] Fix tests Signed-off-by: Varsha U N --- src/packagedcode/pypi.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/packagedcode/pypi.py b/src/packagedcode/pypi.py index 7d97dabe18..1aa503b839 100644 --- a/src/packagedcode/pypi.py +++ b/src/packagedcode/pypi.py @@ -517,10 +517,8 @@ def parse(cls, location, package_only=False): if license_file: extra_data['license_file'] = license_file - classifiers = project_data.get('classifiers', []) - is_private = any('Private ::' in classifier for classifier in classifiers) - cls.is_private=is_private - + classifiers = project_data.get('classifiers', []) + is_private = any('Private ::' in classifier for classifier in classifiers) dependencies = [] parsed_dependencies = get_requires_dependencies( requires=project_data.get("dependencies", []), From 2a35ecf73d91153804a70fab923985742c567cca Mon Sep 17 00:00:00 2001 From: Varsha U N Date: Wed, 26 Feb 2025 21:25:25 +0530 Subject: [PATCH 3/9] Modify test files Signed-off-by: Varsha U N --- src/packagedcode/pypi.py | 3 +- ...te-classifier-pyproject.toml-expected.json | 144 ++++++++++++++---- tests/packagedcode/test_pypi.py | 4 +- 3 files changed, 117 insertions(+), 34 deletions(-) diff --git a/src/packagedcode/pypi.py b/src/packagedcode/pypi.py index 1aa503b839..0a5f043952 100644 --- a/src/packagedcode/pypi.py +++ b/src/packagedcode/pypi.py @@ -546,7 +546,8 @@ def parse(cls, location, package_only=False): dependencies=dependencies, is_private=is_private, extra_data=extra_data, - **urls, + homepage_url=urls.get('homepage'), + download_url=urls.get('download'), ) yield models.PackageData.from_data(package_data, package_only) diff --git a/tests/packagedcode/data/pypi/pyproject-toml/standard/private-classifier-pyproject.toml-expected.json b/tests/packagedcode/data/pypi/pyproject-toml/standard/private-classifier-pyproject.toml-expected.json index ef7fe126ac..a188cdc385 100644 --- a/tests/packagedcode/data/pypi/pyproject-toml/standard/private-classifier-pyproject.toml-expected.json +++ b/tests/packagedcode/data/pypi/pyproject-toml/standard/private-classifier-pyproject.toml-expected.json @@ -1,32 +1,114 @@ [ - { - "datasource_id": "pypi_pyproject_toml", - "type": "pypi", - "primary_language": "Python", - "name": "titanic_ml", - "version": "0.1.0", - "extracted_license_statement": null, - "description": "titanic_ml example package", - "keywords": [], - "parties": [ - { - "type": "author", - "name": "Niels Zeilemaker", - "email": "nielszeilemaker@xebia.com" - } - ], - "dependencies": [ - { - "dependency": "pyspark[ml]", - "version": "" - }, - { - "dependency": "sklearn", - "version": "" - } - ], - "is_private": true, - "extra_data": {}, - "urls": {} - } -] + { + "type": "pypi", + "namespace": null, + "name": "titanic_ml", + "version": "0.1.0", + "qualifiers": {}, + "subpath": null, + "primary_language": "Python", + "description": "titanic_ml example package", + "release_date": null, + "parties": [ + { + "type": "person", + "role": "author", + "name": "Niels Zeilemaker", + "email": "nielszeilemaker@xebia.com", + "url": null + } + ], + "keywords": [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Private :: Do Not Upload" + ], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": true, + "is_virtual": false, + "extra_data": {}, + "dependencies": [ + { + "purl": "pkg:pypi/pyspark", + "extracted_requirement": null, + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/sklearn", + "extracted_requirement": null, + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/tox", + "extracted_requirement": null, + "scope": "dev", + "is_runtime": true, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/pre-commit", + "extracted_requirement": null, + "scope": "dev", + "is_runtime": true, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/bump2version", + "extracted_requirement": null, + "scope": "dev", + "is_runtime": true, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + } + ], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null, + "datasource_id": "pypi_pyproject_toml", + "purl": "pkg:pypi/titanic-ml@0.1.0" + } +] \ No newline at end of file diff --git a/tests/packagedcode/test_pypi.py b/tests/packagedcode/test_pypi.py index 56b5b4f2d2..b503b3e56d 100644 --- a/tests/packagedcode/test_pypi.py +++ b/tests/packagedcode/test_pypi.py @@ -339,11 +339,11 @@ def test_parse_pyproject_toml_standard_lc0(self): expected_loc = self.get_test_loc('pypi/pyproject-toml/standard/lc0-pyproject.toml-expected.json') self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES) - def test_parse_pyproject_toml_private_package(self): + def test_parse_pyproject_toml_private_package(self): test_file = self.get_test_loc('pypi/pyproject-toml/standard/python-private-classifier/pyproject.toml') packages = pypi.PyprojectTomlHandler.parse(test_file) expected_loc = self.get_test_loc('pypi/pyproject-toml/standard/private-classifier-pyproject.toml-expected.json') - self.check_packages_data(packages, expected_loc, regen=REGEN_TEST_FIXTURES) + self.check_packages_data(packages, expected_loc) class TestPoetryHandler(PackageTester): From 19e0fa9327a1b048a952449a5ee7f7926c7593a9 Mon Sep 17 00:00:00 2001 From: Varsha U N Date: Wed, 26 Feb 2025 22:15:20 +0530 Subject: [PATCH 4/9] modify test fails Signed-off-by: Varsha U N --- tests/packagedcode/test_pypi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/packagedcode/test_pypi.py b/tests/packagedcode/test_pypi.py index b503b3e56d..f5bfea8874 100644 --- a/tests/packagedcode/test_pypi.py +++ b/tests/packagedcode/test_pypi.py @@ -343,7 +343,7 @@ def test_parse_pyproject_toml_private_package(self): test_file = self.get_test_loc('pypi/pyproject-toml/standard/python-private-classifier/pyproject.toml') packages = pypi.PyprojectTomlHandler.parse(test_file) expected_loc = self.get_test_loc('pypi/pyproject-toml/standard/private-classifier-pyproject.toml-expected.json') - self.check_packages_data(packages, expected_loc) + self.check_packages_data(packages, expected_loc,regen=REGEN_TEST_FIXTURES) class TestPoetryHandler(PackageTester): From de9250a81374b3cbd4cb30d97004f0ad56b2d293 Mon Sep 17 00:00:00 2001 From: Varsha U N Date: Sat, 5 Apr 2025 21:08:00 +0530 Subject: [PATCH 5/9] Modify get_urls() to accept is_private as an argument Signed-off-by: Varsha U N --- src/packagedcode/pypi.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/packagedcode/pypi.py b/src/packagedcode/pypi.py index 0a5f043952..ff992c87e9 100644 --- a/src/packagedcode/pypi.py +++ b/src/packagedcode/pypi.py @@ -511,14 +511,15 @@ def parse(cls, location, package_only=False): description = project_data.get('description') or '' description = description.strip() + classifiers = project_data.get('classifiers', []) + is_private = any('Private ::' in classifier for classifier in classifiers) + urls, extra_data = get_urls(metainfo=project_data, name=name, version=version) extracted_license_statement, license_file = get_declared_license(project_data) if license_file: extra_data['license_file'] = license_file - classifiers = project_data.get('classifiers', []) - is_private = any('Private ::' in classifier for classifier in classifiers) dependencies = [] parsed_dependencies = get_requires_dependencies( requires=project_data.get("dependencies", []), @@ -546,7 +547,7 @@ def parse(cls, location, package_only=False): dependencies=dependencies, is_private=is_private, extra_data=extra_data, - homepage_url=urls.get('homepage'), + **urls, download_url=urls.get('download'), ) yield models.PackageData.from_data(package_data, package_only) @@ -2285,7 +2286,7 @@ def get_pypi_urls(name, version, **kwargs): ) -def get_urls(metainfo, name, version, poetry=False): +def get_urls(metainfo, name, version,is_private, poetry=False): """ Return a mapping of standard URLs and a mapping of extra-data URls for URLs of this package: @@ -2327,7 +2328,10 @@ def get_urls(metainfo, name, version, poetry=False): # Project-URL: Say Thanks! extra_data = {} - urls = get_pypi_urls(name, version) + if not is_private: + urls = get_pypi_urls(name, version) + else: + urls = {} def add_url(_url, _utype=None, _attribute=None): """ From 12e0a377ca832a446eb27521d3e4465d0f93d808 Mon Sep 17 00:00:00 2001 From: Varsha U N Date: Wed, 16 Apr 2025 23:00:27 +0530 Subject: [PATCH 6/9] Extend support across other handlers Signed-off-by: Varsha U N --- src/packagedcode/pypi.py | 19 ++++++++++++++----- src/packagedcode/utils.py | 4 ++++ 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/packagedcode/pypi.py b/src/packagedcode/pypi.py index ff992c87e9..f94b18eb10 100644 --- a/src/packagedcode/pypi.py +++ b/src/packagedcode/pypi.py @@ -44,6 +44,7 @@ from packagedcode.utils import yield_dependencies_from_package_data from packagedcode.utils import yield_dependencies_from_package_resource from packagedcode.utils import get_base_purl +from packagedcode.utils import is_private_package try: from zipfile import Path as ZipPath @@ -511,9 +512,7 @@ def parse(cls, location, package_only=False): description = project_data.get('description') or '' description = description.strip() - classifiers = project_data.get('classifiers', []) - is_private = any('Private ::' in classifier for classifier in classifiers) - + is_private = is_private_package(project_data.get('classifiers', [])) urls, extra_data = get_urls(metainfo=project_data, name=name, version=version) extracted_license_statement, license_file = get_declared_license(project_data) @@ -552,7 +551,6 @@ def parse(cls, location, package_only=False): ) yield models.PackageData.from_data(package_data, package_only) - def is_poetry_pyproject_toml(location): with open(location, 'r') as file: data = file.read() @@ -741,6 +739,8 @@ def parse(cls, location, package_only=False): ) dependencies.append(dependency.to_dict()) + is_private = is_private_package(poetry_data.get('classifiers', [])) + package_data = dict( datasource_id=cls.datasource_id, type=cls.default_package_type, @@ -1019,6 +1019,9 @@ def parse_metadata(location, datasource_id, package_type, package_only=False): if license_file: extra_data['license_file'] = license_file + classifiers = get_attribute(meta, 'Classifier', multiple=True) + is_private = is_private_package(classifiers) + # FIXME: We are getting dependencies from other sibling files, this is duplicated # data at the package_data level, is this necessary? We also have the entire dependency # relationships here at requires.txt present in ``.egg-info`` should we store these @@ -1039,6 +1042,7 @@ def parse_metadata(location, datasource_id, package_type, package_only=False): dependencies=dependencies, file_references=file_references, extra_data=extra_data, + is_private=is_private, **urls, ) return models.PackageData.from_data(package_data, package_only) @@ -1214,6 +1218,8 @@ def parse(cls, location, package_only=False): if license_file: extra_data['license_file'] = license_file + is_private = is_private_package(setup_args.get('classifiers', [])) + package_data = dict( datasource_id=cls.datasource_id, type=cls.default_package_type, @@ -1343,6 +1349,9 @@ def parse(cls, location, package_only=False): extracted_license_statement = '' extracted_license_statement += f" license_files: {license_file_references}" + classifiers = parser.get('metadata', 'classifiers', fallback='').splitlines() + is_private = is_private_package(classifiers) + package_data = dict( datasource_id=cls.datasource_id, type=cls.default_package_type, @@ -2286,7 +2295,7 @@ def get_pypi_urls(name, version, **kwargs): ) -def get_urls(metainfo, name, version,is_private, poetry=False): +def get_urls(metainfo, name, version, is_private=False, poetry=False): """ Return a mapping of standard URLs and a mapping of extra-data URls for URLs of this package: diff --git a/src/packagedcode/utils.py b/src/packagedcode/utils.py index 59e42714f2..dcc2a15d24 100644 --- a/src/packagedcode/utils.py +++ b/src/packagedcode/utils.py @@ -304,3 +304,7 @@ def is_simple_path(path): def is_simple_path_pattern(path): return path.endswith('*') and path.count('*') == 1 + + +def is_private_package(classifiers): + return any('Private ::' in classifier for classifier in classifiers if classifier) From bb4a59f49213042f7e85382a853b74afc5325084 Mon Sep 17 00:00:00 2001 From: Varsha U N Date: Sat, 26 Apr 2025 12:04:57 +0530 Subject: [PATCH 7/9] Add test-files for handlers Signed-off-by: Varsha U N --- src/packagedcode/pypi.py | 18 +++++++++++------- tests/packagedcode/data/pypi/develop/PKG-INFO | 5 +++++ .../private-classifier-egg-info-expected.json | 10 ++++++++++ .../pypi/setup.py/private-classifier-setup.py | 10 ++++++++++ .../private-classifier-setup.py.expected.json | 12 ++++++++++++ tests/packagedcode/test_pypi.py | 12 +++++++++++- 6 files changed, 59 insertions(+), 8 deletions(-) create mode 100644 tests/packagedcode/data/pypi/develop/PKG-INFO create mode 100644 tests/packagedcode/data/pypi/develop/private-classifier-egg-info-expected.json create mode 100644 tests/packagedcode/data/pypi/setup.py/private-classifier-setup.py create mode 100644 tests/packagedcode/data/pypi/setup.py/private-classifier-setup.py.expected.json diff --git a/src/packagedcode/pypi.py b/src/packagedcode/pypi.py index f94b18eb10..db45c6edaa 100644 --- a/src/packagedcode/pypi.py +++ b/src/packagedcode/pypi.py @@ -513,7 +513,7 @@ def parse(cls, location, package_only=False): description = description.strip() is_private = is_private_package(project_data.get('classifiers', [])) - urls, extra_data = get_urls(metainfo=project_data, name=name, version=version) + urls, extra_data = get_urls(metainfo=project_data, name=name, version=version,is_private=is_private) extracted_license_statement, license_file = get_declared_license(project_data) if license_file: @@ -752,6 +752,7 @@ def parse(cls, location, package_only=False): keywords=get_keywords(poetry_data), parties=get_pyproject_toml_parties(poetry_data), extra_data=extra_data, + is_private=is_private, dependencies=dependencies, **urls, ) @@ -1208,7 +1209,9 @@ def parse(cls, location, package_only=False): # search for possible dunder versions here and elsewhere version = detect_version_attribute(location) - urls, extra_data = get_urls(metainfo=setup_args, name=name, version=version) + is_private = is_private_package(setup_args.get('classifiers', [])) + + urls, extra_data = get_urls(metainfo=setup_args, name=name, version=version,is_private=is_private) dependencies = get_setup_py_dependencies(setup_args) python_requires = get_setup_py_python_requires(setup_args) @@ -1218,7 +1221,6 @@ def parse(cls, location, package_only=False): if license_file: extra_data['license_file'] = license_file - is_private = is_private_package(setup_args.get('classifiers', [])) package_data = dict( datasource_id=cls.datasource_id, @@ -1231,6 +1233,7 @@ def parse(cls, location, package_only=False): extracted_license_statement=extracted_license_statement, dependencies=dependencies, keywords=get_keywords(setup_args), + is_private=is_private, extra_data=extra_data, **urls, ) @@ -1361,6 +1364,7 @@ def parse(cls, location, package_only=False): homepage_url=metadata.get('url'), primary_language=cls.default_primary_language, dependencies=dependent_packages, + is_private=is_private, extracted_license_statement=extracted_license_statement, ) yield models.PackageData.from_data(package_data, package_only) @@ -2337,10 +2341,10 @@ def get_urls(metainfo, name, version, is_private=False, poetry=False): # Project-URL: Say Thanks! extra_data = {} - if not is_private: - urls = get_pypi_urls(name, version) - else: - urls = {} + if is_private: + return {}, {} + + urls = get_pypi_urls(name, version) def add_url(_url, _utype=None, _attribute=None): """ diff --git a/tests/packagedcode/data/pypi/develop/PKG-INFO b/tests/packagedcode/data/pypi/develop/PKG-INFO new file mode 100644 index 0000000000..03d906228f --- /dev/null +++ b/tests/packagedcode/data/pypi/develop/PKG-INFO @@ -0,0 +1,5 @@ +Metadata-Version: 2.1 +Name: example_egg +Version: 1.0.0 +Classifier: Development Status :: 5 - Production/Stable +Classifier: Private :: Do Not Upload \ No newline at end of file diff --git a/tests/packagedcode/data/pypi/develop/private-classifier-egg-info-expected.json b/tests/packagedcode/data/pypi/develop/private-classifier-egg-info-expected.json new file mode 100644 index 0000000000..799474c877 --- /dev/null +++ b/tests/packagedcode/data/pypi/develop/private-classifier-egg-info-expected.json @@ -0,0 +1,10 @@ +[ + { + "type": "pypi", + "name": "example_egg", + "version": "1.0.0", + "is_private": true, + "datasource_id": "pypi_egg_info", + "purl": "pkg:pypi/example-egg@1.0.0" + } +] \ No newline at end of file diff --git a/tests/packagedcode/data/pypi/setup.py/private-classifier-setup.py b/tests/packagedcode/data/pypi/setup.py/private-classifier-setup.py new file mode 100644 index 0000000000..574fc24db9 --- /dev/null +++ b/tests/packagedcode/data/pypi/setup.py/private-classifier-setup.py @@ -0,0 +1,10 @@ +from setuptools import setup + +setup( + name="example_setup", + version="1.0.0", + classifiers=[ + "Development Status :: 5 - Production/Stable", + "Private :: Do Not Upload", + ], +) \ No newline at end of file diff --git a/tests/packagedcode/data/pypi/setup.py/private-classifier-setup.py.expected.json b/tests/packagedcode/data/pypi/setup.py/private-classifier-setup.py.expected.json new file mode 100644 index 0000000000..2babe2ea88 --- /dev/null +++ b/tests/packagedcode/data/pypi/setup.py/private-classifier-setup.py.expected.json @@ -0,0 +1,12 @@ +[ + { + "type": "pypi", + "name": "example_setup", + "version": "1.0.0", + "description": null, + "primary_language": "Python", + "is_private": true, + "datasource_id": "pypi_setup_py", + "purl": "pkg:pypi/example-setup@1.0.0" + } +] \ No newline at end of file diff --git a/tests/packagedcode/test_pypi.py b/tests/packagedcode/test_pypi.py index f5bfea8874..58d7e1ded3 100644 --- a/tests/packagedcode/test_pypi.py +++ b/tests/packagedcode/test_pypi.py @@ -345,7 +345,17 @@ def test_parse_pyproject_toml_private_package(self): expected_loc = self.get_test_loc('pypi/pyproject-toml/standard/private-classifier-pyproject.toml-expected.json') self.check_packages_data(packages, expected_loc,regen=REGEN_TEST_FIXTURES) - + def test_parse_setup_py_private_package(self): + test_file = self.get_test_loc('pypi/setup.py/private-classifier-setup.py') + packages = pypi.PythonSetupPyHandler.parse(test_file) + expected_loc = self.get_test_loc('pypi/setup.py/private-classifier-setup.py-expected.json') + self.check_packages_data(packages, expected_loc, regen=True) + + def test_parse_egg_info_private_package(self): + test_file = self.get_test_loc('pypi/develop/private-classifier.egg-info/PKG-INFO') + packages = pypi.parse_metadata(test_file, datasource_id="pypi_egg_info", package_type="pypi") + expected_loc = self.get_test_loc('pypi/develop/private-classifier-egg-info-expected.json') + self.check_packages_data(packages, expected_loc, regen=True) class TestPoetryHandler(PackageTester): def test_is_pyproject_toml_poetry(self): From 6e1d600071382cc073ddbef0b3e8cb3b94a5b4b4 Mon Sep 17 00:00:00 2001 From: Varsha U N Date: Sun, 27 Apr 2025 22:42:00 +0530 Subject: [PATCH 8/9] fix test failures Signed-off-by: Varsha U N --- .../private-classifier-egg-info-expected.json | 39 +++++++++++++++++++ .../private-classifier-setup.py.expected.json | 39 ++++++++++++++++++- tests/packagedcode/test_pypi.py | 6 +-- 3 files changed, 80 insertions(+), 4 deletions(-) diff --git a/tests/packagedcode/data/pypi/develop/private-classifier-egg-info-expected.json b/tests/packagedcode/data/pypi/develop/private-classifier-egg-info-expected.json index 799474c877..b6ec9bc832 100644 --- a/tests/packagedcode/data/pypi/develop/private-classifier-egg-info-expected.json +++ b/tests/packagedcode/data/pypi/develop/private-classifier-egg-info-expected.json @@ -1,9 +1,48 @@ [ { "type": "pypi", + "namespace": null, "name": "example_egg", "version": "1.0.0", + "qualifiers": {}, + "subpath": null, + "primary_language": "Python", + "description": "", + "release_date": null, + "parties": [], + "keywords": [ + "Development Status :: 5 - Production/Stable", + "Private :: Do Not Upload" + ], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], "is_private": true, + "is_virtual": false, + "extra_data": {}, + "dependencies": [], + "repository_homepage_url": "https://pypi.org/project/example_egg", + "repository_download_url": "https://pypi.org/packages/source/e/example_egg/example_egg-1.0.0.tar.gz", + "api_data_url": "https://pypi.org/pypi/example_egg/1.0.0/json", "datasource_id": "pypi_egg_info", "purl": "pkg:pypi/example-egg@1.0.0" } diff --git a/tests/packagedcode/data/pypi/setup.py/private-classifier-setup.py.expected.json b/tests/packagedcode/data/pypi/setup.py/private-classifier-setup.py.expected.json index 2babe2ea88..66208de11a 100644 --- a/tests/packagedcode/data/pypi/setup.py/private-classifier-setup.py.expected.json +++ b/tests/packagedcode/data/pypi/setup.py/private-classifier-setup.py.expected.json @@ -1,11 +1,48 @@ [ { "type": "pypi", + "namespace": null, "name": "example_setup", "version": "1.0.0", - "description": null, + "qualifiers": {}, + "subpath": null, "primary_language": "Python", + "description": "", + "release_date": null, + "parties": [], + "keywords": [ + "Development Status :: 5 - Production/Stable", + "Private :: Do Not Upload" + ], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], "is_private": true, + "is_virtual": false, + "extra_data": {}, + "dependencies": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null, "datasource_id": "pypi_setup_py", "purl": "pkg:pypi/example-setup@1.0.0" } diff --git a/tests/packagedcode/test_pypi.py b/tests/packagedcode/test_pypi.py index e5cbd6ac21..c3870e37a6 100644 --- a/tests/packagedcode/test_pypi.py +++ b/tests/packagedcode/test_pypi.py @@ -378,14 +378,14 @@ def test_parse_pyproject_toml_private_package(self): def test_parse_setup_py_private_package(self): test_file = self.get_test_loc('pypi/setup.py/private-classifier-setup.py') packages = pypi.PythonSetupPyHandler.parse(test_file) - expected_loc = self.get_test_loc('pypi/setup.py/private-classifier-setup.py-expected.json') + expected_loc = self.get_test_loc('pypi/setup.py/private-classifier-setup.py.expected.json') self.check_packages_data(packages, expected_loc, regen=True) def test_parse_egg_info_private_package(self): - test_file = self.get_test_loc('pypi/develop/private-classifier.egg-info/PKG-INFO') + test_file = self.get_test_loc('pypi/develop/PKG-INFO') packages = pypi.parse_metadata(test_file, datasource_id="pypi_egg_info", package_type="pypi") expected_loc = self.get_test_loc('pypi/develop/private-classifier-egg-info-expected.json') - self.check_packages_data(packages, expected_loc, regen=True) + self.check_packages_data([packages], expected_loc, regen=True) class TestPoetryHandler(PackageTester): def test_is_pyproject_toml_poetry(self): From e7632cc3feba946b179fe53bc929db19bd8f3202 Mon Sep 17 00:00:00 2001 From: Varsha U N Date: Wed, 30 Apr 2025 19:55:45 +0530 Subject: [PATCH 9/9] Move is_private_package to pypi Signed-off-by: Varsha U N --- src/packagedcode/pypi.py | 3 ++- src/packagedcode/utils.py | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/packagedcode/pypi.py b/src/packagedcode/pypi.py index 2d751ea894..d99af91d61 100644 --- a/src/packagedcode/pypi.py +++ b/src/packagedcode/pypi.py @@ -45,7 +45,6 @@ from packagedcode.utils import yield_dependencies_from_package_data from packagedcode.utils import yield_dependencies_from_package_resource from packagedcode.utils import get_base_purl -from packagedcode.utils import is_private_package try: from zipfile import Path as ZipPath @@ -75,6 +74,8 @@ def logger_debug(*args): def logger_debug(*args): return logger.debug(' '.join(isinstance(a, str) and a or repr(a) for a in args)) +def is_private_package(classifiers): + return any('Private ::' in classifier for classifier in classifiers if classifier) class PythonEggPkgInfoFile(models.DatafileHandler): datasource_id = 'pypi_egg_pkginfo' diff --git a/src/packagedcode/utils.py b/src/packagedcode/utils.py index 66fc1cb371..f4249e87dd 100644 --- a/src/packagedcode/utils.py +++ b/src/packagedcode/utils.py @@ -305,5 +305,3 @@ def is_simple_path(path): def is_simple_path_pattern(path): return path.endswith('*') and path.count('*') == 1 -def is_private_package(classifiers): - return any('Private ::' in classifier for classifier in classifiers if classifier)