From 956094a5df1f15a5130dddac479fd04fc4186812 Mon Sep 17 00:00:00 2001 From: Alexander Mazuruk Date: Thu, 10 Jun 2021 11:13:47 +0200 Subject: [PATCH 1/4] Add providing location for fetch_via_{vcs,git} This allows to call fetch_via_{vcs,git} multiple times for a location to have another revision. Example: fetch_via_git("git+https://github.com/nexB/fetchcode.git", location="/tmp/repo") will checkout tip of default branch fetch_via_git("git+https://github.com/nexB/fetchcode.git@ccb7b6199681910ccf047f1a18aa89ece45d665c", location="/tmp/repo") will reset to ccb7b6199681910ccf047f1a18aa89ece45d665c Additionally remove some trailing whitespaces and fix indentation of a docstring. Signed-off-by: Alexander Mazuruk --- src/fetchcode/__init__.py | 4 ++-- src/fetchcode/vcs/__init__.py | 19 ++++++++++--------- src/fetchcode/vcs/git.py | 19 +++++++++++++------ 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/src/fetchcode/__init__.py b/src/fetchcode/__init__.py index c573991f..b44a04aa 100644 --- a/src/fetchcode/__init__.py +++ b/src/fetchcode/__init__.py @@ -41,7 +41,7 @@ def __init__(self, location, content_type, size, url): def fetch_http(url, location): """ Return a `Response` object built from fetching the content at a HTTP/HTTPS based `url` URL string - saving the content in a file at `location` + saving the content in a file at `location` """ r = requests.get(url) with open(location, 'wb') as f: @@ -59,7 +59,7 @@ def fetch_http(url, location): def fetch_ftp(url, location): """ Return a `Response` object built from fetching the content at a FTP based `url` URL string - saving the content in a file at `location` + saving the content in a file at `location` """ url_parts = urlparse(url) diff --git a/src/fetchcode/vcs/__init__.py b/src/fetchcode/vcs/__init__.py index c839aa3b..b8152134 100644 --- a/src/fetchcode/vcs/__init__.py +++ b/src/fetchcode/vcs/__init__.py @@ -29,9 +29,9 @@ class VCSResponse: """ Represent the response from fetching a VCS URL with: -- `dest_dir`: destination of directory -- `vcs_type`: VCS Type of URL (git,bzr,hg,svn) -- `domain` : Source of git VCS (GitHub, Gitlab, Bitbucket) + - `dest_dir`: destination of directory + - `vcs_type`: VCS Type of URL (git,bzr,hg,svn) + - `domain` : Source of git VCS (GitHub, Gitlab, Bitbucket) """ def __init__(self, dest_dir, vcs_type, domain): @@ -40,16 +40,17 @@ def __init__(self, dest_dir, vcs_type, domain): self.domain = domain -def fetch_via_vcs(url): +def fetch_via_vcs(url, location=None): """ Take `url` as input and store the content of it at location specified at `location` string - Return a VCSResponse object + Return a VCSResponse object """ parsed_url = urlparse(url) scheme = parsed_url.scheme domain = parsed_url.netloc - temp = tempfile.mkdtemp() - os.rmdir(temp) + if location is None: + location = tempfile.mkdtemp() + os.rmdir(location) if scheme not in vcs.all_schemes: raise Exception("Not a supported/known scheme.") @@ -58,6 +59,6 @@ def fetch_via_vcs(url): vcs_type = vcs_name backend = vcs.get_backend_for_scheme(scheme) - backend.obtain(dest=temp, url=misc.hide_url(url)) + backend.obtain(dest=location, url=misc.hide_url(url)) - return VCSResponse(dest_dir=temp, vcs_type=vcs_type, domain=domain) + return VCSResponse(dest_dir=location, vcs_type=vcs_type, domain=domain) \ No newline at end of file diff --git a/src/fetchcode/vcs/git.py b/src/fetchcode/vcs/git.py index 9d9c6cca..72bc9708 100644 --- a/src/fetchcode/vcs/git.py +++ b/src/fetchcode/vcs/git.py @@ -19,21 +19,28 @@ from urllib.parse import urlparse from fetchcode.vcs.pip._internal.vcs.git import Git +from fetchcode.vcs.pip._internal.vcs.versioncontrol import RevOptions from fetchcode.vcs.pip._internal.utils import misc from fetchcode.vcs.pip._internal.vcs import vcs from fetchcode.vcs import VCSResponse -def fetch_via_git(url): +def fetch_via_git(url, location=None): + """ + Take `url` as input and store the content of it at location specified at `location` string + If location string is not set, a tempfile.mkdtemp() will be created to store content in. + tempfile.mkdtemp must be cleaned by user manually. + Return a VCSResponse object + """ parsed_url = urlparse(url) scheme = parsed_url.scheme domain = parsed_url.netloc - temp = tempfile.mkdtemp() - os.rmdir(temp) + if location is None: + location = tempfile.mkdtemp() + os.rmdir(location) if scheme not in Git.schemes: raise Exception("Not a Git based scheme.") backend = vcs.get_backend(name="git") - backend.obtain(dest=temp, url=misc.hide_url(url)) - - return VCSResponse(dest_dir=temp, vcs_type="git", domain=domain) + backend.obtain(dest=location, url=misc.hide_url(url)) + return VCSResponse(dest_dir=location, vcs_type="git", domain=domain) \ No newline at end of file From 1231e858efd405e313e792ce3a4b7e073bfcc77b Mon Sep 17 00:00:00 2001 From: Alexander Mazuruk Date: Thu, 10 Jun 2021 17:11:05 +0200 Subject: [PATCH 2/4] Lessen code duplication in vcs tests By using pytest.mark.parametrize decorator. Signed-off-by: Alexander Mazuruk --- tests/test_vcs.py | 241 ++++++------------------------------------ tests/test_vcs_git.py | 64 +++-------- 2 files changed, 48 insertions(+), 257 deletions(-) diff --git a/tests/test_vcs.py b/tests/test_vcs.py index 9d2bfdc1..869bf800 100644 --- a/tests/test_vcs.py +++ b/tests/test_vcs.py @@ -25,213 +25,40 @@ def obtain(dest, url): pass - -@mock.patch("fetchcode.vcs.vcs.get_backend_for_scheme") -def test_fetch_with_git_http_url_returns_a_response(mock_backend): - mock_backend.return_value.obtain = obtain - url = "git+http://github.com/jamesor/mongoose-versioner" - response = fetch_via_vcs(url=url) - assert response.vcs_type == "git" - assert response.domain == "github.com" - - -@mock.patch("fetchcode.vcs.vcs.get_backend_for_scheme") -def test_fetch_with_git_url_returns_a_response(mock_backend): - mock_backend.return_value.obtain = obtain - url = "git://github.com/jamesor/mongoose-versioner" - response = fetch_via_vcs(url=url) - assert response.vcs_type == "git" - assert response.domain == "github.com" - - -@mock.patch("fetchcode.vcs.vcs.get_backend_for_scheme") -def test_fetch_with_git_https_url_returns_a_response(mock_backend): - mock_backend.return_value.obtain = obtain - url = "git+https://github.com/jamesor/mongoose-versioner" - response = fetch_via_vcs(url=url) - assert response.vcs_type == "git" - assert response.domain == "github.com" - - -@mock.patch("fetchcode.vcs.vcs.get_backend_for_scheme") -def test_fetch_with_git_ssh_url_returns_a_response(mock_backend): - mock_backend.return_value.obtain = obtain - url = "git+ssh://github.com/jamesor/mongoose-versioner" - response = fetch_via_vcs(url=url) - assert response.vcs_type == "git" - assert response.domain == "github.com" - - -@mock.patch("fetchcode.vcs.vcs.get_backend_for_scheme") -def test_fetch_with_git_file_url_returns_a_response(mock_backend): - mock_backend.return_value.obtain = obtain - url = "git+file://github.com/jamesor/mongoose-versioner" - response = fetch_via_vcs(url=url) - assert response.vcs_type == "git" - assert response.domain == "github.com" - - -@mock.patch("fetchcode.vcs.vcs.get_backend_for_scheme") -def test_fetch_with_git_ssh_url_returns_a_response(mock_backend): - mock_backend.return_value.obtain = obtain - url = "git+git://github.com/jamesor/mongoose-versioner" - response = fetch_via_vcs(url=url) - assert response.vcs_type == "git" - assert response.domain == "github.com" - - -@mock.patch("fetchcode.vcs.vcs.get_backend_for_scheme") -def test_fetch_with_bzr_http_url_returns_a_response(mock_backend): - mock_backend.return_value.obtain = obtain - url = "bzr+http://gitlab.com/jamesor/mongoose-versioner" - response = fetch_via_vcs(url=url) - assert response.vcs_type == "bzr" - assert response.domain == "gitlab.com" - - -@mock.patch("fetchcode.vcs.vcs.get_backend_for_scheme") -def test_fetch_with_bzr_https_url_returns_a_response(mock_backend): - mock_backend.return_value.obtain = obtain - url = "bzr+https://gitlab.com/jamesor/mongoose-versioner" - response = fetch_via_vcs(url=url) - assert response.vcs_type == "bzr" - assert response.domain == "gitlab.com" - - -@mock.patch("fetchcode.vcs.vcs.get_backend_for_scheme") -def test_fetch_with_bzr_url_returns_a_response(mock_backend): - mock_backend.return_value.obtain = obtain - url = "bzr://gitlab.com/jamesor/mongoose-versioner" - response = fetch_via_vcs(url=url) - assert response.vcs_type == "bzr" - assert response.domain == "gitlab.com" - - -@mock.patch("fetchcode.vcs.vcs.get_backend_for_scheme") -def test_fetch_with_bzr_ssh_url_returns_a_response(mock_backend): - mock_backend.return_value.obtain = obtain - url = "bzr+ssh://gitlab.com/jamesor/mongoose-versioner" - response = fetch_via_vcs(url=url) - assert response.vcs_type == "bzr" - assert response.domain == "gitlab.com" - - -@mock.patch("fetchcode.vcs.vcs.get_backend_for_scheme") -def test_fetch_with_bzr_ftp_url_returns_a_response(mock_backend): - mock_backend.return_value.obtain = obtain - url = "bzr+ftp://gitlab.com/jamesor/mongoose-versioner" - response = fetch_via_vcs(url=url) - assert response.vcs_type == "bzr" - assert response.domain == "gitlab.com" - - -@mock.patch("fetchcode.vcs.vcs.get_backend_for_scheme") -def test_fetch_with_bzr_sftp_url_returns_a_response(mock_backend): - mock_backend.return_value.obtain = obtain - url = "bzr+sftp://gitlab.com/jamesor/mongoose-versioner" - response = fetch_via_vcs(url=url) - assert response.vcs_type == "bzr" - assert response.domain == "gitlab.com" - - -@mock.patch("fetchcode.vcs.vcs.get_backend_for_scheme") -def test_fetch_with_bzr_lp_url_returns_a_response(mock_backend): - mock_backend.return_value.obtain = obtain - url = "bzr+lp://gitlab.com/jamesor/mongoose-versioner" - response = fetch_via_vcs(url=url) - assert response.vcs_type == "bzr" - assert response.domain == "gitlab.com" - - -@mock.patch("fetchcode.vcs.vcs.get_backend_for_scheme") -def test_fetch_with_hg_url_returns_a_response(mock_backend): - mock_backend.return_value.obtain = obtain - url = "hg://bitbucket.com/jamesor/mongoose-versioner" - response = fetch_via_vcs(url=url) - assert response.vcs_type == "hg" - assert response.domain == "bitbucket.com" - - -@mock.patch("fetchcode.vcs.vcs.get_backend_for_scheme") -def test_fetch_with_hg_file_url_returns_a_response(mock_backend): - mock_backend.return_value.obtain = obtain - url = "hg+file://bitbucket.com/jamesor/mongoose-versioner" - response = fetch_via_vcs(url=url) - assert response.vcs_type == "hg" - assert response.domain == "bitbucket.com" - - -@mock.patch("fetchcode.vcs.vcs.get_backend_for_scheme") -def test_fetch_with_hg_http_url_returns_a_response(mock_backend): - mock_backend.return_value.obtain = obtain - url = "hg+http://bitbucket.com/jamesor/mongoose-versioner" - response = fetch_via_vcs(url=url) - assert response.vcs_type == "hg" - assert response.domain == "bitbucket.com" - - -@mock.patch("fetchcode.vcs.vcs.get_backend_for_scheme") -def test_fetch_with_hg_https_url_returns_a_response(mock_backend): - mock_backend.return_value.obtain = obtain - url = "hg+https://bitbucket.com/jamesor/mongoose-versioner" - response = fetch_via_vcs(url=url) - assert response.vcs_type == "hg" - assert response.domain == "bitbucket.com" - - -@mock.patch("fetchcode.vcs.vcs.get_backend_for_scheme") -def test_fetch_with_hg_ssh_url_returns_a_response(mock_backend): - mock_backend.return_value.obtain = obtain - url = "hg+ssh://bitbucket.com/jamesor/mongoose-versioner" - response = fetch_via_vcs(url=url) - assert response.vcs_type == "hg" - assert response.domain == "bitbucket.com" - - -@mock.patch("fetchcode.vcs.vcs.get_backend_for_scheme") -def test_fetch_with_hg_static_http_url_returns_a_response(mock_backend): - mock_backend.return_value.obtain = obtain - url = "hg+static-http://bitbucket.com/jamesor/mongoose-versioner" - response = fetch_via_vcs(url=url) - assert response.vcs_type == "hg" - assert response.domain == "bitbucket.com" - - -@mock.patch("fetchcode.vcs.vcs.get_backend_for_scheme") -def test_fetch_with_svn_url_returns_a_response(mock_backend): - mock_backend.return_value.obtain = obtain - url = "svn://bitbucket.com/jamesor/mongoose-versioner" - response = fetch_via_vcs(url=url) - assert response.vcs_type == "svn" - assert response.domain == "bitbucket.com" - - -@mock.patch("fetchcode.vcs.vcs.get_backend_for_scheme") -def test_fetch_with_svn_http_url_returns_a_response(mock_backend): - mock_backend.return_value.obtain = obtain - url = "svn+http://bitbucket.com/jamesor/mongoose-versioner" - response = fetch_via_vcs(url=url) - assert response.vcs_type == "svn" - assert response.domain == "bitbucket.com" - - -@mock.patch("fetchcode.vcs.vcs.get_backend_for_scheme") -def test_fetch_with_svn_https_url_returns_a_response(mock_backend): - mock_backend.return_value.obtain = obtain - url = "svn+https://bitbucket.com/jamesor/mongoose-versioner" - response = fetch_via_vcs(url=url) - assert response.vcs_type == "svn" - assert response.domain == "bitbucket.com" - - -@mock.patch("fetchcode.vcs.vcs.get_backend_for_scheme") -def test_fetch_with_svn_svn_url_returns_a_response(mock_backend): - mock_backend.return_value.obtain = obtain - url = "svn+svn://bitbucket.com/jamesor/mongoose-versioner" - response = fetch_via_vcs(url=url) - assert response.vcs_type == "svn" - assert response.domain == "bitbucket.com" - +@pytest.mark.parametrize( + "url, vcs_type, domain", + [ + pytest.param("git+http://github.com/jamesor/mongoose-versioner", "git", "github.com", id="git_http"), + pytest.param("git://github.com/jamesor/mongoose-versioner", "git", "github.com", id="git"), + pytest.param("git+https://github.com/jamesor/mongoose-versioner", "git", "github.com", id="git_https"), + pytest.param("git+ssh://github.com/jamesor/mongoose-versioner", "git", "github.com", id="git_ssh"), + pytest.param("git+file://github.com/jamesor/mongoose-versioner", "git", "github.com", id="git_file"), + pytest.param("git+git://github.com/jamesor/mongoose-versioner", "git", "github.com", id="git_git"), + pytest.param("bzr+http://gitlab.com/jamesor/mongoose-versioner", "bzr", "gitlab.com", id="bzr_http"), + pytest.param("bzr+https://gitlab.com/jamesor/mongoose-versioner", "bzr", "gitlab.com", id="bzr_https"), + pytest.param("bzr://gitlab.com/jamesor/mongoose-versioner", "bzr", "gitlab.com", id="bzr"), + pytest.param("bzr+ssh://gitlab.com/jamesor/mongoose-versioner", "bzr", "gitlab.com", id="bzr_ssh"), + pytest.param("bzr+ftp://gitlab.com/jamesor/mongoose-versioner", "bzr", "gitlab.com", id="bzr_ftp"), + pytest.param("bzr+sftp://gitlab.com/jamesor/mongoose-versioner", "bzr", "gitlab.com", id="bzr_sftp"), + pytest.param("bzr+lp://gitlab.com/jamesor/mongoose-versioner", "bzr", "gitlab.com", id="bzr_lp"), + pytest.param("hg://bitbucket.com/jamesor/mongoose-versioner", "hg", "bitbucket.com", id="hg"), + pytest.param("hg+file://bitbucket.com/jamesor/mongoose-versioner", "hg", "bitbucket.com", id="hg_file"), + pytest.param("hg+http://bitbucket.com/jamesor/mongoose-versioner", "hg", "bitbucket.com", id="hg_http"), + pytest.param("hg+https://bitbucket.com/jamesor/mongoose-versioner", "hg", "bitbucket.com", id="hg_https"), + pytest.param("hg+ssh://bitbucket.com/jamesor/mongoose-versioner", "hg", "bitbucket.com", id="hg_ssh"), + pytest.param("hg+static-http://bitbucket.com/jamesor/mongoose-versioner", "hg", "bitbucket.com", id="hg_static_http"), + pytest.param("svn://bitbucket.com/jamesor/mongoose-versioner", "svn", "bitbucket.com", id="svn"), + pytest.param("svn+http://bitbucket.com/jamesor/mongoose-versioner", "svn", "bitbucket.com", id="svn_http"), + pytest.param("svn+https://bitbucket.com/jamesor/mongoose-versioner", "svn", "bitbucket.com", id="svn_https"), + pytest.param("svn+svn://bitbucket.com/jamesor/mongoose-versioner", "svn", "bitbucket.com", id="svn_svn") + ], +) +@mock.patch("fetchcode.vcs.vcs.get_backend_for_scheme") +def test_fetch_via_vcs_returns_response(mock_backend, url, vcs_type, domain): + mock_backend.return_value.obtain = obtain + response = fetch_via_vcs(url=url) + assert response.vcs_type == vcs_type + assert response.domain == domain def test_fetch_with_invalid_scheme(): invalid_urls = [ diff --git a/tests/test_vcs_git.py b/tests/test_vcs_git.py index ec7dcdfd..3972b768 100644 --- a/tests/test_vcs_git.py +++ b/tests/test_vcs_git.py @@ -25,59 +25,23 @@ def obtain(dest, url): pass +@pytest.mark.parametrize( + "url, vcs_type, domain", + [ + pytest.param("git+http://github.com/jamesor/mongoose-versioner", "git", "github.com", id="git_http"), + pytest.param("git://github.com/jamesor/mongoose-versioner", "git", "github.com", id="git"), + pytest.param("git+https://github.com/jamesor/mongoose-versioner", "git", "github.com", id="git_https"), + pytest.param("git+ssh://github.com/jamesor/mongoose-versioner", "git", "github.com", id="git_ssh"), + pytest.param("git+file://github.com/jamesor/mongoose-versioner", "git", "github.com", id="git_file"), + pytest.param("git+git://github.com/jamesor/mongoose-versioner", "git", "github.com", id="git_git") + ], +) @mock.patch("fetchcode.vcs.git.vcs.get_backend") -def test_fetch_with_git_http_url_returns_a_response(mock_backend): +def test_fetch_via_vcs_returns_response(mock_backend, url, vcs_type, domain): mock_backend.return_value.obtain = obtain - url = "git+http://github.com/jamesor/mongoose-versioner" response = fetch_via_git(url=url) - assert response.vcs_type == "git" - assert response.domain == "github.com" - - -@mock.patch("fetchcode.vcs.git.vcs.get_backend") -def test_fetch_with_git_url_returns_a_response(mock_backend): - mock_backend.return_value.obtain = obtain - url = "git://github.com/jamesor/mongoose-versioner" - response = fetch_via_git(url=url) - assert response.vcs_type == "git" - assert response.domain == "github.com" - - -@mock.patch("fetchcode.vcs.git.vcs.get_backend") -def test_fetch_with_git_https_url_returns_a_response(mock_backend): - mock_backend.return_value.obtain = obtain - url = "git+https://github.com/jamesor/mongoose-versioner" - response = fetch_via_git(url=url) - assert response.vcs_type == "git" - assert response.domain == "github.com" - - -@mock.patch("fetchcode.vcs.git.vcs.get_backend") -def test_fetch_with_git_ssh_url_returns_a_response(mock_backend): - mock_backend.return_value.obtain = obtain - url = "git+ssh://github.com/jamesor/mongoose-versioner" - response = fetch_via_git(url=url) - assert response.vcs_type == "git" - assert response.domain == "github.com" - - -@mock.patch("fetchcode.vcs.git.vcs.get_backend") -def test_fetch_with_git_file_url_returns_a_response(mock_backend): - mock_backend.return_value.obtain = obtain - url = "git+file://github.com/jamesor/mongoose-versioner" - response = fetch_via_git(url=url) - assert response.vcs_type == "git" - assert response.domain == "github.com" - - -@mock.patch("fetchcode.vcs.git.vcs.get_backend") -def test_fetch_with_git_ssh_url_returns_a_response(mock_backend): - mock_backend.return_value.obtain = obtain - url = "git+git://github.com/jamesor/mongoose-versioner" - response = fetch_via_git(url=url) - assert response.vcs_type == "git" - assert response.domain == "github.com" - + assert response.vcs_type == vcs_type + assert response.domain == domain def test_fetch_with_git_invalid_scheme(): invalid_urls = [ From 1829791216d46c74229b29541d5bb0a9d5c6ca80 Mon Sep 17 00:00:00 2001 From: Alexander Mazuruk Date: Thu, 10 Jun 2021 14:46:04 +0200 Subject: [PATCH 3/4] Format code with psf/black As recommended by @TG1999 Signed-off-by: Alexander Mazuruk --- src/fetchcode/__init__.py | 14 +- src/fetchcode/commoncode_datautils.py | 126 ++++-- src/fetchcode/packagedcode_models.py | 544 ++++++++++++++------------ src/fetchcode/vcs/__init__.py | 2 +- src/fetchcode/vcs/git.py | 2 +- 5 files changed, 409 insertions(+), 279 deletions(-) diff --git a/src/fetchcode/__init__.py b/src/fetchcode/__init__.py index b44a04aa..d2e301cb 100644 --- a/src/fetchcode/__init__.py +++ b/src/fetchcode/__init__.py @@ -44,11 +44,11 @@ def fetch_http(url, location): saving the content in a file at `location` """ r = requests.get(url) - with open(location, 'wb') as f: + with open(location, "wb") as f: f.write(r.content) - content_type = r.headers.get('content-type') - size = r.headers.get('content-length') + content_type = r.headers.get("content-type") + size = r.headers.get("content-length") size = int(size) if size else None resp = Response(location=location, content_type=content_type, size=size, url=url) @@ -79,8 +79,8 @@ def fetch_ftp(url, location): content_type = None ftp.cwd(dir) - file = 'RETR {}'.format(file) - with open(location, 'wb') as f: + file = "RETR {}".format(file) + with open(location, "wb") as f: ftp.retrbinary(file, f.write) ftp.close() @@ -99,9 +99,9 @@ def fetch(url): url_parts = urlparse(url) scheme = url_parts.scheme - fetchers = {'ftp': fetch_ftp, 'http': fetch_http, 'https': fetch_http} + fetchers = {"ftp": fetch_ftp, "http": fetch_http, "https": fetch_http} if scheme in fetchers: return fetchers.get(scheme)(url, location) - raise Exception('Not a supported/known scheme.') + raise Exception("Not a supported/known scheme.") diff --git a/src/fetchcode/commoncode_datautils.py b/src/fetchcode/commoncode_datautils.py index 4822fbeb..e8bd6536 100644 --- a/src/fetchcode/commoncode_datautils.py +++ b/src/fetchcode/commoncode_datautils.py @@ -39,14 +39,23 @@ """ -HELP_METADATA = '__field_help' -LABEL_METADATA = '__field_label' +HELP_METADATA = "__field_help" +LABEL_METADATA = "__field_label" -def attribute(default=attr.NOTHING, validator=None, - repr=False, eq=True, order=True, # NOQA - init=True, type=None, converter=None, # NOQA - help=None, label=None, metadata=None,): # NOQA +def attribute( + default=attr.NOTHING, + validator=None, + repr=False, + eq=True, + order=True, # NOQA + init=True, + type=None, + converter=None, # NOQA + help=None, + label=None, + metadata=None, +): # NOQA """ A generic attribute with help metadata and that is not included in the representation by default. @@ -67,12 +76,20 @@ def attribute(default=attr.NOTHING, validator=None, init=init, metadata=metadata, type=type, - converter=converter + converter=converter, ) -def Boolean(default=False, validator=None, repr=False, eq=True, order=True, # NOQA - converter=None, label=None, help=None,): # NOQA +def Boolean( + default=False, + validator=None, + repr=False, + eq=True, + order=True, # NOQA + converter=None, + label=None, + help=None, +): # NOQA """ A boolean attribute. """ @@ -90,8 +107,16 @@ def Boolean(default=False, validator=None, repr=False, eq=True, order=True, # N ) -def TriBoolean(default=None, validator=None, repr=False, eq=True, order=True, # NOQA - converter=None, label=None, help=None,): # NOQA +def TriBoolean( + default=None, + validator=None, + repr=False, + eq=True, + order=True, # NOQA + converter=None, + label=None, + help=None, +): # NOQA """ A tri-boolean attribute with possible values of None, True and False. """ @@ -109,8 +134,16 @@ def TriBoolean(default=None, validator=None, repr=False, eq=True, order=True, # ) -def String(default=None, validator=None, repr=False, eq=True, order=True, # NOQA - converter=None, label=None, help=None,): # NOQA +def String( + default=None, + validator=None, + repr=False, + eq=True, + order=True, # NOQA + converter=None, + label=None, + help=None, +): # NOQA """ A string attribute. """ @@ -128,8 +161,16 @@ def String(default=None, validator=None, repr=False, eq=True, order=True, # NOQ ) -def Integer(default=0, validator=None, repr=False, eq=True, order=True, # NOQA - converter=None, label=None, help=None,): # NOQA +def Integer( + default=0, + validator=None, + repr=False, + eq=True, + order=True, # NOQA + converter=None, + label=None, + help=None, +): # NOQA """ An integer attribute. """ @@ -148,8 +189,16 @@ def Integer(default=0, validator=None, repr=False, eq=True, order=True, # NOQA ) -def Float(default=0.0, validator=None, repr=False, eq=True, order=True, # NOQA - converter=None, label=None, help=None,): # NOQA +def Float( + default=0.0, + validator=None, + repr=False, + eq=True, + order=True, # NOQA + converter=None, + label=None, + help=None, +): # NOQA """ A float attribute. """ @@ -167,9 +216,17 @@ def Float(default=0.0, validator=None, repr=False, eq=True, order=True, # NOQA ) -def List(item_type=typing.Any, default=attr.NOTHING, validator=None, - repr=False, eq=True, order=True, # NOQA - converter=None, label=None, help=None,): # NOQA +def List( + item_type=typing.Any, + default=attr.NOTHING, + validator=None, + repr=False, + eq=True, + order=True, # NOQA + converter=None, + label=None, + help=None, +): # NOQA """ A list attribute: the optional item_type defines the type of items it stores. """ @@ -190,9 +247,17 @@ def List(item_type=typing.Any, default=attr.NOTHING, validator=None, ) -def Mapping(value_type=typing.Any, default=attr.NOTHING, validator=None, - repr=False, eq=True, order=True, # NOQA - converter=None, help=None, label=None): # NOQA +def Mapping( + value_type=typing.Any, + default=attr.NOTHING, + validator=None, + repr=False, + eq=True, + order=True, # NOQA + converter=None, + help=None, + label=None, +): # NOQA """ A mapping attribute: the optional value_type defines the type of values it stores. The key is always a string. @@ -221,8 +286,17 @@ def Mapping(value_type=typing.Any, default=attr.NOTHING, validator=None, # FIXME: add proper support for dates!!! ################################################## -def Date(default=None, validator=None, repr=False, eq=True, order=True, # NOQA - converter=None, label=None, help=None,): # NOQA + +def Date( + default=None, + validator=None, + repr=False, + eq=True, + order=True, # NOQA + converter=None, + label=None, + help=None, +): # NOQA """ A date attribute. It always serializes to an ISO date string. Behavior is TBD and for now this is exactly a string. @@ -236,4 +310,4 @@ def Date(default=None, validator=None, repr=False, eq=True, order=True, # NOQA converter=converter, help=help, label=label, - ) \ No newline at end of file + ) diff --git a/src/fetchcode/packagedcode_models.py b/src/fetchcode/packagedcode_models.py index ecf89ac1..ba935b56 100644 --- a/src/fetchcode/packagedcode_models.py +++ b/src/fetchcode/packagedcode_models.py @@ -89,7 +89,9 @@ def logger_debug(*args): logger.setLevel(logging.DEBUG) def logger_debug(*args): - return logger.debug(' '.join(isinstance(a, string_types) and a or repr(a) for a in args)) + return logger.debug( + " ".join(isinstance(a, string_types) and a or repr(a) for a in args) + ) class BaseModel(object): @@ -122,11 +124,11 @@ def fields(cls): return [a.name for a in attr.fields(cls)] -party_person = 'person' +party_person = "person" # often loosely defined -party_project = 'project' +party_project = "project" # more formally defined -party_org = 'organization' +party_org = "organization" PARTY_TYPES = ( None, party_person, @@ -143,27 +145,23 @@ class Party(BaseModel): type = String( validator=choices(PARTY_TYPES), - label='party type', - help='the type of this party: One of: ' - +', '.join(p for p in PARTY_TYPES if p)) + label="party type", + help="the type of this party: One of: " + + ", ".join(p for p in PARTY_TYPES if p), + ) role = String( - label='party role', - help='A role for this party. Something such as author, ' - 'maintainer, contributor, owner, packager, distributor, ' - 'vendor, developer, owner, etc.') + label="party role", + help="A role for this party. Something such as author, " + "maintainer, contributor, owner, packager, distributor, " + "vendor, developer, owner, etc.", + ) - name = String( - label='name', - help='Name of this party.') + name = String(label="name", help="Name of this party.") - email = String( - label='email', - help='Email for this party.') + email = String(label="email", help="Email for this party.") - url = String( - label='url', - help='URL to a primary web page for this party.') + url = String(label="url", help="URL to a primary web page for this party.") @attr.s() @@ -200,41 +198,43 @@ class BasePackage(BaseModel): type = String( repr=True, - label='package type', - help='Optional. A short code to identify what is the type of this ' - 'package. For instance gem for a Rubygem, docker for container, ' - 'pypi for Python Wheel or Egg, maven for a Maven Jar, ' - 'deb for a Debian package, etc.') + label="package type", + help="Optional. A short code to identify what is the type of this " + "package. For instance gem for a Rubygem, docker for container, " + "pypi for Python Wheel or Egg, maven for a Maven Jar, " + "deb for a Debian package, etc.", + ) namespace = String( repr=True, - label='package namespace', - help='Optional namespace for this package.') + label="package namespace", + help="Optional namespace for this package.", + ) - name = String( - repr=True, - label='package name', - help='Name of the package.') + name = String(repr=True, label="package name", help="Name of the package.") version = String( repr=True, - label='package version', - help='Optional version of the package as a string.') + label="package version", + help="Optional version of the package as a string.", + ) qualifiers = Mapping( default=None, value_type=str, converter=lambda v: normalize_qualifiers(v, encode=False), - label='package qualifiers', - help='Optional mapping of key=value pairs qualifiers for this package') + label="package qualifiers", + help="Optional mapping of key=value pairs qualifiers for this package", + ) subpath = String( - label='extra package subpath', - help='Optional extra subpath inside a package and relative to the root ' - 'of this package') + label="extra package subpath", + help="Optional extra subpath inside a package and relative to the root " + "of this package", + ) def __attrs_post_init__(self, *args, **kwargs): - if not self.type and hasattr(self, 'default_type'): + if not self.type and hasattr(self, "default_type"): self.type = self.default_type @property @@ -245,8 +245,13 @@ def purl(self): if not self.name: return return PackageURL( - self.type, self.namespace, self.name, self.version, - self.qualifiers, self.subpath).to_string() + self.type, + self.namespace, + self.name, + self.version, + self.qualifiers, + self.subpath, + ).to_string() def repository_homepage_url(self, baseurl=default_web_baseurl): """ @@ -285,7 +290,7 @@ def set_purl(self, package_url): if not isinstance(package_url, PackageURL): package_url = PackageURL.from_string(package_url) - attribs = ['type', 'namespace', 'name', 'version', 'qualifiers', 'subpath'] + attribs = ["type", "namespace", "name", "version", "qualifiers", "subpath"] for att in attribs: self_val = getattr(self, att) purl_val = getattr(package_url, att) @@ -297,13 +302,13 @@ def to_dict(self, **kwargs): Return an OrderedDict of primitive Python types. """ mapping = attr.asdict(self, dict_factory=OrderedDict) - if not kwargs.get('exclude_properties'): - mapping['purl'] = self.purl - mapping['repository_homepage_url'] = self.repository_homepage_url() - mapping['repository_download_url'] = self.repository_download_url() - mapping['api_data_url'] = self.api_data_url() + if not kwargs.get("exclude_properties"): + mapping["purl"] = self.purl + mapping["repository_homepage_url"] = self.repository_homepage_url() + mapping["repository_download_url"] = self.repository_download_url() + mapping["api_data_url"] = self.api_data_url() if self.qualifiers: - mapping['qualifiers'] = normalize_qualifiers(self.qualifiers, encode=False) + mapping["qualifiers"] = normalize_qualifiers(self.qualifiers, encode=False) return mapping @classmethod @@ -313,6 +318,7 @@ def create(cls, ignore_unknown=True, **kwargs): Optionally `ignore_unknown` attributes provided in `kwargs`. """ from packagedcode import get_package_class + cls = get_package_class(kwargs, default=cls) return super(BasePackage, cls).create(ignore_unknown=ignore_unknown, **kwargs) @@ -325,37 +331,43 @@ class DependentPackage(BaseModel): purl = String( repr=True, - label='Dependent package URL', - help='A compact purl package URL. Typically when there is an unresolved requirement, there is no version. ' - 'If the dependency is resolved, the version should be added to the purl') + label="Dependent package URL", + help="A compact purl package URL. Typically when there is an unresolved requirement, there is no version. " + "If the dependency is resolved, the version should be added to the purl", + ) requirement = String( repr=True, - label='dependent package version requirement', - help='A string defining version(s)requirements. Package-type specific.') + label="dependent package version requirement", + help="A string defining version(s)requirements. Package-type specific.", + ) scope = String( repr=True, - label='dependency scope', - help='The scope of this dependency, such as runtime, install, etc. ' - 'This is package-type specific and is the original scope string.') + label="dependency scope", + help="The scope of this dependency, such as runtime, install, etc. " + "This is package-type specific and is the original scope string.", + ) is_runtime = Boolean( default=True, - label='is runtime flag', - help='True if this dependency is a runtime dependency.') + label="is runtime flag", + help="True if this dependency is a runtime dependency.", + ) is_optional = Boolean( default=False, - label='is optional flag', - help='True if this dependency is an optional dependency') + label="is optional flag", + help="True if this dependency is an optional dependency", + ) is_resolved = Boolean( default=False, - label='is resolved flag', - help='True if this dependency version requirement has ' - 'been resolved and this dependency url points to an ' - 'exact version.') + label="is resolved flag", + help="True if this dependency version requirement has " + "been resolved and this dependency url points to an " + "exact version.", + ) @attr.s() @@ -368,120 +380,122 @@ class Package(BasePackage): default_primary_language = None primary_language = String( - label='Primary programming language', - help='Primary programming language',) + label="Primary programming language", + help="Primary programming language", + ) description = String( - label='Description', - help='Description for this package. ' - 'By convention the first should be a summary when available.') + label="Description", + help="Description for this package. " + "By convention the first should be a summary when available.", + ) - release_date = Date( - label='release date', - help='Release date of the package') + release_date = Date(label="release date", help="Release date of the package") parties = List( item_type=Party, - label='parties', - help='A list of parties such as a person, project or organization.') + label="parties", + help="A list of parties such as a person, project or organization.", + ) - keywords = List( - item_type=str, - label='keywords', - help='A list of keywords.') + keywords = List(item_type=str, label="keywords", help="A list of keywords.") homepage_url = String( - label='homepage URL', - help='URL to the homepage for this package.') + label="homepage URL", help="URL to the homepage for this package." + ) - download_url = String( - label='Download URL', - help='A direct download URL.') + download_url = String(label="Download URL", help="A direct download URL.") - api_url = String( - label='API URL', - help='URL of API for this package.') + api_url = String(label="API URL", help="URL of API for this package.") size = Integer( default=None, - label='download size', - help='size of the package download in bytes') + label="download size", + help="size of the package download in bytes", + ) sha1 = String( - label='SHA1 checksum', - help='SHA1 checksum for this download in hexadecimal') + label="SHA1 checksum", help="SHA1 checksum for this download in hexadecimal" + ) md5 = String( - label='MD5 checksum', - help='MD5 checksum for this download in hexadecimal') + label="MD5 checksum", help="MD5 checksum for this download in hexadecimal" + ) sha256 = String( - label='SHA256 checksum', - help='SHA256 checksum for this download in hexadecimal') + label="SHA256 checksum", help="SHA256 checksum for this download in hexadecimal" + ) sha512 = String( - label='SHA512 checksum', - help='SHA512 checksum for this download in hexadecimal') + label="SHA512 checksum", help="SHA512 checksum for this download in hexadecimal" + ) bug_tracking_url = String( - label='bug tracking URL', - help='URL to the issue or bug tracker for this package') + label="bug tracking URL", + help="URL to the issue or bug tracker for this package", + ) code_view_url = String( - label='code view URL', - help='a URL where the code can be browsed online') + label="code view URL", help="a URL where the code can be browsed online" + ) vcs_url = String( - help='a URL to the VCS repository in the SPDX form of: ' - 'https://github.com/nexb/scancode-toolkit.git@405aaa4b3 ' - 'See SPDX specification "Package Download Location" ' - 'at https://spdx.org/spdx-specification-21-web-version#h.49x2ik5 ') + help="a URL to the VCS repository in the SPDX form of: " + "https://github.com/nexb/scancode-toolkit.git@405aaa4b3 " + 'See SPDX specification "Package Download Location" ' + "at https://spdx.org/spdx-specification-21-web-version#h.49x2ik5 " + ) copyright = String( - label='Copyright', - help='Copyright statements for this package. Typically one per line.') + label="Copyright", + help="Copyright statements for this package. Typically one per line.", + ) license_expression = String( - label='license expression', - help='The license expression for this package typically derived ' - 'from its declared license or .') + label="license expression", + help="The license expression for this package typically derived " + "from its declared license or .", + ) declared_license = String( - label='declared license', - help='The declared license mention, tag or text as found in a ' - 'package manifest.') + label="declared license", + help="The declared license mention, tag or text as found in a " + "package manifest.", + ) - notice_text = String( - label='notice text', - help='A notice text for this package.') + notice_text = String(label="notice text", help="A notice text for this package.") root_path = String( - label='package root path', - help='The path to the root of the package documented in this manifest ' - 'if any, such as a Maven .pom or a npm package.json parent directory.') + label="package root path", + help="The path to the root of the package documented in this manifest " + "if any, such as a Maven .pom or a npm package.json parent directory.", + ) dependencies = List( item_type=DependentPackage, - label='dependencies', - help='A list of DependentPackage for this package. ') + label="dependencies", + help="A list of DependentPackage for this package. ", + ) contains_source_code = TriBoolean( - label='contains source code', - help='Flag set to True if this package contains its own source code, None ' - 'if this is unknown, False if not.') + label="contains source code", + help="Flag set to True if this package contains its own source code, None " + "if this is unknown, False if not.", + ) source_packages = List( item_type=String, - label='List of related source code packages', + label="List of related source code packages", help='A list of related source code Package URLs (aka. "purl") for ' - 'this package. For instance an SRPM is the "source package" for a ' - 'binary RPM.') + 'this package. For instance an SRPM is the "source package" for a ' + "binary RPM.", + ) def __attrs_post_init__(self, *args, **kwargs): - if not self.type and hasattr(self, 'default_type'): + if not self.type and hasattr(self, "default_type"): self.type = self.default_type - if not self.primary_language and hasattr(self, 'default_primary_language'): + if not self.primary_language and hasattr(self, "default_primary_language"): self.primary_language = self.default_primary_language @classmethod @@ -520,7 +534,9 @@ def get_package_resources(cls, package_root, codebase): """ if not Package.is_ignored_package_resource(package_root, codebase): yield package_root - for resource in package_root.walk(codebase, topdown=True, ignored=Package.is_ignored_package_resource): + for resource in package_root.walk( + codebase, topdown=True, ignored=Package.is_ignored_package_resource + ): yield resource @classmethod @@ -533,6 +549,7 @@ def ignore_resource(cls, resource, codebase): @staticmethod def is_ignored_package_resource(resource, codebase): from packagedcode import PACKAGE_TYPES + return any(pt.ignore_resource(resource, codebase) for pt in PACKAGE_TYPES) def compute_normalized_license(self): @@ -591,12 +608,13 @@ def compute_normalized_license(declared_license): return from packagedcode import licensing + try: return licensing.get_normalized_expression(declared_license) except Exception: # FIXME: add logging # we never fail just for this - return 'unknown' + return "unknown" # Package types @@ -606,76 +624,99 @@ def compute_normalized_license(declared_license): @attr.s() class DebianPackage(Package): - metafiles = ('*.control',) - extensions = ('.deb',) - filetypes = ('debian binary package',) - mimetypes = ('application/x-archive', 'application/vnd.debian.binary-package',) - default_type = 'deb' + metafiles = ("*.control",) + extensions = (".deb",) + filetypes = ("debian binary package",) + mimetypes = ( + "application/x-archive", + "application/vnd.debian.binary-package", + ) + default_type = "deb" @attr.s() class JavaJar(Package): - metafiles = ('META-INF/MANIFEST.MF',) - extensions = ('.jar',) - filetypes = ('java archive ', 'zip archive',) - mimetypes = ('application/java-archive', 'application/zip',) - default_type = 'jar' - default_primary_language = 'Java' + metafiles = ("META-INF/MANIFEST.MF",) + extensions = (".jar",) + filetypes = ( + "java archive ", + "zip archive", + ) + mimetypes = ( + "application/java-archive", + "application/zip", + ) + default_type = "jar" + default_primary_language = "Java" @attr.s() class JavaWar(Package): - metafiles = ('WEB-INF/web.xml',) - extensions = ('.war',) - filetypes = ('java archive ', 'zip archive',) - mimetypes = ('application/java-archive', 'application/zip') - default_type = 'war' - default_primary_language = 'Java' + metafiles = ("WEB-INF/web.xml",) + extensions = (".war",) + filetypes = ( + "java archive ", + "zip archive", + ) + mimetypes = ("application/java-archive", "application/zip") + default_type = "war" + default_primary_language = "Java" @attr.s() class JavaEar(Package): - metafiles = ('META-INF/application.xml', 'META-INF/ejb-jar.xml') - extensions = ('.ear',) - filetypes = ('java archive ', 'zip archive',) - mimetypes = ('application/java-archive', 'application/zip') - default_type = 'ear' - default_primary_language = 'Java' + metafiles = ("META-INF/application.xml", "META-INF/ejb-jar.xml") + extensions = (".ear",) + filetypes = ( + "java archive ", + "zip archive", + ) + mimetypes = ("application/java-archive", "application/zip") + default_type = "ear" + default_primary_language = "Java" @attr.s() class Axis2Mar(Package): """Apache Axis2 module""" - metafiles = ('META-INF/module.xml',) - extensions = ('.mar',) - filetypes = ('java archive ', 'zip archive',) - mimetypes = ('application/java-archive', 'application/zip') - default_type = 'axis2' - default_primary_language = 'Java' + + metafiles = ("META-INF/module.xml",) + extensions = (".mar",) + filetypes = ( + "java archive ", + "zip archive", + ) + mimetypes = ("application/java-archive", "application/zip") + default_type = "axis2" + default_primary_language = "Java" @attr.s() class JBossSar(Package): - metafiles = ('META-INF/jboss-service.xml',) - extensions = ('.sar',) - filetypes = ('java archive ', 'zip archive',) - mimetypes = ('application/java-archive', 'application/zip') - default_type = 'jboss' - default_primary_language = 'Java' + metafiles = ("META-INF/jboss-service.xml",) + extensions = (".sar",) + filetypes = ( + "java archive ", + "zip archive", + ) + mimetypes = ("application/java-archive", "application/zip") + default_type = "jboss" + default_primary_language = "Java" @attr.s() class IvyJar(JavaJar): - metafiles = ('ivy.xml',) - default_type = 'ivy' - default_primary_language = 'Java' + metafiles = ("ivy.xml",) + default_type = "ivy" + default_primary_language = "Java" + # FIXME: move to bower.py @attr.s() class BowerPackage(Package): - metafiles = ('bower.json',) - default_type = 'bower' - default_primary_language = 'JavaScript' + metafiles = ("bower.json",) + default_type = "bower" + default_primary_language = "JavaScript" @classmethod def get_package_root(cls, manifest_resource, codebase): @@ -684,9 +725,9 @@ def get_package_root(cls, manifest_resource, codebase): @attr.s() class MeteorPackage(Package): - metafiles = ('package.js',) - default_type = 'meteor' - default_primary_language = 'JavaScript' + metafiles = ("package.js",) + default_type = "meteor" + default_primary_language = "JavaScript" @classmethod def get_package_root(cls, manifest_resource, codebase): @@ -696,27 +737,28 @@ def get_package_root(cls, manifest_resource, codebase): @attr.s() class CpanModule(Package): metafiles = ( - '*.pod', - '*.pm', - 'MANIFEST', - 'Makefile.PL', - 'META.yml', - 'META.json', - '*.meta', - 'dist.ini',) + "*.pod", + "*.pm", + "MANIFEST", + "Makefile.PL", + "META.yml", + "META.json", + "*.meta", + "dist.ini", + ) # TODO: refine me - extensions = ('.tar.gz',) - default_type = 'cpan' - default_primary_language = 'Perl' + extensions = (".tar.gz",) + default_type = "cpan" + default_primary_language = "Perl" # TODO: refine me: Go packages are a mess but something is emerging # TODO: move to and use godeps.py @attr.s() class Godep(Package): - metafiles = ('Godeps',) - default_type = 'golang' - default_primary_language = 'Go' + metafiles = ("Godeps",) + default_type = "golang" + default_primary_language = "Go" @classmethod def get_package_root(cls, manifest_resource, codebase): @@ -735,112 +777,126 @@ def get_package_root(cls, manifest_resource, codebase): @attr.s() class AndroidApp(Package): - filetypes = ('zip archive',) - mimetypes = ('application/zip',) - extensions = ('.apk',) - default_type = 'android' - default_primary_language = 'Java' + filetypes = ("zip archive",) + mimetypes = ("application/zip",) + extensions = (".apk",) + default_type = "android" + default_primary_language = "Java" # see http://tools.android.com/tech-docs/new-build-system/aar-formats @attr.s() class AndroidLibrary(Package): - filetypes = ('zip archive',) - mimetypes = ('application/zip',) + filetypes = ("zip archive",) + mimetypes = ("application/zip",) # note: Apache Axis also uses AAR extensions for plain Jars. # this could be decided based on internal structure - extensions = ('.aar',) - default_type = 'android-lib' - default_primary_language = 'Java' + extensions = (".aar",) + default_type = "android-lib" + default_primary_language = "Java" @attr.s() class MozillaExtension(Package): - filetypes = ('zip archive',) - mimetypes = ('application/zip',) - extensions = ('.xpi',) - default_type = 'mozilla' - default_primary_language = 'JavaScript' + filetypes = ("zip archive",) + mimetypes = ("application/zip",) + extensions = (".xpi",) + default_type = "mozilla" + default_primary_language = "JavaScript" @attr.s() class ChromeExtension(Package): - filetypes = ('data',) - mimetypes = ('application/octet-stream',) - extensions = ('.crx',) - default_type = 'chrome' - default_primary_language = 'JavaScript' + filetypes = ("data",) + mimetypes = ("application/octet-stream",) + extensions = (".crx",) + default_type = "chrome" + default_primary_language = "JavaScript" @attr.s() class IOSApp(Package): - filetypes = ('zip archive',) - mimetypes = ('application/zip',) - extensions = ('.ipa',) - default_type = 'ios' - default_primary_language = 'Objective-C' + filetypes = ("zip archive",) + mimetypes = ("application/zip",) + extensions = (".ipa",) + default_type = "ios" + default_primary_language = "Objective-C" @attr.s() class CabPackage(Package): - filetypes = ('microsoft cabinet',) - mimetypes = ('application/vnd.ms-cab-compressed',) - extensions = ('.cab',) - default_type = 'cab' + filetypes = ("microsoft cabinet",) + mimetypes = ("application/vnd.ms-cab-compressed",) + extensions = (".cab",) + default_type = "cab" @attr.s() class MsiInstallerPackage(Package): - filetypes = ('msi installer',) - mimetypes = ('application/x-msi',) - extensions = ('.msi',) - default_type = 'msi' + filetypes = ("msi installer",) + mimetypes = ("application/x-msi",) + extensions = (".msi",) + default_type = "msi" @attr.s() class InstallShieldPackage(Package): - filetypes = ('installshield',) - mimetypes = ('application/x-dosexec',) - extensions = ('.exe',) - default_type = 'installshield' + filetypes = ("installshield",) + mimetypes = ("application/x-dosexec",) + extensions = (".exe",) + default_type = "installshield" @attr.s() class NSISInstallerPackage(Package): - filetypes = ('nullsoft installer',) - mimetypes = ('application/x-dosexec',) - extensions = ('.exe',) - default_type = 'nsis' + filetypes = ("nullsoft installer",) + mimetypes = ("application/x-dosexec",) + extensions = (".exe",) + default_type = "nsis" @attr.s() class SharPackage(Package): - filetypes = ('posix shell script',) - mimetypes = ('text/x-shellscript',) - extensions = ('.sha', '.shar', '.bin',) - default_type = 'shar' + filetypes = ("posix shell script",) + mimetypes = ("text/x-shellscript",) + extensions = ( + ".sha", + ".shar", + ".bin", + ) + default_type = "shar" @attr.s() class AppleDmgPackage(Package): - filetypes = ('zlib compressed',) - mimetypes = ('application/zlib',) - extensions = ('.dmg', '.sparseimage',) - default_type = 'dmg' + filetypes = ("zlib compressed",) + mimetypes = ("application/zlib",) + extensions = ( + ".dmg", + ".sparseimage", + ) + default_type = "dmg" @attr.s() class IsoImagePackage(Package): - filetypes = ('iso 9660 cd-rom', 'high sierra cd-rom',) - mimetypes = ('application/x-iso9660-image',) - extensions = ('.iso', '.udf', '.img',) - default_type = 'iso' + filetypes = ( + "iso 9660 cd-rom", + "high sierra cd-rom", + ) + mimetypes = ("application/x-iso9660-image",) + extensions = ( + ".iso", + ".udf", + ".img", + ) + default_type = "iso" @attr.s() class SquashfsPackage(Package): - filetypes = ('squashfs',) - default_type = 'squashfs' + filetypes = ("squashfs",) + default_type = "squashfs" -# TODO: Add VM images formats(VMDK, OVA, OVF, VDI, etc) and Docker/other containers \ No newline at end of file +# TODO: Add VM images formats(VMDK, OVA, OVF, VDI, etc) and Docker/other containers diff --git a/src/fetchcode/vcs/__init__.py b/src/fetchcode/vcs/__init__.py index b8152134..e9eeadf2 100644 --- a/src/fetchcode/vcs/__init__.py +++ b/src/fetchcode/vcs/__init__.py @@ -61,4 +61,4 @@ def fetch_via_vcs(url, location=None): backend = vcs.get_backend_for_scheme(scheme) backend.obtain(dest=location, url=misc.hide_url(url)) - return VCSResponse(dest_dir=location, vcs_type=vcs_type, domain=domain) \ No newline at end of file + return VCSResponse(dest_dir=location, vcs_type=vcs_type, domain=domain) diff --git a/src/fetchcode/vcs/git.py b/src/fetchcode/vcs/git.py index 72bc9708..8d7d9bc9 100644 --- a/src/fetchcode/vcs/git.py +++ b/src/fetchcode/vcs/git.py @@ -43,4 +43,4 @@ def fetch_via_git(url, location=None): backend = vcs.get_backend(name="git") backend.obtain(dest=location, url=misc.hide_url(url)) - return VCSResponse(dest_dir=location, vcs_type="git", domain=domain) \ No newline at end of file + return VCSResponse(dest_dir=location, vcs_type="git", domain=domain) From 43ba80c7bc199814afa00fd13f9254dfb5ece225 Mon Sep 17 00:00:00 2001 From: Alexander Mazuruk Date: Thu, 10 Jun 2021 17:15:12 +0200 Subject: [PATCH 4/4] Format tests with psf/black As recommended by @TG1999 Signed-off-by: Alexander Mazuruk --- tests/test_fetch.py | 32 ++++----- tests/test_vcs.py | 157 +++++++++++++++++++++++++++++++++++------- tests/test_vcs_git.py | 40 +++++++++-- 3 files changed, 184 insertions(+), 45 deletions(-) diff --git a/tests/test_fetch.py b/tests/test_fetch.py index 09b0bf4a..0bb31a0a 100644 --- a/tests/test_fetch.py +++ b/tests/test_fetch.py @@ -21,45 +21,45 @@ from fetchcode import fetch -@mock.patch('fetchcode.requests.get') +@mock.patch("fetchcode.requests.get") def test_fetch_http_with_tempfile(mock_get): mock_get.return_value.headers = { - 'content-type': 'image/png', - 'content-length': '1000999', + "content-type": "image/png", + "content-length": "1000999", } - with mock.patch('fetchcode.open', mock.mock_open()) as mocked_file: - url = 'https://raw.githubusercontent.com/TG1999/converge/master/assets/Group%2022.png' + with mock.patch("fetchcode.open", mock.mock_open()) as mocked_file: + url = "https://raw.githubusercontent.com/TG1999/converge/master/assets/Group%2022.png" response = fetch(url=url) assert response is not None assert 1000999 == response.size assert url == response.url - assert 'image/png' == response.content_type + assert "image/png" == response.content_type -@mock.patch('fetchcode.FTP') +@mock.patch("fetchcode.FTP") def test_fetch_with_wrong_url(mock_get): with pytest.raises(Exception) as e_info: - url = 'ftp://speedtest/1KB.zip' + url = "ftp://speedtest/1KB.zip" response = fetch(url=url) - assert 'Not a valid URL' == e_info + assert "Not a valid URL" == e_info -@mock.patch('fetchcode.FTP', autospec=True) +@mock.patch("fetchcode.FTP", autospec=True) def test_fetch_ftp_with_tempfile(mock_ftp_constructor): mock_ftp = mock_ftp_constructor.return_value mock_ftp_constructor.return_value.size.return_value = 1024 - with mock.patch('fetchcode.open', mock.mock_open()) as mocked_file: - response = fetch('ftp://speedtest.tele2.net/1KB.zip') + with mock.patch("fetchcode.open", mock.mock_open()) as mocked_file: + response = fetch("ftp://speedtest.tele2.net/1KB.zip") assert 1024 == response.size - mock_ftp_constructor.assert_called_with('speedtest.tele2.net') + mock_ftp_constructor.assert_called_with("speedtest.tele2.net") assert mock_ftp.login.called == True - mock_ftp.cwd.assert_called_with('/') + mock_ftp.cwd.assert_called_with("/") assert mock_ftp.retrbinary.called def test_fetch_with_scheme_not_present(): with pytest.raises(Exception) as e_info: - url = 'abc://speedtest/1KB.zip' + url = "abc://speedtest/1KB.zip" response = fetch(url=url) - assert 'Not a supported/known scheme.' == e_info + assert "Not a supported/known scheme." == e_info diff --git a/tests/test_vcs.py b/tests/test_vcs.py index 869bf800..a8c64304 100644 --- a/tests/test_vcs.py +++ b/tests/test_vcs.py @@ -25,32 +25,142 @@ def obtain(dest, url): pass + @pytest.mark.parametrize( "url, vcs_type, domain", [ - pytest.param("git+http://github.com/jamesor/mongoose-versioner", "git", "github.com", id="git_http"), - pytest.param("git://github.com/jamesor/mongoose-versioner", "git", "github.com", id="git"), - pytest.param("git+https://github.com/jamesor/mongoose-versioner", "git", "github.com", id="git_https"), - pytest.param("git+ssh://github.com/jamesor/mongoose-versioner", "git", "github.com", id="git_ssh"), - pytest.param("git+file://github.com/jamesor/mongoose-versioner", "git", "github.com", id="git_file"), - pytest.param("git+git://github.com/jamesor/mongoose-versioner", "git", "github.com", id="git_git"), - pytest.param("bzr+http://gitlab.com/jamesor/mongoose-versioner", "bzr", "gitlab.com", id="bzr_http"), - pytest.param("bzr+https://gitlab.com/jamesor/mongoose-versioner", "bzr", "gitlab.com", id="bzr_https"), - pytest.param("bzr://gitlab.com/jamesor/mongoose-versioner", "bzr", "gitlab.com", id="bzr"), - pytest.param("bzr+ssh://gitlab.com/jamesor/mongoose-versioner", "bzr", "gitlab.com", id="bzr_ssh"), - pytest.param("bzr+ftp://gitlab.com/jamesor/mongoose-versioner", "bzr", "gitlab.com", id="bzr_ftp"), - pytest.param("bzr+sftp://gitlab.com/jamesor/mongoose-versioner", "bzr", "gitlab.com", id="bzr_sftp"), - pytest.param("bzr+lp://gitlab.com/jamesor/mongoose-versioner", "bzr", "gitlab.com", id="bzr_lp"), - pytest.param("hg://bitbucket.com/jamesor/mongoose-versioner", "hg", "bitbucket.com", id="hg"), - pytest.param("hg+file://bitbucket.com/jamesor/mongoose-versioner", "hg", "bitbucket.com", id="hg_file"), - pytest.param("hg+http://bitbucket.com/jamesor/mongoose-versioner", "hg", "bitbucket.com", id="hg_http"), - pytest.param("hg+https://bitbucket.com/jamesor/mongoose-versioner", "hg", "bitbucket.com", id="hg_https"), - pytest.param("hg+ssh://bitbucket.com/jamesor/mongoose-versioner", "hg", "bitbucket.com", id="hg_ssh"), - pytest.param("hg+static-http://bitbucket.com/jamesor/mongoose-versioner", "hg", "bitbucket.com", id="hg_static_http"), - pytest.param("svn://bitbucket.com/jamesor/mongoose-versioner", "svn", "bitbucket.com", id="svn"), - pytest.param("svn+http://bitbucket.com/jamesor/mongoose-versioner", "svn", "bitbucket.com", id="svn_http"), - pytest.param("svn+https://bitbucket.com/jamesor/mongoose-versioner", "svn", "bitbucket.com", id="svn_https"), - pytest.param("svn+svn://bitbucket.com/jamesor/mongoose-versioner", "svn", "bitbucket.com", id="svn_svn") + pytest.param( + "git+http://github.com/jamesor/mongoose-versioner", + "git", + "github.com", + id="git_http", + ), + pytest.param( + "git://github.com/jamesor/mongoose-versioner", "git", "github.com", id="git" + ), + pytest.param( + "git+https://github.com/jamesor/mongoose-versioner", + "git", + "github.com", + id="git_https", + ), + pytest.param( + "git+ssh://github.com/jamesor/mongoose-versioner", + "git", + "github.com", + id="git_ssh", + ), + pytest.param( + "git+file://github.com/jamesor/mongoose-versioner", + "git", + "github.com", + id="git_file", + ), + pytest.param( + "git+git://github.com/jamesor/mongoose-versioner", + "git", + "github.com", + id="git_git", + ), + pytest.param( + "bzr+http://gitlab.com/jamesor/mongoose-versioner", + "bzr", + "gitlab.com", + id="bzr_http", + ), + pytest.param( + "bzr+https://gitlab.com/jamesor/mongoose-versioner", + "bzr", + "gitlab.com", + id="bzr_https", + ), + pytest.param( + "bzr://gitlab.com/jamesor/mongoose-versioner", "bzr", "gitlab.com", id="bzr" + ), + pytest.param( + "bzr+ssh://gitlab.com/jamesor/mongoose-versioner", + "bzr", + "gitlab.com", + id="bzr_ssh", + ), + pytest.param( + "bzr+ftp://gitlab.com/jamesor/mongoose-versioner", + "bzr", + "gitlab.com", + id="bzr_ftp", + ), + pytest.param( + "bzr+sftp://gitlab.com/jamesor/mongoose-versioner", + "bzr", + "gitlab.com", + id="bzr_sftp", + ), + pytest.param( + "bzr+lp://gitlab.com/jamesor/mongoose-versioner", + "bzr", + "gitlab.com", + id="bzr_lp", + ), + pytest.param( + "hg://bitbucket.com/jamesor/mongoose-versioner", + "hg", + "bitbucket.com", + id="hg", + ), + pytest.param( + "hg+file://bitbucket.com/jamesor/mongoose-versioner", + "hg", + "bitbucket.com", + id="hg_file", + ), + pytest.param( + "hg+http://bitbucket.com/jamesor/mongoose-versioner", + "hg", + "bitbucket.com", + id="hg_http", + ), + pytest.param( + "hg+https://bitbucket.com/jamesor/mongoose-versioner", + "hg", + "bitbucket.com", + id="hg_https", + ), + pytest.param( + "hg+ssh://bitbucket.com/jamesor/mongoose-versioner", + "hg", + "bitbucket.com", + id="hg_ssh", + ), + pytest.param( + "hg+static-http://bitbucket.com/jamesor/mongoose-versioner", + "hg", + "bitbucket.com", + id="hg_static_http", + ), + pytest.param( + "svn://bitbucket.com/jamesor/mongoose-versioner", + "svn", + "bitbucket.com", + id="svn", + ), + pytest.param( + "svn+http://bitbucket.com/jamesor/mongoose-versioner", + "svn", + "bitbucket.com", + id="svn_http", + ), + pytest.param( + "svn+https://bitbucket.com/jamesor/mongoose-versioner", + "svn", + "bitbucket.com", + id="svn_https", + ), + pytest.param( + "svn+svn://bitbucket.com/jamesor/mongoose-versioner", + "svn", + "bitbucket.com", + id="svn_svn", + ), ], ) @mock.patch("fetchcode.vcs.vcs.get_backend_for_scheme") @@ -60,6 +170,7 @@ def test_fetch_via_vcs_returns_response(mock_backend, url, vcs_type, domain): assert response.vcs_type == vcs_type assert response.domain == domain + def test_fetch_with_invalid_scheme(): invalid_urls = [ "https://github.com/TG1999/fetchcode", diff --git a/tests/test_vcs_git.py b/tests/test_vcs_git.py index 3972b768..28d8014f 100644 --- a/tests/test_vcs_git.py +++ b/tests/test_vcs_git.py @@ -28,12 +28,39 @@ def obtain(dest, url): @pytest.mark.parametrize( "url, vcs_type, domain", [ - pytest.param("git+http://github.com/jamesor/mongoose-versioner", "git", "github.com", id="git_http"), - pytest.param("git://github.com/jamesor/mongoose-versioner", "git", "github.com", id="git"), - pytest.param("git+https://github.com/jamesor/mongoose-versioner", "git", "github.com", id="git_https"), - pytest.param("git+ssh://github.com/jamesor/mongoose-versioner", "git", "github.com", id="git_ssh"), - pytest.param("git+file://github.com/jamesor/mongoose-versioner", "git", "github.com", id="git_file"), - pytest.param("git+git://github.com/jamesor/mongoose-versioner", "git", "github.com", id="git_git") + pytest.param( + "git+http://github.com/jamesor/mongoose-versioner", + "git", + "github.com", + id="git_http", + ), + pytest.param( + "git://github.com/jamesor/mongoose-versioner", "git", "github.com", id="git" + ), + pytest.param( + "git+https://github.com/jamesor/mongoose-versioner", + "git", + "github.com", + id="git_https", + ), + pytest.param( + "git+ssh://github.com/jamesor/mongoose-versioner", + "git", + "github.com", + id="git_ssh", + ), + pytest.param( + "git+file://github.com/jamesor/mongoose-versioner", + "git", + "github.com", + id="git_file", + ), + pytest.param( + "git+git://github.com/jamesor/mongoose-versioner", + "git", + "github.com", + id="git_git", + ), ], ) @mock.patch("fetchcode.vcs.git.vcs.get_backend") @@ -43,6 +70,7 @@ def test_fetch_via_vcs_returns_response(mock_backend, url, vcs_type, domain): assert response.vcs_type == vcs_type assert response.domain == domain + def test_fetch_with_git_invalid_scheme(): invalid_urls = [ "https://github.com/TG1999/fetchcode",