Skip to content

Commit 43dc430

Browse files
AyanSinhaMahapatraAyanSinhaMahapatra
committed
Improve debian package detection
Detect and store more attributes from debian .dsc metadata files. Also properly detect and create packages from control and md5sums files. Reference: aboutcode-org/scancode.io#1151 Reference: aboutcode-org/purldb#3 Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>
1 parent cafcbcf commit 43dc430

File tree

11 files changed

+772
-11
lines changed

11 files changed

+772
-11
lines changed

‎src/packagedcode/debian.py

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,7 @@ def assign_package_to_resources(cls, package, resource, codebase, package_adder)
127127
class DebianControlFileInExtractedDebHandler(models.DatafileHandler):
128128
datasource_id = 'debian_control_extracted_deb'
129129
default_package_type = 'deb'
130-
path_patterns = ('*/control.tar.gz-extract/control',)
130+
path_patterns = ('*/control.tar.gz-extract/control','*/control.tar.xz-extract/control')
131131
description = 'Debian control file - extracted layout'
132132
documentation_url = 'https://www.debian.org/doc/debian-policy/ch-controlfields.html'
133133

@@ -590,7 +590,7 @@ def parse_debian_files_list(location, datasource_id, package_type):
590590
else:
591591
name = None
592592
# For DebianMd5sumFilelistInPackageHandler we cannot infer name
593-
if not name == "md5sums":
593+
if not filename == "md5sums":
594594
name = filename
595595

596596
file_references = []
@@ -663,6 +663,19 @@ def build_package_data(debian_data, datasource_id, package_type='deb', distro=No
663663
party = models.Party(role='maintainer', name=maintainer_name, email=maintainer_email)
664664
parties.append(party)
665665

666+
uploaders = debian_data.get('uploaders')
667+
if uploaders:
668+
for uploader in uploaders.split(", "):
669+
uploader_name, uploader_email = parse_debian_maintainers(uploader)
670+
party = models.Party(role='uploader', name=uploader_name, email=uploader_email)
671+
parties.append(party)
672+
673+
vcs_url = debian_data.get('vcs-git')
674+
if vcs_url and ' ' in vcs_url:
675+
vcs_url = vcs_url.split(' ')[0]
676+
677+
code_view_url = debian_data.get('vcs-browser')
678+
666679
keywords = []
667680
keyword = debian_data.get('section')
668681
if keyword:
@@ -712,6 +725,8 @@ def build_package_data(debian_data, datasource_id, package_type='deb', distro=No
712725
qualifiers=qualifiers,
713726
description=description,
714727
homepage_url=homepage_url,
728+
vcs_url=vcs_url,
729+
code_view_url=code_view_url,
715730
size=size,
716731
source_packages=source_packages,
717732
keywords=keywords,

‎tests/packagedcode/data/debian/control.expected.json

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,13 @@
140140
"name": "Bdale Garbee",
141141
"email": "bdale@gag.com",
142142
"url": null
143+
},
144+
{
145+
"type": null,
146+
"role": "uploader",
147+
"name": "Carl Worth",
148+
"email": "cworth@cworth.org",
149+
"url": null
143150
}
144151
],
145152
"keywords": [
@@ -153,8 +160,8 @@
153160
"sha256": null,
154161
"sha512": null,
155162
"bug_tracking_url": null,
156-
"code_view_url": null,
157-
"vcs_url": null,
163+
"code_view_url": "https://salsa.debian.org/debian/gzip",
164+
"vcs_url": "https://salsa.debian.org/debian/gzip.git",
158165
"copyright": null,
159166
"holder": null,
160167
"declared_license_expression": null,

‎tests/packagedcode/data/debian/dsc_files/adduser_3.118+deb11u1.dsc.expected.json

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,13 @@
1919
"name": "Debian Adduser Developers",
2020
"email": "adduser@packages.debian.org",
2121
"url": null
22+
},
23+
{
24+
"type": null,
25+
"role": "uploader",
26+
"name": "Marc Haber",
27+
"email": "mh+debian-packages@zugschlus.de",
28+
"url": null
2229
}
2330
],
2431
"keywords": [],
@@ -30,8 +37,8 @@
3037
"sha256": null,
3138
"sha512": null,
3239
"bug_tracking_url": null,
33-
"code_view_url": null,
34-
"vcs_url": null,
40+
"code_view_url": "https://salsa.debian.org/debian/adduser",
41+
"vcs_url": "https://salsa.debian.org/debian/adduser.git",
3542
"copyright": null,
3643
"holder": null,
3744
"declared_license_expression": null,
@@ -84,6 +91,13 @@
8491
"name": "Debian Adduser Developers",
8592
"email": "adduser@packages.debian.org",
8693
"url": null
94+
},
95+
{
96+
"type": null,
97+
"role": "uploader",
98+
"name": "Marc Haber",
99+
"email": "mh+debian-packages@zugschlus.de",
100+
"url": null
87101
}
88102
],
89103
"keywords": [],
@@ -95,8 +109,8 @@
95109
"sha256": null,
96110
"sha512": null,
97111
"bug_tracking_url": null,
98-
"code_view_url": null,
99-
"vcs_url": null,
112+
"code_view_url": "https://salsa.debian.org/debian/adduser",
113+
"vcs_url": "https://salsa.debian.org/debian/adduser.git",
100114
"copyright": null,
101115
"holder": null,
102116
"declared_license_expression": null,

‎tests/packagedcode/data/debian/dsc_files/zsh_5.7.1-1+deb10u1.dsc

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
-----BEGIN PGP SIGNED MESSAGE-----
2+
Hash: SHA512
3+
4+
Format: 3.0 (quilt)
5+
Source: zsh
6+
Binary: zsh-common, zsh, zsh-doc, zsh-static, zsh-dev
7+
Architecture: any all
8+
Version: 5.7.1-1+deb10u1
9+
Maintainer: Debian Zsh Maintainers <pkg-zsh-devel@lists.alioth.debian.org>
10+
Uploaders: Michael Prokop <mika@debian.org>, Axel Beckert <abe@debian.org>, Frank Terbeck <ft@bewatermyfriend.org>, Richard Hartmann <richih@debian.org>
11+
Homepage: https://www.zsh.org/
12+
Standards-Version: 4.3.0
13+
Vcs-Browser: https://salsa.debian.org/debian/zsh
14+
Vcs-Git: https://salsa.debian.org/debian/zsh.git -b debian
15+
Testsuite: autopkgtest
16+
Testsuite-Triggers: adequate
17+
Build-Depends: bsdmainutils, cm-super-minimal, debhelper-compat (= 12), dpkg-dev (>= 1.16.2~), ghostscript, groff, groff-base, libcap-dev [linux-any], libncursesw5-dev, libpcre3-dev, texinfo (>= 5~), texlive-fonts-recommended, texlive-latex-base, texlive-latex-recommended, yodl (>= 3.08.01) | yodl (<< 3.08.00)
18+
Package-List:
19+
zsh deb shells optional arch=any
20+
zsh-common deb shells optional arch=all
21+
zsh-dev deb libdevel optional arch=any
22+
zsh-doc deb doc optional arch=all
23+
zsh-static deb shells optional arch=any
24+
Checksums-Sha1:
25+
b2fd47fdb878aa681edc974864e37baae9b0d6b7 2776796 zsh_5.7.1.orig.tar.xz
26+
14a8d38d3fae5b8eec0b124be5c943a60c4e8fec 87028 zsh_5.7.1-1+deb10u1.debian.tar.xz
27+
Checksums-Sha256:
28+
439aafb4341522c307a67a2680e95fadb1b35a5c7f332089b9cc5154496570ca 2776796 zsh_5.7.1.orig.tar.xz
29+
ecbe22ed6a2b8dcaf10eff02b6b66583ce8d108a936624fc424c72188dea1ddd 87028 zsh_5.7.1-1+deb10u1.debian.tar.xz
30+
Files:
31+
acc1a32ef5b3120ead5c6f0d011ceb76 2776796 zsh_5.7.1.orig.tar.xz
32+
d0f5fe26d9548331d9757e26b95c7aaf 87028 zsh_5.7.1-1+deb10u1.debian.tar.xz
33+
34+
-----BEGIN PGP SIGNATURE-----
35+
36+
iQIzBAEBCgAdFiEERoyJeTtCmBnp12Ema+Zjx1o1yXUFAmIMYK4ACgkQa+Zjx1o1
37+
yXWTgw//SzaiMwbl8gSiQ/mB7+KPCeqL0o44q8oVAbj0WWhlh6yVUdjIcoHk6Zr3
38+
Om4b3C5pU1l64jQ/JH5GP4i/7NnHAttobDL5zh9yvXAjUkUSfhBB6mJPeKT2QlHY
39+
3SsvWiwtZOU8hPdwW48ATFiSSiYka+hKEM5nr5f/kI++yg2Rg7d1b5svaD3BMytI
40+
Hl25comMwLgPeL4aIB7j8UL0sQT2zb/O5CK4aZr3CPUxy9OzBAwpQuukNNsJJ3Zg
41+
g+a6Y6ADLAdhkav1JeZIF6LQ4Q9sk4Cqkp+CFenvRtDv2BDdosOXwOnBUBlm1A2j
42+
yPuOcPdgSDqctMohSWVehDqGZK8dPjIOVq/sX3yKzwtNWdil9aErjGWxQVRSMyfg
43+
fqEaCHYdpRpfrE8QWk+7TqovRqv3PaAmY6wahEAd5buniRScDF2yobT95qDOMnfj
44+
fR1YDDH0cRkePHt+VHQUzM1mO4c/5Obhxfd8uExR+8KpO4SnsT8z7wa1J1O+jWGg
45+
88zuhJ2HUCDGHdVlpgksagnNkmbT1Oohml6YfTAtDO4eXXJeTPmSpOYNzRCMsN0D
46+
V1tY+awJuR0VSNjo+3X7DKyC/pKJiCzk+MBVogmFvzfBwJhYiLbVnVElNic/2J2J
47+
Rz6VMPKIYqKUTyIgsQML0IkK8D9Lpp1LHqLNbUVt2aWcYlqgPBU=
48+
=cPZC
49+
-----END PGP SIGNATURE-----

‎tests/packagedcode/data/debian/dsc_files/zsh_5.7.1-1+deb10u1.dsc.expected.json

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
{
2+
"packages": [
3+
{
4+
"type": "deb",
5+
"namespace": "debian",
6+
"name": "zsh",
7+
"version": "5.7.1-1+deb10u1",
8+
"qualifiers": {
9+
"arch": "any all"
10+
},
11+
"subpath": null,
12+
"primary_language": null,
13+
"description": null,
14+
"release_date": null,
15+
"parties": [
16+
{
17+
"type": null,
18+
"role": "maintainer",
19+
"name": "Debian Zsh Maintainers",
20+
"email": "pkg-zsh-devel@lists.alioth.debian.org",
21+
"url": null
22+
},
23+
{
24+
"type": null,
25+
"role": "uploader",
26+
"name": "Michael Prokop",
27+
"email": "mika@debian.org",
28+
"url": null
29+
},
30+
{
31+
"type": null,
32+
"role": "uploader",
33+
"name": "Axel Beckert",
34+
"email": "abe@debian.org",
35+
"url": null
36+
},
37+
{
38+
"type": null,
39+
"role": "uploader",
40+
"name": "Frank Terbeck",
41+
"email": "ft@bewatermyfriend.org",
42+
"url": null
43+
},
44+
{
45+
"type": null,
46+
"role": "uploader",
47+
"name": "Richard Hartmann",
48+
"email": "richih@debian.org",
49+
"url": null
50+
}
51+
],
52+
"keywords": [],
53+
"homepage_url": "https://www.zsh.org/",
54+
"download_url": null,
55+
"size": null,
56+
"sha1": null,
57+
"md5": null,
58+
"sha256": null,
59+
"sha512": null,
60+
"bug_tracking_url": null,
61+
"code_view_url": "https://salsa.debian.org/debian/zsh",
62+
"vcs_url": "https://salsa.debian.org/debian/zsh.git",
63+
"copyright": null,
64+
"holder": null,
65+
"declared_license_expression": null,
66+
"declared_license_expression_spdx": null,
67+
"license_detections": [],
68+
"other_license_expression": null,
69+
"other_license_expression_spdx": null,
70+
"other_license_detections": [],
71+
"extracted_license_statement": null,
72+
"notice_text": null,
73+
"source_packages": [
74+
"pkg:deb/debian/zsh"
75+
],
76+
"extra_data": {},
77+
"repository_homepage_url": null,
78+
"repository_download_url": null,
79+
"api_data_url": null,
80+
"package_uid": "pkg:deb/debian/zsh@5.7.1-1%2Bdeb10u1?arch=any%20all&uuid=fixed-uid-done-for-testing-5642512d1758",
81+
"datafile_paths": [
82+
"zsh_5.7.1-1+deb10u1.dsc"
83+
],
84+
"datasource_ids": [
85+
"debian_source_control_dsc"
86+
],
87+
"purl": "pkg:deb/debian/zsh@5.7.1-1%2Bdeb10u1?arch=any%20all"
88+
}
89+
],
90+
"dependencies": [],
91+
"files": [
92+
{
93+
"path": "zsh_5.7.1-1+deb10u1.dsc",
94+
"type": "file",
95+
"package_data": [
96+
{
97+
"type": "deb",
98+
"namespace": "debian",
99+
"name": "zsh",
100+
"version": "5.7.1-1+deb10u1",
101+
"qualifiers": {
102+
"arch": "any all"
103+
},
104+
"subpath": null,
105+
"primary_language": null,
106+
"description": null,
107+
"release_date": null,
108+
"parties": [
109+
{
110+
"type": null,
111+
"role": "maintainer",
112+
"name": "Debian Zsh Maintainers",
113+
"email": "pkg-zsh-devel@lists.alioth.debian.org",
114+
"url": null
115+
},
116+
{
117+
"type": null,
118+
"role": "uploader",
119+
"name": "Michael Prokop",
120+
"email": "mika@debian.org",
121+
"url": null
122+
},
123+
{
124+
"type": null,
125+
"role": "uploader",
126+
"name": "Axel Beckert",
127+
"email": "abe@debian.org",
128+
"url": null
129+
},
130+
{
131+
"type": null,
132+
"role": "uploader",
133+
"name": "Frank Terbeck",
134+
"email": "ft@bewatermyfriend.org",
135+
"url": null
136+
},
137+
{
138+
"type": null,
139+
"role": "uploader",
140+
"name": "Richard Hartmann",
141+
"email": "richih@debian.org",
142+
"url": null
143+
}
144+
],
145+
"keywords": [],
146+
"homepage_url": "https://www.zsh.org/",
147+
"download_url": null,
148+
"size": null,
149+
"sha1": null,
150+
"md5": null,
151+
"sha256": null,
152+
"sha512": null,
153+
"bug_tracking_url": null,
154+
"code_view_url": "https://salsa.debian.org/debian/zsh",
155+
"vcs_url": "https://salsa.debian.org/debian/zsh.git",
156+
"copyright": null,
157+
"holder": null,
158+
"declared_license_expression": null,
159+
"declared_license_expression_spdx": null,
160+
"license_detections": [],
161+
"other_license_expression": null,
162+
"other_license_expression_spdx": null,
163+
"other_license_detections": [],
164+
"extracted_license_statement": null,
165+
"notice_text": null,
166+
"source_packages": [
167+
"pkg:deb/debian/zsh"
168+
],
169+
"file_references": [],
170+
"extra_data": {},
171+
"dependencies": [],
172+
"repository_homepage_url": null,
173+
"repository_download_url": null,
174+
"api_data_url": null,
175+
"datasource_id": "debian_source_control_dsc",
176+
"purl": "pkg:deb/debian/zsh@5.7.1-1%2Bdeb10u1?arch=any%20all"
177+
}
178+
],
179+
"for_packages": [
180+
"pkg:deb/debian/zsh@5.7.1-1%2Bdeb10u1?arch=any%20all&uuid=fixed-uid-done-for-testing-5642512d1758"
181+
],
182+
"scan_errors": []
183+
}
184+
]
185+
}

0 commit comments

Comments
 (0)