Skip to content

Commit a087f31

Browse files
keshav-spacekeshav-space
authored
Add pipeline to publish scan to federatedcode (#1400)
* Add pipeline to publish scan to federatedcode - Addon pipeline to push package scan to federatedcode Signed-off-by: Keshav Priyadarshi <git@keshav.space> * Add docstrings to pipeline steps Signed-off-by: Keshav Priyadarshi <git@keshav.space> * Store scan result in scancodeio.json file Signed-off-by: Keshav Priyadarshi <git@keshav.space> * Add test for federatedcode Signed-off-by: Keshav Priyadarshi <git@keshav.space> * Add project_purl field to project Signed-off-by: Keshav Priyadarshi <git@keshav.space> * Use project_purl to push scan result Signed-off-by: Keshav Priyadarshi <git@keshav.space> * Update tests Signed-off-by: Keshav Priyadarshi <git@keshav.space> * Validate code format Signed-off-by: Keshav Priyadarshi <git@keshav.space> * Add docs for FederatedCode pipeline Signed-off-by: Keshav Priyadarshi <git@keshav.space> * Remove unused imports Signed-off-by: Keshav Priyadarshi <git@keshav.space> * Omit __init__.py from namespace package directory - namespace package should not contain __init__.py - see https://packaging.python.org/en/latest/guides/packaging-namespace-packages/#native-namespace-packages Signed-off-by: Keshav Priyadarshi <git@keshav.space> * Address reviews Signed-off-by: Keshav Priyadarshi <git@keshav.space> * Address reviews Signed-off-by: Keshav Priyadarshi <git@keshav.space> * Address review Signed-off-by: Keshav Priyadarshi <git@keshav.space> * Fix test Signed-off-by: Keshav Priyadarshi <git@keshav.space> * Add unit test for clean_purl Signed-off-by: Keshav Priyadarshi <git@keshav.space> --------- Signed-off-by: Keshav Priyadarshi <git@keshav.space>
1 parent 6c3f4cd commit a087f31

File tree

16 files changed

+494
-0
lines changed

16 files changed

+494
-0
lines changed

‎CHANGELOG.rst

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,12 @@ v34.9.0 (unreleased)
1818
"policies.yml" files, or global app settings.
1919
https://github.com/aboutcode-org/scancode.io/issues/386
2020

21+
- Add a new ``PublishToFederatedCode`` pipeline (addon) to push scan result
22+
to FederatedCode.
23+
https://github.com/nexB/scancode.io/pull/1400
24+
25+
- Add new ``purl`` field to project model. https://github.com/nexB/scancode.io/pull/1400
26+
2127
v34.8.3 (2024-10-30)
2228
--------------------
2329

‎aboutcode/__init__.py

Whitespace-only changes.

‎docs/application-settings.rst

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -338,6 +338,28 @@ API key using ``MATCHCODEIO_API_KEY``::
338338

339339
MATCHCODEIO_API_KEY=insert_your_api_key_here
340340

341+
.. _scancodeio_settings_federatedcode:
342+
343+
FEDERATEDCODE
344+
^^^^^^^^^^^^^
345+
346+
FederatedCode is decentralized and federated metadata for software applications
347+
stored in Git repositories.
348+
349+
350+
To configure your local environment, set the following in your ``.env`` file::
351+
352+
FEDERATEDCODE_GIT_ACCOUNT_URL=https://<Address to your git account>/
353+
354+
FEDERATEDCODE_GIT_SERVICE_TOKEN=insert_your_git_api_key_here
355+
356+
Also provide the name and email that will be used to sign off on commits to Git repositories::
357+
358+
FEDERATEDCODE_GIT_SERVICE_NAME=insert_name_here
359+
360+
FEDERATEDCODE_GIT_SERVICE_EMAIL=insert_email_here
361+
362+
341363
.. _scancodeio_settings_fetch_authentication:
342364

343365
Fetch Authentication

‎docs/built-in-pipelines.rst

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,20 @@ Populate PurlDB (addon)
188188
:members:
189189
:member-order: bysource
190190

191+
.. _pipeline_publish_to_federatedcode:
192+
193+
Publish To FederatedCode (addon)
194+
--------------------------------
195+
196+
.. warning::
197+
This pipeline requires access to a FederatedCode service.
198+
Refer to :ref:`scancodeio_settings_federatedcode` to configure access to
199+
FederatedCode in your ScanCode.io instance.
200+
201+
.. autoclass:: scanpipe.pipelines.publish_to_federatedcode.PublishToFederatedCode()
202+
:members:
203+
:member-order: bysource
204+
191205
.. _pipeline_scan_codebase:
192206

193207
Scan Codebase

‎scancodeio/settings.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -418,3 +418,10 @@
418418
MATCHCODEIO_USER = env.str("MATCHCODEIO_USER", default="")
419419
MATCHCODEIO_PASSWORD = env.str("MATCHCODEIO_PASSWORD", default="")
420420
MATCHCODEIO_API_KEY = env.str("MATCHCODEIO_API_KEY", default="")
421+
422+
# FederatedCode integration
423+
424+
FEDERATEDCODE_GIT_ACCOUNT_URL = env.str("FEDERATEDCODE_GIT_ACCOUNT_URL", default="")
425+
FEDERATEDCODE_GIT_SERVICE_TOKEN = env.str("FEDERATEDCODE_GIT_SERVICE_TOKEN", default="")
426+
FEDERATEDCODE_GIT_SERVICE_NAME = env.str("FEDERATEDCODE_GIT_SERVICE_NAME", default="")
427+
FEDERATEDCODE_GIT_SERVICE_EMAIL = env.str("FEDERATEDCODE_GIT_SERVICE_EMAIL", default="")

‎scanpipe/api/serializers.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -209,6 +209,7 @@ class Meta:
209209
"name",
210210
"url",
211211
"uuid",
212+
"purl",
212213
"upload_file",
213214
"upload_file_tag",
214215
"input_urls",

‎scanpipe/forms.py

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
from django.core.exceptions import ObjectDoesNotExist
2626
from django.core.exceptions import ValidationError
2727

28+
from packageurl import PackageURL
2829
from taggit.forms import TagField
2930
from taggit.forms import TagWidget
3031

@@ -480,12 +481,31 @@ class Meta:
480481
fields = [
481482
"name",
482483
"notes",
484+
"purl",
483485
]
484486
widgets = {
485487
"name": forms.TextInput(attrs={"class": "input"}),
486488
"notes": forms.Textarea(attrs={"rows": 3, "class": "textarea is-dynamic"}),
489+
"purl": forms.TextInput(
490+
attrs={
491+
"class": "input",
492+
"placeholder": "pkg:npm/lodash@4.7.21",
493+
}
494+
),
487495
}
488496

497+
def clean_purl(self):
498+
"""Validate the Project PURL."""
499+
purl = self.cleaned_data.get("purl")
500+
501+
if purl:
502+
try:
503+
PackageURL.from_string(purl)
504+
except ValueError:
505+
raise forms.ValidationError("PURL must be a valid PackageURL")
506+
507+
return purl
508+
489509
def __init__(self, *args, **kwargs):
490510
"""Load initial values from Project ``settings`` field."""
491511
super().__init__(*args, **kwargs)

‎scanpipe/migrations/0069_project_purl.py

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
# Generated by Django 5.1.3 on 2024-11-08 12:47
2+
3+
from django.db import migrations, models
4+
5+
6+
class Migration(migrations.Migration):
7+
8+
dependencies = [
9+
('scanpipe', '0068_rename_discovered_dependencies_attribute'),
10+
]
11+
12+
operations = [
13+
migrations.AddField(
14+
model_name='project',
15+
name='purl',
16+
field=models.CharField(blank=True, help_text='Package URL for the project, used for pushing project scan result to FederatedCode. This should be the PURL of the input.', max_length=2048),
17+
),
18+
]

‎scanpipe/models.py

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -562,6 +562,16 @@ class Project(UUIDPKModel, ExtraDataFieldMixin, UpdateMixin, models.Model):
562562
notes = models.TextField(blank=True)
563563
settings = models.JSONField(default=dict, blank=True)
564564
labels = TaggableManager(through=UUIDTaggedItem)
565+
purl = models.CharField(
566+
max_length=2048,
567+
blank=True,
568+
help_text=_(
569+
"Package URL (PURL) for the project, required for pushing the project's "
570+
"scan result to FederatedCode. For example, if the input is an input URL "
571+
"like https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz, the "
572+
"corresponding PURL would be pkg:npm/lodash@4.17.21."
573+
),
574+
)
565575

566576
objects = ProjectQuerySet.as_manager()
567577

@@ -705,6 +715,7 @@ def clone(
705715
"""Clone this project using the provided ``clone_name`` as new project name."""
706716
new_project = Project.objects.create(
707717
name=clone_name,
718+
purl=self.purl,
708719
settings=self.settings if copy_settings else {},
709720
)
710721

‎scanpipe/pipelines/publish_to_federatedcode.py

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
# SPDX-License-Identifier: Apache-2.0
2+
#
3+
# http://nexb.com and https://github.com/aboutcode-org/scancode.io
4+
# The ScanCode.io software is licensed under the Apache License version 2.0.
5+
# Data generated with ScanCode.io is provided as-is without warranties.
6+
# ScanCode is a trademark of nexB Inc.
7+
#
8+
# You may not use this software except in compliance with the License.
9+
# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
10+
# Unless required by applicable law or agreed to in writing, software distributed
11+
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
12+
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
13+
# specific language governing permissions and limitations under the License.
14+
#
15+
# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
16+
# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
17+
# ScanCode.io should be considered or used as legal advice. Consult an Attorney
18+
# for any legal advice.
19+
#
20+
# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
21+
# Visit https://github.com/aboutcode-org/scancode.io for support and download.
22+
23+
24+
from scanpipe.pipelines import Pipeline
25+
from scanpipe.pipes import federatedcode
26+
27+
28+
class PublishToFederatedCode(Pipeline):
29+
"""
30+
Publish package scan to FederatedCode.
31+
32+
This pipeline commits the project scan result in FederatedCode Git repository.
33+
It uses ``Project PURL`` to determine the Git repository and the
34+
exact directory path where the scan should be stored.
35+
"""
36+
37+
download_inputs = False
38+
is_addon = True
39+
40+
@classmethod
41+
def steps(cls):
42+
return (
43+
cls.check_federatedcode_eligibility,
44+
cls.get_package_repository,
45+
cls.clone_repository,
46+
cls.add_scan_result,
47+
cls.commit_and_push_changes,
48+
cls.delete_local_clone,
49+
)
50+
51+
def check_federatedcode_eligibility(self):
52+
"""
53+
Check if the project fulfills the following criteria for
54+
pushing the project result to FederatedCode.
55+
"""
56+
federatedcode.check_federatedcode_eligibility(project=self.project)
57+
58+
def get_package_repository(self):
59+
"""Get the Git repository URL and scan path for a given package."""
60+
self.package_git_repo, self.package_scan_file = (
61+
federatedcode.get_package_repository(
62+
project_purl=self.project.purl, logger=self.log
63+
)
64+
)
65+
66+
def clone_repository(self):
67+
"""Clone repository to local_path."""
68+
self.repo = federatedcode.clone_repository(
69+
repo_url=self.package_git_repo,
70+
logger=self.log,
71+
)
72+
73+
def add_scan_result(self):
74+
"""Add package scan result to the local Git repository."""
75+
self.relative_file_path = federatedcode.add_scan_result(
76+
project=self.project,
77+
repo=self.repo,
78+
package_scan_file=self.package_scan_file,
79+
logger=self.log,
80+
)
81+
82+
def commit_and_push_changes(self):
83+
"""Commit and push changes to remote repository."""
84+
federatedcode.commit_and_push_changes(
85+
repo=self.repo,
86+
file_to_commit=str(self.relative_file_path),
87+
purl=self.project.purl,
88+
logger=self.log,
89+
)
90+
self.log(
91+
f"Scan result for '{self.project.purl}' "
92+
f"pushed to '{self.package_git_repo}'"
93+
)
94+
95+
def delete_local_clone(self):
96+
"""Remove local clone."""
97+
federatedcode.delete_local_clone(repo=self.repo)

0 commit comments

Comments
 (0)