Add ability to "group" pipeline steps to control inclusion #1045 (#1055)

Signed-off-by: tdruez <tdruez@nexb.com>

Author: tdruez

CHANGELOG.rst

@@ -4,13 +4,18 @@ Changelog
 v33.2.0 (unreleased)
 --------------------
 
+- Add ability to "group" pipeline steps to control their inclusion in a pipeline run.
+  The groups can be selected in the UI, or provided using the
+  "pipeline_name:group1,group2" syntax in CLI and REST API.
+  https://github.com/nexB/scancode.io/issues/1045
+
 - Refine pipeline choices in the "Add pipeline" modal based on the project context.
    * When there is at least one existing pipeline in the project, the modal now includes
      all addon pipelines along with the existing pipeline for selection.
    * In cases where no pipelines are assigned to the project, the modal displays all
      base (non-addon) pipelines for user selection.
 
-   https://github.com/nexB/scancode.io/issues/
+   https://github.com/nexB/scancode.io/issues/1071
 
 v33.1.0 (2024-02-02)
 --------------------

docs/command-line-interface.rst

@@ -84,6 +84,11 @@ Optional arguments:
 
 - ``--pipeline PIPELINES`` Pipelines names to add on the project.
 
+.. tip::
+    Use the "pipeline_name:group1,group2" syntax to select steps groups:
+
+    ``--pipeline map_deploy_to_develop:Java,JavaScript``
+
 - ``--input-file INPUTS_FILES`` Input file locations to copy in the :guilabel:`input/`
   work directory.
 
@@ -190,6 +195,11 @@ add the docker pipeline to your project::
 
     $ scanpipe add-pipeline --project foo analyze_docker_image
 
+.. tip::
+    Use the "pipeline_name:group1,group2" syntax to select steps groups:
+
+    ``--pipeline map_deploy_to_develop:Java,JavaScript``
+
 
 `$ scanpipe execute --project PROJECT`
 --------------------------------------
@@ -201,6 +211,7 @@ Optional arguments:
 - ``--async`` Add the pipeline run to the tasks queue for execution by a worker instead
   of running in the current thread.
 
+
 `$ scanpipe show-pipeline --project PROJECT`
 --------------------------------------------
 

docs/rest-api.rst

@@ -278,6 +278,11 @@ Data:
     - ``pipeline``: The pipeline name
     - ``execute_now``: ``true`` or ``false``
 
+.. tip::
+    Use the "pipeline_name:group1,group2" syntax to select steps groups:
+
+    ``"pipeline": "map_deploy_to_develop:Java,JavaScript"``
+
 Using cURL:
 
 .. code-block:: console

scanpipe/api/views.py

@@ -247,10 +247,11 @@ def add_pipeline(self, request, *args, **kwargs):
 
         pipeline = request.data.get("pipeline")
         if pipeline:
-            pipeline = scanpipe_app.get_new_pipeline_name(pipeline)
-            if pipeline in scanpipe_app.pipelines:
+            pipeline_name, groups = scanpipe_app.extract_group_from_pipeline(pipeline)
+            pipeline_name = scanpipe_app.get_new_pipeline_name(pipeline_name)
+            if pipeline_name in scanpipe_app.pipelines:
                 execute_now = request.data.get("execute_now")
-                project.add_pipeline(pipeline, execute_now)
+                project.add_pipeline(pipeline_name, execute_now, selected_groups=groups)
                 return Response({"status": "Pipeline added."})
 
             message = {"status": f"{pipeline} is not a valid pipeline."}

scanpipe/apps.py

@@ -189,6 +189,17 @@ def get_new_pipeline_name(pipeline_name):
             return new_name
         return pipeline_name
 
+    @staticmethod
+    def extract_group_from_pipeline(pipeline):
+        pipeline_name = pipeline
+        groups = None
+
+        if ":" in pipeline:
+            pipeline_name, value = pipeline.split(":", maxsplit=1)
+            groups = value.split(",") if value else []
+
+        return pipeline_name, groups
+
     def get_scancode_licenses(self):
         """
         Load licenses-related information from the ScanCode-toolkit ``licensedcode``

scanpipe/forms.py

@@ -100,6 +100,14 @@ def handle_inputs(self, project):
             project.add_input_source(download_url=url)
 
 
+class GroupChoiceField(forms.MultipleChoiceField):
+    widget = forms.CheckboxSelectMultiple
+
+    def valid_value(self, value):
+        """Accept all values."""
+        return True
+
+
 class PipelineBaseForm(forms.Form):
     pipeline = forms.ChoiceField(
         choices=scanpipe_app.get_pipeline_choices(),
@@ -110,12 +118,14 @@ class PipelineBaseForm(forms.Form):
         initial=True,
         required=False,
     )
+    selected_groups = GroupChoiceField(required=False)
 
     def handle_pipeline(self, project):
         pipeline = self.cleaned_data["pipeline"]
         execute_now = self.cleaned_data["execute_now"]
+        selected_groups = self.cleaned_data.get("selected_groups", [])
         if pipeline:
-            project.add_pipeline(pipeline, execute_now)
+            project.add_pipeline(pipeline, execute_now, selected_groups)
 
 
 class ProjectForm(InputsBaseForm, PipelineBaseForm, forms.ModelForm):
@@ -127,6 +137,7 @@ class Meta:
             "input_urls",
             "pipeline",
             "execute_now",
+            "selected_groups",
         ]
 
     def __init__(self, *args, **kwargs):
@@ -158,32 +169,11 @@ def save(self, project):
 
 class AddPipelineForm(PipelineBaseForm):
     pipeline = forms.ChoiceField(
+        choices=scanpipe_app.get_pipeline_choices(),
         widget=forms.RadioSelect(),
         required=True,
     )
 
-    def __init__(self, project_runs=None, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-
-        # The pipeline choices are determined based on the project context:
-        # 1. If no pipelines are assigned to the project:
-        #    Include all base (non-addon) pipelines.
-        # 2. If at least one pipeline already exists on the project:
-        #    Include all addon pipelines and the existing pipeline (useful for
-        #    potential re-runs in debug mode).
-        project_run_names = {run.pipeline_name for run in project_runs or []}
-
-        pipeline_choices = [
-            (name, pipeline_class.get_summary())
-            for name, pipeline_class in scanpipe_app.pipelines.items()
-            # no pipelines are assigned to the project
-            if (not project_runs and not pipeline_class.is_addon)
-            # at least one pipeline already exists on the project
-            or (project_runs and (name in project_run_names or pipeline_class.is_addon))
-        ]
-
-        self.fields["pipeline"].choices = pipeline_choices
-
     def save(self, project):
         self.handle_pipeline(project)
         return project

scanpipe/management/commands/__init__.py

@@ -152,14 +152,22 @@ def add_arguments(self, parser):
             action="append",
             dest="input_files",
             default=list(),
-            help="Input file locations to copy in the input/ work directory.",
+            help=(
+                "Input file locations to copy in the input/ work directory. "
+                'Use the "filename:tag" syntax to tag input files such as '
+                '"path/filename:tag"'
+            ),
         )
         parser.add_argument(
             "--input-url",
             action="append",
             dest="input_urls",
             default=list(),
-            help="Input URLs to download in the input/ work directory.",
+            help=(
+                "Input URLs to download in the input/ work directory. "
+                'Use the "url#tag" syntax to tag downloaded files such as '
+                '"https://url.com/filename#tag"'
+            ),
         )
         parser.add_argument(
             "--copy-codebase",
@@ -251,19 +259,32 @@ def validate_copy_from(copy_from):
             raise CommandError(f"{copy_from} is not a directory")
 
 
-def validate_pipelines(pipeline_names):
+def extract_group_from_pipelines(pipelines):
+    """
+    Add support for the ":group1,group2" suffix in pipeline data.
+
+    For example: "map_deploy_to_develop:Java,JavaScript"
+    """
+    pipelines_data = {}
+    for pipeline in pipelines:
+        pipeline_name, groups = scanpipe_app.extract_group_from_pipeline(pipeline)
+        pipelines_data[pipeline_name] = groups
+    return pipelines_data
+
+
+def validate_pipelines(pipelines_data):
     """Raise an error if one of the `pipeline_names` is not available."""
     # Backward compatibility with old pipeline names.
-    pipeline_names = [
-        scanpipe_app.get_new_pipeline_name(pipeline_name)
-        for pipeline_name in pipeline_names
-    ]
+    pipelines_data = {
+        scanpipe_app.get_new_pipeline_name(pipeline_name): groups
+        for pipeline_name, groups in pipelines_data.items()
+    }
 
-    for pipeline_name in pipeline_names:
+    for pipeline_name in pipelines_data.keys():
         if pipeline_name not in scanpipe_app.pipelines:
             raise CommandError(
                 f"{pipeline_name} is not a valid pipeline. \n"
                 f"Available: {', '.join(scanpipe_app.pipelines.keys())}"
             )
 
-    return pipeline_names
+    return pipelines_data

scanpipe/management/commands/add-pipeline.py

@@ -23,6 +23,7 @@
 from django.template.defaultfilters import pluralize
 
 from scanpipe.management.commands import ProjectCommand
+from scanpipe.management.commands import extract_group_from_pipelines
 from scanpipe.management.commands import validate_pipelines
 
 
@@ -38,13 +39,16 @@ def add_arguments(self, parser):
             help="One or more pipeline names.",
         )
 
-    def handle(self, *pipeline_names, **options):
-        super().handle(*pipeline_names, **options)
+    def handle(self, *pipelines, **options):
+        super().handle(*pipelines, **options)
 
-        pipeline_names = validate_pipelines(pipeline_names)
-        for pipeline_name in pipeline_names:
-            self.project.add_pipeline(pipeline_name)
+        pipelines_data = extract_group_from_pipelines(pipelines)
+        pipelines_data = validate_pipelines(pipelines_data)
 
+        for pipeline_name, selected_groups in pipelines_data.items():
+            self.project.add_pipeline(pipeline_name, selected_groups=selected_groups)
+
+        pipeline_names = pipelines_data.keys()
         msg = (
             f"Pipeline{pluralize(pipeline_names)} {', '.join(pipeline_names)} "
             f"added to the project"

scanpipe/management/commands/create-project.py

@@ -26,6 +26,7 @@
 from django.core.management.base import BaseCommand
 
 from scanpipe.management.commands import AddInputCommandMixin
+from scanpipe.management.commands import extract_group_from_pipelines
 from scanpipe.management.commands import validate_copy_from
 from scanpipe.management.commands import validate_pipelines
 from scanpipe.models import Project
@@ -43,8 +44,9 @@ def add_arguments(self, parser):
             dest="pipelines",
             default=list(),
             help=(
-                "Pipelines names to add to the project."
-                "The pipelines are added and executed based on their given order."
+                "Pipelines names to add to the project. "
+                "The pipelines are added and executed based on their given order. "
+                'Groups can be provided using the "pipeline_name:group1,group2" syntax.'
             ),
         )
         parser.add_argument(
@@ -68,7 +70,7 @@ def add_arguments(self, parser):
 
     def handle(self, *args, **options):
         name = options["name"]
-        pipeline_names = options["pipelines"]
+        pipelines = options["pipelines"]
         input_files = options["input_files"]
         input_urls = options["input_urls"]
         copy_from = options["copy_codebase"]
@@ -84,22 +86,24 @@ def handle(self, *args, **options):
             raise CommandError("\n".join(e.messages))
 
         # Run validation before creating the project in the database
-        pipeline_names = validate_pipelines(pipeline_names)
+        pipelines_data = extract_group_from_pipelines(pipelines)
+        pipelines_data = validate_pipelines(pipelines_data)
+
         input_files_data = self.extract_tag_from_input_files(input_files)
         self.validate_input_files(input_files=input_files_data.keys())
         validate_copy_from(copy_from)
 
-        if execute and not pipeline_names:
+        if execute and not pipelines:
             raise CommandError("The --execute option requires one or more pipelines.")
 
         project.save()
+        self.project = project
         msg = f"Project {name} created with work directory {project.work_directory}"
         self.stdout.write(msg, self.style.SUCCESS)
 
-        for pipeline_name in pipeline_names:
-            project.add_pipeline(pipeline_name)
+        for pipeline_name, selected_groups in pipelines_data.items():
+            self.project.add_pipeline(pipeline_name, selected_groups=selected_groups)
 
-        self.project = project
         if input_files:
             self.handle_input_files(input_files_data)
 

scanpipe/management/commands/show-pipeline.py

@@ -32,4 +32,7 @@ def handle(self, *args, **options):
 
         for run in self.project.runs.all():
             status_code = self.get_run_status_code(run)
-            self.stdout.write(f" [{status_code}] {run.pipeline_name}")
+            output = f" [{status_code}] {run.pipeline_name}"
+            if run.selected_groups:
+                output += f" ({','.join(run.selected_groups)})"
+            self.stdout.write(output)