Add pipeline step selection for a run execution #1303 (#1310)

Signed-off-by: tdruez <tdruez@nexb.com>

Author: tdruez

CHANGELOG.rst

@@ -1,6 +1,16 @@
 Changelog
 =========
 
+v34.7.1 (unreleased)
+--------------------
+
+- Add pipeline step selection for a run execution.
+  This allows to run a pipeline in an advanced mode allowing to skip some steps,
+  or restart from a step, like the last failed step.
+  The steps can be edited from the Run "status" modal using the "Select steps" button.
+  This is an advanced feature and should we used with caution.
+  https://github.com/nexB/scancode.io/issues/1303
+
 v34.7.0 (2024-07-02)
 --------------------
 

scanpipe/api/serializers.py

@@ -114,6 +114,7 @@ class Meta:
             "status",
             "description",
             "selected_groups",
+            "selected_steps",
             "project",
             "uuid",
             "created_date",

scanpipe/forms.py

@@ -29,6 +29,7 @@
 from taggit.forms import TagWidget
 
 from scanpipe.models import Project
+from scanpipe.models import Run
 from scanpipe.pipelines import convert_markdown_to_html
 from scanpipe.pipes import fetch
 
@@ -119,9 +120,11 @@ def handle_inputs(self, project):
             project.add_input_source(download_url=url)
 
 
-class GroupChoiceField(forms.MultipleChoiceField):
+class CheckboxChoiceField(forms.MultipleChoiceField):
     widget = forms.CheckboxSelectMultiple
 
+
+class SelectedGroupsCheckboxChoiceField(CheckboxChoiceField):
     def valid_value(self, value):
         """Accept all values."""
         return True
@@ -137,7 +140,7 @@ class PipelineBaseForm(forms.Form):
         initial=True,
         required=False,
     )
-    selected_groups = GroupChoiceField(required=False)
+    selected_groups = SelectedGroupsCheckboxChoiceField(required=False)
 
     def handle_pipeline(self, project):
         pipeline = self.cleaned_data["pipeline"]
@@ -531,3 +534,31 @@ def clean_clone_name(self):
 
     def save(self, *args, **kwargs):
         return self.project.clone(**self.cleaned_data)
+
+
+class PipelineRunStepSelectionForm(forms.ModelForm):
+    selected_steps = CheckboxChoiceField(required=False)
+
+    class Meta:
+        model = Run
+        fields = [
+            "selected_steps",
+        ]
+
+    def __init__(self, *args, **kwargs):
+        if not kwargs.get("instance"):
+            raise ValueError("An Run object is required to instantiate this form.")
+
+        super().__init__(*args, **kwargs)
+        pipeline_class = self.instance.pipeline_class
+        choices = self.get_step_choices(pipeline_class)
+        self.fields["selected_steps"].choices = choices
+
+        # All step checkboxes are selected by default unless already defined on the run
+        if not self.instance.selected_steps:
+            self.initial["selected_steps"] = [choice[0] for choice in choices]
+
+    @staticmethod
+    def get_step_choices(pipeline_class):
+        """Return a `choices` list of tuple suitable for a Django ChoiceField."""
+        return [(step.__name__, step.__name__) for step in pipeline_class.get_steps()]

scanpipe/migrations/0063_run_selected_steps.py

@@ -0,0 +1,19 @@
+# Generated by Django 5.0.6 on 2024-07-03 09:59
+
+import scanpipe.models
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('scanpipe', '0062_dependency_resolver_update'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='run',
+            name='selected_steps',
+            field=models.JSONField(blank=True, null=True, validators=[scanpipe.models.validate_none_or_list]),
+        ),
+    ]

scanpipe/models.py

@@ -1775,6 +1775,9 @@ class Run(UUIDPKModel, ProjectRelatedModel, AbstractTaskFieldsModel):
     selected_groups = models.JSONField(
         null=True, blank=True, validators=[validate_none_or_list]
     )
+    selected_steps = models.JSONField(
+        null=True, blank=True, validators=[validate_none_or_list]
+    )
 
     objects = RunQuerySet.as_manager()
 

scanpipe/pipelines/__init__.py

@@ -197,6 +197,7 @@ def execute(self):
         self.log(f"Pipeline [{self.pipeline_name}] starting")
 
         steps = self.get_steps(groups=self.run.selected_groups)
+        selected_steps = self.run.selected_steps
 
         if self.download_inputs:
             steps = (self.__class__.download_missing_inputs,) + steps
@@ -207,6 +208,10 @@ def execute(self):
         for current_index, step in enumerate(steps, start=1):
             step_name = step.__name__
 
+            if selected_steps and step_name not in selected_steps:
+                self.log(f"Step [{step_name}] skipped")
+                continue
+
             self.run.set_current_step(f"{current_index}/{steps_count} {step_name}")
             self.log(f"Step [{step_name}] starting")
             step_start_time = timer()

scanpipe/templates/scanpipe/includes/run_step_selection_form.html

@@ -0,0 +1,19 @@
+<div id="run-step-selection-box" class="box has-background-warning-light">
+  <div class="has-text-weight-semibold mb-3">
+    Warning: Step Selection is an advanced feature.<br>
+    Removing steps may compromise data integrity. Use with caution.
+  </div>
+  {% include 'scanpipe/includes/form_errors.html' %}
+  <form hx-post="{% url 'project_run_step_selection' run.uuid %}"
+        hx-target="#run-step-selection-box"
+        hx-swap="outerHTML"
+  >{% csrf_token %}
+    <div class="field">
+      <label class="label">Select steps:</label>
+      <div class="control">
+        {{ form.selected_steps }}
+      </div>
+    </div>
+    <button class="button is-smaller is-link">Update steps</button>
+  </form>
+</div>

scanpipe/templates/scanpipe/modals/run_modal.html

@@ -33,6 +33,7 @@
       if (execute_pipeline_links) {
         execute_pipeline_links.addEventListener('click', displayOverlay);
       }
+      htmx.process($modal);
     }).catch(function (error) {
       console.warn('Error:', error);
     });

scanpipe/templates/scanpipe/modals/run_modal_content.html

@@ -29,10 +29,18 @@
         </div>
       {% endif %}
       {% if run.status == run.Status.NOT_STARTED or run.status == run.Status.QUEUED %}
-        <a href="{% url 'project_delete_pipeline' project.slug run.uuid %}" class="execute-pipeline-link has-text-danger ml-3">
+        <a href="{% url 'project_delete_pipeline' project.slug run.uuid %}" class="execute-pipeline-link has-text-danger ml-2">
           <i class="fa-solid fa-trash-alt"></i>
           Delete pipeline
         </a>
+        <a href="#" class="ml-4"
+           hx-get="{% url 'project_run_step_selection' run.uuid %}"
+           hx-target="#run-step-selection-box"
+           hx-swap="outerHTML"
+        >
+          <i class="fa-regular fa-square-check"></i>
+          Select steps
+        </a>
       {% endif %}
       {% if run.task_exitcode %}
         <div class="control">
@@ -59,6 +67,7 @@
         </a>
       {% endif %}
     </div>
+    <div id="run-step-selection-box"></div>
     <div class="field is-grouped is-grouped-multiline">
       {% if run.execution_time %}
         <div class="control">

scanpipe/tests/data/asgiref/asgiref-3.3.0_load_inventory_expected.json

@@ -16,6 +16,7 @@
           "pipeline_name": "load_inventory",
           "status": "not_started",
           "selected_groups": null,
+          "selected_steps": null,
           "scancodeio_version": "",
           "task_id": null,
           "task_start_date": null,