Add pipeline selected groups in create project API endpoint #1426 (#1427)

* Add pipeline selected groups in create project API endpoint #1426

Signed-off-by: tdruez <tdruez@nexb.com>

* Add proper pipeline validation in the OrderedMultiplePipelineChoiceField #1426

Signed-off-by: tdruez <tdruez@nexb.com>

---------

Signed-off-by: tdruez <tdruez@nexb.com>

Author: tdruez

CHANGELOG.rst

@@ -1,6 +1,12 @@
 Changelog
 =========
 
+v34.9.0 (unreleased)
+--------------------
+
+- Add ability to declared pipeline selected groups in create project REST API endpoint.
+  https://github.com/aboutcode-org/scancode.io/issues/1426
+
 v34.8.3 (2024-10-30)
 --------------------
 

docs/rest-api.rst

@@ -126,6 +126,20 @@ Using cURL:
     To tag the ``upload_file``, you can provide the tag value using the
     ``upload_file_tag`` field.
 
+.. tip::
+
+    You can declare multiple pipelines to be executed at the project creation using a
+    list of pipeline names or a comma-separated string:
+
+    - ``"pipeline": ["scan_single_package", "scan_for_virus"]``
+    - ``"pipeline": "scan_single_package,scan_for_virus"``
+
+.. tip::
+
+    Use the "pipeline_name:group1,group2" syntax to select steps groups:
+
+    ``"pipeline": "map_deploy_to_develop:Java,JavaScript"``
+
 Using Python and the **"requests"** library:
 
 .. code-block:: python

scanpipe/api/serializers.py

@@ -69,8 +69,12 @@ def __init__(self, *args, **kwargs):
         self.fields["pipeline"].choices = scanpipe_app.get_pipeline_choices()
 
 
-class OrderedMultipleChoiceField(serializers.MultipleChoiceField):
-    """Forcing outputs as list() in place of set() to keep the ordering integrity."""
+class OrderedMultiplePipelineChoiceField(serializers.MultipleChoiceField):
+    """
+    Forcing outputs as list() in place of set() to keep the ordering integrity.
+    The field validation is bypassed and delegated to the ``project.add_pipeline``
+    method called in the ``ProjectSerializer.create`` method.
+    """
 
     def to_internal_value(self, data):
         if isinstance(data, str):
@@ -80,15 +84,17 @@ def to_internal_value(self, data):
         if not self.allow_empty and len(data) == 0:
             self.fail("empty")
 
-        # Backward compatibility with old pipeline names.
-        # This will need to be refactored in case this OrderedMultipleChoiceField
-        # class is used for another field that is not ``pipeline`` related.
-        data = [scanpipe_app.get_new_pipeline_name(pipeline) for pipeline in data]
+        # Adds support for providing pipeline names as a comma-separated single string.
+        data = [item.strip() for value in data for item in value.split(",")]
 
-        return [
-            super(serializers.MultipleChoiceField, self).to_internal_value(item)
-            for item in data
-        ]
+        # Pipeline validation
+        for pipeline in data:
+            pipeline_name, _ = scanpipe_app.extract_group_from_pipeline(pipeline)
+            pipeline_name = scanpipe_app.get_new_pipeline_name(pipeline_name)
+            if pipeline_name not in scanpipe_app.pipelines:
+                self.fail("invalid_choice", input=pipeline_name)
+
+        return data
 
     def to_representation(self, value):
         return [self.choice_strings_to_values.get(str(item), item) for item in value]
@@ -163,7 +169,7 @@ class ProjectSerializer(
     TaggitSerializer,
     serializers.ModelSerializer,
 ):
-    pipeline = OrderedMultipleChoiceField(
+    pipeline = OrderedMultiplePipelineChoiceField(
         choices=(),
         required=False,
         write_only=True,
@@ -302,7 +308,7 @@ def create(self, validated_data):
         upload_file = validated_data.pop("upload_file", None)
         upload_file_tag = validated_data.pop("upload_file_tag", "")
         input_urls = validated_data.pop("input_urls", [])
-        pipeline = validated_data.pop("pipeline", [])
+        pipelines = validated_data.pop("pipeline", [])
         execute_now = validated_data.pop("execute_now", False)
         webhook_url = validated_data.pop("webhook_url", None)
         webhooks = validated_data.pop("webhooks", [])
@@ -315,8 +321,10 @@ def create(self, validated_data):
         for url in input_urls:
             project.add_input_source(download_url=url)
 
-        for pipeline_name in pipeline:
-            project.add_pipeline(pipeline_name, execute_now)
+        for pipeline in pipelines:
+            pipeline_name, groups = scanpipe_app.extract_group_from_pipeline(pipeline)
+            pipeline_name = scanpipe_app.get_new_pipeline_name(pipeline_name)
+            project.add_pipeline(pipeline_name, execute_now, selected_groups=groups)
 
         if webhook_url:
             project.add_webhook_subscription(target_url=webhook_url)

scanpipe/tests/test_api.py

@@ -447,19 +447,39 @@ def test_scanpipe_api_project_create_multiple_pipelines(self):
             "pipeline": "analyze_docker_image,scan_single_package",
         }
         response = self.csrf_client.post(self.project_list_url, data)
-        self.assertEqual(status.HTTP_400_BAD_REQUEST, response.status_code)
-        expected = {
+        self.assertEqual(status.HTTP_201_CREATED, response.status_code)
+        self.assertEqual(2, len(response.data["runs"]))
+        self.assertEqual(
+            "analyze_docker_image", response.data["runs"][0]["pipeline_name"]
+        )
+        self.assertEqual(
+            "scan_single_package", response.data["runs"][1]["pipeline_name"]
+        )
+
+        data = {
+            "name": "Mix of string and list plus selected groups",
             "pipeline": [
-                ErrorDetail(
-                    string=(
-                        '"analyze_docker_image,scan_single_package" '
-                        "is not a valid choice."
-                    ),
-                    code="invalid_choice",
-                )
-            ]
+                "analyze_docker_image",
+                "inspect_packages:StaticResolver,scan_single_package",
+            ],
         }
-        self.assertEqual(expected, response.data)
+        response = self.csrf_client.post(self.project_list_url, data)
+        self.assertEqual(status.HTTP_201_CREATED, response.status_code)
+        self.assertEqual(
+            "analyze_docker_image", response.data["runs"][0]["pipeline_name"]
+        )
+        self.assertEqual("inspect_packages", response.data["runs"][1]["pipeline_name"])
+        self.assertEqual(
+            "scan_single_package", response.data["runs"][2]["pipeline_name"]
+        )
+        self.assertEqual(
+            ["StaticResolver"], response.data["runs"][1]["selected_groups"]
+        )
+        runs = Project.objects.get(name=data["name"]).runs.all()
+        self.assertEqual("analyze_docker_image", runs[0].pipeline_name)
+        self.assertEqual("inspect_packages", runs[1].pipeline_name)
+        self.assertEqual("scan_single_package", runs[2].pipeline_name)
+        self.assertEqual(["StaticResolver"], runs[1].selected_groups)
 
     def test_scanpipe_api_project_create_pipeline_old_name_compatibility(self):
         data = {
@@ -498,6 +518,20 @@ def test_scanpipe_api_project_create_labels(self):
         project = Project.objects.get(name=data["name"])
         self.assertEqual(data["labels"], sorted(project.labels.names()))
 
+    def test_scanpipe_api_project_create_pipeline_groups(self):
+        data = {
+            "name": "Project1",
+            "pipeline": "inspect_packages:StaticResolver",
+        }
+        response = self.csrf_client.post(self.project_list_url, data)
+        self.assertEqual(status.HTTP_201_CREATED, response.status_code)
+        self.assertEqual(
+            ["StaticResolver"], response.data["runs"][0]["selected_groups"]
+        )
+        run = Project.objects.get(name="Project1").runs.get()
+        self.assertEqual("inspect_packages", run.pipeline_name)
+        self.assertEqual(["StaticResolver"], run.selected_groups)
+
     def test_scanpipe_api_project_create_webhooks(self):
         data = {
             "name": "Project1",
@@ -862,7 +896,9 @@ def test_scanpipe_api_project_action_add_pipeline_groups(self):
         }
         response = self.csrf_client.post(url, data=data)
         self.assertEqual({"status": "Pipeline added."}, response.data)
-        self.assertEqual("analyze_docker_image", self.project1.runs.get().pipeline_name)
+        run = self.project1.runs.get()
+        self.assertEqual("analyze_docker_image", run.pipeline_name)
+        self.assertEqual(["group1", "group2"], run.selected_groups)
 
     def test_scanpipe_api_project_action_add_input(self):
         url = reverse("project-add-input", args=[self.project1.uuid])