Add filtering by label and pipeline in the flush-projects management command (#1690)

Signed-off-by: tdruez <tdruez@nexb.com>

Author: tdruez

CHANGELOG.rst

@@ -4,6 +4,10 @@ Changelog
 v34.12.0 (unreleased)
 ---------------------
 
+- Add filtering by label and pipeline in the ``flush-projects`` management command.
+  Also, a new ``--dry-run`` option is available to test the filters before applying
+  the deletion.
+
 - Add support for using Package URL (purl) as project input.
   This implementation is based on ``purl2url.get_download_url``.
   https://github.com/aboutcode-org/scancode.io/issues/1383

docs/command-line-interface.rst

@@ -549,6 +549,14 @@ Optional arguments:
 
     scanpipe flush-projects --retain-days 7
 
+- ``--dry-run`` Do not delete any projects; just print the ones that would be flushed.
+
+- ``--label LABELS`` Filter projects by the provided label.
+  Multiple labels can be provided by using this argument multiple times.
+
+- ``--pipeline PIPELINES`` Filter projects by the provided pipeline name.
+  Multiple pipeline name can be provided by using this argument multiple times.
+
 - ``--no-input`` Does not prompt the user for input of any kind.
 
 

pyproject.toml

@@ -36,3 +36,5 @@ max-complexity = 10
 # Allow the usage of assert in the test_spdx file.
 "**/test_spdx.py*" = ["S101"]
 "scanpipe/pipes/spdx.py" = ["UP006", "UP035"]
+# Allow complexity in management commands
+"scanpipe/management/commands/*" = ["C901"]

scanpipe/management/commands/flush-projects.py

@@ -48,28 +48,72 @@ def add_arguments(self, parser):
             ),
             default=0,
         )
+        parser.add_argument(
+            "--label",
+            action="append",
+            dest="labels",
+            default=list(),
+            help=(
+                "Filter projects by the provided label. "
+                "Multiple labels can be provided by using this argument multiple times."
+            ),
+        )
+        parser.add_argument(
+            "--pipeline",
+            action="append",
+            dest="pipelines",
+            default=list(),
+            help=(
+                "Filter projects by the provided pipeline name. "
+                "Multiple pipeline name can be provided by using this argument multiple"
+                " times."
+            ),
+        )
         parser.add_argument(
             "--no-input",
             action="store_false",
             dest="interactive",
             help="Do not prompt the user for input of any kind.",
         )
+        parser.add_argument(
+            "--dry-run",
+            action="store_true",
+            help=(
+                "Do not delete any projects; just print the ones that would be flushed."
+            ),
+        )
 
     def handle(self, *inputs, **options):
         verbosity = options["verbosity"]
         retain_days = options["retain_days"]
-        projects = Project.objects.all()
+        labels = options["labels"]
+        pipelines = options["pipelines"]
+        dry_run = options["dry_run"]
+        projects = Project.objects.order_by("-created_date").distinct()
 
         if retain_days:
             cutoff_date = timezone.now() - datetime.timedelta(days=retain_days)
             projects = projects.filter(created_date__lt=cutoff_date)
 
+        if labels:
+            projects = projects.filter(labels__name__in=labels)
+
+        if pipelines:
+            projects = projects.filter(runs__pipeline_name__in=pipelines)
+
         projects_count = projects.count()
         if projects_count == 0:
             if verbosity > 0:
-                self.stdout.write("No projects to remove.")
+                self.stdout.write("No projects to delete.")
             sys.exit(0)
 
+        if dry_run:
+            msg = self.style.WARNING(f"{projects_count} projects would be deleted:")
+            self.stdout.write(msg)
+            self.stdout.write("\n".join([f"- {project.name}" for project in projects]))
+            return
+            # sys.exit(0)
+
         if options["interactive"]:
             confirm = input(
                 f"You have requested the deletion of {projects_count} "

scanpipe/tests/__init__.py

@@ -49,8 +49,23 @@
 
 
 def make_project(name=None, **data):
+    """
+    Create and return a Project instance.
+    Labels can be provided using the labels=["labels1", "labels2"] argument.
+    """
     name = name or str(uuid.uuid4())[:8]
-    return Project.objects.create(name=name, **data)
+    pipelines = data.pop("pipelines", [])
+    labels = data.pop("labels", [])
+
+    project = Project.objects.create(name=name, **data)
+
+    for pipeline in pipelines:
+        project.add_pipeline(pipeline)
+
+    if labels:
+        project.labels.add(*labels)
+
+    return project
 
 
 def make_resource_file(project, path, **data):

scanpipe/tests/test_commands.py

@@ -319,7 +319,7 @@ def test_scanpipe_management_command_batch_create_global_webhook(
     def test_scanpipe_management_command_add_input_file(self):
         out = StringIO()
 
-        project = Project.objects.create(name="my_project")
+        project = make_project(name="my_project")
         parent_path = Path(__file__).parent
         options = [
             "--input-file",
@@ -346,7 +346,7 @@ def test_scanpipe_management_command_add_input_file(self):
             call_command("add-input", *options, stdout=out)
 
     def test_scanpipe_management_command_add_input_url(self):
-        project = Project.objects.create(name="my_project")
+        project = make_project(name="my_project")
         options = [
             "--input-url",
             "https://example.com/archive.zip",
@@ -368,7 +368,7 @@ def test_scanpipe_management_command_add_input_url(self):
     def test_scanpipe_management_command_add_input_copy_codebase(self):
         out = StringIO()
 
-        project = Project.objects.create(name="my_project")
+        project = make_project(name="my_project")
 
         options = ["--copy-codebase", "non-existing", "--project", project.name]
         expected = "non-existing not found"
@@ -394,7 +394,7 @@ def test_scanpipe_management_command_add_input_copy_codebase(self):
     def test_scanpipe_management_command_add_pipeline(self):
         out = StringIO()
 
-        project = Project.objects.create(name="my_project")
+        project = make_project(name="my_project")
 
         pipelines = [
             self.pipeline_name,
@@ -431,7 +431,7 @@ def test_scanpipe_management_command_add_pipeline(self):
     def test_scanpipe_management_command_add_webhook(self):
         out = StringIO()
 
-        project = Project.objects.create(name="my_project")
+        project = make_project(name="my_project")
 
         options = ["https://example.com/webhook"]
         expected = "the following arguments are required: --project"
@@ -486,7 +486,7 @@ def test_scanpipe_management_command_show_pipeline(self):
             "analyze_root_filesystem_or_vm_image",
         ]
 
-        project = Project.objects.create(name="my_project")
+        project = make_project(name="my_project")
         for pipeline_name in pipeline_names:
             project.add_pipeline(pipeline_name)
 
@@ -513,7 +513,7 @@ def test_scanpipe_management_command_show_pipeline(self):
         self.assertEqual(expected, out.getvalue())
 
     def test_scanpipe_management_command_execute(self):
-        project = Project.objects.create(name="my_project")
+        project = make_project(name="my_project")
         options = ["--project", project.name]
 
         out = StringIO()
@@ -557,7 +557,7 @@ def test_scanpipe_management_command_execute(self):
         self.assertEqual("", run3.task_output)
 
     def test_scanpipe_management_command_execute_project_function(self):
-        project = Project.objects.create(name="my_project")
+        project = make_project(name="my_project")
 
         expected = "No pipelines to run on project my_project"
         with self.assertRaisesMessage(CommandError, expected):
@@ -584,7 +584,7 @@ def test_scanpipe_management_command_execute_project_function(self):
         self.assertIsNone(returned_value)
 
     def test_scanpipe_management_command_status(self):
-        project = Project.objects.create(name="my_project")
+        project = make_project(name="my_project")
         run = project.add_pipeline(self.pipeline_name)
 
         options = ["--project", project.name, "--no-color"]
@@ -657,9 +657,9 @@ def test_scanpipe_management_command_list_pipelines(self):
         self.assertIn("(addon)", output)
 
     def test_scanpipe_management_command_list_project(self):
-        project1 = Project.objects.create(name="project1")
-        project2 = Project.objects.create(name="project2")
-        project3 = Project.objects.create(name="archived", is_archived=True)
+        project1 = make_project(name="project1")
+        project2 = make_project(name="project2")
+        project3 = make_project(name="archived", is_archived=True)
 
         options = []
         out = StringIO()
@@ -686,7 +686,7 @@ def test_scanpipe_management_command_list_project(self):
         self.assertIn(project3.name, output)
 
     def test_scanpipe_management_command_output(self):
-        project = Project.objects.create(name="my_project")
+        project = make_project(name="my_project")
         make_package(project, package_url="pkg:generic/name@1.0")
 
         out = StringIO()
@@ -754,7 +754,7 @@ def test_scanpipe_management_command_output(self):
         self.assertIn('"specVersion": "1.5",', out_value)
 
     def test_scanpipe_management_command_delete_project(self):
-        project = Project.objects.create(name="my_project")
+        project = make_project(name="my_project")
         work_path = project.work_path
         self.assertTrue(work_path.exists())
 
@@ -770,7 +770,7 @@ def test_scanpipe_management_command_delete_project(self):
         self.assertFalse(work_path.exists())
 
     def test_scanpipe_management_command_archive_project(self):
-        project = Project.objects.create(name="my_project")
+        project = make_project(name="my_project")
         (project.input_path / "input_file").touch()
         (project.codebase_path / "codebase_file").touch()
         self.assertEqual(1, len(Project.get_root_content(project.input_path)))
@@ -797,7 +797,7 @@ def test_scanpipe_management_command_archive_project(self):
         self.assertEqual(0, len(Project.get_root_content(project.codebase_path)))
 
     def test_scanpipe_management_command_reset_project(self):
-        project = Project.objects.create(name="my_project")
+        project = make_project(name="my_project")
         project.add_pipeline("analyze_docker_image")
         CodebaseResource.objects.create(project=project, path="filename.ext")
         DiscoveredPackage.objects.create(project=project)
@@ -833,8 +833,8 @@ def test_scanpipe_management_command_reset_project(self):
         self.assertEqual(0, len(Project.get_root_content(project.codebase_path)))
 
     def test_scanpipe_management_command_flush_projects(self):
-        project1 = Project.objects.create(name="project1")
-        project2 = Project.objects.create(name="project2")
+        project1 = make_project("project1")
+        project2 = make_project("project2")
         ten_days_ago = timezone.now() - datetime.timedelta(days=10)
         project2.update(created_date=ten_days_ago)
 
@@ -846,14 +846,58 @@ def test_scanpipe_management_command_flush_projects(self):
         self.assertEqual(expected, out_value)
         self.assertEqual(project1, Project.objects.get())
 
-        Project.objects.create(name="project2")
+        make_project("project2")
+        out = StringIO()
+        options = ["--no-color", "--no-input", "--dry-run"]
+        call_command("flush-projects", *options, stdout=out)
+        out_value = out.getvalue().strip()
+        expected = "2 projects would be deleted:\n- project2\n- project1"
+        self.assertEqual(expected, out_value)
+
         out = StringIO()
         options = ["--no-color", "--no-input"]
         call_command("flush-projects", *options, stdout=out)
         out_value = out.getvalue().strip()
         expected = "2 projects and their related data have been removed."
         self.assertEqual(expected, out_value)
 
+    def test_scanpipe_management_command_flush_projects_filters(self):
+        label1 = "label1"
+        label2 = "label2"
+        make_project("project1", labels=[label1])
+        make_project("project2", labels=[label1, label2])
+        make_project("project3", pipelines=["scan_single_package"])
+
+        base_options = ["--no-color", "--no-input", "--dry-run"]
+
+        out = StringIO()
+        options = base_options + ["--label", label1]
+        call_command("flush-projects", *options, stdout=out)
+        out_value = out.getvalue().strip()
+        expected = "2 projects would be deleted:\n- project2\n- project1"
+        self.assertEqual(expected, out_value)
+
+        out = StringIO()
+        options = base_options + ["--label", label2]
+        call_command("flush-projects", *options, stdout=out)
+        out_value = out.getvalue().strip()
+        expected = "1 projects would be deleted:\n- project2"
+        self.assertEqual(expected, out_value)
+
+        out = StringIO()
+        options = base_options + ["--label", label1, "--label", label2]
+        call_command("flush-projects", *options, stdout=out)
+        out_value = out.getvalue().strip()
+        expected = "2 projects would be deleted:\n- project2\n- project1"
+        self.assertEqual(expected, out_value)
+
+        out = StringIO()
+        options = base_options + ["--pipeline", "scan_single_package"]
+        call_command("flush-projects", *options, stdout=out)
+        out_value = out.getvalue().strip()
+        expected = "1 projects would be deleted:\n- project3"
+        self.assertEqual(expected, out_value)
+
     def test_scanpipe_management_command_create_user(self):
         out = StringIO()
 
@@ -1123,7 +1167,7 @@ def test_scanpipe_management_command_purldb_scan_queue_worker_continue_after_fai
         )
 
     def test_scanpipe_management_command_check_compliance(self):
-        project = Project.objects.create(name="my_project")
+        project = make_project(name="my_project")
 
         out = StringIO()
         options = ["--project", project.name]
@@ -1169,9 +1213,8 @@ def test_scanpipe_management_command_check_compliance(self):
         self.assertEqual(expected, out_value)
 
     def test_scanpipe_management_command_report(self):
-        project1 = make_project("project1")
         label1 = "label1"
-        project1.labels.add(label1)
+        project1 = make_project("project1", labels=[label1])
         make_resource_file(project1, path="file.ext", status=flag.REQUIRES_REVIEW)
         make_project("project2")