Add a report action on project list to export XLSX of packages #1437 (#1517)

Signed-off-by: tdruez <tdruez@nexb.com>

Author: tdruez

CHANGELOG.rst

@@ -20,6 +20,13 @@ v34.9.4 (unreleased)
   and let the data loading process handle the data issues.
   https://github.com/aboutcode-org/scancode.io/issues/1515
 
+- Add a report action on project list to export XLSX containing packages from selected
+  projects.
+  https://github.com/aboutcode-org/scancode.io/issues/1437
+
+- Add a download action on project list to enable bulk download of Project output files.
+  https://github.com/aboutcode-org/scancode.io/issues/1518
+
 v34.9.3 (2024-12-31)
 --------------------
 

scanpipe/forms.py

@@ -256,6 +256,22 @@ class ArchiveProjectForm(forms.Form):
     )
 
 
+class ProjectOutputDownloadForm(forms.Form):
+    output_format = forms.ChoiceField(
+        label="Choose the output format to include in the ZIP file",
+        choices=[
+            ("json", "JSON"),
+            ("xlsx", "XLSX"),
+            ("spdx", "SPDX"),
+            ("cyclonedx", "CycloneDX"),
+            ("attribution", "Attribution"),
+        ],
+        required=True,
+        initial="json",
+        widget=forms.RadioSelect,
+    )
+
+
 class ListTextarea(forms.CharField):
     """
     A Django form field that displays as a textarea and converts each line of input

scanpipe/pipes/output.py

@@ -294,7 +294,9 @@ def to_json(project):
 }
 
 
-def queryset_to_xlsx_worksheet(queryset, workbook, exclude_fields=()):
+def queryset_to_xlsx_worksheet(
+    queryset, workbook, exclude_fields=None, extra_fields=None
+):
     """
     Add a new worksheet to the ``workbook`` ``xlsxwriter.Workbook`` using the
     ``queryset``. The ``queryset`` "model_name" is used as a name for the
@@ -312,7 +314,10 @@ def queryset_to_xlsx_worksheet(queryset, workbook, exclude_fields=()):
 
     fields = get_serializer_fields(model_class)
     exclude_fields = exclude_fields or []
+    extra_fields = extra_fields or []
     fields = [field for field in fields if field not in exclude_fields]
+    if extra_fields:
+        fields.extend(extra_fields)
 
     return _add_xlsx_worksheet(
         workbook=workbook,
@@ -893,3 +898,12 @@ def to_attribution(project):
 
     output_file.write_text(rendered_template)
     return output_file
+
+
+FORMAT_TO_FUNCTION_MAPPING = {
+    "json": to_json,
+    "xlsx": to_xlsx,
+    "spdx": to_spdx,
+    "cyclonedx": to_cyclonedx,
+    "attribution": to_attribution,
+}

scanpipe/templates/scanpipe/dropdowns/project_actions_dropdown.html

@@ -13,6 +13,19 @@
   <div class="dropdown-menu" role="menu">
     <div class="dropdown-content">
       <ul>
+        <li>
+          <a href="#" class="dropdown-item modal-button" data-target="modal-projects-download" aria-haspopup="true">
+            <i class="fa-solid fa-download mr-2"></i>Download
+          </a>
+        </li>
+        <li>
+          <a href="#" class="dropdown-item modal-button" data-target="modal-projects-report" aria-haspopup="true">
+            <i class="fa-solid fa-table-cells mr-2"></i>Report
+          </a>
+        </li>
+        <li>
+          <hr class="dropdown-divider">
+        </li>
         <li>
           <a href="#" class="dropdown-item modal-button" data-target="modal-projects-archive" aria-haspopup="true">
             <i class="fa-solid fa-dice-d6 mr-2"></i>Archive

scanpipe/templates/scanpipe/modals/projects_download_modal.html

@@ -0,0 +1,23 @@
+<div class="modal" id="modal-projects-download">
+  <div class="modal-background"></div>
+  <div class="modal-card">
+    <header class="modal-card-head">
+      <p class="modal-card-title">Download outputs for selected projects as ZIP file</p>
+      <button class="delete" aria-label="close"></button>
+    </header>
+    <form action="{% url 'project_action' %}" method="post" id="download-projects-form">{% csrf_token %}
+      <section class="modal-card-body">
+        <ul class="mb-3">
+          {{ outputs_download_form.as_ul }}
+        </ul>
+      </section>
+      <input type="hidden" name="action" value="download">
+      <footer class="modal-card-foot is-flex is-justify-content-space-between">
+        <button class="button has-text-weight-semibold" type="reset">Cancel</button>
+        <button class="button is-success" type="button" data-action-trigger="download-projects">
+          Download<span class="icon ml-1"><i class="fa-solid fa-download"></i></span>
+        </button>
+      </footer>
+    </form>
+  </div>
+</div>

scanpipe/templates/scanpipe/modals/projects_report_modal.html

@@ -0,0 +1,23 @@
+<div class="modal" id="modal-projects-report">
+  <div class="modal-background"></div>
+  <div class="modal-card">
+    <header class="modal-card-head">
+      <p class="modal-card-title">Report of selected projects</p>
+      <button class="delete" aria-label="close"></button>
+    </header>
+    <form action="{% url 'project_action' %}" method="post" id="report-projects-form">{% csrf_token %}
+      <section class="modal-card-body">
+        <div class="notification is-info has-text-weight-semibold">
+          All the packages for the selected projects will be included in an XLSX report.
+        </div>
+      </section>
+      <input type="hidden" name="action" value="report">
+      <footer class="modal-card-foot is-flex is-justify-content-space-between">
+        <button class="button has-text-weight-semibold" type="reset">Cancel</button>
+        <button class="button is-success" type="button" data-action-trigger="report-projects">
+          Download XLSX<span class="icon ml-1"><i class="fa-solid fa-download"></i></span>
+        </button>
+      </footer>
+    </form>
+  </div>
+</div>

scanpipe/templates/scanpipe/project_list.html

@@ -70,6 +70,8 @@
   </div>
 
   {% include 'scanpipe/modals/run_modal.html' %}
+  {% include "scanpipe/modals/projects_download_modal.html" %}
+  {% include "scanpipe/modals/projects_report_modal.html" %}
   {% include "scanpipe/modals/projects_archive_modal.html" %}
   {% include "scanpipe/modals/projects_delete_modal.html" %}
   {% include "scanpipe/modals/projects_reset_modal.html" %}

scanpipe/views.py

@@ -24,6 +24,7 @@
 import io
 import json
 import operator
+import zipfile
 from collections import Counter
 from contextlib import suppress
 from pathlib import Path
@@ -79,6 +80,7 @@
 from scanpipe.forms import PipelineRunStepSelectionForm
 from scanpipe.forms import ProjectCloneForm
 from scanpipe.forms import ProjectForm
+from scanpipe.forms import ProjectOutputDownloadForm
 from scanpipe.forms import ProjectSettingsForm
 from scanpipe.models import CodebaseRelation
 from scanpipe.models import CodebaseResource
@@ -442,15 +444,30 @@ class ExportXLSXMixin:
 
     export_xlsx_query_param = "export_xlsx"
 
+    def get_export_xlsx_queryset(self):
+        return self.filterset.qs
+
+    def get_export_xlsx_filename(self):
+        return f"{self.project.name}_{self.model._meta.model_name}.xlsx"
+
+    def get_export_xlsx_extra_fields(self):
+        return []
+
     def export_xlsx_file_response(self):
-        filtered_qs = self.filterset.qs
         output_file = io.BytesIO()
+        queryset = self.get_export_xlsx_queryset()
+        extra_fields = self.get_export_xlsx_extra_fields()
         with xlsxwriter.Workbook(output_file) as workbook:
-            output.queryset_to_xlsx_worksheet(filtered_qs, workbook)
+            output.queryset_to_xlsx_worksheet(
+                queryset, workbook, extra_fields=extra_fields
+            )
 
-        filename = f"{self.project.name}_{self.model._meta.model_name}.xlsx"
         output_file.seek(0)
-        return FileResponse(output_file, as_attachment=True, filename=filename)
+        return FileResponse(
+            output_file,
+            as_attachment=True,
+            filename=self.get_export_xlsx_filename(),
+        )
 
     def get_context_data(self, **kwargs):
         context = super().get_context_data(**kwargs)
@@ -574,6 +591,7 @@ class ProjectListView(
     def get_context_data(self, **kwargs):
         context = super().get_context_data(**kwargs)
         context["archive_form"] = ArchiveProjectForm()
+        context["outputs_download_form"] = ProjectOutputDownloadForm()
         return context
 
     def get_queryset(self):
@@ -1137,29 +1155,35 @@ def form_valid(self, form):
 
 
 @method_decorator(require_POST, name="dispatch")
-class ProjectActionView(ConditionalLoginRequired, generic.ListView):
+class ProjectActionView(ConditionalLoginRequired, ExportXLSXMixin, generic.ListView):
     """Call a method for each instance of the selection."""
 
     model = Project
-    allowed_actions = ["archive", "delete", "reset"]
+    allowed_actions = ["archive", "delete", "reset", "report", "download"]
     success_url = reverse_lazy("project_list")
 
     def post(self, request, *args, **kwargs):
         action = request.POST.get("action")
         if action not in self.allowed_actions:
             raise Http404
 
-        selected_ids = request.POST.get("selected_ids", "").split(",")
+        self.selected_project_ids = request.POST.get("selected_ids", "").split(",")
         count = 0
 
         action_kwargs = {}
+        if action == "report":
+            return self.export_xlsx_file_response()
+
+        if action == "download":
+            return self.download_outputs_zip_response()
+
         if action == "archive":
             archive_form = ArchiveProjectForm(request.POST)
             if not archive_form.is_valid():
                 raise Http404
             action_kwargs = archive_form.cleaned_data
 
-        for project_uuid in selected_ids:
+        for project_uuid in self.selected_project_ids:
             if self.perform_action(action, project_uuid, action_kwargs):
                 count += 1
 
@@ -1186,6 +1210,46 @@ def perform_action(self, action, project_uuid, action_kwargs=None):
     def get_success_message(self, action, count):
         return f"{count} projects have been {action}."
 
+    def get_projects_queryset(self):
+        return Project.objects.filter(pk__in=self.selected_project_ids)
+
+    def get_export_xlsx_queryset(self):
+        projects = self.get_projects_queryset()
+        packages = DiscoveredPackage.objects.filter(project__in=projects)
+        packages = packages.select_related("project")
+        return packages
+
+    def get_export_xlsx_extra_fields(self):
+        return ["project"]
+
+    def get_export_xlsx_filename(self):
+        return "report.xlsx"
+
+    def download_outputs_zip_response(self):
+        outputs_download_form = ProjectOutputDownloadForm(self.request.POST)
+        if not outputs_download_form.is_valid():
+            return HttpResponseRedirect(self.success_url)
+
+        output_format = outputs_download_form.cleaned_data["output_format"]
+        output_function = output.FORMAT_TO_FUNCTION_MAPPING.get(output_format)
+        projects = self.get_projects_queryset()
+
+        # In-memory file storage for the zip archive
+        zip_buffer = io.BytesIO()
+        with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zip_file:
+            for project in projects:
+                output_file = output_function(project)
+                filename = output.safe_filename(f"{project.name}_{output_file.name}")
+                with open(output_file, "rb") as f:
+                    zip_file.writestr(filename, f.read())
+
+        zip_buffer.seek(0)
+        return FileResponse(
+            zip_buffer,
+            as_attachment=True,
+            filename="scancodeio_output_files.zip",
+        )
+
 
 class ProjectResetView(ConditionalLoginRequired, generic.DeleteView):
     model = Project