add API end-points for indexing

scanpipe/api/serializers.py

@@ -32,6 +32,7 @@
 from scanpipe.models import CodebaseResource
 from scanpipe.models import DiscoveredDependency
 from scanpipe.models import DiscoveredPackage
+from scanpipe.models import DownloadedPackage
 from scanpipe.models import InputSource
 from scanpipe.models import Project
 from scanpipe.models import ProjectMessage
@@ -520,6 +521,37 @@ class Meta:
         ]
 
 
+class DownloadedPackageSerializer(serializers.ModelSerializer):
+    download_url = serializers.SerializerMethodField()
+    archive_checksum = serializers.CharField(
+        source="package_archive.checksum_sha256", read_only=True
+    )
+
+    class Meta:
+        model = DownloadedPackage
+        fields = [
+            "id",
+            "url",
+            "filename",
+            "download_date",
+            "scan_log",
+            "scan_date",
+            "scancode_version",
+            "pipeline_name",
+            "archive_checksum",
+            "download_url",
+        ]
+        read_only_fields = fields
+
+    def get_download_url(self, obj):
+        request = self.context.get("request")
+        if obj.package_archive.package_file and request:
+            return request.build_absolute_uri(
+                f"/api/projects/{obj.project.uuid}/packages/{obj.id}/download/"
+            )
+        return None
+
+
 def get_model_serializer(model_class):
     """Return a Serializer class that ia related to a given `model_class`."""
     serializer = {
@@ -528,6 +560,7 @@ def get_model_serializer(model_class):
         DiscoveredDependency: DiscoveredDependencySerializer,
         CodebaseRelation: CodebaseRelationSerializer,
         ProjectMessage: ProjectMessageSerializer,
+        DownloadedPackage: DownloadedPackageSerializer,
     }.get(model_class, None)
 
     if not serializer:

scanpipe/api/urls.py

@@ -0,0 +1,35 @@
+# scanpipe/api/urls.py
+# SPDX-License-Identifier: Apache-2.0
+#
+# http://nexb.com and https://github.com/aboutcode-org/scancode.io
+# The ScanCode.io software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode.io is provided as-is without warranties.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+# ScanCode.io should be considered or used as legal advice. Consult an Attorney
+# for any legal advice.
+#
+# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
+# Visit https://github.com/aboutcode-org/scancode.io for support and download.
+
+from rest_framework.routers import DefaultRouter
+
+from scanpipe.api.views import DownloadedPackageViewSet
+
+router = DefaultRouter()
+router.register(
+    r"projects/(?P<project_uuid>[^/.]+)/packages",
+    DownloadedPackageViewSet,
+    basename="downloaded-package",
+)
+
+urlpatterns = router.urls

scanpipe/api/views.py

@@ -21,6 +21,7 @@
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
 import json
+import os
 
 from django.apps import apps
 from django.core.exceptions import ObjectDoesNotExist
@@ -34,12 +35,14 @@
 from rest_framework import status
 from rest_framework import viewsets
 from rest_framework.decorators import action
+from rest_framework.permissions import IsAuthenticatedOrReadOnly
 from rest_framework.response import Response
 
 from scanpipe.api.serializers import CodebaseRelationSerializer
 from scanpipe.api.serializers import CodebaseResourceSerializer
 from scanpipe.api.serializers import DiscoveredDependencySerializer
 from scanpipe.api.serializers import DiscoveredPackageSerializer
+from scanpipe.api.serializers import DownloadedPackageSerializer
 from scanpipe.api.serializers import PipelineSerializer
 from scanpipe.api.serializers import ProjectMessageSerializer
 from scanpipe.api.serializers import ProjectSerializer
@@ -50,6 +53,7 @@
 from scanpipe.filters import ProjectMessageFilterSet
 from scanpipe.filters import RelationFilterSet
 from scanpipe.filters import ResourceFilterSet
+from scanpipe.models import DownloadedPackage
 from scanpipe.models import Project
 from scanpipe.models import Run
 from scanpipe.models import RunInProgressError
@@ -526,3 +530,123 @@ def delete_pipeline(self, request, *args, **kwargs):
 
         run.delete_task()
         return Response({"status": f"Pipeline {run.pipeline_name} deleted."})
+
+
+class DownloadedPackageFilter(django_filters.FilterSet):
+    url = django_filters.CharFilter(lookup_expr="exact")
+    checksum = django_filters.CharFilter(
+        field_name="package_archive__checksum_sha256", lookup_expr="exact"
+    )
+
+    class Meta:
+        model = DownloadedPackage
+        fields = ["url", "checksum"]
+
+
+class DownloadedPackageViewSet(
+    mixins.ListModelMixin,
+    mixins.RetrieveModelMixin,
+    viewsets.GenericViewSet,
+):
+    """
+    A viewset for managing DownloadedPackage instances, providing endpoints to list,
+    retrieve, and download packages by ID, URL, or checksum.
+    """
+
+    queryset = DownloadedPackage.objects.select_related("package_archive")
+    serializer_class = DownloadedPackageSerializer
+    permission_classes = [IsAuthenticatedOrReadOnly]
+    lookup_field = "id"
+    filterset_class = DownloadedPackageFilter
+    filter_backends = [django_filters.rest_framework.DjangoFilterBackend]
+
+    def get_queryset(self):
+        project_uuid = self.kwargs["project_uuid"]
+        try:
+            project = Project.objects.get(uuid=project_uuid)
+            return self.queryset.filter(project=project)
+        except Project.DoesNotExist:
+            return self.queryset.none()
+
+    @action(detail=True, methods=["get"])
+    def download(self, request, project_uuid=None, id=None):
+        """Download a package file by ID."""
+        try:
+            package = self.get_queryset().get(id=id)
+            file_path = package.package_archive.package_file.path
+            if not file_path or not os.path.exists(file_path):
+                return Response({"error": "Package file not found"}, status=404)
+            file_name = os.path.basename(file_path)
+            return FileResponse(
+                open(file_path, "rb"),
+                as_attachment=True,
+                filename=file_name,
+            )
+        except DownloadedPackage.DoesNotExist:
+            return Response({"error": "Package not found"}, status=404)
+
+    @action(detail=False, methods=["get"])
+    def by_url(self, request, project_uuid=None):
+        """Query package details by URL."""
+        url = request.query_params.get("url")
+        if not url:
+            return Response({"error": "URL parameter is required"}, status=400)
+        try:
+            package = self.get_queryset().get(url=url)
+            serializer = self.get_serializer(package)
+            return Response(serializer.data)
+        except DownloadedPackage.DoesNotExist:
+            return Response({"error": "Package not found"}, status=404)
+
+    @action(detail=False, methods=["get"])
+    def download_by_url(self, request, project_uuid=None):
+        """Download a package file by URL."""
+        url = request.query_params.get("url")
+        if not url:
+            return Response({"error": "URL parameter is required"}, status=400)
+        try:
+            package = self.get_queryset().get(url=url)
+            file_path = package.package_archive.package_file.path
+            if not file_path or not os.path.exists(file_path):
+                return Response({"error": "Package file not found"}, status=404)
+            file_name = os.path.basename(file_path)
+            return FileResponse(
+                open(file_path, "rb"),
+                as_attachment=True,
+                filename=file_name,
+            )
+        except DownloadedPackage.DoesNotExist:
+            return Response({"error": "Package not found"}, status=404)
+
+    @action(detail=False, methods=["get"])
+    def by_checksum(self, request, project_uuid=None):
+        """Query package details by SHA256 checksum."""
+        checksum = request.query_params.get("checksum")
+        if not checksum:
+            return Response({"error": "Checksum parameter is required"}, status=400)
+        try:
+            package = self.get_queryset().get(package_archive__checksum_sha256=checksum)
+            serializer = self.get_serializer(package)
+            return Response(serializer.data)
+        except DownloadedPackage.DoesNotExist:
+            return Response({"error": "Package not found"}, status=404)
+
+    @action(detail=False, methods=["get"])
+    def download_by_checksum(self, request, project_uuid=None):
+        """Download a package file by SHA256 checksum."""
+        checksum = request.query_params.get("checksum")
+        if not checksum:
+            return Response({"error": "Checksum parameter is required"}, status=400)
+        try:
+            package = self.get_queryset().get(package_archive__checksum_sha256=checksum)
+            file_path = package.package_archive.package_file.path
+            if not file_path or not os.path.exists(file_path):
+                return Response({"error": "Package file not found"}, status=404)
+            file_name = os.path.basename(file_path)
+            return FileResponse(
+                open(file_path, "rb"),
+                as_attachment=True,
+                filename=file_name,
+            )
+        except DownloadedPackage.DoesNotExist:
+            return Response({"error": "Package not found"}, status=404)

scanpipe/urls.py

@@ -242,4 +242,10 @@
         name="license_list",
     ),
     path("monitor/", include("django_rq.urls")),
+    path(
+        "project/<slug:slug>/", views.ProjectDetailView.as_view(), name="project_detail"
+    ),
+    path("license/", views.LicenseListView.as_view(), name="license_list"),
+    path("monitor/", include("django_rq.urls")),
+    path("api/", include("scanpipe.api.urls")),
 ]