diff --git a/scanpipe/migrations/0074_codebaseresource_parent_path_and_more.py b/scanpipe/migrations/0074_codebaseresource_parent_path_and_more.py new file mode 100644 index 000000000..0f424f678 --- /dev/null +++ b/scanpipe/migrations/0074_codebaseresource_parent_path_and_more.py @@ -0,0 +1,24 @@ +# Generated by Django 5.1.9 on 2025-06-19 21:19 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('scanpipe', '0073_add_sha1_git_checksum'), + ] + + operations = [ + migrations.AddField( + model_name='codebaseresource', + name='parent_path', + field=models.CharField(blank=True, help_text="The path of the resource's parent directory. Set to None for top-level (root) resources. Used to efficiently retrieve a directory's contents.", max_length=2000, null=True), + ), + migrations.AddIndex( + model_name='codebaseresource', + index=models.Index(fields=['project', 'parent_path'], name='scanpipe_co_project_008448_idx'), + ), + ] + + diff --git a/scanpipe/models.py b/scanpipe/models.py index fd72fe711..969166714 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -46,6 +46,7 @@ from django.db import transaction from django.db.models import Case from django.db.models import Count +from django.db.models import Exists from django.db.models import IntegerField from django.db.models import OuterRef from django.db.models import Prefetch @@ -229,7 +230,7 @@ def delete(self, *args, **kwargs): Note that projects with queued or running pipeline runs cannot be deleted. See the `_raise_if_run_in_progress` method. The following if statements should not be triggered unless the `.delete()` - method is directly call from an instance of this class. + method is directly call from a instance of this class. """ with suppress(redis.exceptions.ConnectionError, AttributeError): if self.status == self.Status.RUNNING: @@ -2383,6 +2384,18 @@ def macho_binaries(self): def executable_binaries(self): return self.union(self.win_exes(), self.macho_binaries(), self.elfs()) + def with_children(self, project): + """ + Annotate the QuerySet with has_children field based on whether + each resource has any children (subdirectories/files). + """ + subdirs = CodebaseResource.objects.filter( + project=project, + parent_path=OuterRef("path"), + ) + + return self.annotate(has_children=Exists(subdirs)) + class ScanFieldsModelMixin(models.Model): """Fields returned by the ScanCode-toolkit scans.""" @@ -2688,6 +2701,17 @@ class CodebaseResource( 'Eg.: "/usr/bin/bash" for a path of "tarball-extract/rootfs/usr/bin/bash"' ), ) + + parent_path = models.CharField( + max_length=2000, + blank=True, + help_text=_( + "The path of the resource's parent directory. " + "Set to None for top-level (root) resources. " + "Used to efficiently retrieve a directory's contents." + ), + ) + status = models.CharField( blank=True, max_length=50, @@ -2781,6 +2805,7 @@ class Meta: models.Index(fields=["compliance_alert"]), models.Index(fields=["is_binary"]), models.Index(fields=["is_text"]), + models.Index(fields=["project", "parent_path"]), ] constraints = [ models.UniqueConstraint( @@ -2793,6 +2818,11 @@ class Meta: def __str__(self): return self.path + def save(self, *args, **kwargs): + if self.path and not self.parent_path: + self.parent_path = self.parent_directory() or "" + super().save(*args, **kwargs) + def get_absolute_url(self): return reverse("resource_detail", args=[self.project.slug, self.path]) @@ -2863,7 +2893,8 @@ def get_path_segments_with_subpath(self): def parent_directory(self): """Return the parent path for this CodebaseResource or None.""" - return parent_directory(self.path, with_trail=False) + parent_path = parent_directory(str(self.path), with_trail=False) + return parent_path or None def has_parent(self): """ diff --git a/scanpipe/templates/scanpipe/panels/codebase_tree_panel.html b/scanpipe/templates/scanpipe/panels/codebase_tree_panel.html new file mode 100644 index 000000000..0ccc12b53 --- /dev/null +++ b/scanpipe/templates/scanpipe/panels/codebase_tree_panel.html @@ -0,0 +1,29 @@ + diff --git a/scanpipe/templates/scanpipe/resource_tree.html b/scanpipe/templates/scanpipe/resource_tree.html new file mode 100644 index 000000000..e22de18a5 --- /dev/null +++ b/scanpipe/templates/scanpipe/resource_tree.html @@ -0,0 +1,67 @@ +{% extends "scanpipe/base.html" %} +{% load static humanize %} +{% block title %}ScanCode.io: {{ project.name }} - Resource Tree{% endblock %} + +{% block extrahead %} + +{% endblock %} + +{% block content %} +
+ {% include 'scanpipe/includes/navbar_header.html' %} +
+
+ {% include 'scanpipe/includes/breadcrumb.html' with linked_project=True current="Resource Tree" %} +
+
+
+ +
+
+
+ {% include "scanpipe/panels/codebase_tree_panel.html" with children=children path=path %} +
+
+
+
+
+
+
+{% endblock %} + +{% block scripts %} + +{% endblock %} \ No newline at end of file diff --git a/scanpipe/tests/test_views.py b/scanpipe/tests/test_views.py index 4c06995b1..6311af8ff 100644 --- a/scanpipe/tests/test_views.py +++ b/scanpipe/tests/test_views.py @@ -1576,3 +1576,35 @@ def test_project_codebase_resources_export_json(self): for field in expected_fields: self.assertIn(field, json_data[0]) + + def test_file_tree_base_url_lists_top_level_nodes(self): + make_resource_file(self.project1, path="child1.txt") + make_resource_file(self.project1, path="dir1") + + url = reverse("codebase_resource_tree", kwargs={"slug": self.project1.slug}) + response = self.client.get(url) + children = response.context["children"] + print(response.context) + child1 = children[0] + dir1 = children[1] + + self.assertEqual(child1.path, "child1.txt") + self.assertEqual(dir1.path, "dir1") + + def test_file_tree_nested_url_lists_only_children_of_given_path(self): + make_resource_file(self.project1, path="parent/child1.txt") + make_resource_file(self.project1, path="parent/dir1") + make_resource_file(self.project1, path="parent/dir1/child2.txt") + + url = reverse("codebase_resource_tree", kwargs={"slug": self.project1.slug}) + response = self.client.get(url + "?path=parent&tree=true") + children = response.context["children"] + + child1 = children[0] + dir1 = children[1] + + self.assertEqual(child1.path, "parent/child1.txt") + self.assertEqual(dir1.path, "parent/dir1") + + self.assertFalse(child1.has_children) + self.assertTrue(dir1.has_children) diff --git a/scanpipe/urls.py b/scanpipe/urls.py index c760becbf..a3b09e4dd 100644 --- a/scanpipe/urls.py +++ b/scanpipe/urls.py @@ -121,6 +121,11 @@ views.ProjectCodebaseView.as_view(), name="project_codebase", ), + path( + "project//codebase_tree/", + views.CodebaseResourceTreeView.as_view(), + name="codebase_resource_tree", + ), path( "run//", views.run_detail_view, diff --git a/scanpipe/views.py b/scanpipe/views.py index d630628fb..d93d946b9 100644 --- a/scanpipe/views.py +++ b/scanpipe/views.py @@ -2567,3 +2567,30 @@ def get_node(self, package): if children: node["children"] = children return node + + +class CodebaseResourceTreeView(ConditionalLoginRequired, generic.DetailView): + template_name = "scanpipe/resource_tree.html" + + def get(self, request, *args, **kwargs): + slug = self.kwargs.get("slug") + project = get_object_or_404(Project, slug=slug) + path = request.GET.get("path", "") + + base_qs = ( + CodebaseResource.objects.filter(project=project, parent_path=path) + .only("path", "name", "type") + .order_by("path") + ) + + children = base_qs.with_children(project) + + context = { + "project": project, + "path": path, + "children": children, + } + + if request.GET.get("tree") == "true": + return render(request, "scanpipe/panels/codebase_tree_panel.html", context) + return render(request, self.template_name, context)