From 17d0fc334634e4a58588b2fd4de28272849d4709 Mon Sep 17 00:00:00 2001 From: Aayush Kumar Date: Fri, 27 Jun 2025 18:23:59 +0530 Subject: [PATCH 1/9] add left-pane file tree view and related templates Signed-off-by: Aayush Kumar --- .../scanpipe/panels/file_tree_panel.html | 42 ++++++++++++ .../templates/scanpipe/resource_tree.html | 67 +++++++++++++++++++ scanpipe/tests/test_views.py | 32 +++++++++ scanpipe/urls.py | 5 ++ scanpipe/views.py | 34 ++++++++++ 5 files changed, 180 insertions(+) create mode 100644 scanpipe/templates/scanpipe/panels/file_tree_panel.html create mode 100644 scanpipe/templates/scanpipe/resource_tree.html diff --git a/scanpipe/templates/scanpipe/panels/file_tree_panel.html b/scanpipe/templates/scanpipe/panels/file_tree_panel.html new file mode 100644 index 000000000..00ebbe79a --- /dev/null +++ b/scanpipe/templates/scanpipe/panels/file_tree_panel.html @@ -0,0 +1,42 @@ + diff --git a/scanpipe/templates/scanpipe/resource_tree.html b/scanpipe/templates/scanpipe/resource_tree.html new file mode 100644 index 000000000..c43698404 --- /dev/null +++ b/scanpipe/templates/scanpipe/resource_tree.html @@ -0,0 +1,67 @@ +{% extends "scanpipe/base.html" %} +{% load static %} +{% block title %}ScanCode.io: {{ project.name }} - Resource{% endblock %} + +{% block extrahead %} + +{% endblock %} + +{% block content %} +
+
+
+ {% include "scanpipe/panels/file_tree_panel.html" with children=children path=path %} +
+
+ +
+
+
+
+
+{% endblock %} + +{% block scripts %} + +{% endblock %} \ No newline at end of file diff --git a/scanpipe/tests/test_views.py b/scanpipe/tests/test_views.py index 4c06995b1..ddce511a7 100644 --- a/scanpipe/tests/test_views.py +++ b/scanpipe/tests/test_views.py @@ -1576,3 +1576,35 @@ def test_project_codebase_resources_export_json(self): for field in expected_fields: self.assertIn(field, json_data[0]) + + def test_file_tree_base_url_lists_top_level_nodes(self): + make_resource_file(self.project1, path="child1.txt") + make_resource_file(self.project1, path="dir1") + + url = reverse("file_tree", kwargs={"slug": self.project1.slug}) + response = self.client.get(url) + children = response.context[-1]["children"] + + child1 = children[0] + dir1 = children[1] + + self.assertEqual(child1.path, "child1.txt") + self.assertEqual(dir1.path, "dir1") + + def test_file_tree_nested_url_lists_only_children_of_given_path(self): + make_resource_file(self.project1, path="parent/child1.txt") + make_resource_file(self.project1, path="parent/dir1") + make_resource_file(self.project1, path="parent/dir1/child2.txt") + + url = reverse("file_tree", kwargs={"slug": self.project1.slug}) + response = self.client.get(url + "?path=parent&tree=true") + children = response.context["children"] + + child1 = children[0] + dir1 = children[1] + + self.assertEqual(child1.path, "parent/child1.txt") + self.assertEqual(dir1.path, "parent/dir1") + + self.assertFalse(child1.has_children) + self.assertTrue(dir1.has_children) diff --git a/scanpipe/urls.py b/scanpipe/urls.py index c760becbf..73dccc74a 100644 --- a/scanpipe/urls.py +++ b/scanpipe/urls.py @@ -241,5 +241,10 @@ views.LicenseListView.as_view(), name="license_list", ), + path( + "project//codebase_tree/", + views.CodebaseResourceTreeView.as_view(), + name="file_tree", + ), path("monitor/", include("django_rq.urls")), ] diff --git a/scanpipe/views.py b/scanpipe/views.py index d630628fb..98e5d287b 100644 --- a/scanpipe/views.py +++ b/scanpipe/views.py @@ -37,6 +37,8 @@ from django.core.exceptions import ValidationError from django.core.files.storage.filesystem import FileSystemStorage from django.core.serializers.json import DjangoJSONEncoder +from django.db.models import Exists +from django.db.models import OuterRef from django.db.models import Prefetch from django.db.models.manager import Manager from django.http import FileResponse @@ -2567,3 +2569,35 @@ def get_node(self, package): if children: node["children"] = children return node + + +class CodebaseResourceTreeView(ConditionalLoginRequired, generic.DetailView): + template_name = "scanpipe/resource_tree.html" + + def get(self, request, *args, **kwargs): + slug = self.kwargs.get("slug") + project = get_object_or_404(Project, slug=slug) + path = request.GET.get("path", None) + + base_qs = ( + CodebaseResource.objects.filter(project=project, parent_path=path) + .only("path", "name", "type") + .order_by("path") + ) + + subdirs = CodebaseResource.objects.filter( + project=project, + parent_path=OuterRef("path"), + ) + + children = base_qs.annotate(has_children=Exists(subdirs)) + + context = { + "project": project, + "path": path, + "children": children, + } + + if request.GET.get("tree") == "true": + return render(request, "scanpipe/panels/file_tree_panel.html", context) + return render(request, self.template_name, context) From a9c511837c576808b25829add8b671bfd9f1fc37 Mon Sep 17 00:00:00 2001 From: Aayush Kumar Date: Fri, 27 Jun 2025 18:47:27 +0530 Subject: [PATCH 2/9] temporarily include parent_path field from previous pr for tests Signed-off-by: Aayush Kumar --- ...3_codebaseresource_parent_path_and_more.py | 24 +++++++++++++++++++ scanpipe/models.py | 23 ++++++++++++++++-- 2 files changed, 45 insertions(+), 2 deletions(-) create mode 100644 scanpipe/migrations/0073_codebaseresource_parent_path_and_more.py diff --git a/scanpipe/migrations/0073_codebaseresource_parent_path_and_more.py b/scanpipe/migrations/0073_codebaseresource_parent_path_and_more.py new file mode 100644 index 000000000..86848b879 --- /dev/null +++ b/scanpipe/migrations/0073_codebaseresource_parent_path_and_more.py @@ -0,0 +1,24 @@ +# Generated by Django 5.1.9 on 2025-06-19 21:19 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('scanpipe', '0072_discovereddependency_uuid_unique'), + ] + + operations = [ + migrations.AddField( + model_name='codebaseresource', + name='parent_path', + field=models.CharField(blank=True, help_text="The path of the resource's parent directory. Set to None for top-level (root) resources. Used to efficiently retrieve a directory's contents.", max_length=2000, null=True), + ), + migrations.AddIndex( + model_name='codebaseresource', + index=models.Index(fields=['project', 'parent_path'], name='scanpipe_co_project_008448_idx'), + ), + ] + + diff --git a/scanpipe/models.py b/scanpipe/models.py index fd72fe711..e3ca9aff8 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -229,7 +229,7 @@ def delete(self, *args, **kwargs): Note that projects with queued or running pipeline runs cannot be deleted. See the `_raise_if_run_in_progress` method. The following if statements should not be triggered unless the `.delete()` - method is directly call from an instance of this class. + method is directly call from a instance of this class. """ with suppress(redis.exceptions.ConnectionError, AttributeError): if self.status == self.Status.RUNNING: @@ -2688,6 +2688,18 @@ class CodebaseResource( 'Eg.: "/usr/bin/bash" for a path of "tarball-extract/rootfs/usr/bin/bash"' ), ) + + parent_path = models.CharField( + max_length=2000, + null=True, + blank=True, + help_text=_( + "The path of the resource's parent directory. " + "Set to None for top-level (root) resources. " + "Used to efficiently retrieve a directory's contents." + ), + ) + status = models.CharField( blank=True, max_length=50, @@ -2781,6 +2793,7 @@ class Meta: models.Index(fields=["compliance_alert"]), models.Index(fields=["is_binary"]), models.Index(fields=["is_text"]), + models.Index(fields=["project", "parent_path"]), ] constraints = [ models.UniqueConstraint( @@ -2793,6 +2806,11 @@ class Meta: def __str__(self): return self.path + def save(self, *args, **kwargs): + if self.path and not self.parent_path: + self.parent_path = self.parent_directory() + super().save(*args, **kwargs) + def get_absolute_url(self): return reverse("resource_detail", args=[self.project.slug, self.path]) @@ -2863,7 +2881,8 @@ def get_path_segments_with_subpath(self): def parent_directory(self): """Return the parent path for this CodebaseResource or None.""" - return parent_directory(self.path, with_trail=False) + parent_path = parent_directory(str(self.path), with_trail=False) + return parent_path or None def has_parent(self): """ From 3ebb81e13867de98984c7f4b4d7cbbc877fa0e5f Mon Sep 17 00:00:00 2001 From: Aayush Kumar Date: Thu, 3 Jul 2025 22:03:16 +0530 Subject: [PATCH 3/9] Some formatting changes Signed-off-by: Aayush Kumar --- .../scanpipe/panels/file_tree_panel.html | 37 +++---- .../templates/scanpipe/resource_tree.html | 97 ++++++++++--------- 2 files changed, 62 insertions(+), 72 deletions(-) diff --git a/scanpipe/templates/scanpipe/panels/file_tree_panel.html b/scanpipe/templates/scanpipe/panels/file_tree_panel.html index 00ebbe79a..95da0dd6c 100644 --- a/scanpipe/templates/scanpipe/panels/file_tree_panel.html +++ b/scanpipe/templates/scanpipe/panels/file_tree_panel.html @@ -2,38 +2,25 @@ {% for node in children %}
  • {% if node.is_dir %} -
    - - - - +
    + + + + + {{ node.name }}
    - {% if node.has_children %} {% endif %} - {% else %} -
    - +
    + + + {{ node.name }}
    {% endif %} diff --git a/scanpipe/templates/scanpipe/resource_tree.html b/scanpipe/templates/scanpipe/resource_tree.html index c43698404..03a1f4a60 100644 --- a/scanpipe/templates/scanpipe/resource_tree.html +++ b/scanpipe/templates/scanpipe/resource_tree.html @@ -1,33 +1,39 @@ {% extends "scanpipe/base.html" %} -{% load static %} -{% block title %}ScanCode.io: {{ project.name }} - Resource{% endblock %} +{% load static humanize %} +{% block title %}ScanCode.io: {{ project.name }} - Resource Tree{% endblock %} {% block extrahead %} - + {% endblock %} {% block content %} -
    -
    +
    + {% include 'scanpipe/includes/navbar_header.html' %} +
    +
    + {% include 'scanpipe/includes/breadcrumb.html' with linked_project=True current="Resource Tree" %} +
    +
    +
    + +
    +
    {% include "scanpipe/panels/file_tree_panel.html" with children=children path=path %}
    - -
    +
    @@ -35,33 +41,30 @@ {% endblock %} {% block scripts %} - + chevron.classList.toggle("rotated"); + e.stopPropagation(); + return; + } + }); + {% endblock %} \ No newline at end of file From 44015e710f4144e06c00bfafb4353e8cc4da77f8 Mon Sep 17 00:00:00 2001 From: Aayush Kumar Date: Fri, 4 Jul 2025 10:14:26 +0530 Subject: [PATCH 4/9] bump migration up to resolve failing tests Signed-off-by: Aayush Kumar --- ...4_codebaseresource_parent_path_and_more.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 scanpipe/migrations/0074_codebaseresource_parent_path_and_more.py diff --git a/scanpipe/migrations/0074_codebaseresource_parent_path_and_more.py b/scanpipe/migrations/0074_codebaseresource_parent_path_and_more.py new file mode 100644 index 000000000..0f424f678 --- /dev/null +++ b/scanpipe/migrations/0074_codebaseresource_parent_path_and_more.py @@ -0,0 +1,24 @@ +# Generated by Django 5.1.9 on 2025-06-19 21:19 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('scanpipe', '0073_add_sha1_git_checksum'), + ] + + operations = [ + migrations.AddField( + model_name='codebaseresource', + name='parent_path', + field=models.CharField(blank=True, help_text="The path of the resource's parent directory. Set to None for top-level (root) resources. Used to efficiently retrieve a directory's contents.", max_length=2000, null=True), + ), + migrations.AddIndex( + model_name='codebaseresource', + index=models.Index(fields=['project', 'parent_path'], name='scanpipe_co_project_008448_idx'), + ), + ] + + From e63c0f97ab08515e28ce21f206e07566c976c942 Mon Sep 17 00:00:00 2001 From: Aayush Kumar Date: Fri, 4 Jul 2025 10:19:22 +0530 Subject: [PATCH 5/9] remove conflicting migration Signed-off-by: Aayush Kumar --- ...3_codebaseresource_parent_path_and_more.py | 24 ------------------- 1 file changed, 24 deletions(-) delete mode 100644 scanpipe/migrations/0073_codebaseresource_parent_path_and_more.py diff --git a/scanpipe/migrations/0073_codebaseresource_parent_path_and_more.py b/scanpipe/migrations/0073_codebaseresource_parent_path_and_more.py deleted file mode 100644 index 86848b879..000000000 --- a/scanpipe/migrations/0073_codebaseresource_parent_path_and_more.py +++ /dev/null @@ -1,24 +0,0 @@ -# Generated by Django 5.1.9 on 2025-06-19 21:19 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('scanpipe', '0072_discovereddependency_uuid_unique'), - ] - - operations = [ - migrations.AddField( - model_name='codebaseresource', - name='parent_path', - field=models.CharField(blank=True, help_text="The path of the resource's parent directory. Set to None for top-level (root) resources. Used to efficiently retrieve a directory's contents.", max_length=2000, null=True), - ), - migrations.AddIndex( - model_name='codebaseresource', - index=models.Index(fields=['project', 'parent_path'], name='scanpipe_co_project_008448_idx'), - ), - ] - - From 3d1c4b7afb0120472b40ff56bf9dbfc96df2daff Mon Sep 17 00:00:00 2001 From: Aayush Kumar Date: Tue, 8 Jul 2025 00:41:26 +0530 Subject: [PATCH 6/9] implement suggested changes Signed-off-by: Aayush Kumar --- scanpipe/models.py | 13 +++++++++++++ ...ile_tree_panel.html => codebase_tree_panel.html} | 8 ++++---- scanpipe/templates/scanpipe/resource_tree.html | 11 ++++------- scanpipe/urls.py | 10 +++++----- scanpipe/views.py | 11 ++--------- 5 files changed, 28 insertions(+), 25 deletions(-) rename scanpipe/templates/scanpipe/panels/{file_tree_panel.html => codebase_tree_panel.html} (72%) diff --git a/scanpipe/models.py b/scanpipe/models.py index e3ca9aff8..f712780ed 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -46,6 +46,7 @@ from django.db import transaction from django.db.models import Case from django.db.models import Count +from django.db.models import Exists from django.db.models import IntegerField from django.db.models import OuterRef from django.db.models import Prefetch @@ -2383,6 +2384,18 @@ def macho_binaries(self): def executable_binaries(self): return self.union(self.win_exes(), self.macho_binaries(), self.elfs()) + def with_children(self, project): + """ + Annotate the QuerySet with has_children field based on whether + each resource has any children (subdirectories/files). + """ + subdirs = CodebaseResource.objects.filter( + project=project, + parent_path=OuterRef("path"), + ) + + return self.annotate(has_children=Exists(subdirs)) + class ScanFieldsModelMixin(models.Model): """Fields returned by the ScanCode-toolkit scans.""" diff --git a/scanpipe/templates/scanpipe/panels/file_tree_panel.html b/scanpipe/templates/scanpipe/panels/codebase_tree_panel.html similarity index 72% rename from scanpipe/templates/scanpipe/panels/file_tree_panel.html rename to scanpipe/templates/scanpipe/panels/codebase_tree_panel.html index 95da0dd6c..0ccc12b53 100644 --- a/scanpipe/templates/scanpipe/panels/file_tree_panel.html +++ b/scanpipe/templates/scanpipe/panels/codebase_tree_panel.html @@ -2,8 +2,8 @@ {% for node in children %}
  • {% if node.is_dir %} -
    -