Skip to content

Commit 81e05b9

Browse files
aayushkdevaayushkdev
authored
Added the ability to export the current filtered QuerySet of a 'FilterView' into the JSON format, Solves issue #1319 (#1572)
Added the ability to export the current filtered QuerySet of a FilterView into the JSON format #1319 Signed-off-by: Aayush Kumar <aayush214.kumar@gmail.com>
1 parent a09d98b commit 81e05b9

File tree

4 files changed

+270
-1
lines changed

4 files changed

+270
-1
lines changed

‎CHANGELOG.rst

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,9 @@ v34.10.2 (unreleased)
1313
if available, when the codebase has been sent for matching to MatchCode.
1414
https://github.com/aboutcode-org/scancode.io/pull/1656
1515

16+
- Add the ability to export filtered QuerySet of a FilterView into the JSON format.
17+
https://github.com/aboutcode-org/scancode.io/pull/1572
18+
1619
v34.10.1 (2025-03-26)
1720
---------------------
1821

‎scanpipe/templates/scanpipe/dropdowns/list_actions_dropdown.html

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,10 @@
1010
<div class="dropdown-menu" id="dropdown-menu-action" role="menu">
1111
<div class="dropdown-content">
1212
<a href="?{{ export_xlsx_url_query }}" class="dropdown-item">
13-
<i class="fa-solid fa-download mr-2"></i>Export results as XLSX
13+
<i class="fa-solid fa-file-excel mr-2"></i>Export results as XLSX
14+
</a>
15+
<a href="?{{ export_json_url_query }}" class="dropdown-item">
16+
<i class="fa-solid fa-file-code mr-2"></i>Export results as JSON
1417
</a>
1518
</div>
1619
</div>

‎scanpipe/tests/test_views.py

Lines changed: 208 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929

3030
from django.apps import apps
3131
from django.core.exceptions import SuspiciousFileOperation
32+
from django.http import FileResponse
3233
from django.http.response import Http404
3334
from django.test import TestCase
3435
from django.test import override_settings
@@ -1340,3 +1341,210 @@ def test_scanpipe_views_codebase_resource_details_get_matched_snippet_annotation
13401341
results = CodebaseResourceDetailsView.get_matched_snippet_annotations(resource1)
13411342
expected_results = [{"start_line": 1, "end_line": 6}]
13421343
self.assertEqual(expected_results, results)
1344+
1345+
def test_project_packages_export_json(self):
1346+
make_package(self.project1, package_url="pkg:type/a")
1347+
1348+
url = reverse("project_packages", args=[self.project1.slug])
1349+
response = self.client.get(url + "?export_json=True")
1350+
1351+
self.assertIsInstance(response, FileResponse)
1352+
self.assertEqual(response.get("Content-Type"), "application/json")
1353+
self.assertTrue(response.get("Content-Disposition").startswith("attachment"))
1354+
1355+
file_content = b"".join(response.streaming_content).decode("utf-8")
1356+
json_data = json.loads(file_content)
1357+
1358+
expected_fields = [
1359+
"purl",
1360+
"type",
1361+
"namespace",
1362+
"name",
1363+
"version",
1364+
"qualifiers",
1365+
"subpath",
1366+
"tag",
1367+
"primary_language",
1368+
"description",
1369+
"notes",
1370+
"release_date",
1371+
"parties",
1372+
"keywords",
1373+
"homepage_url",
1374+
"download_url",
1375+
"bug_tracking_url",
1376+
"code_view_url",
1377+
"vcs_url",
1378+
"repository_homepage_url",
1379+
"repository_download_url",
1380+
"api_data_url",
1381+
"size",
1382+
"md5",
1383+
"sha1",
1384+
"sha256",
1385+
"sha512",
1386+
"copyright",
1387+
"holder",
1388+
"declared_license_expression",
1389+
"declared_license_expression_spdx",
1390+
"other_license_expression",
1391+
"other_license_expression_spdx",
1392+
"extracted_license_statement",
1393+
"compliance_alert",
1394+
"notice_text",
1395+
"source_packages",
1396+
"package_uid",
1397+
"is_private",
1398+
"is_virtual",
1399+
"datasource_ids",
1400+
"datafile_paths",
1401+
"file_references",
1402+
"missing_resources",
1403+
"modified_resources",
1404+
]
1405+
1406+
for field in expected_fields:
1407+
self.assertIn(field, json_data[0])
1408+
1409+
def test_project_dependencies_export_json(self):
1410+
make_resource_file(self.project1, "file.ext")
1411+
make_dependency(self.project1)
1412+
1413+
url = reverse("project_dependencies", args=[self.project1.slug])
1414+
response = self.client.get(url + "?export_json=True")
1415+
1416+
self.assertIsInstance(response, FileResponse)
1417+
self.assertEqual(response.get("Content-Type"), "application/json")
1418+
self.assertTrue(response.get("Content-Disposition").startswith("attachment"))
1419+
1420+
file_content = b"".join(response.streaming_content).decode("utf-8")
1421+
json_data = json.loads(file_content)
1422+
1423+
expected_fields = [
1424+
"purl",
1425+
"extracted_requirement",
1426+
"scope",
1427+
"is_runtime",
1428+
"is_optional",
1429+
"is_pinned",
1430+
"is_direct",
1431+
"dependency_uid",
1432+
"for_package_uid",
1433+
"resolved_to_package_uid",
1434+
"datafile_path",
1435+
"datasource_id",
1436+
"package_type",
1437+
]
1438+
1439+
for field in expected_fields:
1440+
self.assertIn(field, json_data[0])
1441+
1442+
def test_project_relations_export_json(self):
1443+
make_relation(
1444+
from_resource=make_resource_file(self.project1, "file1.ext"),
1445+
to_resource=make_resource_file(self.project1, "file2.ext"),
1446+
map_type="path",
1447+
)
1448+
1449+
url = reverse("project_relations", args=[self.project1.slug])
1450+
response = self.client.get(url + "?export_json=True")
1451+
1452+
self.assertIsInstance(response, FileResponse)
1453+
self.assertEqual(response.get("Content-Type"), "application/json")
1454+
self.assertTrue(response.get("Content-Disposition").startswith("attachment"))
1455+
1456+
file_content = b"".join(response.streaming_content).decode("utf-8")
1457+
json_data = json.loads(file_content)
1458+
1459+
expected_fields = [
1460+
"to_resource",
1461+
"status",
1462+
"map_type",
1463+
"score",
1464+
"from_resource",
1465+
]
1466+
1467+
for field in expected_fields:
1468+
self.assertIn(field, json_data[0])
1469+
1470+
def test_project_messages_export_json(self):
1471+
self.project1.add_message("warning")
1472+
1473+
url = reverse("project_messages", args=[self.project1.slug])
1474+
response = self.client.get(url + "?export_json=True")
1475+
1476+
self.assertIsInstance(response, FileResponse)
1477+
self.assertEqual(response.get("Content-Type"), "application/json")
1478+
self.assertTrue(response.get("Content-Disposition").startswith("attachment"))
1479+
1480+
file_content = b"".join(response.streaming_content).decode("utf-8")
1481+
json_data = json.loads(file_content)
1482+
1483+
expected_fields = [
1484+
"uuid",
1485+
"severity",
1486+
"description",
1487+
"model",
1488+
"details",
1489+
"traceback",
1490+
"created_date",
1491+
]
1492+
1493+
for field in expected_fields:
1494+
self.assertIn(field, json_data[0])
1495+
1496+
def test_project_codebase_resources_export_json(self):
1497+
make_resource_file(self.project1, "file.ext")
1498+
1499+
url = reverse("project_resources", args=[self.project1.slug])
1500+
response = self.client.get(url + "?export_json=True")
1501+
1502+
self.assertIsInstance(response, FileResponse)
1503+
self.assertEqual(response.get("Content-Type"), "application/json")
1504+
self.assertTrue(response.get("Content-Disposition").startswith("attachment"))
1505+
1506+
file_content = b"".join(response.streaming_content).decode("utf-8")
1507+
json_data = json.loads(file_content)
1508+
1509+
expected_fields = [
1510+
"path",
1511+
"type",
1512+
"name",
1513+
"status",
1514+
"for_packages",
1515+
"tag",
1516+
"extension",
1517+
"size",
1518+
"mime_type",
1519+
"file_type",
1520+
"programming_language",
1521+
"detected_license_expression",
1522+
"detected_license_expression_spdx",
1523+
"license_detections",
1524+
"license_clues",
1525+
"percentage_of_license_text",
1526+
"compliance_alert",
1527+
"copyrights",
1528+
"holders",
1529+
"authors",
1530+
"package_data",
1531+
"emails",
1532+
"urls",
1533+
"md5",
1534+
"sha1",
1535+
"sha256",
1536+
"sha512",
1537+
"is_binary",
1538+
"is_text",
1539+
"is_archive",
1540+
"is_media",
1541+
"is_legal",
1542+
"is_manifest",
1543+
"is_readme",
1544+
"is_top_level",
1545+
"is_key_file",
1546+
"extra_data",
1547+
]
1548+
1549+
for field in expected_fields:
1550+
self.assertIn(field, json_data[0])

‎scanpipe/views.py

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
from django.core.exceptions import SuspiciousFileOperation
3737
from django.core.exceptions import ValidationError
3838
from django.core.files.storage.filesystem import FileSystemStorage
39+
from django.core.serializers.json import DjangoJSONEncoder
3940
from django.db.models import Prefetch
4041
from django.db.models.manager import Manager
4142
from django.http import FileResponse
@@ -499,6 +500,55 @@ def get(self, request, *args, **kwargs):
499500
return response
500501

501502

503+
class ExportJSONMixin:
504+
"""
505+
Add the ability to export the current filtered QuerySet of a `FilterView`
506+
into JSON format.
507+
"""
508+
509+
export_json_query_param = "export_json"
510+
511+
def get_export_json_queryset(self):
512+
return self.filterset.qs
513+
514+
def get_export_json_filename(self):
515+
return f"{self.project.name}_{self.model._meta.model_name}.json"
516+
517+
def export_json_file_response(self):
518+
from scanpipe.api.serializers import get_model_serializer
519+
520+
queryset = self.get_export_json_queryset()
521+
serializer_class = get_model_serializer(queryset.model)
522+
serializer = serializer_class(queryset, many=True)
523+
serialized_data = json.dumps(serializer.data, indent=2, cls=DjangoJSONEncoder)
524+
525+
output_file = io.BytesIO(serialized_data.encode("utf-8"))
526+
527+
return FileResponse(
528+
output_file,
529+
as_attachment=True,
530+
filename=self.get_export_json_filename(),
531+
content_type="application/json",
532+
)
533+
534+
def get_context_data(self, **kwargs):
535+
context = super().get_context_data(**kwargs)
536+
537+
query_dict = self.request.GET.copy()
538+
query_dict[self.export_json_query_param] = True
539+
context["export_json_url_query"] = query_dict.urlencode()
540+
541+
return context
542+
543+
def get(self, request, *args, **kwargs):
544+
response = super().get(request, *args, **kwargs)
545+
546+
if request.GET.get(self.export_json_query_param):
547+
return self.export_json_file_response()
548+
549+
return response
550+
551+
502552
class FormAjaxMixin:
503553
def is_xhr(self):
504554
return self.request.META.get("HTTP_X_REQUESTED_WITH") == "XMLHttpRequest"
@@ -1543,6 +1593,7 @@ class CodebaseResourceListView(
15431593
ProjectRelatedViewMixin,
15441594
TableColumnsMixin,
15451595
ExportXLSXMixin,
1596+
ExportJSONMixin,
15461597
PaginatedFilterView,
15471598
):
15481599
model = CodebaseResource
@@ -1616,6 +1667,7 @@ class DiscoveredPackageListView(
16161667
ProjectRelatedViewMixin,
16171668
TableColumnsMixin,
16181669
ExportXLSXMixin,
1670+
ExportJSONMixin,
16191671
PaginatedFilterView,
16201672
):
16211673
model = DiscoveredPackage
@@ -1672,6 +1724,7 @@ class DiscoveredDependencyListView(
16721724
ProjectRelatedViewMixin,
16731725
TableColumnsMixin,
16741726
ExportXLSXMixin,
1727+
ExportJSONMixin,
16751728
PaginatedFilterView,
16761729
):
16771730
model = DiscoveredDependency
@@ -1741,6 +1794,7 @@ class ProjectMessageListView(
17411794
ProjectRelatedViewMixin,
17421795
TableColumnsMixin,
17431796
ExportXLSXMixin,
1797+
ExportJSONMixin,
17441798
FilterView,
17451799
):
17461800
model = ProjectMessage
@@ -1765,6 +1819,7 @@ class CodebaseRelationListView(
17651819
PrefetchRelatedViewMixin,
17661820
TableColumnsMixin,
17671821
ExportXLSXMixin,
1822+
ExportJSONMixin,
17681823
PaginatedFilterView,
17691824
):
17701825
model = CodebaseRelation

0 commit comments

Comments
 (0)