Ignore large data files and bump scancode-toolkit (#1508)

* Ignore scanning large data files

Ignore scanning large data files which are larger than 1 MB
to avoid crashing scans on memory spikes.
Also rollback #1504

Reference: aboutcode-org/scancode-toolkit#3711
Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

* Bump scancode-toolkit to version v32.3.1

Also remove platform constraints from rust-inspector and
go-inspector.

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

* Increase size limit to skip scanning data file

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

* Add a scancodeio setting SCANCODEIO_SCAN_MAX_FILE_SIZE

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

* Use scancode with conda bugfix

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

* Address feedback

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

* Bump scancode-toolkit to v32.3.2

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

* Add scan_max_file_size to project settings

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

* Update CHANGELOG and docs on project settings

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

* Add scan_max_file_size to project settings UI

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

---------

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>
Co-authored-by: tdruez <tdruez@nexb.com>

Author: AyanSinhaMahapatra

CHANGELOG.rst

@@ -58,6 +58,16 @@ v34.9.4 (unreleased)
   checkbox on paginated list.
   https://github.com/aboutcode-org/scancode.io/issues/1524
 
+- Update scancode-toolkit to v32.3.2. See CHANGELOG for updates:
+  https://github.com/aboutcode-org/scancode-toolkit/releases/tag/v32.3.2
+  https://github.com/aboutcode-org/scancode-toolkit/releases/tag/v32.3.1
+
+- Adds  a project settings ``scan_max_file_size`` and a scancode.io settings field
+  ``SCANCODEIO_SCAN_MAX_FILE_SIZE`` to skip scanning files above a certain
+  file size (in bytes) as a temporary fix for large memory spikes while
+  scanning for licenses in certain large files.
+  https://github.com/aboutcode-org/scancode-toolkit/issues/3711
+
 v34.9.3 (2024-12-31)
 --------------------
 

docs/application-settings.rst

@@ -165,6 +165,18 @@ The value unit is second and is defined as an integer::
 
 Default: ``120`` (2 minutes)
 
+SCANCODEIO_SCAN_MAX_FILE_SIZE
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Maximum file size allowed for a file to be scanned when scanning a codebase.
+
+The value unit is bytes and is defined as an integer, see the following
+example of setting this at 5 MB::
+
+    SCANCODEIO_SCAN_MAX_FILE_SIZE=5242880
+
+Default: ``None`` (all files will be scanned)
+
 .. _scancodeio_settings_pipelines_dirs:
 
 SCANCODEIO_PIPELINES_DIRS

docs/project-configuration.rst

@@ -54,6 +54,7 @@ Content of a ``scancode-config.yml`` file:
     ignored_patterns:
      - '*.tmp'
      - 'tests/*'
+    scan_max_file_size: 5242880
     ignored_dependency_scopes:
      - package_type: npm
        scope: devDependencies
@@ -86,6 +87,24 @@ product_version
 
 The product version of this project, as specified within the DejaCode application.
 
+scan_max_file_size
+^^^^^^^^^^^^^^^^^^
+
+Maximum file size allowed for a file to be scanned when scanning a codebase.
+
+The value unit is bytes and is defined as an integer, see the following
+example of setting this at 5 MB::
+
+    scan_max_file_size=5242880
+
+Default is ``None``, in which case all files will be scanned.
+
+.. note::
+    This is the same as the scancodeio setting ``SCANCODEIO_SCAN_MAX_FILE_SIZE``
+    set using the .env file, and the project setting ``scan_max_file_size`` takes
+    precedence over the scancodeio setting ``SCANCODEIO_SCAN_MAX_FILE_SIZE``.
+
+
 ignored_patterns
 ^^^^^^^^^^^^^^^^
 

scancodeio/settings.py

@@ -100,6 +100,9 @@
 # Default to 2 minutes.
 SCANCODEIO_SCAN_FILE_TIMEOUT = env.int("SCANCODEIO_SCAN_FILE_TIMEOUT", default=120)
 
+# Default to None which scans all files
+SCANCODEIO_SCAN_MAX_FILE_SIZE = env.int("SCANCODEIO_SCAN_MAX_FILE_SIZE", default=None)
+
 # List views pagination, controls the number of items displayed per page.
 # Syntax in .env: SCANCODEIO_PAGINATE_BY=project=10,project_error=10
 SCANCODEIO_PAGINATE_BY = env.dict(

scanpipe/forms.py

@@ -437,6 +437,7 @@ class ProjectSettingsForm(forms.ModelForm):
         "ignored_vulnerabilities",
         "policies",
         "attribution_template",
+        "scan_max_file_size",
         "product_name",
         "product_version",
     ]
@@ -511,6 +512,15 @@ class ProjectSettingsForm(forms.ModelForm):
         ),
         widget=forms.Textarea(attrs={"class": "textarea is-dynamic", "rows": 3}),
     )
+    scan_max_file_size = forms.IntegerField(
+        label="Max file size to scan",
+        required=False,
+        help_text=(
+            "Maximum file size in bytes which should be skipped from scanning."
+            "File size is in bytes. Example: 5 MB is 5242880 bytes."
+        ),
+        widget=forms.NumberInput(attrs={"class": "input"}),
+    )
     product_name = forms.CharField(
         label="Product name",
         required=False,

scanpipe/models.py

@@ -920,6 +920,16 @@ def get_ignored_dependency_scopes_index(self):
 
         return dict(ignored_scope_index)
 
+    @cached_property
+    def get_scan_max_file_size(self):
+        """
+        Return a the ``scan_max_file_size`` settings value defined in this
+        Project env.
+        """
+        scan_max_file_size = self.get_env(field_name="scan_max_file_size")
+        if scan_max_file_size:
+            return scan_max_file_size
+
     @cached_property
     def ignored_dependency_scopes_index(self):
         """

scanpipe/pipes/flag.py

@@ -20,6 +20,7 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
+
 NO_STATUS = ""
 
 SCANNED = "scanned"
@@ -43,6 +44,7 @@
 IGNORED_DEFAULT_IGNORES = "ignored-default-ignores"
 IGNORED_DATA_FILE_NO_CLUES = "ignored-data-file-no-clues"
 IGNORED_DOC_FILE = "ignored-doc-file"
+IGNORED_BY_MAX_FILE_SIZE = "ignored-by-max-file-size"
 
 COMPLIANCE_LICENSES = "compliance-licenses"
 COMPLIANCE_SOURCEMIRROR = "compliance-sourcemirror"
@@ -102,6 +104,19 @@ def flag_ignored_patterns(project, patterns):
     return update_count
 
 
+def flag_and_ignore_files_over_max_size(resource_qs, file_size_limit):
+    """
+    Flag codebase resources which are over the max file size for scanning
+    and return all other files within the file size limit.
+    """
+    if not file_size_limit:
+        return resource_qs
+
+    return resource_qs.filter(size__gte=file_size_limit).update(
+        status=IGNORED_BY_MAX_FILE_SIZE
+    )
+
+
 def analyze_scanned_files(project):
     """Set the status for CodebaseResource to unknown or no license."""
     scanned_files = project.codebaseresources.files().status(SCANNED)

scanpipe/pipes/scancode.py

@@ -34,6 +34,7 @@
 from django.apps import apps
 from django.conf import settings
 from django.db.models import ObjectDoesNotExist
+from django.db.models import Q
 
 from commoncode import fileutils
 from commoncode.resource import VirtualCodebase
@@ -58,6 +59,7 @@
 Utilities to deal with ScanCode toolkit features and objects.
 """
 
+
 scanpipe_app = apps.get_app_config("scanpipe")
 
 
@@ -291,6 +293,7 @@ def scan_resources(
     save_func,
     scan_func_kwargs=None,
     progress_logger=None,
+    file_size_limit=None,
 ):
     """
     Run the `scan_func` on the codebase resources of the provided `resource_qs`.
@@ -310,9 +313,21 @@ def scan_resources(
     if not scan_func_kwargs:
         scan_func_kwargs = {}
 
-    resource_count = resource_qs.count()
+    # Skip scannning files larger than the specified max size
+    skipped_files_max_size = flag.flag_and_ignore_files_over_max_size(
+        resource_qs=resource_qs,
+        file_size_limit=file_size_limit,
+    )
+    if file_size_limit and skipped_files_max_size:
+        logger.info(
+            f"Skipped {skipped_files_max_size} files over the size of {file_size_limit}"
+        )
+
+    scan_resource_qs = resource_qs.filter(~Q(status=flag.IGNORED_BY_MAX_FILE_SIZE))
+
+    resource_count = scan_resource_qs.count()
     logger.info(f"Scan {resource_count} codebase resources with {scan_func.__name__}")
-    resource_iterator = resource_qs.iterator(chunk_size=2000)
+    resource_iterator = scan_resource_qs.iterator(chunk_size=2000)
     progress = LoopProgress(resource_count, logger=progress_logger)
     max_workers = get_max_workers(keep_available=1)
 
@@ -350,14 +365,7 @@ def scan_resources(
                     "Please ensure that there is at least 2 GB of available memory per "
                     "CPU core for successful execution."
                 )
-
-                resource.project.add_error(
-                    exception=broken_pool_error,
-                    model="scan_resources",
-                    description=message,
-                    object_instance=resource,
-                )
-                continue
+                raise broken_pool_error from InsufficientResourcesError(message)
 
             save_func(resource, scan_results, scan_errors)
 
@@ -374,11 +382,18 @@ def scan_for_files(project, resource_qs=None, progress_logger=None):
     if resource_qs is None:
         resource_qs = project.codebaseresources.no_status()
 
+    # Get max file size limit set in project settings, or alternatively
+    # get it from scancodeio settings
+    file_size_limit = project.get_scan_max_file_size
+    if not file_size_limit:
+        file_size_limit = settings.SCANCODEIO_SCAN_MAX_FILE_SIZE
+
     scan_resources(
         resource_qs=resource_qs,
         scan_func=scan_file,
         save_func=save_scan_file_results,
         progress_logger=progress_logger,
+        file_size_limit=file_size_limit,
     )
 
 

scanpipe/templates/scanpipe/project_settings.html

@@ -114,6 +114,18 @@
                   </div>
                 </div>
 
+                <div class="field">
+                  <label class="label" for="{{ form.scan_max_file_size.id_for_label }}">
+                    {{ form.scan_max_file_size.label }}
+                  </label>
+                  <div class="control">
+                    {{ form.scan_max_file_size }}
+                  </div>
+                  <div class="help">
+                    {{ form.scan_max_file_size.help_text|safe|linebreaksbr }}
+                  </div>
+                </div>
+
                 <div class="field">
                   <label class="label" for="{{ form.ignored_dependency_scopes.id_for_label }}">
                     {{ form.ignored_dependency_scopes.label }}

scanpipe/tests/data/manifests/openpdf-parent-1.3.11_scan_package.json

@@ -19,7 +19,7 @@
       "errors": [],
       "warnings": [],
       "extra_data": {
-        "spdx_license_list_version": "3.25",
+        "spdx_license_list_version": "3.26",
         "files_count": 1
       }
     }