Skip to content

Commit 4391b48

Browse files
tdrueztdruez
authored
Capture errors during the inspect_elf_binaries pipeline execution (#1123)
Signed-off-by: tdruez <tdruez@nexb.com>
1 parent eec8b12 commit 4391b48

File tree

14 files changed

+166
-55
lines changed

14 files changed

+166
-55
lines changed

‎CHANGELOG.rst

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,14 @@ v34.1.0 (unreleased)
1515
Universal Ctags.
1616
https://github.com/nexB/scancode.io/pull/1116
1717

18+
- Capture errors during the `inspect_elf_binaries` pipeline execution.
19+
Errors on resource inspection are stored as project error message instead of global
20+
pipeline failure.
21+
The problematic resource path is stored in the message details and displayed in the
22+
message list UI as a link to the resource details view.
23+
https://github.com/nexB/scancode.io/issues/1121
24+
https://github.com/nexB/scancode.io/issues/1122
25+
1826
v34.0.0 (2024-03-04)
1927
--------------------
2028

‎docs/built-in-pipelines.rst

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ Analyse Docker Windows Image
4545
.. _pipeline_collect_symbols:
4646

4747
Collect Codebase Symbols (addon)
48-
---------------------------------
48+
--------------------------------
4949
.. autoclass:: scanpipe.pipelines.collect_symbols.CollectSymbols()
5050
:members:
5151
:member-order: bysource

‎docs/scanpipe-pipes.rst

Lines changed: 28 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,28 +33,53 @@ Docker
3333
.. automodule:: scanpipe.pipes.docker
3434
:members:
3535

36+
ELF
37+
---
38+
.. automodule:: scanpipe.pipes.elf
39+
:members:
40+
3641
Fetch
3742
-----
3843
.. automodule:: scanpipe.pipes.fetch
3944
:members:
4045
:exclude-members: Download
4146

47+
Flag
48+
----
49+
.. automodule:: scanpipe.pipes.flag
50+
:members:
51+
4252
Input
4353
-----
4454
.. automodule:: scanpipe.pipes.input
4555
:members:
4656

57+
JS
58+
--
59+
.. automodule:: scanpipe.pipes.js
60+
:members:
61+
4762
JVM
4863
---
4964
.. automodule:: scanpipe.pipes.jvm
5065
:members:
5166

67+
MatchCode
68+
---------
69+
.. automodule:: scanpipe.pipes.matchcode
70+
:members:
71+
5272
Output
5373
------
5474
.. automodule:: scanpipe.pipes.output
5575
:members:
5676
:exclude-members: JSONResultsGenerator
5777

78+
PathMap
79+
-------
80+
.. automodule:: scanpipe.pipes.pathmap
81+
:members:
82+
5883
PurlDB
5984
------
6085
.. automodule:: scanpipe.pipes.purldb
@@ -80,9 +105,9 @@ SPDX
80105
.. automodule:: scanpipe.pipes.spdx
81106
:members:
82107

83-
Flag
84-
----
85-
.. automodule:: scanpipe.pipes.flag
108+
Symbols
109+
-------
110+
.. automodule:: scanpipe.pipes.symbols
86111
:members:
87112

88113
VulnerableCode

‎scanpipe/models.py

Lines changed: 55 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1097,12 +1097,20 @@ def get_next_run(self):
10971097
return self.runs.not_started().earliest("created_date")
10981098

10991099
def add_message(
1100-
self, severity, description="", model="", details=None, exception=None
1100+
self,
1101+
severity,
1102+
description="",
1103+
model="",
1104+
details=None,
1105+
exception=None,
1106+
resource=None,
11011107
):
11021108
"""
11031109
Create a ProjectMessage record for this Project.
11041110
11051111
The ``model`` attribute can be provided as a string or as a Model class.
1112+
A ``resource`` can be provided to keep track of the codebase resource that was
1113+
analyzed when the error occurred.
11061114
"""
11071115
if inspect.isclass(model):
11081116
model = model.__name__
@@ -1114,29 +1122,61 @@ def add_message(
11141122
if exception and not description:
11151123
description = str(exception)
11161124

1125+
details = details or {}
1126+
if resource:
1127+
# Do not change this field name as it has special behavior in templates.
1128+
details["resource_path"] = resource.path
1129+
11171130
return ProjectMessage.objects.create(
11181131
project=self,
11191132
severity=severity,
11201133
description=description,
11211134
model=model,
1122-
details=details or {},
1135+
details=details,
11231136
traceback=traceback,
11241137
)
11251138

1126-
def add_info(self, description="", model="", details=None, exception=None):
1139+
def add_info(
1140+
self,
1141+
description="",
1142+
model="",
1143+
details=None,
1144+
exception=None,
1145+
resource=None,
1146+
):
11271147
"""Create an INFO ProjectMessage record for this project."""
11281148
severity = ProjectMessage.Severity.INFO
1129-
return self.add_message(severity, description, model, details, exception)
1149+
return self.add_message(
1150+
severity, description, model, details, exception, resource
1151+
)
11301152

1131-
def add_warning(self, description="", model="", details=None, exception=None):
1153+
def add_warning(
1154+
self,
1155+
description="",
1156+
model="",
1157+
details=None,
1158+
exception=None,
1159+
resource=None,
1160+
):
11321161
"""Create a WARNING ProjectMessage record for this project."""
11331162
severity = ProjectMessage.Severity.WARNING
1134-
return self.add_message(severity, description, model, details, exception)
1163+
return self.add_message(
1164+
severity, description, model, details, exception, resource
1165+
)
11351166

1136-
def add_error(self, description="", model="", details=None, exception=None):
1167+
def add_error(
1168+
self,
1169+
description="",
1170+
model="",
1171+
details=None,
1172+
exception=None,
1173+
resource=None,
1174+
):
11371175
"""Create an ERROR ProjectMessage record using for this project."""
11381176
severity = ProjectMessage.Severity.ERROR
1139-
return self.add_message(severity, description, model, details, exception)
1177+
return self.add_message(
1178+
severity, description, model, details, exception, resource
1179+
)
11401180

11411181
def get_absolute_url(self):
11421182
"""Return this project's details URL."""
@@ -1409,10 +1449,15 @@ def add_error(self, exception):
14091449
Create a ProjectMessage record using the provided ``exception`` Exception
14101450
instance.
14111451
"""
1452+
resource = None
1453+
if isinstance(self, CodebaseResource):
1454+
resource = self
1455+
14121456
return self.project.add_error(
14131457
model=self.__class__,
14141458
details=model_to_dict(self),
14151459
exception=exception,
1460+
resource=resource,
14161461
)
14171462

14181463
def add_errors(self, exceptions):
@@ -2538,12 +2583,9 @@ def create_and_add_package(self, package_data):
25382583
except Exception as exception:
25392584
self.project.add_warning(
25402585
model=DiscoveredPackage,
2541-
details={
2542-
"codebase_resource_path": self.path,
2543-
"codebase_resource_pk": self.pk,
2544-
**package_data,
2545-
},
2586+
details=package_data,
25462587
exception=exception,
2588+
resource=self,
25472589
)
25482590
else:
25492591
self.add_package(package)

‎scanpipe/pipelines/__init__.py

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -230,25 +230,35 @@ def download_missing_inputs(self):
230230
if errors:
231231
raise InputFileError(errors)
232232

233-
def add_error(self, exception):
233+
def add_error(self, exception, resource=None):
234234
"""Create a ``ProjectMessage`` ERROR record on the current `project`."""
235-
self.project.add_error(model=self.pipeline_name, exception=exception)
235+
self.project.add_error(
236+
model=self.pipeline_name,
237+
exception=exception,
238+
resource=resource,
239+
)
236240

237241
@contextmanager
238-
def save_errors(self, *exceptions):
242+
def save_errors(self, *exceptions, **kwargs):
239243
"""
240244
Context manager to save specified exceptions as ``ProjectMessage`` in the
241245
database.
242246
243-
Example in a Pipeline step:
247+
- Example in a Pipeline step::
244248
245249
with self.save_errors(rootfs.DistroNotFound):
246250
rootfs.scan_rootfs_for_system_packages(self.project, rfs)
251+
252+
- Example when iterating over resources::
253+
254+
for resource in self.project.codebaseresources.all():
255+
with self.save_errors(Exception, resource=resource):
256+
analyse(resource)
247257
"""
248258
try:
249259
yield
250260
except exceptions as error:
251-
self.add_error(exception=error)
261+
self.add_error(exception=error, **kwargs)
252262

253263

254264
class Pipeline(BasePipeline):

‎scanpipe/pipelines/inspect_elf_binaries.py

Lines changed: 5 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,8 @@
2020
# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
2121
# Visit https://github.com/nexB/scancode.io for support and download.
2222

23-
from pathlib import Path
24-
25-
from elf_inspector.dwarf import get_dwarf_paths
26-
2723
from scanpipe.pipelines import Pipeline
24+
from scanpipe.pipes.elf import collect_dwarf_source_path_references
2825

2926

3027
class InspectELFBinaries(Pipeline):
@@ -38,10 +35,7 @@ def steps(cls):
3835
return (cls.collect_dwarf_source_path_references,)
3936

4037
def collect_dwarf_source_path_references(self):
41-
"""
42-
Update ``extra_data`` of ELF files with
43-
dwarf data extracted from ELF files.
44-
"""
45-
for elf in self.project.codebaseresources.elfs():
46-
dwarf_paths = get_dwarf_paths(Path(self.project.codebase_path / elf.path))
47-
elf.update_extra_data(dwarf_paths)
38+
"""Collect DWARF paths from ELF files and set values on the extra_data field."""
39+
for elf_resource in self.project.codebaseresources.elfs():
40+
with self.save_errors(Exception, resource=elf_resource):
41+
collect_dwarf_source_path_references(elf_resource)

‎scanpipe/pipes/d2d.py

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -826,15 +826,13 @@ def create_indexes(cls, project, from_about_files, logger=None):
826826
package_data = resolve.resolve_about_package(
827827
input_location=str(about_file_resource.location_path)
828828
)
829-
error_message_details = {
830-
"path": about_file_resource.path,
831-
"package_data": package_data,
832-
}
829+
error_message_details = {"package_data": package_data}
833830
if not package_data:
834831
project.add_error(
835832
description="Cannot create package from ABOUT file",
836833
model="map_about_files",
837834
details=error_message_details,
835+
resource=about_file_resource,
838836
)
839837
continue
840838

@@ -846,6 +844,7 @@ def create_indexes(cls, project, from_about_files, logger=None):
846844
description="ABOUT file does not have about_resource",
847845
model="map_about_files",
848846
details=error_message_details,
847+
resource=about_file_resource,
849848
)
850849
continue
851850
else:

‎scanpipe/pipes/elf.py

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
# SPDX-License-Identifier: Apache-2.0
2+
#
3+
# http://nexb.com and https://github.com/nexB/scancode.io
4+
# The ScanCode.io software is licensed under the Apache License version 2.0.
5+
# Data generated with ScanCode.io is provided as-is without warranties.
6+
# ScanCode is a trademark of nexB Inc.
7+
#
8+
# You may not use this software except in compliance with the License.
9+
# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
10+
# Unless required by applicable law or agreed to in writing, software distributed
11+
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
12+
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
13+
# specific language governing permissions and limitations under the License.
14+
#
15+
# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
16+
# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
17+
# ScanCode.io should be considered or used as legal advice. Consult an Attorney
18+
# for any legal advice.
19+
#
20+
# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
21+
# Visit https://github.com/nexB/scancode.io for support and download.
22+
23+
from elf_inspector.dwarf import get_dwarf_paths
24+
25+
26+
def collect_dwarf_source_path_references(resource):
27+
"""Collect and store the DWARF debug paths of the provided ELF ``resource``."""
28+
dwarf_paths = get_dwarf_paths(resource.location_path)
29+
resource.update_extra_data(dwarf_paths)
30+
return dwarf_paths

‎scanpipe/pipes/resolve.py

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -57,16 +57,13 @@ def get_packages(project, package_registry, manifest_resources, model=None):
5757
return
5858

5959
for resource in manifest_resources:
60-
if packages := get_packages_from_manifest(
61-
input_location=resource.location,
62-
package_registry=package_registry,
63-
):
60+
if packages := get_packages_from_manifest(resource.location, package_registry):
6461
resolved_packages.extend(packages)
6562
else:
6663
project.add_error(
6764
description="No packages could be resolved for",
6865
model=model,
69-
details={"path": resource.path},
66+
resource=resource,
7067
)
7168

7269
return resolved_packages

‎scanpipe/pipes/scancode.py

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -391,11 +391,10 @@ def add_resource_to_package(package_uid, resource, project):
391391
try:
392392
package = project.discoveredpackages.get(package_uid=package_uid)
393393
except ObjectDoesNotExist as error:
394-
details = {
395-
"package_uid": str(package_uid),
396-
"resource": str(resource),
397-
}
398-
project.add_error(error, model="assemble_package", details=details)
394+
details = {"package_uid": str(package_uid)}
395+
project.add_error(
396+
error, model="assemble_package", details=details, resource=resource
397+
)
399398
return
400399

401400
resource.discovered_packages.add(package)

0 commit comments

Comments
 (0)