From fb1346023ed90fc16ab8c39fd2cd9a5f82e03489 Mon Sep 17 00:00:00 2001 From: John Pennycook Date: Wed, 23 Apr 2025 09:45:03 +0100 Subject: [PATCH 1/5] Add regression test for backslash-newline at EOF gcc accepts this with a warning and clang accepts it silently, but CBI exits with a RuntimeError. Signed-off-by: John Pennycook --- tests/preprocessor/__init__.py | 2 ++ tests/preprocessor/test_warnings.py | 42 +++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 tests/preprocessor/__init__.py create mode 100644 tests/preprocessor/test_warnings.py diff --git a/tests/preprocessor/__init__.py b/tests/preprocessor/__init__.py new file mode 100644 index 0000000..93af6d4 --- /dev/null +++ b/tests/preprocessor/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2019 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause diff --git a/tests/preprocessor/test_warnings.py b/tests/preprocessor/test_warnings.py new file mode 100644 index 0000000..77da6e9 --- /dev/null +++ b/tests/preprocessor/test_warnings.py @@ -0,0 +1,42 @@ +# Copyright (C) 2019-2024 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause + +import logging +import os +import tempfile +import unittest +from pathlib import Path + +from codebasin import file_parser + + +class TestPreprocessorWarnings(unittest.TestCase): + """ + Test that preprocessor generates warnings for weird corner cases. + """ + + def setUp(self): + self.cwd = os.getcwd() + + def tearDown(self): + os.chdir(self.cwd) + + def test_backslash_eof(self): + """Check backslash-newline at EOF is only a warning""" + tmp = tempfile.TemporaryDirectory() + path = Path(tmp.name) + os.chdir(tmp.name) + + with open(path / "test.hpp", mode="w") as f: + f.write("#define BAD_MACRO \\\n") + + parser = file_parser.FileParser(path / "test.hpp") + + logging.disable(logging.NOTSET) + logger = logging.getLogger("codebasin") + with self.assertLogs(logger, level="WARNING") as cm: + _ = parser.parse_file() + logging.disable() + self.assertRegex(cm.output[0], "backslash-newline at end of file") + + tmp.cleanup() From bd86a4fba7f90fec41d8a0a97a188ed32f6354dd Mon Sep 17 00:00:00 2001 From: John Pennycook Date: Tue, 22 Apr 2025 16:47:59 +0100 Subject: [PATCH 2/5] Add current file to debug log Signed-off-by: John Pennycook --- codebasin/finder.py | 1 + 1 file changed, 1 insertion(+) diff --git a/codebasin/finder.py b/codebasin/finder.py index 5c376bc..8986974 100644 --- a/codebasin/finder.py +++ b/codebasin/finder.py @@ -176,6 +176,7 @@ def find( leave=False, disable=not show_progress, ): + log.debug(f"Parsing {f}") state.insert_file(f) # Process each tree, by associating nodes with platforms From cd216c206e3cfe7b8009286daed2349ef7f408e1 Mon Sep 17 00:00:00 2001 From: John Pennycook Date: Tue, 22 Apr 2025 17:41:54 +0100 Subject: [PATCH 3/5] Remove "relaxed" parsing mode This parsing mode has never been enabled, and referring to it when parsing fails does not help end users. Instead, we should: - Treat any parsing failure as a bug; and - Selectively permit certain failures (with warnings). Signed-off-by: John Pennycook --- codebasin/file_source.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/codebasin/file_source.py b/codebasin/file_source.py index 94ac5cb..8842963 100644 --- a/codebasin/file_source.py +++ b/codebasin/file_source.py @@ -460,13 +460,11 @@ def logical_result(self): ) -def c_file_source(fp, relaxed=False, directives_only=False): +def c_file_source(fp, directives_only=False): """ Process file fp in terms of logical (sloc) and physical lines of C code. Yield blocks of logical lines of code with physical extents. Return total lines at exit. - Relaxed allows for inconsistent state at the end of parsing, usefule for - special composition cases. directives_only sets up parser to only process directive lines such that the output can be fed to another file source (i.e. Fortran). """ @@ -513,22 +511,21 @@ def c_file_source(fp, relaxed=False, directives_only=False): yield curr_line total_sloc += curr_line.physical_reset() - if not relaxed and not cleaner.state == ["TOPLEVEL"]: + if not cleaner.state == ["TOPLEVEL"]: raise RuntimeError( - "Parser must end at top level without 'relaxed' mode.", + "Parsing failed. Please open a bug report at: " + "https://github.com/intel/code-base-investigator/issues/new?template=bug_report.yml.", # noqa: E501 ) return (total_sloc, total_physical_lines) -def fortran_file_source(fp, relaxed=False): +def fortran_file_source(fp): """ Process file fp in terms of logical (sloc) and physical lines of fixed-form Fortran code. Yield blocks of logical lines of code with physical extents. Return total lines at exit. - Relaxed allows for inconsistent state at the end of parsing, usefule for - special composition cases. """ current_physical_line = one_space_line() @@ -593,9 +590,10 @@ def fortran_file_source(fp, relaxed=False): yield curr_line total_sloc += curr_line.physical_reset() - if not relaxed and not cleaner.state == ["TOPLEVEL"]: + if not cleaner.state == ["TOPLEVEL"]: raise RuntimeError( - "Parser must end at top level without 'relaxed' mode.", + "Parsing failed. Please open a bug report at: " + "https://github.com/intel/code-base-investigator/issues/new?template=bug_report.yml.", # noqa: E501 ) return (total_sloc, total_physical_lines) @@ -642,7 +640,7 @@ def process(self, lineiter): pass -def asm_file_source(fp, relaxed=False): +def asm_file_source(fp): """ Process file fp in terms of logical (sloc) and physical lines of ASM code. Yield blocks of logical lines of code with physical extents. From effdff98f068e197de807e7fbf318f094cb02853 Mon Sep 17 00:00:00 2001 From: John Pennycook Date: Wed, 23 Apr 2025 09:46:33 +0100 Subject: [PATCH 4/5] Accept C files that end in a continuation Signed-off-by: John Pennycook --- codebasin/file_source.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/codebasin/file_source.py b/codebasin/file_source.py index 8842963..b0b5c7d 100644 --- a/codebasin/file_source.py +++ b/codebasin/file_source.py @@ -468,7 +468,6 @@ def c_file_source(fp, directives_only=False): directives_only sets up parser to only process directive lines such that the output can be fed to another file source (i.e. Fortran). """ - current_physical_line = one_space_line() cleaner = c_cleaner(current_physical_line, directives_only) @@ -477,6 +476,7 @@ def c_file_source(fp, directives_only=False): total_sloc = 0 physical_line_num = 0 + continued = False for physical_line_num, line in enumerate(fp, start=1): current_physical_line.__init__() end = len(line) @@ -511,7 +511,14 @@ def c_file_source(fp, directives_only=False): yield curr_line total_sloc += curr_line.physical_reset() - if not cleaner.state == ["TOPLEVEL"]: + + # Even if code is technically wrong, we should only fail when necessary. + parsing_failed = not cleaner.state == ["TOPLEVEL"] + if continued: + log.warning("backslash-newline at end of file") + parsing_failed = False + + if parsing_failed: raise RuntimeError( "Parsing failed. Please open a bug report at: " "https://github.com/intel/code-base-investigator/issues/new?template=bug_report.yml.", # noqa: E501 @@ -590,7 +597,9 @@ def fortran_file_source(fp): yield curr_line total_sloc += curr_line.physical_reset() - if not cleaner.state == ["TOPLEVEL"]: + + parsing_failed = not cleaner.state == ["TOPLEVEL"] + if parsing_failed: raise RuntimeError( "Parsing failed. Please open a bug report at: " "https://github.com/intel/code-base-investigator/issues/new?template=bug_report.yml.", # noqa: E501 From 8d541523c50cf0812922b4cc455e3a49f16751c6 Mon Sep 17 00:00:00 2001 From: John Pennycook Date: Wed, 23 Apr 2025 16:49:35 +0100 Subject: [PATCH 5/5] Fix link to GitHub issues page Ending the warning with a "." might result in a 404, if the terminal decides to treat that as part of the URL. Signed-off-by: John Pennycook --- codebasin/file_source.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codebasin/file_source.py b/codebasin/file_source.py index b0b5c7d..dc6a189 100644 --- a/codebasin/file_source.py +++ b/codebasin/file_source.py @@ -521,7 +521,7 @@ def c_file_source(fp, directives_only=False): if parsing_failed: raise RuntimeError( "Parsing failed. Please open a bug report at: " - "https://github.com/intel/code-base-investigator/issues/new?template=bug_report.yml.", # noqa: E501 + "https://github.com/intel/code-base-investigator/issues/new?template=bug_report.yml", # noqa: E501 ) return (total_sloc, total_physical_lines) @@ -602,7 +602,7 @@ def fortran_file_source(fp): if parsing_failed: raise RuntimeError( "Parsing failed. Please open a bug report at: " - "https://github.com/intel/code-base-investigator/issues/new?template=bug_report.yml.", # noqa: E501 + "https://github.com/intel/code-base-investigator/issues/new?template=bug_report.yml", # noqa: E501 ) return (total_sloc, total_physical_lines)