Merge pull request #2161 from strictdoc-project/stanislaw/multiline_markers

backend/sdoc_source_code: parse multiline @relation markers

Author: stanislaw

strictdoc/backend/sdoc_source_code/helpers/comment_preprocessor.py

@@ -0,0 +1,21 @@
+import re
+from typing import Match
+
+WS = "[ \t]"
+
+
+def preprocess_source_code_comment(comment: str) -> str:
+    """
+    Remove all Doxygen/Python/etc comment markers for processing.
+    """
+
+    def replace_with_spaces(match: Match[str]) -> str:
+        # Return a string of spaces with the same length as the matched text.
+        return " " * len(match.group(0))
+
+    return re.sub(
+        rf"(^/\*\*)|^{WS}*\*/?|(^///)|(^//)|(^#+)",
+        replace_with_spaces,
+        comment,
+        flags=re.MULTILINE,
+    )

strictdoc/backend/sdoc_source_code/marker_parser.py

@@ -1,6 +1,9 @@
 import re
 from typing import List, Optional, Union
 
+from strictdoc.backend.sdoc_source_code.helpers.comment_preprocessor import (
+    preprocess_source_code_comment,
+)
 from strictdoc.backend.sdoc_source_code.models.function_range_marker import (
     FunctionRangeMarker,
 )
@@ -12,9 +15,10 @@
 
 REGEX_REQ = r"[A-Za-z][A-Za-z0-9_\/\.\\-]+"
 REGEX_ROLE = r"[A-Za-z][A-Za-z0-9\\-]+"
+
 # @relation(REQ-1, scope=function) or @relation{REQ-1, scope=function}
 REGEX_MARKER = re.compile(
-    rf"@relation[({{]({REGEX_REQ}(?:, {REGEX_REQ})*), scope=(file|class|function|line|range_start|range_end)(?:, role=({REGEX_ROLE}))?[)}}]"
+    rf"@relation[({{](\s*{REGEX_REQ}(?:,\s+{REGEX_REQ})*),\s+scope=(file|class|function|line|range_start|range_end)(?:,\s+role=({REGEX_ROLE}))?[\s*)}}]"
 )
 
 
@@ -25,37 +29,77 @@ def parse(
         line_start: int,
         line_end: int,
         comment_line_start: int,
-        comment_column_start: int,
         entity_name: Optional[str] = None,
     ) -> List[Union[FunctionRangeMarker, RangeMarker, LineMarker]]:
+        """
+        Parse relation markers from source file comments.
+
+        Before the actual parsing, the function removes all code comment symbols
+        such as /** ... */ or /// Doxygen comments or Python
+
+        The line start/end indicate a range where the comment is located in the
+        source file.
+        The comment_line_start parameter maybe the same as line start but can
+        also be different if the comment is part of a Python or C function in
+        which case the comment_line_start will be several lines below the actual
+        start of the range.
+        """
+
         markers: List[Union[FunctionRangeMarker, RangeMarker, LineMarker]] = []
-        for input_line_idx_, input_line_ in enumerate(
-            input_string.splitlines()
-        ):
-            match = REGEX_MARKER.search(input_line_)
-            if match is None:
-                continue
 
-            assert match.lastindex is not None
-            marker_type = match.group(2)
-            marker_role = match.group(3) if len(match.groups()) >= 3 else None
-            req_list = match.group(1)
+        input_string = preprocess_source_code_comment(input_string)
+
+        matches = REGEX_MARKER.finditer(input_string)
+        for match_ in matches:
+            assert match_.lastindex is not None
+            marker_type = match_.group(2)
+            marker_role = match_.group(3) if len(match_.groups()) >= 3 else None
+            req_list = match_.group(1)
+
+            marker_start_index = match_.start(0)
+
+            marker_start_line = comment_line_start + input_string.count(
+                "\n", 0, marker_start_index
+            )
 
-            first_requirement_index = match.start(1)
+            marker_line_start_index = input_string.rfind(
+                "\n", 0, marker_start_index
+            )
+            marker_line_start_index = (
+                0
+                if marker_line_start_index == -1
+                else marker_line_start_index + 1
+            )
+
+            marker_start_column = (
+                marker_start_index - marker_line_start_index
+            ) + 1
+
+            all_reqs_start_index = match_.start(1)
 
-            current_line = comment_line_start + input_line_idx_
-            first_requirement_column = first_requirement_index + 1
-            if input_line_idx_ == 0:
-                first_requirement_column += comment_column_start - 1
             requirements = []
             for req_match in re.finditer(REGEX_REQ, req_list):
-                req_item = req_match.group(0)  # Matched REQ-XXX item
-                # Calculate actual position relative to the original string
-                start_index = (
-                    req_match.start()
-                )  # Offset by where group 1 starts
+                req_start_index = all_reqs_start_index + req_match.start()
+                last_newline_pos = input_string.rfind("\n", 0, req_start_index)
+
+                line_start_index = (
+                    0 if last_newline_pos == -1 else last_newline_pos + 1
+                )
+
+                req_abs_line = comment_line_start + input_string.count(
+                    "\n", 0, req_start_index
+                )
+
+                first_requirement_index = match_.start(1)
+                first_requirement_column = (
+                    first_requirement_index - line_start_index
+                ) + 1
+
+                req_item = req_match.group(0)
+
+                start_index = req_match.start()
                 requirement = Req(None, req_item)
-                requirement.ng_source_line = current_line
+                requirement.ng_source_line = req_abs_line
                 requirement.ng_source_column = (
                     first_requirement_column + start_index
                 )
@@ -65,11 +109,10 @@ def parse(
                 function_marker = FunctionRangeMarker(
                     None, requirements, scope=marker_type, role=marker_role
                 )
-                function_marker.ng_source_line_begin = line_start
+                function_marker.ng_source_line_begin = marker_start_line
+                function_marker.ng_source_column_begin = marker_start_column
                 function_marker.ng_range_line_begin = line_start
                 function_marker.ng_range_line_end = line_end
-                function_marker.ng_marker_line = current_line
-                function_marker.ng_marker_column = first_requirement_column
                 if marker_type == "file":
                     function_marker.set_description("entire file")
                 elif marker_type == "function":
@@ -85,16 +128,16 @@ def parse(
                     requirements,
                     role=marker_role,
                 )
-                range_marker.ng_source_line_begin = line_start
-                range_marker.ng_source_column_begin = first_requirement_column
+                range_marker.ng_source_line_begin = marker_start_line
+                range_marker.ng_source_column_begin = marker_start_column
                 range_marker.ng_range_line_begin = line_start
                 range_marker.ng_range_line_end = line_end
                 range_marker.ng_new_relation_keyword = True
                 markers.append(range_marker)
             elif marker_type == "line":
                 line_marker = LineMarker(None, requirements, role=marker_role)
-                line_marker.ng_source_line_begin = line_start
-                line_marker.ng_source_column_begin = first_requirement_column
+                line_marker.ng_source_line_begin = marker_start_line
+                line_marker.ng_source_column_begin = marker_start_column
                 line_marker.ng_range_line_begin = line_start
                 line_marker.ng_range_line_end = line_end
                 markers.append(line_marker)

strictdoc/backend/sdoc_source_code/reader.py

@@ -205,10 +205,6 @@ def line_marker_processor(line_marker: LineMarker, parse_context: ParseContext):
 def function_range_marker_processor(
     function_range_marker: FunctionRangeMarker, parse_context: ParseContext
 ):
-    location = get_location(function_range_marker)
-    line = location["line"]
-    column = location["col"]
-
     if (
         len(parse_context.marker_stack) > 0
         and parse_context.marker_stack[-1].ng_is_nodoc
@@ -222,8 +218,6 @@ def function_range_marker_processor(
     function_range_marker.ng_range_line_end = (
         parse_context.file_stats.lines_total
     )
-    function_range_marker.ng_marker_line = line
-    function_range_marker.ng_marker_column = column
 
     for req in function_range_marker.reqs:
         markers = parse_context.map_reqs_to_markers.setdefault(req, [])

strictdoc/backend/sdoc_source_code/reader_c.py

@@ -80,7 +80,6 @@ def read(
                             if input_buffer[-1] == 10
                             else node_.end_point[0] + 1,
                             node_.start_point[0] + 1,
-                            node_.start_point[1] + 1,
                         )
                         for marker_ in markers:
                             if not isinstance(marker_, FunctionRangeMarker):
@@ -179,7 +178,6 @@ def read(
                         function_comment_node.start_point[0] + 1,
                         function_last_line,
                         function_comment_node.start_point[0] + 1,
-                        function_comment_node.start_point[1] + 1,
                         entity_name=function_display_name,
                     )
                     for marker_ in markers:
@@ -278,7 +276,6 @@ def read(
                         function_comment_node.start_point[0] + 1,
                         function_last_line,
                         function_comment_node.start_point[0] + 1,
-                        function_comment_node.start_point[1] + 1,
                         entity_name=function_display_name,
                     )
                     for marker_ in markers:
@@ -327,7 +324,6 @@ def read(
                     node_.start_point[0] + 1,
                     node_.end_point[0] + 1,
                     node_.start_point[0] + 1,
-                    node_.start_point[1] + 1,
                 )
                 for marker_ in markers:
                     if isinstance(marker_, RangeMarker) and (

strictdoc/backend/sdoc_source_code/reader_python.py

@@ -106,7 +106,6 @@ def read(
                             if input_buffer[-1] == 10
                             else node_.end_point[0] + 1,
                             string_content.start_point[0] + 1,
-                            string_content.start_point[1] + 1,
                         )
                         for marker_ in markers:
                             if isinstance(marker_, FunctionRangeMarker) and (
@@ -165,7 +164,6 @@ def read(
                                 node_.start_point[0] + 1,
                                 node_.end_point[0] + 1,
                                 string_content.start_point[0] + 1,
-                                string_content.start_point[1] + 1,
                                 function_name,
                             )
                             for marker_ in markers:
@@ -230,7 +228,6 @@ def read(
                     node_.start_point[0] + 1,
                     node_.end_point[0] + 1,
                     node_.start_point[0] + 1,
-                    node_.start_point[1] + 1,
                     None,
                 )
                 for marker_ in markers:

strictdoc/core/file_traceability_index.py

@@ -711,11 +711,8 @@ def forward_function_marker_from_function(
             parent=None, reqs_objs=reqs, scope=marker_type.value
         )
         function_marker.ng_source_line_begin = function.line_begin
-        function_marker.ng_source_column_begin = 1
         function_marker.ng_range_line_begin = function.line_begin
         function_marker.ng_range_line_end = function.line_end
-        function_marker.ng_marker_line = function.line_begin
-        function_marker.ng_marker_column = 1
         function_marker.role = role
         if description is not None:
             function_marker.set_description(description)

tests/unit/strictdoc/backend/sdoc_source_code/helpers/test_comment_preprocessor.py

@@ -0,0 +1,85 @@
+# The tests in this file has strings with whitespace that must not be linted.
+# ruff: noqa: W291,W293
+from strictdoc.backend.sdoc_source_code.helpers.comment_preprocessor import (
+    preprocess_source_code_comment,
+)
+
+
+def test_001_doxygen_slashes_and_stars():
+    source_input = """\
+/**
+ * @relation(REQ-1, scope=function)
+ */
+"""
+
+    preprocessed_comment = preprocess_source_code_comment(source_input)
+
+    # Note: There are invisible characters.
+    assert (
+        preprocessed_comment
+        == """\
+   
+   @relation(REQ-1, scope=function)
+   
+"""
+    )
+
+
+def test_001_doxygen_three_slashes():
+    source_input = """\
+///
+/// @relation(REQ-1, scope=function)
+///
+"""
+
+    preprocessed_comment = preprocess_source_code_comment(source_input)
+
+    # Note: There are invisible characters.
+    assert (
+        preprocessed_comment
+        == """\
+   
+    @relation(REQ-1, scope=function)
+   
+"""
+    )
+
+
+def test_003_doxygen_two_slashes():
+    source_input = """\
+//
+// @relation(REQ-1, scope=function)
+//
+"""
+
+    preprocessed_comment = preprocess_source_code_comment(source_input)
+
+    # Note: There are invisible characters.
+    assert (
+        preprocessed_comment
+        == """\
+  
+   @relation(REQ-1, scope=function)
+  
+"""
+    )
+
+
+def test_004_python():
+    source_input = """\
+#
+# @relation(REQ-1, scope=function)
+#
+"""
+
+    preprocessed_comment = preprocess_source_code_comment(source_input)
+
+    # Note: There are invisible characters.
+    assert (
+        preprocessed_comment
+        == """\
+ 
+  @relation(REQ-1, scope=function)
+ 
+"""
+    )