Skip to content

Commit 90434d8

Browse files
authored
Merge pull request #2247 from haxtibal/tdmg/robot_src_reader
Robot Framework part 1: Source Reader
2 parents bd0a429 + 0647af5 commit 90434d8

File tree

27 files changed

+513
-4
lines changed

27 files changed

+513
-4
lines changed

docs/strictdoc_01_user_guide.sdoc

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2411,7 +2411,7 @@ MID: 07bfe045a4934dd29d97fdb8ac567b7c
24112411
STATEMENT: >>>
24122412
For parsing source code and calculating traceability to requirements, StrictDoc uses a general parser that is agnostic of specific programming languages and their constructs, such as classes or functions. However, for languages with these constructs, establishing traceability to them can simplify the tracing process.
24132413

2414-
As an experimental option, StrictDoc supports parsing source files of selected programming languages (currently Python and C) to recognize language syntax, primarily enabling traceability of functions (in Python, C, and others) and classes (in Python, C++, and others) to requirements.
2414+
As an experimental option, StrictDoc supports parsing source files of selected programming languages to recognize language syntax, primarily enabling traceability of functions (in Python, C, and others), classes (in Python, C++, and others) and tests (Python, C++, Robot Framework) to requirements.
24152415

24162416
To activate language-aware traceability, configure the project with the following features:
24172417

@@ -2424,7 +2424,7 @@ To activate language-aware traceability, configure the project with the followin
24242424
"SOURCE_FILE_LANGUAGE_PARSERS"
24252425
]
24262426

2427-
Currently, only Python and C/C++ parsers are implemented. Upcoming implementations include parsers for Rust, Bash, and more.
2427+
Currently, Python, C/C++ and Robot Framework parsers are implemented. Upcoming implementations include parsers for Rust, Bash, and more.
24282428
<<<
24292429

24302430
[[/SECTION]]

pyproject.toml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,9 @@ dependencies = [
8989

9090
# HTML2PDF dependencies
9191
"html2pdf4doc >= 0.0.18",
92+
93+
# Robot Framework dependencies
94+
"robotframework >= 4.0.0",
9295
]
9396
# @sdoc[/SDOC-SRS-89]
9497

strictdoc/backend/sdoc_source_code/caching_reader.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,9 @@
1313
from strictdoc.backend.sdoc_source_code.reader_python import (
1414
SourceFileTraceabilityReader_Python,
1515
)
16+
from strictdoc.backend.sdoc_source_code.reader_robot import (
17+
SourceFileTraceabilityReader_Robot,
18+
)
1619
from strictdoc.core.project_config import ProjectConfig
1720

1821

@@ -59,6 +62,7 @@ def _get_reader(
5962
SourceFileTraceabilityReader,
6063
SourceFileTraceabilityReader_Python,
6164
SourceFileTraceabilityReader_C,
65+
SourceFileTraceabilityReader_Robot,
6266
]:
6367
if project_config.is_activated_source_file_language_parsers():
6468
if path_to_file.endswith(".py"):
@@ -72,4 +76,6 @@ def _get_reader(
7276
or path_to_file.endswith(".cpp")
7377
):
7478
return SourceFileTraceabilityReader_C()
79+
if path_to_file.endswith(".robot"):
80+
return SourceFileTraceabilityReader_Robot()
7581
return SourceFileTraceabilityReader()

strictdoc/backend/sdoc_source_code/marker_parser.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ def parse(
3030
line_end: int,
3131
comment_line_start: int,
3232
entity_name: Optional[str] = None,
33+
col_offset: int = 0,
3334
) -> List[Union[FunctionRangeMarker, RangeMarker, LineMarker]]:
3435
"""
3536
Parse relation markers from source file comments.
@@ -72,8 +73,8 @@ def parse(
7273
)
7374

7475
marker_start_column = (
75-
marker_start_index - marker_line_start_index
76-
) + 1
76+
(marker_start_index - marker_line_start_index) + col_offset + 1
77+
)
7778

7879
all_reqs_start_index = match_.start(1)
7980

Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,198 @@
1+
import sys
2+
from typing import List, Union
3+
4+
from robot.api.parsing import (
5+
Comment,
6+
Documentation,
7+
EmptyLine,
8+
ModelVisitor,
9+
Tags,
10+
TestCase,
11+
Token,
12+
get_model,
13+
)
14+
15+
from strictdoc.backend.sdoc.error_handling import StrictDocSemanticError
16+
from strictdoc.backend.sdoc_source_code.constants import FunctionAttribute
17+
from strictdoc.backend.sdoc_source_code.marker_parser import MarkerParser
18+
from strictdoc.backend.sdoc_source_code.models.function import Function
19+
from strictdoc.backend.sdoc_source_code.models.function_range_marker import (
20+
FunctionRangeMarker,
21+
RangeMarkerType,
22+
)
23+
from strictdoc.backend.sdoc_source_code.models.range_marker import (
24+
LineMarker,
25+
RangeMarker,
26+
)
27+
from strictdoc.backend.sdoc_source_code.models.source_file_info import (
28+
SourceFileTraceabilityInfo,
29+
)
30+
from strictdoc.backend.sdoc_source_code.parse_context import ParseContext
31+
from strictdoc.backend.sdoc_source_code.processors.general_language_marker_processors import (
32+
function_range_marker_processor,
33+
line_marker_processor,
34+
range_marker_processor,
35+
source_file_traceability_info_processor,
36+
)
37+
from strictdoc.helpers.file_stats import SourceFileStats
38+
39+
40+
class SdocRelationVisitor(ModelVisitor):
41+
"""
42+
Create functions from test cases in *.robot files and create markers.
43+
44+
Note: ModelVisitor reuses ast.NodeVisitor from Python. We rely on following
45+
behavior.
46+
- It traverses depth first. This order is important to get matching marker
47+
pairs on the ParsersContext.marker_stack.
48+
- It doesn't recurse into subtrees if a custom visit_* method is defined.
49+
This is important to avoid duplicated matches.
50+
"""
51+
52+
def __init__(
53+
self,
54+
traceability_info: SourceFileTraceabilityInfo,
55+
parse_context: ParseContext,
56+
):
57+
super().__init__()
58+
self.traceability_info = traceability_info
59+
self.parse_context = parse_context
60+
61+
def visit_Comment(self, node: Comment) -> None:
62+
"""Create non-function Marker from Comment outside TestCases."""
63+
self._visit_possibly_marked_node(node)
64+
65+
def visit_Documentation(self, node: Documentation) -> None:
66+
"""Create non-function Marker from Documentation outside TestCases."""
67+
self._visit_possibly_marked_node(node)
68+
69+
def visit_Tags(self, node: Tags) -> None:
70+
"""Create non-function Marker from Tags outside TestCases."""
71+
self._visit_possibly_marked_node(node)
72+
73+
def visit_TestCase(self, node: TestCase) -> None:
74+
"""Create function and non-function Marker from TestCases."""
75+
trailing_empty_lines = 0
76+
tc_markers = []
77+
for stmt in node.body:
78+
if isinstance(stmt, EmptyLine):
79+
# trim trailing newlines from test case range
80+
trailing_empty_lines += 1
81+
else:
82+
trailing_empty_lines = 0
83+
84+
if isinstance(stmt, (Comment, Tags, Documentation)):
85+
for token in filter(self._token_filter, stmt.tokens):
86+
markers: List[
87+
Union[FunctionRangeMarker, RangeMarker, LineMarker]
88+
] = MarkerParser.parse(
89+
token.value,
90+
token.lineno,
91+
token.lineno,
92+
token.lineno,
93+
node.name,
94+
token.col_offset,
95+
)
96+
tc_markers.extend(markers)
97+
98+
function_markers = []
99+
for marker_ in tc_markers:
100+
if isinstance(marker_, FunctionRangeMarker):
101+
marker_.ng_range_line_begin = node.lineno
102+
marker_.ng_range_line_end = (
103+
node.end_lineno - trailing_empty_lines
104+
)
105+
function_markers.append(marker_)
106+
function_range_marker_processor(marker_, self.parse_context)
107+
elif isinstance(marker_, RangeMarker):
108+
range_marker_processor(marker_, self.parse_context)
109+
elif isinstance(marker_, LineMarker):
110+
line_marker_processor(marker_, self.parse_context)
111+
112+
self.traceability_info.markers.extend(function_markers)
113+
test_case = Function(
114+
parent=self.traceability_info,
115+
name=node.name,
116+
display_name=node.name,
117+
line_begin=node.lineno,
118+
line_end=node.end_lineno - trailing_empty_lines,
119+
child_functions=[],
120+
markers=function_markers,
121+
attributes={FunctionAttribute.DEFINITION},
122+
)
123+
self.traceability_info.functions.append(test_case)
124+
125+
def _visit_possibly_marked_node(
126+
self, node: Union[Comment, Documentation, Tags]
127+
) -> None:
128+
for token in filter(self._token_filter, node.tokens):
129+
markers: List[
130+
Union[FunctionRangeMarker, RangeMarker, LineMarker]
131+
] = MarkerParser.parse(
132+
token.value,
133+
node.lineno,
134+
node.lineno,
135+
node.lineno,
136+
None,
137+
token.col_offset,
138+
)
139+
for marker_ in markers:
140+
if (
141+
isinstance(marker_, FunctionRangeMarker)
142+
and marker_.scope is RangeMarkerType.FILE
143+
):
144+
# Outside Test Cases only accept scope=file function markers
145+
marker_.ng_range_line_begin = 1
146+
marker_.ng_range_line_end = (
147+
self.parse_context.file_stats.lines_total
148+
)
149+
function_range_marker_processor(marker_, self.parse_context)
150+
self.traceability_info.markers.append(marker_)
151+
elif isinstance(marker_, RangeMarker):
152+
range_marker_processor(marker_, self.parse_context)
153+
elif isinstance(marker_, LineMarker):
154+
line_marker_processor(marker_, self.parse_context)
155+
156+
@staticmethod
157+
def _token_filter(token: Token) -> bool:
158+
if token.type in ("SEPARATOR", "EOL"):
159+
return False
160+
return True
161+
162+
163+
class SourceFileTraceabilityReader_Robot:
164+
def read(
165+
self, input_buffer: str, file_path: str
166+
) -> SourceFileTraceabilityInfo:
167+
traceability_info = SourceFileTraceabilityInfo([])
168+
if len(input_buffer) == 0:
169+
return traceability_info
170+
file_stats = SourceFileStats.create(input_buffer)
171+
parse_context = ParseContext(file_path, file_stats)
172+
robotfw_model = get_model(input_buffer, data_only=False)
173+
visitor = SdocRelationVisitor(traceability_info, parse_context)
174+
visitor.visit(robotfw_model)
175+
source_file_traceability_info_processor(
176+
traceability_info, parse_context
177+
)
178+
return traceability_info
179+
180+
def read_from_file(self, file_path: str) -> SourceFileTraceabilityInfo:
181+
try:
182+
with open(file_path) as file:
183+
sdoc_content = file.read()
184+
sdoc = self.read(sdoc_content, file_path=file_path)
185+
return sdoc
186+
except UnicodeDecodeError:
187+
raise
188+
except StrictDocSemanticError as exc:
189+
print(exc.to_print_message()) # noqa: T201
190+
sys.exit(1)
191+
except Exception as exc: # pylint: disable=broad-except
192+
print( # noqa: T201
193+
f"error: SourceFileTraceabilityReader_Robot: could not parse file: "
194+
f"{file_path}.\n{exc.__class__.__name__}: {exc}"
195+
)
196+
# TODO: when --debug is provided
197+
# traceback.print_exc() # noqa: ERA001
198+
sys.exit(1)

tests/integration/features/file_traceability/20_skip_invalid_utf8/test.itest

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,13 @@ REQUIRES: PYTHON_39_OR_HIGHER
33
# invalid_file.* was created with echo "00 08 00 00" | xxd -r -p > file.bin
44
RUN: %strictdoc export %S --output-dir Output | filecheck %s --dump-input=fail
55
CHECK: warning: Skip tracing binary file {{.*}}invalid_file.bin
6+
CHECK: Reading source: {{.*}}invalid_file.bin
67
CHECK: warning: Skip tracing binary file {{.*}}invalid_file.c
8+
CHECK: Reading source: {{.*}}invalid_file.c
79
CHECK: warning: Skip tracing binary file {{.*}}invalid_file.py
10+
CHECK: Reading source: {{.*}}invalid_file.py
11+
# skip warning check since robotframework doesn't raises unicode error on windows
12+
CHECK: Reading source: {{.*}}invalid_file.robot
813
CHECK: Published: Hello world doc
914

1015
RUN: %check_exists --file "%S/Output/html/_source_files/file.py.html"
Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
*** Settings ***
2+
Documentation Example using the space separated format.
3+
4+
*** Variables ***
5+
# @relation(REQ-1, scope=range_start)
6+
${MESSAGE} Hello, world!
7+
# @relation(REQ-1, scope=range_end)
8+
9+
*** Test Cases ***
10+
My Test
11+
[Documentation] Test following requirements
12+
# @relation(REQ-1, scope=range_start)
13+
Log ${MESSAGE}
14+
My Keyword ${MESSAGE}
15+
# @relation(REQ-1, scope=range_end)
16+
My Other Keyword
Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
[DOCUMENT]
2+
TITLE: Hello world doc
3+
4+
[REQUIREMENT]
5+
UID: REQ-1
6+
TITLE: Requirement Title
7+
STATEMENT: Requirement Statement
Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
[project]
2+
3+
features = [
4+
"REQUIREMENT_TO_SOURCE_TRACEABILITY",
5+
"SOURCE_FILE_LANGUAGE_PARSERS",
6+
"PROJECT_STATISTICS_SCREEN"
7+
]

0 commit comments

Comments
 (0)