From b316b360f7045aef5fa90e6e873edca93f34497b Mon Sep 17 00:00:00 2001 From: Priya Shah Date: Fri, 17 Oct 2025 18:04:30 -0400 Subject: [PATCH 1/5] Fix for junit failure while using with --early-exit when running multiple features FIX #487 --- radish/extensions/junit_xml_writer.py | 53 ++++++++++++++------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/radish/extensions/junit_xml_writer.py b/radish/extensions/junit_xml_writer.py index 75888b16..344ae1da 100644 --- a/radish/extensions/junit_xml_writer.py +++ b/radish/extensions/junit_xml_writer.py @@ -96,32 +96,33 @@ def generate_junit_xml(self, features, marker): for feature in features: if not feature.has_to_run(world.config.scenarios): continue - - testsuite_states = {"failures": 0, "errors": 0, "skipped": 0, "tests": 0} - - for scenario in (s for s in feature.all_scenarios if not isinstance(s, (ScenarioOutline, ScenarioLoop))): - if not scenario.has_to_run(world.config.scenarios): - continue - - testsuite_states["tests"] += 1 - if scenario.state in [ - Step.State.UNTESTED, - Step.State.PENDING, - Step.State.SKIPPED, - ]: - testsuite_states["skipped"] += 1 - if scenario.state is Step.State.FAILED: - testsuite_states["failures"] += 1 - - testsuite_element = etree.Element( - "testsuite", - name=feature.sentence, - failures=str(testsuite_states["failures"]), - errors=str(testsuite_states["errors"]), - skipped=str(testsuite_states["skipped"]), - tests=str(testsuite_states["tests"]), - time="%.3f" % feature.duration.total_seconds(), - ) + + if feature.state in [Step.State.PASSED, Step.State.FAILED]: + testsuite_states = {"failures": 0, "errors": 0, "skipped": 0, "tests": 0} + + for scenario in (s for s in feature.all_scenarios if not isinstance(s, (ScenarioOutline, ScenarioLoop))): + if not scenario.has_to_run(world.config.scenarios): + continue + + testsuite_states["tests"] += 1 + if scenario.state in [ + Step.State.UNTESTED, + Step.State.PENDING, + Step.State.SKIPPED, + ]: + testsuite_states["skipped"] += 1 + if scenario.state is Step.State.FAILED: + testsuite_states["failures"] += 1 + + testsuite_element = etree.Element( + "testsuite", + name=feature.sentence, + failures=str(testsuite_states["failures"]), + errors=str(testsuite_states["errors"]), + skipped=str(testsuite_states["skipped"]), + tests=str(testsuite_states["tests"]), + time="%.3f" % feature.duration.total_seconds(), + ) for scenario in (s for s in feature.all_scenarios if not isinstance(s, (ScenarioOutline, ScenarioLoop))): if not scenario.has_to_run(world.config.scenarios): From 150699c0362c7aa39b553806f5b385323e0584cb Mon Sep 17 00:00:00 2001 From: Priya Shah Date: Fri, 17 Oct 2025 22:11:30 +0000 Subject: [PATCH 2/5] Fixd formatting/linting issue --- radish/extensions/junit_xml_writer.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/radish/extensions/junit_xml_writer.py b/radish/extensions/junit_xml_writer.py index 344ae1da..8d829a1d 100644 --- a/radish/extensions/junit_xml_writer.py +++ b/radish/extensions/junit_xml_writer.py @@ -96,11 +96,13 @@ def generate_junit_xml(self, features, marker): for feature in features: if not feature.has_to_run(world.config.scenarios): continue - + if feature.state in [Step.State.PASSED, Step.State.FAILED]: testsuite_states = {"failures": 0, "errors": 0, "skipped": 0, "tests": 0} - for scenario in (s for s in feature.all_scenarios if not isinstance(s, (ScenarioOutline, ScenarioLoop))): + for scenario in ( + s for s in feature.all_scenarios if not isinstance(s, (ScenarioOutline, ScenarioLoop)) + ): if not scenario.has_to_run(world.config.scenarios): continue From 8081b7e7123a2b7d2a756e1938f41b1aab9254a4 Mon Sep 17 00:00:00 2001 From: Alexandre hassan Date: Fri, 31 Oct 2025 18:45:04 +0000 Subject: [PATCH 3/5] add unit tests for early exit junit, and fix testcase_element missing --- .gitignore | 2 + radish/extensions/junit_xml_writer.py | 77 +++++++++---------- .../unit/extensions/test_junit_xml_writer.py | 35 ++++++++- 3 files changed, 72 insertions(+), 42 deletions(-) diff --git a/.gitignore b/.gitignore index d0343efc..26acc08f 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,5 @@ README.rst # IDE configuration .idea/ +.venv/ +.vscode/ \ No newline at end of file diff --git a/radish/extensions/junit_xml_writer.py b/radish/extensions/junit_xml_writer.py index 8d829a1d..50d34cfa 100644 --- a/radish/extensions/junit_xml_writer.py +++ b/radish/extensions/junit_xml_writer.py @@ -97,50 +97,50 @@ def generate_junit_xml(self, features, marker): if not feature.has_to_run(world.config.scenarios): continue - if feature.state in [Step.State.PASSED, Step.State.FAILED]: - testsuite_states = {"failures": 0, "errors": 0, "skipped": 0, "tests": 0} - - for scenario in ( - s for s in feature.all_scenarios if not isinstance(s, (ScenarioOutline, ScenarioLoop)) - ): - if not scenario.has_to_run(world.config.scenarios): - continue - - testsuite_states["tests"] += 1 - if scenario.state in [ - Step.State.UNTESTED, - Step.State.PENDING, - Step.State.SKIPPED, - ]: - testsuite_states["skipped"] += 1 - if scenario.state is Step.State.FAILED: - testsuite_states["failures"] += 1 - - testsuite_element = etree.Element( - "testsuite", - name=feature.sentence, - failures=str(testsuite_states["failures"]), - errors=str(testsuite_states["errors"]), - skipped=str(testsuite_states["skipped"]), - tests=str(testsuite_states["tests"]), - time="%.3f" % feature.duration.total_seconds(), - ) + testsuite_states = {"failures": 0, "errors": 0, "skipped": 0, "tests": 0} for scenario in (s for s in feature.all_scenarios if not isinstance(s, (ScenarioOutline, ScenarioLoop))): if not scenario.has_to_run(world.config.scenarios): continue - if scenario.state not in [ + testsuite_states["tests"] += 1 + if scenario.state in [ Step.State.UNTESTED, Step.State.PENDING, Step.State.SKIPPED, ]: - testcase_element = etree.Element( - "testcase", - classname=feature.sentence, - name=scenario.sentence, - time="%.3f" % scenario.duration.total_seconds(), - ) + testsuite_states["skipped"] += 1 + if scenario.state is Step.State.FAILED: + testsuite_states["failures"] += 1 + + if feature.starttime and feature.endtime: + feature_duration = feature.duration.total_seconds() + else: + feature_duration = 0 + testsuite_element = etree.Element( + "testsuite", + name=feature.sentence, + failures=str(testsuite_states["failures"]), + errors=str(testsuite_states["errors"]), + skipped=str(testsuite_states["skipped"]), + tests=str(testsuite_states["tests"]), + time=f"{feature_duration:.3f}", + ) + + for scenario in (s for s in feature.all_scenarios if not isinstance(s, (ScenarioOutline, ScenarioLoop))): + if not scenario.has_to_run(world.config.scenarios): + continue + + if scenario.starttime and scenario.endtime: + testcase_duration = scenario.duration.total_seconds() + else: + testcase_duration = 0 + testcase_element = etree.Element( + "testcase", + classname=feature.sentence, + name=scenario.sentence, + time=f"{testcase_duration:.3f}", + ) if world.config.junit_relaxed: properties_element = etree.Element("properties") @@ -182,10 +182,5 @@ def generate_junit_xml(self, features, marker): testsuites_element.append(testsuite_element) - content = etree.tostring( - testsuites_element, - pretty_print=True, - xml_declaration=True, - encoding="utf-8", - ) + content = etree.tostring(testsuites_element, pretty_print=True, xml_declaration=True, encoding="utf-8") self._write_xml_to_disk(content) diff --git a/tests/unit/extensions/test_junit_xml_writer.py b/tests/unit/extensions/test_junit_xml_writer.py index 473790d5..0765ebce 100644 --- a/tests/unit/extensions/test_junit_xml_writer.py +++ b/tests/unit/extensions/test_junit_xml_writer.py @@ -7,6 +7,7 @@ Copyright: MIT, Timo Furrer """ +import re from datetime import datetime, timezone import pytest @@ -16,6 +17,7 @@ from radish.feature import Feature from radish.model import Tag from radish.scenario import Scenario +from radish.stepmodel import Step from radish.terrain import world @@ -27,7 +29,7 @@ def test_empty_feature_list(): writer.generate_junit_xml(no_features, "marker-is-ignored") -def test_singel_feature_list(mocker): +def test_single_feature_list(mocker): stub = mocker.patch("radish.extensions.junit_xml_writer.JUnitXMLWriter._write_xml_to_disk") first_feature = Feature(1, "Feature", "I am a feature", "foo.feature", 1, tags=None) @@ -108,3 +110,34 @@ def test_relaxed_mode_adding_tags_to_junit(mocker): assert "author" in str(stub.call_args[0]) assert "batman" in str(stub.call_args[0]) + + +def test_early_exit_feature_list(mocker): + stub = mocker.patch("radish.extensions.junit_xml_writer.JUnitXMLWriter._write_xml_to_disk") + + first_feature = Feature(1, "Feature", "I am a feature", "foo.feature", 1, tags=None) + first_feature.starttime = datetime.now(timezone.utc) + first_feature.endtime = datetime.now(timezone.utc) + second_feature = Feature(2, "Feature", "Did not run", "foo.feature", 1, tags=None) + # second_feature.state = Step.State.UNTESTED + scenario = Scenario( + 1, "Scenario", "Did not run", "foo.feature", 1, parent=None, tags=None, preconditions=None, background=None + ) + scenario.steps = [Step(1, "Foo", "foo.feature", 2, None, False)] + second_feature.scenarios = [scenario] + assert second_feature.state not in [Step.State.PASSED, Step.State.FAILED] + + features = [first_feature, second_feature] + + writer = JUnitXMLWriter() + writer.generate_junit_xml(features, "marker-is-ignored") + + result = str(stub.call_args[0]) + feature_regex = re.compile(r"]*name=\"([^\"]+)\"([^>]*)>") + matches = feature_regex.findall(result) + assert len(matches) == 2 + f1_match = next(m for m in matches if m[0] == "I am a feature") + f2_match = next(m for m in matches if m[0] == "Did not run") + assert 'tests="0"' in f1_match[1] # f1 contains no scenarios + assert 'skipped="1"' in f2_match[1] # f2 contains one untested scenario (it was skipped) + assert " Date: Fri, 31 Oct 2025 19:16:38 +0000 Subject: [PATCH 4/5] remove commented out code --- tests/unit/extensions/test_junit_xml_writer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/extensions/test_junit_xml_writer.py b/tests/unit/extensions/test_junit_xml_writer.py index 0765ebce..f4a32ade 100644 --- a/tests/unit/extensions/test_junit_xml_writer.py +++ b/tests/unit/extensions/test_junit_xml_writer.py @@ -119,7 +119,6 @@ def test_early_exit_feature_list(mocker): first_feature.starttime = datetime.now(timezone.utc) first_feature.endtime = datetime.now(timezone.utc) second_feature = Feature(2, "Feature", "Did not run", "foo.feature", 1, tags=None) - # second_feature.state = Step.State.UNTESTED scenario = Scenario( 1, "Scenario", "Did not run", "foo.feature", 1, parent=None, tags=None, preconditions=None, background=None ) From becca9c12dbf9dd0e8acaf1e217765be7675a608 Mon Sep 17 00:00:00 2001 From: fliiiix Date: Sat, 1 Nov 2025 08:40:30 +0100 Subject: [PATCH 5/5] Add exploratory test feature aborted Run with: radish -b radish features --junit-xml=result.xml Produces junit xml: ``` ``` --- .../features/01-SumNumbers.feature | 18 ++++++++++++++ .../features/02-NeverRuns.feature | 17 +++++++++++++ .../abortedfeature/radish/steps.py | 24 +++++++++++++++++++ .../abortedfeature/radish/terrain.py | 6 +++++ 4 files changed, 65 insertions(+) create mode 100644 tests/exploratory/abortedfeature/features/01-SumNumbers.feature create mode 100644 tests/exploratory/abortedfeature/features/02-NeverRuns.feature create mode 100644 tests/exploratory/abortedfeature/radish/steps.py create mode 100644 tests/exploratory/abortedfeature/radish/terrain.py diff --git a/tests/exploratory/abortedfeature/features/01-SumNumbers.feature b/tests/exploratory/abortedfeature/features/01-SumNumbers.feature new file mode 100644 index 00000000..3bb67c52 --- /dev/null +++ b/tests/exploratory/abortedfeature/features/01-SumNumbers.feature @@ -0,0 +1,18 @@ +Feature: Test summing numbers + In order to test the basic + features of radish I test + to sum numbers. + + Scenario: Sum two numbers + Given I have the number 5 + And I have the number 3 + When I sum them + Then I expect the result to be 8 + + Scenario: Sum three numbers + Given I have the number 5 + And I have the number 3 + And I have the number 2 + When I sum them + Then I expect the result to be 10 + Then I crash diff --git a/tests/exploratory/abortedfeature/features/02-NeverRuns.feature b/tests/exploratory/abortedfeature/features/02-NeverRuns.feature new file mode 100644 index 00000000..44d2c9e4 --- /dev/null +++ b/tests/exploratory/abortedfeature/features/02-NeverRuns.feature @@ -0,0 +1,17 @@ +Feature: Test summing numbers + In order to test the basic + features of radish I test + to sum numbers. + + Scenario: Sum two numbers + Given I have the number 5 + And I have the number 3 + When I sum them + Then I expect the result to be 8 + + Scenario: Sum three numbers + Given I have the number 5 + And I have the number 3 + And I have the number 2 + When I sum them + Then I expect the result to be 10 diff --git a/tests/exploratory/abortedfeature/radish/steps.py b/tests/exploratory/abortedfeature/radish/steps.py new file mode 100644 index 00000000..415ca06d --- /dev/null +++ b/tests/exploratory/abortedfeature/radish/steps.py @@ -0,0 +1,24 @@ +import sys + +from radish import then, when +from radish.stepregistry import step + + +@step("I have the number {number:g}") +def have_number(step, number): + step.context.numbers.append(int(number)) + + +@when("I sum them") +def sum_numbers(step): + step.context.result = sum(step.context.numbers) + + +@then("I expect the result to be {result:g}") +def expect_result(step, result): + assert step.context.result == result + + +@then("I crash") +def crash(step): + sys.exit(1) diff --git a/tests/exploratory/abortedfeature/radish/terrain.py b/tests/exploratory/abortedfeature/radish/terrain.py new file mode 100644 index 00000000..e45102a3 --- /dev/null +++ b/tests/exploratory/abortedfeature/radish/terrain.py @@ -0,0 +1,6 @@ +from radish import before + + +@before.each_scenario +def init_numbers(scenario): + scenario.context.numbers = []