Skip to content

Commit 6ba3a7e

Browse files
mrsheepukmrsheepuk
authored
Mark junit test cases as skipped if no pickle step results available (#597)
* Mark junit test cases as skipped if no pickle step results available * Add tests
1 parent 9b4d5e9 commit 6ba3a7e

File tree

6 files changed

+265
-15
lines changed

6 files changed

+265
-15
lines changed

‎internal/formatters/fmt_junit.go

Lines changed: 54 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"time"
1111

1212
"github.com/cucumber/godog/formatters"
13+
"github.com/cucumber/godog/internal/models"
1314
"github.com/cucumber/godog/internal/utils"
1415
)
1516

@@ -47,6 +48,40 @@ func junitTimeDuration(from, to time.Time) string {
4748
return strconv.FormatFloat(to.Sub(from).Seconds(), 'f', -1, 64)
4849
}
4950

51+
// getPickleResult deals with the fact that if there's no result due to 'StopOnFirstFailure' being
52+
// set, MustGetPickleResult panics.
53+
func (f *JUnit) getPickleResult(pickleID string) (res *models.PickleResult) {
54+
defer func() {
55+
if r := recover(); r != nil {
56+
res = nil
57+
}
58+
}()
59+
pr := f.Storage.MustGetPickleResult(pickleID)
60+
res = &pr
61+
return
62+
}
63+
64+
func (f *JUnit) getPickleStepResult(stepID string) (res *models.PickleStepResult) {
65+
defer func() {
66+
if r := recover(); r != nil {
67+
res = nil
68+
}
69+
}()
70+
psr := f.Storage.MustGetPickleStepResult(stepID)
71+
res = &psr
72+
return
73+
}
74+
75+
func (f *JUnit) getPickleStepResultsByPickleID(pickleID string) (res []models.PickleStepResult) {
76+
defer func() {
77+
if r := recover(); r != nil {
78+
res = nil
79+
}
80+
}()
81+
res = f.Storage.MustGetPickleStepResultsByPickleID(pickleID)
82+
return
83+
}
84+
5085
func (f *JUnit) buildJUNITPackageSuite() JunitPackageSuite {
5186
features := f.Storage.MustGetFeatures()
5287
sort.Sort(sortFeaturesByName(features))
@@ -79,33 +114,37 @@ func (f *JUnit) buildJUNITPackageSuite() JunitPackageSuite {
79114
var outlineNo = make(map[string]int)
80115
for idx, pickle := range pickles {
81116
tc := junitTestCase{}
82-
83-
pickleResult := f.Storage.MustGetPickleResult(pickle.Id)
84-
85-
if idx == 0 {
86-
firstPickleStartedAt = pickleResult.StartedAt
117+
tc.Name = pickle.Name
118+
if testcaseNames[tc.Name] > 1 {
119+
outlineNo[tc.Name] = outlineNo[tc.Name] + 1
120+
tc.Name += fmt.Sprintf(" #%d", outlineNo[tc.Name])
87121
}
88122

89-
lastPickleFinishedAt = pickleResult.StartedAt
123+
pickleResult := f.getPickleResult(pickle.Id)
124+
if pickleResult == nil {
125+
tc.Status = skipped.String()
126+
} else {
127+
if idx == 0 {
128+
firstPickleStartedAt = pickleResult.StartedAt
129+
}
130+
lastPickleFinishedAt = pickleResult.StartedAt
131+
}
90132

91133
if len(pickle.Steps) > 0 {
92134
lastStep := pickle.Steps[len(pickle.Steps)-1]
93-
lastPickleStepResult := f.Storage.MustGetPickleStepResult(lastStep.Id)
94-
lastPickleFinishedAt = lastPickleStepResult.FinishedAt
135+
if lastPickleStepResult := f.getPickleStepResult(lastStep.Id); lastPickleStepResult != nil {
136+
lastPickleFinishedAt = lastPickleStepResult.FinishedAt
137+
}
95138
}
96139

97-
tc.Time = junitTimeDuration(pickleResult.StartedAt, lastPickleFinishedAt)
98-
99-
tc.Name = pickle.Name
100-
if testcaseNames[tc.Name] > 1 {
101-
outlineNo[tc.Name] = outlineNo[tc.Name] + 1
102-
tc.Name += fmt.Sprintf(" #%d", outlineNo[tc.Name])
140+
if pickleResult != nil {
141+
tc.Time = junitTimeDuration(pickleResult.StartedAt, lastPickleFinishedAt)
103142
}
104143

105144
ts.Tests++
106145
suite.Tests++
107146

108-
pickleStepResults := f.Storage.MustGetPickleStepResultsByPickleID(pickle.Id)
147+
pickleStepResults := f.getPickleStepResultsByPickleID(pickle.Id)
109148
for _, stepResult := range pickleStepResults {
110149
pickleStep := f.Storage.MustGetPickleStep(stepResult.PickleStepID)
111150

‎internal/formatters/fmt_junit_test.go

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
package formatters_test
2+
3+
import (
4+
"bytes"
5+
"encoding/xml"
6+
"fmt"
7+
"testing"
8+
9+
"github.com/cucumber/godog"
10+
)
11+
12+
func Test_JUnitFormatter_StopOnFirstFailure(t *testing.T) {
13+
featureFile := "formatter-tests/features/stop_on_first_failure.feature"
14+
15+
// First, verify the normal output (without StopOnFirstFailure)
16+
var normalBuf bytes.Buffer
17+
normalOpts := godog.Options{
18+
Format: "junit",
19+
Paths: []string{featureFile},
20+
Output: &normalBuf,
21+
Strict: true,
22+
}
23+
24+
normalSuite := godog.TestSuite{
25+
Name: "Normal Run",
26+
ScenarioInitializer: func(sc *godog.ScenarioContext) {
27+
setupStopOnFailureSteps(sc)
28+
},
29+
Options: &normalOpts,
30+
}
31+
if status := normalSuite.Run(); status != 1 {
32+
t.Fatalf("Expected suite to have status 1, but got %d", status)
33+
}
34+
35+
// Parse the XML output
36+
var normalResult JunitPackageSuite
37+
err := xml.Unmarshal(normalBuf.Bytes(), &normalResult)
38+
if err != nil {
39+
t.Fatalf("Failed to parse XML output: %v", err)
40+
}
41+
42+
// Now run with StopOnFirstFailure
43+
var stopBuf bytes.Buffer
44+
stopOpts := godog.Options{
45+
Format: "junit",
46+
Paths: []string{featureFile},
47+
Output: &stopBuf,
48+
Strict: true,
49+
StopOnFailure: true,
50+
}
51+
52+
stopSuite := godog.TestSuite{
53+
Name: "Stop On First Failure",
54+
ScenarioInitializer: func(sc *godog.ScenarioContext) {
55+
setupStopOnFailureSteps(sc)
56+
},
57+
Options: &stopOpts,
58+
}
59+
if status := stopSuite.Run(); status != 1 {
60+
t.Fatalf("Expected suite to have status 1, but got %d", status)
61+
}
62+
63+
// Parse the XML output
64+
var stopResult JunitPackageSuite
65+
err = xml.Unmarshal(stopBuf.Bytes(), &stopResult)
66+
if err != nil {
67+
t.Fatalf("Failed to parse XML output: %v", err)
68+
}
69+
70+
// Verify the second test case is marked as skipped when StopOnFirstFailure is enabled
71+
if len(stopResult.TestSuites) == 0 || len(stopResult.TestSuites[0].TestCases) < 2 {
72+
t.Fatal("Expected at least 2 test cases in the results")
73+
}
74+
75+
// In a normal run, second test case should not be skipped
76+
if normalResult.TestSuites[0].TestCases[1].Status == "skipped" {
77+
t.Errorf("In normal run, second test case should not be skipped")
78+
}
79+
80+
// In stop on failure run, second test case should be skipped
81+
if stopResult.TestSuites[0].TestCases[1].Status != "skipped" {
82+
t.Errorf("In stop on failure run, second test case should be skipped, but got %s",
83+
stopResult.TestSuites[0].TestCases[1].Status)
84+
}
85+
}
86+
87+
// setupStopOnFailureSteps registers the step definitions for the stop-on-failure test
88+
func setupStopOnFailureSteps(sc *godog.ScenarioContext) {
89+
sc.Step(`^a passing step$`, func() error {
90+
return nil
91+
})
92+
sc.Step(`^a failing step$`, func() error {
93+
return fmt.Errorf("step failed")
94+
})
95+
}
96+
97+
// JunitPackageSuite represents the JUnit XML structure for test suites
98+
type JunitPackageSuite struct {
99+
XMLName xml.Name `xml:"testsuites"`
100+
Name string `xml:"name,attr"`
101+
Tests int `xml:"tests,attr"`
102+
Skipped int `xml:"skipped,attr"`
103+
Failures int `xml:"failures,attr"`
104+
Errors int `xml:"errors,attr"`
105+
Time string `xml:"time,attr"`
106+
TestSuites []*JunitTestSuite `xml:"testsuite"`
107+
}
108+
109+
type JunitTestSuite struct {
110+
XMLName xml.Name `xml:"testsuite"`
111+
Name string `xml:"name,attr"`
112+
Tests int `xml:"tests,attr"`
113+
Skipped int `xml:"skipped,attr"`
114+
Failures int `xml:"failures,attr"`
115+
Errors int `xml:"errors,attr"`
116+
Time string `xml:"time,attr"`
117+
TestCases []*JunitTestCase `xml:"testcase"`
118+
}
119+
120+
type JunitTestCase struct {
121+
XMLName xml.Name `xml:"testcase"`
122+
Name string `xml:"name,attr"`
123+
Status string `xml:"status,attr"`
124+
Time string `xml:"time,attr"`
125+
Failure *JunitFailure `xml:"failure,omitempty"`
126+
Error []*JunitError `xml:"error,omitempty"`
127+
}
128+
129+
type JunitFailure struct {
130+
Message string `xml:"message,attr"`
131+
Type string `xml:"type,attr,omitempty"`
132+
}
133+
134+
type JunitError struct {
135+
XMLName xml.Name `xml:"error,omitempty"`
136+
Message string `xml:"message,attr"`
137+
Type string `xml:"type,attr"`
138+
}

‎internal/formatters/formatter-tests/features/stop_on_first_failure.feature

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
Feature: Stop on first failure
2+
3+
Scenario: First scenario - should run and fail
4+
Given a passing step
5+
When a failing step
6+
Then a passing step
7+
8+
Scenario: Second scenario - should be skipped
9+
Given a passing step
10+
Then a passing step

‎internal/formatters/formatter-tests/junit,pretty/stop_on_first_failure

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
<bold-white>Feature:</bold-white> Stop on first failure
2+
3+
<bold-white>Scenario:</bold-white> First scenario - should run and fail <bold-black># formatter-tests/features/stop_on_first_failure.feature:3</bold-black>
4+
<green>Given</green> <green>a passing step</green> <bold-black># fmt_output_test.go:XXX -> github.com/cucumber/godog/internal/formatters_test.passingStepDef</bold-black>
5+
<red>When</red> <red>a failing step</red> <bold-black># fmt_output_test.go:XXX -> github.com/cucumber/godog/internal/formatters_test.failingStepDef</bold-black>
6+
<bold-red>step failed</bold-red>
7+
<cyan>Then</cyan> <cyan>a passing step</cyan> <bold-black># fmt_output_test.go:XXX -> github.com/cucumber/godog/internal/formatters_test.passingStepDef</bold-black>
8+
9+
<bold-white>Scenario:</bold-white> Second scenario - should be skipped <bold-black># formatter-tests/features/stop_on_first_failure.feature:8</bold-black>
10+
<green>Given</green> <green>a passing step</green> <bold-black># fmt_output_test.go:XXX -> github.com/cucumber/godog/internal/formatters_test.passingStepDef</bold-black>
11+
<green>Then</green> <green>a passing step</green> <bold-black># fmt_output_test.go:XXX -> github.com/cucumber/godog/internal/formatters_test.passingStepDef</bold-black>
12+
<?xml version="1.0" encoding="UTF-8"?>
13+
<testsuites name="junit,pretty" tests="2" skipped="0" failures="1" errors="0" time="0">
14+
<testsuite name="Stop on first failure" tests="2" skipped="0" failures="1" errors="0" time="0">
15+
<testcase name="First scenario - should run and fail" status="failed" time="0">
16+
<failure message="Step a failing step: step failed"></failure>
17+
<error message="Step a passing step" type="skipped"></error>
18+
</testcase>
19+
<testcase name="Second scenario - should be skipped" status="passed" time="0"></testcase>
20+
</testsuite>
21+
</testsuites>
22+
--- <red>Failed steps:</red>
23+
24+
<red>Scenario: First scenario - should run and fail</red> <bold-black># formatter-tests/features/stop_on_first_failure.feature:3</bold-black>
25+
<red>When a failing step</red> <bold-black># formatter-tests/features/stop_on_first_failure.feature:5</bold-black>
26+
<red>Error: </red><bold-red>step failed</bold-red>
27+
28+
29+
2 scenarios (<green>1 passed</green>, <red>1 failed</red>)
30+
5 steps (<green>3 passed</green>, <red>1 failed</red>, <cyan>1 skipped</cyan>)
31+
0s

‎internal/formatters/formatter-tests/junit/stop_on_first_failure

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<testsuites name="junit" tests="2" skipped="0" failures="1" errors="0" time="0">
3+
<testsuite name="Stop on first failure" tests="2" skipped="0" failures="1" errors="0" time="0">
4+
<testcase name="First scenario - should run and fail" status="failed" time="0">
5+
<failure message="Step a failing step: step failed"></failure>
6+
<error message="Step a passing step" type="skipped"></error>
7+
</testcase>
8+
<testcase name="Second scenario - should be skipped" status="passed" time="0"></testcase>
9+
</testsuite>
10+
</testsuites>

‎internal/formatters/formatter-tests/pretty/stop_on_first_failure

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
<bold-white>Feature:</bold-white> Stop on first failure
2+
3+
<bold-white>Scenario:</bold-white> First scenario - should run and fail <bold-black># formatter-tests/features/stop_on_first_failure.feature:3</bold-black>
4+
<green>Given</green> <green>a passing step</green> <bold-black># fmt_output_test.go:XXX -> github.com/cucumber/godog/internal/formatters_test.passingStepDef</bold-black>
5+
<red>When</red> <red>a failing step</red> <bold-black># fmt_output_test.go:XXX -> github.com/cucumber/godog/internal/formatters_test.failingStepDef</bold-black>
6+
<bold-red>step failed</bold-red>
7+
<cyan>Then</cyan> <cyan>a passing step</cyan> <bold-black># fmt_output_test.go:XXX -> github.com/cucumber/godog/internal/formatters_test.passingStepDef</bold-black>
8+
9+
<bold-white>Scenario:</bold-white> Second scenario - should be skipped <bold-black># formatter-tests/features/stop_on_first_failure.feature:8</bold-black>
10+
<green>Given</green> <green>a passing step</green> <bold-black># fmt_output_test.go:XXX -> github.com/cucumber/godog/internal/formatters_test.passingStepDef</bold-black>
11+
<green>Then</green> <green>a passing step</green> <bold-black># fmt_output_test.go:XXX -> github.com/cucumber/godog/internal/formatters_test.passingStepDef</bold-black>
12+
13+
--- <red>Failed steps:</red>
14+
15+
<red>Scenario: First scenario - should run and fail</red> <bold-black># formatter-tests/features/stop_on_first_failure.feature:3</bold-black>
16+
<red>When a failing step</red> <bold-black># formatter-tests/features/stop_on_first_failure.feature:5</bold-black>
17+
<red>Error: </red><bold-red>step failed</bold-red>
18+
19+
20+
2 scenarios (<green>1 passed</green>, <red>1 failed</red>)
21+
5 steps (<green>3 passed</green>, <red>1 failed</red>, <cyan>1 skipped</cyan>)
22+
0s

0 commit comments

Comments
 (0)