Skip to content

Commit 37358b9

Browse files
nikita-tkachenko-datadognikita-tkachenko-datadog
authored
Update Spock ITR instrumentation to skip spec setup methods if all features in a spec are skipped (#6782)
1 parent 04f98d5 commit 37358b9

File tree

15 files changed

+1077
-23
lines changed

15 files changed

+1077
-23
lines changed

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/events/TestEventsHandlerImpl.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -291,6 +291,11 @@ public boolean skip(TestIdentifier test) {
291291
return testModule.skip(test);
292292
}
293293

294+
@Override
295+
public boolean isSkippable(TestIdentifier test) {
296+
return testModule.isSkippable(test);
297+
}
298+
294299
@Override
295300
@Nonnull
296301
public TestRetryPolicy retryPolicy(TestIdentifier test) {

‎dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/main/java/datadog/trace/instrumentation/junit5/JUnit5SpockItrInstrumentation.java

Lines changed: 31 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,14 @@
1313
import datadog.trace.api.civisibility.config.TestIdentifier;
1414
import datadog.trace.bootstrap.CallDepthThreadLocalMap;
1515
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
16-
import java.util.Collection;
1716
import java.util.Set;
1817
import net.bytebuddy.asm.Advice;
1918
import net.bytebuddy.description.type.TypeDescription;
2019
import net.bytebuddy.matcher.ElementMatcher;
21-
import org.junit.platform.engine.TestTag;
20+
import org.junit.platform.engine.TestDescriptor;
2221
import org.junit.platform.engine.support.hierarchical.Node;
2322
import org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService;
23+
import org.spockframework.runtime.SpecNode;
2424
import org.spockframework.runtime.SpockNode;
2525

2626
@AutoService(Instrumenter.class)
@@ -102,16 +102,38 @@ public static void shouldBeSkipped(
102102
return;
103103
}
104104

105-
Collection<TestTag> tags = SpockUtils.getTags(spockNode);
106-
for (TestTag tag : tags) {
107-
if (InstrumentationBridge.ITR_UNSKIPPABLE_TAG.equals(tag.getName())) {
108-
return;
109-
}
105+
if (SpockUtils.isUnskippable(spockNode)) {
106+
return;
110107
}
111108

112-
TestIdentifier test = SpockUtils.toTestIdentifier(spockNode);
113-
if (test != null && TestEventsHandlerHolder.TEST_EVENTS_HANDLER.skip(test)) {
109+
if (spockNode instanceof SpecNode) {
110+
// suite
111+
SpecNode specNode = (SpecNode) spockNode;
112+
Set<? extends TestDescriptor> features = specNode.getChildren();
113+
for (TestDescriptor feature : features) {
114+
if (feature instanceof SpockNode && SpockUtils.isUnskippable((SpockNode<?>) feature)) {
115+
return;
116+
}
117+
118+
TestIdentifier featureIdentifier = SpockUtils.toTestIdentifier(feature);
119+
if (featureIdentifier == null
120+
|| !TestEventsHandlerHolder.TEST_EVENTS_HANDLER.isSkippable(featureIdentifier)) {
121+
return;
122+
}
123+
}
124+
125+
// all children are skippable
126+
for (TestDescriptor feature : features) {
127+
TestEventsHandlerHolder.TEST_EVENTS_HANDLER.skip(SpockUtils.toTestIdentifier(feature));
128+
}
114129
skipResult = Node.SkipResult.skip(InstrumentationBridge.ITR_SKIP_REASON);
130+
131+
} else {
132+
// individual test case
133+
TestIdentifier test = SpockUtils.toTestIdentifier(spockNode);
134+
if (test != null && TestEventsHandlerHolder.TEST_EVENTS_HANDLER.skip(test)) {
135+
skipResult = Node.SkipResult.skip(InstrumentationBridge.ITR_SKIP_REASON);
136+
}
115137
}
116138
}
117139

‎dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/main/java/datadog/trace/instrumentation/junit5/SpockUtils.java

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package datadog.trace.instrumentation.junit5;
22

3+
import datadog.trace.api.civisibility.InstrumentationBridge;
34
import datadog.trace.api.civisibility.config.TestIdentifier;
45
import java.lang.invoke.MethodHandle;
56
import java.lang.reflect.Method;
@@ -64,6 +65,16 @@ public static Collection<TestTag> getTags(SpockNode<?> spockNode) {
6465
}
6566
}
6667

68+
public static boolean isUnskippable(SpockNode<?> spockNode) {
69+
Collection<TestTag> tags = SpockUtils.getTags(spockNode);
70+
for (TestTag tag : tags) {
71+
if (InstrumentationBridge.ITR_UNSKIPPABLE_TAG.equals(tag.getName())) {
72+
return true;
73+
}
74+
}
75+
return false;
76+
}
77+
6778
public static Method getTestMethod(MethodSource methodSource) {
6879
String methodName = methodSource.getMethodName();
6980
if (methodName == null) {

‎dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/SpockTest.groovy

Lines changed: 24 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,10 @@ import org.example.TestFailedParameterizedSpock
66
import org.example.TestFailedSpock
77
import org.example.TestFailedThenSucceedParameterizedSpock
88
import org.example.TestFailedThenSucceedSpock
9+
import org.example.TestParameterizedSetupSpecSpock
910
import org.example.TestParameterizedSpock
1011
import org.example.TestSucceedAndFailedSpock
12+
import org.example.TestSucceedSetupSpecSpock
1113
import org.example.TestSucceedSpock
1214
import org.example.TestSucceedSpockSlow
1315
import org.example.TestSucceedSpockUnskippable
@@ -47,13 +49,21 @@ class SpockTest extends CiVisibilityInstrumentationTest {
4749
assertSpansData(testcaseName, expectedTracesCount)
4850

4951
where:
50-
testcaseName | tests | expectedTracesCount | skippedTests
51-
"test-itr-skipping" | [TestSucceedSpock] | 2 | [new TestIdentifier("org.example.TestSucceedSpock", "test success", null, null)]
52-
"test-itr-skipping-parameterized" | [TestParameterizedSpock] | 3 | [
52+
testcaseName | tests | expectedTracesCount | skippedTests
53+
"test-itr-skipping" | [TestSucceedSpock] | 2 | [new TestIdentifier("org.example.TestSucceedSpock", "test success", null, null)]
54+
"test-itr-skipping-parameterized" | [TestParameterizedSpock] | 3 | [
5355
new TestIdentifier("org.example.TestParameterizedSpock", "test add 1 and 2", '{"metadata":{"test_name":"test add 1 and 2"}}', null)
5456
]
55-
"test-itr-unskippable" | [TestSucceedSpockUnskippable] | 2 | [new TestIdentifier("org.example.TestSucceedSpockUnskippable", "test success", null, null)]
56-
"test-itr-unskippable-suite" | [TestSucceedSpockUnskippableSuite] | 2 | [new TestIdentifier("org.example.TestSucceedSpockUnskippableSuite", "test success", null, null)]
57+
"test-itr-unskippable" | [TestSucceedSpockUnskippable] | 2 | [new TestIdentifier("org.example.TestSucceedSpockUnskippable", "test success", null, null)]
58+
"test-itr-unskippable-suite" | [TestSucceedSpockUnskippableSuite] | 2 | [new TestIdentifier("org.example.TestSucceedSpockUnskippableSuite", "test success", null, null)]
59+
"test-itr-skipping-spec-setup" | [TestSucceedSetupSpecSpock] | 2 | [
60+
new TestIdentifier("org.example.TestSucceedSetupSpecSpock", "test success", null, null),
61+
new TestIdentifier("org.example.TestSucceedSetupSpecSpock", "test another success", null, null)
62+
]
63+
"test-itr-not-skipping-spec-setup" | [TestSucceedSetupSpecSpock] | 2 | [new TestIdentifier("org.example.TestSucceedSetupSpecSpock", "test success", null, null)]
64+
"test-itr-not-skipping-parameterized-spec-setup" | [TestParameterizedSetupSpecSpock] | 2 | [
65+
new TestIdentifier("org.example.TestParameterizedSetupSpecSpock", "test add 1 and 2", '{"metadata":{"test_name":"test add 1 and 2"}}', null)
66+
]
5767
}
5868

5969
def "test flaky retries #testcaseName"() {
@@ -86,18 +96,18 @@ class SpockTest extends CiVisibilityInstrumentationTest {
8696
assertSpansData(testcaseName, expectedTracesCount)
8797

8898
where:
89-
testcaseName | tests | expectedTracesCount | knownTestsList
90-
"test-efd-known-test" | [TestSucceedSpock] | 2 | [new TestIdentifier("org.example.TestSucceedSpock", "test success", null, null)]
91-
"test-efd-known-parameterized-test" | [TestParameterizedSpock] | 3 | [
99+
testcaseName | tests | expectedTracesCount | knownTestsList
100+
"test-efd-known-test" | [TestSucceedSpock] | 2 | [new TestIdentifier("org.example.TestSucceedSpock", "test success", null, null)]
101+
"test-efd-known-parameterized-test" | [TestParameterizedSpock] | 3 | [
92102
new TestIdentifier("org.example.TestParameterizedSpock", "test add 1 and 2", null, null),
93103
new TestIdentifier("org.example.TestParameterizedSpock", "test add 4 and 4", null, null)
94104
]
95-
"test-efd-new-test" | [TestSucceedSpock] | 4 | []
96-
"test-efd-new-parameterized-test" | [TestParameterizedSpock] | 7 | []
97-
"test-efd-known-tests-and-new-test" | [TestParameterizedSpock] | 5 | [new TestIdentifier("org.example.TestParameterizedSpock", "test add 1 and 2", null, null)]
98-
"test-efd-new-slow-test" | [TestSucceedSpockSlow] | 3 | [] // is executed only twice
99-
"test-efd-new-very-slow-test" | [TestSucceedSpockVerySlow] | 2 | [] // is executed only once
100-
"test-efd-faulty-session-threshold" | [TestSucceedAndFailedSpock] | 8 | []
105+
"test-efd-new-test" | [TestSucceedSpock] | 4 | []
106+
"test-efd-new-parameterized-test" | [TestParameterizedSpock] | 7 | []
107+
"test-efd-known-tests-and-new-test" | [TestParameterizedSpock] | 5 | [new TestIdentifier("org.example.TestParameterizedSpock", "test add 1 and 2", null, null)]
108+
"test-efd-new-slow-test" | [TestSucceedSpockSlow] | 3 | [] // is executed only twice
109+
"test-efd-new-very-slow-test" | [TestSucceedSpockVerySlow] | 2 | [] // is executed only once
110+
"test-efd-faulty-session-threshold" | [TestSucceedAndFailedSpock] | 8 | []
101111
}
102112

103113
private static void runTests(List<Class<?>> classes) {

‎dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/org/example/TestParameterizedSetupSpecSpock.groovy

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
package org.example
2+
3+
import datadog.trace.bootstrap.instrumentation.api.AgentScope
4+
import datadog.trace.bootstrap.instrumentation.api.AgentSpan
5+
import datadog.trace.bootstrap.instrumentation.api.AgentTracer
6+
import datadog.trace.bootstrap.instrumentation.api.ScopeSource
7+
import spock.lang.Specification
8+
9+
class TestParameterizedSetupSpecSpock extends Specification {
10+
11+
def setupSpec() {
12+
AgentTracer.TracerAPI agentTracer = AgentTracer.get()
13+
AgentSpan span = agentTracer.buildSpan("spock-manual", "spec-setup").start()
14+
try (AgentScope scope = agentTracer.activateSpan(span, ScopeSource.MANUAL)) {
15+
// manually trace spec setup to check whether ITR skips it or not
16+
}
17+
span.finish()
18+
}
19+
20+
def "test add #a and #b"() {
21+
expect:
22+
a + b == c
23+
24+
where:
25+
a | b | c
26+
1 | 2 | 3
27+
4 | 4 | 8
28+
}
29+
}

‎dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/org/example/TestSucceedSetupSpecSpock.groovy

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
package org.example
2+
3+
import datadog.trace.bootstrap.instrumentation.api.AgentScope
4+
import datadog.trace.bootstrap.instrumentation.api.AgentSpan
5+
import datadog.trace.bootstrap.instrumentation.api.AgentTracer
6+
import datadog.trace.bootstrap.instrumentation.api.ScopeSource
7+
import spock.lang.Specification
8+
9+
class TestSucceedSetupSpecSpock extends Specification {
10+
11+
def setupSpec() {
12+
AgentTracer.TracerAPI agentTracer = AgentTracer.get()
13+
AgentSpan span = agentTracer.buildSpan("spock-manual", "spec-setup").start()
14+
try (AgentScope scope = agentTracer.activateSpan(span, ScopeSource.MANUAL)) {
15+
// manually trace spec setup to check whether ITR skips it or not
16+
}
17+
span.finish()
18+
}
19+
20+
def "test success"() {
21+
expect:
22+
1 == 1
23+
}
24+
25+
def "test another success"() {
26+
expect:
27+
1 == 1
28+
}
29+
}

‎dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-not-skipping-parameterized-spec-setup/coverages.ftl

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
[ ]

0 commit comments

Comments
 (0)