diff --git a/docs/extensions.adoc b/docs/extensions.adoc index 004a3c72b5..7240b099b3 100644 --- a/docs/extensions.adoc +++ b/docs/extensions.adoc @@ -44,6 +44,18 @@ runner { See the <> section for a detailed description. +=== Test Order Configuration + +[source,groovy] +---- +runner { + orderer new RandomSpecOrderer() // randomize specification run order +} +---- + +Instead of the default run order, you can configure Spock to execute specifications and/or features e.g. in random, +alphabetical or manually assigned, annotation-based order. See the <<#_run_order>> section for more details. + == Built-In Extensions Most of Spock's built-in extensions are _annotation-driven_. In other words, they are triggered by annotating a @@ -709,6 +721,114 @@ runner { } ---- +=== Run Order + +Ideally, automated tests in general and Spock specifications in particular should be independent of each other. The same +applies to feature methods within a specification. Therefore, you should not rely on any specific order of execution +(run order). + +Nevertheless, you have options to influence the run order, using the <> and a set of built-in +orderers derived from super class `SpecOrderer`. Please check the Javadocs for package +`org.spockframework.runtime.extension.builtin.orderer` for more details. You can also write your own `SpecOrderer`, if +none of the built-in ones satisfies your needs. + +Please note that `@Stepwise` always trumps any run order you might have configured, i.e. `@Stepwise` "wins" against +`SpecOrderer`. + +==== Random Run Order + +One helpful way to heuristically increase your confidence that your tests are indeed independent of each other, is to +explicitly say goodbye to deterministic run order by randomizing it. + +[source,groovy] +---- +import org.spockframework.runtime.extension.builtin.orderer.RandomSpecOrderer + +runner { + orderer new RandomSpecOrderer( + true, // Randomize overall specification run order + true, // Randomize the run order of feature methods within specifications + System.currentTimeMillis() // Set a fixed value, if you want repeatable pseudo-random numbers. + // This might be helpful for reproducing issues when debugging your tests. + ) +} +---- + +==== Alphabetical Run Order + +Less useful than random run order, but available anyway, is a way to execute specifications and/or features +alphabetically, based on their display names and a simple `String.compareTo(String)` (no fancy locale-based collation). +The default sorting direction is ascending, optionally you can also sort elements in descending order. + +[source,groovy] +---- +import org.spockframework.runtime.extension.builtin.orderer.AlphabeticalSpecOrderer + +runner { + orderer new AlphabeticalSpecOrderer( + true, // Run specifications in alphabetical order by display name + true, // Run feature methods within specifications in alphabetical order by display name + false // Sort in ascending order (use 'true' for descending order) + ) +} +---- + +==== Annotation-Based Run Order + +If you want to basically retain Spock's default run order for most or at least some of your specifications and/or +feature methods, but modify it for particular specs/features, or take it to the extreme and manually assign run orders +everywhere, use the `@Order(int)` annotation in combination with the annotation-based orderer: + +[source,groovy] +---- +import org.spockframework.runtime.extension.builtin.orderer.AnnotatationBasedSpecOrderer + +runner { + orderer new AnnotatationBasedSpecOrderer() +} +---- + +Please note, that `@Order` annotations have no effect whatsoever, if `AnnotatationBasedSpecOrderer` is not configured +as the active orderer. E.g., you cannot expect to be able to use random ordering in combination with manually assigning +run orders via annotations for some exceptions. Annotation-based ordering must be explicitly activated and is only +available as a modification of Spock's default run order. + +Using `@Order`, the basic idea is to assume unannotated specifications and features to all carry an implicit `@Order(0)` +annotation. If you wish to run some specs/features before others, assign them a lower (negative) run order. If you want +to run them after the default-ordered elements, assign them a higher (positive) order number: + +[source,groovy] +---- +@Order(1) // Execute after default-ordered specs +class FirstSpec extends Specification { + // Execute features in order 'three', 'one', 'two' in ascending order of assigned @Order values + @Order(2) def one() { expect: true } + @Order(3) def two() { expect: true } + @Order(1) def three() { expect: true } +} + +@Order(-1) // Execute before default-ordered specs +class SecondSpec extends Specification { + def foo() { expect: true } // Default order + @Order(99) def bar() { expect: true } // Execute after default-ordered features + @Order(-5) def zot() { expect: true } // Execute before default-ordered features +} + +// Default order +class ThirdSpec extends Specification { + def "some feature"() { expect: true } // Default order + @Order(1) def "another feature"() { expect: true } // Execute after default-ordered features + def "one more feature"() { expect: true } // Default order +} + +// Default order +class FourthSpec extends Specification { + def 'feature X'() { expect: true } // Default order + def 'feature M'() { expect: true } // Default order + @Order(-1) def 'feature D'() { expect: true } // Execute before default-ordered features +} +---- + == Third-Party Extensions You can find a list of third-party extensions in the https://github.com/spockframework/spock/wiki/Third-Party-Extensions[Spock Wiki]. @@ -733,13 +853,17 @@ fully-qualified class name in a file `META-INF/services/org.spockframework.runti class path. As soon as these two conditions are satisfied, the extension is automatically loaded and used when Spock is running. -`IGlobalExtension` has the following three methods: +`IGlobalExtension` has the following four methods: `start()`:: This is called once at the very start of the Spock execution. +`initSpecs(Collection specs)`:: + This is called once, before visiting single specifications later on in `visitSpec`. It enables global extensions to + view all specifications as an ensemble, e.g. for iterating over them and rearranging their execution order. + `visitSpec(SpecInfo spec)`:: - This is called once for each specification. In this method you can prepare a specification with your extension magic, + This is called once for each specification. In this method, you can prepare a specification with your extension magic, like attaching interceptors to various interception points as described in the chapter <>. `stop()`:: diff --git a/docs/release_notes.adoc b/docs/release_notes.adoc index 45652ce7d0..614be023b9 100644 --- a/docs/release_notes.adoc +++ b/docs/release_notes.adoc @@ -5,6 +5,12 @@ include::include.adoc[] == 2.4 (tbd) +* New lifecycle method `IGlobalExtension.initSpecs` spockPull:1631[] +* Support for setting execution order (a.k.a. run order) in `SpecInfo`, can be used by extensions spockPull:1631[] +* New built-in `OrderExtension` supports run order modification for specifications, features or a combination of both. + Built-in orderers can create random, alphabetical or user-defined ordering, the latter using the new `@Order(int)` + annotation. See manual section <>. spockPull:1631[] + == 2.4-M1 (2022-11-30) * Fix issues with Spring 6/Spring Boot 3 spockPull:1541[] diff --git a/spock-core/src/main/java/org/spockframework/runtime/ExtensionRunner.java b/spock-core/src/main/java/org/spockframework/runtime/ExtensionRunner.java index 687bb37619..ecb81804e5 100644 --- a/spock-core/src/main/java/org/spockframework/runtime/ExtensionRunner.java +++ b/spock-core/src/main/java/org/spockframework/runtime/ExtensionRunner.java @@ -14,6 +14,7 @@ package org.spockframework.runtime; +import org.junit.platform.engine.TestDescriptor; import org.spockframework.runtime.extension.*; import org.spockframework.runtime.model.*; import org.spockframework.util.Nullable; @@ -30,6 +31,8 @@ */ @SuppressWarnings("rawtypes") public class ExtensionRunner { + private static final SpecInfo EMPTY_SPEC = new SpecInfo(); + private final SpecInfo spec; private final IExtensionRegistry extensionRegistry; private final IConfigurationRegistry configurationRegistry; @@ -42,6 +45,20 @@ public ExtensionRunner(SpecInfo spec, IExtensionRegistry extensionRegistry, ICon this.configurationRegistry = configurationRegistry; } + public ExtensionRunner(IExtensionRegistry extensionRegistry, IConfigurationRegistry configurationRegistry) { + this(EMPTY_SPEC, extensionRegistry, configurationRegistry); + } + + public void initGlobalExtensions(Set testDescriptors) { + List specs = testDescriptors.stream() + .filter(testDescriptor -> testDescriptor instanceof SpecNode) + .map(testDescriptor -> ((SpecNode) testDescriptor).getNodeInfo()) + .collect(toList()); + for (IGlobalExtension extension : extensionRegistry.getGlobalExtensions()) { + extension.initSpecs(specs); + } + } + public void run() { runGlobalExtensions(); runAnnotationDrivenExtensions(); diff --git a/spock-core/src/main/java/org/spockframework/runtime/RunContext.java b/spock-core/src/main/java/org/spockframework/runtime/RunContext.java index eefe913732..c1301e09ae 100644 --- a/spock-core/src/main/java/org/spockframework/runtime/RunContext.java +++ b/spock-core/src/main/java/org/spockframework/runtime/RunContext.java @@ -81,6 +81,10 @@ public ExtensionRunner createExtensionRunner(SpecInfo spec) { return new ExtensionRunner(spec, globalExtensionRegistry, globalExtensionRegistry); } + public ExtensionRunner createExtensionRunner() { + return new ExtensionRunner(globalExtensionRegistry, globalExtensionRegistry); + } + public PlatformParameterizedSpecRunner createSpecRunner(SpecInfo spec) { return new PlatformParameterizedSpecRunner( new MasterRunSupervisor(spec, createStackTraceFilter(spec), diffedObjectRenderer)); diff --git a/spock-core/src/main/java/org/spockframework/runtime/SpecProcessor.java b/spock-core/src/main/java/org/spockframework/runtime/SpecProcessor.java new file mode 100644 index 0000000000..479efb3c39 --- /dev/null +++ b/spock-core/src/main/java/org/spockframework/runtime/SpecProcessor.java @@ -0,0 +1,35 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * https://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.spockframework.runtime; + +import org.spockframework.runtime.extension.builtin.orderer.SpecOrderer; +import org.spockframework.runtime.model.SpecInfo; + +import java.util.Collection; + +/** + * Generic bulk processor for a collection of {@link SpecInfo} elements + * + * @see SpecOrderer + */ +public interface SpecProcessor { + /** + * Bulk-process a collection of {@link SpecInfo} elements in-place, i.e. do not return anything but operate on the + * elements given, changing their state if necessary. + * + * @param specs spec-info instances to be processed + */ + void process(Collection specs); +} diff --git a/spock-core/src/main/java/org/spockframework/runtime/SpockEngineDiscoveryPostProcessor.java b/spock-core/src/main/java/org/spockframework/runtime/SpockEngineDiscoveryPostProcessor.java index 6c900e1704..961a95b7bc 100644 --- a/spock-core/src/main/java/org/spockframework/runtime/SpockEngineDiscoveryPostProcessor.java +++ b/spock-core/src/main/java/org/spockframework/runtime/SpockEngineDiscoveryPostProcessor.java @@ -5,6 +5,10 @@ import org.junit.platform.engine.*; +import java.util.Set; + +import static java.util.Comparator.comparingInt; + class SpockEngineDiscoveryPostProcessor { private static final Object[] EMPTY_ARGS = new Object[0]; @@ -12,8 +16,11 @@ class SpockEngineDiscoveryPostProcessor { SpockEngineDescriptor postProcessEngineDescriptor(UniqueId uniqueId, RunContext runContext, SpockEngineDescriptor engineDescriptor) { SpockEngineDescriptor processedEngineDescriptor = new SpockEngineDescriptor(uniqueId, runContext); - engineDescriptor.getChildren().stream() + Set testDescriptors = engineDescriptor.getChildren(); + initSpecNodes(testDescriptors, runContext); + testDescriptors.stream() .map(child -> processSpecNode(child, runContext)) + .sorted(comparingInt(child -> child instanceof SpecNode ? ((SpecNode) child).getNodeInfo().getExecutionOrder() : 0)) .forEach(processedEngineDescriptor::addChild); return processedEngineDescriptor; } @@ -43,6 +50,10 @@ private UniqueId toUniqueId(UniqueId parentId, FeatureInfo feature) { return parentId.append("feature", feature.getFeatureMethod().getReflection().getName()); } + private void initSpecNodes(Set testDescriptors, RunContext runContext) { + runContext.createExtensionRunner().initGlobalExtensions(testDescriptors); + } + private TestDescriptor processSpecNode(TestDescriptor child, RunContext runContext) { if (child instanceof SpecNode) { SpecNode specNode = (SpecNode) child; diff --git a/spock-core/src/main/java/org/spockframework/runtime/extension/IGlobalExtension.java b/spock-core/src/main/java/org/spockframework/runtime/extension/IGlobalExtension.java index f9315864c7..059438e5a7 100644 --- a/spock-core/src/main/java/org/spockframework/runtime/extension/IGlobalExtension.java +++ b/spock-core/src/main/java/org/spockframework/runtime/extension/IGlobalExtension.java @@ -18,10 +18,13 @@ import org.spockframework.runtime.model.SpecInfo; +import java.util.Collection; + // TODO: start/stop lifecycle // TODO: design threading model public interface IGlobalExtension { - default void start() {}; - default void visitSpec(SpecInfo spec){}; - default void stop(){}; + default void start() {} + default void initSpecs(Collection specs) {} + default void visitSpec(SpecInfo spec) {} + default void stop() {} } diff --git a/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/OrderExtension.java b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/OrderExtension.java new file mode 100644 index 0000000000..2b77d70477 --- /dev/null +++ b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/OrderExtension.java @@ -0,0 +1,34 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * https://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.spockframework.runtime.extension.builtin; + +import org.spockframework.runtime.extension.IGlobalExtension; +import org.spockframework.runtime.model.SpecInfo; +import spock.config.RunnerConfiguration; + +import java.util.Collection; + +public class OrderExtension implements IGlobalExtension { + private final RunnerConfiguration config; + + public OrderExtension(RunnerConfiguration config) { + this.config = config; + } + + @Override + public void initSpecs(Collection specs) { + config.orderer.process(specs); + } +} diff --git a/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/AlphabeticalSpecOrderer.java b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/AlphabeticalSpecOrderer.java new file mode 100644 index 0000000000..dfab34cd06 --- /dev/null +++ b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/AlphabeticalSpecOrderer.java @@ -0,0 +1,87 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * https://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.spockframework.runtime.extension.builtin.orderer; + +import org.spockframework.runtime.model.FeatureInfo; +import org.spockframework.runtime.model.SpecInfo; + +import java.util.Collection; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Orderer capable of assigning specification and/or feature method run orders according to spec/feature display names, + * comparing them alphabetically. There is no locale-specific collation, only simple string comparison based on the + * default JVM locale, using by {@link String#compareTo(String)}. + */ +public class AlphabeticalSpecOrderer extends SpecOrderer { + private final boolean ascending; + + /** + * Create an alphabetical spec orderer + * + * @param orderSpecs modify specification run order (yes/no)? + * @param orderFeatures modify feature run order within a specification (yes/no)? + * @param ascending sort in ascending order (yes/no)? + */ + public AlphabeticalSpecOrderer(boolean orderSpecs, boolean orderFeatures, boolean ascending) { + super(orderSpecs, orderFeatures); + this.ascending = ascending; + } + + /** + * Create an alphabetical spec orderer with a default ascending sort order + * + * @param orderSpecs modify specification run order (yes/no)? + * @param orderFeatures modify feature run order within a specification (yes/no)? + * @see #AlphabeticalSpecOrderer(boolean, boolean, boolean) + */ + public AlphabeticalSpecOrderer(boolean orderSpecs, boolean orderFeatures) { + this(orderSpecs, orderFeatures, true); + } + + /** + * Create an alphabetical spec orderer with default values. This is a shorthand for calling + * {@link #AlphabeticalSpecOrderer(boolean, boolean, boolean)} with parameters {@code true, true, true}. + */ + public AlphabeticalSpecOrderer() { + this(true, true); + } + + @Override + protected void orderSpecs(Collection specs) { + AtomicInteger i = new AtomicInteger(); + specs.stream() + .sorted((o1, o2) -> ascending + ? o1.getDisplayName().compareTo(o2.getDisplayName()) + : o2.getDisplayName().compareTo(o1.getDisplayName()) + ) + .forEach(specInfo -> specInfo.setExecutionOrder(i.getAndIncrement())); + } + + @Override + protected void orderFeatures(Collection features) { + AtomicInteger i = new AtomicInteger(); + features.stream() + .sorted((o1, o2) -> ascending + ? o1.getDisplayName().compareTo(o2.getDisplayName()) + : o2.getDisplayName().compareTo(o1.getDisplayName()) + ) + .forEach(featureInfo -> featureInfo.setExecutionOrder(i.getAndIncrement())); + } + + public boolean isAscending() { + return ascending; + } +} diff --git a/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/AnnotatationBasedSpecOrderer.java b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/AnnotatationBasedSpecOrderer.java new file mode 100644 index 0000000000..ddbe80b85a --- /dev/null +++ b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/AnnotatationBasedSpecOrderer.java @@ -0,0 +1,51 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * https://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.spockframework.runtime.extension.builtin.orderer; + +import org.spockframework.runtime.model.FeatureInfo; +import org.spockframework.runtime.model.SpecInfo; +import spock.lang.Order; + +import java.util.Collection; + +/** + * Spec orderer for usef-defined, manual specification and/or feature method ordering, to be used in connection with + * {@link Order @Order} annotations. See the Spock user manual for more details. + */ +public class AnnotatationBasedSpecOrderer extends SpecOrderer { + /** + * Create an annotation-based spec orderer + * @see Order + */ + public AnnotatationBasedSpecOrderer() { + super(true, true); + } + + @Override + protected void orderSpecs(Collection specs) { + for (SpecInfo spec : specs) { + Order orderAnnotation = spec.getAnnotation(Order.class); + spec.setExecutionOrder(orderAnnotation == null ? 0 : orderAnnotation.value()); + } + } + + @Override + protected void orderFeatures(Collection features) { + for (FeatureInfo feature : features) { + Order orderAnnotation = feature.getFeatureMethod().getAnnotation(Order.class); + feature.setExecutionOrder(orderAnnotation == null ? 0 : orderAnnotation.value()); + } + } +} diff --git a/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/DefaultSpecOrderer.java b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/DefaultSpecOrderer.java new file mode 100644 index 0000000000..216e3c0b50 --- /dev/null +++ b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/DefaultSpecOrderer.java @@ -0,0 +1,38 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * https://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.spockframework.runtime.extension.builtin.orderer; + +import org.spockframework.runtime.model.FeatureInfo; +import org.spockframework.runtime.model.SpecInfo; + +import java.util.Collection; + +/** + * No-op spec orderer, used as a default if no other orderer is configured + */ +public class DefaultSpecOrderer extends SpecOrderer { + /** + * Create a no-op spec orderer + */ + public DefaultSpecOrderer() { + super(false, false); + } + + @Override + protected void orderSpecs(Collection specs) { } + + @Override + protected void orderFeatures(Collection features) { } +} diff --git a/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/RandomSpecOrderer.java b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/RandomSpecOrderer.java new file mode 100644 index 0000000000..7e6eb67584 --- /dev/null +++ b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/RandomSpecOrderer.java @@ -0,0 +1,74 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * https://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.spockframework.runtime.extension.builtin.orderer; + +import org.spockframework.runtime.model.FeatureInfo; +import org.spockframework.runtime.model.SpecInfo; + +import java.util.Collection; +import java.util.Random; + +/** + * Orderer capable of randomizing specification and/or feature method run order + */ +public class RandomSpecOrderer extends SpecOrderer { + private final Random random; + + /** + * Create a random spec orderer + * + * @param orderSpecs modify specification run order (yes/no)? + * @param orderFeatures modify feature run order within a specification (yes/no)? + * @param seed random seed to be used in {@link Random#Random(long)}; setting this to a fixed value enables + * you to create test runs with repeatable pseudo-random run ordering, which can be helpful when + * e.g. debugging tests failing due to a particular run order, making them more independent of + * each other in the process + */ + public RandomSpecOrderer(boolean orderSpecs, boolean orderFeatures, long seed) { + super(orderSpecs, orderFeatures); + random = new Random(seed); + } + + /** + * Create a random spec orderer with a default random seed of {@code System.currentTimeMillis()} + * + * @param orderSpecs modify specification run order (yes/no)? + * @param orderFeatures modify feature run order within a specification (yes/no)? + * @see #RandomSpecOrderer(boolean, boolean, long) + */ + public RandomSpecOrderer(boolean orderSpecs, boolean orderFeatures) { + this(orderSpecs, orderFeatures, System.currentTimeMillis()); + } + + /** + * Create a random spec orderer with default values. This is a shorthand for calling + * {@link #RandomSpecOrderer(boolean, boolean, long)} with parameters {@code true, true, System.currentTimeMillis()}. + */ + public RandomSpecOrderer() { + this(true, true); + } + + @Override + protected void orderSpecs(Collection specs) { + for (SpecInfo spec : specs) + spec.setExecutionOrder(random.nextInt()); + } + + @Override + protected void orderFeatures(Collection features) { + for (FeatureInfo feature : features) + feature.setExecutionOrder(random.nextInt()); + } +} diff --git a/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/SpecOrderer.java b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/SpecOrderer.java new file mode 100644 index 0000000000..255530c5f5 --- /dev/null +++ b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/SpecOrderer.java @@ -0,0 +1,108 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * https://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.spockframework.runtime.extension.builtin.orderer; + +import org.spockframework.runtime.SpecProcessor; +import org.spockframework.runtime.model.FeatureInfo; +import org.spockframework.runtime.model.SpecInfo; + +import java.util.Collection; + +/** + * Abstract base class for specification and feature method orderers, i.e. workers modifying the corresponding execution + * order properties of spec-info and feature-info instances. + * + * @see DefaultSpecOrderer + * @see RandomSpecOrderer + * @see AnnotatationBasedSpecOrderer + * @see AlphabeticalSpecOrderer + */ +public abstract class SpecOrderer implements SpecProcessor { + protected final boolean orderSpecs; + protected final boolean orderFeatures; + + /** + * Create a spec-orderer with a user-defined operational scope + * + * @param orderSpecs modify specification run order (yes/no)? + * @param orderFeatures modify feature run order within a specification (yes/no)? + */ + public SpecOrderer(boolean orderSpecs, boolean orderFeatures) { + this.orderSpecs = orderSpecs; + this.orderFeatures = orderFeatures; + } + + /** + * Dispatch to {@link #orderSpecs(Collection)} and then to {@link #orderFeatures(Collection)} for each spec-info + * + * @param specs spec-info instances to be processed + */ + @Override + public void process(Collection specs) { + if (orderSpecs) + orderSpecs(specs); + if (!orderFeatures) + return; + for (SpecInfo spec : specs) + orderFeatures(spec.getAllFeatures()); + } + + /** + * Assign values to specification run orders. Implementors are expected to modify the corresponding object state + * in place, e.g. like this: + * 
+   * for (SpecInfo spec : specs)
+   *   spec.setExecutionOrder(random.nextInt());
+   *
+ * Or maybe: + * 
+   * AtomicInteger i = new AtomicInteger();
+   * specs.stream()
+   *   .sorted(myComparator)
+   *   .forEach(specInfo -> specInfo.setExecutionOrder(i.getAndIncrement()));
+   *
+ * + * @param specs spec-info instances to be ordered + */ + + protected abstract void orderSpecs(Collection specs); + + /** + * Assign values to feature run orders. Implementors are expected to modify the corresponding object state + * in place, e.g. like this: + * 
+   * for (FeatureInfo feature : features)
+   *   feature.setExecutionOrder(random.nextInt());
+   *
+ * Or maybe: + * 
+   * AtomicInteger i = new AtomicInteger();
+   * features.stream()
+   *   .sorted(myComparator)
+   *   .forEach(featureInfo -> featureInfo.setExecutionOrder(i.getAndIncrement()));
+   *
+ * + * @param features feature-info instances to be ordered + */ + protected abstract void orderFeatures(Collection features); + + public boolean isOrderSpecs() { + return orderSpecs; + } + + public boolean isOrderFeatures() { + return orderFeatures; + } +} diff --git a/spock-core/src/main/java/org/spockframework/runtime/model/FeatureInfo.java b/spock-core/src/main/java/org/spockframework/runtime/model/FeatureInfo.java index 7c0f341d34..d3b633b672 100644 --- a/spock-core/src/main/java/org/spockframework/runtime/model/FeatureInfo.java +++ b/spock-core/src/main/java/org/spockframework/runtime/model/FeatureInfo.java @@ -14,7 +14,6 @@ */ public class FeatureInfo extends SpecElementInfo implements ITestTaggable { private int declarationOrder; // per spec class - private int executionOrder; // per spec inheritance chain private final List parameterNames = new ArrayList<>(); private final List dataVariables = new ArrayList<>(); @@ -59,14 +58,6 @@ public void setDeclarationOrder(int declarationOrder) { this.declarationOrder = declarationOrder; } - public int getExecutionOrder() { - return executionOrder; - } - - public void setExecutionOrder(int executionOrder) { - this.executionOrder = executionOrder; - } - public List getParameterNames() { return parameterNames; } diff --git a/spock-core/src/main/java/org/spockframework/runtime/model/SpecElementInfo.java b/spock-core/src/main/java/org/spockframework/runtime/model/SpecElementInfo.java index dd58692020..188c3137b3 100644 --- a/spock-core/src/main/java/org/spockframework/runtime/model/SpecElementInfo.java +++ b/spock-core/src/main/java/org/spockframework/runtime/model/SpecElementInfo.java @@ -31,9 +31,11 @@ public abstract class SpecElementInfo

tags = new ArrayList<>(); private final List attachments = new ArrayList<>(); private final List interceptors = new ArrayList<>(); + @Override public boolean isSkipped() { return skipped; @@ -75,6 +77,14 @@ public void setExcluded(boolean excluded) { this.excluded = excluded; } + public int getExecutionOrder() { + return executionOrder; + } + + public void setExecutionOrder(int executionOrder) { + this.executionOrder = executionOrder; + } + @Override public List getTags() { return tags; diff --git a/spock-core/src/main/java/spock/config/RunnerConfiguration.java b/spock-core/src/main/java/spock/config/RunnerConfiguration.java index 5a01be9887..9afe511c0d 100644 --- a/spock-core/src/main/java/spock/config/RunnerConfiguration.java +++ b/spock-core/src/main/java/spock/config/RunnerConfiguration.java @@ -14,21 +14,26 @@ package spock.config; +import org.spockframework.runtime.extension.builtin.orderer.DefaultSpecOrderer; +import org.spockframework.runtime.extension.builtin.orderer.SpecOrderer; + /** * Configuration settings for the spec runner. * *

Example: * 

+ * import org.spockframework.runtime.extension.builtin.orderer.RandomSpecOrderer
  * import some.pkg.Fast
  * import some.pkg.IntegrationSpec
  *
  * runner {
- *   include Fast // could be either an annotation or a (base) class
+ *   include Fast  // could be either an annotation or a (base) class
  *   exclude {
  *     annotation some.pkg.Slow
  *     baseClass IntegrationSpec
  *   }
- *   filterStackTrace true // this is the default
+ *   filterStackTrace true            // this is the default
+ *   orderer new RandomSpecOrderer()  // DefaultSpecOrderer (no-op) is the default
  * }
  *
*/ @@ -37,6 +42,7 @@ public class RunnerConfiguration { public IncludeExcludeCriteria include = new IncludeExcludeCriteria(); public IncludeExcludeCriteria exclude = new IncludeExcludeCriteria(); public ParallelConfiguration parallel = new ParallelConfiguration(); + public SpecOrderer orderer = new DefaultSpecOrderer(); public boolean filterStackTrace = true; public boolean optimizeRunOrder = false; } diff --git a/spock-core/src/main/java/spock/lang/Order.java b/spock-core/src/main/java/spock/lang/Order.java new file mode 100644 index 0000000000..39341e8467 --- /dev/null +++ b/spock-core/src/main/java/spock/lang/Order.java @@ -0,0 +1,40 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * https://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package spock.lang; + +import org.spockframework.runtime.extension.builtin.OrderExtension; +import org.spockframework.runtime.extension.builtin.orderer.AnnotatationBasedSpecOrderer; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Assigns an execution order to a specification or feature. + *

+ * Annotations of this type are picked up by the {@link OrderExtension}, if and only if the + * {@link AnnotatationBasedSpecOrderer} is activated in the Spock configuration file: 

+ * runner {
+ *   orderer new AnnotatationBasedSpecOrderer()
+ * }
+ *

+ * See the Spock user manual for more details. + */ +@Retention(RetentionPolicy.RUNTIME) +@Target({ ElementType.TYPE, ElementType.METHOD }) +public @interface Order { + int value(); +} diff --git a/spock-core/src/main/resources/META-INF/services/org.spockframework.runtime.extension.IGlobalExtension b/spock-core/src/main/resources/META-INF/services/org.spockframework.runtime.extension.IGlobalExtension index 778826130b..77c6e009f6 100644 --- a/spock-core/src/main/resources/META-INF/services/org.spockframework.runtime.extension.IGlobalExtension +++ b/spock-core/src/main/resources/META-INF/services/org.spockframework.runtime.extension.IGlobalExtension @@ -1,3 +1,4 @@ org.spockframework.runtime.extension.builtin.IncludeExcludeExtension org.spockframework.runtime.extension.builtin.OptimizeRunOrderExtension +org.spockframework.runtime.extension.builtin.OrderExtension org.spockframework.runtime.extension.builtin.UnrollExtension diff --git a/spock-specs/src/test/groovy/org/spockframework/runtime/ExtensionClassesLoaderSpec.groovy b/spock-specs/src/test/groovy/org/spockframework/runtime/ExtensionClassesLoaderSpec.groovy index a0d72daf58..3c0075d810 100644 --- a/spock-specs/src/test/groovy/org/spockframework/runtime/ExtensionClassesLoaderSpec.groovy +++ b/spock-specs/src/test/groovy/org/spockframework/runtime/ExtensionClassesLoaderSpec.groovy @@ -1,3 +1,17 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * https://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.spockframework.runtime import org.spockframework.report.log.ReportLogConfiguration @@ -15,7 +29,7 @@ class ExtensionClassesLoaderSpec extends Specification { def result = new ExtensionClassesLoader().loadExtensionClassesFromDefaultLocation() then: - result == [IncludeExcludeExtension, OptimizeRunOrderExtension, UnrollExtension] + result == [IncludeExcludeExtension, OptimizeRunOrderExtension, OrderExtension, UnrollExtension] } def "loads global ConfigObjects"() { diff --git a/spock-specs/src/test/groovy/org/spockframework/smoke/extension/OrderExtensionSpec.groovy b/spock-specs/src/test/groovy/org/spockframework/smoke/extension/OrderExtensionSpec.groovy new file mode 100644 index 0000000000..d42454581c --- /dev/null +++ b/spock-specs/src/test/groovy/org/spockframework/smoke/extension/OrderExtensionSpec.groovy @@ -0,0 +1,304 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * https://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.spockframework.smoke.extension + +import groovy.transform.Canonical +import groovy.transform.ToString +import org.spockframework.EmbeddedSpecification +import org.spockframework.runtime.extension.builtin.OrderExtension +import org.spockframework.runtime.extension.builtin.orderer.AlphabeticalSpecOrderer +import org.spockframework.runtime.extension.builtin.orderer.AnnotatationBasedSpecOrderer +import org.spockframework.runtime.extension.builtin.orderer.RandomSpecOrderer +import spock.lang.Order +import spock.lang.Retry + +import static spock.lang.Retry.Mode.SETUP_FEATURE_CLEANUP + +class OrderExtensionSpec extends EmbeddedSpecification { + + @Canonical + @ToString(includePackage = false) + static class SpecExecution { + String spec + List features = [] + } + + static ThreadLocal> executionsTL = new ThreadLocal<>() + + List specs + + def setup() { + executionsTL.set([]) + compiler.addClassMemberImport(OrderExtensionSpec) + compiler.addClassImport(Order) + specs = compiler.compileWithImports(""" +abstract class BaseSpec extends Specification { + @Shared execution = new SpecExecution(spec: this.class.simpleName) + def setup() { execution.features << specificationContext.currentFeature.name } + def cleanupSpec() { executionsTL.get() << execution } +} + +class FirstSpec extends BaseSpec { + def one() { expect: true } + def two() { expect: true } + def three() { expect: true } +} + +@Order(-1) +class SecondSpec extends BaseSpec { + def foo() { expect: true } + @Order(99) def bar() { expect: true } + @Order(-5) def zot() { expect: true } +} + +class ThirdSpec extends BaseSpec { + def "some feature"() { expect: true } + @Order(1) def "another feature"() { expect: true } + def "one more feature"() { expect: true } +} + +class FourthSpec extends BaseSpec { + def 'feature X'() { expect: true } + def 'feature M'() { expect: true } + @Order(-1) def 'feature D'() { expect: true } +} + """) + } + + def 'default order'() { + runner.configurationScript = { + runner {} + } + runner.extensionClasses << OrderExtension + + when: + runner.runClasses(specs) + def executions = executionsTL.get() + + then: + executions*.spec == ['FirstSpec', 'SecondSpec', 'ThirdSpec', 'FourthSpec'] + executions*.features == [ + ['one', 'two', 'three'], + ['foo', 'bar', 'zot'], + ['some feature', 'another feature', 'one more feature'], + ['feature X', 'feature M', 'feature D'] + ] + } + + // Retry, if random order accidentally equals default order + @Retry(count = 50, mode = SETUP_FEATURE_CLEANUP) + def 'random spec order'() { + runner.configurationScript = { + runner { + orderer new RandomSpecOrderer(true, false) + } + } + runner.extensionClasses << OrderExtension + + when: + runner.runClasses(specs) + def executions = executionsTL.get() + + then: + executions*.spec != ['FirstSpec', 'SecondSpec', 'ThirdSpec', 'FourthSpec'] + executions*.features.containsAll([ + ['one', 'two', 'three'], + ['foo', 'bar', 'zot'], + ['some feature', 'another feature', 'one more feature'], + ['feature X', 'feature M', 'feature D'] + ]) + } + + // Retry, if random order accidentally equals default order + @Retry(count = 50, mode = SETUP_FEATURE_CLEANUP) + def 'random feature order'() { + runner.configurationScript = { + runner { + orderer new RandomSpecOrderer(false, true) + } + } + runner.extensionClasses << OrderExtension + + when: + runner.runClasses(specs) + def executions = executionsTL.get() + + then: + executions*.spec == ['FirstSpec', 'SecondSpec', 'ThirdSpec', 'FourthSpec'] + !executions*.features.containsAll([ + ['one', 'two', 'three'], + ['foo', 'bar', 'zot'], + ['some feature', 'another feature', 'one more feature'], + ['feature X', 'feature M', 'feature D'] + ]) + } + + // Retry, if random order accidentally equals default order + @Retry(count = 50, mode = SETUP_FEATURE_CLEANUP) + def 'random spec and feature order'() { + runner.configurationScript = { + runner { + orderer new RandomSpecOrderer(true, true) + } + } + runner.extensionClasses << OrderExtension + + when: + runner.runClasses(specs) + def executions = executionsTL.get() + + then: + executions*.spec != ['FirstSpec', 'SecondSpec', 'ThirdSpec', 'FourthSpec'] + !executions*.features.containsAll([ + ['one', 'two', 'three'], + ['foo', 'bar', 'zot'], + ['some feature', 'another feature', 'one more feature'], + ['feature X', 'feature M', 'feature D'] + ]) + } + + def 'repeatable, random-seeded spec and feature order'() { + runner.configurationScript = { + runner { + orderer new RandomSpecOrderer(true, true, 42L) + } + } + runner.extensionClasses << OrderExtension + + when: + runner.runClasses(specs) + def executions = executionsTL.get() + + then: + executions*.spec == ['SecondSpec', 'ThirdSpec', 'FourthSpec', 'FirstSpec'] + executions*.features == [ + ['bar', 'zot', 'foo'], + ['another feature', 'one more feature', 'some feature'], + ['feature M', 'feature X', 'feature D'], + ['one', 'two', 'three'] + ] + } + + def 'alphabetical spec order'() { + runner.configurationScript = { + runner { + orderer new AlphabeticalSpecOrderer(true, false) + } + } + runner.extensionClasses << OrderExtension + + when: + runner.runClasses(specs) + def executions = executionsTL.get() + + then: + executions*.spec == ['FirstSpec', 'FourthSpec', 'SecondSpec', 'ThirdSpec'] + executions*.features == [ + ['one', 'two', 'three'], + ['feature X', 'feature M', 'feature D'], + ['foo', 'bar', 'zot'], + ['some feature', 'another feature', 'one more feature'] + ] + } + + def 'alphabetical feature order'() { + runner.configurationScript = { + runner { + orderer new AlphabeticalSpecOrderer(false, true) + } + } + runner.extensionClasses << OrderExtension + + when: + runner.runClasses(specs) + def executions = executionsTL.get() + + then: + executions*.spec == ['FirstSpec', 'SecondSpec', 'ThirdSpec', 'FourthSpec'] + executions*.features == [ + ['one', 'three', 'two'], + ['bar', 'foo', 'zot'], + ['another feature', 'one more feature', 'some feature'], + ['feature D', 'feature M', 'feature X'] + ] + } + + def 'alphabetical spec and feature order'() { + runner.configurationScript = { + runner { + orderer new AlphabeticalSpecOrderer(true, true) + } + } + runner.extensionClasses << OrderExtension + + when: + runner.runClasses(specs) + def executions = executionsTL.get() + + then: + executions*.spec == ['FirstSpec', 'FourthSpec', 'SecondSpec', 'ThirdSpec'] + executions*.features == [ + ['one', 'three', 'two'], + ['feature D', 'feature M', 'feature X'], + ['bar', 'foo', 'zot'], + ['another feature', 'one more feature', 'some feature'] + ] + } + + def 'descending, alphabetical spec and feature order'() { + runner.configurationScript = { + runner { + orderer new AlphabeticalSpecOrderer(true, true, false) + } + } + runner.extensionClasses << OrderExtension + + when: + runner.runClasses(specs) + def executions = executionsTL.get() + + then: + executions*.spec == ['FirstSpec', 'FourthSpec', 'SecondSpec', 'ThirdSpec'].reverse() + executions*.features == [ + ['one', 'three', 'two'].reverse(), + ['feature D', 'feature M', 'feature X'].reverse(), + ['bar', 'foo', 'zot'].reverse(), + ['another feature', 'one more feature', 'some feature'].reverse() + ].reverse() + } + + def 'annotation-based spec and feature order'() { + runner.configurationScript = { + runner { + orderer new AnnotatationBasedSpecOrderer() + } + } + runner.extensionClasses << OrderExtension + + when: + runner.runClasses(specs) + def executions = executionsTL.get() + + then: + executions*.spec == ['SecondSpec', 'FirstSpec', 'ThirdSpec', 'FourthSpec'] + executions*.features == [ + ['zot', 'foo', 'bar'], + ['one', 'two', 'three'], + ['some feature', 'one more feature', 'another feature'], + ['feature D', 'feature X', 'feature M'] + ] + } + +}