diff --git a/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java b/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java index 332c31169f..d77b3cfb70 100644 --- a/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java +++ b/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java @@ -310,6 +310,22 @@ public class IntegrationTestMojo @Parameter( property = "failsafe.runOrder", defaultValue = "filesystem" ) private String runOrder; + /** + * Sets the random seed that will be used to order the tests if {@code failsafe.runOrder} is set to {@code random}. + *
+ *
+ * If no seeds are set and {@code failsafe.runOrder} is set to {@code random}, then the seed used will be + * outputted (search for "To reproduce ordering use flag -Dfailsafe.runOrder.random.seed"). + *
+ *
+ * To deterministically reproduce any random test order that was run before, simply set the seed to + * be the same value. + * + * @since 3.0.0-M6 + */ + @Parameter( property = "failsafe.runOrder.random.seed" ) + private Long runOrderRandomSeed; + /** * A file containing include patterns, each in a next line. Blank lines, or lines starting with # are ignored. * If {@code includes} are also specified, these patterns are appended. Example with path, simple and regex @@ -891,6 +907,18 @@ public void setRunOrder( String runOrder ) this.runOrder = runOrder; } + @Override + public Long getRunOrderRandomSeed() + { + return runOrderRandomSeed; + } + + @Override + public void setRunOrderRandomSeed( Long runOrderRandomSeed ) + { + this.runOrderRandomSeed = runOrderRandomSeed; + } + @Override public File getIncludesFile() { diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java index 64975b4fc6..5c2eea3d39 100644 --- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java +++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java @@ -865,6 +865,10 @@ public abstract class AbstractSurefireMojo public abstract void setRunOrder( String runOrder ); + public abstract Long getRunOrderRandomSeed(); + + public abstract void setRunOrderRandomSeed( Long runOrderRandomSeed ); + protected abstract void handleSummary( RunResult summary, Exception firstForkException ) throws MojoExecutionException, MojoFailureException; @@ -1129,6 +1133,7 @@ boolean verifyParameters() warnIfNotApplicableSkipAfterFailureCount(); warnIfIllegalTempDir(); warnIfForkCountIsZero(); + printDefaultSeedIfNecessary(); } return true; } @@ -1285,7 +1290,7 @@ private RunResult executeProvider( @Nonnull ProviderInfo provider, @Nonnull Defa ClassLoaderConfiguration classLoaderConfiguration = getClassLoaderConfiguration(); provider.addProviderProperties(); RunOrderParameters runOrderParameters = - new RunOrderParameters( getRunOrder(), getStatisticsFile( getConfigChecksum() ) ); + new RunOrderParameters( getRunOrder(), getStatisticsFile( getConfigChecksum() ), getRunOrderRandomSeed() ); if ( isNotForking() ) { @@ -3051,6 +3056,17 @@ private void warnIfIllegalTempDir() throws MojoFailureException } } + private void printDefaultSeedIfNecessary() + { + if ( getRunOrderRandomSeed() == null && getRunOrder().equals( RunOrder.RANDOM.name() ) ) + { + setRunOrderRandomSeed( System.nanoTime() ); + getConsoleLogger().info( + "Tests will run in random order. To reproduce ordering use flag -D" + + getPluginName() + ".runOrder.random.seed=" + getRunOrderRandomSeed() ); + } + } + final class TestNgProviderInfo implements ProviderInfo { diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/BooterSerializer.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/BooterSerializer.java index c8aee3c049..1e00f16f72 100644 --- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/BooterSerializer.java +++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/BooterSerializer.java @@ -56,6 +56,7 @@ import static org.apache.maven.surefire.booter.BooterConstants.PLUGIN_PID; import static org.apache.maven.surefire.booter.BooterConstants.PROCESS_CHECKER; import static org.apache.maven.surefire.booter.BooterConstants.PROVIDER_CONFIGURATION; +import static org.apache.maven.surefire.booter.BooterConstants.RUN_ORDER_RANDOM_SEED; import static org.apache.maven.surefire.booter.BooterConstants.REPORTSDIRECTORY; import static org.apache.maven.surefire.booter.BooterConstants.REQUESTEDTEST; import static org.apache.maven.surefire.booter.BooterConstants.RERUN_FAILING_TESTS_COUNT; @@ -161,6 +162,7 @@ File serialize( KeyValueSource sourceProperties, ProviderConfiguration providerC { properties.setProperty( RUN_ORDER, RunOrder.asString( runOrderParameters.getRunOrder() ) ); properties.setProperty( RUN_STATISTICS_FILE, runOrderParameters.getRunStatisticsFile() ); + properties.setProperty( RUN_ORDER_RANDOM_SEED, runOrderParameters.getRunOrderRandomSeed() ); } ReporterConfiguration reporterConfiguration = providerConfiguration.getReporterConfiguration(); diff --git a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoJava7PlusTest.java b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoJava7PlusTest.java index 7d5343c7ae..c9dbf5cde6 100644 --- a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoJava7PlusTest.java +++ b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoJava7PlusTest.java @@ -839,6 +839,18 @@ public void setRunOrder( String runOrder ) } + @Override + public Long getRunOrderRandomSeed() + { + return null; + } + + @Override + public void setRunOrderRandomSeed( Long runOrderRandomSeed ) + { + + } + @Override protected void handleSummary( RunResult summary, Exception firstForkException ) { diff --git a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoTest.java b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoTest.java index a08224cae2..06e52be8c1 100644 --- a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoTest.java +++ b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoTest.java @@ -2381,6 +2381,18 @@ public void setRunOrder( String runOrder ) } + @Override + public Long getRunOrderRandomSeed() + { + return null; + } + + @Override + public void setRunOrderRandomSeed( Long runOrderRandomSeed ) + { + + } + @Override protected void handleSummary( RunResult summary, Exception firstForkException ) { diff --git a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/MojoMocklessTest.java b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/MojoMocklessTest.java index afcb64201e..1c9b89f6ae 100644 --- a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/MojoMocklessTest.java +++ b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/MojoMocklessTest.java @@ -709,6 +709,18 @@ public void setRunOrder( String runOrder ) } + @Override + public Long getRunOrderRandomSeed() + { + return null; + } + + @Override + public void setRunOrderRandomSeed( Long runOrderRandomSeed ) + { + + } + @Override public String[] getDependenciesToScan() { diff --git a/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java b/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java index f2b4fdcd85..34ed33ceba 100644 --- a/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java +++ b/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java @@ -292,6 +292,22 @@ public class SurefirePlugin @Parameter( property = "surefire.runOrder", defaultValue = "filesystem" ) private String runOrder; + /** + * Sets the random seed that will be used to order the tests if {@code surefire.runOrder} is set to {@code random}. + *
+ *
+ * If no seeds are set and {@code surefire.runOrder} is set to {@code random}, then the seed used will be + * outputted (search for "To reproduce ordering use flag -Dsurefire.runOrder.random.seed"). + *
+ *
+ * To deterministically reproduce any random test order that was run before, simply set the seed to + * be the same value. + * + * @since 3.0.0-M6 + */ + @Parameter( property = "surefire.runOrder.random.seed" ) + private Long runOrderRandomSeed; + /** * A file containing include patterns. Blank lines, or lines starting with # are ignored. If {@code includes} are * also specified, these patterns are appended. Example with path, simple and regex includes: @@ -794,6 +810,18 @@ public void setRunOrder( String runOrder ) this.runOrder = runOrder; } + @Override + public Long getRunOrderRandomSeed() + { + return runOrderRandomSeed; + } + + @Override + public void setRunOrderRandomSeed( Long runOrderRandomSeed ) + { + this.runOrderRandomSeed = runOrderRandomSeed; + } + @Override public File getIncludesFile() { diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/api/testset/RunOrderParameters.java b/surefire-api/src/main/java/org/apache/maven/surefire/api/testset/RunOrderParameters.java index 1fd2c3f66b..07e3b3ea98 100644 --- a/surefire-api/src/main/java/org/apache/maven/surefire/api/testset/RunOrderParameters.java +++ b/surefire-api/src/main/java/org/apache/maven/surefire/api/testset/RunOrderParameters.java @@ -31,16 +31,34 @@ public class RunOrderParameters private File runStatisticsFile; + private Long runOrderRandomSeed; + public RunOrderParameters( RunOrder[] runOrder, File runStatisticsFile ) { this.runOrder = runOrder; this.runStatisticsFile = runStatisticsFile; + this.runOrderRandomSeed = null; } public RunOrderParameters( String runOrder, File runStatisticsFile ) { this.runOrder = runOrder == null ? RunOrder.DEFAULT : RunOrder.valueOfMulti( runOrder ); this.runStatisticsFile = runStatisticsFile; + this.runOrderRandomSeed = null; + } + + public RunOrderParameters( RunOrder[] runOrder, File runStatisticsFile, Long runOrderRandomSeed ) + { + this.runOrder = runOrder; + this.runStatisticsFile = runStatisticsFile; + this.runOrderRandomSeed = runOrderRandomSeed; + } + + public RunOrderParameters( String runOrder, File runStatisticsFile, Long runOrderRandomSeed ) + { + this.runOrder = runOrder == null ? RunOrder.DEFAULT : RunOrder.valueOfMulti( runOrder ); + this.runStatisticsFile = runStatisticsFile; + this.runOrderRandomSeed = runOrderRandomSeed; } public static RunOrderParameters alphabetical() @@ -53,6 +71,16 @@ public RunOrder[] getRunOrder() return runOrder; } + public Long getRunOrderRandomSeed() + { + return runOrderRandomSeed; + } + + public void setRunOrderRandomSeed( Long runOrderRandomSeed ) + { + this.runOrderRandomSeed = runOrderRandomSeed; + } + public File getRunStatisticsFile() { return runStatisticsFile; diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java b/surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java index c897bb7815..8ced06afa5 100644 --- a/surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java +++ b/surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java @@ -28,6 +28,7 @@ import java.util.Comparator; import java.util.LinkedHashSet; import java.util.List; +import java.util.Random; /** * Applies the final runorder of the tests @@ -45,12 +46,21 @@ public class DefaultRunOrderCalculator private final int threadCount; + private final Random random; + public DefaultRunOrderCalculator( RunOrderParameters runOrderParameters, int threadCount ) { this.runOrderParameters = runOrderParameters; this.threadCount = threadCount; this.runOrder = runOrderParameters.getRunOrder(); this.sortOrder = this.runOrder.length > 0 ? getSortOrderComparator( this.runOrder[0] ) : null; + Long runOrderRandomSeed = runOrderParameters.getRunOrderRandomSeed(); + if ( runOrderRandomSeed == null ) + { + runOrderRandomSeed = System.nanoTime(); + runOrderParameters.setRunOrderRandomSeed( runOrderRandomSeed ); + } + this.random = new Random( runOrderRandomSeed ); } @Override @@ -72,7 +82,7 @@ private void orderTestClasses( List> testClasses, RunOrder runOrder ) { if ( RunOrder.RANDOM.equals( runOrder ) ) { - Collections.shuffle( testClasses ); + Collections.shuffle( testClasses, random ); } else if ( RunOrder.FAILEDFIRST.equals( runOrder ) ) { diff --git a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterConstants.java b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterConstants.java index fa664beac5..ae950df4f8 100644 --- a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterConstants.java +++ b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterConstants.java @@ -46,6 +46,7 @@ private BooterConstants() public static final String SOURCE_DIRECTORY = "testSuiteDefinitionTestSourceDirectory"; public static final String TEST_CLASSES_DIRECTORY = "testClassesDirectory"; public static final String RUN_ORDER = "runOrder"; + public static final String RUN_ORDER_RANDOM_SEED = "runOrderRandomSeed"; public static final String RUN_STATISTICS_FILE = "runStatisticsFile"; public static final String TEST_SUITE_XML_FILES = "testSuiteXmlFiles"; public static final String PROVIDER_CONFIGURATION = "providerConfiguration"; diff --git a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterDeserializer.java b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterDeserializer.java index 4ec8cec4ed..6a6cfaef23 100644 --- a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterDeserializer.java +++ b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterDeserializer.java @@ -102,6 +102,7 @@ public ProviderConfiguration deserialize() final List testSuiteXmlFiles = properties.getStringList( TEST_SUITE_XML_FILES ); final File testClassesDirectory = properties.getFileProperty( TEST_CLASSES_DIRECTORY ); final String runOrder = properties.getProperty( RUN_ORDER ); + final Long runOrderRandomSeed = properties.getLongProperty( RUN_ORDER_RANDOM_SEED ); final String runStatisticsFile = properties.getProperty( RUN_STATISTICS_FILE ); final int rerunFailingTestsCount = properties.getIntProperty( RERUN_FAILING_TESTS_COUNT ); @@ -111,7 +112,8 @@ public ProviderConfiguration deserialize() properties.getBooleanProperty( FAILIFNOTESTS ), runOrder ); RunOrderParameters runOrderParameters - = new RunOrderParameters( runOrder, runStatisticsFile == null ? null : new File( runStatisticsFile ) ); + = new RunOrderParameters( runOrder, runStatisticsFile == null ? null : new File( runStatisticsFile ), + runOrderRandomSeed ); TestArtifactInfo testNg = new TestArtifactInfo( testNgVersion, testArtifactClassifier ); TestRequest testSuiteDefinition = diff --git a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/SurefireReflector.java b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/SurefireReflector.java index 7f4c355996..b10b322792 100644 --- a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/SurefireReflector.java +++ b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/SurefireReflector.java @@ -178,10 +178,11 @@ private Object createRunOrderParameters( RunOrderParameters runOrderParameters ) return null; } //Can't use the constructor with the RunOrder parameter. Using it causes some integration tests to fail. - Class[] arguments = { String.class, File.class }; + Class[] arguments = { String.class, File.class, Long.class }; Constructor constructor = getConstructor( this.runOrderParameters, arguments ); File runStatisticsFile = runOrderParameters.getRunStatisticsFile(); - return newInstance( constructor, RunOrder.asString( runOrderParameters.getRunOrder() ), runStatisticsFile ); + return newInstance( constructor, RunOrder.asString( runOrderParameters.getRunOrder() ), runStatisticsFile, + runOrderParameters.getRunOrderRandomSeed() ); } private Object createTestArtifactInfo( TestArtifactInfo testArtifactInfo ) diff --git a/surefire-booter/src/test/java/org/apache/maven/surefire/booter/SurefireReflectorTest.java b/surefire-booter/src/test/java/org/apache/maven/surefire/booter/SurefireReflectorTest.java index d236d6ab25..44e462714f 100644 --- a/surefire-booter/src/test/java/org/apache/maven/surefire/booter/SurefireReflectorTest.java +++ b/surefire-booter/src/test/java/org/apache/maven/surefire/booter/SurefireReflectorTest.java @@ -119,6 +119,20 @@ public void testRunOrderParameters() assertTrue( isCalled( foo ) ); } + public void testRunOrderParametersWithRunOrderRandomSeed() + { + SurefireReflector surefireReflector = getReflector(); + Object foo = getFoo(); + + // Arbitrary random seed that should be ignored because RunOrder is not RANDOM + Long runOrderRandomSeed = 5L; + + RunOrderParameters runOrderParameters = new RunOrderParameters( RunOrder.DEFAULT, new File( "." ), + runOrderRandomSeed ); + surefireReflector.setRunOrderParameters( foo, runOrderParameters ); + assertTrue( isCalled( foo ) ); + } + public void testNullRunOrderParameters() { SurefireReflector surefireReflector = getReflector(); diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/RunOrderIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/RunOrderIT.java index 430c9bf714..0032e8de74 100644 --- a/surefire-its/src/test/java/org/apache/maven/surefire/its/RunOrderIT.java +++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/RunOrderIT.java @@ -19,6 +19,7 @@ * under the License. */ +import java.util.Arrays; import java.util.Calendar; import org.apache.maven.it.VerificationException; import org.apache.maven.surefire.its.fixture.OutputValidator; @@ -60,6 +61,43 @@ public void testAlphabeticalJUnit5() assertTestnamesAppearInSpecificOrder( validator, TESTS_IN_ALPHABETICAL_ORDER ); } + @Test + public void testRandomJUnit4DifferentSeed() + throws Exception + { + long seed = 0L; + OutputValidator validator = executeWithRandomOrder( "junit4", seed ); + String[] expected = validator.getStringsOrderInLog( TESTS_IN_ALPHABETICAL_ORDER ); + for ( long i = seed; i < 5 + seed; i++ ) + { + OutputValidator validator2 = executeWithRandomOrder( "junit4", i ); + String[] observed = validator2.getStringsOrderInLog( TESTS_IN_ALPHABETICAL_ORDER ); + if ( ! Arrays.equals( expected, observed ) ) + { + return; + } + } + throw new VerificationException( "All random orders with the different seeds produced the same orders" ); + } + + @Test + public void testRandomJUnit4SameSeed() + throws Exception + { + long seed = 0L; + OutputValidator validator = executeWithRandomOrder( "junit4", seed ); + String[] expected = validator.getStringsOrderInLog( TESTS_IN_ALPHABETICAL_ORDER ); + for ( long i = 0; i < 5; i++ ) + { + OutputValidator validator2 = executeWithRandomOrder( "junit4", seed ); + String[] observed = validator2.getStringsOrderInLog( TESTS_IN_ALPHABETICAL_ORDER ); + if ( ! Arrays.equals( expected, observed ) ) + { + throw new VerificationException( "Random orders with the same seed produced different orders" ); + } + } + } + @Test public void testReverseAlphabeticalJUnit4() throws Exception @@ -149,6 +187,17 @@ private OutputValidator executeWithRunOrder( String runOrder, String profile ) .verifyErrorFree( 3 ); } + private OutputValidator executeWithRandomOrder( String profile, long seed ) + { + return unpack() + .activateProfile( profile ) + .forkMode( getForkMode() ) + .runOrder( "random" ) + .runOrderRandomSeed( String.valueOf( seed ) ) + .executeTest() + .verifyErrorFree( 3 ); + } + protected String getForkMode() { return "once"; diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/OutputValidator.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/OutputValidator.java index d36c6a6065..14b360ca62 100644 --- a/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/OutputValidator.java +++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/OutputValidator.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.nio.charset.Charset; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import org.apache.commons.io.FileUtils; @@ -223,6 +224,32 @@ public File getBaseDir() return baseDir; } + public String[] getStringsOrderInLog( String[] strings ) + throws VerificationException + { + String[] retArr = new String[strings.length]; + List strList = new ArrayList( Arrays.asList( strings ) ); + int i = 0; + for ( String line : loadLogLines() ) + { + for ( int j = 0; j < strList.size(); j++ ) + { + if ( line.startsWith( strList.get( j ) ) ) + { + retArr[i] = strList.get( j ); + ++i; + if ( i == strings.length ) + { + return retArr; + } + strList.remove( j ); + break; + } + } + } + return retArr; + } + public boolean stringsAppearInSpecificOrderInLog( String[] strings ) throws VerificationException { diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/SurefireLauncher.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/SurefireLauncher.java index a5970d4cf4..03421725b4 100755 --- a/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/SurefireLauncher.java +++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/fixture/SurefireLauncher.java @@ -289,6 +289,12 @@ public SurefireLauncher runOrder( String runOrder ) return this; } + public SurefireLauncher runOrderRandomSeed( String runOrderRandomSeed ) + { + mavenLauncher.sysProp( "surefire.runOrder.random.seed", runOrderRandomSeed ); + return this; + } + public SurefireLauncher failIfNoTests( boolean fail ) { mavenLauncher.sysProp( "failIfNoTests", fail );