[SUREFIRE-756] Allow ability to capture executed random runOrder for …

…re-play purposes
TestingResearchIllinois · Aug 4, 2020 · a669c19 · a669c19
1 parent 6e895b1
commit a669c19
Show file tree

Hide file tree

Showing 18 changed files with 231 additions and 11 deletions.
diff --git a/...n-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java b/...n-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java
@@ -310,6 +310,22 @@ public class IntegrationTestMojo
     @Parameter( property = "failsafe.runOrder", defaultValue = "filesystem" )
     private String runOrder;
 
+    /**
+     * Sets the random seed that will be used to order the tests if {@code failsafe.runOrder} is set to {@code random}.
+     * <br>
+     * <br>
+     * If no seeds are set and {@code failsafe.runOrder} is set to {@code random}, then the seed used will be
+     * outputted (search for "To reproduce ordering use flag -Dfailsafe.seed").
+     * <br>
+     * <br>
+     * To deterministically reproduce any random test order that was run before, simply set the seed to 
+     * be the same value.
+     *
+     * @since 3.0.0-M6
+     */
+    @Parameter( property = "failsafe.seed" )
+    private Long randomSeed;
+
     /**
      * A file containing include patterns, each in a next line. Blank lines, or lines starting with # are ignored.
      * If {@code includes} are also specified, these patterns are appended. Example with path, simple and regex
@@ -891,6 +907,18 @@ public void setRunOrder( String runOrder )
         this.runOrder = runOrder;
     }
 
+    @Override
+    public Long getRandomSeed()
+    {
+        return randomSeed;
+    }
+
+    @Override
+    public void setRandomSeed( Long randomSeed )
+    {
+        this.randomSeed = randomSeed;
+    }
+
     @Override
     public File getIncludesFile()
     {

diff --git a/...-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java b/...-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java
@@ -865,6 +865,10 @@ public abstract class AbstractSurefireMojo
 
     public abstract void setRunOrder( String runOrder );
 
+    public abstract Long getRandomSeed();
+
+    public abstract void setRandomSeed( Long seed );
+
     protected abstract void handleSummary( RunResult summary, Exception firstForkException )
         throws MojoExecutionException, MojoFailureException;
 
@@ -1129,6 +1133,7 @@ boolean verifyParameters()
             warnIfNotApplicableSkipAfterFailureCount();
             warnIfIllegalTempDir();
             warnIfForkCountIsZero();
+            printDefaultSeedIfNecessary();
         }
         return true;
     }
@@ -1285,7 +1290,7 @@ private RunResult executeProvider( @Nonnull ProviderInfo provider, @Nonnull Defa
         ClassLoaderConfiguration classLoaderConfiguration = getClassLoaderConfiguration();
         provider.addProviderProperties();
         RunOrderParameters runOrderParameters =
-            new RunOrderParameters( getRunOrder(), getStatisticsFile( getConfigChecksum() ) );
+            new RunOrderParameters( getRunOrder(), getStatisticsFile( getConfigChecksum() ), getRandomSeed() );
 
         if ( isNotForking() )
         {
@@ -3051,6 +3056,17 @@ private void warnIfIllegalTempDir() throws MojoFailureException
         }
     }
 
+    private void printDefaultSeedIfNecessary()
+    {
+        if ( getRandomSeed() == null && getRunOrder().equals( RunOrder.RANDOM.name() ) )
+        {
+            setRandomSeed( System.nanoTime() );
+            getConsoleLogger().info(
+                "Tests will run in random order. To reproduce ordering use flag -D"
+                    + getPluginName() + ".seed=" + getRandomSeed() );
+        }
+    }
+
     final class TestNgProviderInfo
         implements ProviderInfo
     {

diff --git a/...-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/BooterSerializer.java b/...-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/BooterSerializer.java
@@ -56,6 +56,7 @@
 import static org.apache.maven.surefire.booter.BooterConstants.PLUGIN_PID;
 import static org.apache.maven.surefire.booter.BooterConstants.PROCESS_CHECKER;
 import static org.apache.maven.surefire.booter.BooterConstants.PROVIDER_CONFIGURATION;
+import static org.apache.maven.surefire.booter.BooterConstants.RANDOM_SEED;
 import static org.apache.maven.surefire.booter.BooterConstants.REPORTSDIRECTORY;
 import static org.apache.maven.surefire.booter.BooterConstants.REQUESTEDTEST;
 import static org.apache.maven.surefire.booter.BooterConstants.RERUN_FAILING_TESTS_COUNT;
@@ -161,6 +162,7 @@ File serialize( KeyValueSource sourceProperties, ProviderConfiguration providerC
         {
             properties.setProperty( RUN_ORDER, RunOrder.asString( runOrderParameters.getRunOrder() ) );
             properties.setProperty( RUN_STATISTICS_FILE, runOrderParameters.getRunStatisticsFile() );
+            properties.setProperty( RANDOM_SEED, runOrderParameters.getRandomSeed() );
         }
 
         ReporterConfiguration reporterConfiguration = providerConfiguration.getReporterConfiguration();

diff --git a/...mon/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoJava7PlusTest.java b/...mon/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoJava7PlusTest.java
@@ -839,6 +839,18 @@ public void setRunOrder( String runOrder )
 
         }
 
+        @Override
+        public Long getRandomSeed()
+        {
+            return null;
+        }
+
+        @Override
+        public void setRandomSeed( Long seed )
+        {
+
+        }
+
         @Override
         protected void handleSummary( RunResult summary, Exception firstForkException )
         {

diff --git a/...efire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoTest.java b/...efire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoTest.java
@@ -2381,6 +2381,18 @@ public void setRunOrder( String runOrder )
 
         }
 
+        @Override
+        public Long getRandomSeed()
+        {
+            return null;
+        }
+
+        @Override
+        public void setRandomSeed( Long seed )
+        {
+
+        }
+
         @Override
         protected void handleSummary( RunResult summary, Exception firstForkException )
         {

diff --git a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/MojoMocklessTest.java b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/MojoMocklessTest.java
@@ -709,6 +709,18 @@ public void setRunOrder( String runOrder )
 
         }
 
+        @Override
+        public Long getRandomSeed()
+        {
+            return null;
+        }
+
+        @Override
+        public void setRandomSeed( Long seed )
+        {
+
+        }        
+
         @Override
         public String[] getDependenciesToScan()
         {

diff --git a/...pache/maven/plugin/surefire/booterclient/BooterDeserializerProviderConfigurationTest.java b/...pache/maven/plugin/surefire/booterclient/BooterDeserializerProviderConfigurationTest.java
@@ -276,7 +276,7 @@ private ProviderConfiguration getTestProviderConfiguration( DirectoryScannerPara
             new TestRequest( getSuiteXmlFileStrings(), getTestSourceDirectory(),
                              new TestListResolver( USER_REQUESTED_TEST + "#aUserRequestedTestMethod" ),
                     RERUN_FAILING_TEST_COUNT );
-        RunOrderParameters runOrderParameters = new RunOrderParameters( RunOrder.DEFAULT, null );
+        RunOrderParameters runOrderParameters = new RunOrderParameters( RunOrder.DEFAULT, null, null );
         return new ProviderConfiguration( directoryScannerParameters, runOrderParameters, true, reporterConfiguration,
                 new TestArtifactInfo( "5.0", "ABC" ), testSuiteDefinition, new HashMap<String, String>(), TEST_TYPED,
                 readTestsFromInStream, cli, 0, Shutdown.DEFAULT, 0 );

diff --git a/...apache/maven/plugin/surefire/booterclient/BooterDeserializerStartupConfigurationTest.java b/...apache/maven/plugin/surefire/booterclient/BooterDeserializerStartupConfigurationTest.java
@@ -197,7 +197,7 @@ private ProviderConfiguration getProviderConfiguration()
             new TestRequest( Arrays.asList( getSuiteXmlFileStrings() ), getTestSourceDirectory(),
                              new TestListResolver( "aUserRequestedTest#aUserRequestedTestMethod" ) );
 
-        RunOrderParameters runOrderParameters = new RunOrderParameters( RunOrder.DEFAULT, null );
+        RunOrderParameters runOrderParameters = new RunOrderParameters( RunOrder.DEFAULT, null, null );
         return new ProviderConfiguration( directoryScannerParameters, runOrderParameters, true, reporterConfiguration,
                 new TestArtifactInfo( "5.0", "ABC" ), testSuiteDefinition, new HashMap<String, String>(),
                 BooterDeserializerProviderConfigurationTest.TEST_TYPED, true, cli, 0, Shutdown.DEFAULT, 0 );

diff --git a/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java b/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java
@@ -292,6 +292,22 @@ public class SurefirePlugin
     @Parameter( property = "surefire.runOrder", defaultValue = "filesystem" )
     private String runOrder;
 
+    /**
+     * Sets the random seed that will be used to order the tests if {@code surefire.runOrder} is set to {@code random}.
+     * <br>
+     * <br>
+     * If no seeds are set and {@code surefire.runOrder} is set to {@code random}, then the seed used will be
+     * outputted (search for "To reproduce ordering use flag -Dsurefire.seed").
+     * <br>
+     * <br>
+     * To deterministically reproduce any random test order that was run before, simply set the seed to 
+     * be the same value.
+     *
+     * @since 3.0.0-M6
+     */
+    @Parameter( property = "surefire.seed" )
+    private Long randomSeed;
+
     /**
      * A file containing include patterns. Blank lines, or lines starting with # are ignored. If {@code includes} are
      * also specified, these patterns are appended. Example with path, simple and regex includes:
@@ -794,6 +810,18 @@ public void setRunOrder( String runOrder )
         this.runOrder = runOrder;
     }
 
+    @Override
+    public Long getRandomSeed()
+    {
+        return randomSeed;
+    }
+
+    @Override
+    public void setRandomSeed( Long randomSeed )
+    {
+        this.randomSeed = randomSeed;
+    }
+
     @Override
     public File getIncludesFile()
     {

diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/api/testset/RunOrderParameters.java b/surefire-api/src/main/java/org/apache/maven/surefire/api/testset/RunOrderParameters.java
@@ -31,28 +31,42 @@ public class RunOrderParameters
 
     private File runStatisticsFile;
 
-    public RunOrderParameters( RunOrder[] runOrder, File runStatisticsFile )
+    private Long randomSeed;
+
+    public RunOrderParameters( RunOrder[] runOrder, File runStatisticsFile, Long randomSeed )
     {
         this.runOrder = runOrder;
         this.runStatisticsFile = runStatisticsFile;
+        this.randomSeed = randomSeed;
     }
 
-    public RunOrderParameters( String runOrder, File runStatisticsFile )
+    public RunOrderParameters( String runOrder, File runStatisticsFile, Long randomSeed )
     {
         this.runOrder = runOrder == null ? RunOrder.DEFAULT : RunOrder.valueOfMulti( runOrder );
         this.runStatisticsFile = runStatisticsFile;
+        this.randomSeed = randomSeed;
     }
 
     public static RunOrderParameters alphabetical()
     {
-        return new RunOrderParameters( new RunOrder[]{ RunOrder.ALPHABETICAL }, null );
+        return new RunOrderParameters( new RunOrder[]{ RunOrder.ALPHABETICAL }, null, null );
     }
 
     public RunOrder[] getRunOrder()
     {
         return runOrder;
     }
 
+    public Long getRandomSeed()
+    {
+        return randomSeed;
+    }
+
+    public void setRandomSeed( long randomSeed )
+    {
+        this.randomSeed = randomSeed;
+    }
+
     public File getRunStatisticsFile()
     {
         return runStatisticsFile;

diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java b/surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java
@@ -28,6 +28,7 @@
 import java.util.Comparator;
 import java.util.LinkedHashSet;
 import java.util.List;
+import java.util.Random;
 
 /**
  * Applies the final runorder of the tests
@@ -45,12 +46,21 @@ public class DefaultRunOrderCalculator
 
     private final int threadCount;
 
+    private final Random random;
+
     public DefaultRunOrderCalculator( RunOrderParameters runOrderParameters, int threadCount )
     {
         this.runOrderParameters = runOrderParameters;
         this.threadCount = threadCount;
         this.runOrder = runOrderParameters.getRunOrder();
         this.sortOrder = this.runOrder.length > 0 ? getSortOrderComparator( this.runOrder[0] ) : null;
+        Long seed = runOrderParameters.getRandomSeed();
+        if ( seed == null )
+        {
+            seed = System.nanoTime();
+            runOrderParameters.setRandomSeed( seed );
+        }
+        this.random = new Random( seed );
     }
 
     @Override
@@ -72,7 +82,7 @@ private void orderTestClasses( List<Class<?>> testClasses, RunOrder runOrder )
     {
         if ( RunOrder.RANDOM.equals( runOrder ) )
         {
-            Collections.shuffle( testClasses );
+            Collections.shuffle( testClasses, random );
         }
         else if ( RunOrder.FAILEDFIRST.equals( runOrder ) )
         {

diff --git a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterConstants.java b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterConstants.java
@@ -46,6 +46,7 @@ private BooterConstants()
     public static final String SOURCE_DIRECTORY = "testSuiteDefinitionTestSourceDirectory";
     public static final String TEST_CLASSES_DIRECTORY = "testClassesDirectory";
     public static final String RUN_ORDER = "runOrder";
+    public static final String RANDOM_SEED = "randomSeed";
     public static final String RUN_STATISTICS_FILE = "runStatisticsFile";
     public static final String TEST_SUITE_XML_FILES = "testSuiteXmlFiles";
     public static final String PROVIDER_CONFIGURATION = "providerConfiguration";

diff --git a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterDeserializer.java b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterDeserializer.java
@@ -102,6 +102,7 @@ public ProviderConfiguration deserialize()
         final List<String> testSuiteXmlFiles = properties.getStringList( TEST_SUITE_XML_FILES );
         final File testClassesDirectory = properties.getFileProperty( TEST_CLASSES_DIRECTORY );
         final String runOrder = properties.getProperty( RUN_ORDER );
+        final Long randomSeed = properties.getLongProperty( RANDOM_SEED );
         final String runStatisticsFile = properties.getProperty( RUN_STATISTICS_FILE );
 
         final int rerunFailingTestsCount = properties.getIntProperty( RERUN_FAILING_TESTS_COUNT );
@@ -111,7 +112,8 @@ public ProviderConfiguration deserialize()
                                             properties.getBooleanProperty( FAILIFNOTESTS ), runOrder );
 
         RunOrderParameters runOrderParameters
-                = new RunOrderParameters( runOrder, runStatisticsFile == null ? null : new File( runStatisticsFile ) );
+                = new RunOrderParameters( runOrder, runStatisticsFile == null ? null : new File( runStatisticsFile ),
+                                          randomSeed );
 
         TestArtifactInfo testNg = new TestArtifactInfo( testNgVersion, testArtifactClassifier );
         TestRequest testSuiteDefinition =

diff --git a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/SurefireReflector.java b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/SurefireReflector.java
@@ -178,10 +178,11 @@ private Object createRunOrderParameters( RunOrderParameters runOrderParameters )
             return null;
         }
         //Can't use the constructor with the RunOrder parameter. Using it causes some integration tests to fail.
-        Class<?>[] arguments = { String.class, File.class };
+        Class<?>[] arguments = { String.class, File.class, Long.class };
         Constructor<?> constructor = getConstructor( this.runOrderParameters, arguments );
         File runStatisticsFile = runOrderParameters.getRunStatisticsFile();
-        return newInstance( constructor, RunOrder.asString( runOrderParameters.getRunOrder() ), runStatisticsFile );
+        return newInstance( constructor, RunOrder.asString( runOrderParameters.getRunOrder() ), runStatisticsFile,
+                            runOrderParameters.getRandomSeed() );
     }
 
     private Object createTestArtifactInfo( TestArtifactInfo testArtifactInfo )

diff --git a/surefire-booter/src/test/java/org/apache/maven/surefire/booter/SurefireReflectorTest.java b/surefire-booter/src/test/java/org/apache/maven/surefire/booter/SurefireReflectorTest.java
@@ -114,7 +114,7 @@ public void testRunOrderParameters()
         SurefireReflector surefireReflector = getReflector();
         Object foo = getFoo();
 
-        RunOrderParameters runOrderParameters = new RunOrderParameters( RunOrder.DEFAULT, new File( "." ) );
+        RunOrderParameters runOrderParameters = new RunOrderParameters( RunOrder.DEFAULT, new File( "." ), null );
         surefireReflector.setRunOrderParameters( foo, runOrderParameters );
         assertTrue( isCalled( foo ) );
     }