[SUREFIRE-756] Allow ability to capture executed random runOrder for …

…re-play purposes
TestingResearchIllinois · Aug 25, 2020 · 009bbbb · 009bbbb
1 parent 6e895b1
commit 009bbbb
Show file tree

Hide file tree

Showing 15 changed files with 239 additions and 5 deletions.
diff --git a/...n-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java b/...n-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java
@@ -310,6 +310,22 @@ public class IntegrationTestMojo
     @Parameter( property = "failsafe.runOrder", defaultValue = "filesystem" )
     private String runOrder;
 
+    /**
+     * Sets the random seed that will be used to order the tests if {@code failsafe.runOrder} is set to {@code random}.
+     * <br>
+     * <br>
+     * If no seeds are set and {@code failsafe.runOrder} is set to {@code random}, then the seed used will be
+     * outputted (search for "To reproduce ordering use flag -Dfailsafe.runOrder.random.seed").
+     * <br>
+     * <br>
+     * To deterministically reproduce any random test order that was run before, simply set the seed to 
+     * be the same value.
+     *
+     * @since 3.0.0-M6
+     */
+    @Parameter( property = "failsafe.runOrder.random.seed" )
+    private Long runOrderRandomSeed;
+
     /**
      * A file containing include patterns, each in a next line. Blank lines, or lines starting with # are ignored.
      * If {@code includes} are also specified, these patterns are appended. Example with path, simple and regex
@@ -891,6 +907,18 @@ public void setRunOrder( String runOrder )
         this.runOrder = runOrder;
     }
 
+    @Override
+    public Long getRunOrderRandomSeed()
+    {
+        return runOrderRandomSeed;
+    }
+
+    @Override
+    public void setRunOrderRandomSeed( Long runOrderRandomSeed )
+    {
+        this.runOrderRandomSeed = runOrderRandomSeed;
+    }
+
     @Override
     public File getIncludesFile()
     {

diff --git a/...-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java b/...-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java
@@ -865,6 +865,10 @@ public abstract class AbstractSurefireMojo
 
     public abstract void setRunOrder( String runOrder );
 
+    public abstract Long getRunOrderRandomSeed();
+
+    public abstract void setRunOrderRandomSeed( Long runOrderRandomSeed );
+
     protected abstract void handleSummary( RunResult summary, Exception firstForkException )
         throws MojoExecutionException, MojoFailureException;
 
@@ -1129,6 +1133,7 @@ boolean verifyParameters()
             warnIfNotApplicableSkipAfterFailureCount();
             warnIfIllegalTempDir();
             warnIfForkCountIsZero();
+            printDefaultSeedIfNecessary();
         }
         return true;
     }
@@ -1285,7 +1290,7 @@ private RunResult executeProvider( @Nonnull ProviderInfo provider, @Nonnull Defa
         ClassLoaderConfiguration classLoaderConfiguration = getClassLoaderConfiguration();
         provider.addProviderProperties();
         RunOrderParameters runOrderParameters =
-            new RunOrderParameters( getRunOrder(), getStatisticsFile( getConfigChecksum() ) );
+            new RunOrderParameters( getRunOrder(), getStatisticsFile( getConfigChecksum() ), getRunOrderRandomSeed() );
 
         if ( isNotForking() )
         {
@@ -3051,6 +3056,17 @@ private void warnIfIllegalTempDir() throws MojoFailureException
         }
     }
 
+    private void printDefaultSeedIfNecessary()
+    {
+        if ( getRunOrderRandomSeed() == null && getRunOrder().equals( RunOrder.RANDOM.name() ) )
+        {
+            setRunOrderRandomSeed( System.nanoTime() );
+            getConsoleLogger().info(
+                "Tests will run in random order. To reproduce ordering use flag -D"
+                    + getPluginName() + ".runOrder.random.seed=" + getRunOrderRandomSeed() );
+        }
+    }
+
     final class TestNgProviderInfo
         implements ProviderInfo
     {

diff --git a/...-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/BooterSerializer.java b/...-common/src/main/java/org/apache/maven/plugin/surefire/booterclient/BooterSerializer.java
@@ -56,6 +56,7 @@
 import static org.apache.maven.surefire.booter.BooterConstants.PLUGIN_PID;
 import static org.apache.maven.surefire.booter.BooterConstants.PROCESS_CHECKER;
 import static org.apache.maven.surefire.booter.BooterConstants.PROVIDER_CONFIGURATION;
+import static org.apache.maven.surefire.booter.BooterConstants.RUN_ORDER_RANDOM_SEED;
 import static org.apache.maven.surefire.booter.BooterConstants.REPORTSDIRECTORY;
 import static org.apache.maven.surefire.booter.BooterConstants.REQUESTEDTEST;
 import static org.apache.maven.surefire.booter.BooterConstants.RERUN_FAILING_TESTS_COUNT;
@@ -161,6 +162,7 @@ File serialize( KeyValueSource sourceProperties, ProviderConfiguration providerC
         {
             properties.setProperty( RUN_ORDER, RunOrder.asString( runOrderParameters.getRunOrder() ) );
             properties.setProperty( RUN_STATISTICS_FILE, runOrderParameters.getRunStatisticsFile() );
+            properties.setProperty( RUN_ORDER_RANDOM_SEED, runOrderParameters.getRunOrderRandomSeed() );
         }
 
         ReporterConfiguration reporterConfiguration = providerConfiguration.getReporterConfiguration();

diff --git a/...mon/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoJava7PlusTest.java b/...mon/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoJava7PlusTest.java
@@ -839,6 +839,18 @@ public void setRunOrder( String runOrder )
 
         }
 
+        @Override
+        public Long getRunOrderRandomSeed()
+        {
+            return null;
+        }
+
+        @Override
+        public void setRunOrderRandomSeed( Long runOrderRandomSeed )
+        {
+
+        }
+
         @Override
         protected void handleSummary( RunResult summary, Exception firstForkException )
         {

diff --git a/...efire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoTest.java b/...efire-common/src/test/java/org/apache/maven/plugin/surefire/AbstractSurefireMojoTest.java
@@ -2381,6 +2381,18 @@ public void setRunOrder( String runOrder )
 
         }
 
+        @Override
+        public Long getRunOrderRandomSeed()
+        {
+            return null;
+        }
+
+        @Override
+        public void setRunOrderRandomSeed( Long runOrderRandomSeed )
+        {
+
+        }
+
         @Override
         protected void handleSummary( RunResult summary, Exception firstForkException )
         {

diff --git a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/MojoMocklessTest.java b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/MojoMocklessTest.java
@@ -709,6 +709,18 @@ public void setRunOrder( String runOrder )
 
         }
 
+        @Override
+        public Long getRunOrderRandomSeed()
+        {
+            return null;
+        }
+
+        @Override
+        public void setRunOrderRandomSeed( Long runOrderRandomSeed )
+        {
+
+        }
+
         @Override
         public String[] getDependenciesToScan()
         {

diff --git a/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java b/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java
@@ -292,6 +292,22 @@ public class SurefirePlugin
     @Parameter( property = "surefire.runOrder", defaultValue = "filesystem" )
     private String runOrder;
 
+    /**
+     * Sets the random seed that will be used to order the tests if {@code surefire.runOrder} is set to {@code random}.
+     * <br>
+     * <br>
+     * If no seeds are set and {@code surefire.runOrder} is set to {@code random}, then the seed used will be
+     * outputted (search for "To reproduce ordering use flag -Dsurefire.runOrder.random.seed").
+     * <br>
+     * <br>
+     * To deterministically reproduce any random test order that was run before, simply set the seed to 
+     * be the same value.
+     *
+     * @since 3.0.0-M6
+     */
+    @Parameter( property = "surefire.runOrder.random.seed" )
+    private Long runOrderRandomSeed;
+
     /**
      * A file containing include patterns. Blank lines, or lines starting with # are ignored. If {@code includes} are
      * also specified, these patterns are appended. Example with path, simple and regex includes:
@@ -794,6 +810,18 @@ public void setRunOrder( String runOrder )
         this.runOrder = runOrder;
     }
 
+    @Override
+    public Long getRunOrderRandomSeed()
+    {
+        return runOrderRandomSeed;
+    }
+
+    @Override
+    public void setRunOrderRandomSeed( Long runOrderRandomSeed )
+    {
+        this.runOrderRandomSeed = runOrderRandomSeed;
+    }
+
     @Override
     public File getIncludesFile()
     {

diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/api/testset/RunOrderParameters.java b/surefire-api/src/main/java/org/apache/maven/surefire/api/testset/RunOrderParameters.java
@@ -31,16 +31,34 @@ public class RunOrderParameters
 
     private File runStatisticsFile;
 
+    private Long runOrderRandomSeed;
+
     public RunOrderParameters( RunOrder[] runOrder, File runStatisticsFile )
     {
         this.runOrder = runOrder;
         this.runStatisticsFile = runStatisticsFile;
+        this.runOrderRandomSeed = null;
     }
 
     public RunOrderParameters( String runOrder, File runStatisticsFile )
     {
         this.runOrder = runOrder == null ? RunOrder.DEFAULT : RunOrder.valueOfMulti( runOrder );
         this.runStatisticsFile = runStatisticsFile;
+        this.runOrderRandomSeed = null;
+    }
+
+    public RunOrderParameters( RunOrder[] runOrder, File runStatisticsFile, Long runOrderRandomSeed )
+    {
+        this.runOrder = runOrder;
+        this.runStatisticsFile = runStatisticsFile;
+        this.runOrderRandomSeed = runOrderRandomSeed;
+    }
+
+    public RunOrderParameters( String runOrder, File runStatisticsFile, Long runOrderRandomSeed )
+    {
+        this.runOrder = runOrder == null ? RunOrder.DEFAULT : RunOrder.valueOfMulti( runOrder );
+        this.runStatisticsFile = runStatisticsFile;
+        this.runOrderRandomSeed = runOrderRandomSeed;
     }
 
     public static RunOrderParameters alphabetical()
@@ -53,6 +71,16 @@ public RunOrder[] getRunOrder()
         return runOrder;
     }
 
+    public Long getRunOrderRandomSeed()
+    {
+        return runOrderRandomSeed;
+    }
+
+    public void setRunOrderRandomSeed( Long runOrderRandomSeed )
+    {
+        this.runOrderRandomSeed = runOrderRandomSeed;
+    }
+
     public File getRunStatisticsFile()
     {
         return runStatisticsFile;

diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java b/surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java
@@ -28,6 +28,7 @@
 import java.util.Comparator;
 import java.util.LinkedHashSet;
 import java.util.List;
+import java.util.Random;
 
 /**
  * Applies the final runorder of the tests
@@ -45,12 +46,21 @@ public class DefaultRunOrderCalculator
 
     private final int threadCount;
 
+    private final Random random;
+
     public DefaultRunOrderCalculator( RunOrderParameters runOrderParameters, int threadCount )
     {
         this.runOrderParameters = runOrderParameters;
         this.threadCount = threadCount;
         this.runOrder = runOrderParameters.getRunOrder();
         this.sortOrder = this.runOrder.length > 0 ? getSortOrderComparator( this.runOrder[0] ) : null;
+        Long runOrderRandomSeed = runOrderParameters.getRunOrderRandomSeed();
+        if ( runOrderRandomSeed == null )
+        {
+            runOrderRandomSeed = System.nanoTime();
+            runOrderParameters.setRunOrderRandomSeed( runOrderRandomSeed );
+        }
+        this.random = new Random( runOrderRandomSeed );
     }
 
     @Override
@@ -72,7 +82,7 @@ private void orderTestClasses( List<Class<?>> testClasses, RunOrder runOrder )
     {
         if ( RunOrder.RANDOM.equals( runOrder ) )
         {
-            Collections.shuffle( testClasses );
+            Collections.shuffle( testClasses, random );
         }
         else if ( RunOrder.FAILEDFIRST.equals( runOrder ) )
         {

diff --git a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterConstants.java b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterConstants.java
@@ -46,6 +46,7 @@ private BooterConstants()
     public static final String SOURCE_DIRECTORY = "testSuiteDefinitionTestSourceDirectory";
     public static final String TEST_CLASSES_DIRECTORY = "testClassesDirectory";
     public static final String RUN_ORDER = "runOrder";
+    public static final String RUN_ORDER_RANDOM_SEED = "runOrderRandomSeed";
     public static final String RUN_STATISTICS_FILE = "runStatisticsFile";
     public static final String TEST_SUITE_XML_FILES = "testSuiteXmlFiles";
     public static final String PROVIDER_CONFIGURATION = "providerConfiguration";

diff --git a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterDeserializer.java b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/BooterDeserializer.java
@@ -102,6 +102,7 @@ public ProviderConfiguration deserialize()
         final List<String> testSuiteXmlFiles = properties.getStringList( TEST_SUITE_XML_FILES );
         final File testClassesDirectory = properties.getFileProperty( TEST_CLASSES_DIRECTORY );
         final String runOrder = properties.getProperty( RUN_ORDER );
+        final Long runOrderRandomSeed = properties.getLongProperty( RUN_ORDER_RANDOM_SEED );
         final String runStatisticsFile = properties.getProperty( RUN_STATISTICS_FILE );
 
         final int rerunFailingTestsCount = properties.getIntProperty( RERUN_FAILING_TESTS_COUNT );
@@ -111,7 +112,8 @@ public ProviderConfiguration deserialize()
                                             properties.getBooleanProperty( FAILIFNOTESTS ), runOrder );
 
         RunOrderParameters runOrderParameters
-                = new RunOrderParameters( runOrder, runStatisticsFile == null ? null : new File( runStatisticsFile ) );
+                = new RunOrderParameters( runOrder, runStatisticsFile == null ? null : new File( runStatisticsFile ),
+                                          runOrderRandomSeed );
 
         TestArtifactInfo testNg = new TestArtifactInfo( testNgVersion, testArtifactClassifier );
         TestRequest testSuiteDefinition =

diff --git a/surefire-booter/src/main/java/org/apache/maven/surefire/booter/SurefireReflector.java b/surefire-booter/src/main/java/org/apache/maven/surefire/booter/SurefireReflector.java
@@ -178,10 +178,11 @@ private Object createRunOrderParameters( RunOrderParameters runOrderParameters )
             return null;
         }
         //Can't use the constructor with the RunOrder parameter. Using it causes some integration tests to fail.
-        Class<?>[] arguments = { String.class, File.class };
+        Class<?>[] arguments = { String.class, File.class, Long.class };
         Constructor<?> constructor = getConstructor( this.runOrderParameters, arguments );
         File runStatisticsFile = runOrderParameters.getRunStatisticsFile();
-        return newInstance( constructor, RunOrder.asString( runOrderParameters.getRunOrder() ), runStatisticsFile );
+        return newInstance( constructor, RunOrder.asString( runOrderParameters.getRunOrder() ), runStatisticsFile,
+                            runOrderParameters.getRunOrderRandomSeed() );
     }
 
     private Object createTestArtifactInfo( TestArtifactInfo testArtifactInfo )