Support for test method ordering using only runOrder configuration

surefire-api/src/main/java/org/apache/maven/surefire/api/testset/TestListResolver.java

@@ -22,7 +22,9 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.LinkedHashSet;
+import java.util.Map;
 import java.util.Set;
 
 import static java.util.Collections.unmodifiableSet;
@@ -61,6 +63,8 @@ public class TestListResolver
 
     private final boolean hasExcludedMethodPatterns;
 
+    private final Map<String, Integer> patternMapper = new HashMap<String, Integer>();
+
     public TestListResolver( Collection<String> tests )
     {
         final IncludedExcludedPatterns patterns = new IncludedExcludedPatterns();
@@ -208,6 +212,7 @@ public boolean shouldRun( String testClassFile, String methodName )
         else
         {
             boolean shouldRun = false;
+            ResolvedTest matchedFilter = null;
 
             if ( getIncludedPatterns().isEmpty() )
             {
@@ -220,6 +225,7 @@ public boolean shouldRun( String testClassFile, String methodName )
                     if ( filter.matchAsInclusive( testClassFile, methodName ) )
                     {
                         shouldRun = true;
+                        matchedFilter = filter;
                         break;
                     }
                 }
@@ -236,6 +242,15 @@ public boolean shouldRun( String testClassFile, String methodName )
                     }
                 }
             }
+
+            if ( shouldRun )
+            {
+                String test = testClassFile + "#" + methodName;
+                if ( ! this.patternMapper.containsKey( test ) )
+                {
+                    this.patternMapper.put( test, new ArrayList<>( this.includedPatterns ).indexOf( matchedFilter ) );
+                }
+            }
             return shouldRun;
         }
     }
@@ -514,4 +529,31 @@ private static boolean haveMethodPatterns( Set<ResolvedTest> patterns )
         }
         return false;
     }
+
+    public Integer testOrderComparator( String className1, String className2, String methodName1, String methodName2 )
+    {
+        String classFileName1 = toClassFileName( className1 );
+        String classFileName2 = toClassFileName( className2 );
+        boolean shouldRunMethodName1 = shouldRun( classFileName1 , methodName1 );
+        boolean shouldRunMethodName2 = shouldRun( classFileName2 , methodName2 );
+        if ( ! shouldRunMethodName1 )
+        {
+            return -1;
+        }
+        if ( ! shouldRunMethodName2 )
+        {
+            return 1;
+        }
+
+        String test1 = classFileName1 + "#" + methodName1;
+        String test2 = classFileName2 + "#" + methodName2;
+        if ( patternMapper.get( test1 ) < patternMapper.get( test2 ) )
+        {
+            return -1;
+        }
+        else
+        {
+            return 1;
+        }
+    }
 }

surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java

@@ -21,12 +21,21 @@
 
 import org.apache.maven.surefire.api.runorder.RunEntryStatisticsMap;
 import org.apache.maven.surefire.api.testset.RunOrderParameters;
+import org.apache.maven.surefire.api.testset.TestListResolver;
 
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.nio.file.Files;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Calendar;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Random;
 
@@ -48,6 +57,8 @@ public class DefaultRunOrderCalculator
 
     private final Random random;
 
+    private final TestListResolver testListResolver;
+
     public DefaultRunOrderCalculator( RunOrderParameters runOrderParameters, int threadCount )
     {
         this.runOrderParameters = runOrderParameters;
@@ -61,6 +72,14 @@ public DefaultRunOrderCalculator( RunOrderParameters runOrderParameters, int thr
             runOrderParameters.setRunOrderRandomSeed( runOrderRandomSeed );
         }
         this.random = new Random( runOrderRandomSeed );
+        if ( RunOrder.TESTORDER.equals( getRunOrderMethod() ) )
+        {
+            this.testListResolver = getTestListResolver();
+        }
+        else
+        {
+            this.testListResolver = null;
+        }
     }
 
     @Override
@@ -78,9 +97,80 @@ public TestsToRun orderTestClasses( TestsToRun scannedClasses )
         return new TestsToRun( new LinkedHashSet<>( result ) );
     }
 
+    @Override
+    public Comparator<String> comparatorForTestMethods()
+    {
+        RunOrder methodRunOrder = getRunOrderMethod();
+        if ( RunOrder.TESTORDER.equals( methodRunOrder ) )
+        {
+            return new Comparator<String>()
+            {
+                @Override
+                public int compare( String o1, String o2 )
+                {
+                    String[] classAndMethod1 = getClassAndMethod( o1 );
+                    String className1 = classAndMethod1[0];
+                    String methodName1 = classAndMethod1[1];
+                    String[] classAndMethod2 = getClassAndMethod( o2 );
+                    String className2 = classAndMethod2[0];
+                    String methodName2 = classAndMethod2[1];
+                    return testListResolver.testOrderComparator( className1, className2, methodName1, methodName2 );
+                }
+            };
+        }
+        else
+        {
+            return null;
+        }
+    }
+
+    public TestListResolver getTestListResolver()
+    {
+        String orderParam = parseTestOrder( System.getProperty( "test" ) );
+        if ( orderParam == null  )
+        {
+            throw new IllegalStateException( "TestListResolver in RunOrderCalculator should be used only when "
+                    + "system property -Dtest is set and runOrder is testorder" );
+        }
+        return new TestListResolver( Arrays.asList( orderParam.split( "," ) ) );
+    }
+
+    public String[] getClassAndMethod( String request )
+    {
+        String[] classAndMethod = { request, request };
+        if ( request.contains( "(" ) )
+        {
+            String[] nameSplit1 = request.split( "\\(" );
+            classAndMethod[0] = nameSplit1[1].substring( 0, nameSplit1[1].length() - 1 );
+            classAndMethod[1] = nameSplit1[0];
+        }
+        return classAndMethod;
+    }
+
+    private RunOrder getRunOrderMethod()
+    {
+        if ( runOrder.length > 1 && Arrays.asList( runOrder ).contains( RunOrder.TESTORDER ) )
+        {
+            // Use of testorder and other runOrders are currently not supported
+            throw new IllegalStateException( "Expected only testorder. Got: " + runOrder.length );
+        }
+        return runOrder[0];
+    }
+
     private void orderTestClasses( List<Class<?>> testClasses, RunOrder runOrder )
     {
-        if ( RunOrder.RANDOM.equals( runOrder ) )
+        if ( RunOrder.TESTORDER.equals( runOrder ) )
+        {
+            Collections.sort( testClasses, new Comparator<Class<?>>()
+                    {
+                        @Override
+                        public int compare( Class<?> o1, Class<?> o2 )
+                        {
+                            return testListResolver.testOrderComparator( o1.getName(), o2.getName(), null, null );
+                        }
+                    } );
+        }
+        else if ( RunOrder.RANDOM.equals( runOrder ) )
         {
             Collections.shuffle( testClasses, random );
         }
@@ -106,6 +196,33 @@ else if ( sortOrder != null )
         }
     }
 
+    private String parseTestOrder( String s )
+    {
+        // if s is a file, then parse each line of the file as a test
+        if ( s != null && s != "" )
+        {
+            File f = new File( s );
+            if ( f.exists() && !f.isDirectory ( ) )
+            {
+                try
+                {
+                    List<String> l = Files.readAllLines( f.toPath(), Charset.defaultCharset( ) );
+                    StringBuilder sb = new StringBuilder( );
+                    for ( String sd : l )
+                    {
+                        sb.append( sd + "," );
+                    }
+                    String sd = sb.toString( );
+                    return sd.substring( 0 , sd.length( ) - 1 );
+                }
+                catch ( IOException e )
+                {
+                }
+            }
+        }
+        return s;
+    }
+
     private Comparator<Class> getSortOrderComparator( RunOrder runOrder )
     {
         if ( RunOrder.ALPHABETICAL.equals( runOrder ) )

surefire-api/src/main/java/org/apache/maven/surefire/api/util/RunOrder.java

@@ -44,6 +44,8 @@ public class RunOrder
 
     public static final RunOrder FAILEDFIRST = new RunOrder( "failedfirst" );
 
+    public static final RunOrder TESTORDER = new RunOrder( "testorder" );
+
     public static final RunOrder[] DEFAULT = new RunOrder[]{ FILESYSTEM };
 
     /**
@@ -108,7 +110,8 @@ private static String createMessageForMissingRunOrder( String name )
 
     private static RunOrder[] values()
     {
-        return new RunOrder[]{ ALPHABETICAL, FILESYSTEM, HOURLY, RANDOM, REVERSE_ALPHABETICAL, BALANCED, FAILEDFIRST };
+        return new RunOrder[]{ ALPHABETICAL, FILESYSTEM, HOURLY, RANDOM, REVERSE_ALPHABETICAL, BALANCED, FAILEDFIRST,
+                               TESTORDER };
     }
 
     public static String asString( RunOrder[] runOrder )

surefire-api/src/main/java/org/apache/maven/surefire/api/util/RunOrderCalculator.java

@@ -19,10 +19,14 @@
  * under the License.
  */
 
+import java.util.Comparator;
+
 /**
  * @author Kristian Rosenvold
  */
 public interface RunOrderCalculator
 {
     TestsToRun orderTestClasses( TestsToRun scannedClasses );
+
+    Comparator<String> comparatorForTestMethods();
 }

surefire-api/src/test/java/org/apache/maven/surefire/api/testset/TestListResolverTest.java

@@ -27,6 +27,7 @@
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.LinkedHashSet;
+import java.util.List;
 import java.util.Set;
 
 import static java.util.Collections.addAll;
@@ -502,4 +503,65 @@ private static Set<ResolvedTest> resolveClass( String patterns )
         }
         return resolved;
     }
+
+    public void testOrderComparatorTest()
+    {
+        List<String> orderParamList = new ArrayList<String>();
+        orderParamList.add( "TestClass1#testa2d" );
+        orderParamList.add( "TestClass1#testabc" );
+        orderParamList.add( "TestClass1#testa1b" );
+        orderParamList.add( "TestClass2#testa1b" );
+        orderParamList.add( "TestClass2#testaBc" );
+        TestListResolver tlr = new TestListResolver( orderParamList );
+        String className = "TestClass1";
+        String className2 = "TestClass2";
+        assertEquals( ( int ) tlr.testOrderComparator( className, className, "testa2d", "testa1b" ), -1 );
+        assertEquals( ( int ) tlr.testOrderComparator( className, className, "testa2d", "testabc" ), -1 );
+        assertEquals( ( int ) tlr.testOrderComparator( className, className, "testa1b", "testabc" ), 1 );
+        assertEquals( ( int ) tlr.testOrderComparator( className, className, "testa2d", "testaBc" ), 1 );
+        assertEquals( ( int ) tlr.testOrderComparator( className, className, "testa3d", "testa1b" ), -1 );
+        assertEquals( ( int ) tlr.testOrderComparator( className, className2, "testa2d", "testa1b" ), -1 );
+        assertEquals( ( int ) tlr.testOrderComparator( className2, className, "testaBc", "testa1b" ), 1 );
+        assertEquals( ( int ) tlr.testOrderComparator( className, className2, "testa3d", "testa1b" ), -1 );
+        assertEquals( ( int ) tlr.testOrderComparator( className, className2, "testa2d", "testabc" ), 1 );
+    }
+
+    public void testRegexMethodOrderComparator()
+    {
+        List<String> orderParamList = new ArrayList<String>();
+        orderParamList.add( "TestClass1#testa?c" );
+        orderParamList.add( "TestClass1#testa?b" );
+        orderParamList.add( "TestClass2#test?1*" );
+        orderParamList.add( "!TestClass1#testa4b" );
+        orderParamList.add( "!TestClass2#test11MyTest" );
+        TestListResolver tlr = new TestListResolver( orderParamList );
+        String className = "TestClass1";
+        String className2 = "TestClass2";
+        assertEquals( ( int ) tlr.testOrderComparator( className, className, "testabc", "testa1b" ), -1 );
+        assertEquals( ( int ) tlr.testOrderComparator( className, className, "testaBc", "testa2b" ), -1 );
+        assertEquals( ( int ) tlr.testOrderComparator( className, className, "testa1b", "testa3c" ), 1 );
+        assertEquals( ( int ) tlr.testOrderComparator( className, className, "testa1b", "testa4b" ), 1 );
+        assertEquals( ( int ) tlr.testOrderComparator( className, className, "testa4b", "testabc" ), -1 );
+        assertEquals( ( int ) tlr.testOrderComparator( className, className2, "testa1b", "test1123" ), -1 );
+        assertEquals( ( int ) tlr.testOrderComparator( className2, className, "testa1b", "testa1b" ), 1 );
+        assertEquals( ( int ) tlr.testOrderComparator( className2, className2, "testa1b", "test1123" ), 1 );
+        assertEquals( ( int ) tlr.testOrderComparator( className2, className2, "test1123", "test11MyTest" ), 1 );
+        assertEquals( ( int ) tlr.testOrderComparator( className2, className2, "test11MyTest", "test456" ), -1 );
+    }
+
+    public void testRegexClassOrderComparator()
+    {
+        List<String> orderParamList = new ArrayList<String>();
+        orderParamList.add( "My2*Test.java" );
+        orderParamList.add( "???My1*Test" );
+        orderParamList.add( "!abcMy1PeaceTest" );
+        TestListResolver tlr = new TestListResolver( orderParamList );
+        String className = "My2ConnectTest";
+        String className2 = "456My1ConnectTest";
+        String className3 = "abcMy1PeaceTest";
+        assertEquals( ( int ) tlr.testOrderComparator( className, className2, null, null ), -1 );
+        assertEquals( ( int ) tlr.testOrderComparator( className2, className, null, null ), 1 );
+        assertEquals( ( int ) tlr.testOrderComparator( className3, className2, null, null ), -1 );
+        assertEquals( ( int ) tlr.testOrderComparator( className, className3, null, null ), 1 );
+    }
 }