Skip to content

Commit

Permalink
Enabling Surefire to run test classes and test methods in any order s…
Browse files Browse the repository at this point in the history 
…pecified by a runOrder
  • Loading branch information
@winglam
winglam committed Apr 15, 2021
1 parent d5beee5 commit 6c49ced
Show file tree
Hide file tree
Showing 10 changed files with 326 additions and 9 deletions.
42 changes: 42 additions & 0 deletions surefire-api/src/main/java/org/apache/maven/surefire/api/testset/TestListResolver.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,9 @@
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Set;

import static java.util.Collections.unmodifiableSet;
Expand Down Expand Up @@ -61,6 +63,8 @@ public class TestListResolver

private final boolean hasExcludedMethodPatterns;

private final Map<String, Integer> patternMapper = new HashMap<String, Integer>();

public TestListResolver( Collection<String> tests )
{
final IncludedExcludedPatterns patterns = new IncludedExcludedPatterns();
Expand Down Expand Up @@ -208,6 +212,7 @@ public boolean shouldRun( String testClassFile, String methodName )
else
{
boolean shouldRun = false;
ResolvedTest matchedFilter = null;

if ( getIncludedPatterns().isEmpty() )
{
Expand All @@ -220,6 +225,7 @@ public boolean shouldRun( String testClassFile, String methodName )
if ( filter.matchAsInclusive( testClassFile, methodName ) )
{
shouldRun = true;
matchedFilter = filter;
break;
}
}
Expand All @@ -236,6 +242,15 @@ public boolean shouldRun( String testClassFile, String methodName )
}
}
}

if ( shouldRun )
{
String test = testClassFile + "#" + methodName;
if ( ! this.patternMapper.containsKey( test ) )
{
this.patternMapper.put( test, new ArrayList<>( this.includedPatterns ).indexOf( matchedFilter ) );
}
}
return shouldRun;
}
}
Expand Down Expand Up @@ -514,4 +529,31 @@ private static boolean haveMethodPatterns( Set<ResolvedTest> patterns )
}
return false;
}

public Integer testOrderComparator( String className1, String className2, String methodName1, String methodName2 )
{
String classFileName1 = toClassFileName( className1 );
String classFileName2 = toClassFileName( className2 );
boolean shouldRunMethodName1 = shouldRun( classFileName1 , methodName1 );
boolean shouldRunMethodName2 = shouldRun( classFileName2 , methodName2 );
if ( ! shouldRunMethodName1 )
{
return -1;
}
if ( ! shouldRunMethodName2 )
{
return 1;
}

String test1 = classFileName1 + "#" + methodName1;
String test2 = classFileName2 + "#" + methodName2;
if ( patternMapper.get( test1 ) < patternMapper.get( test2 ) )
{
return -1;
}
else
{
return 1;
}
}
}
119 changes: 118 additions & 1 deletion surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,21 @@

import org.apache.maven.surefire.api.runorder.RunEntryStatisticsMap;
import org.apache.maven.surefire.api.testset.RunOrderParameters;
import org.apache.maven.surefire.api.testset.TestListResolver;

import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Random;

Expand All @@ -48,6 +57,8 @@ public class DefaultRunOrderCalculator

private final Random random;

private final TestListResolver testListResolver;

public DefaultRunOrderCalculator( RunOrderParameters runOrderParameters, int threadCount )
{
this.runOrderParameters = runOrderParameters;
Expand All @@ -61,6 +72,14 @@ public DefaultRunOrderCalculator( RunOrderParameters runOrderParameters, int thr
runOrderParameters.setRunOrderRandomSeed( runOrderRandomSeed );
}
this.random = new Random( runOrderRandomSeed );
if ( RunOrder.TESTORDER.equals( getRunOrderMethod() ) )
{
this.testListResolver = getTestListResolver();
}
else
{
this.testListResolver = null;
}
}

@Override
Expand All @@ -78,9 +97,80 @@ public TestsToRun orderTestClasses( TestsToRun scannedClasses )
return new TestsToRun( new LinkedHashSet<>( result ) );
}

@Override
public Comparator<String> comparatorForTestMethods()
{
RunOrder methodRunOrder = getRunOrderMethod();
if ( RunOrder.TESTORDER.equals( methodRunOrder ) )
{
return new Comparator<String>()
{
@Override
public int compare( String o1, String o2 )
{
String[] classAndMethod1 = getClassAndMethod( o1 );
String className1 = classAndMethod1[0];
String methodName1 = classAndMethod1[1];
String[] classAndMethod2 = getClassAndMethod( o2 );
String className2 = classAndMethod2[0];
String methodName2 = classAndMethod2[1];
return testListResolver.testOrderComparator( className1, className2, methodName1, methodName2 );
}
};
}
else
{
return null;
}
}

public TestListResolver getTestListResolver()
{
String orderParam = parseTestOrder( System.getProperty( "test" ) );
if ( orderParam == null )
{
throw new IllegalStateException( "TestListResolver in RunOrderCalculator should be used only when "
+ "system property -Dtest is set and runOrder is testorder" );
}
return new TestListResolver( Arrays.asList( orderParam.split( "," ) ) );
}

public String[] getClassAndMethod( String request )
{
String[] classAndMethod = { request, request };
if ( request.contains( "(" ) )
{
String[] nameSplit1 = request.split( "\\(" );
classAndMethod[0] = nameSplit1[1].substring( 0, nameSplit1[1].length() - 1 );
classAndMethod[1] = nameSplit1[0];
}
return classAndMethod;
}

private RunOrder getRunOrderMethod()
{
if ( runOrder.length > 1 && Arrays.asList( runOrder ).contains( RunOrder.TESTORDER ) )
{
// Use of testorder and other runOrders are currently not supported
throw new IllegalStateException( "Expected only testorder. Got: " + runOrder.length );
}
return runOrder[0];
}

private void orderTestClasses( List<Class<?>> testClasses, RunOrder runOrder )
{
if ( RunOrder.RANDOM.equals( runOrder ) )
if ( RunOrder.TESTORDER.equals( runOrder ) )
{
Collections.sort( testClasses, new Comparator<Class<?>>()
{
@Override
public int compare( Class<?> o1, Class<?> o2 )
{
return testListResolver.testOrderComparator( o1.getName(), o2.getName(), null, null );
}
} );
}
else if ( RunOrder.RANDOM.equals( runOrder ) )
{
Collections.shuffle( testClasses, random );
}
Expand All @@ -106,6 +196,33 @@ else if ( sortOrder != null )
}
}

private String parseTestOrder( String s )
{
// if s is a file, then parse each line of the file as a test
if ( s != null && s != "" )
{
File f = new File( s );
if ( f.exists() && !f.isDirectory ( ) )
{
try
{
List<String> l = Files.readAllLines( f.toPath(), Charset.defaultCharset( ) );
StringBuilder sb = new StringBuilder( );
for ( String sd : l )
{
sb.append( sd + "," );
}
String sd = sb.toString( );
return sd.substring( 0 , sd.length( ) - 1 );
}
catch ( IOException e )
{
}
}
}
return s;
}

private Comparator<Class> getSortOrderComparator( RunOrder runOrder )
{
if ( RunOrder.ALPHABETICAL.equals( runOrder ) )
Expand Down
5 changes: 4 additions & 1 deletion surefire-api/src/main/java/org/apache/maven/surefire/api/util/RunOrder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ public class RunOrder

public static final RunOrder FAILEDFIRST = new RunOrder( "failedfirst" );

public static final RunOrder TESTORDER = new RunOrder( "testorder" );

public static final RunOrder[] DEFAULT = new RunOrder[]{ FILESYSTEM };

/**
Expand Down Expand Up @@ -108,7 +110,8 @@ private static String createMessageForMissingRunOrder( String name )

private static RunOrder[] values()
{
return new RunOrder[]{ ALPHABETICAL, FILESYSTEM, HOURLY, RANDOM, REVERSE_ALPHABETICAL, BALANCED, FAILEDFIRST };
return new RunOrder[]{ ALPHABETICAL, FILESYSTEM, HOURLY, RANDOM, REVERSE_ALPHABETICAL, BALANCED, FAILEDFIRST,
TESTORDER };
}

public static String asString( RunOrder[] runOrder )
Expand Down
4 changes: 4 additions & 0 deletions surefire-api/src/main/java/org/apache/maven/surefire/api/util/RunOrderCalculator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,14 @@
* under the License.
*/

import java.util.Comparator;

/**
* @author Kristian Rosenvold
*/
public interface RunOrderCalculator
{
TestsToRun orderTestClasses( TestsToRun scannedClasses );

Comparator<String> comparatorForTestMethods();
}
62 changes: 62 additions & 0 deletions surefire-api/src/test/java/org/apache/maven/surefire/api/testset/TestListResolverTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;

import static java.util.Collections.addAll;
Expand Down Expand Up @@ -502,4 +503,65 @@ private static Set<ResolvedTest> resolveClass( String patterns )
}
return resolved;
}

public void testOrderComparatorTest()
{
List<String> orderParamList = new ArrayList<String>();
orderParamList.add( "TestClass1#testa2d" );
orderParamList.add( "TestClass1#testabc" );
orderParamList.add( "TestClass1#testa1b" );
orderParamList.add( "TestClass2#testa1b" );
orderParamList.add( "TestClass2#testaBc" );
TestListResolver tlr = new TestListResolver( orderParamList );
String className = "TestClass1";
String className2 = "TestClass2";
assertEquals( ( int ) tlr.testOrderComparator( className, className, "testa2d", "testa1b" ), -1 );
assertEquals( ( int ) tlr.testOrderComparator( className, className, "testa2d", "testabc" ), -1 );
assertEquals( ( int ) tlr.testOrderComparator( className, className, "testa1b", "testabc" ), 1 );
assertEquals( ( int ) tlr.testOrderComparator( className, className, "testa2d", "testaBc" ), 1 );
assertEquals( ( int ) tlr.testOrderComparator( className, className, "testa3d", "testa1b" ), -1 );
assertEquals( ( int ) tlr.testOrderComparator( className, className2, "testa2d", "testa1b" ), -1 );
assertEquals( ( int ) tlr.testOrderComparator( className2, className, "testaBc", "testa1b" ), 1 );
assertEquals( ( int ) tlr.testOrderComparator( className, className2, "testa3d", "testa1b" ), -1 );
assertEquals( ( int ) tlr.testOrderComparator( className, className2, "testa2d", "testabc" ), 1 );
}

public void testRegexMethodOrderComparator()
{
List<String> orderParamList = new ArrayList<String>();
orderParamList.add( "TestClass1#testa?c" );
orderParamList.add( "TestClass1#testa?b" );
orderParamList.add( "TestClass2#test?1*" );
orderParamList.add( "!TestClass1#testa4b" );
orderParamList.add( "!TestClass2#test11MyTest" );
TestListResolver tlr = new TestListResolver( orderParamList );
String className = "TestClass1";
String className2 = "TestClass2";
assertEquals( ( int ) tlr.testOrderComparator( className, className, "testabc", "testa1b" ), -1 );
assertEquals( ( int ) tlr.testOrderComparator( className, className, "testaBc", "testa2b" ), -1 );
assertEquals( ( int ) tlr.testOrderComparator( className, className, "testa1b", "testa3c" ), 1 );
assertEquals( ( int ) tlr.testOrderComparator( className, className, "testa1b", "testa4b" ), 1 );
assertEquals( ( int ) tlr.testOrderComparator( className, className, "testa4b", "testabc" ), -1 );
assertEquals( ( int ) tlr.testOrderComparator( className, className2, "testa1b", "test1123" ), -1 );
assertEquals( ( int ) tlr.testOrderComparator( className2, className, "testa1b", "testa1b" ), 1 );
assertEquals( ( int ) tlr.testOrderComparator( className2, className2, "testa1b", "test1123" ), 1 );
assertEquals( ( int ) tlr.testOrderComparator( className2, className2, "test1123", "test11MyTest" ), 1 );
assertEquals( ( int ) tlr.testOrderComparator( className2, className2, "test11MyTest", "test456" ), -1 );
}

public void testRegexClassOrderComparator()
{
List<String> orderParamList = new ArrayList<String>();
orderParamList.add( "My2*Test.java" );
orderParamList.add( "???My1*Test" );
orderParamList.add( "!abcMy1PeaceTest" );
TestListResolver tlr = new TestListResolver( orderParamList );
String className = "My2ConnectTest";
String className2 = "456My1ConnectTest";
String className3 = "abcMy1PeaceTest";
assertEquals( ( int ) tlr.testOrderComparator( className, className2, null, null ), -1 );
assertEquals( ( int ) tlr.testOrderComparator( className2, className, null, null ), 1 );
assertEquals( ( int ) tlr.testOrderComparator( className3, className2, null, null ), -1 );
assertEquals( ( int ) tlr.testOrderComparator( className, className3, null, null ), 1 );
}
}
Loading

0 comments on commit 6c49ced

Please sign in to comment.