Skip to content

Commit

Permalink
Enabling Surefire to run test classes and test methods in any order s…
Browse files Browse the repository at this point in the history 
…pecified by a runOrder
  • Loading branch information
@winglam
winglam committed Apr 16, 2021
1 parent d5beee5 commit a92ea27
Show file tree
Hide file tree
Showing 10 changed files with 288 additions and 9 deletions.
35 changes: 35 additions & 0 deletions surefire-api/src/main/java/org/apache/maven/surefire/api/testset/TestListResolver.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,9 @@
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Set;

import static java.util.Collections.unmodifiableSet;
Expand Down Expand Up @@ -61,6 +63,8 @@ public class TestListResolver

private final boolean hasExcludedMethodPatterns;

private final Map<String, Integer> patternMapper = new HashMap<String, Integer>();

public TestListResolver( Collection<String> tests )
{
final IncludedExcludedPatterns patterns = new IncludedExcludedPatterns();
Expand Down Expand Up @@ -208,6 +212,7 @@ public boolean shouldRun( String testClassFile, String methodName )
else
{
boolean shouldRun = false;
ResolvedTest matchedFilter = null;

if ( getIncludedPatterns().isEmpty() )
{
Expand All @@ -220,6 +225,7 @@ public boolean shouldRun( String testClassFile, String methodName )
if ( filter.matchAsInclusive( testClassFile, methodName ) )
{
shouldRun = true;
matchedFilter = filter;
break;
}
}
Expand All @@ -236,6 +242,15 @@ public boolean shouldRun( String testClassFile, String methodName )
}
}
}

if ( shouldRun )
{
String test = testClassFile + "#" + methodName;
if ( ! this.patternMapper.containsKey( test ) )
{
this.patternMapper.put( test, new ArrayList<>( this.includedPatterns ).indexOf( matchedFilter ) );
}
}
return shouldRun;
}
}
Expand Down Expand Up @@ -514,4 +529,24 @@ private static boolean haveMethodPatterns( Set<ResolvedTest> patterns )
}
return false;
}

public Integer testOrderComparator( String className1, String className2, String methodName1, String methodName2 )
{
String classFileName1 = toClassFileName( className1 );
String classFileName2 = toClassFileName( className2 );
boolean shouldRunMethodName1 = shouldRun( classFileName1 , methodName1 );
boolean shouldRunMethodName2 = shouldRun( classFileName2 , methodName2 );
if ( ! shouldRunMethodName1 )
{
return -1;
}
if ( ! shouldRunMethodName2 )
{
return 1;
}

String test1 = classFileName1 + "#" + methodName1;
String test2 = classFileName2 + "#" + methodName2;
return patternMapper.get( test1 ) - patternMapper.get( test2 );
}
}
85 changes: 84 additions & 1 deletion surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,10 @@

import org.apache.maven.surefire.api.runorder.RunEntryStatisticsMap;
import org.apache.maven.surefire.api.testset.RunOrderParameters;
import org.apache.maven.surefire.api.testset.TestListResolver;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Collections;
import java.util.Comparator;
Expand All @@ -48,6 +50,8 @@ public class DefaultRunOrderCalculator

private final Random random;

private final TestListResolver testListResolver;

public DefaultRunOrderCalculator( RunOrderParameters runOrderParameters, int threadCount )
{
this.runOrderParameters = runOrderParameters;
Expand All @@ -61,6 +65,14 @@ public DefaultRunOrderCalculator( RunOrderParameters runOrderParameters, int thr
runOrderParameters.setRunOrderRandomSeed( runOrderRandomSeed );
}
this.random = new Random( runOrderRandomSeed );
if ( RunOrder.TESTORDER.equals( getRunOrderMethod() ) )
{
this.testListResolver = getTestListResolver();
}
else
{
this.testListResolver = null;
}
}

@Override
Expand All @@ -78,9 +90,80 @@ public TestsToRun orderTestClasses( TestsToRun scannedClasses )
return new TestsToRun( new LinkedHashSet<>( result ) );
}

@Override
public Comparator<String> comparatorForTestMethods()
{
RunOrder methodRunOrder = getRunOrderMethod();
if ( RunOrder.TESTORDER.equals( methodRunOrder ) )
{
return new Comparator<String>()
{
@Override
public int compare( String o1, String o2 )
{
String[] classAndMethod1 = getClassAndMethod( o1 );
String className1 = classAndMethod1[0];
String methodName1 = classAndMethod1[1];
String[] classAndMethod2 = getClassAndMethod( o2 );
String className2 = classAndMethod2[0];
String methodName2 = classAndMethod2[1];
return testListResolver.testOrderComparator( className1, className2, methodName1, methodName2 );
}
};
}
else
{
return null;
}
}

public TestListResolver getTestListResolver()
{
String orderParam = System.getProperty( "test" );
if ( orderParam == null )
{
throw new IllegalStateException( "TestListResolver in RunOrderCalculator should be used only when "
+ "system property -Dtest is set and runOrder is testorder" );
}
return new TestListResolver( Arrays.asList( orderParam.split( "," ) ) );
}

public String[] getClassAndMethod( String request )
{
String[] classAndMethod = { request, request };
if ( request.contains( "(" ) )
{
String[] nameSplit1 = request.split( "\\(" );
classAndMethod[0] = nameSplit1[1].substring( 0, nameSplit1[1].length() - 1 );
classAndMethod[1] = nameSplit1[0];
}
return classAndMethod;
}

private RunOrder getRunOrderMethod()
{
if ( runOrder.length > 1 && Arrays.asList( runOrder ).contains( RunOrder.TESTORDER ) )
{
// Use of testorder and other runOrders are currently not supported
throw new IllegalStateException( "Expected only testorder. Got: " + runOrder.length );
}
return runOrder[0];
}

private void orderTestClasses( List<Class<?>> testClasses, RunOrder runOrder )
{
if ( RunOrder.RANDOM.equals( runOrder ) )
if ( RunOrder.TESTORDER.equals( runOrder ) )
{
Collections.sort( testClasses, new Comparator<Class<?>>()
{
@Override
public int compare( Class<?> o1, Class<?> o2 )
{
return testListResolver.testOrderComparator( o1.getName(), o2.getName(), null, null );
}
} );
}
else if ( RunOrder.RANDOM.equals( runOrder ) )
{
Collections.shuffle( testClasses, random );
}
Expand Down
5 changes: 4 additions & 1 deletion surefire-api/src/main/java/org/apache/maven/surefire/api/util/RunOrder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ public class RunOrder

public static final RunOrder FAILEDFIRST = new RunOrder( "failedfirst" );

public static final RunOrder TESTORDER = new RunOrder( "testorder" );

public static final RunOrder[] DEFAULT = new RunOrder[]{ FILESYSTEM };

/**
Expand Down Expand Up @@ -108,7 +110,8 @@ private static String createMessageForMissingRunOrder( String name )

private static RunOrder[] values()
{
return new RunOrder[]{ ALPHABETICAL, FILESYSTEM, HOURLY, RANDOM, REVERSE_ALPHABETICAL, BALANCED, FAILEDFIRST };
return new RunOrder[]{ ALPHABETICAL, FILESYSTEM, HOURLY, RANDOM, REVERSE_ALPHABETICAL, BALANCED, FAILEDFIRST,
TESTORDER };
}

public static String asString( RunOrder[] runOrder )
Expand Down
4 changes: 4 additions & 0 deletions surefire-api/src/main/java/org/apache/maven/surefire/api/util/RunOrderCalculator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,14 @@
* under the License.
*/

import java.util.Comparator;

/**
* @author Kristian Rosenvold
*/
public interface RunOrderCalculator
{
TestsToRun orderTestClasses( TestsToRun scannedClasses );

Comparator<String> comparatorForTestMethods();
}
65 changes: 65 additions & 0 deletions surefire-api/src/test/java/org/apache/maven/surefire/api/testset/TestListResolverTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;

import static java.util.Collections.addAll;
Expand Down Expand Up @@ -502,4 +503,68 @@ private static Set<ResolvedTest> resolveClass( String patterns )
}
return resolved;
}

public void testOrderComparatorTest()
{
List<String> orderParamList = new ArrayList<String>();
orderParamList.add( "TestClass1#testa2d" );
orderParamList.add( "TestClass1#testabc" );
orderParamList.add( "TestClass1#testa1b" );
orderParamList.add( "TestClass2#testa1b" );
orderParamList.add( "TestClass2#testaBc" );
TestListResolver tlr = new TestListResolver( orderParamList );
String className = "TestClass1";
String className2 = "TestClass2";
assertTrue( ( int ) tlr.testOrderComparator( className, className, "testa2d", "testa1b" ) < 0 );
assertTrue( ( int ) tlr.testOrderComparator( className, className, "testa2d", "testabc" ) < 0 );
assertTrue( ( int ) tlr.testOrderComparator( className, className, "testa1b", "testabc" ) > 0 );
assertTrue( ( int ) tlr.testOrderComparator( className, className, "testa2d", "testaBc" ) > 0 );
assertTrue( ( int ) tlr.testOrderComparator( className, className, "testa3d", "testa1b" ) < 0 );
assertTrue( ( int ) tlr.testOrderComparator( className, className2, "testa2d", "testa1b" ) < 0 );
assertTrue( ( int ) tlr.testOrderComparator( className2, className, "testaBc", "testa1b" ) > 0 );
assertTrue( ( int ) tlr.testOrderComparator( className, className2, "testa3d", "testa1b" ) < 0 );
assertTrue( ( int ) tlr.testOrderComparator( className, className2, "testa2d", "testabc" ) > 0 );
assertTrue( ( int ) tlr.testOrderComparator( className, className, "testa2d", "testa2d" ) == 0 );
}

public void testRegexMethodOrderComparator()
{
List<String> orderParamList = new ArrayList<String>();
orderParamList.add( "TestClass1#testa?c" );
orderParamList.add( "TestClass1#testa?b" );
orderParamList.add( "TestClass2#test?1*" );
orderParamList.add( "!TestClass1#testa4b" );
orderParamList.add( "!TestClass2#test11MyTest" );
TestListResolver tlr = new TestListResolver( orderParamList );
String className = "TestClass1";
String className2 = "TestClass2";
assertTrue( ( int ) tlr.testOrderComparator( className, className, "testabc", "testa1b" ) < 0 );
assertTrue( ( int ) tlr.testOrderComparator( className, className, "testaBc", "testa2b" ) < 0 );
assertTrue( ( int ) tlr.testOrderComparator( className, className, "testa1b", "testa3c" ) > 0 );
assertTrue( ( int ) tlr.testOrderComparator( className, className, "testa1b", "testa4b" ) > 0 );
assertTrue( ( int ) tlr.testOrderComparator( className, className, "testa4b", "testabc" ) < 0 );
assertTrue( ( int ) tlr.testOrderComparator( className, className2, "testa1b", "test1123" ) < 0 );
assertTrue( ( int ) tlr.testOrderComparator( className2, className, "testa1b", "testa1b" ) > 0 );
assertTrue( ( int ) tlr.testOrderComparator( className2, className2, "testa1b", "test1123" ) == 0 );
assertTrue( ( int ) tlr.testOrderComparator( className2, className2, "test1123", "test11MyTest" ) > 0 );
assertTrue( ( int ) tlr.testOrderComparator( className2, className2, "test11MyTest", "test456" ) < 0 );
assertTrue( ( int ) tlr.testOrderComparator( className, className, "testa1c", "testa1c" ) == 0 );
}

public void testRegexClassOrderComparator()
{
List<String> orderParamList = new ArrayList<String>();
orderParamList.add( "My2*Test.java" );
orderParamList.add( "???My1*Test" );
orderParamList.add( "!abcMy1PeaceTest" );
TestListResolver tlr = new TestListResolver( orderParamList );
String className = "My2ConnectTest";
String className2 = "456My1ConnectTest";
String className3 = "abcMy1PeaceTest";
assertTrue( ( int ) tlr.testOrderComparator( className, className2, null, null ) < 0 );
assertTrue( ( int ) tlr.testOrderComparator( className2, className, null, null ) > 0 );
assertTrue( ( int ) tlr.testOrderComparator( className3, className2, null, null ) < 0 );
assertTrue( ( int ) tlr.testOrderComparator( className, className3, null, null ) > 0 );
assertTrue( ( int ) tlr.testOrderComparator( className, className, null, null ) == 0 );
}
}
51 changes: 51 additions & 0 deletions surefire-api/src/test/java/org/apache/maven/surefire/api/util/RunOrderCalculatorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,10 @@
* under the License.
*/

import java.util.Arrays;
import java.util.Comparator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;

import org.apache.maven.surefire.api.testset.RunOrderParameters;
Expand Down Expand Up @@ -59,4 +62,52 @@ static class B
{

}

public void testOrderTestMethods()
{
RunOrderParameters runOrderParameters = new RunOrderParameters( "testorder" , null );
System.setProperty( "test", "TestClass#a2d,TestClass#aBc,TestClass#abc,TestClass#a1b" );
DefaultRunOrderCalculator runOrderCalculator = new DefaultRunOrderCalculator( runOrderParameters, 1 );
Comparator<String> testOrderRunOrderComparator = runOrderCalculator.comparatorForTestMethods();
String[] strArray = { "abc(TestClass)", "a1b(TestClass)", "a2d(TestClass)", "aBc(TestClass)" };
List<String> actual = Arrays.asList( strArray );
actual.sort( testOrderRunOrderComparator );
String[] strArray2 = { "a2d(TestClass)", "aBc(TestClass)", "abc(TestClass)", "a1b(TestClass)" };
List<String> expected = Arrays.asList( strArray2 );
assertEquals( actual, expected );
}

public void testOrderTestClassesAndMethods()
{
RunOrderParameters runOrderParameters = new RunOrderParameters( "testorder" , null );
System.setProperty( "test", "TestClass1#a2d,TestClass2#aBc,TestClass2#abc,TestClass2#a1b" );
DefaultRunOrderCalculator runOrderCalculator = new DefaultRunOrderCalculator( runOrderParameters, 1 );
Comparator<String> testOrderRunOrderComparator = runOrderCalculator.comparatorForTestMethods();
String[] strArray = { "abc(TestClass2)", "a1b(TestClass2)", "a2d(TestClass1)", "aBc(TestClass2)" };
List<String> actual = Arrays.asList( strArray );
actual.sort( testOrderRunOrderComparator );
String[] strArray2 = { "a2d(TestClass1)", "aBc(TestClass2)", "abc(TestClass2)", "a1b(TestClass2)" };
List<String> expected = Arrays.asList( strArray2 );
assertEquals( actual, expected );
}

public void testOrderTestRegexClassesAndMethods()
{
RunOrderParameters runOrderParameters = new RunOrderParameters( "testorder" , null );
System.setProperty( "test", "Amber*Test#a?c,My???Test#test*" );
DefaultRunOrderCalculator runOrderCalculator = new DefaultRunOrderCalculator( runOrderParameters, 1 );
Comparator<String> testOrderRunOrderComparator = runOrderCalculator.comparatorForTestMethods();
String[] strArray = { "abc(AmberGoodTest)",
"testabc(MyabcTest)",
"a2c(AmberBadTest)",
"testefg(MyefgTest)",
"aBc(AmberGoodTest)" };
List<String> actual = Arrays.asList( strArray );
actual.sort( testOrderRunOrderComparator );
assertEquals( runOrderCalculator.getClassAndMethod( actual.get( 0 ) )[0].substring( 0, 5 ), "Amber" );
assertEquals( runOrderCalculator.getClassAndMethod( actual.get( 1 ) )[0].substring( 0, 5 ), "Amber" );
assertEquals( runOrderCalculator.getClassAndMethod( actual.get( 2 ) )[0].substring( 0, 5 ), "Amber" );
assertEquals( runOrderCalculator.getClassAndMethod( actual.get( 3 ) )[0].substring( 0, 2 ), "My" );
assertEquals( runOrderCalculator.getClassAndMethod( actual.get( 4 ) )[0].substring( 0, 2 ), "My" );
}
}
Loading

0 comments on commit a92ea27

Please sign in to comment.