Merge pull request #26 from Workday/fully_qualify_anntations_and_fix_…

…options_metadata

fix matching logic for annotations and fix options metadata

README.md

@@ -76,7 +76,7 @@ With arguments :
 ```console
 java -jar torque-runner-$torque_version.jar \
 --test-apks myLibrary1-androidTest.apk myLibrary2-androidTest.apk \
---annotations MediumTest \
+--includedAnnotations MediumTest \
 --chunk-size 8 \
 --output-directory artifacts/torque-output
 ```

gradle.properties

@@ -1,4 +1,4 @@
-VERSION_NAME=1.1.1
+VERSION_NAME=1.1.2
 GROUP=com.workday
 POM_DESCRIPTION=A Reactive Android instrumentation test orchestrator with multi-library-modules-testing and test pooling/grouping support.
 POM_URL=https://github.com/Workday/torque

torque-core/src/main/kotlin/com/workday/torque/ModuleTestParser.kt

@@ -1,5 +1,6 @@
 package com.workday.torque
 
+import com.linkedin.dex.parser.TestAnnotation
 import com.linkedin.dex.parser.TestMethod
 import com.workday.torque.pooling.ModuleInfo
 import com.workday.torque.pooling.TestModule
@@ -30,12 +31,11 @@ class ModuleTestParser(private val args: Args, private val apkTestParser: ApkTes
 			includedAnnotations: List<String>,
 			excludedAnnotations: List<String>
     ): Boolean {
-        return annotations.all { includedAnnotations.hasAnnotation(it.name) }
-                && excludedAnnotations.all { includedAnnotations.hasAnnotation(it).not() }
+        return includedAnnotations.all { annotations.hasAnnotation(it) } && excludedAnnotations.all { annotations.hasAnnotation(it).not() }
     }
 
-    private fun List<String>.hasAnnotation(annotation: String): Boolean {
-        return any { it.contains(annotation) }
+    private fun List<TestAnnotation>.hasAnnotation(annotation: String): Boolean {
+        return any { it.name.contains(annotation) }
     }
 
     private fun List<TestMethod>.filterClassRegexes(regexStrings: List<String>): List<TestMethod> {

torque-core/src/test/kotlin/com/workday/torque/ApkTestParserSpec.kt

@@ -1,9 +1,9 @@
 package com.workday.torque
 
 import com.linkedin.dex.parser.TestMethod
-import com.workday.torque.utils.RUNS_WITH_ANDROID_ANNOTATION
-import com.workday.torque.utils.TEST_ANNOTATION
-import com.workday.torque.utils.THROWS_EXCEPTION_ANNOTATION
+import com.workday.torque.utils.RUNS_WITH_ANDROID_ANNOTATION_QUALIFIED
+import com.workday.torque.utils.TEST_ANNOTATION_QUALIFIED
+import com.workday.torque.utils.THROWS_EXCEPTION_ANNOTATION_QUALIFIED
 import com.workday.torque.utils.assertAnnotationArrays
 import org.assertj.core.api.Assertions.assertThat
 import org.jetbrains.spek.api.Spek
@@ -31,9 +31,9 @@ class ApkTestParserSpec : Spek({
         it("parses tests list correctly") {
             val expectedTestMethods = listOf(
                     TestMethod("test.test.myapplication.ExampleInstrumentedTest#useAppContext",
-                            listOf(RUNS_WITH_ANDROID_ANNOTATION,
-                                    THROWS_EXCEPTION_ANNOTATION,
-                                    TEST_ANNOTATION)
+                            listOf(RUNS_WITH_ANDROID_ANNOTATION_QUALIFIED,
+                                    THROWS_EXCEPTION_ANNOTATION_QUALIFIED,
+                                    TEST_ANNOTATION_QUALIFIED)
                     ))
             val actualTestMethods = apkTestParser.getTests(testApkPath)
             expectedTestMethods.zip(actualTestMethods).forEach { (expected, actual) ->

torque-core/src/test/kotlin/com/workday/torque/ModuleTestParserSpec.kt

@@ -35,29 +35,29 @@ class ModuleTestParserSpec : Spek(
 		}
 	}
 
-	context("parse tests with different test annotations") {
+	context("parse tests with different qualified test annotations") {
 		val noAnnotationsTest = TestMethod("com.company.mymodule.test#testNoAnnotations",
-				listOf(TEST_ANNOTATION,
-						METADATA_ANNOTATION))
+				listOf(TEST_ANNOTATION_QUALIFIED,
+						METADATA_ANNOTATION_QUALIFIED))
 		val ignoredTest = TestMethod("com.company.mymodule.test#testIgnored",
-				listOf(IGNORE_TEST_ANNOTATION,
-						TEST_ANNOTATION,
-						METADATA_ANNOTATION))
+				listOf(IGNORE_TEST_ANNOTATION_QUALIFIED,
+						TEST_ANNOTATION_QUALIFIED,
+						METADATA_ANNOTATION_QUALIFIED))
 		val mediumTest = TestMethod("com.company.mymodule.test#testMediumTest",
-				listOf(MEDIUM_TEST_ANNOTATION,
-						TEST_ANNOTATION,
-						METADATA_ANNOTATION))
+				listOf(MEDIUM_TEST_ANNOTATION_QUALIFIED,
+						TEST_ANNOTATION_QUALIFIED,
+						METADATA_ANNOTATION_QUALIFIED))
 		val flakyMediumTest = TestMethod("com.company.mymodule.test#testMultiAnnotations",
-				listOf(MEDIUM_TEST_ANNOTATION,
-						FLAKY_TEST_ANNOTATION,
-						TEST_ANNOTATION,
-						METADATA_ANNOTATION))
+				listOf(MEDIUM_TEST_ANNOTATION_QUALIFIED,
+						FLAKY_TEST_ANNOTATION_QUALIFIED,
+						TEST_ANNOTATION_QUALIFIED,
+						METADATA_ANNOTATION_QUALIFIED))
 		val ignoredFlakyMediumTest = TestMethod("com.company.mymodule.test#testPositiveNegativeAnnotations",
-				listOf(IGNORE_TEST_ANNOTATION,
-						MEDIUM_TEST_ANNOTATION,
-						FLAKY_TEST_ANNOTATION,
-						TEST_ANNOTATION,
-						METADATA_ANNOTATION))
+				listOf(IGNORE_TEST_ANNOTATION_QUALIFIED,
+						MEDIUM_TEST_ANNOTATION_QUALIFIED,
+						FLAKY_TEST_ANNOTATION_QUALIFIED,
+						TEST_ANNOTATION_QUALIFIED,
+						METADATA_ANNOTATION_QUALIFIED))
 		val testMethods = listOf(
 				noAnnotationsTest,
 				ignoredTest,
@@ -77,7 +77,112 @@ class ModuleTestParserSpec : Spek(
 		val testApkPath = "some_path"
 		val testModuleInfo = TestModuleInfo(ModuleInfo(testPackage, testApkPath), testRunner, ModuleInfo(targetPackage, targetApkPath))
 
-		given("no Annotations and with default Ignore NotAnnotations") {
+		given("no Annotations and with default org.junit.Ignore Excluded Annotations") {
+			val args = Args().apply {
+				testApkPaths = listOf(testApkPath)
+				appApkPath = targetApkPath
+			}
+
+			it("filters out org.junit.Ignored tests only") {
+				val expectedTestMethods = listOf(
+						noAnnotationsTest,
+						mediumTest,
+						flakyMediumTest)
+				val expectedTestModules = listOf(TestModule(testModuleInfo, expectedTestMethods))
+				val actualTestModules = ModuleTestParser(args, apkTestParser).parseTestsFromModuleApks()
+				assertTestModules(expectedTestModules, actualTestModules)
+			}
+		}
+
+		given("androidx.test.filters.MediumTest Annotations and with default Ignore Excluded Annotations") {
+			val args = Args().apply {
+				testApkPaths = listOf(testApkPath)
+				appApkPath = targetApkPath
+				includedAnnotations = listOf("androidx.test.filters.MediumTest")
+			}
+
+			it("filters to androidx.test.filters.MediumTest only") {
+				val expectedTestMethods = listOf(mediumTest, flakyMediumTest)
+				val expectedTestModules = listOf(TestModule(testModuleInfo, expectedTestMethods))
+				val actualTestModules = ModuleTestParser(args, apkTestParser).parseTestsFromModuleApks()
+				assertTestModules(expectedTestModules, actualTestModules)
+			}
+		}
+
+		given("androidx.test.filters.MediumTest androidx.test.filters.FlakyTest Annotations and with default Ignore Excluded Annotations") {
+			val args = Args().apply {
+				testApkPaths = listOf(testApkPath)
+				appApkPath = targetApkPath
+				includedAnnotations = listOf("androidx.test.filters.MediumTest", "androidx.test.filters.FlakyTest")
+			}
+
+			it("filters to androidx.test.filters.MediumTest androidx.test.filters.FlakyTest tests only") {
+				val expectedTestMethods = listOf(flakyMediumTest)
+				val expectedTestModules = listOf(TestModule(testModuleInfo, expectedTestMethods))
+				val actualTestModules = ModuleTestParser(args, apkTestParser).parseTestsFromModuleApks()
+				assertTestModules(expectedTestModules, actualTestModules)
+			}
+		}
+
+		given("FlakyTest Excluded Annotations") {
+			val args = Args().apply {
+				testApkPaths = listOf(testApkPath)
+				appApkPath = targetApkPath
+				excludedAnnotations = listOf("androidx.test.filters.FlakyTest")
+			}
+
+			it("filters out androidx.test.filters.FlakyTest tests only") {
+				val expectedTestMethods = listOf(noAnnotationsTest, ignoredTest, mediumTest)
+				val expectedTestModules = listOf(TestModule(testModuleInfo, expectedTestMethods))
+				val actualTestModules = ModuleTestParser(args, apkTestParser).parseTestsFromModuleApks()
+				assertTestModules(expectedTestModules, actualTestModules)
+			}
+		}
+	}
+
+	context("parse tests with different unqualified test annotations") {
+		val noAnnotationsTest = TestMethod("com.company.mymodule.test#testNoAnnotations",
+				listOf(TEST_ANNOTATION_QUALIFIED,
+						METADATA_ANNOTATION_QUALIFIED))
+		val ignoredTest = TestMethod("com.company.mymodule.test#testIgnored",
+				listOf(IGNORE_TEST_ANNOTATION_QUALIFIED,
+						TEST_ANNOTATION_QUALIFIED,
+						METADATA_ANNOTATION_QUALIFIED))
+		val mediumTest = TestMethod("com.company.mymodule.test#testMediumTest",
+				listOf(MEDIUM_TEST_ANNOTATION_QUALIFIED,
+						TEST_ANNOTATION_QUALIFIED,
+						METADATA_ANNOTATION_QUALIFIED))
+		val flakyMediumTest = TestMethod("com.company.mymodule.test#testMultiAnnotations",
+				listOf(MEDIUM_TEST_ANNOTATION_QUALIFIED,
+						FLAKY_TEST_ANNOTATION_QUALIFIED,
+						TEST_ANNOTATION_QUALIFIED,
+						METADATA_ANNOTATION_QUALIFIED))
+		val ignoredFlakyMediumTest = TestMethod("com.company.mymodule.test#testPositiveNegativeAnnotations",
+				listOf(IGNORE_TEST_ANNOTATION_QUALIFIED,
+						MEDIUM_TEST_ANNOTATION_QUALIFIED,
+						FLAKY_TEST_ANNOTATION_QUALIFIED,
+						TEST_ANNOTATION_QUALIFIED,
+						METADATA_ANNOTATION_QUALIFIED))
+		val testMethods = listOf(
+				noAnnotationsTest,
+				ignoredTest,
+				mediumTest,
+				flakyMediumTest,
+				ignoredFlakyMediumTest)
+		val testPackage = ApkPackage.Valid("com.company.mymodule.test")
+		val targetPackage = ApkPackage.Valid("test.test.myapplication")
+		val testRunner = TestRunner.Valid("android.support.test.runner.AndroidJUnitRunner")
+		val apkTestParser = mockk<ApkTestParser> {
+			every { getValidatedTestPackage(any()) } returns testPackage
+			every { getValidatedTargetPackage(any()) } returns targetPackage
+			every { getValidatedTestRunner(any()) } returns testRunner
+			every { getTests(any()) } returns testMethods
+		}
+		val targetApkPath = "some_app_path"
+		val testApkPath = "some_path"
+		val testModuleInfo = TestModuleInfo(ModuleInfo(testPackage, testApkPath), testRunner, ModuleInfo(targetPackage, targetApkPath))
+
+		given("no Annotations and with default Ignore Excluded Annotations") {
 			val args = Args().apply {
 				testApkPaths = listOf(testApkPath)
 				appApkPath = targetApkPath
@@ -94,26 +199,26 @@ class ModuleTestParserSpec : Spek(
 			}
 		}
 
-		given("MediumTest Annotations and with default Ignore NotAnnotations") {
+		given("MediumTest Annotations and with default Ignore Excluded Annotations") {
 			val args = Args().apply {
 				testApkPaths = listOf(testApkPath)
 				appApkPath = targetApkPath
 				includedAnnotations = listOf("MediumTest")
 			}
 
-			it("filters to MediumTests only") {
+			it("filters to MediumTest only") {
 				val expectedTestMethods = listOf(mediumTest, flakyMediumTest)
 				val expectedTestModules = listOf(TestModule(testModuleInfo, expectedTestMethods))
 				val actualTestModules = ModuleTestParser(args, apkTestParser).parseTestsFromModuleApks()
 				assertTestModules(expectedTestModules, actualTestModules)
 			}
 		}
 
-		given("MediumTest FlakyTest Annotations and with default Ignore NotAnnotations") {
+		given("MediumTest FlakyTest Annotations and with default Ignore Excluded Annotations") {
 			val args = Args().apply {
 				testApkPaths = listOf(testApkPath)
 				appApkPath = targetApkPath
-				includedAnnotations = listOf("android.support.test.filters.MediumTest", "android.support.test.filters.FlakyTest")
+				includedAnnotations = listOf("MediumTest", "FlakyTest")
 			}
 
 			it("filters to MediumTest FlakyTest tests only") {
@@ -124,11 +229,11 @@ class ModuleTestParserSpec : Spek(
 			}
 		}
 
-		given("FlakyTest ExcludedAnnotations") {
+		given("FlakyTest Excluded Annotations") {
 			val args = Args().apply {
 				testApkPaths = listOf(testApkPath)
 				appApkPath = targetApkPath
-				excludedAnnotations = listOf("android.support.test.filters.FlakyTest")
+				excludedAnnotations = listOf("FlakyTest")
 			}
 
 			it("filters out FlakyTest tests only") {
@@ -205,21 +310,21 @@ class ModuleTestParserSpec : Spek(
 
 	context("parse tests with annotations and regex") {
 		val noAnnotationsSpecificTest = TestMethod("com.company.mymodule.SomeSpecificTest#someTest",
-				listOf(TEST_ANNOTATION,
-						METADATA_ANNOTATION))
+				listOf(TEST_ANNOTATION_QUALIFIED,
+						METADATA_ANNOTATION_QUALIFIED))
 		val mediumSpecificTest = TestMethod("com.company.mymodule.SomeSpecificTest#someTest",
-				listOf(MEDIUM_TEST_ANNOTATION,
-						TEST_ANNOTATION,
-						METADATA_ANNOTATION))
+				listOf(MEDIUM_TEST_ANNOTATION_QUALIFIED,
+						TEST_ANNOTATION_QUALIFIED,
+						METADATA_ANNOTATION_QUALIFIED))
 		val mediumOtherTest = TestMethod("com.company.mymodule.SomeOtherTest#someTest",
-				listOf(MEDIUM_TEST_ANNOTATION,
-						TEST_ANNOTATION,
-						METADATA_ANNOTATION))
+				listOf(MEDIUM_TEST_ANNOTATION_QUALIFIED,
+						TEST_ANNOTATION_QUALIFIED,
+						METADATA_ANNOTATION_QUALIFIED))
 		val flakyMediumTest = TestMethod("com.company.mymodule.SomeSpecificTest#someTest",
-				listOf(MEDIUM_TEST_ANNOTATION,
-						FLAKY_TEST_ANNOTATION,
-						TEST_ANNOTATION,
-						METADATA_ANNOTATION))
+				listOf(MEDIUM_TEST_ANNOTATION_QUALIFIED,
+						FLAKY_TEST_ANNOTATION_QUALIFIED,
+						TEST_ANNOTATION_QUALIFIED,
+						METADATA_ANNOTATION_QUALIFIED))
 		val testMethods = listOf(
 				noAnnotationsSpecificTest,
 				mediumSpecificTest,
@@ -242,8 +347,8 @@ class ModuleTestParserSpec : Spek(
 			val args = Args().apply {
 				testApkPaths = listOf(testApkPath)
 				appApkPath = targetApkPath
-				includedAnnotations = listOf("android.support.test.filters.MediumTest")
-				excludedAnnotations = listOf("android.support.test.filters.FlakyTest")
+				includedAnnotations = listOf("MediumTest")
+				excludedAnnotations = listOf("FlakyTest")
 				testClassRegexes = listOf("[a-zA-Z]+SpecificTest")
 			}
 
@@ -263,9 +368,9 @@ private fun createExpectedTestModules(testApkPath: String, moduleCount: Int): Li
 	val testRunner = TestRunner.Valid("android.support.test.runner.AndroidJUnitRunner")
 	val testMethods = listOf(
 			TestMethod("test.test.myapplication.ExampleInstrumentedTest#useAppContext",
-					listOf(RUNS_WITH_ANDROID_ANNOTATION,
-							THROWS_EXCEPTION_ANNOTATION,
-							TEST_ANNOTATION)
+					listOf(RUNS_WITH_ANDROID_ANNOTATION_QUALIFIED,
+							THROWS_EXCEPTION_ANNOTATION_QUALIFIED,
+							TEST_ANNOTATION_QUALIFIED)
 			))
 	val moduleInfo = TestModuleInfo(ModuleInfo(testPackage, testApkPath), testRunner, ModuleInfo(targetPackage, ""))
 	return MutableList(moduleCount) { TestModule(moduleInfo, testMethods) }

torque-core/src/test/kotlin/com/workday/torque/utils/TestUtils.kt

@@ -8,13 +8,13 @@ import org.assertj.core.api.Assertions
 import kotlin.test.assertEquals
 import kotlin.test.assertTrue
 
-val THROWS_EXCEPTION_ANNOTATION = TestAnnotation("dalvik.annotation.Throws", values = mapOf("value" to DecodedValue.DecodedArrayValue(values= arrayOf(DecodedValue.DecodedType(value="Ljava/lang/Exception;")))), inherited = false)
-val TEST_ANNOTATION = TestAnnotation(name = "org.junit.Test", values = mapOf(), inherited = false)
-val RUNS_WITH_ANDROID_ANNOTATION = TestAnnotation("org.junit.runner.RunWith", values = mapOf("value" to DecodedValue.DecodedType(value="Landroid/support/test/runner/AndroidJUnit4;")), inherited = true)
-val MEDIUM_TEST_ANNOTATION = TestAnnotation("android.support.test.filters.MediumTest", values = mapOf(), inherited = false)
-val METADATA_ANNOTATION = TestAnnotation("kotlin.Metadata", values = mapOf(), inherited = false)
-val FLAKY_TEST_ANNOTATION = TestAnnotation("android.support.test.filters.FlakyTest", values = mapOf(), inherited = false)
-val IGNORE_TEST_ANNOTATION = TestAnnotation("org.junit.Ignore", values = mapOf(), inherited = false)
+val THROWS_EXCEPTION_ANNOTATION_QUALIFIED = TestAnnotation("dalvik.annotation.Throws", values = mapOf("value" to DecodedValue.DecodedArrayValue(values= arrayOf(DecodedValue.DecodedType(value="Ljava/lang/Exception;")))), inherited = false)
+val TEST_ANNOTATION_QUALIFIED = TestAnnotation(name = "org.junit.Test", values = mapOf(), inherited = false)
+val RUNS_WITH_ANDROID_ANNOTATION_QUALIFIED = TestAnnotation("org.junit.runner.RunWith", values = mapOf("value" to DecodedValue.DecodedType(value="Landroid/support/test/runner/AndroidJUnit4;")), inherited = true)
+val MEDIUM_TEST_ANNOTATION_QUALIFIED = TestAnnotation("androidx.test.filters.MediumTest", values = mapOf(), inherited = false)
+val METADATA_ANNOTATION_QUALIFIED = TestAnnotation("kotlin.Metadata", values = mapOf(), inherited = false)
+val FLAKY_TEST_ANNOTATION_QUALIFIED = TestAnnotation("androidx.test.filters.FlakyTest", values = mapOf(), inherited = false)
+val IGNORE_TEST_ANNOTATION_QUALIFIED = TestAnnotation("org.junit.Ignore", values = mapOf(), inherited = false)
 
 fun createTestMethodsList(size: Int): MutableList<TestMethod> {
     return MutableList(size) { TestMethod("test#somemethod", listOf(TestAnnotation("org.junit.Test", values = mapOf(), inherited = false))) }

torque-gradle-plugin/src/main/kotlin/com/workday/torque/gradle/TorqueRunTask.kt

@@ -7,14 +7,14 @@ import org.gradle.api.tasks.options.Option
 open class TorqueRunTask: DefaultTask() {
 
     @get: Input
-    @set: Option(option = "annotations", description = "annotations for tests to be ran")
+    @set: Option(description = "annotations for tests to be ran")
     var includedAnnotations: List<String> = emptyList()
 
     @get: Input
-    @set: Option(option = "notAnnotations", description = "annotations for tests not to be ran")
+    @set: Option(description = "annotations for tests to be ignored")
     var excludedAnnotations: List<String> = emptyList()
 
     @get: Input
-    @set: Option(option = "testClassRegexes", description = "regex for tests to be ran")
+    @set: Option(description = "regex for tests to be ran")
     var testClassRegexes: List<String> = emptyList()
 }