Merge pull request #16 from Workday/instrumentation_coverage_reports

instrumentation coverage reports

.gitignore

@@ -1,14 +1,16 @@
-.gradle
+gs.gradle
 /local.properties
-/.idea/
+.idea/
 *.iml
 .DS_Store
 build/
 /artifacts
 /captures
 /gradle/local.properties
 /docs
-/torque-output*
+torque-output/
+.gradle/
+*.output
 
 # Share code style.
 !.idea/codeStyleSettings.xml
@@ -17,3 +19,7 @@ build/
 node_modules/
 npm-debug.log*
 /test/
+local.properties
+failed-install-output.txt
+successful-install-output.txt
+test-result.txt

.travis.yml

@@ -1,8 +1,20 @@
-language: java
+language: android
+dist: trusty
+android:
+  components:
+    - tools
+    - platform-tools
+    - build-tools-26.0.2
+    - android-26
+    - add-on
+    - extra
+  licenses:
+    - 'android-sdk-preview-license-52d11cd2'
+    - 'android-sdk-license-.+'
+    - 'google-gdk-license-.+'
 script:
 #  https://issues.sonatype.org/browse/OSSRH-38347 Change to auto release or switch repository (jfrog etc.)
-#- "./gradlew clean build uploadArchives -s -PNEXUS_USERNAME=$NEXUS_USERNAME -PNEXUS_PASSWORD=$NEXUS_PASSWORD"
-- "./gradlew clean build -s"
+- "./gradlew clean build uploadArchives -s -PNEXUS_USERNAME=$NEXUS_USERNAME -PNEXUS_PASSWORD=$NEXUS_PASSWORD"
 
 deploy:
   provider: releases

dependencies.gradle

@@ -1,6 +1,6 @@
 ext.versions = [
-        androidGradlePlugin: '3.2.0',
-        kotlin             : '1.3.20',
+        androidGradlePlugin: '3.6.0',
+        kotlin             : '1.3.71',
         kotlinCoroutines   : '1.1.1',
         rxJava             : '1.3.0',
         rxJava2            : '2.2.2',
@@ -12,8 +12,8 @@ ext.versions = [
         gson               : '2.8.0',
         dexParser          : '1.1.0',
         junit              : '4.12',
-        junitPlatform      : '1.0.0-M4',
-        spek               : '1.1.2',
+        junitPlatform      : '1.2.0',
+        spek               : '1.2.1',
         assertJ            : '3.5.2',
         mockk              : '1.9',
 ]

gradle.properties

@@ -1,4 +1,4 @@
-VERSION_NAME=1.0.5
+VERSION_NAME=1.0.6
 GROUP=com.workday
 
 POM_DESCRIPTION=A Reactive Android instrumentation test orchestrator with multi-library-modules-testing and test pooling/grouping support.

gradle/wrapper/gradle-wrapper.properties

@@ -3,4 +3,4 @@ distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-4.10-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-6.5.1-all.zip

torque-core/src/main/kotlin/com/workday/torque/Args.kt

@@ -117,6 +117,13 @@ data class Args(
         )
         var testFilesPullDeviceDirectory: String = DEFAULT_FILES_PULL_DEVICE_DIR_PATH,
 
+        @Parameter(
+                names = ["--test-coverage-enabled"],
+                arity = 1,
+                description = "Instrument test coverage file generation. Setting this to true will create a coverage-reports directory in your test file directory containing any generated coverage files"
+        )
+        var testCoverageEnabled: Boolean = false,
+
         @Parameter(
                 names = ["--test-files-pull-host-directory"],
                 description = "Directory on the Torque run host machine to pull test files into. Setting this and --file-pull-device-directory will enable pulling of the folders." +

torque-core/src/main/kotlin/com/workday/torque/TestChunkRunner.kt

@@ -2,11 +2,13 @@ package com.workday.torque
 
 import com.gojuno.commander.os.Notification
 import com.workday.torque.pooling.TestChunk
+import io.reactivex.Completable
 import io.reactivex.Single
 import kotlinx.coroutines.rx2.await
 import java.util.concurrent.TimeUnit
 
 class TestChunkRunner(
+        private val args: Args,
         private val adbDevice: AdbDevice,
         private val logcatFileIO: LogcatFileIO,
         private val installer: Installer,
@@ -17,32 +19,48 @@ class TestChunkRunner(
         return try {
             adbDevice.log("Starting tests...")
             installer.ensureTestPackageInstalled(args, testChunk)
+            makeTestFileDirectories(args).await()
             runAndParseTests(args.chunkTimeoutSeconds, testChunk).await()
         } catch (e: Exception) {
             adbDevice.log("TestChunk run crashed with exception: ${e.message}")
             createCrashedAdbDeviceTestResults(testChunk, e)
         }
     }
 
+    private fun makeTestFileDirectories(args: Args): Completable {
+        return processRunner.runAdb(commandAndArgs = listOf(
+                        "-s", adbDevice.id,
+                        "shell", "mkdir -p ${args.testFilesPullDeviceDirectory}/coverage-reports"
+                ),
+                        destroyOnUnsubscribe = true)
+                .ofType(Notification.Exit::class.java)
+                .doOnError { error -> adbDevice.log("Failed to mkdir on ${adbDevice.tag}, filepath: ${args.testFilesPullDeviceDirectory}/coverage-reports, failed: $error") }
+                .ignoreElements()
+    }
+
     private fun runAndParseTests(chunkTimeoutSeconds: Long, testChunk: TestChunk): Single<List<AdbDeviceTestResult>> {
         val testPackageName = testChunk.testModuleInfo.moduleInfo.apkPackage.value
         val testRunnerClass = testChunk.testModuleInfo.testRunner.value
+        val coverageFileName = testChunk.testMethods.joinToString(",") { it.testName } + ".ec"
         val testMethodsArgs = "-e class " + testChunk.testMethods.joinToString(",") { it.testName }
         val timeout = Timeout(chunkTimeoutSeconds.toInt(), TimeUnit.SECONDS)
-
-        return processRunner.runAdb(
-                commandAndArgs = listOf(
-                        "-s", adbDevice.id,
-                        "shell", "am instrument -w -r $testMethodsArgs $testPackageName/$testRunnerClass"
-                ),
-                timeout = timeout)
+        val runCommand = if(args.testCoverageEnabled && shouldPullTestFiles(args)) {
+            "am instrument -w -r -e coverage true -e \"coverageFile ${args.testFilesPullDeviceDirectory}/coverage-reports/$coverageFileName\" \"$testMethodsArgs $testPackageName/$testRunnerClass\""
+        } else {
+            "am instrument -w -r \"$testMethodsArgs $testPackageName/$testRunnerClass\""
+        }
+        return processRunner.runAdb(commandAndArgs = listOf("-s", adbDevice.id, "shell", runCommand), timeout = timeout)
                 .ofType(Notification.Start::class.java)
                 .flatMap { instrumentationReader.readTestResults(it.output, chunkTimeoutSeconds) }
                 .doOnNext { instrumentationTestResult -> logTestResult(instrumentationTestResult) }
                 .map { instrumentationTestResult -> createAdbDeviceTestResult(instrumentationTestResult) }
                 .toList()
     }
 
+    private fun shouldPullTestFiles(args: Args) : Boolean {
+        return args.testFilesPullDeviceDirectory.isNotEmpty() && args.testFilesPullHostDirectory.isNotEmpty()
+    }
+
     private fun logTestResult(testResult: InstrumentationTestResult) {
         val status = when (testResult.status) {
             is InstrumentationTestResult.Status.Passed -> "passed"

torque-core/src/main/kotlin/com/workday/torque/TestRunFactory.kt

@@ -23,7 +23,7 @@ class TestRunFactory {
             logcatRecorder: LogcatRecorder = LogcatRecorder(adbDevice, logcatFileIO),
             installer: Installer = Installer(adbDevice),
             filePuller: FilePuller = FilePuller(adbDevice),
-            testChunkRunner: TestChunkRunner = TestChunkRunner(adbDevice, logcatFileIO, installer),
+            testChunkRunner: TestChunkRunner = TestChunkRunner(args, adbDevice, logcatFileIO, installer),
             testChunkRetryer: TestChunkRetryer = TestChunkRetryer(adbDevice, args, logcatFileIO, testChunkRunner, installer)
     ): Single<AdbDeviceTestSession> {
         val testSession = AdbDeviceTestSession(adbDevice = adbDevice,

torque-core/src/test/kotlin/com/workday/torque/ArgsSpec.kt

@@ -228,4 +228,21 @@ class ArgsSpec : Spek(
             }
         }
     }
+
+    context("parse args with explicitly passed --test-coverage-enabled") {
+
+        listOf(true, false).forEach { testCoverageEnabled ->
+
+            context("--test-coverage-enabled $testCoverageEnabled") {
+
+                val args by memoized {
+                    parseArgs(rawArgsWithOnlyRequiredFields + arrayOf("--test-coverage-enabled", "$testCoverageEnabled"))
+                }
+
+                it("parses --test-coverage-enabled correctly") {
+                    assertThat(args.testCoverageEnabled).isEqualTo(testCoverageEnabled)
+                }
+            }
+        }
+    }
 })

torque-core/src/test/kotlin/com/workday/torque/TestChunkRunnerSpec.kt

@@ -22,13 +22,15 @@ import kotlin.test.assertEquals
 class TestChunkRunnerSpec : Spek(
 {
     context("Running a test chunk") {
+        val args = Args()
         val adbDevice = mockk<AdbDevice>(relaxed = true)
         val logcatFileIO by memoized { mockk<LogcatFileIO>() }
         val installer = mockk<Installer>(relaxed = true)
         val processRunner by memoized { mockk<ProcessRunner>() }
         val instrumentationReader by memoized { mockk<InstrumentationReader>() }
         val testChunkRunner by memoized {
-            TestChunkRunner(adbDevice,
+            TestChunkRunner(args,
+                            adbDevice,
                             logcatFileIO,
                             installer,
                             processRunner,
@@ -59,9 +61,7 @@ class TestChunkRunnerSpec : Spek(
 
 
             on("timeout per chunk of 75 seconds") {
-                val args = Args().apply {
-                    chunkTimeoutSeconds = 75
-                }
+                args.chunkTimeoutSeconds = 75
 
                 it("runs the process with 75 seconds timeout") {
                     runBlocking {
@@ -85,7 +85,7 @@ class TestChunkRunnerSpec : Spek(
 
                     val adbDeviceTestResults = runBlocking {
                         testChunkRunner.run(args, testChunk)
-                    }!!
+                    }
 
                     assertEquals(expectedTestResults, adbDeviceTestResults)
                 }