Extract code coverage from Android device

core/src/main/kotlin/com/malinskiy/marathon/analytics/tracker/TrackerFactory.kt

@@ -4,6 +4,7 @@ import com.google.gson.Gson
 import com.malinskiy.marathon.analytics.tracker.local.DeviceTracker
 import com.malinskiy.marathon.analytics.tracker.local.JUnitTracker
 import com.malinskiy.marathon.analytics.tracker.local.RawTestResultTracker
+import com.malinskiy.marathon.analytics.tracker.local.TestCoverageTracker
 import com.malinskiy.marathon.analytics.tracker.local.TestResultsTracker
 import com.malinskiy.marathon.analytics.tracker.remote.influx.InfluxDbProvider
 import com.malinskiy.marathon.analytics.tracker.remote.influx.InfluxDbTracker
@@ -12,6 +13,7 @@ import com.malinskiy.marathon.execution.Configuration
 import com.malinskiy.marathon.io.FileManager
 import com.malinskiy.marathon.report.allure.AllureTestListener
 import com.malinskiy.marathon.report.internal.DeviceInfoReporter
+import com.malinskiy.marathon.report.internal.TestCoverageReporter
 import com.malinskiy.marathon.report.internal.TestResultReporter
 import com.malinskiy.marathon.report.junit.JUnitReporter
 import java.io.File
@@ -26,13 +28,17 @@ internal class TrackerFactory(private val configuration: Configuration,
     val allureTracker = AllureTestListener(configuration, File(configuration.outputDir, "allure-results"))
 
     fun create(): Tracker {
-        val defaultTrackers = listOf(
+        val defaultTrackers = if (configuration.isCodeCoverageEnabled) {
+            listOf(TestCoverageTracker(TestCoverageReporter(fileManager)))
+        } else {
+            listOf(
                 JUnitTracker(JUnitReporter(fileManager)),
                 DeviceTracker(deviceInfoReporter),
                 TestResultsTracker(testResultReporter),
                 rawTestResultTracker,
                 allureTracker
-        )
+            )
+        }
         return when {
             configuration.analyticsConfiguration is InfluxDbConfiguration -> {
                 val config = configuration.analyticsConfiguration

core/src/main/kotlin/com/malinskiy/marathon/analytics/tracker/local/TestCoverageTracker.kt

@@ -0,0 +1,19 @@
+package com.malinskiy.marathon.analytics.tracker.local
+
+import com.malinskiy.marathon.analytics.tracker.NoOpTracker
+import com.malinskiy.marathon.device.DeviceInfo
+import com.malinskiy.marathon.device.DevicePoolId
+import com.malinskiy.marathon.execution.TestResult
+import com.malinskiy.marathon.report.internal.TestCoverageReporter
+
+internal class TestCoverageTracker(private val testCoverageReporter: TestCoverageReporter) :
+    NoOpTracker() {
+
+    override fun trackTestFinished(
+        poolId: DevicePoolId,
+        device: DeviceInfo,
+        testResult: TestResult
+    ) {
+        testCoverageReporter.testFinished(poolId, device, testResult)
+    }
+}

core/src/main/kotlin/com/malinskiy/marathon/execution/Attachment.kt

@@ -7,11 +7,13 @@ data class Attachment(val file: File, val type: AttachmentType)
 enum class AttachmentType {
     SCREENSHOT,
     VIDEO,
+    COVERAGE,
     LOG;
 
     fun toMimeType() = when(this) {
         SCREENSHOT -> "image/gif"
         VIDEO -> "video/mp4"
+        COVERAGE -> "application/ec"
         LOG -> "text/txt"
     }
 }

core/src/main/kotlin/com/malinskiy/marathon/io/FileType.kt

@@ -7,6 +7,6 @@ enum class FileType(val dir: String, val suffix: String) {
     LOG("logs", "log"),
     DEVICE_INFO("devices", "json"),
     VIDEO("video", "mp4"),
-    SCREENSHOT("screenshot", "gif")
-
+    SCREENSHOT("screenshot", "gif"),
+    COVERAGE("coverage", "ec")
 }

core/src/main/kotlin/com/malinskiy/marathon/report/internal/TestCoverageReporter.kt

@@ -0,0 +1,15 @@
+package com.malinskiy.marathon.report.internal
+
+import com.malinskiy.marathon.device.DeviceInfo
+import com.malinskiy.marathon.device.DevicePoolId
+import com.malinskiy.marathon.execution.TestResult
+import com.malinskiy.marathon.io.FileManager
+import com.malinskiy.marathon.io.FileType
+
+class TestCoverageReporter(private val fileManager: FileManager) {
+
+    fun testFinished(poolId: DevicePoolId, device: DeviceInfo, testResult: TestResult) {
+        val file = fileManager.createFile(FileType.COVERAGE, poolId, device, testResult.test)
+        testResult.attachments.first().file.copyTo(file, overwrite = true)
+    }
+}

vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/AndroidDevice.kt

@@ -11,6 +11,7 @@ import com.malinskiy.marathon.android.executor.listeners.DebugTestRunListener
 import com.malinskiy.marathon.android.executor.listeners.LogCatListener
 import com.malinskiy.marathon.android.executor.listeners.NoOpTestRunListener
 import com.malinskiy.marathon.android.executor.listeners.ProgressTestRunListener
+import com.malinskiy.marathon.android.executor.listeners.TestCoverageResultListener
 import com.malinskiy.marathon.android.executor.listeners.TestRunResultsListener
 import com.malinskiy.marathon.android.executor.listeners.screenshot.ScreenCapturerTestRunListener
 import com.malinskiy.marathon.android.executor.listeners.video.ScreenRecorderTestRunListener
@@ -164,14 +165,20 @@ class AndroidDevice(val ddmsDevice: IDevice,
 
         val timer = SystemTimer()
 
+        val coverageListener = TestCoverageResultListener(fileManager, devicePoolId, this, testBatch, deferred, timer)
+
         return CompositeTestRunListener(
-                listOf(
+                if (configuration.isCodeCoverageEnabled) {
+                    listOf(coverageListener)
+                } else {
+                    listOf(
                         recorderListener,
                         logCatListener,
                         TestRunResultsListener(testBatch, this, deferred, timer, attachmentProviders),
                         DebugTestRunListener(this),
                         ProgressTestRunListener(this, devicePoolId, progressReporter)
-                )
+                    )
+                }
         )
     }
 

vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/RemoteFileManager.kt

@@ -53,6 +53,10 @@ class RemoteFileManager(private val device: IDevice) {
         return remoteFileForTest(videoFileName(test))
     }
 
+    fun remoteCoverageForTest(): String {
+        return remoteFileForTest("coverage.ec")
+    }
+
     private fun remoteFileForTest(filename: String): String {
         return "$outputDir/$filename"
     }

vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/executor/listeners/TestCoverageResultListener.kt

@@ -0,0 +1,204 @@
+package com.malinskiy.marathon.android.executor.listeners
+
+import com.android.ddmlib.TimeoutException
+import com.android.ddmlib.testrunner.TestIdentifier
+import com.android.ddmlib.testrunner.TestResult
+import com.android.ddmlib.testrunner.TestRunResult
+import com.malinskiy.marathon.android.AndroidDevice
+import com.malinskiy.marathon.android.exception.deviceLostRegex
+import com.malinskiy.marathon.android.toMarathonStatus
+import com.malinskiy.marathon.android.toTest
+import com.malinskiy.marathon.device.Device
+import com.malinskiy.marathon.device.DevicePoolId
+import com.malinskiy.marathon.device.toDeviceInfo
+import com.malinskiy.marathon.execution.Attachment
+import com.malinskiy.marathon.execution.AttachmentType
+import com.malinskiy.marathon.execution.TestBatchResults
+import com.malinskiy.marathon.execution.TestStatus
+import com.malinskiy.marathon.io.FileManager
+import com.malinskiy.marathon.io.FileType
+import com.malinskiy.marathon.log.MarathonLogging
+import com.malinskiy.marathon.report.attachment.AttachmentListener
+import com.malinskiy.marathon.report.attachment.AttachmentProvider
+import com.malinskiy.marathon.test.Test
+import com.malinskiy.marathon.test.TestBatch
+import com.malinskiy.marathon.test.toTestName
+import com.malinskiy.marathon.time.Timer
+import kotlinx.coroutines.CompletableDeferred
+import kotlin.system.measureTimeMillis
+
+class TestCoverageResultListener(
+    private val fileManager: FileManager,
+    private val pool: DevicePoolId,
+    private val device: AndroidDevice,
+    private val testBatch: TestBatch,
+    private val deferred: CompletableDeferred<TestBatchResults>,
+    private val timer: Timer
+) : AbstractTestRunResultListener(), AttachmentProvider, AttachmentListener {
+
+    private val logger = MarathonLogging.logger("CoverageListener")
+
+    val attachmentListeners = mutableListOf<AttachmentListener>()
+
+    private val attachments: MutableMap<Test, MutableList<Attachment>> = mutableMapOf()
+
+    init {
+        registerListener(this)
+    }
+
+    override fun onAttachment(test: Test, attachment: Attachment) {
+        val list = attachments[test]
+        if (list == null) {
+            attachments[test] = mutableListOf()
+        }
+
+        attachments[test]!!.add(attachment)
+    }
+
+    override fun handleTestRunResults(runResult: TestRunResult) {
+        val tests = testBatch.tests.associateBy { it.identifier() }
+
+        val passed = mutableListOf<com.malinskiy.marathon.execution.TestResult>()
+        val failed = mutableListOf<com.malinskiy.marathon.execution.TestResult>()
+        val infraFailures = mutableListOf<com.malinskiy.marathon.execution.TestResult>()
+        val incomplete = mutableListOf<com.malinskiy.marathon.execution.TestResult>()
+
+        runResult.testResults.forEach { entry ->
+            val result = entry.toTestResult(device)
+            if (result.test.method == "null") return@forEach
+
+            when (entry.value.status) {
+                TestResult.TestStatus.PASSED, TestResult.TestStatus.ASSUMPTION_FAILURE, TestResult.TestStatus.IGNORED -> passed.add(
+                    result
+                )
+
+                TestResult.TestStatus.FAILURE -> {
+                    val isTimedOut =
+                        result.stacktrace?.contains(TimeoutException::class.java.canonicalName)
+                            ?: false
+                    val isDeviceLost = result.stacktrace?.contains(deviceLostRegex) ?: false
+
+                    if (isTimedOut || isDeviceLost) {
+                        logger.warn { "infraFailure = ${result.test.toTestName()}, ${device.serialNumber}" }
+                        infraFailures.add(result)
+                    } else {
+                        failed.add(result)
+                    }
+                }
+
+                TestResult.TestStatus.INCOMPLETE, null -> {
+                    logger.warn { "uncompleted = ${result.test.toTestName()}, ${device.serialNumber}" }
+                    incomplete.add(result)
+                }
+            }
+        }
+
+        val noStatus = tests
+            .filterNot { expectedTest ->
+                runResult.testResults.containsKey(expectedTest.key)
+            }
+            .values
+            .createUncompletedTestResults(runResult, device)
+
+        noStatus.forEach {
+            logger.warn { "noStatus = ${it.test.toTestName()}, ${device.serialNumber}" }
+        }
+
+        logger.debug(
+            "Batch test results: " +
+                    "passed=${passed.size}; " +
+                    "failed=${failed.size}; " +
+                    "infraFailures=${infraFailures.size}; " +
+                    "uncompleted=${incomplete.size}; " +
+                    "noStatus=${noStatus.size}"
+        )
+        runResult.testResults.forEach { entry ->
+            pullCoverageFile(entry.key)
+            removeCoverageFile()
+            val result = entry.toTestResult(device)
+
+
+            if (result.isSuccess) {
+                passed.add(result)
+            } else {
+                failed.add(result)
+            }
+            deferred.complete(
+                TestBatchResults(
+                    device, passed, failed,
+                    uncompleted = emptyList()
+                )
+            )
+        }
+    }
+
+    private fun pullCoverageFile(test: TestIdentifier) {
+        val localCoverageFile =
+            fileManager.createFile(FileType.COVERAGE, pool, device.toDeviceInfo(), test.toTest())
+        val remoteFilePath = device.fileManager.remoteCoverageForTest()
+        val millis = measureTimeMillis {
+            device.fileManager.pullFile(remoteFilePath, localCoverageFile.toString())
+        }
+        logger.trace { "Pulling file finished in ${millis}ms $remoteFilePath " }
+        attachmentListeners.forEach {
+            it.onAttachment(
+                test.toTest(),
+                Attachment(localCoverageFile, AttachmentType.COVERAGE)
+            )
+        }
+    }
+
+    private fun removeCoverageFile() {
+        val remoteFilePath = device.fileManager.remoteCoverageForTest()
+        val millis = measureTimeMillis {
+            device.fileManager.removeRemotePath(remoteFilePath)
+        }
+        logger.trace { "Removed file in ${millis}ms $remoteFilePath" }
+    }
+
+    override fun registerListener(listener: AttachmentListener) {
+        attachmentListeners.add(listener)
+    }
+
+    private fun Test.identifier() = TestIdentifier("$pkg.$clazz", method)
+
+    private fun Map.Entry<TestIdentifier, TestResult>.toTestResult(device: Device): com.malinskiy.marathon.execution.TestResult {
+        val testInstanceFromBatch =
+            testBatch.tests.find { "${it.pkg}.${it.clazz}" == key.className && it.method == key.testName }
+        val test = key.toTest()
+        val attachments = attachments[test] ?: emptyList<Attachment>()
+        return com.malinskiy.marathon.execution.TestResult(
+            test = testInstanceFromBatch ?: test,
+            device = device.toDeviceInfo(),
+            status = value.status.toMarathonStatus(),
+            startTime = value.startTime,
+            endTime = value.endTime,
+            stacktrace = value.stackTrace,
+            attachments = attachments
+        )
+    }
+
+    private fun Collection<Test>.createUncompletedTestResults(
+        testRunResult: TestRunResult,
+        device: Device
+    ): Collection<com.malinskiy.marathon.execution.TestResult> {
+
+        val lastCompletedTestEndTime = testRunResult
+            .testResults
+            .values
+            .maxBy { it.endTime }
+            ?.endTime
+            ?: timer.currentTimeMillis()
+
+        return map {
+            com.malinskiy.marathon.execution.TestResult(
+                it,
+                device.toDeviceInfo(),
+                TestStatus.INCOMPLETE,
+                lastCompletedTestEndTime,
+                lastCompletedTestEndTime,
+                testRunResult.runFailureMessage
+            )
+        }
+    }
+}