Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Extract code coverage from Android device #21

Open
wants to merge 23 commits into
base: feature/agoda-android-ci
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import com.google.gson.Gson
import com.malinskiy.marathon.analytics.tracker.local.DeviceTracker
import com.malinskiy.marathon.analytics.tracker.local.JUnitTracker
import com.malinskiy.marathon.analytics.tracker.local.RawTestResultTracker
import com.malinskiy.marathon.analytics.tracker.local.TestCoverageTracker
import com.malinskiy.marathon.analytics.tracker.local.TestResultsTracker
import com.malinskiy.marathon.analytics.tracker.remote.influx.InfluxDbProvider
import com.malinskiy.marathon.analytics.tracker.remote.influx.InfluxDbTracker
Expand All @@ -12,6 +13,7 @@ import com.malinskiy.marathon.execution.Configuration
import com.malinskiy.marathon.io.FileManager
import com.malinskiy.marathon.report.allure.AllureTestListener
import com.malinskiy.marathon.report.internal.DeviceInfoReporter
import com.malinskiy.marathon.report.internal.TestCoverageReporter
import com.malinskiy.marathon.report.internal.TestResultReporter
import com.malinskiy.marathon.report.junit.JUnitReporter
import java.io.File
Expand All @@ -26,13 +28,24 @@ internal class TrackerFactory(private val configuration: Configuration,
val allureTracker = AllureTestListener(configuration, File(configuration.outputDir, "allure-results"))

fun create(): Tracker {
val defaultTrackers = listOf(
val defaultTrackers = if (configuration.isCodeCoverageEnabled) {
listOf(
TestCoverageTracker(TestCoverageReporter(fileManager)),
JUnitTracker(JUnitReporter(fileManager)),
DeviceTracker(deviceInfoReporter),
TestResultsTracker(testResultReporter),
rawTestResultTracker,
allureTracker
)
)
} else {
listOf(
JUnitTracker(JUnitReporter(fileManager)),
DeviceTracker(deviceInfoReporter),
TestResultsTracker(testResultReporter),
rawTestResultTracker,
allureTracker
)
}
return when {
configuration.analyticsConfiguration is InfluxDbConfiguration -> {
val config = configuration.analyticsConfiguration
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
package com.malinskiy.marathon.analytics.tracker.local

import com.malinskiy.marathon.analytics.tracker.NoOpTracker
import com.malinskiy.marathon.device.DeviceInfo
import com.malinskiy.marathon.device.DevicePoolId
import com.malinskiy.marathon.execution.TestResult
import com.malinskiy.marathon.report.internal.TestCoverageReporter

internal class TestCoverageTracker(private val testCoverageReporter: TestCoverageReporter) :
NoOpTracker() {

override fun trackTestFinished(
poolId: DevicePoolId,
device: DeviceInfo,
testResult: TestResult
) {
testCoverageReporter.testFinished(poolId, device, testResult)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -5,25 +5,35 @@ import com.malinskiy.marathon.device.DeviceInfo
import com.malinskiy.marathon.device.DevicePoolId
import com.malinskiy.marathon.execution.TestResult
import com.malinskiy.marathon.execution.TestStatus
import com.malinskiy.marathon.log.MarathonLogging
import com.malinskiy.marathon.test.toSafeTestName
import org.influxdb.InfluxDB
import org.influxdb.dto.Point
import java.util.concurrent.TimeUnit

internal class InfluxDbTracker(private val influxDb: InfluxDB) : NoOpTracker() {

private val logger = MarathonLogging.logger("InfluxDB")

override fun trackRawTestRun(poolId: DevicePoolId, device: DeviceInfo, testResult: TestResult) {
influxDb.write(Point.measurement("tests")
.time(System.currentTimeMillis(), TimeUnit.MILLISECONDS)
.tag("testname", testResult.test.toSafeTestName())
.tag("package", testResult.test.pkg)
.tag("class", testResult.test.clazz)
.tag("method", testResult.test.method)
.tag("deviceSerial", device.serialNumber)
.addField("ignored", if (testResult.isIgnored) 1.0 else 0.0)
.addField("success", if (testResult.status == TestStatus.PASSED) 1.0 else 0.0)
.addField("duration", testResult.durationMillis())
.build())
val point = Point.measurement("tests")
.time(System.currentTimeMillis(), TimeUnit.MILLISECONDS)
.tag("testname", testResult.test.toSafeTestName())
.tag("package", testResult.test.pkg)
.tag("class", testResult.test.clazz)
.tag("method", testResult.test.method)
.tag("deviceSerial", device.serialNumber)
.addField("ignored", if (testResult.isIgnored) 1.0 else 0.0)
.addField("success", if (testResult.status == TestStatus.PASSED) 1.0 else 0.0)
.addField("duration", testResult.durationMillis())
.build()
runCatching {
influxDb.write(point)
}.onSuccess {
logger.trace { "Tracked in influxDB $point" }
}.onFailure {
logger.error(it) { "Error writing to influxDB with $point" }
}
Comment on lines +19 to +36
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We are having an issue writing to influxDB 400 Bad Request, just want to log the request marathon is sending.

}

override fun terminate() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,13 @@ data class Attachment(val file: File, val type: AttachmentType)
enum class AttachmentType {
SCREENSHOT,
VIDEO,
COVERAGE,
LOG;

fun toMimeType() = when(this) {
SCREENSHOT -> "image/gif"
VIDEO -> "video/mp4"
COVERAGE -> "application/ec"
LOG -> "text/txt"
}
}
4 changes: 2 additions & 2 deletions core/src/main/kotlin/com/malinskiy/marathon/io/FileType.kt
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,6 @@ enum class FileType(val dir: String, val suffix: String) {
LOG("logs", "log"),
DEVICE_INFO("devices", "json"),
VIDEO("video", "mp4"),
SCREENSHOT("screenshot", "gif")

SCREENSHOT("screenshot", "gif"),
COVERAGE("coverage", "ec")
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
package com.malinskiy.marathon.report.internal

import com.malinskiy.marathon.device.DeviceInfo
import com.malinskiy.marathon.device.DevicePoolId
import com.malinskiy.marathon.execution.TestResult
import com.malinskiy.marathon.io.FileManager
import com.malinskiy.marathon.io.FileType

class TestCoverageReporter(private val fileManager: FileManager) {

fun testFinished(poolId: DevicePoolId, device: DeviceInfo, testResult: TestResult) {
val file = fileManager.createFile(FileType.COVERAGE, poolId, device, testResult.test)
file.setWritable(true)
testResult.attachments.forEach {
val bytes = it.file.readBytes()
file.appendBytes(bytes)
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import com.malinskiy.marathon.android.executor.listeners.DebugTestRunListener
import com.malinskiy.marathon.android.executor.listeners.LogCatListener
import com.malinskiy.marathon.android.executor.listeners.NoOpTestRunListener
import com.malinskiy.marathon.android.executor.listeners.ProgressTestRunListener
import com.malinskiy.marathon.android.executor.listeners.TestCoverageResultListener
import com.malinskiy.marathon.android.executor.listeners.TestRunResultsListener
import com.malinskiy.marathon.android.executor.listeners.screenshot.ScreenCapturerTestRunListener
import com.malinskiy.marathon.android.executor.listeners.video.ScreenRecorderTestRunListener
Expand Down Expand Up @@ -164,14 +165,27 @@ class AndroidDevice(val ddmsDevice: IDevice,

val timer = SystemTimer()

val coverageListener = TestCoverageResultListener(fileManager, devicePoolId, this)

return CompositeTestRunListener(
listOf(
if (configuration.isCodeCoverageEnabled) {
listOf(
coverageListener,
recorderListener,
logCatListener,
TestRunResultsListener(testBatch, this, deferred, timer, attachmentProviders),
DebugTestRunListener(this),
ProgressTestRunListener(this, devicePoolId, progressReporter)
)
)
} else {
listOf(
recorderListener,
logCatListener,
TestRunResultsListener(testBatch, this, deferred, timer, attachmentProviders),
DebugTestRunListener(this),
ProgressTestRunListener(this, devicePoolId, progressReporter)
)
}
)
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,10 @@ class RemoteFileManager(private val device: IDevice) {
return remoteFileForTest(videoFileName(test))
}

fun remoteCoverageForTest(): String {
return remoteFileForTest("coverage.ec")
}

private fun remoteFileForTest(filename: String): String {
return "$outputDir/$filename"
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
package com.malinskiy.marathon.android.executor.listeners

import com.android.ddmlib.testrunner.TestIdentifier
import com.malinskiy.marathon.android.AndroidDevice
import com.malinskiy.marathon.android.toTest
import com.malinskiy.marathon.device.DevicePoolId
import com.malinskiy.marathon.device.toDeviceInfo
import com.malinskiy.marathon.execution.Attachment
import com.malinskiy.marathon.execution.AttachmentType
import com.malinskiy.marathon.io.FileManager
import com.malinskiy.marathon.io.FileType
import com.malinskiy.marathon.log.MarathonLogging
import com.malinskiy.marathon.report.attachment.AttachmentListener
import com.malinskiy.marathon.report.attachment.AttachmentProvider
import kotlin.system.measureTimeMillis

class TestCoverageResultListener(
private val fileManager: FileManager,
private val pool: DevicePoolId,
private val device: AndroidDevice
) : NoOpTestRunListener(), AttachmentProvider {

private val logger = MarathonLogging.logger("CoverageListener")

private val attachmentListeners = mutableListOf<AttachmentListener>()

private lateinit var lastTestIdentifier: TestIdentifier

override fun testEnded(test: TestIdentifier, testMetrics: Map<String, String>) {
super.testEnded(test, testMetrics)
lastTestIdentifier = test
}

override fun testRunEnded(elapsedTime: Long, runMetrics: Map<String, String>) {
super.testRunEnded(elapsedTime, runMetrics)
pullCoverageFile(lastTestIdentifier)
removeCoverageFile()
}

private fun pullCoverageFile(test: TestIdentifier) {
val localCoverageFile =
fileManager.createFile(FileType.COVERAGE, pool, device.toDeviceInfo(), test.toTest())
val remoteFilePath = device.fileManager.remoteCoverageForTest()
logger.trace { "Pulling from $remoteFilePath" }
val millis = measureTimeMillis {
device.fileManager.pullFile(remoteFilePath, localCoverageFile.toString())
}
logger.trace { "Pulling file finished in ${millis}ms $remoteFilePath " }
attachmentListeners.forEach {
it.onAttachment(
test.toTest(),
Attachment(localCoverageFile, AttachmentType.COVERAGE)
)
}
}

private fun removeCoverageFile() {
val remoteFilePath = device.fileManager.remoteCoverageForTest()
val millis = measureTimeMillis {
device.fileManager.removeRemotePath(remoteFilePath)
}
logger.trace { "Removed file in ${millis}ms $remoteFilePath" }
}

override fun registerListener(listener: AttachmentListener) {
attachmentListeners.add(listener)
}
}