diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ModuleExecutionSettingsFactoryImpl.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ModuleExecutionSettingsFactoryImpl.java index 9f0d08d14a5..7ce67824d55 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ModuleExecutionSettingsFactoryImpl.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ModuleExecutionSettingsFactoryImpl.java @@ -60,24 +60,31 @@ public ModuleExecutionSettings create(JvmInfo jvmInfo, @Nullable String moduleNa buildTracerEnvironment(repositoryRoot, jvmInfo, moduleName); CiVisibilitySettings ciVisibilitySettings = getCiVisibilitySettings(tracerEnvironment); - boolean codeCoverageEnabled = isCodeCoverageEnabled(ciVisibilitySettings); boolean itrEnabled = isItrEnabled(ciVisibilitySettings); + boolean codeCoverageEnabled = isCodeCoverageEnabled(ciVisibilitySettings); + boolean testSkippingEnabled = isTestSkippingEnabled(ciVisibilitySettings); boolean flakyTestRetriesEnabled = isFlakyTestRetriesEnabled(ciVisibilitySettings); boolean earlyFlakeDetectionEnabled = isEarlyFlakeDetectionEnabled(ciVisibilitySettings); Map systemProperties = getPropertiesPropagatedToChildProcess( - codeCoverageEnabled, itrEnabled, flakyTestRetriesEnabled, earlyFlakeDetectionEnabled); + itrEnabled, + codeCoverageEnabled, + testSkippingEnabled, + flakyTestRetriesEnabled, + earlyFlakeDetectionEnabled); LOGGER.info( "CI Visibility settings ({}, {}):\n" - + "Per-test code coverage - {},\n" + "Intelligent Test Runner - {},\n" + + "Per-test code coverage - {},\n" + + "Tests skipping - {},\n" + "Early flakiness detection - {},\n" - + "Flaky test retries - {}", + + "Auto test retries - {}", repositoryRoot, jvmInfo, - codeCoverageEnabled, itrEnabled, + codeCoverageEnabled, + testSkippingEnabled, earlyFlakeDetectionEnabled, flakyTestRetriesEnabled); @@ -104,8 +111,9 @@ public ModuleExecutionSettings create(JvmInfo jvmInfo, @Nullable String moduleNa List coverageEnabledPackages = getCoverageEnabledPackages(codeCoverageEnabled); return new ModuleExecutionSettings( - codeCoverageEnabled, itrEnabled, + codeCoverageEnabled, + testSkippingEnabled, flakyTestRetriesEnabled, // knownTests being null covers the following cases: // - early flake detection is disabled in remote settings @@ -177,7 +185,12 @@ private CiVisibilitySettings getCiVisibilitySettings(TracerEnvironment tracerEnv } private boolean isItrEnabled(CiVisibilitySettings ciVisibilitySettings) { - return ciVisibilitySettings.isTestsSkippingEnabled() && config.isCiVisibilityItrEnabled(); + return ciVisibilitySettings.isItrEnabled() && config.isCiVisibilityItrEnabled(); + } + + private boolean isTestSkippingEnabled(CiVisibilitySettings ciVisibilitySettings) { + return ciVisibilitySettings.isTestsSkippingEnabled() + && config.isCiVisibilityTestSkippingEnabled(); } private boolean isCodeCoverageEnabled(CiVisibilitySettings ciVisibilitySettings) { @@ -196,8 +209,9 @@ private boolean isEarlyFlakeDetectionEnabled(CiVisibilitySettings ciVisibilitySe } private Map getPropertiesPropagatedToChildProcess( - boolean codeCoverageEnabled, boolean itrEnabled, + boolean codeCoverageEnabled, + boolean testSkippingEnabled, boolean flakyTestRetriesEnabled, boolean earlyFlakeDetectionEnabled) { Map propagatedSystemProperties = new HashMap<>(); @@ -212,14 +226,19 @@ private Map getPropertiesPropagatedToChildProcess( } } + propagatedSystemProperties.put( + Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_ITR_ENABLED), + Boolean.toString(itrEnabled)); + propagatedSystemProperties.put( Strings.propertyNameToSystemPropertyName( CiVisibilityConfig.CIVISIBILITY_CODE_COVERAGE_ENABLED), Boolean.toString(codeCoverageEnabled)); propagatedSystemProperties.put( - Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_ITR_ENABLED), - Boolean.toString(itrEnabled)); + Strings.propertyNameToSystemPropertyName( + CiVisibilityConfig.CIVISIBILITY_TEST_SKIPPING_ENABLED), + Boolean.toString(testSkippingEnabled)); propagatedSystemProperties.put( Strings.propertyNameToSystemPropertyName( diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ModuleExecutionSettingsSerializer.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ModuleExecutionSettingsSerializer.java index db0ba596f61..ab2697f54bc 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ModuleExecutionSettingsSerializer.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ModuleExecutionSettingsSerializer.java @@ -11,17 +11,19 @@ public class ModuleExecutionSettingsSerializer { - private static final int CODE_COVERAGE_ENABLED_FLAG = 1; - private static final int ITR_ENABLED_FLAG = 2; - private static final int FLAKY_TEST_RETRIES_ENABLED_FLAG = 4; + private static final int ITR_ENABLED_FLAG = 1; + private static final int CODE_COVERAGE_ENABLED_FLAG = 2; + private static final int TEST_SKIPPING_ENABLED_FLAG = 4; + private static final int FLAKY_TEST_RETRIES_ENABLED_FLAG = 8; public static ByteBuffer serialize(ModuleExecutionSettings settings) { Serializer s = new Serializer(); byte flags = (byte) - ((settings.isCodeCoverageEnabled() ? CODE_COVERAGE_ENABLED_FLAG : 0) - | (settings.isItrEnabled() ? ITR_ENABLED_FLAG : 0) + ((settings.isItrEnabled() ? ITR_ENABLED_FLAG : 0) + | (settings.isCodeCoverageEnabled() ? CODE_COVERAGE_ENABLED_FLAG : 0) + | (settings.isTestSkippingEnabled() ? TEST_SKIPPING_ENABLED_FLAG : 0) | (settings.isFlakyTestRetriesEnabled() ? FLAKY_TEST_RETRIES_ENABLED_FLAG : 0)); s.write(flags); @@ -45,8 +47,9 @@ public static ByteBuffer serialize(ModuleExecutionSettings settings) { public static ModuleExecutionSettings deserialize(ByteBuffer buffer) { byte flags = Serializer.readByte(buffer); - boolean codeCoverageEnabled = (flags & CODE_COVERAGE_ENABLED_FLAG) != 0; boolean itrEnabled = (flags & ITR_ENABLED_FLAG) != 0; + boolean codeCoverageEnabled = (flags & CODE_COVERAGE_ENABLED_FLAG) != 0; + boolean testSkippingEnabled = (flags & TEST_SKIPPING_ENABLED_FLAG) != 0; boolean flakyTestRetriesEnabled = (flags & FLAKY_TEST_RETRIES_ENABLED_FLAG) != 0; EarlyFlakeDetectionSettings earlyFlakeDetectionSettings = @@ -69,8 +72,9 @@ public static ModuleExecutionSettings deserialize(ByteBuffer buffer) { List codeCoveragePackages = Serializer.readStringList(buffer); return new ModuleExecutionSettings( - codeCoverageEnabled, itrEnabled, + codeCoverageEnabled, + testSkippingEnabled, flakyTestRetriesEnabled, earlyFlakeDetectionSettings, systemProperties, diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/CoverageProbeStoreFactory.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/CoverageProbeStoreFactory.java index 3c9b4f7fb95..772d165f46a 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/CoverageProbeStoreFactory.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/CoverageProbeStoreFactory.java @@ -1,8 +1,9 @@ package datadog.trace.civisibility.coverage; +import datadog.trace.api.civisibility.config.TestIdentifier; import datadog.trace.api.civisibility.coverage.CoverageProbeStore; import datadog.trace.civisibility.source.SourcePathResolver; public interface CoverageProbeStoreFactory extends CoverageProbeStore.Registry { - CoverageProbeStore create(SourcePathResolver sourcePathResolver); + CoverageProbeStore create(TestIdentifier testIdentifier, SourcePathResolver sourcePathResolver); } diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/NoopCoverageProbeStore.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/NoopCoverageProbeStore.java index 20f4492c1ab..cee1070e39d 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/NoopCoverageProbeStore.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/NoopCoverageProbeStore.java @@ -1,12 +1,13 @@ package datadog.trace.civisibility.coverage; +import datadog.trace.api.civisibility.config.TestIdentifier; import datadog.trace.api.civisibility.coverage.CoverageProbeStore; import datadog.trace.api.civisibility.coverage.TestReport; import datadog.trace.civisibility.source.SourcePathResolver; import javax.annotation.Nullable; public class NoopCoverageProbeStore implements CoverageProbeStore { - private static final CoverageProbeStore INSTANCE = new NoopCoverageProbeStore(); + public static final CoverageProbeStore INSTANCE = new NoopCoverageProbeStore(); @Override public void record(Class clazz) {} @@ -33,7 +34,8 @@ public static final class NoopCoverageProbeStoreFactory implements CoverageProbe public void setTotalProbeCount(String className, int totalProbeCount) {} @Override - public CoverageProbeStore create(SourcePathResolver sourcePathResolver) { + public CoverageProbeStore create( + TestIdentifier testIdentifier, SourcePathResolver sourcePathResolver) { return INSTANCE; } } diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/SegmentlessTestProbes.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/SegmentlessTestProbes.java index 3e21dd01adc..788d925e42e 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/SegmentlessTestProbes.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/SegmentlessTestProbes.java @@ -1,5 +1,6 @@ package datadog.trace.civisibility.coverage; +import datadog.trace.api.civisibility.config.TestIdentifier; import datadog.trace.api.civisibility.coverage.CoverageProbeStore; import datadog.trace.api.civisibility.coverage.TestReport; import datadog.trace.api.civisibility.coverage.TestReportFileEntry; @@ -178,7 +179,8 @@ public void setTotalProbeCount(String className, int totalProbeCount) { } @Override - public CoverageProbeStore create(SourcePathResolver sourcePathResolver) { + public CoverageProbeStore create( + TestIdentifier testIdentifier, SourcePathResolver sourcePathResolver) { return new SegmentlessTestProbes(sourcePathResolver, metricCollector); } } diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/SkippableAwareCoverageProbeStoreFactory.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/SkippableAwareCoverageProbeStoreFactory.java new file mode 100644 index 00000000000..e1c85fbef3d --- /dev/null +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/SkippableAwareCoverageProbeStoreFactory.java @@ -0,0 +1,38 @@ +package datadog.trace.civisibility.coverage; + +import datadog.trace.api.civisibility.config.TestIdentifier; +import datadog.trace.api.civisibility.coverage.CoverageProbeStore; +import datadog.trace.civisibility.source.SourcePathResolver; +import java.util.Collection; + +/** + * Coverage store factory returns no-op stores for skippable tests. This is done to reduce coverage + * overhead. The idea is that if a test is skippable then it means none of the files it covers were + * changed. If none of the files were changed then gathering coverage for the test make no sense, + * because it will be the same as previously gathered coverage that the backend already has. + */ +public class SkippableAwareCoverageProbeStoreFactory implements CoverageProbeStoreFactory { + private final Collection skippableTests; + private final CoverageProbeStoreFactory delegate; + + public SkippableAwareCoverageProbeStoreFactory( + Collection skippableTests, CoverageProbeStoreFactory delegate) { + this.skippableTests = skippableTests; + this.delegate = delegate; + } + + @Override + public CoverageProbeStore create( + TestIdentifier testIdentifier, SourcePathResolver sourcePathResolver) { + if (skippableTests.contains(testIdentifier)) { + return NoopCoverageProbeStore.INSTANCE; + } else { + return delegate.create(testIdentifier, sourcePathResolver); + } + } + + @Override + public void setTotalProbeCount(String className, int totalProbeCount) { + delegate.setTotalProbeCount(className, totalProbeCount); + } +} diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/TestProbes.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/TestProbes.java index dacb74f8640..f1761227f60 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/TestProbes.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/TestProbes.java @@ -1,5 +1,6 @@ package datadog.trace.civisibility.coverage; +import datadog.trace.api.civisibility.config.TestIdentifier; import datadog.trace.api.civisibility.coverage.CoverageProbeStore; import datadog.trace.api.civisibility.coverage.TestReport; import datadog.trace.api.civisibility.coverage.TestReportFileEntry; @@ -235,7 +236,8 @@ public void setTotalProbeCount(String className, int totalProbeCount) { } @Override - public CoverageProbeStore create(SourcePathResolver sourcePathResolver) { + public CoverageProbeStore create( + TestIdentifier testIdentifier, SourcePathResolver sourcePathResolver) { return new TestProbes(sourcePathResolver, metricCollector); } } diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/AbstractTestModule.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/AbstractTestModule.java index 99d3ec7e0f7..92224505e67 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/AbstractTestModule.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/AbstractTestModule.java @@ -12,7 +12,6 @@ import datadog.trace.bootstrap.instrumentation.api.Tags; import datadog.trace.civisibility.InstrumentationType; import datadog.trace.civisibility.codeowners.Codeowners; -import datadog.trace.civisibility.coverage.CoverageProbeStoreFactory; import datadog.trace.civisibility.decorator.TestDecorator; import datadog.trace.civisibility.source.MethodLinesResolver; import datadog.trace.civisibility.source.SourcePathResolver; @@ -30,7 +29,6 @@ public abstract class AbstractTestModule { protected final SourcePathResolver sourcePathResolver; protected final Codeowners codeowners; protected final MethodLinesResolver methodLinesResolver; - protected final CoverageProbeStoreFactory coverageProbeStoreFactory; private final Consumer onSpanFinish; public AbstractTestModule( @@ -45,7 +43,6 @@ public AbstractTestModule( SourcePathResolver sourcePathResolver, Codeowners codeowners, MethodLinesResolver methodLinesResolver, - CoverageProbeStoreFactory coverageProbeStoreFactory, Consumer onSpanFinish) { this.sessionId = sessionId; this.moduleName = moduleName; @@ -55,7 +52,6 @@ public AbstractTestModule( this.sourcePathResolver = sourcePathResolver; this.codeowners = codeowners; this.methodLinesResolver = methodLinesResolver; - this.coverageProbeStoreFactory = coverageProbeStoreFactory; this.onSpanFinish = onSpanFinish; if (startTime != null) { diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestFrameworkModule.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestFrameworkModule.java index af5f8a893b2..42723fcffd1 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestFrameworkModule.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestFrameworkModule.java @@ -15,14 +15,6 @@ TestSuiteImpl testSuiteStart( boolean parallelized, TestFrameworkInstrumentation instrumentation); - /** - * Checks if a given test can be skipped with Intelligent Test Runner or not - * - * @param test Test to be checked - * @return {@code true} if the test can be skipped, {@code false} otherwise - */ - boolean isSkippable(TestIdentifier test); - /** * Checks if a given test is "new" or not. A test is considered "new" if the backend has no * information about it. @@ -33,6 +25,14 @@ TestSuiteImpl testSuiteStart( */ boolean isNew(TestIdentifier test); + /** + * Checks if a given test should be skipped with Intelligent Test Runner or not + * + * @param test Test to be checked + * @return {@code true} if the test can be skipped, {@code false} otherwise + */ + boolean shouldBeSkipped(TestIdentifier test); + /** * Checks if a given test can be skipped with Intelligent Test Runner or not. If the test is * considered skippable, the count of skippable tests is incremented. diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestImpl.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestImpl.java index 4f1a306ae57..852581c0d49 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestImpl.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestImpl.java @@ -8,6 +8,7 @@ import datadog.trace.api.civisibility.DDTest; import datadog.trace.api.civisibility.InstrumentationBridge; import datadog.trace.api.civisibility.InstrumentationTestBridge; +import datadog.trace.api.civisibility.config.TestIdentifier; import datadog.trace.api.civisibility.coverage.CoverageBridge; import datadog.trace.api.civisibility.coverage.CoverageProbeStore; import datadog.trace.api.civisibility.domain.TestContext; @@ -56,6 +57,7 @@ public TestImpl( String moduleName, String testSuiteName, String testName, + @Nullable String testParameters, @Nullable String itrCorrelationId, @Nullable Long startTime, @Nullable Class testClass, @@ -76,7 +78,9 @@ public TestImpl( this.suiteId = suiteId; this.onSpanFinish = onSpanFinish; - CoverageProbeStore probeStore = coverageProbeStoreFactory.create(sourcePathResolver); + TestIdentifier identifier = new TestIdentifier(testSuiteName, testName, testParameters, null); + CoverageProbeStore probeStore = + coverageProbeStoreFactory.create(identifier, sourcePathResolver); CoverageBridge.setThreadLocalCoverageProbeStore(probeStore); this.context = new TestContextImpl(probeStore); diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestSuiteImpl.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestSuiteImpl.java index e2859fab3a6..eb5436980fd 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestSuiteImpl.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestSuiteImpl.java @@ -38,7 +38,7 @@ public class TestSuiteImpl implements DDTestSuite { private final InstrumentationType instrumentationType; private final TestFrameworkInstrumentation instrumentation; private final Config config; - CiVisibilityMetricCollector metricCollector; + private final CiVisibilityMetricCollector metricCollector; private final TestDecorator testDecorator; private final SourcePathResolver sourcePathResolver; private final Codeowners codeowners; @@ -189,6 +189,14 @@ public void end(@Nullable Long endTime) { @Override public TestImpl testStart( String testName, @Nullable Method testMethod, @Nullable Long startTime) { + return testStart(testName, null, testMethod, startTime); + } + + public TestImpl testStart( + String testName, + @Nullable String testParameters, + @Nullable Method testMethod, + @Nullable Long startTime) { return new TestImpl( sessionId, moduleId, @@ -196,6 +204,7 @@ public TestImpl testStart( moduleName, testSuiteName, testName, + testParameters, itrCorrelationId, startTime, testClass, diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/BuildSystemModuleImpl.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/BuildSystemModuleImpl.java index aae7efd8701..c77b93b0f10 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/BuildSystemModuleImpl.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/BuildSystemModuleImpl.java @@ -9,7 +9,6 @@ import datadog.trace.bootstrap.instrumentation.api.Tags; import datadog.trace.civisibility.InstrumentationType; import datadog.trace.civisibility.codeowners.Codeowners; -import datadog.trace.civisibility.coverage.CoverageProbeStoreFactory; import datadog.trace.civisibility.coverage.CoverageUtils; import datadog.trace.civisibility.decorator.TestDecorator; import datadog.trace.civisibility.domain.AbstractTestModule; @@ -40,7 +39,7 @@ public class BuildSystemModuleImpl extends AbstractTestModule implements BuildSy private final TestModuleRegistry testModuleRegistry; private final LongAdder testsSkipped = new LongAdder(); private volatile boolean codeCoverageEnabled; - private volatile boolean itrEnabled; + private volatile boolean testSkippingEnabled; private final Object coverageDataLock = new Object(); @GuardedBy("coverageDataLock") @@ -61,7 +60,6 @@ public BuildSystemModuleImpl( SourcePathResolver sourcePathResolver, Codeowners codeowners, MethodLinesResolver methodLinesResolver, - CoverageProbeStoreFactory coverageProbeStoreFactory, RepoIndexProvider repoIndexProvider, TestModuleRegistry testModuleRegistry, Consumer onSpanFinish) { @@ -77,7 +75,6 @@ public BuildSystemModuleImpl( sourcePathResolver, codeowners, methodLinesResolver, - coverageProbeStoreFactory, onSpanFinish); this.repoRoot = repoRoot; this.signalServerAddress = signalServerAddress; @@ -124,8 +121,8 @@ public void onModuleExecutionResultReceived(ModuleExecutionResult result) { if (result.isCoverageEnabled()) { codeCoverageEnabled = true; } - if (result.isItrEnabled()) { - itrEnabled = true; + if (result.isTestSkippingEnabled()) { + testSkippingEnabled = true; } if (result.isEarlyFlakeDetectionEnabled()) { setTag(Tags.TEST_EARLY_FLAKE_ENABLED, true); @@ -163,7 +160,7 @@ public void end(@Nullable Long endTime) { setTag(Tags.TEST_CODE_COVERAGE_ENABLED, true); } - if (itrEnabled) { + if (testSkippingEnabled) { setTag(Tags.TEST_ITR_TESTS_SKIPPING_ENABLED, true); setTag(Tags.TEST_ITR_TESTS_SKIPPING_TYPE, "test"); diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/BuildSystemSessionImpl.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/BuildSystemSessionImpl.java index 9a25dfca73d..ec532c7d956 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/BuildSystemSessionImpl.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/BuildSystemSessionImpl.java @@ -54,7 +54,7 @@ public class BuildSystemSessionImpl extends AbstractTestSession implements Build private final RepoIndexProvider repoIndexProvider; protected final LongAdder testsSkipped = new LongAdder(); private volatile boolean codeCoverageEnabled; - private volatile boolean itrEnabled; + private volatile boolean testSkippingEnabled; private final Object coverageDataLock = new Object(); @GuardedBy("coverageDataLock") @@ -114,8 +114,8 @@ private SignalResponse onModuleExecutionResultReceived(ModuleExecutionResult res if (result.isCoverageEnabled()) { codeCoverageEnabled = true; } - if (result.isItrEnabled()) { - itrEnabled = true; + if (result.isTestSkippingEnabled()) { + testSkippingEnabled = true; } if (result.isEarlyFlakeDetectionEnabled()) { setTag(Tags.TEST_EARLY_FLAKE_ENABLED, true); @@ -166,8 +166,9 @@ private SignalResponse onModuleExecutionSettingsRequestReceived(ModuleSettingsRe ModuleExecutionSettings moduleSettings = new ModuleExecutionSettings( - settings.isCodeCoverageEnabled(), settings.isItrEnabled(), + settings.isCodeCoverageEnabled(), + settings.isTestSkippingEnabled(), settings.isFlakyTestRetriesEnabled(), settings.getEarlyFlakeDetectionSettings(), settings.getSystemProperties(), @@ -191,7 +192,7 @@ public void end(@Nullable Long endTime) { setTag(Tags.TEST_CODE_COVERAGE_ENABLED, true); } - if (itrEnabled) { + if (testSkippingEnabled) { setTag(Tags.TEST_ITR_TESTS_SKIPPING_ENABLED, true); setTag(Tags.TEST_ITR_TESTS_SKIPPING_TYPE, "test"); @@ -269,7 +270,6 @@ public BuildSystemModuleImpl testModuleStart( sourcePathResolver, codeowners, methodLinesResolver, - coverageProbeStoreFactory, repoIndexProvider, testModuleRegistry, SpanUtils.propagateCiVisibilityTagsTo(span)); diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/ProxyTestModule.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/ProxyTestModule.java index a554b48d58d..59723ae38a0 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/ProxyTestModule.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/ProxyTestModule.java @@ -13,6 +13,7 @@ import datadog.trace.civisibility.InstrumentationType; import datadog.trace.civisibility.codeowners.Codeowners; import datadog.trace.civisibility.coverage.CoverageProbeStoreFactory; +import datadog.trace.civisibility.coverage.SkippableAwareCoverageProbeStoreFactory; import datadog.trace.civisibility.decorator.TestDecorator; import datadog.trace.civisibility.domain.TestFrameworkModule; import datadog.trace.civisibility.domain.TestSuiteImpl; @@ -59,6 +60,7 @@ public class ProxyTestModule implements TestFrameworkModule { private final CoverageProbeStoreFactory coverageProbeStoreFactory; private final LongAdder testsSkipped = new LongAdder(); private final Collection skippableTests; + private final boolean testSkippingEnabled; private final boolean flakyTestRetriesEnabled; @Nullable private final Collection flakyTests; private final Collection knownTests; @@ -91,9 +93,14 @@ public ProxyTestModule( this.sourcePathResolver = sourcePathResolver; this.codeowners = codeowners; this.methodLinesResolver = methodLinesResolver; - this.coverageProbeStoreFactory = coverageProbeStoreFactory; this.itrCorrelationId = executionSettings.getItrCorrelationId(); + + this.testSkippingEnabled = executionSettings.isTestSkippingEnabled(); this.skippableTests = new HashSet<>(executionSettings.getSkippableTests(moduleName)); + this.coverageProbeStoreFactory = + executionSettings.isItrEnabled() + ? new SkippableAwareCoverageProbeStoreFactory(skippableTests, coverageProbeStoreFactory) + : coverageProbeStoreFactory; this.flakyTestRetriesEnabled = executionSettings.isFlakyTestRetriesEnabled(); Collection flakyTests = executionSettings.getFlakyTests(moduleName); @@ -106,18 +113,18 @@ public ProxyTestModule( } @Override - public boolean isSkippable(TestIdentifier test) { - return test != null && skippableTests.contains(test); + public boolean isNew(TestIdentifier test) { + return knownTests != null && !knownTests.contains(test.withoutParameters()); } @Override - public boolean isNew(TestIdentifier test) { - return knownTests != null && !knownTests.contains(test.withoutParameters()); + public boolean shouldBeSkipped(TestIdentifier test) { + return testSkippingEnabled && test != null && skippableTests.contains(test); } @Override public boolean skip(TestIdentifier test) { - if (isSkippable(test)) { + if (shouldBeSkipped(test)) { testsSkipped.increment(); return true; } else { @@ -160,7 +167,7 @@ public void end(@Nullable Long endTime) { private void sendModuleExecutionResult() { boolean coverageEnabled = config.isCiVisibilityCodeCoverageEnabled(); - boolean itrEnabled = config.isCiVisibilityItrEnabled(); + boolean testSkippingEnabled = config.isCiVisibilityTestSkippingEnabled(); boolean earlyFlakeDetectionEnabled = earlyFlakeDetectionSettings.isEnabled(); boolean earlyFlakeDetectionFaulty = earlyFlakeDetectionEnabled @@ -173,7 +180,7 @@ private void sendModuleExecutionResult() { parentProcessSessionId, parentProcessModuleId, coverageEnabled, - itrEnabled, + testSkippingEnabled, earlyFlakeDetectionEnabled, earlyFlakeDetectionFaulty, testsSkippedTotal, diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/headless/HeadlessTestModule.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/headless/HeadlessTestModule.java index 675422902e7..4798c1dfe85 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/headless/HeadlessTestModule.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/headless/HeadlessTestModule.java @@ -14,6 +14,7 @@ import datadog.trace.civisibility.InstrumentationType; import datadog.trace.civisibility.codeowners.Codeowners; import datadog.trace.civisibility.coverage.CoverageProbeStoreFactory; +import datadog.trace.civisibility.coverage.SkippableAwareCoverageProbeStoreFactory; import datadog.trace.civisibility.decorator.TestDecorator; import datadog.trace.civisibility.domain.AbstractTestModule; import datadog.trace.civisibility.domain.TestFrameworkModule; @@ -44,13 +45,14 @@ public class HeadlessTestModule extends AbstractTestModule implements TestFramew private final LongAdder testsSkipped = new LongAdder(); private final String itrCorrelationId; private final Collection skippableTests; + private final CoverageProbeStoreFactory coverageProbeStoreFactory; private final boolean flakyTestRetriesEnabled; @Nullable private final Collection flakyTests; private final Collection knownTests; private final EarlyFlakeDetectionSettings earlyFlakeDetectionSettings; private final AtomicInteger earlyFlakeDetectionsUsed = new AtomicInteger(0); private final boolean codeCoverageEnabled; - private final boolean itrEnabled; + private final boolean testSkippingEnabled; public HeadlessTestModule( AgentSpan.Context sessionSpanContext, @@ -78,13 +80,16 @@ public HeadlessTestModule( sourcePathResolver, codeowners, methodLinesResolver, - coverageProbeStoreFactory, onSpanFinish); codeCoverageEnabled = executionSettings.isCodeCoverageEnabled(); - itrEnabled = executionSettings.isItrEnabled(); + testSkippingEnabled = executionSettings.isTestSkippingEnabled(); itrCorrelationId = executionSettings.getItrCorrelationId(); skippableTests = new HashSet<>(executionSettings.getSkippableTests(moduleName)); + this.coverageProbeStoreFactory = + executionSettings.isItrEnabled() + ? new SkippableAwareCoverageProbeStoreFactory(skippableTests, coverageProbeStoreFactory) + : coverageProbeStoreFactory; flakyTestRetriesEnabled = executionSettings.isFlakyTestRetriesEnabled(); Collection flakyTests = executionSettings.getFlakyTests(moduleName); @@ -97,18 +102,18 @@ public HeadlessTestModule( } @Override - public boolean isSkippable(TestIdentifier test) { - return test != null && skippableTests.contains(test); + public boolean isNew(TestIdentifier test) { + return knownTests != null && !knownTests.contains(test.withoutParameters()); } @Override - public boolean isNew(TestIdentifier test) { - return knownTests != null && !knownTests.contains(test.withoutParameters()); + public boolean shouldBeSkipped(TestIdentifier test) { + return testSkippingEnabled && test != null && skippableTests.contains(test); } @Override public boolean skip(TestIdentifier test) { - if (isSkippable(test)) { + if (shouldBeSkipped(test)) { testsSkipped.increment(); return true; } else { @@ -148,7 +153,7 @@ public void end(@Nullable Long endTime) { setTag(Tags.TEST_CODE_COVERAGE_ENABLED, true); } - if (itrEnabled) { + if (testSkippingEnabled) { setTag(Tags.TEST_ITR_TESTS_SKIPPING_ENABLED, true); setTag(Tags.TEST_ITR_TESTS_SKIPPING_TYPE, "test"); diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/manualapi/ManualApiTestModule.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/manualapi/ManualApiTestModule.java index 1a53f32c323..0a050f4eea6 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/manualapi/ManualApiTestModule.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/manualapi/ManualApiTestModule.java @@ -22,6 +22,9 @@ * datadog.trace.api.civisibility.CIVisibility}) */ public class ManualApiTestModule extends AbstractTestModule implements DDTestModule { + + private final CoverageProbeStoreFactory coverageProbeStoreFactory; + public ManualApiTestModule( AgentSpan.Context sessionSpanContext, long sessionId, @@ -47,8 +50,8 @@ public ManualApiTestModule( sourcePathResolver, codeowners, methodLinesResolver, - coverageProbeStoreFactory, onSpanFinish); + this.coverageProbeStoreFactory = coverageProbeStoreFactory; } @Override diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/events/TestEventsHandlerImpl.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/events/TestEventsHandlerImpl.java index 53393444e2f..882a6a4d155 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/events/TestEventsHandlerImpl.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/events/TestEventsHandlerImpl.java @@ -132,7 +132,7 @@ public void onTestStart( } TestSuiteImpl testSuite = inProgressTestSuites.get(suiteDescriptor); - TestImpl test = testSuite.testStart(testName, testMethod, null); + TestImpl test = testSuite.testStart(testName, testParameters, testMethod, null); TestIdentifier thisTest = new TestIdentifier(testSuiteName, testName, testParameters, null); if (testModule.isNew(thisTest)) { @@ -160,7 +160,7 @@ public void onTestStart( test.setTag(Tags.TEST_ITR_UNSKIPPABLE, true); metricCollector.add(CiVisibilityCountMetric.ITR_UNSKIPPABLE, 1, EventType.TEST); - if (testModule.isSkippable(thisTest)) { + if (testModule.shouldBeSkipped(thisTest)) { test.setTag(Tags.TEST_ITR_FORCED_RUN, true); metricCollector.add(CiVisibilityCountMetric.ITR_FORCED_RUN, 1, EventType.TEST); } @@ -247,8 +247,8 @@ public boolean skip(TestIdentifier test) { } @Override - public boolean isSkippable(TestIdentifier test) { - return testModule.isSkippable(test); + public boolean shouldBeSkipped(TestIdentifier test) { + return testModule.shouldBeSkipped(test); } @Override diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/ipc/ModuleExecutionResult.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/ipc/ModuleExecutionResult.java index 8ddf7f2b193..4d32f6dd4ce 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/ipc/ModuleExecutionResult.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/ipc/ModuleExecutionResult.java @@ -9,14 +9,14 @@ public class ModuleExecutionResult implements Signal { private static final int COVERAGE_ENABLED_FLAG = 1; - private static final int ITR_ENABLED_FLAG = 2; + private static final int TEST_SKIPPING_ENABLED_FLAG = 2; private static final int EARLY_FLAKE_DETECTION_ENABLED_FLAG = 4; private static final int EARLY_FLAKE_DETECTION_FAULTY_FLAG = 8; private final long sessionId; private final long moduleId; private final boolean coverageEnabled; - private final boolean itrEnabled; + private final boolean testSkippingEnabled; private final boolean earlyFlakeDetectionEnabled; private final boolean earlyFlakeDetectionFaulty; private final long testsSkippedTotal; @@ -27,7 +27,7 @@ public ModuleExecutionResult( long sessionId, long moduleId, boolean coverageEnabled, - boolean itrEnabled, + boolean testSkippingEnabled, boolean earlyFlakeDetectionEnabled, boolean earlyFlakeDetectionFaulty, long testsSkippedTotal, @@ -36,7 +36,7 @@ public ModuleExecutionResult( this.sessionId = sessionId; this.moduleId = moduleId; this.coverageEnabled = coverageEnabled; - this.itrEnabled = itrEnabled; + this.testSkippingEnabled = testSkippingEnabled; this.earlyFlakeDetectionEnabled = earlyFlakeDetectionEnabled; this.earlyFlakeDetectionFaulty = earlyFlakeDetectionFaulty; this.testsSkippedTotal = testsSkippedTotal; @@ -56,8 +56,8 @@ public boolean isCoverageEnabled() { return coverageEnabled; } - public boolean isItrEnabled() { - return itrEnabled; + public boolean isTestSkippingEnabled() { + return testSkippingEnabled; } public boolean isEarlyFlakeDetectionEnabled() { @@ -93,7 +93,7 @@ public boolean equals(Object o) { return sessionId == that.sessionId && moduleId == that.moduleId && coverageEnabled == that.coverageEnabled - && itrEnabled == that.itrEnabled + && testSkippingEnabled == that.testSkippingEnabled && testsSkippedTotal == that.testsSkippedTotal && Objects.equals(testFrameworks, that.testFrameworks) && Arrays.equals(coverageData, that.coverageData); @@ -105,7 +105,7 @@ public int hashCode() { sessionId, moduleId, coverageEnabled, - itrEnabled, + testSkippingEnabled, testsSkippedTotal, testFrameworks, Arrays.hashCode(coverageData)); @@ -120,8 +120,8 @@ public String toString() { + moduleId + ", coverageEnabled=" + coverageEnabled - + ", itrEnabled=" - + itrEnabled + + ", testSkippingEnabled=" + + testSkippingEnabled + ", itrTestsSkipped=" + testsSkippedTotal + '}'; @@ -142,8 +142,8 @@ public ByteBuffer serialize() { if (coverageEnabled) { flags |= COVERAGE_ENABLED_FLAG; } - if (itrEnabled) { - flags |= ITR_ENABLED_FLAG; + if (testSkippingEnabled) { + flags |= TEST_SKIPPING_ENABLED_FLAG; } if (earlyFlakeDetectionEnabled) { flags |= EARLY_FLAKE_DETECTION_ENABLED_FLAG; @@ -166,7 +166,7 @@ public static ModuleExecutionResult deserialize(ByteBuffer buffer) { int flags = Serializer.readByte(buffer); boolean coverageEnabled = (flags & COVERAGE_ENABLED_FLAG) != 0; - boolean itrEnabled = (flags & ITR_ENABLED_FLAG) != 0; + boolean testSkippingEnabled = (flags & TEST_SKIPPING_ENABLED_FLAG) != 0; boolean earlyFlakeDetectionEnabled = (flags & EARLY_FLAKE_DETECTION_ENABLED_FLAG) != 0; boolean earlyFlakeDetectionFaulty = (flags & EARLY_FLAKE_DETECTION_FAULTY_FLAG) != 0; @@ -179,7 +179,7 @@ public static ModuleExecutionResult deserialize(ByteBuffer buffer) { sessionId, moduleId, coverageEnabled, - itrEnabled, + testSkippingEnabled, earlyFlakeDetectionEnabled, earlyFlakeDetectionFaulty, testsSkippedTotal, diff --git a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/TestImplTest.groovy b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/TestImplTest.groovy index 7702a46c7c8..82407276715 100644 --- a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/TestImplTest.groovy +++ b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/TestImplTest.groovy @@ -123,6 +123,7 @@ class TestImplTest extends DDSpecification { "moduleName", "suiteName", "testName", + "testParameters", null, null, null, diff --git a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/ModuleExecutionSettingsSerializerTest.groovy b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/ModuleExecutionSettingsSerializerTest.groovy index 07ff57e6fd1..b72953279b9 100644 --- a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/ModuleExecutionSettingsSerializerTest.groovy +++ b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/ModuleExecutionSettingsSerializerTest.groovy @@ -17,8 +17,8 @@ class ModuleExecutionSettingsSerializerTest extends Specification { where: settings << [ - new ModuleExecutionSettings(false, false, false, EarlyFlakeDetectionSettings.DEFAULT, [:], null, [:], null, [:], []), - new ModuleExecutionSettings(true, false, true, + new ModuleExecutionSettings(false, false, false, false, EarlyFlakeDetectionSettings.DEFAULT, [:], null, [:], null, [:], []), + new ModuleExecutionSettings(true, true, false, true, new EarlyFlakeDetectionSettings(true, [], 10), ["a": "b", "propName": "propValue"], "", @@ -26,7 +26,7 @@ class ModuleExecutionSettingsSerializerTest extends Specification { [new TestIdentifier("suite", "name", null, null)], ["bundle": [new TestIdentifier("a", "b", "c", null)]], ["a", "bcde", "f", "ghhi"]), - new ModuleExecutionSettings(false, true, false, + new ModuleExecutionSettings(false, false, true, false, new EarlyFlakeDetectionSettings(true, [new EarlyFlakeDetectionSettings.ExecutionsByDuration(10, 20)], 10), ["a": "b", "propName": "propValue"], "itrCorrelationId", @@ -34,7 +34,7 @@ class ModuleExecutionSettingsSerializerTest extends Specification { [new TestIdentifier("suite", "name", null, null), new TestIdentifier("a", "b", "c", null)], ["bundle": [new TestIdentifier("a", "b", "c", null), new TestIdentifier("aa", "bb", "cc", null)]], ["a", "bcde", "f", "ghhi"]), - new ModuleExecutionSettings(true, true, true, + new ModuleExecutionSettings(true, true, true, true, new EarlyFlakeDetectionSettings(true, [ new EarlyFlakeDetectionSettings.ExecutionsByDuration(10, 20), new EarlyFlakeDetectionSettings.ExecutionsByDuration(30, 40) diff --git a/dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy b/dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy index fdca26edfdd..2830d3a53dd 100644 --- a/dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy +++ b/dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy @@ -99,7 +99,9 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner { (CiVisibilityConfig.CIVISIBILITY_FLAKY_RETRY_ENABLED) : String.valueOf(flakyRetryEnabled), (CiVisibilityConfig.CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED): String.valueOf(earlyFlakinessDetectionEnabled) ] - return new ModuleExecutionSettings(false, + return new ModuleExecutionSettings( + itrEnabled, + false, itrEnabled, flakyRetryEnabled, earlyFlakinessDetectionEnabled diff --git a/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/main/java/datadog/trace/instrumentation/junit4/JUnit4CucumberItrInstrumentation.java b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/main/java/datadog/trace/instrumentation/junit4/JUnit4CucumberItrInstrumentation.java index 8d7671a963e..a93d2751830 100644 --- a/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/main/java/datadog/trace/instrumentation/junit4/JUnit4CucumberItrInstrumentation.java +++ b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/main/java/datadog/trace/instrumentation/junit4/JUnit4CucumberItrInstrumentation.java @@ -33,7 +33,7 @@ public JUnit4CucumberItrInstrumentation() { @Override public boolean isApplicable(Set enabledSystems) { - return super.isApplicable(enabledSystems) && Config.get().isCiVisibilityItrEnabled(); + return super.isApplicable(enabledSystems) && Config.get().isCiVisibilityTestSkippingEnabled(); } @Override diff --git a/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-itr-skipping/coverages.ftl b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-itr-skipping/coverages.ftl index 874bae33a26..fe51488c706 100644 --- a/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-itr-skipping/coverages.ftl +++ b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-itr-skipping/coverages.ftl @@ -1,9 +1 @@ -[ { - "test_session_id" : ${content_test_session_id}, - "test_suite_id" : ${content_test_suite_id}, - "span_id" : ${content_span_id}, - "files" : [ { - "filename" : "org/example/cucumber/calculator/basic_arithmetic.feature", - "segments" : [ ] - } ] -} ] \ No newline at end of file +[] diff --git a/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-itr-unskippable-suite/coverages.ftl b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-itr-unskippable-suite/coverages.ftl index 6fec5542b9d..fe51488c706 100644 --- a/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-itr-unskippable-suite/coverages.ftl +++ b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-itr-unskippable-suite/coverages.ftl @@ -1,9 +1 @@ -[ { - "test_session_id" : ${content_test_session_id}, - "test_suite_id" : ${content_test_suite_id}, - "span_id" : ${content_span_id}, - "files" : [ { - "filename" : "org/example/cucumber/calculator/basic_arithmetic_unskippable_suite.feature", - "segments" : [ ] - } ] -} ] \ No newline at end of file +[] diff --git a/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-itr-unskippable/coverages.ftl b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-itr-unskippable/coverages.ftl index c8492b3d13b..fe51488c706 100644 --- a/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-itr-unskippable/coverages.ftl +++ b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-itr-unskippable/coverages.ftl @@ -1,9 +1 @@ -[ { - "test_session_id" : ${content_test_session_id}, - "test_suite_id" : ${content_test_suite_id}, - "span_id" : ${content_span_id}, - "files" : [ { - "filename" : "org/example/cucumber/calculator/basic_arithmetic_unskippable.feature", - "segments" : [ ] - } ] -} ] \ No newline at end of file +[] diff --git a/dd-java-agent/instrumentation/junit-4.10/src/main/java/datadog/trace/instrumentation/junit4/JUnit4ItrInstrumentation.java b/dd-java-agent/instrumentation/junit-4.10/src/main/java/datadog/trace/instrumentation/junit4/JUnit4ItrInstrumentation.java index dfa68421a7b..d3f32d8f526 100644 --- a/dd-java-agent/instrumentation/junit-4.10/src/main/java/datadog/trace/instrumentation/junit4/JUnit4ItrInstrumentation.java +++ b/dd-java-agent/instrumentation/junit-4.10/src/main/java/datadog/trace/instrumentation/junit4/JUnit4ItrInstrumentation.java @@ -35,7 +35,7 @@ public JUnit4ItrInstrumentation() { @Override public boolean isApplicable(Set enabledSystems) { - return super.isApplicable(enabledSystems) && Config.get().isCiVisibilityItrEnabled(); + return super.isApplicable(enabledSystems) && Config.get().isCiVisibilityTestSkippingEnabled(); } @Override diff --git a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/main/java/datadog/trace/instrumentation/junit5/JUnit5CucumberItrInstrumentation.java b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/main/java/datadog/trace/instrumentation/junit5/JUnit5CucumberItrInstrumentation.java index d917bf19bd1..e2b344f37ab 100644 --- a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/main/java/datadog/trace/instrumentation/junit5/JUnit5CucumberItrInstrumentation.java +++ b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/main/java/datadog/trace/instrumentation/junit5/JUnit5CucumberItrInstrumentation.java @@ -37,7 +37,7 @@ public ElementMatcher.Junction classLoaderMatcher() { @Override public boolean isApplicable(Set enabledSystems) { - return super.isApplicable(enabledSystems) && Config.get().isCiVisibilityItrEnabled(); + return super.isApplicable(enabledSystems) && Config.get().isCiVisibilityTestSkippingEnabled(); } @Override diff --git a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-itr-unskippable-suite/coverages.ftl b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-itr-unskippable-suite/coverages.ftl index 6fec5542b9d..fe51488c706 100644 --- a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-itr-unskippable-suite/coverages.ftl +++ b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-itr-unskippable-suite/coverages.ftl @@ -1,9 +1 @@ -[ { - "test_session_id" : ${content_test_session_id}, - "test_suite_id" : ${content_test_suite_id}, - "span_id" : ${content_span_id}, - "files" : [ { - "filename" : "org/example/cucumber/calculator/basic_arithmetic_unskippable_suite.feature", - "segments" : [ ] - } ] -} ] \ No newline at end of file +[] diff --git a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-itr-unskippable/coverages.ftl b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-itr-unskippable/coverages.ftl index c8492b3d13b..fe51488c706 100644 --- a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-itr-unskippable/coverages.ftl +++ b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-itr-unskippable/coverages.ftl @@ -1,9 +1 @@ -[ { - "test_session_id" : ${content_test_session_id}, - "test_suite_id" : ${content_test_suite_id}, - "span_id" : ${content_span_id}, - "files" : [ { - "filename" : "org/example/cucumber/calculator/basic_arithmetic_unskippable.feature", - "segments" : [ ] - } ] -} ] \ No newline at end of file +[] diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/main/java/datadog/trace/instrumentation/junit5/JUnit5SpockItrInstrumentation.java b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/main/java/datadog/trace/instrumentation/junit5/JUnit5SpockItrInstrumentation.java index 103fe442579..cd6fa5e2f09 100644 --- a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/main/java/datadog/trace/instrumentation/junit5/JUnit5SpockItrInstrumentation.java +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/main/java/datadog/trace/instrumentation/junit5/JUnit5SpockItrInstrumentation.java @@ -38,7 +38,7 @@ public ElementMatcher.Junction classLoaderMatcher() { @Override public boolean isApplicable(Set enabledSystems) { - return super.isApplicable(enabledSystems) && Config.get().isCiVisibilityItrEnabled(); + return super.isApplicable(enabledSystems) && Config.get().isCiVisibilityTestSkippingEnabled(); } @Override @@ -117,7 +117,7 @@ public static void shouldBeSkipped( TestIdentifier featureIdentifier = SpockUtils.toTestIdentifier(feature); if (featureIdentifier == null - || !TestEventsHandlerHolder.TEST_EVENTS_HANDLER.isSkippable(featureIdentifier)) { + || !TestEventsHandlerHolder.TEST_EVENTS_HANDLER.shouldBeSkipped(featureIdentifier)) { return; } } diff --git a/dd-java-agent/instrumentation/junit-5.3/src/main/java/datadog/trace/instrumentation/junit5/JUnit5ItrInstrumentation.java b/dd-java-agent/instrumentation/junit-5.3/src/main/java/datadog/trace/instrumentation/junit5/JUnit5ItrInstrumentation.java index b48932ecfb3..d36b72d359b 100644 --- a/dd-java-agent/instrumentation/junit-5.3/src/main/java/datadog/trace/instrumentation/junit5/JUnit5ItrInstrumentation.java +++ b/dd-java-agent/instrumentation/junit-5.3/src/main/java/datadog/trace/instrumentation/junit5/JUnit5ItrInstrumentation.java @@ -33,7 +33,7 @@ public JUnit5ItrInstrumentation() { @Override public boolean isApplicable(Set enabledSystems) { - return super.isApplicable(enabledSystems) && Config.get().isCiVisibilityItrEnabled(); + return super.isApplicable(enabledSystems) && Config.get().isCiVisibilityTestSkippingEnabled(); } @Override diff --git a/dd-java-agent/instrumentation/karate/src/main/java/datadog/trace/instrumentation/karate/KarateTracingHook.java b/dd-java-agent/instrumentation/karate/src/main/java/datadog/trace/instrumentation/karate/KarateTracingHook.java index 4d5a2a7fa84..d51f7520154 100644 --- a/dd-java-agent/instrumentation/karate/src/main/java/datadog/trace/instrumentation/karate/KarateTracingHook.java +++ b/dd-java-agent/instrumentation/karate/src/main/java/datadog/trace/instrumentation/karate/KarateTracingHook.java @@ -97,7 +97,7 @@ public boolean beforeScenario(ScenarioRuntime sr) { String parameters = KarateUtils.getParameters(scenario); Collection categories = scenario.getTagsEffective().getTagKeys(); - if (Config.get().isCiVisibilityItrEnabled() + if (Config.get().isCiVisibilityTestSkippingEnabled() && !categories.contains(InstrumentationBridge.ITR_UNSKIPPABLE_TAG)) { TestIdentifier skippableTest = KarateUtils.toTestIdentifier(scenario); if (TestEventsHandlerHolder.TEST_EVENTS_HANDLER.skip(skippableTest)) { diff --git a/dd-java-agent/instrumentation/scalatest/src/main/java/datadog/trace/instrumentation/scalatest/ScalatestItrInstrumentation.java b/dd-java-agent/instrumentation/scalatest/src/main/java/datadog/trace/instrumentation/scalatest/ScalatestItrInstrumentation.java index 8bd3be134df..99541ceeac6 100644 --- a/dd-java-agent/instrumentation/scalatest/src/main/java/datadog/trace/instrumentation/scalatest/ScalatestItrInstrumentation.java +++ b/dd-java-agent/instrumentation/scalatest/src/main/java/datadog/trace/instrumentation/scalatest/ScalatestItrInstrumentation.java @@ -30,7 +30,7 @@ public ScalatestItrInstrumentation() { @Override public boolean isApplicable(Set enabledSystems) { - return super.isApplicable(enabledSystems) && Config.get().isCiVisibilityItrEnabled(); + return super.isApplicable(enabledSystems) && Config.get().isCiVisibilityTestSkippingEnabled(); } @Override diff --git a/dd-java-agent/instrumentation/testng/src/main/java/datadog/trace/instrumentation/testng/TestNGItrInstrumentation.java b/dd-java-agent/instrumentation/testng/src/main/java/datadog/trace/instrumentation/testng/TestNGItrInstrumentation.java index e3f7e7fdd06..74e61f1fd32 100644 --- a/dd-java-agent/instrumentation/testng/src/main/java/datadog/trace/instrumentation/testng/TestNGItrInstrumentation.java +++ b/dd-java-agent/instrumentation/testng/src/main/java/datadog/trace/instrumentation/testng/TestNGItrInstrumentation.java @@ -26,7 +26,7 @@ public TestNGItrInstrumentation() { @Override public boolean isApplicable(Set enabledSystems) { - return super.isApplicable(enabledSystems) && Config.get().isCiVisibilityItrEnabled(); + return super.isApplicable(enabledSystems) && Config.get().isCiVisibilityTestSkippingEnabled(); } @Override diff --git a/dd-smoke-tests/backend-mock/src/main/groovy/datadog/smoketest/MockBackend.groovy b/dd-smoke-tests/backend-mock/src/main/groovy/datadog/smoketest/MockBackend.groovy index b3beb4f7c17..de49d71d51a 100644 --- a/dd-smoke-tests/backend-mock/src/main/groovy/datadog/smoketest/MockBackend.groovy +++ b/dd-smoke-tests/backend-mock/src/main/groovy/datadog/smoketest/MockBackend.groovy @@ -31,6 +31,7 @@ class MockBackend implements AutoCloseable { private final Collection> skippableTests = new CopyOnWriteArrayList<>() private final Collection> flakyTests = new CopyOnWriteArrayList<>() + private boolean itrEnabled = true private boolean codeCoverageEnabled = true private boolean testsSkippingEnabled = true @@ -100,10 +101,18 @@ class MockBackend implements AutoCloseable { // it may choose to compress the response or not based on its size, // so smaller responses (like those of /setting endpoint) are uncompressed, // while the larger ones (skippable and flaky test lists) are compressed - response.status(200).send(('{ "data": { "type": "ci_app_tracers_test_service_settings", "id": "uuid", "attributes": { ' - + '"code_coverage": ' + codeCoverageEnabled - + ', "tests_skipping": ' + testsSkippingEnabled - + ', "flaky_test_retries_enabled": ' + flakyRetriesEnabled + '} } }').bytes) + response.status(200).send(("""{ + "data": { + "type": "ci_app_tracers_test_service_settings", + "id": "uuid", + "attributes": { + "itr_enabled": $itrEnabled, + "code_coverage": $codeCoverageEnabled, + "tests_skipping": $testsSkippingEnabled, + "flaky_test_retries_enabled": $flakyRetriesEnabled + } + } + }""").bytes) } prefix("/api/v2/ci/tests/skippable") { diff --git a/dd-trace-api/src/main/java/datadog/trace/api/config/CiVisibilityConfig.java b/dd-trace-api/src/main/java/datadog/trace/api/config/CiVisibilityConfig.java index e9d96de0b97..f6391e58b33 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/config/CiVisibilityConfig.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/config/CiVisibilityConfig.java @@ -41,6 +41,8 @@ public final class CiVisibilityConfig { public static final String CIVISIBILITY_SIGNAL_CLIENT_TIMEOUT_MILLIS = "civisibility.signal.client.timeout.millis"; public static final String CIVISIBILITY_ITR_ENABLED = "civisibility.itr.enabled"; + public static final String CIVISIBILITY_TEST_SKIPPING_ENABLED = + "civisibility.test.skipping.enabled"; public static final String CIVISIBILITY_CIPROVIDER_INTEGRATION_ENABLED = "civisibility.ciprovider.integration.enabled"; public static final String CIVISIBILITY_REPO_INDEX_SHARING_ENABLED = diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index 725cf28a09f..b8777f487d7 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -203,6 +203,7 @@ import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_SOURCE_DATA_ENABLED; import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_SOURCE_DATA_ROOT_CHECK_ENABLED; import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_TELEMETRY_ENABLED; +import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_TEST_SKIPPING_ENABLED; import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_TRACE_SANITATION_ENABLED; import static datadog.trace.api.config.CrashTrackingConfig.CRASH_TRACKING_AGENTLESS; import static datadog.trace.api.config.CrashTrackingConfig.CRASH_TRACKING_AGENTLESS_DEFAULT; @@ -799,6 +800,7 @@ static class HostNameHolder { private final int ciVisibilitySignalServerPort; private final int ciVisibilitySignalClientTimeoutMillis; private final boolean ciVisibilityItrEnabled; + private final boolean ciVisibilityTestSkippingEnabled; private final boolean ciVisibilityCiProviderIntegrationEnabled; private final boolean ciVisibilityRepoIndexSharingEnabled; private final int ciVisibilityModuleExecutionSettingsCacheSize; @@ -1816,6 +1818,8 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment()) ciVisibilitySignalClientTimeoutMillis = configProvider.getInteger(CIVISIBILITY_SIGNAL_CLIENT_TIMEOUT_MILLIS, 10_000); ciVisibilityItrEnabled = configProvider.getBoolean(CIVISIBILITY_ITR_ENABLED, true); + ciVisibilityTestSkippingEnabled = + configProvider.getBoolean(CIVISIBILITY_TEST_SKIPPING_ENABLED, true); ciVisibilityCiProviderIntegrationEnabled = configProvider.getBoolean(CIVISIBILITY_CIPROVIDER_INTEGRATION_ENABLED, true); ciVisibilityRepoIndexSharingEnabled = @@ -3103,6 +3107,10 @@ public boolean isCiVisibilityItrEnabled() { return ciVisibilityItrEnabled; } + public boolean isCiVisibilityTestSkippingEnabled() { + return ciVisibilityTestSkippingEnabled; + } + public boolean isCiVisibilityCiProviderIntegrationEnabled() { return ciVisibilityCiProviderIntegrationEnabled; } diff --git a/internal-api/src/main/java/datadog/trace/api/civisibility/config/ModuleExecutionSettings.java b/internal-api/src/main/java/datadog/trace/api/civisibility/config/ModuleExecutionSettings.java index 8f2abd32afb..b2a990de7cf 100644 --- a/internal-api/src/main/java/datadog/trace/api/civisibility/config/ModuleExecutionSettings.java +++ b/internal-api/src/main/java/datadog/trace/api/civisibility/config/ModuleExecutionSettings.java @@ -14,6 +14,7 @@ public class ModuleExecutionSettings { false, false, false, + false, EarlyFlakeDetectionSettings.DEFAULT, Collections.emptyMap(), null, @@ -22,8 +23,9 @@ public class ModuleExecutionSettings { null, Collections.emptyList()); - private final boolean codeCoverageEnabled; private final boolean itrEnabled; + private final boolean codeCoverageEnabled; + private final boolean testSkippingEnabled; private final boolean flakyTestRetriesEnabled; private final EarlyFlakeDetectionSettings earlyFlakeDetectionSettings; private final Map systemProperties; @@ -34,8 +36,9 @@ public class ModuleExecutionSettings { private final List coverageEnabledPackages; public ModuleExecutionSettings( - boolean codeCoverageEnabled, boolean itrEnabled, + boolean codeCoverageEnabled, + boolean testSkippingEnabled, boolean flakyTestRetriesEnabled, EarlyFlakeDetectionSettings earlyFlakeDetectionSettings, Map systemProperties, @@ -44,8 +47,9 @@ public ModuleExecutionSettings( Collection flakyTests, Map> knownTestsByModule, List coverageEnabledPackages) { - this.codeCoverageEnabled = codeCoverageEnabled; this.itrEnabled = itrEnabled; + this.codeCoverageEnabled = codeCoverageEnabled; + this.testSkippingEnabled = testSkippingEnabled; this.flakyTestRetriesEnabled = flakyTestRetriesEnabled; this.earlyFlakeDetectionSettings = earlyFlakeDetectionSettings; this.systemProperties = systemProperties; @@ -56,12 +60,20 @@ public ModuleExecutionSettings( this.coverageEnabledPackages = coverageEnabledPackages; } + /** + * @return {@code true} if ITR is enabled. Enabled ITR does not necessarily imply test skipping: + * for an excluded branch ITR will be enabled, but not skipping. + */ + public boolean isItrEnabled() { + return itrEnabled; + } + public boolean isCodeCoverageEnabled() { return codeCoverageEnabled; } - public boolean isItrEnabled() { - return itrEnabled; + public boolean isTestSkippingEnabled() { + return testSkippingEnabled; } public boolean isFlakyTestRetriesEnabled() { @@ -128,8 +140,9 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; ModuleExecutionSettings that = (ModuleExecutionSettings) o; - return codeCoverageEnabled == that.codeCoverageEnabled - && itrEnabled == that.itrEnabled + return itrEnabled == that.itrEnabled + && codeCoverageEnabled == that.codeCoverageEnabled + && testSkippingEnabled == that.testSkippingEnabled && Objects.equals(earlyFlakeDetectionSettings, that.earlyFlakeDetectionSettings) && Objects.equals(systemProperties, that.systemProperties) && Objects.equals(itrCorrelationId, that.itrCorrelationId) @@ -142,8 +155,9 @@ public boolean equals(Object o) { @Override public int hashCode() { return Objects.hash( - codeCoverageEnabled, itrEnabled, + codeCoverageEnabled, + testSkippingEnabled, earlyFlakeDetectionSettings, systemProperties, itrCorrelationId, diff --git a/internal-api/src/main/java/datadog/trace/api/civisibility/events/TestEventsHandler.java b/internal-api/src/main/java/datadog/trace/api/civisibility/events/TestEventsHandler.java index 6b73b74a3ca..6c6f4101560 100644 --- a/internal-api/src/main/java/datadog/trace/api/civisibility/events/TestEventsHandler.java +++ b/internal-api/src/main/java/datadog/trace/api/civisibility/events/TestEventsHandler.java @@ -73,7 +73,7 @@ void onTestIgnore( boolean skip(TestIdentifier test); - boolean isSkippable(TestIdentifier test); + boolean shouldBeSkipped(TestIdentifier test); @Nonnull TestRetryPolicy retryPolicy(TestIdentifier test);