Skip to content

Commit

Permalink
👷 split test execution to avoir breakage
Browse files Browse the repository at this point in the history
Split tests to avoid stupid test flakyness due to bad global state / mocking.
  • Loading branch information
CedricGatay committed Aug 3, 2021
1 parent f3e4e85 commit 907584a
Show file tree
Hide file tree
Showing 6 changed files with 297 additions and 4 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -557,6 +557,8 @@
65E3A56F2270CB7A004156E9 /* CR_InterstitialViewController.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = CR_InterstitialViewController.h; sourceTree = "<group>"; };
65E3A5702270CB7A004156E9 /* CR_InterstitialViewController.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = CR_InterstitialViewController.m; sourceTree = "<group>"; };
7B8F678826B03DC200C20CBF /* CriteoPublisherSdk-Functional.xctestplan */ = {isa = PBXFileReference; lastKnownFileType = text; path = "CriteoPublisherSdk-Functional.xctestplan"; sourceTree = "<group>"; };
7BF5E9DE26B9354A00690F70 /* FlakyTest.xctestplan */ = {isa = PBXFileReference; lastKnownFileType = text; path = FlakyTest.xctestplan; sourceTree = "<group>"; };
7BF5E9DF26B9767600690F70 /* LoggingTests.xctestplan */ = {isa = PBXFileReference; lastKnownFileType = text; path = LoggingTests.xctestplan; sourceTree = "<group>"; };
7D00672A243CD6F900E5D50D /* CR_CASObjectQueue+ArraySet.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = "CR_CASObjectQueue+ArraySet.m"; sourceTree = "<group>"; };
7D00672B243CD6F900E5D50D /* CR_CASObjectQueue+ArraySet.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "CR_CASObjectQueue+ArraySet.h"; sourceTree = "<group>"; };
7D006732243DB74B00E5D50D /* CR_FeedbackMessageFunctionalTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = CR_FeedbackMessageFunctionalTests.m; sourceTree = "<group>"; };
Expand Down Expand Up @@ -1655,6 +1657,8 @@
E1F2737121B5F08D00A2FBFA /* Tests */ = {
isa = PBXGroup;
children = (
7BF5E9DF26B9767600690F70 /* LoggingTests.xctestplan */,
7BF5E9DE26B9354A00690F70 /* FlakyTest.xctestplan */,
4607BD0D24AF4F8300F18AF5 /* CriteoPublisherSdk.xctestplan */,
7B8F678826B03DC200C20CBF /* CriteoPublisherSdk-Functional.xctestplan */,
4646EF182514CEF2009E4D16 /* CriteoPublisherSdk-Standalone.xctestplan */,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,12 @@
<TestPlanReference
reference = "container:Tests/CriteoPublisherSdk-Functional.xctestplan">
</TestPlanReference>
<TestPlanReference
reference = "container:Tests/FlakyTest.xctestplan">
</TestPlanReference>
<TestPlanReference
reference = "container:Tests/LoggingTests.xctestplan">
</TestPlanReference>
</TestPlans>
<Testables>
<TestableReference
Expand All @@ -55,6 +61,16 @@
ReferencedContainer = "container:CriteoPublisherSdk.xcodeproj">
</BuildableReference>
</TestableReference>
<TestableReference
skipped = "NO">
<BuildableReference
BuildableIdentifier = "primary"
BlueprintIdentifier = "E1F2736C21B5F08D00A2FBFA"
BuildableName = "CriteoPublisherSdkTests.xctest"
BlueprintName = "CriteoPublisherSdkTests"
ReferencedContainer = "container:CriteoPublisherSdk.xcodeproj">
</BuildableReference>
</TestableReference>
</Testables>
</TestAction>
<LaunchAction
Expand Down
4 changes: 4 additions & 0 deletions CriteoPublisherSdk/Tests/CriteoPublisherSdk.xctestplan
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,9 @@
{
"skippedTests" : [
"CRInterstitialTests\/testWithRendering",
"CR_BidManagerIntegrationTests",
"CR_BidManagerTests",
"CR_CacheManagerTests",
"CR_ContextualFunctionalTests",
"CR_DataConsentFunctionalTests",
"CR_DfpBannerFunctionalTests",
Expand All @@ -36,6 +39,7 @@
"CR_HeaderBiddingDictionaryFunctionalTests",
"CR_InHouseFunctionalTests",
"CR_LoggingFunctionalTests",
"CR_LoggingTests",
"CR_MopubBannerFunctionalTests",
"CR_MopubInterstitialFunctionalTests",
"CR_NativeAdFunctionalTests",
Expand Down
136 changes: 136 additions & 0 deletions CriteoPublisherSdk/Tests/FlakyTest.xctestplan
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
{
"configurations" : [
{
"id" : "920EB707-7EE7-4FBC-AA98-EE969E00F4B1",
"name" : "Configuration 1",
"options" : {

}
}
],
"defaultOptions" : {

},
"testTargets" : [
{
"skippedTests" : [
"CRAdUnitTests",
"CRBannerAdUnitTests",
"CRBannerViewDelegateTests",
"CRBannerViewTests",
"CRBidTests",
"CREmailHasherTests",
"CRInterstitialAdUnitTests",
"CRInterstitialDelegateTests",
"CRInterstitialTests",
"CRInterstitialViewControllerTests",
"CRMediaViewTests",
"CRNativeAdTests",
"CRNativeAdUnitTests",
"CRNativeAdViewTests",
"CRNativeLoaderTests",
"CR_AdUnitHelperTests",
"CR_AdUnitHelperTests\/testAdUnitToCacheAdUnit",
"CR_AdUnitHelperTests\/testBannerAdUnitsToCacheAdUnits",
"CR_AdUnitHelperTests\/testInterstitialAdUnitsToCacheAdUnits",
"CR_AdUnitHelperTests\/testNativeAdUnitToCacheAdUnit",
"CR_ApiHandlerTests",
"CR_AppEventsIntegrationTests",
"CR_AppEventsTests",
"CR_BidFetchTrackerTests",
"CR_BidManagerFeedbackSendingTests",
"CR_BidManagerFeedbackTests",
"CR_BidManagerHelperTests",
"CR_BidManagerIntegrationTests\/test_given2AdUnits_whenPrefetchBid_thenGet2Bids",
"CR_BidManagerIntegrationTests\/test_givenCriteo_whenRegisterAdUnit_thenGetBid",
"CR_BidManagerIntegrationTests\/test_givenPrefetchingBid_whenExpiredBid_shouldProvideEmptyBid",
"CR_BidManagerIntegrationTests\/test_givenPrefetchingBid_whenGetBid_thenDontFetchBidAgain",
"CR_BidManagerIntegrationTests\/test_givenPrefetchingBid_whenGetImmediateBid_shouldProvideBidWithTtlSetToDefaultOne",
"CR_BidManagerIntegrationTests\/test_givenPrefetchingBid_whenMissingBid_shouldProvideEmptyBid",
"CR_BidManagerIntegrationTests\/test_givenPrefetchingBid_whenNoBidModeBid_shouldProvideEmptyBid",
"CR_BidManagerIntegrationTests\/test_givenPrefetchingBid_whenNoContent_shouldProvideEmptyBid",
"CR_BidManagerIntegrationTests\/test_givenPrefetchingBid_whenSilenceModeBid_shouldProvideEmptyBid",
"CR_BidManagerIntegrationTests\/test_givenPrefetchingBid_whenUserSilenceModeBid_shouldProvideEmptyBid",
"CR_BidPerformanceTests",
"CR_BidRequestSerializerSwiftTests",
"CR_BidRequestSerializerTests",
"CR_CASBoundedFileObjectQueueTests",
"CR_CacheAdUnitTests",
"CR_CdbBidTests",
"CR_CdbResponseTests",
"CR_ConfigManagerTests",
"CR_ConfigTests",
"CR_ContextualFunctionalTests",
"CR_DataConsentFunctionalTests",
"CR_DataProtectionConsentTests",
"CR_DefaultFileManipulatorTests",
"CR_DefaultMediaDownloaderTests",
"CR_DeviceInfoTests",
"CR_DfpBannerFunctionalTests",
"CR_DfpInterstitialFunctionalTests",
"CR_DfpNativeFunctionalTests",
"CR_DisplaySizeInjectorTests",
"CR_FeedbackControllerTests",
"CR_FeedbackFeatureGuardTests",
"CR_FeedbackFileManagerTests",
"CR_FeedbackMessageFunctionalTests",
"CR_FeedbackMessageTests",
"CR_FeedbackSerializerTests",
"CR_FeedbackStorageTests",
"CR_GdprSerializerTests",
"CR_GdprTests",
"CR_HeaderBiddingDictionaryFunctionalTests",
"CR_HeaderBiddingTests",
"CR_HttpTests",
"CR_ImageCacheTests",
"CR_ImpressionDetectionTests",
"CR_InHouseFunctionalTests",
"CR_IntegrationRegistryTests",
"CR_IntegrationsTestBase",
"CR_InternalContextProviderTests",
"CR_LogHandlerTests",
"CR_LoggingFunctionalTests",
"CR_LoggingTests",
"CR_MopubBannerFunctionalTests",
"CR_MopubInterstitialFunctionalTests",
"CR_NativeAdFunctionalTests",
"CR_NativeAdvertiserTests",
"CR_NativeAssetsTests",
"CR_NativeImageTests",
"CR_NativePrivacyTests",
"CR_NativeProductTests",
"CR_NetworkManagerTests",
"CR_ProfileIdFunctionalTests",
"CR_RegistrationFunctionalTests",
"CR_RemoteConfigRequestTests",
"CR_RemoteLogHandlerTest",
"CR_RemoteLogRecordSerializerTests",
"CR_RemoteLogRecordTests",
"CR_RemoteLogStorageTests",
"CR_SKAdNetworkParametersTests",
"CR_SafeMediaDownloaderTests",
"CR_StandaloneBannerFunctionalTests",
"CR_ThreadManagerTests",
"CR_URLOpenerTests",
"CR_URLRequestTests",
"CR_URLResolverTests",
"CR_UniqueIdGeneratorTests",
"Criteo",
"CriteoTests",
"NSArray_CriteoTests",
"NSDictionary_CriteoTests",
"NSError_CriteoTests",
"NSObject_CriteoTests",
"NSString_CriteoUrlTests",
"SKAdNetworkInfoTests",
"UIView_CriteoTests"
],
"target" : {
"containerPath" : "container:CriteoPublisherSdk.xcodeproj",
"identifier" : "E1F2736C21B5F08D00A2FBFA",
"name" : "CriteoPublisherSdkTests"
}
}
],
"version" : 1
}
124 changes: 124 additions & 0 deletions CriteoPublisherSdk/Tests/LoggingTests.xctestplan
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
{
"configurations" : [
{
"id" : "D186CD90-2920-4261-A78A-4D806D25E2F4",
"name" : "Configuration 1",
"options" : {

}
}
],
"defaultOptions" : {

},
"testTargets" : [
{
"skippedTests" : [
"CRAdUnitTests",
"CRBannerAdUnitTests",
"CRBannerViewDelegateTests",
"CRBannerViewTests",
"CRBidTests",
"CREmailHasherTests",
"CRInterstitialAdUnitTests",
"CRInterstitialDelegateTests",
"CRInterstitialTests",
"CRInterstitialViewControllerTests",
"CRMediaViewTests",
"CRNativeAdTests",
"CRNativeAdUnitTests",
"CRNativeAdViewTests",
"CRNativeLoaderTests",
"CR_AdUnitHelperTests",
"CR_ApiHandlerTests",
"CR_AppEventsIntegrationTests",
"CR_AppEventsTests",
"CR_BidFetchTrackerTests",
"CR_BidManagerFeedbackSendingTests",
"CR_BidManagerFeedbackTests",
"CR_BidManagerHelperTests",
"CR_BidManagerIntegrationTests",
"CR_BidManagerTests",
"CR_BidPerformanceTests",
"CR_BidRequestSerializerSwiftTests",
"CR_BidRequestSerializerTests",
"CR_CASBoundedFileObjectQueueTests",
"CR_CacheAdUnitTests",
"CR_CacheManagerTests",
"CR_CdbBidTests",
"CR_CdbResponseTests",
"CR_ConfigManagerTests",
"CR_ConfigTests",
"CR_ContextualFunctionalTests",
"CR_DataConsentFunctionalTests",
"CR_DataProtectionConsentTests",
"CR_DefaultFileManipulatorTests",
"CR_DefaultMediaDownloaderTests",
"CR_DeviceInfoTests",
"CR_DfpBannerFunctionalTests",
"CR_DfpInterstitialFunctionalTests",
"CR_DfpNativeFunctionalTests",
"CR_DisplaySizeInjectorTests",
"CR_FeedbackControllerTests",
"CR_FeedbackFeatureGuardTests",
"CR_FeedbackFileManagerTests",
"CR_FeedbackMessageFunctionalTests",
"CR_FeedbackMessageTests",
"CR_FeedbackSerializerTests",
"CR_FeedbackStorageTests",
"CR_GdprSerializerTests",
"CR_GdprTests",
"CR_HeaderBiddingDictionaryFunctionalTests",
"CR_HeaderBiddingTests",
"CR_HttpTests",
"CR_ImageCacheTests",
"CR_ImpressionDetectionTests",
"CR_InHouseFunctionalTests",
"CR_IntegrationRegistryTests",
"CR_IntegrationsTestBase",
"CR_InternalContextProviderTests",
"CR_LogHandlerTests",
"CR_LoggingFunctionalTests",
"CR_MopubBannerFunctionalTests",
"CR_MopubInterstitialFunctionalTests",
"CR_NativeAdFunctionalTests",
"CR_NativeAdvertiserTests",
"CR_NativeAssetsTests",
"CR_NativeImageTests",
"CR_NativePrivacyTests",
"CR_NativeProductTests",
"CR_NetworkManagerTests",
"CR_ProfileIdFunctionalTests",
"CR_RegistrationFunctionalTests",
"CR_RemoteConfigRequestTests",
"CR_RemoteLogHandlerTest",
"CR_RemoteLogRecordSerializerTests",
"CR_RemoteLogRecordTests",
"CR_RemoteLogStorageTests",
"CR_SKAdNetworkParametersTests",
"CR_SafeMediaDownloaderTests",
"CR_StandaloneBannerFunctionalTests",
"CR_ThreadManagerTests",
"CR_URLOpenerTests",
"CR_URLRequestTests",
"CR_URLResolverTests",
"CR_UniqueIdGeneratorTests",
"Criteo",
"CriteoTests",
"NSArray_CriteoTests",
"NSDictionary_CriteoTests",
"NSError_CriteoTests",
"NSObject_CriteoTests",
"NSString_CriteoUrlTests",
"SKAdNetworkInfoTests",
"UIView_CriteoTests"
],
"target" : {
"containerPath" : "container:CriteoPublisherSdk.xcodeproj",
"identifier" : "E1F2736C21B5F08D00A2FBFA",
"name" : "CriteoPublisherSdkTests"
}
}
],
"version" : 1
}
17 changes: 13 additions & 4 deletions fastlane/Fastfile
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,21 @@ platform :ios do
scan(scheme: "CriteoAdViewer")
scan(scheme: "CriteoGoogleAdapter")
scan(scheme: "CriteoMoPubAdapter")
multi_scan(
# split tests as they are breaking each others (probably due to mocking or global state)
scan(
workspace: "CriteoPublisherSdk.xcworkspace",
scheme: "CriteoPublisherSdk",
testplan: "CriteoPublisherSdk",
try_count: 5,
parallel_testrun_count: 2
testplan: "CriteoPublisherSdk"
)
scan(
workspace: "CriteoPublisherSdk.xcworkspace",
scheme: "CriteoPublisherSdk",
testplan: "FlakyTest"
)
scan(
workspace: "CriteoPublisherSdk.xcworkspace",
scheme: "CriteoPublisherSdk",
testplan: "LoggingTests"
)
pod_lib_lint
end
Expand Down

0 comments on commit 907584a

Please sign in to comment.