Multiple blocking rule lists

Sources/BrowserServicesKit/ContentBlocking/ContentBlockerRulesIdentifier.swift

@@ -21,13 +21,14 @@ import Foundation
 
 public class ContentBlockerRulesIdentifier: Equatable {
 
+    private let name: String
     private let tdsEtag: String
     private let tempListEtag: String
     private let allowListEtag: String
     private let unprotectedSitesHash: String
 
     public var stringValue: String {
-        return tdsEtag + tempListEtag + unprotectedSitesHash
+        return name + tdsEtag + tempListEtag + unprotectedSitesHash
     }
 
     public struct Difference: OptionSet {
@@ -70,8 +71,9 @@ public class ContentBlockerRulesIdentifier: Equatable {
         return domains.joined().sha1
     }
 
-    public init(tdsEtag: String, tempListEtag: String?, allowListEtag: String?, unprotectedSitesHash: String?) {
+    public init(name: String, tdsEtag: String, tempListEtag: String?, allowListEtag: String?, unprotectedSitesHash: String?) {
 
+        self.name = Self.normalize(identifier: name)
         self.tdsEtag = Self.normalize(identifier: tdsEtag)
         self.tempListEtag = Self.normalize(identifier: tempListEtag)
         self.allowListEtag = Self.normalize(identifier: allowListEtag)

Sources/BrowserServicesKit/ContentBlocking/ContentBlockerRulesSource.swift

@@ -21,36 +21,66 @@ import Foundation
 import TrackerRadarKit
 
 /**
- Represents all sources used to build Content Blocking Rules along with their state information.
+ Represents all sources used to build Content Blocking Rules.
  */
-public protocol ContentBlockerRulesSource {
+public protocol ContentBlockerRulesListsSource {
+
+    var contentBlockerRulesLists: [ContentBlockerRulesList] { get }
+}
+
+/**
+ Represents sources used to prepare exceptions to content blocking Rules.
+ */
+public protocol ContentBlockerRulesExceptionsSource {
 
-    var trackerData: TrackerDataManager.DataSet? { get }
-    var embeddedTrackerData: TrackerDataManager.DataSet { get }
     var tempListEtag: String { get }
     var tempList: [String] { get }
     var allowListEtag: String { get }
     var allowList: [TrackerException] { get }
     var unprotectedSites: [String] { get }
-
 }
 
-public class DefaultContentBlockerRulesSource: ContentBlockerRulesSource {
+public struct ContentBlockerRulesList {
 
-    let trackerDataManager: TrackerDataManager
-    let privacyConfigManager: PrivacyConfigurationManager
+    public let trackerData: TrackerDataManager.DataSet?
+    public let fallbackTrackerData: TrackerDataManager.DataSet
 
-    public init(trackerDataManager: TrackerDataManager, privacyConfigManager: PrivacyConfigurationManager) {
-        self.trackerDataManager = trackerDataManager
-        self.privacyConfigManager = privacyConfigManager
+    public let name: String
+
+    public init(name: String,
+                trackerData: TrackerDataManager.DataSet?,
+                fallbackTrackerData: TrackerDataManager.DataSet) {
+        self.name = name
+        self.trackerData = trackerData
+        self.fallbackTrackerData = fallbackTrackerData
+    }
+}
+
+open class DefaultContentBlockerRulesListsSource: ContentBlockerRulesListsSource {
+
+    public struct Constants {
+        public static let trackerDataSetRulesListName = "TrackerDataSet"
     }
 
-    public var trackerData: TrackerDataManager.DataSet? {
-        return trackerDataManager.fetchedData
+    private let trackerDataManger: TrackerDataManager
+
+    public init(trackerDataManger: TrackerDataManager) {
+        self.trackerDataManger = trackerDataManger
     }
+
+    open var contentBlockerRulesLists: [ContentBlockerRulesList] {
+        return [ContentBlockerRulesList(name: Constants.trackerDataSetRulesListName,
+                                        trackerData: trackerDataManger.fetchedData,
+                                        fallbackTrackerData: trackerDataManger.embeddedData)]
+    }
+}
 
-    public var embeddedTrackerData: TrackerDataManager.DataSet {
-        return trackerDataManager.embeddedData
+public class DefaultContentBlockerRulesExceptionsSource: ContentBlockerRulesExceptionsSource {
+
+    let privacyConfigManager: PrivacyConfigurationManager
+
+    public init(privacyConfigManager: PrivacyConfigurationManager) {
+        self.privacyConfigManager = privacyConfigManager
     }
 
     public var tempListEtag: String {

Sources/BrowserServicesKit/ContentBlocking/ContentBlockerRulesSourceManager.swift

@@ -25,6 +25,7 @@ import TrackerRadarKit
  */
 public class ContentBlockerRulesSourceIdentifiers {
 
+    public let name: String
     public let tdsIdentifier: String
 
     public internal(set) var tempListIdentifier: String?
@@ -33,12 +34,14 @@ public class ContentBlockerRulesSourceIdentifiers {
 
     public internal(set) var unprotectedSitesIdentifier: String?
 
-    init(tdsIdentfier: String) {
+    init(name: String, tdsIdentfier: String) {
+        self.name = name
         self.tdsIdentifier = tdsIdentfier
     }
 
     public var rulesIdentifier: ContentBlockerRulesIdentifier {
-        ContentBlockerRulesIdentifier(tdsEtag: tdsIdentifier,
+        ContentBlockerRulesIdentifier(name: name,
+                                      tdsEtag: tdsIdentifier,
                                       tempListEtag: tempListIdentifier,
                                       allowListEtag: allowListIdentifier,
                                       unprotectedSitesHash: unprotectedSitesIdentifier)
@@ -58,9 +61,9 @@ public class ContentBlockerRulesSourceModel: ContentBlockerRulesSourceIdentifier
 
     var unprotectedSites = [String]()
 
-    init(tdsIdentfier: String, tds: TrackerData) {
+    init(name: String, tdsIdentfier: String, tds: TrackerData) {
         self.tds = tds
-        super.init(tdsIdentfier: tdsIdentfier)
+        super.init(name: name, tdsIdentfier: tdsIdentfier)
     }
 }
 
@@ -70,33 +73,42 @@ public class ContentBlockerRulesSourceModel: ContentBlockerRulesSourceIdentifier
 public class ContentBlockerRulesSourceManager {
 
     /**
-     Data source for all of the inputs used for compilation.
+     Data source for all of the exception info used during compilation.
      */
-    private let dataSource: ContentBlockerRulesSource
+    private let exceptionsSource: ContentBlockerRulesExceptionsSource
+
+    var rulesList: ContentBlockerRulesList
 
     /**
      Identifiers of sources that have caused compilation process to fail.
      */
     public private(set) var brokenSources: ContentBlockerRulesSourceIdentifiers?
+    public private(set) var fallbackTDSFailure = false
 
     private let errorReporting: EventMapping<ContentBlockerDebugEvents>?
 
-    init(dataSource: ContentBlockerRulesSource, errorReporting: EventMapping<ContentBlockerDebugEvents>? = nil) {
-        self.dataSource = dataSource
+    init(rulesList: ContentBlockerRulesList,
+         exceptionsSource: ContentBlockerRulesExceptionsSource,
+         errorReporting: EventMapping<ContentBlockerDebugEvents>? = nil) {
+        self.rulesList = rulesList
+        self.exceptionsSource = exceptionsSource
         self.errorReporting = errorReporting
     }
 
     /**
-     Create Source Model based on data source and knowne broken sources.
+     Create Source Model based on data source and known broken sources.
 
      This method takes into account changes to `dataSource` that could fix previously corrupted data set - in such case `brokenSources` state is updated.
      */
-    func makeModel() -> ContentBlockerRulesSourceModel {
+    func makeModel() -> ContentBlockerRulesSourceModel? {
+        guard !fallbackTDSFailure else {
+            return nil
+        }
 
         // Fetch identifiers up-front
-        let tempListIdentifier = dataSource.tempListEtag
-        let allowListIdentifier = dataSource.allowListEtag
-        let unprotectedSites = dataSource.unprotectedSites
+        let tempListIdentifier = exceptionsSource.tempListEtag
+        let allowListIdentifier = exceptionsSource.allowListEtag
+        let unprotectedSites = exceptionsSource.unprotectedSites
         let unprotectedSitesIdentifier = ContentBlockerRulesIdentifier.hash(domains: unprotectedSites)
 
         // In case of any broken input that has been changed, reset the broken state and retry full compilation
@@ -108,25 +120,27 @@ public class ContentBlockerRulesSourceManager {
 
         // Check which Tracker Data Set to use - fallback to embedded one in case of any issues.
         let result: ContentBlockerRulesSourceModel
-        if let trackerData = dataSource.trackerData,
+        if let trackerData = rulesList.trackerData,
            trackerData.etag != brokenSources?.tdsIdentifier {
-            result = ContentBlockerRulesSourceModel(tdsIdentfier: trackerData.etag,
-                                                   tds: trackerData.tds)
+            result = ContentBlockerRulesSourceModel(name: rulesList.name,
+                                                    tdsIdentfier: trackerData.etag,
+                                                    tds: trackerData.tds)
         } else {
-            result = ContentBlockerRulesSourceModel(tdsIdentfier: dataSource.embeddedTrackerData.etag,
-                                                   tds: dataSource.embeddedTrackerData.tds)
+            result = ContentBlockerRulesSourceModel(name: rulesList.name,
+                                                    tdsIdentfier: rulesList.fallbackTrackerData.etag,
+                                                    tds: rulesList.fallbackTrackerData.tds)
         }
 
         if tempListIdentifier != brokenSources?.tempListIdentifier {
-            let tempListDomains = dataSource.tempList
+            let tempListDomains = exceptionsSource.tempList
             if !tempListDomains.isEmpty {
                 result.tempListIdentifier = tempListIdentifier
                 result.tempList = tempListDomains
             }
         }
 
         if allowListIdentifier != brokenSources?.allowListIdentifier {
-            let allowList = dataSource.allowList
+            let allowList = exceptionsSource.allowList
             if !allowList.isEmpty {
                 result.allowListIdentifier = allowListIdentifier
                 result.allowList = allowList
@@ -147,26 +161,32 @@ public class ContentBlockerRulesSourceManager {
      Process information about last failed compilation in order to update `brokenSources` state.
      */
     func compilationFailed(for input: ContentBlockerRulesSourceIdentifiers, with error: Error) {
-
-        if input.tdsIdentifier != dataSource.embeddedTrackerData.etag {
+        
+        if input.tdsIdentifier != rulesList.fallbackTrackerData.etag {
             // We failed compilation for non-embedded TDS, marking it as broken.
-            brokenSources = ContentBlockerRulesSourceIdentifiers(tdsIdentfier: input.tdsIdentifier)
+            brokenSources = ContentBlockerRulesSourceIdentifiers(name: rulesList.name,
+                                                                 tdsIdentfier: input.tdsIdentifier)
+
             errorReporting?.fire(.contentBlockingTDSCompilationFailed,
+                                 scope: input.name,
                                  error: error,
                                  parameters: [ContentBlockerDebugEvents.Parameters.etag: input.tdsIdentifier])
         } else if input.tempListIdentifier != nil {
             brokenSources?.tempListIdentifier = input.tempListIdentifier
             errorReporting?.fire(.contentBlockingTempListCompilationFailed,
+                                 scope: input.name,
                                  error: error,
                                  parameters: [ContentBlockerDebugEvents.Parameters.etag: input.tempListIdentifier ?? "empty"])
         } else if input.allowListIdentifier != nil {
             brokenSources?.allowListIdentifier = input.allowListIdentifier
             errorReporting?.fire(.contentBlockingAllowListCompilationFailed,
+                                 scope: input.name,
                                  error: error,
                                  parameters: [ContentBlockerDebugEvents.Parameters.etag: input.allowListIdentifier ?? "empty"])
         } else if input.unprotectedSitesIdentifier != nil {
             brokenSources?.unprotectedSitesIdentifier = input.unprotectedSitesIdentifier
             errorReporting?.fire(.contentBlockingUnpSitesCompilationFailed,
+                                 scope: input.name,
                                  error: error)
         } else {
             // We failed for embedded data, this is unlikely.
@@ -176,11 +196,16 @@ public class ContentBlockerRulesSourceManager {
             let params = [ContentBlockerDebugEvents.Parameters.errorDescription: errorDesc.isEmpty ? "empty" : errorDesc]
 
             errorReporting?.fire(.contentBlockingFallbackCompilationFailed,
+                                 scope: input.name,
                                  error: error,
                                  parameters: params,
                                  onComplete: { _ in
-                fatalError("Could not compile embedded rules list")
+                if input.name == DefaultContentBlockerRulesListsSource.Constants.trackerDataSetRulesListName {
+                    fatalError("Could not compile embedded rules list")
+                }
             })
+
+            fallbackTDSFailure = true
         }
     }
 }

Sources/BrowserServicesKit/ContentBlocking/EventMapping.swift

@@ -20,15 +20,18 @@
 import Foundation
 
 public class EventMapping<BSKEvent> {
-    public typealias Mapping = (_ event: BSKEvent, _ error: Error?, _ params: [String: String]?, _ onComplete: @escaping (Error?) -> Void) -> Void
+    public typealias Mapping = (_ event: BSKEvent,
+                                _ scope: String?,
+                                _ error: Error?,
+                                _ params: [String: String]?, _ onComplete: @escaping (Error?) -> Void) -> Void
 
     private let eventMapper: Mapping
 
     public init(mapping: @escaping Mapping) {
         eventMapper = mapping
     }
 
-    public func fire(_ event: BSKEvent, error: Error? = nil, parameters: [String: String]? = nil, onComplete: @escaping (Error?) -> Void = {_ in }) {
-        eventMapper(event, error, parameters, onComplete)
+    public func fire(_ event: BSKEvent, scope: String? = nil, error: Error? = nil, parameters: [String: String]? = nil, onComplete: @escaping (Error?) -> Void = {_ in }) {
+        eventMapper(event, scope, error, parameters, onComplete)
     }
 }

Sources/BrowserServicesKit/ContentBlocking/UserScripts/ContentBlockerRulesUserScript.swift

@@ -23,6 +23,7 @@ import TrackerRadarKit
 public protocol ContentBlockerRulesUserScriptDelegate: NSObjectProtocol {
 
     func contentBlockerRulesUserScriptShouldProcessTrackers(_ script: ContentBlockerRulesUserScript) -> Bool
+    func contentBlockerRulesUserScriptShouldProcessCTLTrackers(_ script: ContentBlockerRulesUserScript) -> Bool
     func contentBlockerRulesUserScript(_ script: ContentBlockerRulesUserScript,
                                        detectedTracker tracker: DetectedTracker)
 
@@ -32,20 +33,31 @@ public protocol ContentBlockerUserScriptConfig: UserScriptSourceProviding {
 
     var privacyConfiguration: PrivacyConfiguration { get }
     var trackerData: TrackerData? { get }
-
+    var ctlTrackerData: TrackerData? { get }
 }
 
 public class DefaultContentBlockerUserScriptConfig: ContentBlockerUserScriptConfig {
 
     public let privacyConfiguration: PrivacyConfiguration
     public let trackerData: TrackerData?
+    public let ctlTrackerData: TrackerData?
 
     public private(set) var source: String
 
     public init(privacyConfiguration: PrivacyConfiguration,
-                trackerData: TrackerData?) { // This should be non-optional
+                trackerData: TrackerData?, // This should be non-optional
+                ctlTrackerData: TrackerData?,
+                trackerDataManager: TrackerDataManager? = nil) {
+
+        if trackerData == nil {
+            // Fallback to embedded
+            self.trackerData = trackerDataManager?.trackerData
+        } else {
+            self.trackerData = trackerData
+        }
+
         self.privacyConfiguration = privacyConfiguration
-        self.trackerData = trackerData
+        self.ctlTrackerData = ctlTrackerData
 
         source = ContentBlockerRulesUserScript.generateSource(privacyConfiguration: privacyConfiguration)
     }
@@ -91,6 +103,7 @@ open class ContentBlockerRulesUserScript: NSObject, UserScript {
     public func userContentController(_ userContentController: WKUserContentController, didReceive message: WKScriptMessage) {
         guard let delegate = delegate else { return }
         guard delegate.contentBlockerRulesUserScriptShouldProcessTrackers(self) else { return }
+        let ctlEnabled = delegate.contentBlockerRulesUserScriptShouldProcessCTLTrackers(self)
 
         guard let dict = message.body as? [String: Any] else { return }
 
@@ -105,6 +118,22 @@ open class ContentBlockerRulesUserScript: NSObject, UserScript {
         }
 
         let privacyConfiguration = configuration.privacyConfiguration
+
+        if ctlEnabled, let ctlTrackerData = configuration.ctlTrackerData {
+            let resolver = TrackerResolver(tds: ctlTrackerData,
+                                           unprotectedSites: privacyConfiguration.userUnprotectedDomains,
+                                           tempList: temporaryUnprotectedDomains)
+
+            if let tracker = resolver.trackerFromUrl(trackerUrlString,
+                                                     pageUrlString: pageUrlStr,
+                                                     resourceType: resourceType,
+                                                     potentiallyBlocked: blocked && privacyConfiguration.isEnabled(featureKey: .contentBlocking)) {
+                if tracker.blocked {
+                    delegate.contentBlockerRulesUserScript(self, detectedTracker: tracker)
+                    return
+                }
+            }
+        }
 
         let resolver = TrackerResolver(tds: currentTrackerData,
                                        unprotectedSites: privacyConfiguration.userUnprotectedDomains,