Merge branch 'main' into sam/data-broker-remote-messaging

# By Dominik Kapusta (2) and others
# Via GitHub
* main:
  Make sure when we set custom config url, we don't expect etag in return (#1994)
  Add PixelKit source parameter (#1989)
  Fix internal user toggling (#2000)
  Show alert and display warning icon in Sync Settings when data syncing is disabled (#1996)
  DBP: Integrate subscription account authentication to DBP (#1995)
  Improve bookmarks html reader (#1986)
  Add Sync feature flags (#1992)
  Add daily stats pixel (#1993)
  Do not reload DBP tab when switching to it (#1942)
  Fix: external application requests via redirect URLs shows wrong origin. (#1900)

# Conflicts:
#	DuckDuckGo/Statistics/PixelEvent.swift

DuckDuckGo.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved

@@ -14,8 +14,8 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/duckduckgo/BrowserServicesKit",
       "state" : {
-        "revision" : "ae9e9180f74d92c83fc3cc1d2fc23f4855fb361c",
-        "version" : "95.0.0"
+        "revision" : "ea133abe237b6cb57a4237e0373318a40c10afc2",
+        "version" : "98.0.1"
       }
     },
     {
@@ -108,6 +108,24 @@
         "version" : "1.3.0"
       }
     },
+    {
+      "identity" : "swift-snapshot-testing",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/pointfreeco/swift-snapshot-testing",
+      "state" : {
+        "revision" : "59b663f68e69f27a87b45de48cb63264b8194605",
+        "version" : "1.15.1"
+      }
+    },
+    {
+      "identity" : "swift-syntax",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/apple/swift-syntax.git",
+      "state" : {
+        "revision" : "6ad4ea24b01559dde0773e3d091f1b9e36175036",
+        "version" : "509.0.2"
+      }
+    },
     {
       "identity" : "swifter",
       "kind" : "remoteSourceControl",
@@ -129,7 +147,7 @@
     {
       "identity" : "trackerradarkit",
       "kind" : "remoteSourceControl",
-      "location" : "https://github.com/duckduckgo/TrackerRadarKit.git",
+      "location" : "https://github.com/duckduckgo/TrackerRadarKit",
       "state" : {
         "revision" : "a6b7ba151d9dc6684484f3785293875ec01cc1ff",
         "version" : "1.2.2"

DuckDuckGo/Application/AppDelegate.swift

@@ -69,10 +69,15 @@ final class AppDelegate: NSObject, NSApplicationDelegate, FileDownloadManagerDel
     private(set) var syncDataProviders: SyncDataProviders!
     private(set) var syncService: DDGSyncing?
     private var isSyncInProgressCancellable: AnyCancellable?
+    private var syncFeatureFlagsCancellable: AnyCancellable?
     private var screenLockedCancellable: AnyCancellable?
     private var emailCancellables = Set<AnyCancellable>()
     let bookmarksManager = LocalBookmarkManager.shared
 
+#if DBP && SUBSCRIPTION
+    private let dataBrokerProtectionSubscriptionEventHandler = DataBrokerProtectionSubscriptionEventHandler()
+#endif
+
     private var didFinishLaunching = false
 
 #if SPARKLE
@@ -242,6 +247,10 @@ final class AppDelegate: NSObject, NSApplicationDelegate, FileDownloadManagerDel
         UNUserNotificationCenter.current().delegate = self
 #endif
 
+#if DBP && SUBSCRIPTION
+        dataBrokerProtectionSubscriptionEventHandler.registerForSubscriptionAccountManagerEvents()
+#endif
+
 #if DBP
         DataBrokerProtectionAppEvents().applicationDidFinishLaunching()
 #endif
@@ -339,7 +348,13 @@ final class AppDelegate: NSObject, NSApplicationDelegate, FileDownloadManagerDel
         let environment = defaultEnvironment
 #endif
         let syncDataProviders = SyncDataProviders(bookmarksDatabase: BookmarkDatabase.shared.db)
-        let syncService = DDGSync(dataProvidersSource: syncDataProviders, errorEvents: SyncErrorHandler(), log: OSLog.sync, environment: environment)
+        let syncService = DDGSync(
+            dataProvidersSource: syncDataProviders,
+            errorEvents: SyncErrorHandler(),
+            privacyConfigurationManager: ContentBlocking.shared.privacyConfigurationManager,
+            log: OSLog.sync,
+            environment: environment
+        )
         syncService.initializeIfNeeded()
         syncDataProviders.setUpDatabaseCleaners(syncService: syncService)
 
@@ -356,12 +371,43 @@ final class AppDelegate: NSObject, NSApplicationDelegate, FileDownloadManagerDel
         isSyncInProgressCancellable = syncService.isSyncInProgressPublisher
             .filter { $0 }
             .asVoid()
-            .prefix(1)
-            .sink {
+            .sink { [weak syncService] in
                 Pixel.fire(.syncDaily, limitTo: .dailyFirst)
+                syncService?.syncDailyStats.sendStatsIfNeeded(handler: { params in
+                    Pixel.fire(.syncSuccessRateDaily, withAdditionalParameters: params)
+                })
             }
 
         subscribeSyncQueueToScreenLockedNotifications()
+        subscribeToSyncFeatureFlags(syncService)
+    }
+
+    @UserDefaultsWrapper(key: .syncDidShowSyncPausedByFeatureFlagAlert, defaultValue: false)
+    private var syncDidShowSyncPausedByFeatureFlagAlert: Bool
+
+    private func subscribeToSyncFeatureFlags(_ syncService: DDGSync) {
+        syncFeatureFlagsCancellable = syncService.featureFlagsPublisher
+            .dropFirst()
+            .map { $0.contains(.dataSyncing) }
+            .receive(on: DispatchQueue.main)
+            .sink { [weak self, weak syncService] isDataSyncingAvailable in
+                if isDataSyncingAvailable {
+                    self?.syncDidShowSyncPausedByFeatureFlagAlert = false
+                } else if syncService?.authState == .active, self?.syncDidShowSyncPausedByFeatureFlagAlert == false {
+                    let isSyncUIVisible = syncService?.featureFlags.contains(.userInterface) == true
+                    let alert = NSAlert.dataSyncingDisabledByFeatureFlag(showLearnMore: isSyncUIVisible)
+                    let response = alert.runModal()
+                    self?.syncDidShowSyncPausedByFeatureFlagAlert = true
+
+                    switch response {
+                    case .alertSecondButtonReturn:
+                        alert.window.sheetParent?.endSheet(alert.window)
+                        WindowControllersManager.shared.showPreferencesTab(withSelectedPane: .sync)
+                    default:
+                        break
+                    }
+                }
+            }
     }
 
     private func subscribeSyncQueueToScreenLockedNotifications() {

DuckDuckGo/Bookmarks/Services/LocalBookmarkStore.swift

@@ -279,9 +279,6 @@ final class LocalBookmarkStore: BookmarkStore {
                     // will return the children of the root folder, as the root folder is an implementation detail of the bookmarks store.
                     let rootFolder = self.bookmarksRoot(in: context)
                     let orphanedEntities = BookmarkUtils.fetchOrphanedEntities(context)
-                    if !orphanedEntities.isEmpty {
-                        self.reportOrphanedBookmarksIfNeeded()
-                    }
                     results = (rootFolder?.childrenArray ?? []) + orphanedEntities
                 case .favorites:
                     results = self.favoritesRoot(in: context)?.favoritesArray ?? []
@@ -300,15 +297,6 @@ final class LocalBookmarkStore: BookmarkStore {
         }
     }
 
-    private func reportOrphanedBookmarksIfNeeded() {
-        Task { @MainActor in
-            guard let syncService = NSApp.delegateTyped.syncService, syncService.authState == .inactive else {
-                return
-            }
-            Pixel.fire(.debug(event: .orphanedBookmarksPresent))
-        }
-    }
-
     func save(bookmark: Bookmark, parent: BookmarkFolder?, index: Int?, completion: @escaping (Bool, Error?) -> Void) {
         applyChangesAndSave(changes: { [weak self] context in
             guard let self = self else {

DuckDuckGo/Common/Extensions/NSAlertExtension.swift

@@ -221,6 +221,18 @@ extension NSAlert {
         return alert
     }
 
+    static func dataSyncingDisabledByFeatureFlag(showLearnMore: Bool, upgradeRequired: Bool = false) -> NSAlert {
+        let alert = NSAlert()
+        alert.messageText = UserText.syncPausedTitle
+        alert.informativeText = upgradeRequired ? UserText.syncUnavailableMessageUpgradeRequired : UserText.syncUnavailableMessage
+        alert.alertStyle = .warning
+        alert.addButton(withTitle: UserText.ok)
+        if showLearnMore {
+            alert.addButton(withTitle: UserText.learnMore)
+        }
+        return alert
+    }
+
     static func customConfigurationAlert(configurationUrl: String) -> NSAlert {
         let alert = NSAlert()
         alert.messageText = "Set custom configuration URL:"

DuckDuckGo/Common/Localizables/UserText.swift

@@ -503,6 +503,9 @@ struct UserText {
     static let syncBookmarkPausedAlertDescription = NSLocalizedString("alert.sync-bookmarks-paused-description", value: "You have exceeded the bookmarks sync limit. Try deleting some bookmarks. Until this is resolved your bookmarks will not be backed up.", comment: "Description for alert shown when sync bookmarks paused for too many items")
     static let syncCredentialsPausedAlertTitle = NSLocalizedString("alert.sync-credentials-paused-title", value: "Passwords Sync is Paused", comment: "Title for alert shown when sync credentials paused for too many items")
     static let syncCredentialsPausedAlertDescription = NSLocalizedString("alert.sync-credentials-paused-description", value: "You have exceeded the passwords sync limit. Try deleting some passwords. Until this is resolved your passwords will not be backed up.", comment: "Description for alert shown when sync credentials paused for too many items")
+    static let syncPausedTitle = NSLocalizedString("alert.sync.warning.sync-paused", value: "Sync & Backup is Paused", comment: "Title of the warning message")
+    static let syncUnavailableMessage = NSLocalizedString("alert.sync.warning.sync-unavailable-message", value: "Sorry, but Sync & Backup is currently unavailable. Please try again later.", comment: "Data syncing unavailable warning message")
+    static let syncUnavailableMessageUpgradeRequired = NSLocalizedString("alert.sync.warning.data-syncing-disabled-upgrade-required", value: "Sorry, but Sync & Backup is no longer available in this app version. Please update DuckDuckGo to the latest version to continue.", comment: "Data syncing unavailable warning message")
     static let defaultBrowser = NSLocalizedString("preferences.default-browser", value: "Default Browser", comment: "Show default browser preferences")
     static let appearance = NSLocalizedString("preferences.appearance", value: "Appearance", comment: "Show appearance preferences")
     static let privacy = NSLocalizedString("preferences.privacy", value: "Privacy", comment: "Show privacy browser preferences")

DuckDuckGo/Common/Utilities/UserDefaultsWrapper.swift

@@ -41,6 +41,8 @@ public struct UserDefaultsWrapper<T> {
         case configStoragePrivacyConfigurationEtag = "config.storage.privacyconfiguration.etag"
         case configFBConfigEtag = "config.storage.fbconfig.etag"
 
+        case configLastInstalled = "config.last.installed"
+
         case fireproofDomains = "com.duckduckgo.fireproofing.allowedDomains"
         case areDomainsMigratedToETLDPlus1 = "com.duckduckgo.are-domains-migrated-to-etldplus1"
         case unprotectedDomains = "com.duckduckgo.contentblocker.unprotectedDomains"
@@ -176,6 +178,7 @@ public struct UserDefaultsWrapper<T> {
         case syncIsEligibleForFaviconsFetcherOnboarding = "sync.is-eligible-for-favicons-fetcher-onboarding"
         case syncDidPresentFaviconsFetcherOnboarding = "sync.did-present-favicons-fetcher-onboarding"
         case syncDidMigrateToImprovedListsHandling = "sync.did-migrate-to-improved-lists-handling"
+        case syncDidShowSyncPausedByFeatureFlagAlert = "sync.did-show-sync-paused-by-feature-flag-alert"
     }
 
     enum RemovedKeys: String, CaseIterable {

DuckDuckGo/Configuration/ConfigurationManager.swift

@@ -61,6 +61,9 @@ final class ConfigurationManager {
     @UserDefaultsWrapper(key: .configLastUpdated, defaultValue: .distantPast)
     private(set) var lastUpdateTime: Date
 
+    @UserDefaultsWrapper(key: .configLastInstalled, defaultValue: nil)
+    private(set) var lastConfigurationInstallDate: Date?
+
     private var timerCancellable: AnyCancellable?
     private var lastRefreshCheckTime: Date = Date()
 
@@ -97,9 +100,9 @@ final class ConfigurationManager {
         os_log("last refresh check %{public}s", log: .config, type: .default, String(describing: lastRefreshCheckTime))
     }
 
-    private func refreshNow() async {
+    private func refreshNow(isDebug: Bool = false) async {
         let updateTrackerBlockingDependenciesTask = Task {
-            let didFetchAnyTrackerBlockingDependencies = await fetchTrackerBlockingDependencies()
+            let didFetchAnyTrackerBlockingDependencies = await fetchTrackerBlockingDependencies(isDebug: isDebug)
             if didFetchAnyTrackerBlockingDependencies {
                 updateTrackerBlockingDependencies()
                 tryAgainLater()
@@ -134,13 +137,13 @@ final class ConfigurationManager {
         log()
     }
 
-    private func fetchTrackerBlockingDependencies() async -> Bool {
+    private func fetchTrackerBlockingDependencies(isDebug: Bool) async -> Bool {
         var didFetchAnyTrackerBlockingDependencies = false
 
         var tasks = [Configuration: Task<(), Swift.Error>]()
         tasks[.trackerDataSet] = Task { try await fetcher.fetch(.trackerDataSet) }
         tasks[.surrogates] = Task { try await fetcher.fetch(.surrogates) }
-        tasks[.privacyConfiguration] = Task { try await fetcher.fetch(.privacyConfiguration) }
+        tasks[.privacyConfiguration] = Task { try await fetcher.fetch(.privacyConfiguration, isDebug: isDebug) }
 
         for (configuration, task) in tasks {
             do {
@@ -184,9 +187,9 @@ final class ConfigurationManager {
 
     private var isReadyToRefresh: Bool { Date().timeIntervalSince(lastUpdateTime) > Constants.refreshPeriodSeconds }
 
-    public func forceRefresh() {
+    public func forceRefresh(isDebug: Bool = false) {
         Task {
-            await refreshNow()
+            await refreshNow(isDebug: isDebug)
         }
     }
 
@@ -200,6 +203,7 @@ final class ConfigurationManager {
     }
 
     private func updateTrackerBlockingDependencies() {
+        lastConfigurationInstallDate = Date()
         ContentBlocking.shared.trackerDataManager.reload(etag: ConfigurationStore.shared.loadEtag(for: .trackerDataSet),
                                                          data: ConfigurationStore.shared.loadData(for: .trackerDataSet))
         ContentBlocking.shared.privacyConfigurationManager.reload(etag: ConfigurationStore.shared.loadEtag(for: .privacyConfiguration),

DuckDuckGo/ContentBlocker/AppPrivacyConfigurationDataProvider.swift

@@ -22,8 +22,8 @@ import BrowserServicesKit
 final class AppPrivacyConfigurationDataProvider: EmbeddedDataProvider {
 
     public struct Constants {
-        public static let embeddedDataETag = "\"ca66d409eb00e5c19f3a0abae449dd1a\""
-        public static let embeddedDataSHA = "42f9d3064372bc85ac8ee37afe883ed4741d6a3cfcb9ce927c2f732c3f694140"
+        public static let embeddedDataETag = "\"8e38db3f042f4a2a5c9f7ca8d33f17c9\""
+        public static let embeddedDataSHA = "a9fc8ee927a37d1b5545377ff29de798f3ec3b8a757430c3af72d13a3844c258"
     }
 
     var embeddedDataEtag: String {

DuckDuckGo/ContentBlocker/Mocks/MockPrivacyConfiguration.swift

@@ -51,6 +51,16 @@ final class MockPrivacyConfiguration: PrivacyConfiguration {
     func userDisabledProtection(forDomain: String) {}
 }
 
+final class MockInternalUserStoring: InternalUserStoring {
+    var isInternalUser: Bool = false
+}
+
+extension DefaultInternalUserDecider {
+    convenience init(mockedStore: MockInternalUserStoring = MockInternalUserStoring()) {
+        self.init(store: mockedStore)
+    }
+}
+
 final class MockPrivacyConfigurationManager: NSObject, PrivacyConfigurationManaging {
     var embeddedConfigData: BrowserServicesKit.PrivacyConfigurationManager.ConfigurationData {
         fatalError("not implemented")
@@ -70,6 +80,7 @@ final class MockPrivacyConfigurationManager: NSObject, PrivacyConfigurationManag
 
     var updatesPublisher: AnyPublisher<Void, Never> = Just(()).eraseToAnyPublisher()
     var privacyConfig: PrivacyConfiguration = MockPrivacyConfiguration()
+    var internalUserDecider: InternalUserDecider = DefaultInternalUserDecider()
 }
 
 #endif