clean up clear data code

.maestro/data_clearing_tests/01_fire_proofing.yml

@@ -0,0 +1,75 @@
+appId: com.duckduckgo.mobile.ios
+tags:
+    - dataclearing
+
+---
+
+# Set up 
+- clearState
+- launchApp
+- runFlow: 
+    when: 
+      visible: 
+        text: "Let’s Do It!"
+        index: 0
+    file: ../shared/onboarding.yaml
+
+# Load Site
+- assertVisible:
+    id: "searchEntry"
+- tapOn: 
+    id: "searchEntry"
+- inputText: "https://setcookie.net"
+- pressKey: Enter
+
+# Set a cookie
+- assertVisible: "Cookie Test"
+- tapOn: "Cookie name"
+- inputText: "TestName"
+- tapOn: "Cookie value"
+- inputText: "TestValue"
+- scrollUntilVisible: 
+    centerElement: true
+    element:
+      text: "Submit"
+- tapOn: "Submit"
+
+# Fireproof the site
+- tapOn: "Browsing Menu"
+- tapOn: "Fireproof This Site"
+- tapOn: "Fireproof"
+- assertVisible: "setcookie.net is now Fireproof"
+
+# Fire Button - twice, just to be sure
+- tapOn: "Close Tabs and Clear Data"
+- tapOn:
+    id: "alert.forget-data.confirm"
+- assertVisible:
+    id: "searchEntry"
+- tapOn: "Close Tabs and Clear Data"
+- tapOn:
+    id: "alert.forget-data.confirm"
+
+# Validate Cookie was retained
+- tapOn: 
+    id: "searchEntry"
+- inputText: "https://setcookie.net"
+- pressKey: Enter
+- assertVisible: "TestName = TestValue"
+
+# Remove fireproofing
+- tapOn: "Browsing Menu"
+- tapOn: "Remove Fireproofing"
+
+# Fire Button
+- tapOn: "Close Tabs and Clear Data"
+- tapOn:
+    id: "alert.forget-data.confirm"
+
+# Validate Cookie was removed
+- tapOn: 
+    id: "searchEntry"
+- inputText: "https://setcookie.net"
+- pressKey: Enter
+- assertVisible: "Cookie Test"
+- assertVisible: "Received no cookies."

.maestro/data_clearing_tests/02_duckduckgo_settings.yml

@@ -0,0 +1,44 @@
+appId: com.duckduckgo.mobile.ios
+tags:
+    - privacy
+
+---
+
+# Set up 
+- clearState
+- launchApp
+- runFlow: 
+    when: 
+      visible: 
+        text: "Let’s Do It!"
+        index: 0
+    file: ../shared/onboarding.yaml
+
+# Load Site
+- assertVisible:
+    id: "searchEntry"
+- tapOn: 
+    id: "searchEntry"
+- inputText: "privacy blogs"
+- pressKey: Enter
+
+# Change settings
+- tapOn: "Safe search: moderate ▼"
+- tapOn: "Off"
+
+# Fire Button - twice, just to be sure
+- tapOn: "Close Tabs and Clear Data"
+- tapOn:
+    id: "alert.forget-data.confirm"
+- assertVisible:
+    id: "searchEntry"
+- tapOn: "Close Tabs and Clear Data"
+- tapOn:
+    id: "alert.forget-data.confirm"
+
+# Validate Cookie was retained
+- tapOn: 
+    id: "searchEntry"
+- inputText: "creepy trackers"
+- pressKey: Enter
+- assertVisible: "Safe search: off ▼"

Core/CookieStorage.swift

@@ -20,6 +20,12 @@
 import Common
 import Foundation
 
+/// Class for persisting cookies for fire proofed sites to work around a WKWebView / DataStore bug which does not let data get persisted until the webview has loaded.
+///
+/// Privacy information:
+/// * The Fire Button does not delete the user's DuckDuckGo search settings, which are saved as cookies. Removing these cookies would reset them and have undesired consequences, i.e. changing the theme, default language, etc.
+/// * The Fire Button also does not delete temporary cookies associated with 'surveys.duckduckgo.com'. When we launch surveys to help us understand issues that impact users over time, we use this cookie to temporarily store anonymous survey answers, before deleting the cookie. Cookie storage duration is communicated to users before they opt to submit survey answers.
+/// * These cookies are not stored in a personally identifiable way. For example, the large size setting is stored as 's=l.' More info in https://duckduckgo.com/privacy
 public class CookieStorage {
 
     struct Keys {
@@ -31,7 +37,7 @@ public class CookieStorage {
 
     var isConsumed: Bool {
         get {
-            userDefaults.bool(forKey: Keys.consumed, defaultValue: false)
+            return userDefaults.bool(forKey: Keys.consumed, defaultValue: false)
         }
         set {
             userDefaults.set(newValue, forKey: Keys.consumed)
@@ -77,15 +83,20 @@ public class CookieStorage {
         self.userDefaults = userDefaults
     }
 
-    enum CookieDomainsOnUpdate {
+    /// Used when debugging (e.g. on the simulator).
+    enum CookieDomainsOnUpdateDiagnostic {
         case empty
         case match
         case missing
         case different
+        case notConsumed
     }
 
+    /// Update ALL cookies. The absence of cookie domains here indicateds they have been removed by the website, so be sure to call this with all cookies that might need to be persisted even if those websites have not been visited yet.
     @discardableResult
-    func updateCookies(_ cookies: [HTTPCookie], keepingPreservedLogins preservedLogins: PreserveLogins) -> CookieDomainsOnUpdate {
+    func updateCookies(_ cookies: [HTTPCookie], keepingPreservedLogins preservedLogins: PreserveLogins) -> CookieDomainsOnUpdateDiagnostic {
+        guard isConsumed else { return .notConsumed }
+
         isConsumed = false
 
         let persisted = self.cookies
@@ -109,7 +120,9 @@ public class CookieStorage {
             persistedDomains: persistedCookiesByDomain.keys.sorted()
         )
 
-        updatedCookiesByDomain.keys.forEach {
+        let cookieDomains = Set(updatedCookiesByDomain.keys.map { $0 } + persistedCookiesByDomain.keys.map { $0 })
+
+        cookieDomains.forEach {
             persistedCookiesByDomain[$0] = updatedCookiesByDomain[$0]
         }
 
@@ -128,7 +141,7 @@ public class CookieStorage {
         return diagnosticResult
     }
 
-    private func evaluateDomains(updatedDomains: [String], persistedDomains: [String]) -> CookieDomainsOnUpdate {
+    private func evaluateDomains(updatedDomains: [String], persistedDomains: [String]) -> CookieDomainsOnUpdateDiagnostic {
         if persistedDomains.isEmpty {
             return .empty
         } else if updatedDomains.count < persistedDomains.count {

Core/DataStoreWarmup.swift

@@ -0,0 +1,72 @@
+//
+//  DataStoreWarmup.swift
+//  DuckDuckGo
+//
+//  Copyright © 2024 DuckDuckGo. All rights reserved.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+//
+
+import Combine
+import WebKit
+
+/// WKWebsiteDataStore is basically non-functional until a web view has been instanciated and a page is successfully loaded.
+public class DataStoreWarmup {
+
+    public init() { }
+
+    @MainActor
+    public func ensureReady() async {
+        await BlockingNavigationDelegate().loadInBackgroundWebView(url: URL(string: "about:blank")!)
+    }
+
+}
+
+private class BlockingNavigationDelegate: NSObject, WKNavigationDelegate {
+
+    let finished = PassthroughSubject<Void, Never>()
+
+    func webView(_ webView: WKWebView, decidePolicyFor navigationAction: WKNavigationAction) async -> WKNavigationActionPolicy {
+        return .allow
+    }
+
+    func webView(_ webView: WKWebView, didFinish navigation: WKNavigation!) {
+        finished.send()
+    }
+
+    func webViewWebContentProcessDidTerminate(_ webView: WKWebView) {
+        Pixel.fire(pixel: .webKitDidTerminateDuringWarmup)
+        // We won't get a `didFinish` if the webview crashes
+        finished.send()
+    }
+
+    var cancellable: AnyCancellable?
+    func waitForLoad() async {
+        await withCheckedContinuation { continuation in
+            cancellable = finished.sink { _ in
+                continuation.resume()
+            }
+        }
+    }
+
+    @MainActor
+    func loadInBackgroundWebView(url: URL) async {
+        let config = WKWebViewConfiguration.persistent()
+        let webView = WKWebView(frame: .zero, configuration: config)
+        webView.navigationDelegate = self
+        let request = URLRequest(url: url)
+        webView.load(request)
+        await waitForLoad()
+    }
+
+}

Core/PixelEvent.swift

@@ -394,7 +394,8 @@ extension Pixel {
 
         case webKitDidTerminate
         case webKitTerminationDidReloadCurrentTab
-
+        case webKitDidTerminateDuringWarmup
+
         case backgroundTaskSubmissionFailed
 
         case blankOverlayNotDismissed
@@ -891,6 +892,7 @@ extension Pixel.Event {
         case .ampBlockingRulesCompilationFailed: return "m_debug_amp_rules_compilation_failed"
 
         case .webKitDidTerminate: return "m_d_wkt"
+        case .webKitDidTerminateDuringWarmup: return "m_d_webkit-terminated-during-warmup"
         case .webKitTerminationDidReloadCurrentTab: return "m_d_wktct"
 
         case .backgroundTaskSubmissionFailed: return "m_bt_rf"
@@ -926,7 +928,7 @@ extension Pixel.Event {
         case .debugCannotClearObservationsDatabase: return "m_d_cannot_clear_observations_database"
         case .debugWebsiteDataStoresNotClearedMultiple: return "m_d_wkwebsitedatastoresnotcleared_multiple"
         case .debugWebsiteDataStoresNotClearedOne: return "m_d_wkwebsitedatastoresnotcleared_one"
-        case .debugCookieCleanupError: return "m_cookie_cleanup_error"
+        case .debugCookieCleanupError: return "m_d_cookie-cleanup-error"
 
         // MARK: Ad Attribution
 

Core/PreserveLogins.swift

@@ -46,7 +46,6 @@ public class PreserveLogins {
         return allowedDomains.contains(where: { $0 == cookieDomain
             || ".\($0)" == cookieDomain
             || (cookieDomain.hasPrefix(".") && $0.hasSuffix(cookieDomain)) })
-
     }
 
     public func remove(domain: String) {