Merge #4301

4301: Merge day! r=rocketsroger a=pocmo

New GeckoView versions!

* browser-engine-gecko-beta (69) -> browser-engine-gecko-release (69)
* browser-engine-gecko-nightly (70) -> browser-engine-gecko-beta (70)
* browser-engine-gecko-nightly -> 71

This time I took some notes. I'll turn this into a documentation and then next time someone else can try them and perform the merge day changes. :)

This also needs to be added to the changelog. But I'm heading out now for today. Will add that later.



Co-authored-by: Sebastian Kaspari <s.kaspari@gmail.com>

buildSrc/src/main/java/Gecko.kt

@@ -6,17 +6,17 @@ internal object GeckoVersions {
     /**
      * GeckoView Nightly Version.
      */
-    const val nightly_version = "70.0.20190901094958"
+    const val nightly_version = "71.0.20190903094847"
 
     /**
      * GeckoView Beta Version.
      */
-    const val beta_version = "69.0.20190824140027"
+    const val beta_version = "70.0.20190902085437"
 
     /**
      * GeckoView Release Version.
      */
-    const val release_version = "68.0.20190711090008"
+    const val release_version = "69.0.20190903125908"
 }
 
 @Suppress("MaxLineLength")

components/browser/engine-gecko-beta/build.gradle

@@ -2,6 +2,10 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
+plugins {
+    id "com.jetbrains.python.envs" version "0.0.26"
+}
+
 apply plugin: 'com.android.library'
 apply plugin: 'kotlin-android'
 
@@ -26,6 +30,11 @@ android {
     }
 }
 
+// Set configuration for the Glean parser to extract metrics.yaml
+// file from AAR dependencies of this project rather than look
+// for it into the project directory.
+ext.allowMetricsFromAAR = true
+
 dependencies {
     implementation project(':concept-engine')
     implementation project(':concept-fetch')
@@ -40,17 +49,24 @@ dependencies {
     testImplementation Dependencies.androidx_test_core
     testImplementation Dependencies.androidx_test_junit
     testImplementation Dependencies.testing_robolectric
+    testImplementation Dependencies.testing_coroutines
     testImplementation Dependencies.testing_mockito
     testImplementation Dependencies.testing_mockwebserver
-    testImplementation Dependencies.testing_coroutines
     testImplementation project(':support-test')
     testImplementation project(':tooling-fetch-tests')
 
+    // We only compile against Glean. It's up to the app to add those dependencies
+    // if it wants to collect GeckoView telemetry through the Glean SDK.
+    compileOnly project(":service-glean")
+    testImplementation project(":service-glean")
+    testImplementation Dependencies.androidx_work_testing
+
     androidTestImplementation Dependencies.androidx_test_core
     androidTestImplementation Dependencies.androidx_test_runner
     androidTestImplementation Dependencies.androidx_test_rules
     androidTestImplementation project(':tooling-fetch-tests')
 }
 
+apply from: '../../../components/service/glean/scripts/sdk_generator.gradle'
 apply from: '../../../publish.gradle'
 ext.configurePublish(config.componentsGroupId, archivesBaseName, project.ext.description)

components/browser/engine-gecko-beta/metrics.yaml

@@ -0,0 +1,29 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# IMPORTANT NOTE: this file is here only as a safety measure, to make
+# sure the correct code is generated even though the GeckoView AAR file
+# reports an empty metrics.yaml file. The metric in this file is currently
+# disabled and not supposed to collect any data.
+
+$schema: moz://mozilla.org/schemas/glean/metrics/1-0-0
+
+test.glean.geckoview:
+  streaming:
+    type: timing_distribution
+    gecko_datapoint: TELEMETRY_TEST_STREAMING
+    disabled: true
+    description: |
+      A test-only, disabled metric. This is required to guarantee
+      that a `GleanGeckoHistogramMapping` is always generated, even
+      though the GeckoView AAR exports no metric. Please note that
+      the data-review field below contains no review, since this
+      metric is disabled and not allowed to collect any data.
+    bugs:
+      - 1566374
+    data_reviews:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1566374
+    notification_emails:
+      - glean-team@mozilla.com
+    expires: never

components/browser/engine-gecko-beta/src/main/java/mozilla/components/browser/engine/gecko/GeckoEngine.kt

@@ -12,8 +12,8 @@ import mozilla.components.browser.engine.gecko.mediaquery.toGeckoValue
 import mozilla.components.browser.engine.gecko.webextension.GeckoWebExtension
 import mozilla.components.concept.engine.Engine
 import mozilla.components.concept.engine.EngineSession
-import mozilla.components.concept.engine.EngineSession.SafeBrowsingPolicy
 import mozilla.components.concept.engine.EngineSession.TrackingProtectionPolicy
+import mozilla.components.concept.engine.EngineSession.SafeBrowsingPolicy
 import mozilla.components.concept.engine.EngineSessionState
 import mozilla.components.concept.engine.EngineView
 import mozilla.components.concept.engine.Settings
@@ -160,24 +160,15 @@ class GeckoEngine(
         override var safeBrowsingPolicy: Array<SafeBrowsingPolicy> =
             arrayOf(SafeBrowsingPolicy.RECOMMENDED)
             set(value) {
-                val safeBrowsingCategories = value.sumBy { it.id }
-                val trackingCategories =
-                    trackingProtectionPolicy?.trackingCategories?.sumBy { it.id }
-                        ?: SafeBrowsingPolicy.NONE.id
-
-                runtime.settings.contentBlocking.categories =
-                    safeBrowsingCategories + trackingCategories
+                val policy = value.sumBy { it.id }
+                runtime.settings.contentBlocking.setSafeBrowsing(policy)
                 field = value
             }
 
         override var trackingProtectionPolicy: TrackingProtectionPolicy? = null
             set(value) {
                 value?.let { policy ->
-
-                    val trackingCategories = policy.trackingCategories.sumBy { it.id } +
-                        safeBrowsingPolicy.sumBy { it.id }
-
-                    runtime.settings.contentBlocking.categories = trackingCategories
+                    runtime.settings.contentBlocking.setAntiTracking(policy.trackingCategories.sumBy { it.id })
                     runtime.settings.contentBlocking.cookieBehavior = policy.cookiePolicy.id
                     defaultSettings?.trackingProtectionPolicy = value
                     field = value
@@ -251,6 +242,7 @@ class GeckoEngine(
             this.automaticFontSizeAdjustment = it.automaticFontSizeAdjustment
             this.automaticLanguageAdjustment = it.automaticLanguageAdjustment
             this.trackingProtectionPolicy = it.trackingProtectionPolicy
+            this.safeBrowsingPolicy = arrayOf(SafeBrowsingPolicy.RECOMMENDED)
             this.remoteDebuggingEnabled = it.remoteDebuggingEnabled
             this.testingModeEnabled = it.testingModeEnabled
             this.userAgentString = it.userAgentString

components/browser/engine-gecko-beta/src/main/java/mozilla/components/browser/engine/gecko/GeckoEngineSession.kt

@@ -8,10 +8,13 @@ import android.annotation.SuppressLint
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.Job
+import kotlinx.coroutines.MainScope
+import kotlinx.coroutines.launch
 import kotlinx.coroutines.runBlocking
 import mozilla.components.browser.engine.gecko.media.GeckoMediaDelegate
 import mozilla.components.browser.engine.gecko.permission.GeckoPermissionRequest
 import mozilla.components.browser.engine.gecko.prompt.GeckoPromptDelegate
+import mozilla.components.browser.engine.gecko.window.GeckoWindowRequest
 import mozilla.components.browser.errorpages.ErrorType
 import mozilla.components.concept.engine.EngineSession
 import mozilla.components.concept.engine.EngineSessionState
@@ -27,7 +30,6 @@ import mozilla.components.support.ktx.android.util.Base64
 import mozilla.components.support.ktx.kotlin.isEmail
 import mozilla.components.support.ktx.kotlin.isGeoLocation
 import mozilla.components.support.ktx.kotlin.isPhone
-import mozilla.components.support.utils.DownloadUtils
 import org.json.JSONObject
 import org.mozilla.geckoview.AllowOrDeny
 import org.mozilla.geckoview.ContentBlocking
@@ -53,7 +55,8 @@ class GeckoEngineSession(
             .build()
         GeckoSession(settings)
     },
-    private val context: CoroutineContext = Dispatchers.IO
+    private val context: CoroutineContext = Dispatchers.IO,
+    openGeckoSession: Boolean = true
 ) : CoroutineScope, EngineSession() {
 
     internal lateinit var geckoSession: GeckoSession
@@ -85,7 +88,7 @@ class GeckoEngineSession(
         get() = context + job
 
     init {
-        createGeckoSession()
+        createGeckoSession(shouldOpen = openGeckoSession)
     }
 
     /**
@@ -302,11 +305,8 @@ class GeckoEngineSession(
             session: GeckoSession,
             request: NavigationDelegate.LoadRequest
         ): GeckoResult<AllowOrDeny> {
-            // TODO use onNewSession and create window request:
-            // https://github.com/mozilla-mobile/android-components/issues/1503
             if (request.target == GeckoSession.NavigationDelegate.TARGET_WINDOW_NEW) {
-                geckoSession.loadUri(request.uri)
-                return GeckoResult.fromValue(AllowOrDeny.DENY)
+                return GeckoResult.fromValue(AllowOrDeny.ALLOW)
             }
 
             val response = settings.requestInterceptor?.onLoadRequest(
@@ -348,7 +348,15 @@ class GeckoEngineSession(
         override fun onNewSession(
             session: GeckoSession,
             uri: String
-        ): GeckoResult<GeckoSession> = GeckoResult.fromValue(null)
+        ): GeckoResult<GeckoSession> {
+            val newEngineSession = GeckoEngineSession(runtime, privateMode, defaultSettings, openGeckoSession = false)
+            notifyObservers {
+                MainScope().launch {
+                    onOpenWindowRequest(GeckoWindowRequest(uri, newEngineSession))
+                }
+            }
+            return GeckoResult.fromValue(newEngineSession.geckoSession)
+        }
 
         override fun onLoadError(
             session: GeckoSession,
@@ -530,13 +538,11 @@ class GeckoEngineSession(
 
         override fun onExternalResponse(session: GeckoSession, response: GeckoSession.WebResponseInfo) {
             notifyObservers {
-                val fileName = response.filename
-                    ?: DownloadUtils.guessFileName(null, response.uri, response.contentType)
                 onExternalResource(
                         url = response.uri,
                         contentLength = response.contentLength,
                         contentType = response.contentType,
-                        fileName = fileName)
+                        fileName = response.filename)
             }
         }
 
@@ -571,38 +577,76 @@ class GeckoEngineSession(
                 onTrackerBlocked(event.toTracker())
             }
         }
+
+        override fun onContentLoaded(session: GeckoSession, event: ContentBlocking.BlockEvent) {
+            notifyObservers {
+                onTrackerLoaded(event.toTracker())
+            }
+        }
     }
 
     private fun ContentBlocking.BlockEvent.toTracker(): Tracker {
         val blockedContentCategories = mutableListOf<TrackingProtectionPolicy.TrackingCategory>()
 
-        if (categories.contains(ContentBlocking.AT_AD)) {
+        if (antiTrackingCategory.contains(ContentBlocking.AntiTracking.AD)) {
             blockedContentCategories.add(TrackingProtectionPolicy.TrackingCategory.AD)
         }
 
-        if (categories.contains(ContentBlocking.AT_ANALYTIC)) {
+        if (antiTrackingCategory.contains(ContentBlocking.AntiTracking.ANALYTIC)) {
             blockedContentCategories.add(TrackingProtectionPolicy.TrackingCategory.ANALYTICS)
         }
 
-        if (categories.contains(ContentBlocking.AT_SOCIAL)) {
+        if (antiTrackingCategory.contains(ContentBlocking.AntiTracking.SOCIAL)) {
             blockedContentCategories.add(TrackingProtectionPolicy.TrackingCategory.SOCIAL)
         }
 
-        if (categories.contains(ContentBlocking.AT_FINGERPRINTING)) {
+        if (antiTrackingCategory.contains(ContentBlocking.AntiTracking.FINGERPRINTING)) {
             blockedContentCategories.add(TrackingProtectionPolicy.TrackingCategory.FINGERPRINTING)
         }
 
-        if (categories.contains(ContentBlocking.AT_CRYPTOMINING)) {
+        if (antiTrackingCategory.contains(ContentBlocking.AntiTracking.CRYPTOMINING)) {
             blockedContentCategories.add(TrackingProtectionPolicy.TrackingCategory.CRYPTOMINING)
         }
-        if (categories.contains(ContentBlocking.AT_CONTENT)) {
+
+        if (antiTrackingCategory.contains(ContentBlocking.AntiTracking.CONTENT)) {
             blockedContentCategories.add(TrackingProtectionPolicy.TrackingCategory.CONTENT)
         }
 
-        if (categories.contains(ContentBlocking.AT_TEST)) {
+        if (antiTrackingCategory.contains(ContentBlocking.AntiTracking.TEST)) {
             blockedContentCategories.add(TrackingProtectionPolicy.TrackingCategory.TEST)
         }
-        return Tracker(uri, blockedContentCategories)
+
+        return Tracker(
+            url = uri,
+            trackingCategories = blockedContentCategories,
+            cookiePolicies = getCookiePolicies()
+        )
+    }
+
+    private fun ContentBlocking.BlockEvent.getCookiePolicies(): List<TrackingProtectionPolicy.CookiePolicy> {
+        val cookiesPolicies = mutableListOf<TrackingProtectionPolicy.CookiePolicy>()
+
+        if (cookieBehaviorCategory == ContentBlocking.CookieBehavior.ACCEPT_ALL) {
+            cookiesPolicies.add(TrackingProtectionPolicy.CookiePolicy.ACCEPT_ALL)
+        }
+
+        if (cookieBehaviorCategory.contains(ContentBlocking.CookieBehavior.ACCEPT_FIRST_PARTY)) {
+            cookiesPolicies.add(TrackingProtectionPolicy.CookiePolicy.ACCEPT_ONLY_FIRST_PARTY)
+        }
+
+        if (cookieBehaviorCategory.contains(ContentBlocking.CookieBehavior.ACCEPT_NONE)) {
+            cookiesPolicies.add(TrackingProtectionPolicy.CookiePolicy.ACCEPT_NONE)
+        }
+
+        if (cookieBehaviorCategory.contains(ContentBlocking.CookieBehavior.ACCEPT_NON_TRACKERS)) {
+            cookiesPolicies.add(TrackingProtectionPolicy.CookiePolicy.ACCEPT_NON_TRACKERS)
+        }
+
+        if (cookieBehaviorCategory.contains(ContentBlocking.CookieBehavior.ACCEPT_VISITED)) {
+            cookiesPolicies.add(TrackingProtectionPolicy.CookiePolicy.ACCEPT_VISITED)
+        }
+
+        return cookiesPolicies
     }
 
     private operator fun Int.contains(mask: Int): Boolean {
@@ -689,7 +733,7 @@ class GeckoEngineSession(
         }
     }
 
-    private fun createGeckoSession() {
+    private fun createGeckoSession(shouldOpen: Boolean = true) {
         this.geckoSession = geckoSessionProvider()
 
         defaultSettings?.trackingProtectionPolicy?.let { enableTrackingProtection(it) }
@@ -699,7 +743,9 @@ class GeckoEngineSession(
         defaultSettings?.userAgentString?.let { geckoSession.settings.userAgentOverride = it }
         defaultSettings?.suspendMediaWhenInactive?.let { geckoSession.settings.suspendMediaWhenInactive = it }
 
-        geckoSession.open(runtime)
+        if (shouldOpen) {
+            geckoSession.open(runtime)
+        }
 
         geckoSession.navigationDelegate = createNavigationDelegate()
         geckoSession.progressDelegate = createProgressDelegate()

components/browser/engine-gecko-beta/src/main/java/mozilla/components/browser/engine/gecko/glean/GeckoAdapter.kt

@@ -0,0 +1,27 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package mozilla.components.browser.engine.gecko.glean
+
+import mozilla.components.browser.engine.gecko.GleanMetrics.GleanGeckoHistogramMapping
+import org.mozilla.geckoview.RuntimeTelemetry
+
+/**
+ * This implements a [RuntimeTelemetry.Delegate] that dispatches Gecko runtime
+ * telemetry to the Glean SDK.
+ *
+ * Metrics defined in the `metrics.yaml` file in Gecko's mozilla-central repository
+ * will be automatically dispatched to the Glean SDK and sent through the requested
+ * pings.
+ *
+ * This can be used, in products collecting data through the Glean SDK, by
+ * providing an instance to `GeckoRuntimeSettings.Builder().telemetryDelegate`.
+ */
+class GeckoAdapter : RuntimeTelemetry.Delegate {
+    override fun onTelemetryReceived(metric: RuntimeTelemetry.Metric) {
+        // Note that the `GleanGeckoHistogramMapping` is automatically generated at
+        // build time by the Glean SDK parsers.
+        GleanGeckoHistogramMapping[metric.name]?.accumulateSamples(metric.values)
+    }
+}