Merge #5386

5386: Closes #5371 - Introduce 'sync' ping, add 'syncUuid' to all sync-related pings  r=csadilek,Dexterp37,boek a=grigoryk

This patch introduce an 'overarching' sync ping, which is meant to contain information
describing a sync overall, vs individual engine runs. Currently it contains global sync
errors.

syncUuid was also added to all sync-related pings, allowing us to tie together in an
analysis all pings that were emitted as part of a single "sync".




Co-authored-by: Grisha Kruglov <gkruglov@mozilla.com>

components/service/firefox-accounts/src/main/java/mozilla/components/service/fxa/sync/WorkManagerSyncManager.kt

@@ -410,22 +410,7 @@ class WorkManagerSyncWorker(
         }
 
         // Process telemetry.
-        syncResult.telemetry?.let {
-            // Yes, this is non-ideal...
-            // But, what this does: individual 'process' function will report global sync errors
-            // as part of its corresponding ping. We don't want to report a global sync error multiple times,
-            // so we check for the boolean flag that indicates if this happened or not.
-            // There's a complete mismatch between what Glean supports and what we need it to do here.
-            // Glean doesn't support "nested metrics" and so we resort to these hacks.
-            // It shouldn't matter in which order these 'process' functions are called.
-            var noGlobalErrorsReported = SyncTelemetry.processBookmarksPing(it)
-            if (noGlobalErrorsReported) {
-                noGlobalErrorsReported = SyncTelemetry.processHistoryPing(it)
-            }
-            if (noGlobalErrorsReported) {
-                SyncTelemetry.processPasswordsPing(it)
-            }
-        }
+        syncResult.telemetry?.let { SyncTelemetry.processSyncTelemetry(it) }
 
         // Finally, declare success, failure or request a retry based on 'sync status'.
         return when (syncResult.status) {

components/service/sync-logins/src/main/java/mozilla/components/service/sync/logins/SyncableLoginsStorage.kt

@@ -255,7 +255,7 @@ class SyncableLoginsStorage(
     @Throws(SyncAuthInvalidException::class, RequestFailedException::class, LoginsStorageException::class)
     suspend fun sync(syncInfo: SyncUnlockInfo): SyncTelemetryPing = withContext(coroutineContext) {
         conn.getStorage().sync(syncInfo).also {
-            SyncTelemetry.processPasswordsPing(it)
+            SyncTelemetry.processLoginsPing(it)
         }
     }
 }

components/support/sync-telemetry/docs/metrics.md

@@ -10,6 +10,7 @@ This means you might have to go searching through the dependency tree to get a f
  - [bookmarks-sync](#bookmarks-sync)
  - [history-sync](#history-sync)
  - [logins-sync](#logins-sync)
+ - [sync](#sync)
 
 
 ## bookmarks-sync
@@ -37,6 +38,7 @@ The following metrics are added to the ping:
 | bookmarks_sync.remote_tree_problems |[labeled_counter](https://mozilla.github.io/glean/book/user/metrics/labeled_counters.html) |Records counts for structure problems and divergences in the remote bookmarks tree. These are documented in https://github.com/mozilla/dogear/blob/fbade15f2a4f11215e30b8f428a0a8df3defeaec/src/tree.rs#L1273-L1294.  |[1](https://github.com/mozilla-mobile/android-components/pull/3092)|<ul><li>orphans</li><li>misparented_roots</li><li>multiple_parents_by_children</li><li>missing_parent_guids</li><li>non_folder_parent_guids</li><li>parent_child_disagreements</li><li>missing_children</li></ul>|never |
 | bookmarks_sync.started_at |[datetime](https://mozilla.github.io/glean/book/user/metrics/datetime.html) |Records when the bookmark sync started.  |[1](https://github.com/mozilla-mobile/android-components/pull/3092)||never |
 | bookmarks_sync.uid |[string](https://mozilla.github.io/glean/book/user/metrics/string.html) |The user's hashed Firefox Account ID.  |[1](https://github.com/mozilla-mobile/android-components/pull/3092)||never |
+| sync.sync_uuid |[uuid](https://mozilla.github.io/glean/book/user/metrics/uuid.html) |Unique identifier for this sync, used to correlate together individual pings for data types that were synchronized together (history, bookmarks, logins). If a data type is synchronized by itself via the legacy 'sync' API (as opposed to the Sync Manager), then this field will not be set on the corresponding ping.  |[1](https://github.com/mozilla-mobile/android-components/pull/5386#pullrequestreview-392363687)||never |
 
 ## history-sync
 
@@ -62,6 +64,7 @@ The following metrics are added to the ping:
 | history_sync.outgoing_batches |[counter](https://mozilla.github.io/glean/book/user/metrics/counter.html) |Records the number of batches needed to upload all outgoing records. The Sync server has a hard limit on the number of records (and request body bytes) on the number of records that can fit into a single batch, and large syncs may require multiple batches.  |[1](https://github.com/mozilla-mobile/android-components/pull/3092)||never |
 | history_sync.started_at |[datetime](https://mozilla.github.io/glean/book/user/metrics/datetime.html) |Records when the history sync started.  |[1](https://github.com/mozilla-mobile/android-components/pull/3092)||never |
 | history_sync.uid |[string](https://mozilla.github.io/glean/book/user/metrics/string.html) |The user's hashed Firefox Account ID.  |[1](https://github.com/mozilla-mobile/android-components/pull/3092)||never |
+| sync.sync_uuid |[uuid](https://mozilla.github.io/glean/book/user/metrics/uuid.html) |Unique identifier for this sync, used to correlate together individual pings for data types that were synchronized together (history, bookmarks, logins). If a data type is synchronized by itself via the legacy 'sync' API (as opposed to the Sync Manager), then this field will not be set on the corresponding ping.  |[1](https://github.com/mozilla-mobile/android-components/pull/5386#pullrequestreview-392363687)||never |
 
 ## logins-sync
 
@@ -87,6 +90,19 @@ The following metrics are added to the ping:
 | logins_sync.outgoing_batches |[counter](https://mozilla.github.io/glean/book/user/metrics/counter.html) |Records the number of batches needed to upload all outgoing records. The Sync server has a hard limit on the number of records (and request body bytes) on the number of records that can fit into a single batch, and large syncs may require multiple batches.  |[1](https://github.com/mozilla-mobile/android-components/pull/5294)||never |
 | logins_sync.started_at |[datetime](https://mozilla.github.io/glean/book/user/metrics/datetime.html) |Records when the passwords sync started.  |[1](https://github.com/mozilla-mobile/android-components/pull/5294)||never |
 | logins_sync.uid |[string](https://mozilla.github.io/glean/book/user/metrics/string.html) |The user's hashed Firefox Account ID.  |[1](https://github.com/mozilla-mobile/android-components/pull/5294)||never |
+| sync.sync_uuid |[uuid](https://mozilla.github.io/glean/book/user/metrics/uuid.html) |Unique identifier for this sync, used to correlate together individual pings for data types that were synchronized together (history, bookmarks, logins). If a data type is synchronized by itself via the legacy 'sync' API (as opposed to the Sync Manager), then this field will not be set on the corresponding ping.  |[1](https://github.com/mozilla-mobile/android-components/pull/5386#pullrequestreview-392363687)||never |
+
+## sync
+
+A summary ping, sent every time a sync is performed. During each Sync one or more data types could be synchronized, depending on which data types user configured to sync. Alongside with 'sync' ping one or more individual data type specific pings will be sent. For example, if history and bookmarks data types are configured to be synchronized, the following pings will be sent: 'sync', 'history-sync' and 'bookmarks-sync'. Alternatively, if only history is configured to be synchronized then 'sync' and 'history-sync' pings will be sent. In case of a "global failure" where none of the data type syncs could even start, e.g. device is offline, only the 'sync' ping will be sent. This ping doesn't include the `client_id` because it reports a hashed version of the user's Firefox Account ID.
+
+
+The following metrics are added to the ping:
+
+| Name | Type | Description | Data reviews | Extras | Expiration |
+| --- | --- | --- | --- | --- | --- |
+| sync.failure_reason |[labeled_string](https://mozilla.github.io/glean/book/user/metrics/labeled_strings.html) |Records a global sync failure: either due to an authentication error, unexpected exception, or other error that caused the sync to fail. Error strings are truncated and sanitized to omit PII, like URLs and file system paths.  |[1](https://github.com/mozilla-mobile/android-components/pull/3092)|<ul><li>other</li><li>unexpected</li><li>auth</li></ul>|never |
+| sync.sync_uuid |[uuid](https://mozilla.github.io/glean/book/user/metrics/uuid.html) |Unique identifier for this sync, used to correlate together individual pings for data types that were synchronized together (history, bookmarks, logins). If a data type is synchronized by itself via the legacy 'sync' API (as opposed to the Sync Manager), then this field will not be set on the corresponding ping.  |[1](https://github.com/mozilla-mobile/android-components/pull/5386#pullrequestreview-392363687)||never |
 
 
 <!-- AUTOGENERATED BY glean_parser.  DO NOT EDIT. -->

components/support/sync-telemetry/metrics.yaml

@@ -4,6 +4,47 @@
 
 $schema: moz://mozilla.org/schemas/glean/metrics/1-0-0
 
+sync:
+  sync_uuid:
+    type: uuid
+    description: >
+      Unique identifier for this sync, used to correlate together individual pings for data types that
+      were synchronized together (history, bookmarks, logins). If a data type is synchronized by itself via
+      the legacy 'sync' API (as opposed to the Sync Manager), then this field will not be set on the corresponding ping.
+    send_in_pings:
+      - sync
+      - history-sync
+      - bookmarks-sync
+      - logins-sync
+    bugs:
+      - https://github.com/mozilla-mobile/android-components/issues/5371
+    data_reviews:
+      - https://github.com/mozilla-mobile/android-components/pull/5386#pullrequestreview-392363687
+    notification_emails:
+      - sync-core@mozilla.com
+    expires: never
+    lifetime: ping
+  failure_reason:
+    type: labeled_string
+    labels:
+      - other
+      - unexpected
+      - auth
+    description: >
+      Records a global sync failure: either due to an authentication error, unexpected exception,
+      or other error that caused the sync to fail. Error strings are truncated and sanitized to omit
+      PII, like URLs and file system paths.
+    send_in_pings:
+      - sync
+    bugs:
+      - https://github.com/mozilla-mobile/android-components/pull/3092
+    data_reviews:
+      - https://github.com/mozilla-mobile/android-components/pull/3092
+    notification_emails:
+      - sync-core@mozilla.com
+    expires: never
+    lifetime: ping
+
 # `history-sync`, `logins-sync` and `bookmarks-sync` metrics all use the same structure,
 # but must be specified individually. We can't define them once and use
 # `send_in_pings` because the stores might be synced in parallel, and we can't

components/support/sync-telemetry/pings.yaml

@@ -4,6 +4,23 @@
 
 $schema: moz://mozilla.org/schemas/glean/pings/1-0-0
 
+sync:
+  description: >
+    A summary ping, sent every time a sync is performed. During each Sync one or more data types
+    could be synchronized, depending on which data types user configured to sync. Alongside with 'sync' ping
+    one or more individual data type specific pings will be sent. For example, if history and bookmarks
+    data types are configured to be synchronized, the following pings will be sent: 'sync', 'history-sync'
+    and 'bookmarks-sync'. Alternatively, if only history is configured to be synchronized then 'sync' and 'history-sync'
+    pings will be sent. In case of a "global failure" where none of the data type syncs could even start,
+    e.g. device is offline, only the 'sync' ping will be sent.
+    This ping doesn't include the `client_id` because it reports a hashed version of the user's Firefox Account ID.
+  include_client_id: false
+  bugs:
+    - https://github.com/mozilla-mobile/android-components/issues/5371
+  notification_emails:
+    - sync-core@mozilla.com
+  data_reviews:
+    - https://github.com/mozilla-mobile/android-components/pull/5386#pullrequestreview-392363687
 history-sync:
   description: >
     A ping sent for every history sync. It doesn't include the `client_id`