Make flush write count threshold/time configurable

glean-core/android/src/main/java/mozilla/telemetry/glean/Glean.kt

@@ -260,6 +260,8 @@ open class GleanInternalAPI internal constructor() {
                 experimentationId = configuration.experimentationId,
                 enableInternalPings = configuration.enableInternalPings,
                 pingSchedule = emptyMap(),
+                pingLifetimeThreshold = configuration.pingLifetimeThreshold.toULong(),
+                pingLifetimeMaxTime = configuration.pingLifetimeMaxTime.toULong(),
             )
             val clientInfo = getClientInfo(configuration, buildInfo)
             val callbacks = OnGleanEventsImpl(this@GleanInternalAPI)

glean-core/android/src/main/java/mozilla/telemetry/glean/config/Configuration.kt

@@ -25,6 +25,8 @@ import mozilla.telemetry.glean.net.PingUploader
  *           to be sent with all pings.
  * @property enableInternalPings Whether to enable internal pings.
  * @property delayPingLifetimeIo Whether Glean should delay persistence of data from metrics with ping lifetime.
+ * @property pingLifetimeThreshold Write count threshold when to auto-flush. `0` disables it.
+ * @property pingLifetimeMaxTime After what time to auto-flush. 0 disables it.
  */
 data class Configuration @JvmOverloads constructor(
     val serverEndpoint: String = DEFAULT_TELEMETRY_ENDPOINT,
@@ -40,6 +42,8 @@ data class Configuration @JvmOverloads constructor(
     val experimentationId: String? = null,
     val enableInternalPings: Boolean = true,
     val delayPingLifetimeIo: Boolean = true,
+    val pingLifetimeThreshold: Int = 1000,
+    val pingLifetimeMaxTime: Int = 0,
 ) {
     companion object {
         /**

glean-core/ios/Glean/Config/Configuration.swift

@@ -13,6 +13,8 @@ public struct Configuration {
     let enableEventTimestamps: Bool
     let experimentationId: String?
     let enableInternalPings: Bool
+    let pingLifetimeThreshold: Int
+    let pingLifetimeMaxTime: Int
 
     struct Constants {
         static let defaultTelemetryEndpoint = "https://incoming.telemetry.mozilla.org"
@@ -32,6 +34,8 @@ public struct Configuration {
     ///   * experimentationId An experimentation identifier derived by the application
     ///   to be sent with all pings.
     ///   * enableInternalPings Whether to enable internal pings.
+    ///   * pingLifetimeThreshold Write count threshold when to auto-flush. `0` disables it.
+    ///   * pingLifetimeMaxTime After what time to auto-flush. 0 disables it.
     public init(
         maxEvents: Int32? = nil,
         channel: String? = nil,
@@ -40,7 +44,9 @@ public struct Configuration {
         logLevel: LevelFilter? = nil,
         enableEventTimestamps: Bool = true,
         experimentationId: String? = nil,
-        enableInternalPings: Bool = true
+        enableInternalPings: Bool = true,
+        pingLifetimeThreshold: Int = 0,
+        pingLifetimeMaxTime: Int = 0
     ) {
         self.serverEndpoint = serverEndpoint ?? Constants.defaultTelemetryEndpoint
         self.maxEvents = maxEvents
@@ -50,5 +56,7 @@ public struct Configuration {
         self.enableEventTimestamps = enableEventTimestamps
         self.experimentationId = experimentationId
         self.enableInternalPings = enableInternalPings
+        self.pingLifetimeThreshold = pingLifetimeThreshold
+        self.pingLifetimeMaxTime = pingLifetimeMaxTime
     }
 }

glean-core/ios/Glean/Glean.swift

@@ -196,7 +196,9 @@ public class Glean {
             enableEventTimestamps: configuration.enableEventTimestamps,
             experimentationId: configuration.experimentationId,
             enableInternalPings: configuration.enableInternalPings,
-            pingSchedule: [:]
+            pingSchedule: [:],
+            pingLifetimeThreshold: UInt64(configuration.pingLifetimeThreshold),
+            pingLifetimeMaxTime: UInt64(configuration.pingLifetimeMaxTime)
         )
         let clientInfo = getClientInfo(configuration, buildInfo: buildInfo)
         let callbacks = OnGleanEventsImpl(glean: self)

glean-core/python/glean/glean.py

@@ -235,6 +235,8 @@ def initialize(
             experimentation_id=configuration.experimentation_id,
             enable_internal_pings=configuration.enable_internal_pings,
             ping_schedule={},
+            ping_lifetime_threshold=0,
+            ping_lifetime_max_time=0,
         )
 
         _uniffi.glean_initialize(cfg, client_info, callbacks)

glean-core/python/glean/net/ping_upload_worker.py

@@ -120,6 +120,8 @@ def _process(data_dir: Path, application_id: str, configuration) -> bool:
             experimentation_id=None,
             enable_internal_pings=False,
             ping_schedule={},
+            ping_lifetime_threshold=0,
+            ping_lifetime_max_time=0,
         )
         if not glean_initialize_for_subprocess(cfg):
             log.error("Couldn't initialize Glean in subprocess")

glean-core/rlb/examples/ping-lifetime-flush.rs

@@ -92,6 +92,8 @@ fn main() {
         .with_use_core_mps(false)
         .with_uploader(uploader)
         .with_delay_ping_lifetime_io(true)
+        .with_ping_lifetime_threshold(1000)
+        .with_ping_lifetime_max_time(Duration::from_millis(2000))
         .build();
 
     let client_info = ClientInfoMetrics {

glean-core/rlb/src/configuration.rs

@@ -8,6 +8,7 @@ use crate::net::PingUploader;
 
 use std::collections::HashMap;
 use std::path::PathBuf;
+use std::time::Duration;
 
 /// The default server pings are sent to.
 pub(crate) const DEFAULT_GLEAN_ENDPOINT: &str = "https://incoming.telemetry.mozilla.org";
@@ -53,6 +54,10 @@ pub struct Configuration {
     /// Maps a ping name to a list of pings to schedule along with it.
     /// Only used if the ping's own ping schedule list is empty.
     pub ping_schedule: HashMap<String, Vec<String>>,
+    /// Write count threshold when to auto-flush. `0` disables it.
+    pub ping_lifetime_threshold: usize,
+    /// After what time to auto-flush. 0 disables it.
+    pub ping_lifetime_max_time: Duration,
 }
 
 /// Configuration builder.
@@ -105,6 +110,10 @@ pub struct Builder {
     /// Maps a ping name to a list of pings to schedule along with it.
     /// Only used if the ping's own ping schedule list is empty.
     pub ping_schedule: HashMap<String, Vec<String>>,
+    /// Write count threshold when to auto-flush. `0` disables it.
+    pub ping_lifetime_threshold: usize,
+    /// After what time to auto-flush. 0 disables it.
+    pub ping_lifetime_max_time: Duration,
 }
 
 impl Builder {
@@ -130,6 +139,8 @@ impl Builder {
             experimentation_id: None,
             enable_internal_pings: true,
             ping_schedule: HashMap::new(),
+            ping_lifetime_threshold: 0,
+            ping_lifetime_max_time: Duration::ZERO,
         }
     }
 
@@ -151,6 +162,8 @@ impl Builder {
             experimentation_id: self.experimentation_id,
             enable_internal_pings: self.enable_internal_pings,
             ping_schedule: self.ping_schedule,
+            ping_lifetime_threshold: self.ping_lifetime_threshold,
+            ping_lifetime_max_time: self.ping_lifetime_max_time,
         }
     }
 
@@ -213,4 +226,16 @@ impl Builder {
         self.ping_schedule = value;
         self
     }
+
+    /// Write count threshold when to auto-flush. `0` disables it.
+    pub fn with_ping_lifetime_threshold(mut self, value: usize) -> Self {
+        self.ping_lifetime_threshold = value;
+        self
+    }
+
+    /// After what time to auto-flush. 0 disables it.
+    pub fn with_ping_lifetime_max_time(mut self, value: Duration) -> Self {
+        self.ping_lifetime_max_time = value;
+        self
+    }
 }

glean-core/rlb/src/lib.rs

@@ -124,6 +124,8 @@ fn initialize_internal(cfg: Configuration, client_info: ClientInfoMetrics) -> Op
         experimentation_id: cfg.experimentation_id,
         enable_internal_pings: cfg.enable_internal_pings,
         ping_schedule: cfg.ping_schedule,
+        ping_lifetime_threshold: cfg.ping_lifetime_threshold as u64,
+        ping_lifetime_max_time: cfg.ping_lifetime_max_time.as_millis() as u64,
     };
 
     glean_core::glean_initialize(core_cfg, client_info.into(), callbacks);

glean-core/src/core/mod.rs

@@ -6,6 +6,7 @@ use std::collections::HashMap;
 use std::path::{Path, PathBuf};
 use std::sync::atomic::{AtomicU8, Ordering};
 use std::sync::{Arc, Mutex};
+use std::time::Duration;
 
 use chrono::{DateTime, FixedOffset};
 use once_cell::sync::OnceCell;
@@ -122,6 +123,8 @@ where
 ///     experimentation_id: None,
 ///     enable_internal_pings: true,
 ///     ping_schedule: Default::default(),
+///     ping_lifetime_threshold: 1000,
+///     ping_lifetime_max_time: 2000,
 /// };
 /// let mut glean = Glean::new(cfg).unwrap();
 /// let ping = PingType::new("sample", true, false, true, true, true, vec![], vec![]);
@@ -250,7 +253,14 @@ impl Glean {
         // Creating the data store creates the necessary path as well.
         // If that fails we bail out and don't initialize further.
         let data_path = Path::new(&cfg.data_path);
-        glean.data_store = Some(Database::new(data_path, cfg.delay_ping_lifetime_io)?);
+        let ping_lifetime_threshold = cfg.ping_lifetime_threshold as usize;
+        let ping_lifetime_max_time = Duration::from_millis(cfg.ping_lifetime_max_time);
+        glean.data_store = Some(Database::new(
+            data_path,
+            cfg.delay_ping_lifetime_io,
+            ping_lifetime_threshold,
+            ping_lifetime_max_time,
+        )?);
 
         // Set experimentation identifier (if any)
         if let Some(experimentation_id) = &cfg.experimentation_id {
@@ -329,6 +339,8 @@ impl Glean {
             experimentation_id: None,
             enable_internal_pings,
             ping_schedule: Default::default(),
+            ping_lifetime_threshold: 0,
+            ping_lifetime_max_time: 0,
         };
 
         let mut glean = Self::new(cfg).unwrap();